From 93adfafe60b8f3b473b5097c2b80497abe8f5300 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 23 Nov 2023 19:49:16 +0800 Subject: [PATCH 01/46] feat(Rust os-agent, proxy): add rust version os-agent and proxy To decrease the memory usage, we develop os-agent and proxy in Rust Signed-off-by: Yuhang Wei --- .gitignore | 7 + KubeOS-Rust/Cargo.lock | 2841 +++++++++++++++++ KubeOS-Rust/Cargo.toml | 15 + KubeOS-Rust/agent/Cargo.toml | 20 + KubeOS-Rust/agent/src/function.rs | 36 + KubeOS-Rust/agent/src/main.rs | 75 + KubeOS-Rust/agent/src/rpc/agent.rs | 32 + KubeOS-Rust/agent/src/rpc/agent_impl.rs | 221 ++ KubeOS-Rust/agent/src/rpc/mod.rs | 19 + KubeOS-Rust/cli/Cargo.toml | 15 + KubeOS-Rust/cli/src/client.rs | 77 + KubeOS-Rust/cli/src/lib.rs | 14 + KubeOS-Rust/cli/src/method/callable_method.rs | 26 + KubeOS-Rust/cli/src/method/cleanup.rs | 34 + KubeOS-Rust/cli/src/method/configure.rs | 41 + KubeOS-Rust/cli/src/method/mod.rs | 19 + KubeOS-Rust/cli/src/method/prepare_upgrade.rs | 41 + KubeOS-Rust/cli/src/method/request.rs | 58 + KubeOS-Rust/cli/src/method/rollback.rs | 34 + KubeOS-Rust/cli/src/method/upgrade.rs | 34 + KubeOS-Rust/manager/Cargo.toml | 22 + KubeOS-Rust/manager/src/api/agent_status.rs | 49 + KubeOS-Rust/manager/src/api/mod.rs | 17 + KubeOS-Rust/manager/src/api/types.rs | 67 + KubeOS-Rust/manager/src/lib.rs | 15 + KubeOS-Rust/manager/src/sys_mgmt/config.rs | 723 +++++ .../manager/src/sys_mgmt/containerd_image.rs | 359 +++ KubeOS-Rust/manager/src/sys_mgmt/mod.rs | 21 + KubeOS-Rust/manager/src/sys_mgmt/values.rs | 35 + KubeOS-Rust/manager/src/utils/common.rs | 312 ++ .../manager/src/utils/container_image.rs | 271 ++ KubeOS-Rust/manager/src/utils/executor.rs | 101 + .../manager/src/utils/image_manager.rs | 260 ++ KubeOS-Rust/manager/src/utils/mod.rs | 23 + KubeOS-Rust/manager/src/utils/partition.rs | 110 + KubeOS-Rust/proxy/Cargo.toml | 31 + KubeOS-Rust/proxy/src/controller/apiclient.rs | 172 + .../proxy/src/controller/controller.rs | 502 +++ KubeOS-Rust/proxy/src/controller/crd.rs | 77 + KubeOS-Rust/proxy/src/controller/drain.rs | 650 ++++ KubeOS-Rust/proxy/src/controller/mod.rs | 23 + KubeOS-Rust/proxy/src/controller/utils.rs | 155 + KubeOS-Rust/proxy/src/controller/values.rs | 49 + KubeOS-Rust/proxy/src/main.rs | 52 + 44 files changed, 7755 insertions(+) create mode 100644 KubeOS-Rust/Cargo.lock create mode 100644 KubeOS-Rust/Cargo.toml create mode 100644 KubeOS-Rust/agent/Cargo.toml create mode 100644 KubeOS-Rust/agent/src/function.rs create mode 100644 KubeOS-Rust/agent/src/main.rs create mode 100644 KubeOS-Rust/agent/src/rpc/agent.rs create mode 100644 KubeOS-Rust/agent/src/rpc/agent_impl.rs create mode 100644 KubeOS-Rust/agent/src/rpc/mod.rs create mode 100644 KubeOS-Rust/cli/Cargo.toml create mode 100644 KubeOS-Rust/cli/src/client.rs create mode 100644 KubeOS-Rust/cli/src/lib.rs create mode 100644 KubeOS-Rust/cli/src/method/callable_method.rs create mode 100644 KubeOS-Rust/cli/src/method/cleanup.rs create mode 100644 KubeOS-Rust/cli/src/method/configure.rs create mode 100644 KubeOS-Rust/cli/src/method/mod.rs create mode 100644 KubeOS-Rust/cli/src/method/prepare_upgrade.rs create mode 100644 KubeOS-Rust/cli/src/method/request.rs create mode 100644 KubeOS-Rust/cli/src/method/rollback.rs create mode 100644 KubeOS-Rust/cli/src/method/upgrade.rs create mode 100644 KubeOS-Rust/manager/Cargo.toml create mode 100644 KubeOS-Rust/manager/src/api/agent_status.rs create mode 100644 KubeOS-Rust/manager/src/api/mod.rs create mode 100644 KubeOS-Rust/manager/src/api/types.rs create mode 100644 KubeOS-Rust/manager/src/lib.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/config.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/mod.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/values.rs create mode 100644 KubeOS-Rust/manager/src/utils/common.rs create mode 100644 KubeOS-Rust/manager/src/utils/container_image.rs create mode 100644 KubeOS-Rust/manager/src/utils/executor.rs create mode 100644 KubeOS-Rust/manager/src/utils/image_manager.rs create mode 100644 KubeOS-Rust/manager/src/utils/mod.rs create mode 100644 KubeOS-Rust/manager/src/utils/partition.rs create mode 100644 KubeOS-Rust/proxy/Cargo.toml create mode 100644 KubeOS-Rust/proxy/src/controller/apiclient.rs create mode 100644 KubeOS-Rust/proxy/src/controller/controller.rs create mode 100644 KubeOS-Rust/proxy/src/controller/crd.rs create mode 100644 KubeOS-Rust/proxy/src/controller/drain.rs create mode 100644 KubeOS-Rust/proxy/src/controller/mod.rs create mode 100644 KubeOS-Rust/proxy/src/controller/utils.rs create mode 100644 KubeOS-Rust/proxy/src/controller/values.rs create mode 100644 KubeOS-Rust/proxy/src/main.rs diff --git a/.gitignore b/.gitignore index 5e56e040..4d173c5e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,8 @@ +# vscode settings +.vscode + +# rust dependencies +target/ + +# KubeOS bin /bin diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock new file mode 100644 index 00000000..f6906039 --- /dev/null +++ b/KubeOS-Rust/Cargo.lock @@ -0,0 +1,2841 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom 0.2.10", + "instant", + "rand 0.8.5", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "jsonrpc", + "log", + "manager", + "serde", + "serde_json", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "dyn-clone" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags 1.3.2", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "globset" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "h2" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +dependencies = [ + "bytes 1.5.0", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.14.0", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes 1.5.0", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes 1.5.0", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +dependencies = [ + "bytes 1.5.0", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio 1.14.0", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio 1.14.0", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes 1.5.0", + "hyper", + "native-tls", + "tokio 1.14.0", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-patch" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce" +dependencies = [ + "serde", + "serde_json", + "treediff", +] + +[[package]] +name = "jsonpath_lib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "jsonrpc" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd8d6b3f301ba426b30feca834a2a18d48d5b54e5065496b5c1b05537bee3639" +dependencies = [ + "base64", + "serde", + "serde_json", +] + +[[package]] +name = "jsonrpc-core" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0745a6379e3edc893c84ec203589790774e4247420033e71a76d3ab4687991fa" +dependencies = [ + "futures 0.1.31", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "jsonrpc-derive" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99a847f9ec7bb52149b2786a17c9cb260d6effc6b8eeb8c16b343a487a7563a3" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "jsonrpc-ipc-server" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf50e53e4eea8f421a7316c5f63e395f7bc7c4e786a6dc54d76fab6ff7aa7ce7" +dependencies = [ + "jsonrpc-core", + "jsonrpc-server-utils", + "log", + "parity-tokio-ipc", + "parking_lot 0.10.2", + "tokio-service", +] + +[[package]] +name = "jsonrpc-server-utils" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f1f3990650c033bd8f6bd46deac76d990f9bbfb5f8dc8c4767bf0a00392176" +dependencies = [ + "bytes 0.4.12", + "globset", + "jsonrpc-core", + "lazy_static", + "log", + "tokio 0.1.22", + "tokio-codec", + "unicase", +] + +[[package]] +name = "k8s-openapi" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c" +dependencies = [ + "base64", + "bytes 1.5.0", + "chrono", + "http", + "percent-encoding", + "serde", + "serde-value", + "serde_json", + "url", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "kube" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4b96944d327b752df4f62f3a31d8694892af06fb585747c0b5e664927823d1a" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-derive", + "kube-runtime", +] + +[[package]] +name = "kube-client" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232db1af3d3680f9289cf0b4db51b2b9fee22550fc65d25869e39b23e0aaa696" +dependencies = [ + "base64", + "bytes 1.5.0", + "chrono", + "dirs-next", + "either", + "futures 0.3.29", + "http", + "http-body", + "hyper", + "hyper-timeout", + "hyper-tls", + "jsonpath_lib", + "k8s-openapi", + "kube-core", + "openssl", + "pem", + "pin-project", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tokio 1.14.0", + "tokio-native-tls", + "tokio-util 0.6.10", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de491f8c9ee97117e0b47a629753e939c2392d5d0a40f6928e582a5fba328098" +dependencies = [ + "chrono", + "form_urlencoded", + "http", + "json-patch", + "k8s-openapi", + "once_cell", + "schemars", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "kube-derive" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbb86bb3607245a67c8ad3a52aff41108f36b0d1e9e3e82ffb5760bfd84b965" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 1.0.109", +] + +[[package]] +name = "kube-runtime" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710729592eb30219b4e84898e91dc991fe09ccafe2c17fec4e45c3426c61abe0" +dependencies = [ + "backoff", + "dashmap", + "derivative", + "futures 0.3.29", + "json-patch", + "k8s-openapi", + "kube-client", + "pin-project", + "serde", + "serde_json", + "smallvec 1.11.1", + "thiserror", + "tokio 1.14.0", + "tokio-util 0.6.10", + "tracing", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" + +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.0", + "libc", + "redox_syscall 0.4.1", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4dcd960cc540667f619483fc99102f88d6118b87730e24e8fbe8054b7445e4" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "manager" +version = "0.1.0" +dependencies = [ + "anyhow", + "env_logger", + "lazy_static", + "log", + "mockall", + "nix", + "predicates", + "regex", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.2", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +dependencies = [ + "libc", + "log", + "miow 0.3.7", + "ntapi", + "winapi 0.3.9", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log", + "mio 0.6.23", + "miow 0.3.7", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "mockall" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4a1c770583dac7ab5e2f6c139153b783a53a1bbee9729613f193e59828326" +dependencies = [ + "cfg-if 1.0.0", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "net2" +version = "0.2.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if 1.0.0", + "libc", + "memoffset 0.7.1", + "pin-utils", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "ntapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.3", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" + +[[package]] +name = "openssl" +version = "0.10.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" +dependencies = [ + "bitflags 2.4.0", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os-agent" +version = "0.1.0" +dependencies = [ + "anyhow", + "env_logger", + "jsonrpc-core", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "lazy_static", + "log", + "manager", + "nix", + "serde", + "serde_json", +] + +[[package]] +name = "parity-tokio-ipc" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e57fea504fea33f9fbb5f49f378359030e7e026a6ab849bb9e8f0787376f1bf" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "libc", + "log", + "mio-named-pipes", + "miow 0.3.7", + "rand 0.7.3", + "tokio 0.1.22", + "tokio-named-pipes", + "tokio-uds", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api", + "parking_lot_core 0.6.3", + "rustc_version", +] + +[[package]] +name = "parking_lot" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +dependencies = [ + "lock_api", + "parking_lot_core 0.7.3", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version", + "smallvec 0.6.14", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93f386bb233083c799e6e642a9d73db98c24a5deeb95ffc85bf281255dffc98" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "smallvec 1.11.1", + "winapi 0.3.9", +] + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64", +] + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "predicates" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc3d91237f5de3bcd9d927e24d03b495adb6135097b001cea7403e2d573d00a9" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proxy" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "cli", + "env_logger", + "futures 0.3.29", + "h2", + "k8s-openapi", + "kube", + "log", + "manager", + "regex", + "reqwest", + "schemars", + "serde", + "serde_json", + "snafu", + "socket2", + "thiserror", + "thread_local", + "tokio 1.14.0", + "tokio-retry", + "tracing", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.10", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom 0.2.10", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "reqwest" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" +dependencies = [ + "base64", + "bytes 1.5.0", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio 1.14.0", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "schemars" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1847b767a3d62d95cbf3d8a9f0e421cf57a0d8aa4f411d4b16525afb0284d4ed" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4d7e1b012cb3d9129567661a63755ea4b8a7386d339dc945ae187e403c6743" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "serde", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c4437699b6d34972de58652c68b98cb5b53a4199ab126db8e20ec8ded29a721" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_json" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap", + "ryu", + "serde", + "yaml-rust", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" + +[[package]] +name = "snafu" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +dependencies = [ + "doc-comment", + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if 1.0.0", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys", +] + +[[package]] +name = "termcolor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "mio 0.6.23", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + +[[package]] +name = "tokio" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" +dependencies = [ + "autocfg", + "bytes 1.5.0", + "libc", + "memchr", + "mio 0.7.14", + "num_cpus", + "once_cell", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "tokio-io", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.31", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils", + "futures 0.1.31", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures 0.1.31", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio 1.14.0", +] + +[[package]] +name = "tokio-macros" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tokio-named-pipes" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d282d483052288b2308ba5ee795f5673b159c9bdf63c385a05609da782a5eae" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "mio 0.6.23", + "mio-named-pipes", + "tokio 0.1.22", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio 1.14.0", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils", + "futures 0.1.31", + "lazy_static", + "log", + "mio 0.6.23", + "num_cpus", + "parking_lot 0.9.0", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio 1.14.0", +] + +[[package]] +name = "tokio-service" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" +dependencies = [ + "futures 0.1.31", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.31", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "mio 0.6.23", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque", + "crossbeam-queue", + "crossbeam-utils", + "futures 0.1.31", + "lazy_static", + "log", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils", + "futures 0.1.31", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", + "mio 0.6.23", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "libc", + "log", + "mio 0.6.23", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes 1.5.0", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "slab", + "tokio 1.14.0", +] + +[[package]] +name = "tokio-util" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" +dependencies = [ + "bytes 1.5.0", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio 1.14.0", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio 1.14.0", + "tokio-util 0.7.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8" +dependencies = [ + "base64", + "bitflags 1.3.2", + "bytes 1.5.0", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "treediff" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff" +dependencies = [ + "serde_json", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" + +[[package]] +name = "web-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml new file mode 100644 index 00000000..64ad4be8 --- /dev/null +++ b/KubeOS-Rust/Cargo.toml @@ -0,0 +1,15 @@ +[workspace] +members = [ + "manager", + "agent", + "cli", + "proxy", +] + +[profile.release] +opt-level = 's' +debug = false +rpath = false +debug-assertions = false +overflow-checks = false +lto = true diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml new file mode 100644 index 00000000..fd7d3905 --- /dev/null +++ b/KubeOS-Rust/agent/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "os-agent" +version = "0.1.0" +edition = "2021" +description = "KubeOS os-agent" +license = "MulanPSL-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +manager = { package = "manager", path = "../manager" } +jsonrpc-core = { version = "15.1" } +jsonrpc-derive = { version = "15.1" } +jsonrpc-ipc-server = { version = "15.1" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } +log = { version = "= 0.4.15" } +anyhow = { version = "1.0" } +env_logger = { version = "0.9" } +lazy_static = { version = "1.4" } +nix = { version = "0.26.2" } diff --git a/KubeOS-Rust/agent/src/function.rs b/KubeOS-Rust/agent/src/function.rs new file mode 100644 index 00000000..8fb8a4e3 --- /dev/null +++ b/KubeOS-Rust/agent/src/function.rs @@ -0,0 +1,36 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub use jsonrpc_core::Result as RpcResult; +use jsonrpc_core::{Error, ErrorCode}; +pub use jsonrpc_derive::rpc; +use log::error; + +const RPC_OP_ERROR: i64 = -1; + +pub struct RpcFunction; + +impl RpcFunction { + pub fn call(f: F) -> RpcResult + where + F: FnOnce() -> anyhow::Result, + { + (f)().map_err(|e| { + error!("{:?}", e); + Error { + code: ErrorCode::ServerError(RPC_OP_ERROR), + message: format!("{:?}", e), + data: None, + } + }) + } +} diff --git a/KubeOS-Rust/agent/src/main.rs b/KubeOS-Rust/agent/src/main.rs new file mode 100644 index 00000000..2201c202 --- /dev/null +++ b/KubeOS-Rust/agent/src/main.rs @@ -0,0 +1,75 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{self, DirBuilder, Permissions}, + os::unix::fs::{DirBuilderExt, PermissionsExt}, + path::Path, +}; + +use env_logger::{Builder, Env, Target}; +use jsonrpc_core::{IoHandler, IoHandlerExtension}; +use jsonrpc_ipc_server::ServerBuilder; + +mod function; +mod rpc; + +use log::info; +use rpc::{Agent, AgentImpl}; + +const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; +const CARGO_PKG_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); + +fn start_and_run(sock_path: &str) { + let socket_path = Path::new(sock_path); + + // Create directory for socket if it doesn't exist + if let Some(dir_path) = socket_path.parent() { + if !dir_path.exists() { + DirBuilder::new() + .mode(0o750) + .create(dir_path) + .expect("Couldn't create directory for socket"); + } + } + + // Add RPC methods to IoHandler + let mut io = IoHandler::new(); + AgentImpl::default().to_delegate().augment(&mut io); + + // Build and start server + let builder = ServerBuilder::new(io); + let server = builder.start(sock_path).expect("Couldn't open socket"); + + let gid = nix::unistd::getgid(); + nix::unistd::chown(socket_path, Some(nix::unistd::ROOT), Some(gid)) + .expect("Couldn't set socket group"); + + // Set socket permissions to 0640 + let socket_permissions = Permissions::from_mode(0o640); + fs::set_permissions(socket_path, socket_permissions).expect("Couldn't set socket permissions"); + + info!("os-agent started, waiting for requests..."); + server.wait(); +} + +fn main() { + Builder::from_env(Env::default().default_filter_or("info")) + .target(Target::Stdout) + .init(); + + info!( + "os-agent version is: {}", + CARGO_PKG_VERSION.unwrap_or("NOT FOUND") + ); + start_and_run(SOCK_PATH); +} diff --git a/KubeOS-Rust/agent/src/rpc/agent.rs b/KubeOS-Rust/agent/src/rpc/agent.rs new file mode 100644 index 00000000..97eb4566 --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/agent.rs @@ -0,0 +1,32 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use super::function::{rpc, RpcResult}; +use manager::api::{ConfigureRequest, Response, UpgradeRequest}; + +#[rpc(server)] +pub trait Agent { + #[rpc(name = "prepare_upgrade")] + fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult; + + #[rpc(name = "upgrade")] + fn upgrade(&self) -> RpcResult; + + #[rpc(name = "cleanup")] + fn cleanup(&self) -> RpcResult; + + #[rpc(name = "configure")] + fn configure(&self, req: ConfigureRequest) -> RpcResult; + + #[rpc(name = "rollback")] + fn rollback(&self) -> RpcResult; +} diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs new file mode 100644 index 00000000..c752ffac --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -0,0 +1,221 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{sync::Mutex, thread, time::Duration}; + +use anyhow::{anyhow, Result}; +use log::{debug, error, info}; +use nix::{sys::reboot::RebootMode, unistd::sync}; + +use super::{ + agent::Agent, + function::{RpcFunction, RpcResult}, +}; +use manager::{ + api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, + sys_mgmt::{CtrImageHandler, CONFIG_TEMPLATE}, + utils::{ + clean_env, get_partition_info, switch_boot_menuentry, PreparePath, RealCommandExecutor, + UpgradeImageManager, + }, +}; + +pub struct AgentImpl { + mutex: Mutex<()>, + disable_reboot: bool, +} + +impl Agent for AgentImpl { + fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult { + RpcFunction::call(|| self.prepare_upgrade_impl(req)) + } + + fn upgrade(&self) -> RpcResult { + RpcFunction::call(|| self.upgrade_impl()) + } + + fn cleanup(&self) -> RpcResult { + RpcFunction::call(|| self.cleanup_impl()) + } + + fn configure(&self, req: ConfigureRequest) -> RpcResult { + RpcFunction::call(|| self.configure_impl(req)) + } + + fn rollback(&self) -> RpcResult { + RpcFunction::call(|| self.rollback_impl()) + } +} + +impl Default for AgentImpl { + fn default() -> Self { + Self { + mutex: Mutex::new(()), + disable_reboot: false, + } + } +} + +impl AgentImpl { + pub fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result { + let _lock = self.mutex.lock().unwrap(); + debug!("Received an 'prepare upgrade' request: {:?}", req); + info!("Start to upgrade to version: {}", req.version); + + let handler: Box> = match req.image_type.as_str() { + "containerd" => Box::new(ImageType::Containerd(CtrImageHandler::default())), + _ => return Err(anyhow!("Invalid image type \"{}\"", req.image_type)), + }; + + let image_manager = handler.download_image(&req)?; + info!( + "Ready to install image: {:?}", + image_manager.paths.image_path.display() + ); + + Ok(Response { + status: AgentStatus::UpgradeReady, + }) + } + + pub fn upgrade_impl(&self) -> Result { + let _lock = self.mutex.lock().unwrap(); + info!("Start to upgrade"); + let command_executor = RealCommandExecutor {}; + let (_, next_partition_info) = get_partition_info(&command_executor)?; + let image_manager = UpgradeImageManager::new( + PreparePath::default(), + next_partition_info, + command_executor, + ); + image_manager.install()?; + self.reboot()?; + Ok(Response { + status: AgentStatus::Upgraded, + }) + } + + pub fn cleanup_impl(&self) -> Result { + let _lock = self.mutex.lock().unwrap(); + info!("Start to cleanup"); + let paths = PreparePath::default(); + clean_env(paths.update_path, paths.mount_path, paths.image_path)?; + Ok(Response { + status: AgentStatus::NotApplied, + }) + } + + pub fn configure_impl(&self, mut req: ConfigureRequest) -> Result { + let _lock = self.mutex.lock().unwrap(); + debug!("Received a 'configure' request: {:?}", req); + info!("Start to configure"); + let config_map = &*CONFIG_TEMPLATE; + for config in req.configs.iter_mut() { + let config_type = &config.model; + if let Some(configuration) = config_map.get(config_type) { + debug!("Found configuration type: \"{}\"", config_type); + configuration.set_config(config)?; + } else { + error!("Unknown configuration type: \"{}\"", config_type); + Err(anyhow!("Unknown configuration type: \"{}\"", config_type))?; + } + } + Ok(Response { + status: AgentStatus::Configured, + }) + } + + pub fn rollback_impl(&self) -> Result { + let _lock = self.mutex.lock().unwrap(); + info!("Start to rollback"); + let command_executor = RealCommandExecutor {}; + let (_, next_partition_info) = get_partition_info(&command_executor)?; + switch_boot_menuentry( + &command_executor, + manager::sys_mgmt::DEFAULT_GRUBENV_PATH, + &next_partition_info.menuentry, + )?; + info!( + "Switch to boot partition: {}, device: {}", + next_partition_info.menuentry, next_partition_info.device + ); + self.reboot()?; + Ok(Response { + status: AgentStatus::Rollbacked, + }) + } + + pub fn reboot(&self) -> Result<()> { + info!("Wait to reboot"); + thread::sleep(Duration::from_secs(1)); + sync(); + if self.disable_reboot { + return Ok(()); + } + nix::sys::reboot::reboot(RebootMode::RB_AUTOBOOT)?; + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + use manager::api::Sysconfig; + use std::collections::HashMap; + + #[test] + fn configure_impl_tests() { + let agent = AgentImpl::default(); + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "kernel.sysctl".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure_impl(req).unwrap(); + assert_eq!( + res, + Response { + status: AgentStatus::Configured, + } + ); + + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "invalid".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure_impl(req); + assert!(res.is_err()); + } + + #[test] + fn upgrade_impl_tests() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + let agent = AgentImpl::default(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "xxx".into(), + image_type: "xxx".into(), + container_image: "xxx".into(), + }; + let res = agent.prepare_upgrade_impl(req); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/agent/src/rpc/mod.rs b/KubeOS-Rust/agent/src/rpc/mod.rs new file mode 100644 index 00000000..976356be --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/mod.rs @@ -0,0 +1,19 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use super::function; + +mod agent; +mod agent_impl; + +pub use agent::*; +pub use agent_impl::*; diff --git a/KubeOS-Rust/cli/Cargo.toml b/KubeOS-Rust/cli/Cargo.toml new file mode 100644 index 00000000..18ea908a --- /dev/null +++ b/KubeOS-Rust/cli/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "cli" +version = "0.1.0" +edition = "2021" +description = "KubeOS os-agent client" +license = "MulanPSL-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +kubeos-manager = { package = "manager", path = "../manager" } +jsonrpc = { version = "0.13", features = ["simple_uds"] } +log = { version = "0.4" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } +anyhow = { version = "1.0" } diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs new file mode 100644 index 00000000..71121fe0 --- /dev/null +++ b/KubeOS-Rust/cli/src/client.rs @@ -0,0 +1,77 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::path::Path; + +use jsonrpc::{ + simple_uds::UdsTransport, Client as JsonRPCClient, Request as JsonRPCRequest, + Response as JsonRPCResponse, +}; +use serde_json::value::RawValue; + +pub struct Client { + json_rpc_client: JsonRPCClient, +} + +pub struct Request<'a>(JsonRPCRequest<'a>); + +impl<'a> Request<'a> {} + +impl Client { + pub fn new>(socket_path: P) -> Self { + let client = Client { + json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)), + }; + client + } + + pub fn build_request<'a>( + &self, + command: &'a str, + params: &'a Vec>, + ) -> Request<'a> { + let json_rpc_request = self.json_rpc_client.build_request(command, ¶ms); + let request = Request(json_rpc_request); + request + } + + pub fn send_request(&self, request: Request) -> Result { + let response = self.json_rpc_client.send_request(request.0); + response + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::method::{callable_method::RpcMethod, configure::ConfigureMethod}; + use kubeos_manager::api; + + #[test] + #[ignore] + fn test_client() { + let socket_path = "/home/yuhang/os-agent-rust.sock"; + let cli = Client::new(socket_path); + + let configured = api::AgentStatus::Configured; + let resp = api::Response { status: configured }; + let config_request = api::ConfigureRequest { + configs: vec![api::Sysconfig { + model: "kernel.sysctl".into(), + config_path: "".into(), + contents: std::collections::hash_map::HashMap::new(), + }], + }; + let config_resp = ConfigureMethod::new(config_request).call(&cli).unwrap(); + assert_eq!(resp, config_resp); + } +} diff --git a/KubeOS-Rust/cli/src/lib.rs b/KubeOS-Rust/cli/src/lib.rs new file mode 100644 index 00000000..cd66d72f --- /dev/null +++ b/KubeOS-Rust/cli/src/lib.rs @@ -0,0 +1,14 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod client; +pub mod method; diff --git a/KubeOS-Rust/cli/src/method/callable_method.rs b/KubeOS-Rust/cli/src/method/callable_method.rs new file mode 100644 index 00000000..d59ebd62 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/callable_method.rs @@ -0,0 +1,26 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde_json::value::RawValue; + +use super::request::{parse_error, request}; +use crate::client::Client; + +pub trait RpcMethod { + type Response: serde::de::DeserializeOwned; + fn command_name(&self) -> &'static str; + fn command_params(&self) -> Vec>; + fn call(&self, client: &Client) -> Result { + let response = request(client, self.command_name(), self.command_params())?; + response.result().map_err(|e| parse_error(e)) + } +} diff --git a/KubeOS-Rust/cli/src/method/cleanup.rs b/KubeOS-Rust/cli/src/method/cleanup.rs new file mode 100644 index 00000000..48a03bc8 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/cleanup.rs @@ -0,0 +1,34 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde_json::value::RawValue; + +use crate::method::callable_method::RpcMethod; +use kubeos_manager::api; + +pub struct CleanupMethod {} + +impl CleanupMethod { + pub fn new() -> Self { + CleanupMethod {} + } +} + +impl RpcMethod for CleanupMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "cleanup" + } + fn command_params(&self) -> Vec> { + vec![] + } +} diff --git a/KubeOS-Rust/cli/src/method/configure.rs b/KubeOS-Rust/cli/src/method/configure.rs new file mode 100644 index 00000000..ddfeb05f --- /dev/null +++ b/KubeOS-Rust/cli/src/method/configure.rs @@ -0,0 +1,41 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde_json::value::{to_raw_value, RawValue}; + +use crate::method::callable_method::RpcMethod; +use kubeos_manager::api; + +pub struct ConfigureMethod { + req: api::ConfigureRequest, +} + +impl ConfigureMethod { + pub fn new(req: api::ConfigureRequest) -> Self { + ConfigureMethod { req } + } + + pub fn set_configure_request(&mut self, req: api::ConfigureRequest) -> &Self { + self.req = req; + self + } +} + +impl RpcMethod for ConfigureMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "configure" + } + fn command_params(&self) -> Vec> { + vec![to_raw_value(&self.req).unwrap()] + } +} diff --git a/KubeOS-Rust/cli/src/method/mod.rs b/KubeOS-Rust/cli/src/method/mod.rs new file mode 100644 index 00000000..b04b0fd8 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/mod.rs @@ -0,0 +1,19 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod callable_method; +pub mod cleanup; +pub mod configure; +pub mod prepare_upgrade; +pub mod request; +pub mod rollback; +pub mod upgrade; diff --git a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs new file mode 100644 index 00000000..dd3157df --- /dev/null +++ b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs @@ -0,0 +1,41 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde_json::value::{to_raw_value, RawValue}; + +use crate::method::callable_method::RpcMethod; +use kubeos_manager::api; + +pub struct PrepareUpgradeMethod { + req: api::UpgradeRequest, +} + +impl PrepareUpgradeMethod { + pub fn new(req: api::UpgradeRequest) -> Self { + PrepareUpgradeMethod { req } + } + + pub fn set_prepare_upgrade_request(&mut self, req: api::UpgradeRequest) -> &Self { + self.req = req; + self + } +} + +impl RpcMethod for PrepareUpgradeMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "prepare_upgrade" + } + fn command_params(&self) -> Vec> { + vec![to_raw_value(&self.req).unwrap()] + } +} diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs new file mode 100644 index 00000000..4e3dbec6 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/request.rs @@ -0,0 +1,58 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::anyhow; +use jsonrpc::{Error, Response}; +use log::debug; +use serde_json::value::RawValue; + +use crate::client::Client; + +pub fn request( + client: &Client, + command: &str, + params: Vec>, +) -> Result { + let request = client.build_request(command, ¶ms); + let response = client.send_request(request).map_err(|e| parse_error(e)); + debug!("{:#?}", response); + response +} + +pub fn parse_error(error: Error) -> anyhow::Error { + match error { + Error::Transport(e) => { + anyhow!( + "Cannot connect to KubeOS os-agent unix socket, {}", + e.source() + .map(|e| e.to_string()) + .unwrap_or_else(|| "Connection timeout".to_string()) + ) + } + Error::Json(e) => { + debug!("Json parse error: {:?}", e); + anyhow!("Failed to parse response") + } + Error::Rpc(ref e) => match e.message == "Method not found" { + true => { + anyhow!("Method is unimplemented") + } + false => { + anyhow!("{}", e.message) + } + }, + _ => { + debug!("{:?}", error); + anyhow!("Response is invalid") + } + } +} diff --git a/KubeOS-Rust/cli/src/method/rollback.rs b/KubeOS-Rust/cli/src/method/rollback.rs new file mode 100644 index 00000000..5b9b0fde --- /dev/null +++ b/KubeOS-Rust/cli/src/method/rollback.rs @@ -0,0 +1,34 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde_json::value::RawValue; + +use crate::method::callable_method::RpcMethod; +use kubeos_manager::api; + +pub struct RollbackMethod {} + +impl RollbackMethod { + pub fn new() -> Self { + RollbackMethod {} + } +} + +impl RpcMethod for RollbackMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "rollback" + } + fn command_params(&self) -> Vec> { + vec![] + } +} diff --git a/KubeOS-Rust/cli/src/method/upgrade.rs b/KubeOS-Rust/cli/src/method/upgrade.rs new file mode 100644 index 00000000..9098e197 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/upgrade.rs @@ -0,0 +1,34 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde_json::value::RawValue; + +use crate::method::callable_method::RpcMethod; +use kubeos_manager::api; + +pub struct UpgradeMethod {} + +impl UpgradeMethod { + pub fn new() -> Self { + UpgradeMethod {} + } +} + +impl RpcMethod for UpgradeMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "upgrade" + } + fn command_params(&self) -> Vec> { + vec![] + } +} diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml new file mode 100644 index 00000000..39f49efd --- /dev/null +++ b/KubeOS-Rust/manager/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "manager" +version = "0.1.0" +edition = "2021" +description = "KubeOS os-agent manager" +license = "MulanPSL-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dev-dependencies] +tempfile = "3.2" +mockall = { version = "=0.11.3" } +predicates = "=2.0.1" + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } +log = { version = "0.4" } +anyhow = { version = "1.0" } +env_logger = { version = "0.9" } +lazy_static = { version = "1.4" } +regex = { version = "1.7.3" } +nix = { version = "0.26.2" } diff --git a/KubeOS-Rust/manager/src/api/agent_status.rs b/KubeOS-Rust/manager/src/api/agent_status.rs new file mode 100644 index 00000000..7a2edf7d --- /dev/null +++ b/KubeOS-Rust/manager/src/api/agent_status.rs @@ -0,0 +1,49 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde::{Deserialize, Serialize}; + +const AGENT_STATUS_UNKNOWN: &str = "UNKNOWN"; +const AGENT_STATUS_NOT_APPLIED: &str = "NOT-APPLIED"; +const AGENT_STATUS_UPGRADEREADY: &str = "UPGRADE-READY"; +const AGENT_STATUS_UPGRADED: &str = "UPGRADED"; +const AGENT_STATUS_ROLLBACKED: &str = "ROLLBACKED"; +const AGENT_STATUS_CONFIGURED: &str = "CONFIGURED"; + +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum AgentStatus { + Unknown, + NotApplied, + UpgradeReady, + Upgraded, + Rollbacked, + Configured, +} + +impl Default for AgentStatus { + fn default() -> Self { + Self::Unknown + } +} + +impl std::fmt::Display for AgentStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + AgentStatus::Unknown => AGENT_STATUS_UNKNOWN, + AgentStatus::NotApplied => AGENT_STATUS_NOT_APPLIED, + AgentStatus::UpgradeReady => AGENT_STATUS_UPGRADEREADY, + AgentStatus::Upgraded => AGENT_STATUS_UPGRADED, + AgentStatus::Rollbacked => AGENT_STATUS_ROLLBACKED, + AgentStatus::Configured => AGENT_STATUS_CONFIGURED, + }) + } +} diff --git a/KubeOS-Rust/manager/src/api/mod.rs b/KubeOS-Rust/manager/src/api/mod.rs new file mode 100644 index 00000000..01c9df1a --- /dev/null +++ b/KubeOS-Rust/manager/src/api/mod.rs @@ -0,0 +1,17 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod agent_status; +mod types; + +pub use agent_status::*; +pub use types::*; diff --git a/KubeOS-Rust/manager/src/api/types.rs b/KubeOS-Rust/manager/src/api/types.rs new file mode 100644 index 00000000..e21f55bf --- /dev/null +++ b/KubeOS-Rust/manager/src/api/types.rs @@ -0,0 +1,67 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::agent_status::*; +use crate::{ + sys_mgmt::CtrImageHandler, + utils::{CommandExecutor, UpgradeImageManager}, +}; + +#[derive(Deserialize, Serialize, Debug)] +pub struct UpgradeRequest { + pub version: String, + pub check_sum: String, + pub image_type: String, + pub container_image: String, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct KeyInfo { + pub value: String, + pub operation: String, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct Sysconfig { + pub model: String, + pub config_path: String, + pub contents: HashMap, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct ConfigureRequest { + pub configs: Vec, +} + +#[derive(Deserialize, Serialize, Debug, PartialEq)] +pub struct Response { + pub status: AgentStatus, +} + +pub enum ImageType { + Containerd(CtrImageHandler), +} + +impl ImageType { + pub fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result> { + match self { + ImageType::Containerd(handler) => handler.download_image(req), + } + } +} +pub trait ImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result>; +} diff --git a/KubeOS-Rust/manager/src/lib.rs b/KubeOS-Rust/manager/src/lib.rs new file mode 100644 index 00000000..b45cab99 --- /dev/null +++ b/KubeOS-Rust/manager/src/lib.rs @@ -0,0 +1,15 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod api; +pub mod sys_mgmt; +pub mod utils; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs new file mode 100644 index 00000000..01a09d70 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -0,0 +1,723 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + collections::HashMap, + fs::{self, File}, + io::{self, BufRead, BufWriter, Write}, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + string::String, +}; + +use anyhow::{anyhow, Context, Ok, Result}; +use lazy_static::lazy_static; +use log::{debug, info, trace, warn}; +use regex::Regex; + +use super::{api::*, values}; +use crate::utils::*; + +lazy_static! { + pub static ref CONFIG_TEMPLATE: HashMap> = { + let mut config_map = HashMap::new(); + config_map.insert( + values::KERNEL_SYSCTL.to_string(), + Box::new(KernelSysctl::new(values::DEFAULT_PROC_PATH)) as Box, + ); + config_map.insert( + values::KERNEL_SYSCTL_PERSIST.to_string(), + Box::new(KernelSysctlPersist) as Box, + ); + config_map.insert( + values::GRUB_CMDLINE_CURRENT.to_string(), + Box::new(GrubCmdline { + grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), + is_cur_partition: true, + }) as Box, + ); + config_map.insert( + values::GRUB_CMDLINE_NEXT.to_string(), + Box::new(GrubCmdline { + grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), + is_cur_partition: false, + }) as Box, + ); + config_map + }; +} + +pub trait Configuration { + fn set_config(&self, config: &mut Sysconfig) -> Result<()>; +} + +pub struct KernelSysctl { + pub proc_path: String, +} +pub struct KernelSysctlPersist; +pub struct GrubCmdline { + pub grub_path: String, + pub is_cur_partition: bool, +} + +impl Configuration for KernelSysctl { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + info!("Start set kernel.sysctl"); + for (key, key_info) in config.contents.iter() { + let proc_path = self.get_proc_path(key); + if key_info.operation == "delete" { + warn!("Failed to delete kernel.sysctl config with key \"{}\"", key); + } else if key_info.value != "" && key_info.operation == "" { + fs::write(&proc_path, format!("{}\n", &key_info.value).as_bytes()).with_context( + || format!("Failed to write kernel.sysctl with key: \"{}\"", key), + )?; + info!("Configured kernel.sysctl {}={}", key, key_info.value); + } else { + warn!( + "Failed to parse kernel.sysctl, key: \"{}\", value: \"{}\", operation: \"{}\"", + key, key_info.value, key_info.operation + ); + } + } + Ok(()) + } +} + +impl KernelSysctl { + fn new(proc_path: &str) -> Self { + Self { + proc_path: String::from(proc_path), + } + } + + fn get_proc_path(&self, key: &str) -> PathBuf { + let path_str = format!("{}{}", self.proc_path, key.replace(".", "/")); + Path::new(&path_str).to_path_buf() + } +} + +impl Configuration for KernelSysctlPersist { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + info!("Start set kernel.sysctl.persist"); + let mut config_path = &values::DEFAULT_KERNEL_CONFIG_PATH.to_string(); + if config.config_path != "" { + config_path = &config.config_path; + } + debug!("kernel.sysctl.persist config_path: \"{}\"", config_path); + create_config_file(config_path)?; + let configs = get_and_set_configs(&mut config.contents, config_path)?; + write_configs_to_file(config_path, &configs)?; + Ok(()) + } +} + +fn create_config_file(config_path: &str) -> Result<()> { + if !is_file_exist(config_path) { + let f = fs::File::create(config_path)?; + let metadata = f.metadata()?; + let mut permissions = metadata.permissions(); + permissions.set_mode(values::DEFAULT_KERNEL_CONFIG_PERM); + debug!("Create file {} with permission 0644", config_path); + } + Ok(()) +} + +fn get_and_set_configs( + expect_configs: &mut HashMap, + config_path: &str, +) -> Result> { + let f = File::open(config_path)?; + let mut configs_write = Vec::new(); + for line in io::BufReader::new(f).lines() { + let line = line?; + // if line is a comment or blank + if line.starts_with("#") || line.starts_with(";") || line.trim().is_empty() { + configs_write.push(line); + continue; + } + let config_kv: Vec<&str> = line.splitn(2, '=').map(|s| s.trim()).collect(); + // if config_kv is not a key-value pair + if config_kv.len() != 2 { + return Err(anyhow!("could not parse sysctl config {}", line)); + } + let new_key_info = expect_configs.get(config_kv[0]); + let new_config = match new_key_info { + Some(new_key_info) if new_key_info.operation == "delete" => { + handle_delete_key(&config_kv, new_key_info) + } + Some(new_key_info) => handle_update_key(&config_kv, new_key_info), + None => config_kv.join("="), + }; + configs_write.push(new_config); + expect_configs.remove(config_kv[0]); + } + let new_config = handle_add_key(expect_configs, false); + configs_write.extend(new_config); + Ok(configs_write) +} + +fn write_configs_to_file(config_path: &str, configs: &Vec) -> Result<()> { + info!("Write configuration to file \"{}\"", config_path); + let f = File::create(config_path)?; + let mut w = BufWriter::new(f); + for line in configs { + if line == "" { + continue; + } + writeln!(w, "{}", line.as_str())?; + } + w.flush() + .with_context(|| format!("Failed to flush file {}", config_path))?; + w.get_mut() + .sync_all() + .with_context(|| format!("Failed to sync"))?; + debug!("Write configuration to file \"{}\" success", config_path); + Ok(()) +} + +fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String { + let key = config_kv[0]; + if config_kv.len() == 1 && new_config_info.value == "" { + info!("Delete configuration key: \"{}\"", key); + return String::from(""); + } else if config_kv.len() == 1 && new_config_info.value != "" { + warn!( + "Failed to delete key \"{}\" with inconsistent values \"nil\" and \"{}\"", + key, new_config_info.value + ); + return key.to_string(); + } + let old_value = config_kv[1]; + if old_value != new_config_info.value { + warn!( + "Failed to delete key \"{}\" with inconsistent values \"{}\" and \"{}\"", + key, old_value, new_config_info.value + ); + return config_kv.join("="); + } + info!("Delete configuration {}={}", key, old_value); + String::from("") +} + +fn handle_update_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String { + let key = config_kv[0]; + if new_config_info.operation != "" { + warn!( + "Unknown operation \"{}\", updating key \"{}\" with value \"{}\" by default", + new_config_info.operation, key, new_config_info.value + ); + } + if config_kv.len() == values::ONLY_KEY && new_config_info.value == "" { + return key.to_string(); + } + let new_value = new_config_info.value.trim(); + if config_kv.len() == values::ONLY_KEY && new_config_info.value != "" { + info!("Update configuration \"{}={}\"", key, new_value); + return format!("{}={}", key, new_value); + } + if new_config_info.value == "" { + warn!("Failed to update key \"{}\" with \"null\" value", key); + return config_kv.join("="); + } + info!("Update configuration \"{}={}\"", key, new_value); + format!("{}={}", key, new_value) +} + +fn handle_add_key( + expect_configs: &HashMap, + is_only_key_valid: bool, +) -> Vec { + let mut configs_write = Vec::new(); + for (key, config_info) in expect_configs.iter() { + if config_info.operation == "delete" { + warn!("Failed to delete inexistent key: \"{}\"", key); + continue; + } + if key == "" || key.contains("=") { + warn!( + "Failed to add \"null\" key or key containing \"=\", key: \"{}\"", + key + ); + continue; + } + if config_info.operation != "" { + warn!( + "Unknown operation \"{}\", adding key \"{}\" with value \"{}\" by default", + config_info.operation, key, config_info.value + ); + } + let (k, v) = (key.trim(), config_info.value.trim()); + if v == "" && is_only_key_valid { + info!("Add configuration \"{}\"", k); + configs_write.push(k.to_string()); + } else if v == "" { + warn!("Failed to add key \"{}\" with \"null\" value", k); + } else { + info!("Add configuration \"{}={}\"", k, v); + configs_write.push(format!("{}={}", k, v)); + } + } + configs_write +} + +impl Configuration for GrubCmdline { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + if self.is_cur_partition { + info!("Start set grub.cmdline.current configuration"); + } else { + info!("Start set grub.cmdline.next configuration"); + } + if !is_file_exist(&self.grub_path) { + return Err(anyhow!("Failed to find grub.cfg file")); + } + if cfg!(test) {} + let config_partition = if cfg!(test) { + self.is_cur_partition + } else { + self.get_config_partition(RealCommandExecutor {})? + }; + debug!( + "Config_partition: {} (false means partition A, true means partition B)", + config_partition + ); + let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition)?; + write_configs_to_file(&self.grub_path, &configs)?; + Ok(()) + } +} + +impl GrubCmdline { + // get_config_partition returns false if the menuentry to be configured is A, true for menuentry B + fn get_config_partition(&self, executor: T) -> Result { + let (_, next_partition) = get_partition_info(&executor)?; + let mut flag = false; + if next_partition.menuentry == "B" { + flag = true + } + Ok(self.is_cur_partition != flag) + } +} + +fn get_and_set_grubcfg( + expect_configs: &mut HashMap, + grub_path: &str, + config_partition: bool, +) -> Result> { + let f = File::open(grub_path)?; + let re_find_cur_linux = r"^\s*linux.*root=.*"; + let re = Regex::new(re_find_cur_linux)?; + let mut configs_write = Vec::new(); + let mut match_config_partition = false; + for line in io::BufReader::new(f).lines() { + let mut line = line?; + if re.is_match(&line) { + if match_config_partition == config_partition { + line = modify_boot_cfg(expect_configs, &line)?; + } + match_config_partition = true; + } + configs_write.push(line); + } + Ok(configs_write) +} + +fn modify_boot_cfg(expect_configs: &mut HashMap, line: &String) -> Result { + trace!( + "Match partition that need to be configured, entering modify_boot_cfg, linux line: {}", + line + ); + let mut new_configs = vec![" ".to_string()]; + let olg_configs: Vec<&str> = line.split(' ').collect(); + for old_config in olg_configs { + if old_config == "" { + continue; + } + // At most 2 substrings can be returned to satisfy the case like root=UUID=xxxx + let config = old_config.splitn(2, "=").collect::>(); + if config.len() != values::ONLY_KEY && config.len() != values::KV_PAIR { + return Err(anyhow!( + "Failed to parse grub.cfg linux line {}", + old_config + )); + } + let new_key_info = expect_configs.get(config[0]); + let new_config = match new_key_info { + Some(new_key_info) if new_key_info.operation == "delete" => { + handle_delete_key(&config, new_key_info) + } + Some(new_key_info) => handle_update_key(&config, new_key_info), + None => config.join("="), + }; + if !new_config.is_empty() { + new_configs.push(new_config); + } + expect_configs.remove(config[0]); + } + let new_config = handle_add_key(expect_configs, true); + new_configs.extend(new_config); + Ok(new_configs.join(" ")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sys_mgmt::{ + GRUB_CMDLINE_CURRENT, GRUB_CMDLINE_NEXT, KERNEL_SYSCTL, KERNEL_SYSCTL_PERSIST, + }; + use mockall::{mock, predicate::*}; + use std::fs; + use tempfile::{NamedTempFile, TempDir}; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_config_partition() { + init(); + let mut grub_cmdline = GrubCmdline { + grub_path: String::from(""), + is_cur_partition: true, + }; + let mut executor = MockCommandExec::new(); + + // the output shows that current root menuentry is A + let command_output1 = + "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor + .expect_run_command_with_output() + .times(1) + .returning(|_, _| Ok(command_output1.to_string())); + + let result = grub_cmdline.get_config_partition(executor).unwrap(); + // it should return false because the current root menuentry is A and we want to configure current partition + assert_eq!(result, false); + + let mut executor = MockCommandExec::new(); + + // the output shows that current root menuentry is A + let command_output1 = + "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor + .expect_run_command_with_output() + .times(1) + .returning(|_, _| Ok(command_output1.to_string())); + grub_cmdline.is_cur_partition = false; + let result = grub_cmdline.get_config_partition(executor).unwrap(); + // it should return true because the current root menuentry is A and we want to configure next partition + assert_eq!(result, true); + } + + #[test] + fn test_kernel_sysctl() { + init(); + let tmp_dir = TempDir::new().unwrap(); + assert_eq!(tmp_dir.path().exists(), true); + let kernel_sysctl = KernelSysctl::new(tmp_dir.path().to_str().unwrap()); + + let config_detail = HashMap::from([ + ( + "a".to_string(), + KeyInfo { + value: "1".to_string(), + operation: "".to_string(), + }, + ), + ( + "b".to_string(), + KeyInfo { + value: "2".to_string(), + operation: "delete".to_string(), + }, + ), + ( + "c".to_string(), + KeyInfo { + value: "3".to_string(), + operation: "add".to_string(), + }, + ), + ( + "d".to_string(), + KeyInfo { + value: "".to_string(), + operation: "".to_string(), + }, + ), + ( + "e".to_string(), + KeyInfo { + value: "".to_string(), + operation: "delete".to_string(), + }, + ), + ]); + + let mut config = Sysconfig { + model: KERNEL_SYSCTL.to_string(), + config_path: String::from(""), + contents: config_detail, + }; + kernel_sysctl.set_config(&mut config).unwrap(); + + let result = + fs::read_to_string(format!("{}{}", tmp_dir.path().to_str().unwrap(), "a")).unwrap(); + assert_eq!(result, "1\n"); + } + + #[test] + fn test_kernel_sysctl_persist() { + init(); + let comment = r"# This file is managed by KubeOS for unit testing."; + // create a tmp file with comment + let mut tmp_file = tempfile::NamedTempFile::new().unwrap(); + writeln!(tmp_file, "{}", comment).unwrap(); + writeln!(tmp_file, "a=0").unwrap(); + let kernel_sysctl_persist = KernelSysctlPersist {}; + let config_detail = HashMap::from([ + ( + "a".to_string(), + KeyInfo { + value: "1".to_string(), + operation: "".to_string(), + }, + ), + ( + "b".to_string(), + KeyInfo { + value: "2".to_string(), + operation: "delete".to_string(), + }, + ), + ( + "c".to_string(), + KeyInfo { + value: "3".to_string(), + operation: "add".to_string(), + }, + ), + ]); + let mut config = Sysconfig { + model: KERNEL_SYSCTL_PERSIST.to_string(), + config_path: String::from(tmp_file.path().to_str().unwrap()), + contents: config_detail, + }; + kernel_sysctl_persist.set_config(&mut config).unwrap(); + let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); + let expected_res = format!("{}\n{}\n{}\n", comment, "a=1", "c=3"); + assert_eq!(result, expected_res); + + // test config_path is empty + // remember modify DEFAULT_KERNEL_CONFIG_PATH first + // let config_detail = HashMap::from([ + // ( + // "aaa".to_string(), + // KeyInfo { + // value: "3".to_string(), + // operation: "add".to_string(), + // }, + // ), + // ( + // "bbb".to_string(), + // KeyInfo { + // value: "1".to_string(), + // operation: "delete".to_string(), + // }, + // ), + // ]); + // config.config_path = "".to_string(); + // config.contents = config_detail; + // kernel_sysctl_persist.set_config(&mut config).unwrap(); + // let result = fs::read_to_string(crate::sys_mgmt::DEFAULT_KERNEL_CONFIG_PATH).unwrap(); + // let expected_res = format!("{}\n", "aaa=3",); + // assert_eq!(result, expected_res); + } + + #[test] + fn write_configs_to_file_tests() { + init(); + let path = "/home/yuhang/abc.txt"; + let configs = vec!["a=1".to_string(), "b=2".to_string()]; + write_configs_to_file(&path.to_string(), &configs).unwrap(); + } + + #[test] + fn test_grub_cmdline() { + init(); + let mut tmp_file = NamedTempFile::new().unwrap(); + let mut grub_cmdline = GrubCmdline { + grub_path: tmp_file.path().to_str().unwrap().to_string(), + is_cur_partition: true, + }; + let grub_cfg = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +} + +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=UUID=2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +}"; + writeln!(tmp_file, "{}", grub_cfg).unwrap(); + let config_first_part = HashMap::from([ + ( + "debug".to_string(), + KeyInfo { + value: "".to_string(), + operation: "".to_string(), + }, + ), + ( + "quiet".to_string(), + KeyInfo { + value: "".to_string(), + operation: "delete".to_string(), + }, + ), + ( + "panic".to_string(), + KeyInfo { + value: "5".to_string(), + operation: "".to_string(), + }, + ), + ( + "nomodeset".to_string(), + KeyInfo { + value: "".to_string(), + operation: "update".to_string(), + }, + ), + ( + "oops".to_string(), + KeyInfo { + value: "".to_string(), + operation: "".to_string(), + }, + ), + ( + "".to_string(), + KeyInfo { + value: "test".to_string(), + operation: "".to_string(), + }, + ), + ( + "selinux".to_string(), + KeyInfo { + value: "1".to_string(), + operation: "delete".to_string(), + }, + ), + ( + "acpi".to_string(), + KeyInfo { + value: "off".to_string(), + operation: "delete".to_string(), + }, + ), + ( + "ro".to_string(), + KeyInfo { + value: "1".to_string(), + operation: "".to_string(), + }, + ), + ]); + let mut config = Sysconfig { + model: GRUB_CMDLINE_CURRENT.to_string(), + config_path: String::new(), + contents: config_first_part, + }; + grub_cmdline.set_config(&mut config).unwrap(); + grub_cmdline.is_cur_partition = false; + let config_second = HashMap::from([ + ( + "pci".to_string(), + KeyInfo { + value: "nomis".to_string(), + operation: "".to_string(), + }, + ), + ( + "panic".to_string(), + KeyInfo { + value: "5".to_string(), + operation: "".to_string(), + }, + ), + ]); + config.contents = config_second; + config.model = GRUB_CMDLINE_NEXT.to_string(); + grub_cmdline.set_config(&mut config).unwrap(); + let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); + let expected_res = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 pci=nomis + initrd /boot/initramfs.img +} +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=UUID=2 ro=1 rootfstype=ext4 nomodeset oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 debug + initrd /boot/initramfs.img +} +"; + assert_eq!(result, expected_res); + } + + #[test] + fn test_create_config_file() { + init(); + let tmp_file = "/tmp/kubeos-test-create-config-file.txt"; + create_config_file(&tmp_file).unwrap(); + assert!(is_file_exist(&tmp_file)); + fs::remove_file(tmp_file).unwrap(); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs new file mode 100644 index 00000000..b4bdd2c8 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -0,0 +1,359 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs, os::unix::fs::PermissionsExt, path::Path}; + +use anyhow::{anyhow, Result}; +use log::{debug, info}; + +use super::api::{ImageHandler, UpgradeRequest}; +use crate::sys_mgmt::{IMAGE_PERMISSION, NEED_GB_SIZE, PERSIST_DIR}; +use crate::utils::*; + +pub struct CtrImageHandler { + pub paths: PreparePath, + pub executor: T, +} + +const DEFAULT_NAMESPACE: &str = "k8s.io"; + +impl ImageHandler for CtrImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + perpare_env(&self.paths, NEED_GB_SIZE, PERSIST_DIR, IMAGE_PERMISSION)?; + self.get_image(req)?; + self.get_rootfs_archive(req, IMAGE_PERMISSION)?; + + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new( + self.paths.clone(), + next_partition_info, + self.executor.clone(), + ); + img_manager.create_os_image(IMAGE_PERMISSION) + } +} + +impl Default for CtrImageHandler { + fn default() -> Self { + Self { + paths: PreparePath::default(), + executor: RealCommandExecutor {}, + } + } +} + +impl CtrImageHandler { + #[cfg(test)] + fn new(paths: PreparePath, executor: T) -> Self { + Self { paths, executor } + } + + fn get_image(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + is_valid_image_name(image_name)?; + info!("Start pull image {}", image_name); + let containerd_command: String; + if is_command_available("crictl", &self.executor) { + containerd_command = "crictl".to_string(); + } else { + containerd_command = "ctr".to_string(); + } + pull_image(&containerd_command, image_name, &self.executor)?; + info!("Start check image digest"); + check_oci_image_digest_match( + &containerd_command, + image_name, + &req.check_sum, + &self.executor, + )?; + Ok(()) + } + + fn get_rootfs_archive(&self, req: &UpgradeRequest, permission: u32) -> Result<()> { + let image_name = &req.container_image; + let mount_path = &self.paths.mount_path.to_str().ok_or_else(|| { + anyhow!( + "Failed to get mount path: {}", + self.paths.mount_path.display() + ) + })?; + info!("Start get rootfs {}", image_name); + self.check_and_unmount(mount_path)?; + self.executor.run_command( + "ctr", + &[ + "-n", + DEFAULT_NAMESPACE, + "images", + "mount", + "--rw", + image_name, + mount_path, + ], + )?; + // copy os.tar from mount_path to its partent dir + self.copy_file( + &self.paths.mount_path.join(&self.paths.rootfs_file), + &self.paths.tar_path, + permission, + )?; + self.check_and_unmount(mount_path)?; + Ok(()) + } + + fn check_and_unmount(&self, mount_path: &str) -> Result<()> { + let ctr_snapshot_cmd = format!( + "ctr -n={} snapshots ls | grep {} | awk '{{print $1}}'", + DEFAULT_NAMESPACE, mount_path + ); + let exist_snapshot = self + .executor + .run_command_with_output("bash", &["-c", &ctr_snapshot_cmd])?; + if !exist_snapshot.is_empty() { + self.executor.run_command( + "ctr", + &["-n", DEFAULT_NAMESPACE, "images", "unmount", mount_path], + )?; + self.executor.run_command( + "ctr", + &["-n", DEFAULT_NAMESPACE, "snapshots", "remove", mount_path], + )?; + } + Ok(()) + } + + fn copy_file, Q: AsRef>( + &self, + src: P, + dst: Q, + permission: u32, + ) -> Result<()> { + let copied_bytes = fs::copy(src.as_ref(), dst.as_ref())?; + debug!( + "Copy {} to {}, total bytes: {}", + src.as_ref().display(), + dst.as_ref().display(), + copied_bytes + ); + fs::set_permissions(dst, fs::Permissions::from_mode(permission))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mockall::mock; + use std::io::Write; + use std::path::Path; + use std::path::PathBuf; + use tempfile::NamedTempFile; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + }; + // mock is_command_available + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "/bin/sh" && args.contains(&"command -v crictl")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock pull_image + mock_executor + .expect_run_command() + .withf(|cmd, args| { + cmd == "crictl" + && args.contains(&"pull") + && args.contains(&"docker.io/library/busybox:latest") + }) + .times(1) + .returning(|_, _| Ok(())); + // mock get_oci_image_digest + let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]"; + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "crictl" + && args.contains(&"inspecti") + && args.contains(&"{{.status.repoDigests}}") + }) + .times(1) + .returning(|_, _| Ok(command_output2.to_string())); + let ctr = CtrImageHandler::new(PreparePath::default(), mock_executor); + let result = ctr.get_image(&req); + assert!(result.is_ok()); + } + + #[test] + fn test_get_rootfs_archive() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + }; + + // mock check_and_unmount + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure + .times(1) + .returning(|_, _| Ok("".to_string())); + + // mock ctr mount rw + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[4] == "--rw") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + // create temp file for copy + let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file."); + writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file."); + + // Get the path of the temporary file and the path where it should be copied. + let src_dir = tmp_file.path().parent().unwrap(); + let src_file_name = tmp_file + .path() + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file."); + let dst_path = dst_file.path().to_path_buf(); + + let paths = PreparePath { + update_path: PathBuf::new(), + image_path: PathBuf::new(), + mount_path: src_dir.to_path_buf(), + rootfs_file: src_file_name.clone(), + tar_path: dst_path.clone(), + }; + + // mock check_and_unmount + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure + .times(1) + .returning(|_, _| Ok("".to_string())); + + let ctr = CtrImageHandler::new(paths, mock_executor); + let result = ctr.get_rootfs_archive(&req, IMAGE_PERMISSION); + assert!(result.is_ok()); + } + + #[test] + fn test_copy_file() { + // Setup: Create a temporary file and write some data to it. + let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file."); + writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file."); + + // Get the path of the temporary file and the path where it should be copied. + let src_path = tmp_file.path().to_str().unwrap().to_string(); + let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file."); + let dst_path = dst_file.path().to_str().unwrap().to_string(); + + let ctr = CtrImageHandler::default(); + let result = ctr.copy_file(&src_path, &dst_path, IMAGE_PERMISSION); + + assert!(result.is_ok()); + + let expected_content = "Hello, world!\n"; + let actual_content = + fs::read_to_string(&dst_path).expect("Failed to read destination file."); + assert_eq!(expected_content, actual_content); + + // Assert the file permission + let metadata = fs::metadata(&dst_path).expect("Failed to read destination file."); + let expected_permission = 0o100600; + assert_eq!(metadata.permissions().mode(), expected_permission); + } + + #[test] + fn test_check_and_unmount() { + let mut mock_executor = MockCommandExec::new(); + + // When `run_command_with_output` is called with "bash" and the specific args, it will return Ok("snapshot_exists"). + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") + .times(1) + .returning(|_, _| Ok("snapshot_exists".to_string())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"images")) + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"snapshots")) + .times(1) + .returning(|_, _| Ok(())); + + let result = CtrImageHandler::new(PreparePath::default(), mock_executor) + .check_and_unmount("test_mount_path"); + + assert!(result.is_ok()); + } + + #[test] + #[ignore] + fn test_download_image() { + init(); + let ctr = CtrImageHandler { + paths: PreparePath::default(), + executor: RealCommandExecutor {}, + }; + let update_req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: "docker.io/library/busybox:latest".to_string(), + check_sum: "".to_string(), + }; + ctr.download_image(&update_req).unwrap(); + let tar_path = "/persist/KubeOS-Update/os.tar"; + assert_eq!(true, Path::new(tar_path).exists()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/mod.rs b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs new file mode 100644 index 00000000..446d072c --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs @@ -0,0 +1,21 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use super::api; + +mod config; +mod containerd_image; +mod values; + +pub use config::*; +pub use containerd_image::*; +pub use values::*; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/values.rs b/KubeOS-Rust/manager/src/sys_mgmt/values.rs new file mode 100644 index 00000000..3452b4ad --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/values.rs @@ -0,0 +1,35 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub const KERNEL_SYSCTL: &str = "kernel.sysctl"; +pub const KERNEL_SYSCTL_PERSIST: &str = "kernel.sysctl.persist"; +pub const GRUB_CMDLINE_CURRENT: &str = "grub.cmdline.current"; +pub const GRUB_CMDLINE_NEXT: &str = "grub.cmdline.next"; + +pub const DEFAULT_PROC_PATH: &str = "/proc/sys/"; +pub const DEFAULT_KERNEL_CONFIG_PATH: &str = "/etc/sysctl.conf"; +pub const DEFAULT_GRUB_CFG_PATH: &str = "/boot/efi/EFI/openEuler/grub.cfg"; +pub const DEFAULT_GRUBENV_PATH: &str = "/boot/efi/EFI/openEuler/grubenv"; + +pub const PERSIST_DIR: &str = "/persist"; +pub const ROOTFS_ARCHIVE: &str = "os.tar"; +pub const UPDATE_DIR: &str = "KubeOS-Update"; +pub const MOUNT_DIR: &str = "kubeos-update"; +pub const OS_IMAGE_NAME: &str = "update.img"; + +pub const DEFAULT_KERNEL_CONFIG_PERM: u32 = 0o644; +pub const DEFAULT_GRUB_CFG_PERM: u32 = 0o751; +pub const IMAGE_PERMISSION: u32 = 0o600; + +pub const ONLY_KEY: usize = 1; +pub const KV_PAIR: usize = 2; +pub const NEED_GB_SIZE: i64 = 3; diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs new file mode 100644 index 00000000..cc826dd4 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -0,0 +1,312 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs, + os::linux::fs::MetadataExt, + os::unix::fs::DirBuilderExt, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Result}; +use log::{debug, info}; +use nix::{mount, mount::MntFlags}; + +use super::executor::CommandExecutor; +use crate::sys_mgmt::{MOUNT_DIR, OS_IMAGE_NAME, PERSIST_DIR, ROOTFS_ARCHIVE, UPDATE_DIR}; + +#[derive(Clone)] +pub struct PreparePath { + pub update_path: PathBuf, // update_path: /persist/KubeOS-Update + pub mount_path: PathBuf, // mount_path: /persist/KubeOS-Update/kubeos-update + pub tar_path: PathBuf, // tar_path: /persist/KubeOS-Update/os.tar + pub image_path: PathBuf, // image_path: /persist/update.img + pub rootfs_file: String, // rootfs_file: os.tar +} + +impl Default for PreparePath { + fn default() -> Self { + let update_pathbuf = Path::new(PERSIST_DIR).join(UPDATE_DIR); + let persist_dir = Path::new(PERSIST_DIR); + Self { + update_path: update_pathbuf.clone(), + mount_path: update_pathbuf.join(MOUNT_DIR), + tar_path: update_pathbuf.join(ROOTFS_ARCHIVE), + image_path: persist_dir.join(OS_IMAGE_NAME), + rootfs_file: ROOTFS_ARCHIVE.to_string(), + } + } +} + +pub fn is_file_exist>(path: P) -> bool { + path.as_ref().exists() +} + +pub fn perpare_env( + prepare_path: &PreparePath, + need_gb: i64, + persist_path: &str, + permission: u32, +) -> Result<()> { + info!("Prepare environment to upgrade"); + check_disk_size(need_gb, persist_path)?; + clean_env( + &prepare_path.update_path, + &prepare_path.mount_path, + &prepare_path.image_path, + )?; + fs::DirBuilder::new() + .recursive(true) + .mode(permission) + .create(&prepare_path.mount_path)?; + Ok(()) +} + +pub fn check_disk_size(need_gb: i64, path: &str) -> Result<()> { + info!("Check if there is enough disk space to upgrade"); + let kb = 1024; + let fs_stat = nix::sys::statfs::statfs(path)?; + let need_disk_size = need_gb * kb * kb * kb; + let available_blocks = i64::try_from(fs_stat.blocks_available())?; + let available_space = available_blocks * fs_stat.block_size(); + if available_space < need_disk_size { + return Err(anyhow!("Space is not enough for downloading")); + } + Ok(()) +} + +// clean_env will umount the mount path and delete all files in /persist/KubeOS-Update and update.img +pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> +where + P: AsRef, +{ + info!("Clean upgrade environment"); + if is_mounted(&mount_path)? { + debug!("Umount {}", mount_path.as_ref().display()); + if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) { + return Err(anyhow!( + "Failed to umount {} in clean_env: {}", + mount_path.as_ref().display(), + errno + )); + } + } + // losetup -D? + delete_file_or_dir(update_path)?; + delete_file_or_dir(image_path)?; + Ok(()) +} + +pub fn delete_file_or_dir>(path: P) -> Result<()> { + if is_file_exist(&path) { + if fs::metadata(&path)?.is_file() { + debug!("Delete file {}", path.as_ref().display()); + fs::remove_file(&path)?; + } else { + debug!("Delete directory {}", path.as_ref().display()); + fs::remove_dir_all(&path)?; + } + } + Ok(()) +} + +pub fn is_command_available(command: &str, command_executor: &T) -> bool { + match command_executor.run_command( + "/bin/sh", + &["-c", format!("command -v {}", command).as_str()], + ) { + Ok(_) => { + debug!("command {} is available", command); + true + } + Err(_) => { + debug!("command {} is not available", command); + false + } + } +} + +pub fn is_mounted>(mount_path: P) -> Result { + if !is_file_exist(&mount_path) { + return Ok(false); + } + // Get device ID of mountPath + let mount_meta = fs::symlink_metadata(&mount_path)?; + let dev = mount_meta.st_dev(); + + // Get device ID of mountPath's parent directory + let parent = mount_path.as_ref().parent().ok_or_else(|| { + anyhow!( + "Failed to get parent directory of {}", + mount_path.as_ref().display() + ) + })?; + let parent_meta = fs::symlink_metadata(parent)?; + let dev_parent = parent_meta.st_dev(); + Ok(dev != dev_parent) +} + +pub fn switch_boot_menuentry( + command_executor: &T, + grub_env_path: &str, + next_menuentry: &str, +) -> Result<()> { + if get_boot_mode() == "uefi" { + command_executor.run_command( + "grub2-editenv", + &[ + grub_env_path, + "set", + format!("saved_entry={}", next_menuentry).as_str(), + ], + )?; + } else { + command_executor.run_command("grub2-set-default", &[next_menuentry])?; + } + Ok(()) +} + +pub fn get_boot_mode() -> String { + if is_file_exist("/sys/firmware/efi") { + "uefi".into() + } else { + "bios".into() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mockall::{mock, predicate::*}; + use tempfile::NamedTempFile; + use tempfile::TempDir; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_file_exist() { + init(); + let path = "/tmp/test_is_file_exist"; + assert_eq!(is_file_exist(path), false); + + let file = NamedTempFile::new().unwrap(); + assert_eq!(is_file_exist(file.path().to_str().unwrap()), true); + + let tmp_dir = TempDir::new().unwrap(); + assert_eq!(is_file_exist(tmp_dir.path().to_str().unwrap()), true); + } + + #[test] + fn test_prepare_env() { + init(); + let paths = PreparePath { + update_path: PathBuf::from("/tmp/test_prepare_env"), + mount_path: PathBuf::from("/tmp/test_prepare_env/kubeos-update"), + tar_path: PathBuf::from("/tmp/test_prepare_env/os.tar"), + image_path: PathBuf::from("/tmp/test_prepare_env/update.img"), + rootfs_file: "os.tar".to_string(), + }; + perpare_env(&paths, 1, "/home", 0o700).unwrap(); + } + + #[test] + fn test_check_disk_size() { + init(); + let path = "/home"; + let need_gb = 1; + let result = check_disk_size(need_gb, path); + assert!(result.is_ok()); + let need_gb = 1000; + let result = check_disk_size(need_gb, path); + assert!(result.is_err()); + } + + #[test] + fn test_clean_env() { + init(); + let update_path = "/tmp/test_clean_env"; + let mount_path = "/tmp/test_clean_env/kubeos-update"; + let image_path = "/tmp/test_clean_env/update.img"; + clean_env( + &update_path.to_string(), + &mount_path.to_string(), + &image_path.to_string(), + ) + .unwrap(); + } + + #[test] + fn test_delete_file_or_dir() { + init(); + let path = "/tmp/test_delete_file"; + fs::File::create(path).unwrap(); + assert_eq!(Path::new(path).exists(), true); + delete_file_or_dir(&path.to_string()).unwrap(); + assert_eq!(Path::new(path).exists(), false); + + let path = "/tmp/test_dir"; + fs::create_dir(path).unwrap(); + assert_eq!(Path::new(path).exists(), true); + delete_file_or_dir(&path.to_string()).unwrap(); + assert_eq!(Path::new(path).exists(), false); + + let path = "/tmp/nonexist"; + delete_file_or_dir(path).unwrap(); + + let path = PathBuf::new(); + delete_file_or_dir(path).unwrap(); + } + + #[test] + fn test_switch_boot_menuentry() { + init(); + let grubenv_path = "/boot/efi/EFI/openEuler/grubenv"; + let next_menuentry = "B"; + let mut mock = MockCommandExec::new(); + mock.expect_run_command() + .withf(move |name, args| { + name == "grub2-editenv" + && args[0] == grubenv_path + && args[2] == format!("saved_entry={}", next_menuentry).as_str() + }) + .times(1) // Expect it to be called once + .returning(move |_, _| Ok(())); + + switch_boot_menuentry(&mock, grubenv_path, next_menuentry).unwrap() + } + + #[test] + #[ignore] + fn test_get_boot_mode() { + init(); + let boot_mode = get_boot_mode(); + assert!(boot_mode == "uefi"); + } +} diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs new file mode 100644 index 00000000..8c5bfaf7 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/container_image.rs @@ -0,0 +1,271 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::{anyhow, Result}; +use log::{debug, trace}; +use regex::Regex; + +use super::executor::CommandExecutor; + +pub fn is_valid_image_name(image: &str) -> Result<()> { + let pattern = r"^(?P[a-z0-9\-.]+\.[a-z0-9\-]+:?[0-9]*)?/?((?P[a-zA-Z0-9-_]+?)|(?P[a-zA-Z0-9-_]+?)/(?P[a-zA-Z-_]+?))(?P(?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$"; + let reg_ex = Regex::new(pattern)?; + if !reg_ex.is_match(image) { + return Err(anyhow!("Invalid image name: {}", image)); + } + trace!("Image name {} is valid", image); + Ok(()) +} + +pub fn check_oci_image_digest_match( + container_runtime: &str, + image_name: &str, + check_sum: &str, + command_executor: &T, +) -> Result<()> { + let image_digests = get_oci_image_digest(container_runtime, image_name, command_executor)?; + if image_digests != check_sum { + return Err(anyhow!( + "Image digest mismatch, expect {}, got {}", + check_sum, + image_digests + )); + } + Ok(()) +} + +pub fn get_oci_image_digest( + container_runtime: &str, + image_name: &str, + executor: &T, +) -> Result { + let cmd_output: String; + match container_runtime { + "crictl" => { + cmd_output = executor.run_command_with_output( + "crictl", + &[ + "inspecti", + "--output", + "go-template", + "--template", + "{{.status.repoDigests}}", + image_name, + ], + )?; + } + "docker" => { + cmd_output = executor.run_command_with_output( + "docker", + &["inspect", "--format", "{{.RepoDigests}}", image_name], + )?; + } + "ctr" => { + cmd_output = executor.run_command_with_output( + "ctr", + &[ + "-n", + "k8s.io", + "images", + "ls", + &format!("name=={}", image_name), + ], + )?; + // Split by whitespaces, we get vec like [REF TYPE DIGEST SIZE PLATFORMS LABELS x x x x x x] + // get the 8th element, and split by ':' to get the digest + let fields: Vec<&str> = cmd_output.split_whitespace().collect(); + if let Some(digest) = fields.get(8).and_then(|field| field.split(':').nth(1)) { + trace!("get_oci_image_digest: {}", digest); + return Ok(digest.to_string()); + } else { + return Err(anyhow!( + "Failed to get digest from ctr command output: {}", + cmd_output + )); + } + } + _ => { + return Err(anyhow!( + "Container runtime {} cannot be recognized", + container_runtime + )); + } + } + + // Parse the cmd_output to extract the digest + let parts: Vec<&str> = cmd_output.split('@').collect(); + if let Some(last_part) = parts.last() { + if last_part.starts_with("sha256") { + let parsed_parts: Vec<&str> = last_part.trim_matches(|c| c == ']').split(':').collect(); + // After spliiing by ':', we should get vec like [sha256, digests] + if parsed_parts.len() == 2 { + trace!("get_oci_image_digest: {}", parsed_parts[1]); + return Ok(parsed_parts[1].to_string()); // 1 is the index of digests + } + } + } + + Err(anyhow!( + "Failed to get digest from command output: {}", + cmd_output + )) +} + +pub fn pull_image(runtime: &str, image_name: &str, executor: &T) -> Result<()> { + debug!("Pull image {}", image_name); + match runtime { + "crictl" => { + executor.run_command("crictl", &["pull", image_name])?; + } + "ctr" => { + executor.run_command( + "ctr", + &[ + &"-n", + "k8s.io", + "images", + "pull", + "--hosts-dir", + "/etc/containerd/certs.d", + image_name, + ], + )?; + } + "docker" => { + executor.run_command("docker", &["pull", image_name])?; + } + _ => { + return Err(anyhow!( + "Container runtime {} cannot be recognized", + runtime + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use mockall::{mock, predicate::*}; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_valid_image_name() { + init(); + let out = is_valid_image_name("nginx").unwrap(); + assert_eq!(out, ()); + let out = is_valid_image_name( + "docker.example.com:5000/gmr/alpine@sha256:11111111111111111111111111111111", + ) + .unwrap(); + assert_eq!(out, ()); + let out = is_valid_image_name( + "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8", + ); + match out { + Ok(_) => assert_eq!(true, false), + Err(_) => assert_eq!(true, true), + } + } + + #[test] + fn test_get_oci_image_digest() { + init(); + let mut mock = MockCommandExec::new(); + let container_runtime = "ctr"; + let image_name = "docker.io/nginx:latest"; + let command_output1 = "REF TYPE DIGEST SIZE PLATFORMS LABELS\ndocker.io/nginx:latest text/html sha256:1111 132.5 KIB - -\n"; + mock.expect_run_command_with_output() + .times(1) + .returning(|_, _| Ok(command_output1.to_string())); + let out1 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); + let expect_output = "1111"; + assert_eq!(out1, expect_output); + + let container_runtime = "crictl"; + let command_output2 = "[docker.io/nginx@sha256:1111]"; + mock.expect_run_command_with_output() + .times(1) + .returning(|_, _| Ok(command_output2.to_string())); + let out2 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); + assert_eq!(out2, expect_output); + } + + #[test] + fn test_check_oci_image_digest_match() { + init(); + let mut mock = MockCommandExec::new(); + let image_name = "docker.io/nginx:latest"; + let container_runtime = "crictl"; + let command_output = "[docker.io/nginx@sha256:1111]"; + let check_sum = "1111"; + mock.expect_run_command_with_output() + .times(1) + .returning(|_, _| Ok(command_output.to_string())); + let result = check_oci_image_digest_match(container_runtime, image_name, check_sum, &mock); + assert!(result.is_ok()); + } + + #[test] + fn test_pull_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.len() == 2 && args[0] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[3] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.len() == 2 && args[0] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + let image_name = "docker.io/nginx:latest"; + let result = pull_image("crictl", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("ctr", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("docker", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("aaa", image_name, &mock_executor); + assert!(result.is_err()); + } +} diff --git a/KubeOS-Rust/manager/src/utils/executor.rs b/KubeOS-Rust/manager/src/utils/executor.rs new file mode 100644 index 00000000..5c70c8e4 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/executor.rs @@ -0,0 +1,101 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::process::Command; + +use anyhow::{anyhow, Result}; +use log::trace; + +pub trait CommandExecutor: Clone { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; +} + +#[derive(Clone)] +pub struct RealCommandExecutor {} + +impl CommandExecutor for RealCommandExecutor { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()> { + let output = Command::new(name).args(args).output()?; + if !output.status.success() { + let error_message = String::from_utf8_lossy(&output.stderr); + return Err(anyhow!( + "Failed to run command: {} {:?}, stderr: {}", + name, + args, + error_message + )); + } + trace!("run_command: {} {:?} done", name, args); + Ok(()) + } + + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result { + let output = Command::new(name).args(args).output()?; + if !output.status.success() { + let error_message = String::from_utf8_lossy(&output.stderr); + return Err(anyhow!( + "Failed to run command: {} {:?}, stderr: {}", + name, + args, + error_message + )); + } + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + trace!("run_command_with_output: {} {:?} done", name, args); + Ok(stdout.trim_end_matches("\n").to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_run_command_with_output() { + init(); + let executor: RealCommandExecutor = RealCommandExecutor {}; + + // test run_command_with_output + let output = executor + .run_command_with_output("echo", &["hello", "world"]) + .unwrap(); + assert_eq!(output, "hello world"); + let out = executor + .run_command_with_output("sh", &["-c", format!("command -v {}", "cat").as_str()]) + .unwrap(); + assert_eq!(out, "/usr/bin/cat"); + let out = executor + .run_command_with_output("sh", &["-c", format!("command -v {}", "apple").as_str()]); + assert!(out.is_err()); + } + + #[test] + fn test_run_command() { + init(); + let executor: RealCommandExecutor = RealCommandExecutor {}; + // test run_command + let out = executor.run_command("sh", &["-c", format!("command -v {}", "apple").as_str()]); + assert!(out.is_err()); + + let out = executor.run_command("sh", &["-c", format!("command -v {}", "cat").as_str()]); + assert!(out.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs new file mode 100644 index 00000000..42b5a745 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/image_manager.rs @@ -0,0 +1,260 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{self, Permissions}, + os::unix::fs::PermissionsExt, + path::PathBuf, +}; + +use anyhow::{Context, Result}; +use log::{debug, info}; + +use super::{ + clean_env, + common::{delete_file_or_dir, PreparePath}, + executor::CommandExecutor, + partition::PartitionInfo, +}; +use crate::{sys_mgmt::DEFAULT_GRUBENV_PATH, utils::switch_boot_menuentry}; + +pub struct UpgradeImageManager { + pub paths: PreparePath, + pub next_partition: PartitionInfo, + pub executor: T, +} + +impl UpgradeImageManager { + pub fn new(paths: PreparePath, next_partition: PartitionInfo, executor: T) -> Self { + Self { + paths, + next_partition, + executor, + } + } + + fn image_path_str(&self) -> Result<&str> { + self.paths + .image_path + .to_str() + .context("Failed to convert image path to string") + } + + fn mount_path_str(&self) -> Result<&str> { + self.paths + .mount_path + .to_str() + .context("Failed to convert mount path to string") + } + + fn tar_path_str(&self) -> Result<&str> { + self.paths + .tar_path + .to_str() + .context("Failed to convert tar path to string") + } + + pub fn create_image_file(&self, permission: u32) -> Result<()> { + let image_str = self.image_path_str()?; + + debug!("Create image {}", image_str); + self.executor.run_command( + "dd", + &[ + "if=/dev/zero", + &format!("of={}", image_str), + "bs=2M", + "count=1024", + ], + )?; + fs::set_permissions(&self.paths.image_path, Permissions::from_mode(permission))?; + Ok(()) + } + + pub fn format_image(&self) -> Result<()> { + let image_str = self.image_path_str()?; + debug!("Format image {}", image_str); + self.executor.run_command( + format!("mkfs.{}", self.next_partition.fs_type).as_str(), + &[ + "-L", + format!("ROOT-{}", self.next_partition.menuentry).as_str(), + image_str, + ], + )?; + Ok(()) + } + + pub fn mount_image(&self) -> Result<()> { + let image_str = self.image_path_str()?; + let mount_str = self.mount_path_str()?; + debug!("Mount {} to {}", image_str, mount_str); + self.executor + .run_command("mount", &["-o", "loop", image_str, mount_str])?; + Ok(()) + } + + pub fn extract_tar_to_image(&self) -> Result<()> { + let tar_str = self.tar_path_str()?; + let mount_str = self.mount_path_str()?; + debug!("Extract {} to mounted path {}", tar_str, mount_str); + self.executor + .run_command("tar", &["-xvf", tar_str, "-C", mount_str])?; + Ok(()) + } + + pub fn create_os_image(self, permission: u32) -> Result { + self.create_image_file(permission)?; + self.format_image()?; + self.mount_image()?; + self.extract_tar_to_image()?; + // Pass empty image_path to clean_env to avoid delete image file + clean_env( + &self.paths.update_path, + &self.paths.mount_path, + &PathBuf::new(), + )?; + Ok(self) + } + + pub fn install(&self) -> Result<()> { + let image_str = self.image_path_str()?; + let device = self.next_partition.device.as_str(); + let menuentry = self.next_partition.menuentry.as_str(); + self.executor.run_command( + "dd", + &[ + format!("if={}", image_str).as_str(), + format!("of={}", device).as_str(), + "bs=8M", + ], + )?; + debug!("Install image {} to {} done", image_str, device); + // based on boot mode use different command to switch boot partition + switch_boot_menuentry(&self.executor, DEFAULT_GRUBENV_PATH, menuentry)?; + info!( + "Switch to boot partition: {}, device: {}", + menuentry, device + ); + delete_file_or_dir(image_str)?; + debug!("Remove image {}", image_str); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mockall::{mock, predicate::*}; + use std::{fs, io::Write, path::Path}; + use tempfile::NamedTempFile; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_update_image_manager() { + init(); + // create a dir in tmp dir + let tmp_dir = "/tmp/test_update_image_manager"; + let img_path = format!("{}/test_image", tmp_dir); + let mut temp_file = NamedTempFile::new().unwrap(); + write!(temp_file, "test content").unwrap(); // Writing s + fs::create_dir(tmp_dir).unwrap(); + let clone_img_path = img_path.clone(); + + let mut mock = MockCommandExec::new(); + //mock create_image_file + mock.expect_run_command() + .withf(|name, args| name == "dd" && args[0] == "if=/dev/zero") + .times(1) // Expect it to be called once + .returning(move |_, _| { + // simulate 'dd' by copying the contents of the temporary file + std::fs::copy(temp_file.path(), &clone_img_path).unwrap(); + Ok(()) + }); + + //mock format_image + mock.expect_run_command() + .withf(|name, args| name == "mkfs.ext4" && args[1] == "ROOT-B") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock mount_image + mock.expect_run_command() + .withf(|name, _| name == "mount") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock extract_tar_to_image + mock.expect_run_command() + .withf(|name, args| name == "tar" && args[0] == "-xvf") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock install->dd + mock.expect_run_command() + .withf(|name, _| name == "dd") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock install->grub2-set-default + mock.expect_run_command() + .withf(|name, args| { + name == "grub2-editenv" + && args[0] == "/boot/efi/EFI/openEuler/grubenv" + && args[1] == "set" + && args[2] == "saved_entry=B" + }) + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + let img_manager = UpgradeImageManager::new( + PreparePath { + update_path: tmp_dir.into(), + image_path: img_path.into(), + mount_path: "/tmp/update/mount".into(), + tar_path: "/tmp/update/image.tar".into(), + rootfs_file: "image.tar".into(), + }, + PartitionInfo { + device: "/dev/sda3".into(), + fs_type: "ext4".into(), + menuentry: "B".into(), + }, + mock, + ); + + let img_manager = img_manager.create_os_image(0o755).unwrap(); + let result = img_manager.install(); + assert!(result.is_ok()); + + assert_eq!(Path::new(&tmp_dir).exists(), false); + } +} diff --git a/KubeOS-Rust/manager/src/utils/mod.rs b/KubeOS-Rust/manager/src/utils/mod.rs new file mode 100644 index 00000000..caf406e3 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/mod.rs @@ -0,0 +1,23 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod common; +mod container_image; +mod executor; +mod image_manager; +mod partition; + +pub use common::*; +pub use container_image::*; +pub use executor::*; +pub use image_manager::*; +pub use partition::*; diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs new file mode 100644 index 00000000..8d59e174 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/partition.rs @@ -0,0 +1,110 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::{anyhow, Result}; +use log::{debug, trace}; + +use super::executor::CommandExecutor; + +#[derive(PartialEq, Debug, Default)] +pub struct PartitionInfo { + pub device: String, + pub menuentry: String, + pub fs_type: String, +} + +pub fn get_partition_info( + executor: &T, +) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> { + let lsblk = executor.run_command_with_output("lsblk", &["-lno", "NAME,MOUNTPOINTS,FSTYPE"])?; + // After split whitespace, the root directory line should have 3 elements, which are "sda2 / ext4". + let mut cur_partition = PartitionInfo::default(); + let mut next_partition = PartitionInfo::default(); + let splitted_len = 3; + trace!("get_partition_info lsblk command output:\n{}", lsblk); + for line in lsblk.lines() { + let res: Vec<&str> = line.split_whitespace().collect(); + if res.len() == splitted_len && res[1] == "/" { + debug!("root directory line: device={}, fs_type={}", res[0], res[2]); + cur_partition.device = format!("/dev/{}", res[0]).to_string(); + cur_partition.fs_type = res[2].to_string(); + next_partition.fs_type = res[2].to_string(); + if res[0].contains("2") { + cur_partition.menuentry = String::from("A"); + next_partition.menuentry = String::from("B"); + next_partition.device = format!("/dev/{}", res[0].replace("2", "3")).to_string(); + } else if res[0].contains("3") { + cur_partition.menuentry = String::from("B"); + next_partition.menuentry = String::from("A"); + next_partition.device = format!("/dev/{}", res[0].replace("3", "2")).to_string(); + } + } + } + if cur_partition.device.is_empty() { + return Err(anyhow!( + "Failed to get partition info, lsblk output: {}", + lsblk + )); + } + Ok((cur_partition, next_partition)) +} + +#[cfg(test)] +mod tests { + use super::*; + use mockall::{mock, predicate::*}; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_partition_info() { + init(); + let command_output1 = + "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + let mut mock = MockCommandExec::new(); + mock.expect_run_command_with_output() + .times(1) + .returning(|_, _| Ok(command_output1.to_string())); + let res = get_partition_info(&mock).unwrap(); + let expect_res = ( + PartitionInfo { + device: "/dev/sda2".to_string(), + menuentry: "A".to_string(), + fs_type: "ext4".to_string(), + }, + PartitionInfo { + device: "/dev/sda3".to_string(), + menuentry: "B".to_string(), + fs_type: "ext4".to_string(), + }, + ); + assert_eq!(res, expect_res); + } +} diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml new file mode 100644 index 00000000..58cb1f51 --- /dev/null +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "proxy" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +kube = { version = "0.66.0", features = ["runtime", "derive"] } +k8s-openapi = { version = "0.13.1", features = ["v1_22"] } +tokio = { version = "=1.14.0", features = ["rt-multi-thread", "macros"] } +anyhow = "1.0.44" +futures = "0.3.17" +serde = { version = "1.0.130", features = ["derive"] } +serde_json = "1.0.68" +thiserror = "1.0.29" +env_logger = "0.9.0" +tracing = "0.1.29" +schemars = "=0.8.10" +socket2 = "=0.4.9" +log = "=0.4.15" +thread_local = "=1.1.4" +async-trait = "0.1" +regex = "=1.7.3" +chrono = { version = "0.4", default-features = false, features = ["std"] } +snafu = "0.7" +h2 = "=0.3.16" +tokio-retry = "0.3" +reqwest = { version = "=0.11.10", default-features = false, features = [ "json" ] } +cli = { version = "0.1.0", path = "../cli" } +manager = { version = "0.1.0", path = "../manager" } diff --git a/KubeOS-Rust/proxy/src/controller/apiclient.rs b/KubeOS-Rust/proxy/src/controller/apiclient.rs new file mode 100644 index 00000000..5ee28004 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/apiclient.rs @@ -0,0 +1,172 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use super::crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}; +use super::values::{LABEL_OSINSTANCE, NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}; +use anyhow::Result; +use apiclient_error::Error; +use async_trait::async_trait; +use kube::{ + api::{Api, ObjectMeta, Patch, PatchParams, PostParams}, + Client, +}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceSpecPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + spec: OSInstanceSpec, +} + +impl Default for OSInstanceSpecPatch { + fn default() -> Self { + OSInstanceSpecPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: None, + upgradeconfigs: None, + }, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceStatusPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + status: Option, +} + +impl Default for OSInstanceStatusPatch { + fn default() -> Self { + OSInstanceStatusPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + status: Some(OSInstanceStatus { + sysconfigs: None, + upgradeconfigs: None, + }), + } + } +} + +#[derive(Clone)] +pub struct ControllerClient { + pub client: Client, +} + +impl ControllerClient { + pub fn new(client: Client) -> Self { + ControllerClient { client } + } +} + +#[async_trait] +pub trait ApplyApi: Clone + Sized + Send + Sync { + async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error>; + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error>; + async fn update_osinstance_status( + &self, + node_name: &str, + namespace: &str, + status: &Option, + ) -> Result<(), Error>; +} + +#[async_trait] +impl ApplyApi for ControllerClient { + async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error> { + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + let osinstance = OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: None, + upgradeconfigs: None, + }, + status: None, + }; + let osi_api = Api::namespaced(self.client.clone(), namespace); + osi_api.create(&PostParams::default(), &osinstance).await?; + Ok(()) + } + + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_spec_patch = OSInstanceSpecPatch { + spec: spec.clone(), + ..Default::default() + }; + osi_api + .patch( + node_name, + &PatchParams::default(), + &Patch::Merge(&osi_spec_patch), + ) + .await?; + Ok(()) + } + + async fn update_osinstance_status( + &self, + node_name: &str, + namespace: &str, + status: &Option, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_status_patch = OSInstanceStatusPatch { + status: status.clone(), + ..Default::default() + }; + osi_api + .patch_status( + node_name, + &PatchParams::default(), + &Patch::Merge(&osi_status_patch), + ) + .await?; + Ok(()) + } +} +pub mod apiclient_error { + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs new file mode 100644 index 00000000..3a0f1a7c --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -0,0 +1,502 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use super::crd::{Content, OSInstance, OS}; +use super::drain::drain_os; +use super::utils::{check_version, get_config_version, ConfigOperation, ConfigType}; +use super::values::{ + LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, + OPERATION_TYPE_UPGRADE, REQUEUE_ERROR, REQUEUE_NORMAL, +}; +use super::{ + apiclient::{ApplyApi, ControllerClient}, + crd::Configs, +}; +use anyhow::Result; +use cli::{ + client::Client as AgentClient, + method::{ + callable_method::RpcMethod, cleanup::CleanupMethod, configure::ConfigureMethod, + prepare_upgrade::PrepareUpgradeMethod, rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::{Api, PostParams}, + core::ErrorResponse, + runtime::controller::{Context, ReconcilerAction}, + Client, ResourceExt, +}; +use log::{debug, error, info}; +use manager::api::{ConfigureRequest, KeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest}; +use reconciler_error::Error; +use std::collections::HashMap; +use std::env; + +pub async fn reconcile( + os: OS, + ctx: Context>, +) -> Result { + debug!("start reconcile"); + let proxy_controller = ctx.get_ref(); + let os_cr = &os; + let node_name = env::var("NODE_NAME")?; + let namespace: String = os_cr.namespace().ok_or(Error::MissingObjectKey { + resource: "os".to_string(), + value: "namespace".to_string(), + })?; + proxy_controller + .check_osi_exisit(&namespace, &node_name) + .await?; + let controller_res = proxy_controller + .get_resources(&namespace, &node_name) + .await?; + let node = controller_res.node; + let mut osinstance = controller_res.osinstance; + let node_os_image = &node + .status + .as_ref() + .ok_or(Error::MissingSubResource { + value: String::from("node.status"), + })? + .node_info + .as_ref() + .ok_or(Error::MissingSubResource { + value: String::from("node.status.node_info"), + })? + .os_image; + debug!( + "os expected osversion is {},actual osversion is {}", + os_cr.spec.osversion, node_os_image + ); + if check_version(&os_cr.spec.osversion, &node_os_image) { + match ConfigType::SysConfig.check_config_version(&os, &osinstance) { + ConfigOperation::Reassign => { + debug!("start reassign"); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.sysconfigs.as_ref()), + ConfigType::SysConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + ConfigOperation::UpdateConfig => { + debug!("start update config"); + osinstance.spec.sysconfigs = os_cr.spec.sysconfigs.clone(); + proxy_controller + .controller_client + .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec) + .await?; + return Ok(REQUEUE_ERROR); + } + _ => {} + } + proxy_controller + .set_config(&mut osinstance, ConfigType::SysConfig) + .await?; + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.sysconfigs.as_ref()), + ConfigType::SysConfig, + ) + .await?; + } else { + if os_cr.spec.opstype == NODE_STATUS_CONFIG { + return Err(Error::UpgradeBeforeConfig); + } + match ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) { + ConfigOperation::Reassign => { + debug!("start reassign"); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + _ => {} + } + if node.labels().contains_key(LABEL_UPGRADING) { + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + info!( + "node has upgrade label ,but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin" + ); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + proxy_controller + .set_config(&mut osinstance, ConfigType::UpgradeConfig) + .await?; + proxy_controller.upgrade_node(os_cr, &node).await?; + } + } + Ok(REQUEUE_NORMAL) +} + +pub fn error_policy( + error: &Error, + _ctx: Context>, +) -> ReconcilerAction { + error!("Reconciliation error:{}", error.to_string()); + REQUEUE_ERROR +} + +struct ControllerResources { + osinstance: OSInstance, + node: Node, +} +pub struct ProxyController { + k8s_client: Client, + controller_client: T, + agent_client: AgentClient, +} + +impl ProxyController { + pub fn new(k8s_client: Client, controller_client: T, agent_client: AgentClient) -> Self { + ProxyController { + k8s_client, + controller_client, + agent_client, + } + } +} + +impl ProxyController { + async fn check_osi_exisit(&self, namespace: &str, node_name: &str) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + match osi_api.get(node_name).await { + Ok(osi) => { + debug!("osinstance is exist {:?}", osi.name()); + return Ok(()); + } + Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { + info!("Create OSInstance {}", node_name); + self.controller_client + .create_osinstance(node_name, namespace) + .await?; + Ok(()) + } + Err(err) => Err(Error::KubeError { source: err }), + } + } + + async fn get_resources( + &self, + namespace: &str, + node_name: &str, + ) -> Result { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + let osinstance_cr = osi_api.get(node_name).await?; + let node_api: Api = Api::all(self.k8s_client.clone()); + let node_cr = node_api.get(node_name).await?; + Ok(ControllerResources { + osinstance: osinstance_cr, + node: node_cr, + }) + } + + async fn refresh_node( + &self, + mut node: Node, + osinstance: OSInstance, + os_config_version: &str, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start refresh_node"); + let node_api: Api = Api::all(self.k8s_client.clone()); + let labels = node.labels_mut(); + if labels.contains_key(LABEL_UPGRADING) { + labels.remove(LABEL_UPGRADING); + node = node_api + .replace(&node.name(), &PostParams::default(), &node) + .await?; + } + if let Some(node_spec) = &node.spec { + if let Some(node_unschedulable) = node_spec.unschedulable { + if node_unschedulable { + node_api.uncordon(&node.name()).await?; + info!("Uncordon successfully node{}", node.name()); + } + } + } + self.update_node_status(osinstance, os_config_version, config_type) + .await?; + Ok(()) + } + + async fn update_node_status( + &self, + mut osinstance: OSInstance, + os_config_version: &str, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start update_node_status"); + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + return Ok(()); + } + let upgradeconfig_spec_version = + get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + let sysconfig_spec_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + let sysconfig_status_version: String; + if let Some(osinstance_status) = osinstance.status.as_ref() { + sysconfig_status_version = get_config_version(osinstance_status.sysconfigs.as_ref()); + } else { + sysconfig_status_version = get_config_version(None); + } + if sysconfig_spec_version == sysconfig_status_version + || (config_type == ConfigType::SysConfig && os_config_version != sysconfig_spec_version) + || (config_type == ConfigType::UpgradeConfig + && os_config_version != upgradeconfig_spec_version) + { + let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); + self.controller_client + .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec) + .await?; + } + Ok(()) + } + + async fn update_osi_status( + &self, + osinstance: &mut OSInstance, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start update_osi_status"); + config_type.set_osi_status_config(osinstance); + debug!("osinstance status is update to {:?}", osinstance.status); + let namespace = &osinstance.namespace().ok_or(Error::MissingObjectKey { + resource: "osinstance".to_string(), + value: "namespace".to_string(), + })?; + self.controller_client + .update_osinstance_status(&osinstance.name(), &namespace, &osinstance.status) + .await?; + Ok(()) + } + + async fn set_config( + &self, + osinstance: &mut OSInstance, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start set_config"); + let config_info = config_type.check_config_start(osinstance); + if config_info.need_config { + match config_info.configs.and_then(convert_to_agent_config) { + Some(agent_configs) => { + let config_request = ConfigureRequest { + configs: agent_configs, + }; + match ConfigureMethod::new(config_request).call(&self.agent_client) { + Ok(_resp) => {} + Err(e) => { + return Err(Error::AgentError { source: e }); + } + } + } + None => { + info!("config is none, no need to config"); + } + }; + self.update_osi_status(osinstance, config_type).await?; + } + Ok(()) + } + + async fn upgrade_node(&self, os_cr: &OS, node: &Node) -> Result<(), Error> { + debug!("start upgrade node"); + + match os_cr.spec.opstype.as_str() { + OPERATION_TYPE_UPGRADE => { + let upgrade_request = UpgradeRequest { + version: os_cr.spec.osversion.clone(), + image_type: os_cr.spec.imagetype.clone(), + check_sum: os_cr.spec.checksum.clone(), + container_image: os_cr.spec.containerimage.clone(), + }; + match PrepareUpgradeMethod::new(upgrade_request).call(&self.agent_client) { + Ok(_resp) => {} + Err(e) => { + return Err(Error::AgentError { source: e }); + } + } + match self + .evict_node(&node.name(), os_cr.spec.evictpodforce) + .await + { + Ok(()) => {} + Err(e) => { + match CleanupMethod::new().call(&self.agent_client) { + Ok(_resp) => {} + Err(agent_error) => { + return Err(Error::AgentError { + source: agent_error, + }); + } + } + return Err(e); + } + } + match UpgradeMethod::new().call(&self.agent_client) { + Ok(_resp) => {} + Err(e) => { + return Err(Error::AgentError { source: e }); + } + } + } + OPERATION_TYPE_ROLLBACK => { + self.evict_node(&node.name(), os_cr.spec.evictpodforce) + .await?; + match RollbackMethod::new().call(&self.agent_client) { + Ok(_resp) => {} + Err(e) => { + return Err(Error::AgentError { source: e }); + } + } + } + _ => { + return Err(Error::OperationError { + value: os_cr.spec.opstype.clone(), + }); + } + } + Ok(()) + } + + async fn evict_node(&self, node_name: &str, evict_pod_force: bool) -> Result<(), Error> { + debug!("start evict_node"); + let node_api = Api::all(self.k8s_client.clone()); + node_api.cordon(node_name).await?; + info!("Cordon node Successfully{}, start drain nodes", node_name); + match self.drain_node(node_name, evict_pod_force).await { + Ok(()) => {} + Err(e) => { + node_api.uncordon(node_name).await?; + info!("Drain node {} error, uncordon node successfully", node_name); + return Err(e); + } + } + Ok(()) + } + + async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> { + use crate::controller::drain::error::DrainError::*; + match drain_os(&self.k8s_client.clone(), node_name, force).await { + Err(FindTargetPods { source, .. }) => Err(Error::KubeError { source: source }), + Err(DeletePodsError { errors, .. }) => Err(Error::DrainNodeError { + value: errors.join("; "), + }), + _ => Ok(()), + } + } +} + +fn convert_to_agent_config(configs: Configs) -> Option> { + let mut agent_configs: Vec = Vec::new(); + if let Some(config_list) = configs.configs { + for config in config_list.into_iter() { + match config.contents.and_then(convert_to_config_hashmap) { + Some(contents_tmp) => { + let config_tmp = AgentSysconfig { + model: config.model.unwrap_or_default(), + config_path: config.configpath.unwrap_or_default(), + contents: contents_tmp, + }; + agent_configs.push(config_tmp) + } + None => { + info!("model {} which has configpath {} do not has any contents no need to configure",config.model.unwrap_or_default(),config.configpath.unwrap_or_default()); + continue; + } + }; + } + if agent_configs.len() == 0 { + info!("no contents in all models, no need to configure"); + return None; + } + return Some(agent_configs); + } + return None; +} + +fn convert_to_config_hashmap(contents: Vec) -> Option> { + let mut contents_tmp: HashMap = HashMap::new(); + for content in contents.into_iter() { + let key_info = KeyInfo { + value: content.value.unwrap_or_default(), + operation: content.operation.unwrap_or_default(), + }; + contents_tmp.insert(content.key.unwrap_or_default(), key_info); + } + return Some(contents_tmp); +} + +pub mod reconciler_error { + use crate::controller::apiclient::apiclient_error; + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + + #[error("Create/Patch OSInstance reported error: {source}")] + ApplyApiError { + #[from] + source: apiclient_error::Error, + }, + + #[error("Cannot get environment NODE_NAME, error: {source}")] + EnvError { + #[from] + source: std::env::VarError, + }, + + #[error("{}.metadata.{} is not exist", resource, value)] + MissingObjectKey { resource: String, value: String }, + + #[error("Cannot get {}, {} is None", value, value)] + MissingSubResource { value: String }, + + #[error("operation {} cannot be recognized", value)] + OperationError { value: String }, + + #[error("Expect OS Version is not same with Node OS Version, please upgrade first")] + UpgradeBeforeConfig, + + #[error("os-agent reported error:{source}")] + AgentError { source: anyhow::Error }, + #[error("Error when drain node, error reported: {}", value)] + DrainNodeError { value: String }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/crd.rs b/KubeOS-Rust/proxy/src/controller/crd.rs new file mode 100644 index 00000000..9f01a964 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/crd.rs @@ -0,0 +1,77 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube( + group = "upgrade.openeuler.org", + version = "v1alpha1", + kind = "OS", + plural = "os", + singular = "os", + namespaced +)] +pub struct OSSpec { + pub osversion: String, + pub maxunavailable: i64, + pub checksum: String, + pub imagetype: String, + pub containerimage: String, + pub opstype: String, + pub evictpodforce: bool, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube( + group = "upgrade.openeuler.org", + version = "v1alpha1", + kind = "OSInstance", + plural = "osinstances", + singular = "osinstance", + status = "OSInstanceStatus", + namespaced +)] +pub struct OSInstanceSpec { + pub nodestatus: String, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct OSInstanceStatus { + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Configs { + pub version: Option, + pub configs: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Config { + pub model: Option, + pub configpath: Option, + pub contents: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Content { + pub key: Option, + pub value: Option, + pub operation: Option, +} diff --git a/KubeOS-Rust/proxy/src/controller/drain.rs b/KubeOS-Rust/proxy/src/controller/drain.rs new file mode 100644 index 00000000..a586d949 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/drain.rs @@ -0,0 +1,650 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use futures::{stream, StreamExt}; +use k8s_openapi::api::core::v1::{Pod, PodSpec, PodStatus}; +use kube::{ + api::{EvictParams, ListParams}, + core::ObjectList, + Api, Client, ResourceExt, +}; +use reqwest::StatusCode; +use tokio::time::{sleep, Duration, Instant}; +use tokio_retry::{ + strategy::{jitter, ExponentialBackoff}, + RetryIf, +}; +use tracing::{event, Level}; + +use self::error::DrainError; +use super::values::{ + EVERY_DELETION_CHECK, EVERY_EVICTION_RETRY, MAX_EVICT_POD_NUM, MAX_RETRIES_TIMES, + RETRY_BASE_DELAY, RETRY_MAX_DELAY, TIMEOUT, +}; + +pub(crate) async fn drain_os( + client: &Client, + node_name: &str, + force: bool, +) -> Result<(), error::DrainError> { + let pods_list = get_pods_deleted(client, node_name, force).await?; + + stream::iter(pods_list) + .for_each_concurrent(MAX_EVICT_POD_NUM, move |pod| { + let k8s_client = client.clone(); + async move { + if evict_pod(&k8s_client, &pod, force).await.is_ok() { + wait_for_deletion(&k8s_client, &pod).await.ok(); + } + } + }) + .await; + + Ok(()) +} + +async fn get_pods_deleted( + client: &Client, + node_name: &str, + force: bool, +) -> Result, error::DrainError> { + let lp = ListParams { + field_selector: Some(format!("spec.nodeName={}", node_name)), + ..Default::default() + }; + let pods_api: Api = Api::all(client.clone()); + let pods: ObjectList = match pods_api.list(&lp).await { + Ok(pods @ ObjectList { .. }) => pods, + Err(err) => { + return Err(DrainError::FindTargetPods { + source: err, + node_name: node_name.to_string(), + }); + } + }; + let mut filterd_pods_list: Vec = Vec::new(); + let mut filterd_err: Vec = Vec::new(); + let pod_filter = CombinedFilter::new(force); + for pod in pods.into_iter() { + let filter_result = pod_filter.filter(&pod); + if filter_result.status == PodDeleteStatus::Error { + filterd_err.push(filter_result.desc); + continue; + } + if filter_result.result { + filterd_pods_list.push(pod); + } + } + if filterd_err.len() > 0 { + return Err(error::DrainError::DeletePodsError { + errors: filterd_err, + }); + } + Ok(filterd_pods_list.into_iter()) +} + +async fn evict_pod( + k8s_client: &kube::Client, + pod: &Pod, + force: bool, +) -> Result<(), error::EvictionError> { + let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); + + let error_handling_strategy = if force { + ErrorHandleStrategy::RetryStrategy + } else { + ErrorHandleStrategy::TolerateStrategy + }; + + RetryIf::spawn( + error_handling_strategy.retry_strategy(), + || async { + loop { + let eviction_result = pod_api.evict(&pod.name_any(), &EvictParams::default()).await; + + match eviction_result { + Ok(_) => { + pod.name(); + event!(Level::INFO, "Successfully evicted Pod '{}'", pod.name_any()); + break; + } + Err(kube::Error::Api(e)) => { + let status_code = StatusCode::from_u16(e.code); + match status_code { + Ok(StatusCode::TOO_MANY_REQUESTS) => { + event!( + Level::ERROR, + "Too many requests when creating Eviction for Pod '{}': '{}'. This is likely due to respecting a Pod Disruption Budget. Retrying in {:.2}s.", + pod.name_any(), + e, + EVERY_EVICTION_RETRY.as_secs_f64() + ); + sleep(EVERY_EVICTION_RETRY).await; + continue; + } + Ok(StatusCode::INTERNAL_SERVER_ERROR) => { + event!( + Level::ERROR, + "Error when evicting Pod '{}': '{}'. Check for misconfigured PodDisruptionBudgets. Retrying in {:.2}s.", + pod.name_any(), + e, + EVERY_EVICTION_RETRY.as_secs_f64() + ); + sleep(EVERY_EVICTION_RETRY).await; + continue; + } + Ok(StatusCode::NOT_FOUND) => { + return Err(error::EvictionError::NonRetriableEviction { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(StatusCode::FORBIDDEN) => { + return Err(error::EvictionError::NonRetriableEviction { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(_) => { + event!( + Level::ERROR, + "Error when evicting Pod '{}': '{}'.", + pod.name_any(), + e + ); + return Err(error::EvictionError::RetriableEviction { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Err(_) => { + event!( + Level::ERROR, + "Received invalid response code from Kubernetes API: '{}'", + e + ); + return Err(error::EvictionError::RetriableEviction { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + } + } + Err(e) => { + event!(Level::ERROR, "Eviction failed: '{}'. Retrying...", e); + return Err(error::EvictionError::RetriableEviction { + source: e, + pod_name: pod.name_any(), + }); + } + } + } + Ok(()) + }, + error_handling_strategy + ).await +} + +async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), error::DrainError> { + let start_time = Instant::now(); + + let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); + loop { + match pod_api.get(&pod.name_any()).await { + Err(kube::Error::Api(e)) if e.code == 404 => { + event!(Level::INFO, "Pod {} deleted.", pod.name_any()); + break; + } + + Ok(p) if p.uid() != pod.uid() => { + let name = p + .metadata + .name + .clone() + .or_else(|| p.metadata.generate_name.clone()) + .unwrap_or_default(); + event!(Level::INFO, "Pod {} deleted.", name); + break; + } + + Ok(_) => { + event!( + Level::DEBUG, + "Pod '{}' not yet deleted. Waiting {}s.", + pod.name_any(), + EVERY_DELETION_CHECK.as_secs_f64() + ); + } + + Err(e) => { + event!( + Level::ERROR, + "Could not determine if Pod '{}' has been deleted: '{}'. Waiting {}s.", + pod.name_any(), + e, + EVERY_DELETION_CHECK.as_secs_f64() + ); + } + } + if start_time.elapsed() > TIMEOUT { + return Err(error::DrainError::WaitForDeletion { + pod_name: pod.name_any(), + max_wait: TIMEOUT, + }); + } else { + sleep(EVERY_DELETION_CHECK).await; + } + } + Ok(()) +} +fn get_pod_api_with_namespace(client: &kube::Client, pod: &Pod) -> Api { + match pod.metadata.namespace.as_ref() { + Some(namespace) => Api::namespaced(client.clone(), namespace), + None => Api::default_namespaced(client.clone()), + } +} +trait NameAny { + fn name_any(self: &Self) -> String; +} + +impl NameAny for &Pod { + fn name_any(self: &Self) -> String { + self.metadata + .name + .clone() + .or_else(|| self.metadata.generate_name.clone()) + .unwrap_or_default() + } +} +trait PodFilter { + fn filter(self: &Self, pod: &Pod) -> Box; +} + +struct FinishedOrFailedFilter {} +impl PodFilter for FinishedOrFailedFilter { + fn filter(self: &Self, pod: &Pod) -> Box { + return match pod.status.as_ref() { + Some(PodStatus { + phase: Some(phase), .. + }) if phase == "Failed" || phase == "Succeeded" => { + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + } + _ => FilterResult::create_filter_result(false, "", PodDeleteStatus::Okay), + }; + } +} +struct DaemonFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for DaemonFilter { + fn filter(self: &Self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = + self.finished_or_failed_filter.filter(pod).as_ref() + { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + return match pod.metadata.owner_references.as_ref() { + Some(owner_references) + if owner_references.iter().any(|reference| { + reference.controller.unwrap_or(false) && reference.kind == "DaemonSet" + }) => + { + if self.force { + let description = format!( + "Ignore Pod '{}': Pod is member of a DaemonSet", + pod.name_any() + ); + Box::new(FilterResult { + result: false, + desc: description, + status: PodDeleteStatus::Warning, + }) + } else { + let description = format!( + "Cannot drain Pod '{}': Pod is member of a DaemonSet", + pod.name_any() + ); + Box::new(FilterResult { + result: false, + desc: description, + status: PodDeleteStatus::Error, + }) + } + } + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} +impl DaemonFilter { + fn new(force: bool) -> DaemonFilter { + return DaemonFilter { + finished_or_failed_filter: FinishedOrFailedFilter {}, + force: force, + }; + } +} + +struct MirrorFilter {} +impl PodFilter for MirrorFilter { + fn filter(self: &Self, pod: &Pod) -> Box { + return match pod.metadata.annotations.as_ref() { + Some(annotations) if annotations.contains_key("kubernetes.io/config.mirror") => { + let description = format!( + "Ignore Pod '{}': Pod is a static Mirror Pod", + pod.name_any() + ); + FilterResult::create_filter_result( + false, + &description.to_string(), + PodDeleteStatus::Warning, + ) + } + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} + +struct LocalStorageFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for LocalStorageFilter { + fn filter(self: &Self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = + self.finished_or_failed_filter.filter(pod).as_ref() + { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + return match pod.spec.as_ref() { + Some(PodSpec { + volumes: Some(volumes), + .. + }) if volumes.iter().any(|volume| volume.empty_dir.is_some()) => { + if self.force { + let description = format!( + "Force draining Pod '{}': Pod has local storage", + pod.name_any() + ); + Box::new(FilterResult { + result: true, + desc: description, + status: PodDeleteStatus::Warning, + }) + } else { + let description = format!( + "Cannot drain Pod '{}': Pod has local Storage", + pod.name_any() + ); + Box::new(FilterResult { + result: false, + desc: description, + status: PodDeleteStatus::Error, + }) + } + } + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} +impl LocalStorageFilter { + fn new(force: bool) -> LocalStorageFilter { + return LocalStorageFilter { + finished_or_failed_filter: FinishedOrFailedFilter {}, + force: force, + }; + } +} +struct UnreplicatedFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for UnreplicatedFilter { + fn filter(self: &Self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = + self.finished_or_failed_filter.filter(pod).as_ref() + { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + let is_replicated = pod.metadata.owner_references.is_some(); + + if is_replicated { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + return if !is_replicated && self.force { + let description = format!("Force drain Pod '{}': Pod is unreplicated", pod.name_any()); + Box::new(FilterResult { + result: true, + desc: description, + status: PodDeleteStatus::Warning, + }) + } else { + let description = format!("Cannot drain Pod '{}': Pod is unreplicated", pod.name_any()); + Box::new(FilterResult { + result: false, + desc: description, + status: PodDeleteStatus::Error, + }) + }; + } +} +impl UnreplicatedFilter { + fn new(force: bool) -> UnreplicatedFilter { + return UnreplicatedFilter { + finished_or_failed_filter: FinishedOrFailedFilter {}, + force: force, + }; + } +} + +struct DeletedFilter { + delete_wait_timeout: Duration, +} +impl PodFilter for DeletedFilter { + fn filter(self: &Self, pod: &Pod) -> Box { + let now = Instant::now().elapsed(); + return match pod.metadata.deletion_timestamp.as_ref() { + Some(time) + if time.0.timestamp() != 0 + && now - Duration::from_secs(time.0.timestamp() as u64) + >= self.delete_wait_timeout => + { + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + } + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} + +struct CombinedFilter { + deleted_filter: DeletedFilter, + daemon_filter: DaemonFilter, + mirror_filter: MirrorFilter, + local_storage_filter: LocalStorageFilter, + unreplicated_filter: UnreplicatedFilter, +} +impl PodFilter for CombinedFilter { + fn filter(self: &Self, pod: &Pod) -> Box { + let mut filter_res = self.deleted_filter.filter(pod); + if !filter_res.result { + event!(Level::INFO, filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status.clone(), + }); + } + filter_res = self.daemon_filter.filter(pod); + if !filter_res.result { + event!(Level::INFO, filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status.clone(), + }); + } + filter_res = self.mirror_filter.filter(pod); + if !filter_res.result { + event!(Level::INFO, filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status.clone(), + }); + } + filter_res = self.local_storage_filter.filter(pod); + if !filter_res.result { + event!(Level::INFO, filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status.clone(), + }); + } + filter_res = self.unreplicated_filter.filter(pod); + if !filter_res.result { + event!(Level::INFO, filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status.clone(), + }); + } + + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } +} +impl CombinedFilter { + fn new(force: bool) -> CombinedFilter { + return CombinedFilter { + deleted_filter: DeletedFilter { + delete_wait_timeout: TIMEOUT, + }, + daemon_filter: DaemonFilter::new(force), + mirror_filter: MirrorFilter {}, + local_storage_filter: LocalStorageFilter::new(force), + unreplicated_filter: UnreplicatedFilter::new(force), + }; + } +} + +#[derive(PartialEq, Clone, Copy)] +enum PodDeleteStatus { + Okay, + Warning, + Error, +} +struct FilterResult { + result: bool, + desc: String, + status: PodDeleteStatus, +} +impl FilterResult { + fn create_filter_result( + result: bool, + desc: &str, + status: PodDeleteStatus, + ) -> Box { + Box::new(FilterResult { + result: result, + desc: desc.to_string(), + status: status, + }) + } +} + +enum ErrorHandleStrategy { + RetryStrategy, + TolerateStrategy, +} + +impl ErrorHandleStrategy { + fn retry_strategy(&self) -> impl Iterator { + let backoff = ExponentialBackoff::from_millis(RETRY_BASE_DELAY.as_millis() as u64) + .max_delay(RETRY_MAX_DELAY) + .map(jitter); + + return match self { + Self::TolerateStrategy => { + return backoff.take(0); + } + + Self::RetryStrategy => backoff.take(MAX_RETRIES_TIMES), + }; + } +} + +impl tokio_retry::Condition for ErrorHandleStrategy { + fn should_retry(&mut self, error: &error::EvictionError) -> bool { + match self { + Self::TolerateStrategy => false, + Self::RetryStrategy => { + if let error::EvictionError::RetriableEviction { .. } = error { + true + } else { + false + } + } + } + } +} + +pub mod error { + use snafu::Snafu; + use tokio::time::Duration; + + #[derive(Debug, Snafu)] + #[snafu(visibility(pub))] + pub enum DrainError { + #[snafu(display("Unable to find drainable Pods for Node '{}': '{}'", node_name, source))] + FindTargetPods { + source: kube::Error, + node_name: String, + }, + + #[snafu( + display( + "Pod '{}' was not deleted in the time allocated ({:.2}s).", + pod_name, + max_wait.as_secs_f64() + ) + )] + WaitForDeletion { + pod_name: String, + max_wait: Duration, + }, + DeletePodsError { + errors: Vec, + }, + } + + #[derive(Debug, Snafu)] + #[snafu(visibility(pub))] + pub enum EvictionError { + #[snafu(display("Unable to evict Pod '{}': '{}'", pod_name, source))] + RetriableEviction { + source: kube::Error, + pod_name: String, + }, + + #[snafu(display("Unable to evict Pod '{}': '{}'", pod_name, source))] + /// A fatal error occurred while attempting to evict a Pod. This will not be retried. + NonRetriableEviction { + source: kube::Error, + pod_name: String, + }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs new file mode 100644 index 00000000..e2e06493 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -0,0 +1,23 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod apiclient; +mod controller; +mod crd; +mod drain; +mod utils; +mod values; + +pub use apiclient::ControllerClient; +pub use controller::{error_policy, reconcile, reconciler_error::Error, ProxyController}; +pub use crd::OS; +pub use values::SOCK_PATH; diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs new file mode 100644 index 00000000..26a0d18a --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/utils.rs @@ -0,0 +1,155 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use super::crd::{Configs, OSInstance, OSInstanceStatus, OS}; +use super::values::{NODE_STATUS_CONFIG, NODE_STATUS_IDLE, NODE_STATUS_UPGRADE}; +use log::{debug, info}; + +#[derive(PartialEq, Clone, Copy)] +pub enum ConfigType { + UpgradeConfig, + SysConfig, +} + +pub enum ConfigOperation { + DoNothing, + Reassign, + UpdateConfig, +} + +pub struct ConfigInfo { + pub need_config: bool, + pub configs: Option, +} + +impl ConfigType { + pub fn check_config_version(&self, os: &OS, osinstance: &OSInstance) -> ConfigOperation { + debug!("start check_config_version"); + let node_status = &osinstance.spec.nodestatus; + if node_status == NODE_STATUS_IDLE { + debug!("======node status is idle======"); + return ConfigOperation::DoNothing; + }; + match self { + ConfigType::UpgradeConfig => { + let os_config_version = get_config_version(os.spec.upgradeconfigs.as_ref()); + let osi_config_version = + get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + debug!("=======os upgradeconfig version is{},osinstance spec upragdeconfig version is{}",os_config_version,osi_config_version); + if !check_version(&os_config_version, &osi_config_version) { + info!("os.spec.upgradeconfig.version is not equal to oninstance.spec.upragdeconfig.version, operation: reassgin upgrade to get newest upgradeconfigs"); + return ConfigOperation::Reassign; + } + } + ConfigType::SysConfig => { + let os_config_version = get_config_version(os.spec.sysconfigs.as_ref()); + let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + debug!( + "=======os sysconfig version is{},osinstance spec sysconfig version is{}", + os_config_version, osi_config_version + ); + if !check_version(&os_config_version, &osi_config_version) { + if node_status == NODE_STATUS_CONFIG { + info!("os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: reassgin config to get newest sysconfigs"); + return ConfigOperation::Reassign; + } + if node_status == NODE_STATUS_UPGRADE { + info!("os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: update osinstance.spec.sysconfig and reconcile"); + return ConfigOperation::UpdateConfig; + } + } + } + }; + ConfigOperation::DoNothing + } + pub fn check_config_start(&self, osinstance: &OSInstance) -> ConfigInfo { + debug!("start check_config_start"); + let spec_config_version: String; + let status_config_version: String; + let configs: Option; + match self { + ConfigType::UpgradeConfig => { + spec_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + if let Some(osinstance_status) = osinstance.status.as_ref() { + status_config_version = + get_config_version(osinstance_status.upgradeconfigs.as_ref()); + } else { + status_config_version = get_config_version(None); + } + configs = osinstance.spec.upgradeconfigs.clone(); + } + ConfigType::SysConfig => { + spec_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + if let Some(osinstance_status) = osinstance.status.as_ref() { + status_config_version = + get_config_version(osinstance_status.sysconfigs.as_ref()); + } else { + status_config_version = get_config_version(None); + } + configs = osinstance.spec.sysconfigs.clone(); + } + } + debug!( + "=======osinstance soec config version is {},status config version is {}", + spec_config_version, status_config_version + ); + if spec_config_version != status_config_version + && osinstance.spec.nodestatus != NODE_STATUS_IDLE + { + return ConfigInfo { + need_config: true, + configs: configs, + }; + } + return ConfigInfo { + need_config: false, + configs: None, + }; + } + pub fn set_osi_status_config(&self, osinstance: &mut OSInstance) { + match self { + ConfigType::UpgradeConfig => { + if let Some(osi_status) = &mut osinstance.status { + osi_status.upgradeconfigs = osinstance.spec.upgradeconfigs.clone(); + } else { + osinstance.status = Some(OSInstanceStatus { + upgradeconfigs: osinstance.spec.upgradeconfigs.clone(), + sysconfigs: None, + }) + } + } + ConfigType::SysConfig => { + if let Some(osi_status) = &mut osinstance.status { + osi_status.sysconfigs = osinstance.spec.sysconfigs.clone(); + } else { + osinstance.status = Some(OSInstanceStatus { + upgradeconfigs: None, + sysconfigs: osinstance.spec.sysconfigs.clone(), + }) + } + } + } + } +} + +pub fn check_version(version_a: &str, version_b: &str) -> bool { + version_a.eq(version_b) +} + +pub fn get_config_version(configs: Option<&Configs>) -> String { + if let Some(configs) = configs { + if let Some(version) = configs.version.as_ref() { + return version.to_string(); + } + }; + String::from("") +} diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs new file mode 100644 index 00000000..f5d41965 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/values.rs @@ -0,0 +1,49 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::runtime::controller::ReconcilerAction; +use tokio::time::Duration; + +pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; +pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; + +pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; +pub const OSINSTANCE_KIND: &str = "OSInstance"; + +pub const NODE_STATUS_IDLE: &str = "idle"; +pub const NODE_STATUS_UPGRADE: &str = "upgrade"; +pub const NODE_STATUS_CONFIG: &str = "config"; + +pub const OPERATION_TYPE_UPGRADE: &str = "upgrade"; +pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; + +pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; + +pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { + requeue_after: Some(Duration::from_secs(15)), +}; + +pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { + requeue_after: Some(Duration::from_secs(1)), +}; + +pub const MAX_EVICT_POD_NUM: usize = 5; + +pub const EVERY_EVICTION_RETRY: Duration = Duration::from_secs(5); + +pub const EVERY_DELETION_CHECK: Duration = Duration::from_secs(5); + +pub const TIMEOUT: Duration = Duration::from_secs(u64::MAX); + +pub const RETRY_BASE_DELAY: Duration = Duration::from_millis(100); +pub const RETRY_MAX_DELAY: Duration = Duration::from_secs(20); +pub const MAX_RETRIES_TIMES: usize = 10; diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs new file mode 100644 index 00000000..43610a64 --- /dev/null +++ b/KubeOS-Rust/proxy/src/main.rs @@ -0,0 +1,52 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::Result; +use env_logger::{Builder, Env, Target}; +use futures::StreamExt; +use kube::{ + api::{Api, ListParams}, + client::Client, + runtime::controller::{Context, Controller}, +}; +use log::{error, info}; +mod controller; +use cli::client::Client as AgentClient; +use controller::{error_policy, reconcile, ControllerClient, ProxyController, OS, SOCK_PATH}; + +const PROXY_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); +#[tokio::main] +async fn main() -> Result<()> { + Builder::from_env(Env::default().default_filter_or("info")) + .target(Target::Stdout) + .init(); + let client = Client::try_default().await?; + let os: Api = Api::all(client.clone()); + let controller_client = ControllerClient::new(client.clone()); + let agent_client = AgentClient::new(SOCK_PATH); + let proxy_controller = ProxyController::new(client, controller_client, agent_client); + info!( + "os-proxy version is {}, start renconcile", + PROXY_VERSION.unwrap_or("Not Found") + ); + Controller::new(os, ListParams::default()) + .run(reconcile, error_policy, Context::new(proxy_controller)) + .for_each(|res| async move { + match res { + Ok(_o) => {} + Err(e) => error!("reconcile failed: {}", e.to_string()), + } + }) + .await; + info!("os-proxy terminated"); + Ok(()) +} -- Gitee From fd8a3362ca3d6ecbdb2869bd61a87de84eced2fe Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Wed, 29 Nov 2023 10:28:21 +0800 Subject: [PATCH 02/46] fix(rust os-agent, proxy): move dd into prepare_upgrade dd update.img to device may takes a while in huawei cloud environment. Considering move dd process into prepare_upgrade stage. The risk is that it damages another partition resulting in unable to rollback to previous version. Signed-off-by: Yuhang Wei --- KubeOS-Rust/agent/src/rpc/agent_impl.rs | 21 +++++++++++-------- KubeOS-Rust/manager/src/api/agent_status.rs | 3 +++ .../manager/src/sys_mgmt/containerd_image.rs | 4 ++-- KubeOS-Rust/manager/src/utils/common.rs | 7 ++++--- .../manager/src/utils/image_manager.rs | 20 ++---------------- .../proxy/src/controller/controller.rs | 21 +++---------------- 6 files changed, 26 insertions(+), 50 deletions(-) diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs index c752ffac..95a28348 100644 --- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -22,10 +22,9 @@ use super::{ }; use manager::{ api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, - sys_mgmt::{CtrImageHandler, CONFIG_TEMPLATE}, + sys_mgmt::{CtrImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH}, utils::{ clean_env, get_partition_info, switch_boot_menuentry, PreparePath, RealCommandExecutor, - UpgradeImageManager, }, }; @@ -69,7 +68,7 @@ impl AgentImpl { pub fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result { let _lock = self.mutex.lock().unwrap(); debug!("Received an 'prepare upgrade' request: {:?}", req); - info!("Start to upgrade to version: {}", req.version); + info!("Start preparing for upgrading to version: {}", req.version); let handler: Box> = match req.image_type.as_str() { "containerd" => Box::new(ImageType::Containerd(CtrImageHandler::default())), @@ -81,6 +80,7 @@ impl AgentImpl { "Ready to install image: {:?}", image_manager.paths.image_path.display() ); + image_manager.install()?; Ok(Response { status: AgentStatus::UpgradeReady, @@ -92,12 +92,15 @@ impl AgentImpl { info!("Start to upgrade"); let command_executor = RealCommandExecutor {}; let (_, next_partition_info) = get_partition_info(&command_executor)?; - let image_manager = UpgradeImageManager::new( - PreparePath::default(), - next_partition_info, - command_executor, + + // based on boot mode use different command to switch boot partition + let device = next_partition_info.device.as_str(); + let menuentry = next_partition_info.menuentry.as_str(); + switch_boot_menuentry(&command_executor, DEFAULT_GRUBENV_PATH, menuentry)?; + info!( + "Switch to boot partition: {}, device: {}", + menuentry, device ); - image_manager.install()?; self.reboot()?; Ok(Response { status: AgentStatus::Upgraded, @@ -110,7 +113,7 @@ impl AgentImpl { let paths = PreparePath::default(); clean_env(paths.update_path, paths.mount_path, paths.image_path)?; Ok(Response { - status: AgentStatus::NotApplied, + status: AgentStatus::CleanedUp, }) } diff --git a/KubeOS-Rust/manager/src/api/agent_status.rs b/KubeOS-Rust/manager/src/api/agent_status.rs index 7a2edf7d..c3152257 100644 --- a/KubeOS-Rust/manager/src/api/agent_status.rs +++ b/KubeOS-Rust/manager/src/api/agent_status.rs @@ -18,6 +18,7 @@ const AGENT_STATUS_UPGRADEREADY: &str = "UPGRADE-READY"; const AGENT_STATUS_UPGRADED: &str = "UPGRADED"; const AGENT_STATUS_ROLLBACKED: &str = "ROLLBACKED"; const AGENT_STATUS_CONFIGURED: &str = "CONFIGURED"; +const AGENT_STATUS_CLEANEDUP: &str = "CLEANEDUP"; #[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] pub enum AgentStatus { @@ -27,6 +28,7 @@ pub enum AgentStatus { Upgraded, Rollbacked, Configured, + CleanedUp, } impl Default for AgentStatus { @@ -44,6 +46,7 @@ impl std::fmt::Display for AgentStatus { AgentStatus::Upgraded => AGENT_STATUS_UPGRADED, AgentStatus::Rollbacked => AGENT_STATUS_ROLLBACKED, AgentStatus::Configured => AGENT_STATUS_CONFIGURED, + AgentStatus::CleanedUp => AGENT_STATUS_CLEANEDUP, }) } } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs index b4bdd2c8..479740e4 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -60,7 +60,7 @@ impl CtrImageHandler { fn get_image(&self, req: &UpgradeRequest) -> Result<()> { let image_name = &req.container_image; is_valid_image_name(image_name)?; - info!("Start pull image {}", image_name); + info!("Start pulling image {}", image_name); let containerd_command: String; if is_command_available("crictl", &self.executor) { containerd_command = "crictl".to_string(); @@ -68,7 +68,7 @@ impl CtrImageHandler { containerd_command = "ctr".to_string(); } pull_image(&containerd_command, image_name, &self.executor)?; - info!("Start check image digest"); + info!("Start checking image digest"); check_oci_image_digest_match( &containerd_command, image_name, diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs index cc826dd4..7bb75a84 100644 --- a/KubeOS-Rust/manager/src/utils/common.rs +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -18,7 +18,7 @@ use std::{ }; use anyhow::{anyhow, Result}; -use log::{debug, info}; +use log::{debug, info, trace}; use nix::{mount, mount::MntFlags}; use super::executor::CommandExecutor; @@ -72,7 +72,7 @@ pub fn perpare_env( } pub fn check_disk_size(need_gb: i64, path: &str) -> Result<()> { - info!("Check if there is enough disk space to upgrade"); + trace!("Check if there is enough disk space to upgrade"); let kb = 1024; let fs_stat = nix::sys::statfs::statfs(path)?; let need_disk_size = need_gb * kb * kb * kb; @@ -81,6 +81,7 @@ pub fn check_disk_size(need_gb: i64, path: &str) -> Result<()> { if available_space < need_disk_size { return Err(anyhow!("Space is not enough for downloading")); } + debug!("There is enough disk space to upgrade"); Ok(()) } @@ -89,7 +90,7 @@ pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> where P: AsRef, { - info!("Clean upgrade environment"); + info!("Clean up the residual upgrade environment"); if is_mounted(&mount_path)? { debug!("Umount {}", mount_path.as_ref().display()); if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) { diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs index 42b5a745..ca48712f 100644 --- a/KubeOS-Rust/manager/src/utils/image_manager.rs +++ b/KubeOS-Rust/manager/src/utils/image_manager.rs @@ -25,7 +25,6 @@ use super::{ executor::CommandExecutor, partition::PartitionInfo, }; -use crate::{sys_mgmt::DEFAULT_GRUBENV_PATH, utils::switch_boot_menuentry}; pub struct UpgradeImageManager { pub paths: PreparePath, @@ -129,7 +128,6 @@ impl UpgradeImageManager { pub fn install(&self) -> Result<()> { let image_str = self.image_path_str()?; let device = self.next_partition.device.as_str(); - let menuentry = self.next_partition.menuentry.as_str(); self.executor.run_command( "dd", &[ @@ -139,14 +137,11 @@ impl UpgradeImageManager { ], )?; debug!("Install image {} to {} done", image_str, device); - // based on boot mode use different command to switch boot partition - switch_boot_menuentry(&self.executor, DEFAULT_GRUBENV_PATH, menuentry)?; info!( - "Switch to boot partition: {}, device: {}", - menuentry, device + "Device {} is overwritten and unable to rollback to the previous version anymore if the eviction of node fails", + device ); delete_file_or_dir(image_str)?; - debug!("Remove image {}", image_str); Ok(()) } } @@ -224,17 +219,6 @@ mod tests { .times(1) // Expect it to be called once .returning(|_, _| Ok(())); - //mock install->grub2-set-default - mock.expect_run_command() - .withf(|name, args| { - name == "grub2-editenv" - && args[0] == "/boot/efi/EFI/openEuler/grubenv" - && args[1] == "set" - && args[2] == "saved_entry=B" - }) - .times(1) // Expect it to be called once - .returning(|_, _| Ok(())); - let img_manager = UpgradeImageManager::new( PreparePath { update_path: tmp_dir.into(), diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index 3a0f1a7c..6410cce1 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -25,7 +25,7 @@ use anyhow::Result; use cli::{ client::Client as AgentClient, method::{ - callable_method::RpcMethod, cleanup::CleanupMethod, configure::ConfigureMethod, + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, rollback::RollbackMethod, upgrade::UpgradeMethod, }, }; @@ -348,23 +348,8 @@ impl ProxyController { return Err(Error::AgentError { source: e }); } } - match self - .evict_node(&node.name(), os_cr.spec.evictpodforce) - .await - { - Ok(()) => {} - Err(e) => { - match CleanupMethod::new().call(&self.agent_client) { - Ok(_resp) => {} - Err(agent_error) => { - return Err(Error::AgentError { - source: agent_error, - }); - } - } - return Err(e); - } - } + self.evict_node(&node.name(), os_cr.spec.evictpodforce) + .await?; match UpgradeMethod::new().call(&self.agent_client) { Ok(_resp) => {} Err(e) => { -- Gitee From b0e86e17400436be59caf6bdf3a6485c8800d64e Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 11 Dec 2023 20:23:31 +0800 Subject: [PATCH 03/46] build(manager): modify dependencies for adapting rust 1.57 Signed-off-by: Yuhang Wei --- KubeOS-Rust/Cargo.lock | 46 +++++++++++++++++++++++----------- KubeOS-Rust/Cargo.toml | 1 + KubeOS-Rust/manager/Cargo.toml | 2 +- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index f6906039..8069b70c 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -399,9 +399,12 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] [[package]] name = "float-cmp" @@ -793,6 +796,17 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.3", + "libc", + "windows-sys", +] + [[package]] name = "iovec" version = "0.1.4" @@ -1081,9 +1095,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" @@ -1555,15 +1569,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.6" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" +checksum = "da1c2388b1513e1b605fcec39a95e0a9e8ef088f71443ef37099fa9ae6673fcb" [[package]] name = "predicates-tree" -version = "1.0.9" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +checksum = "4d86de6de25020a36c6d3643a86d9a6a9f552107c0559c60ea03551b5e16c032" dependencies = [ "predicates-core", "termtree", @@ -1792,12 +1806,13 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.9" +version = "0.37.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" dependencies = [ - "bitflags 2.4.0", + "bitflags 1.3.2", "errno", + "io-lifetimes", "libc", "linux-raw-sys", "windows-sys", @@ -2068,10 +2083,11 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ + "autocfg", "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", @@ -2090,9 +2106,9 @@ dependencies = [ [[package]] name = "termtree" -version = "0.4.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" [[package]] name = "thiserror" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml index 64ad4be8..c1299f2d 100644 --- a/KubeOS-Rust/Cargo.toml +++ b/KubeOS-Rust/Cargo.toml @@ -5,6 +5,7 @@ members = [ "cli", "proxy", ] +resolver = "2" [profile.release] opt-level = 's' diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml index 39f49efd..0b82b17d 100644 --- a/KubeOS-Rust/manager/Cargo.toml +++ b/KubeOS-Rust/manager/Cargo.toml @@ -7,7 +7,7 @@ license = "MulanPSL-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dev-dependencies] -tempfile = "3.2" +tempfile = "3.6.0" mockall = { version = "=0.11.3" } predicates = "=2.0.1" -- Gitee From 7526a9d6338d2e6218e890f3263d46377243658f Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 11 Dec 2023 20:24:10 +0800 Subject: [PATCH 04/46] fix(manager): remove useless code Signed-off-by: Yuhang Wei --- KubeOS-Rust/manager/src/sys_mgmt/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs index 01a09d70..deec26ea 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -279,7 +279,6 @@ impl Configuration for GrubCmdline { if !is_file_exist(&self.grub_path) { return Err(anyhow!("Failed to find grub.cfg file")); } - if cfg!(test) {} let config_partition = if cfg!(test) { self.is_cur_partition } else { -- Gitee From a97c76a1302188138a3b80ef6265cbbfef35e182 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Tue, 26 Dec 2023 18:31:50 +0800 Subject: [PATCH 05/46] proxy:add agentclient to proxy and modify drain Add agentclient to proxy, decouple proxy and agent, modify drain, unify the use of log library, log printing, and variable names.Delete snafu and tracing from Cargo.toml for deleting no use packages. Signed-off-by: liyuanr --- KubeOS-Rust/Cargo.lock | 36 ---- KubeOS-Rust/proxy/Cargo.toml | 4 +- .../proxy/src/controller/agentclient.rs | 173 ++++++++++++++++++ .../proxy/src/controller/controller.rs | 79 ++++---- KubeOS-Rust/proxy/src/controller/drain.rs | 163 ++++++++--------- KubeOS-Rust/proxy/src/controller/mod.rs | 2 + KubeOS-Rust/proxy/src/main.rs | 5 +- 7 files changed, 293 insertions(+), 169 deletions(-) create mode 100644 KubeOS-Rust/proxy/src/controller/agentclient.rs diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 8069b70c..08ad969e 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -330,12 +330,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "downcast" version = "0.11.0" @@ -622,12 +616,6 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "hermit-abi" version = "0.1.19" @@ -1621,13 +1609,11 @@ dependencies = [ "schemars", "serde", "serde_json", - "snafu", "socket2", "thiserror", "thread_local", "tokio 1.14.0", "tokio-retry", - "tracing", ] [[package]] @@ -2021,28 +2007,6 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" -[[package]] -name = "snafu" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" -dependencies = [ - "doc-comment", - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "socket2" version = "0.4.9" diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index 58cb1f51..a0c48358 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -2,6 +2,8 @@ name = "proxy" version = "0.1.0" edition = "2021" +description = "KubeOS os-proxy" +license = "MulanPSL-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -15,7 +17,6 @@ serde = { version = "1.0.130", features = ["derive"] } serde_json = "1.0.68" thiserror = "1.0.29" env_logger = "0.9.0" -tracing = "0.1.29" schemars = "=0.8.10" socket2 = "=0.4.9" log = "=0.4.15" @@ -23,7 +24,6 @@ thread_local = "=1.1.4" async-trait = "0.1" regex = "=1.7.3" chrono = { version = "0.4", default-features = false, features = ["std"] } -snafu = "0.7" h2 = "=0.3.16" tokio-retry = "0.3" reqwest = { version = "=0.11.10", default-features = false, features = [ "json" ] } diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs new file mode 100644 index 00000000..4d954ee8 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs @@ -0,0 +1,173 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, + prepare_upgrade::PrepareUpgradeMethod, rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; + +use agent_call::AgentCallClient; +use agent_error::Error; +use manager::api::{ + ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest, +}; +use std::collections::HashMap; +use std::path::Path; + +pub struct UpgradeInfo { + pub version: String, + pub image_type: String, + pub check_sum: String, + pub container_image: String, +} + +pub struct ConfigInfo { + pub configs: Vec, +} + +pub struct Sysconfig { + pub model: String, + pub config_path: String, + pub contents: HashMap, +} + +pub struct KeyInfo { + pub value: String, + pub operation: String, +} + +pub trait AgentMethod { + fn prepare_upgrade_method( + &self, + upgrade_info: UpgradeInfo, + agent_call: AgentCallClient, + ) -> Result<(), Error>; + fn upgrade_method(&self, agent_call: AgentCallClient) -> Result<(), Error>; + fn rollback_method(&self, agent_call: AgentCallClient) -> Result<(), Error>; + fn configure_method( + &self, + config_info: ConfigInfo, + agent_call: AgentCallClient, + ) -> Result<(), Error>; +} + +pub mod agent_call { + use super::{Client, Error, RpcMethod}; + #[derive(Default)] + pub struct AgentCallClient {} + + impl AgentCallClient { + pub fn call_agent( + &self, + client: &Client, + method: T, + ) -> Result<(), Error> { + match method.call(client) { + Ok(_resp) => Ok(()), + Err(e) => Err(Error::AgentError { source: e }), + } + } + } +} + +pub struct AgentClient { + pub agent_client: Client, +} + +impl AgentClient { + pub fn new>(socket_path: P) -> Self { + AgentClient { + agent_client: Client::new(socket_path), + } + } +} + +impl AgentMethod for AgentClient { + fn prepare_upgrade_method( + &self, + upgrade_info: UpgradeInfo, + agent_call: AgentCallClient, + ) -> Result<(), Error> { + let upgrade_request = UpgradeRequest { + version: upgrade_info.version, + image_type: upgrade_info.image_type, + check_sum: upgrade_info.check_sum, + container_image: upgrade_info.container_image, + }; + match agent_call.call_agent( + &self.agent_client, + PrepareUpgradeMethod::new(upgrade_request), + ) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn upgrade_method(&self, agent_call: AgentCallClient) -> Result<(), Error> { + match agent_call.call_agent(&self.agent_client, UpgradeMethod::new()) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn rollback_method(&self, agent_call: AgentCallClient) -> Result<(), Error> { + match agent_call.call_agent(&self.agent_client, RollbackMethod::new()) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn configure_method( + &self, + config_info: ConfigInfo, + agent_call: AgentCallClient, + ) -> Result<(), Error> { + let mut agent_configs: Vec = Vec::new(); + for config in config_info.configs { + let mut contents_tmp: HashMap = HashMap::new(); + for (key, key_info) in config.contents.iter() { + contents_tmp.insert( + key.to_string(), + AgentKeyInfo { + value: key_info.value.clone(), + operation: key_info.operation.clone(), + }, + ); + } + agent_configs.push(AgentSysconfig { + model: config.model, + config_path: config.config_path, + contents: contents_tmp, + }) + } + let config_request = ConfigureRequest { + configs: agent_configs, + }; + match agent_call.call_agent(&self.agent_client, ConfigureMethod::new(config_request)) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } +} + +pub mod agent_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("{source}")] + AgentError { source: anyhow::Error }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index 6410cce1..da2e031e 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -10,25 +10,20 @@ * See the Mulan PSL v2 for more details. */ -use super::crd::{Content, OSInstance, OS}; -use super::drain::drain_os; -use super::utils::{check_version, get_config_version, ConfigOperation, ConfigType}; -use super::values::{ - LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, - OPERATION_TYPE_UPGRADE, REQUEUE_ERROR, REQUEUE_NORMAL, -}; use super::{ - apiclient::{ApplyApi, ControllerClient}, - crd::Configs, -}; -use anyhow::Result; -use cli::{ - client::Client as AgentClient, - method::{ - callable_method::RpcMethod, configure::ConfigureMethod, - prepare_upgrade::PrepareUpgradeMethod, rollback::RollbackMethod, upgrade::UpgradeMethod, + agentclient::{ + agent_call::AgentCallClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo, + }, + apiclient::ApplyApi, + crd::{Configs, Content, OSInstance, OS}, + drain::drain_os, + utils::{check_version, get_config_version, ConfigOperation, ConfigType}, + values::{ + LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, + OPERATION_TYPE_UPGRADE, REQUEUE_ERROR, REQUEUE_NORMAL, }, }; +use anyhow::Result; use k8s_openapi::api::core::v1::Node; use kube::{ api::{Api, PostParams}, @@ -37,14 +32,13 @@ use kube::{ Client, ResourceExt, }; use log::{debug, error, info}; -use manager::api::{ConfigureRequest, KeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest}; use reconciler_error::Error; use std::collections::HashMap; use std::env; pub async fn reconcile( os: OS, - ctx: Context>, + ctx: Context>, ) -> Result { debug!("start reconcile"); let proxy_controller = ctx.get_ref(); @@ -159,7 +153,7 @@ pub async fn reconcile( pub fn error_policy( error: &Error, - _ctx: Context>, + _ctx: Context>, ) -> ReconcilerAction { error!("Reconciliation error:{}", error.to_string()); REQUEUE_ERROR @@ -169,14 +163,14 @@ struct ControllerResources { osinstance: OSInstance, node: Node, } -pub struct ProxyController { +pub struct ProxyController { k8s_client: Client, controller_client: T, - agent_client: AgentClient, + agent_client: U, } -impl ProxyController { - pub fn new(k8s_client: Client, controller_client: T, agent_client: AgentClient) -> Self { +impl ProxyController { + pub fn new(k8s_client: Client, controller_client: T, agent_client: U) -> Self { ProxyController { k8s_client, controller_client, @@ -185,7 +179,7 @@ impl ProxyController { } } -impl ProxyController { +impl ProxyController { async fn check_osi_exisit(&self, namespace: &str, node_name: &str) -> Result<(), Error> { let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); match osi_api.get(node_name).await { @@ -312,10 +306,13 @@ impl ProxyController { if config_info.need_config { match config_info.configs.and_then(convert_to_agent_config) { Some(agent_configs) => { - let config_request = ConfigureRequest { - configs: agent_configs, - }; - match ConfigureMethod::new(config_request).call(&self.agent_client) { + let agent_call_client = AgentCallClient::default(); + match self.agent_client.configure_method( + ConfigInfo { + configs: agent_configs, + }, + agent_call_client, + ) { Ok(_resp) => {} Err(e) => { return Err(Error::AgentError { source: e }); @@ -333,16 +330,19 @@ impl ProxyController { async fn upgrade_node(&self, os_cr: &OS, node: &Node) -> Result<(), Error> { debug!("start upgrade node"); - match os_cr.spec.opstype.as_str() { OPERATION_TYPE_UPGRADE => { - let upgrade_request = UpgradeRequest { + let upgrade_info = UpgradeInfo { version: os_cr.spec.osversion.clone(), image_type: os_cr.spec.imagetype.clone(), check_sum: os_cr.spec.checksum.clone(), container_image: os_cr.spec.containerimage.clone(), }; - match PrepareUpgradeMethod::new(upgrade_request).call(&self.agent_client) { + let agent_call_client = AgentCallClient::default(); + match self + .agent_client + .prepare_upgrade_method(upgrade_info, agent_call_client) + { Ok(_resp) => {} Err(e) => { return Err(Error::AgentError { source: e }); @@ -350,7 +350,8 @@ impl ProxyController { } self.evict_node(&node.name(), os_cr.spec.evictpodforce) .await?; - match UpgradeMethod::new().call(&self.agent_client) { + let agent_call_client = AgentCallClient::default(); + match self.agent_client.upgrade_method(agent_call_client) { Ok(_resp) => {} Err(e) => { return Err(Error::AgentError { source: e }); @@ -360,7 +361,8 @@ impl ProxyController { OPERATION_TYPE_ROLLBACK => { self.evict_node(&node.name(), os_cr.spec.evictpodforce) .await?; - match RollbackMethod::new().call(&self.agent_client) { + let agent_call_client = AgentCallClient::default(); + match self.agent_client.rollback_method(agent_call_client) { Ok(_resp) => {} Err(e) => { return Err(Error::AgentError { source: e }); @@ -395,7 +397,6 @@ impl ProxyController { async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> { use crate::controller::drain::error::DrainError::*; match drain_os(&self.k8s_client.clone(), node_name, force).await { - Err(FindTargetPods { source, .. }) => Err(Error::KubeError { source: source }), Err(DeletePodsError { errors, .. }) => Err(Error::DrainNodeError { value: errors.join("; "), }), @@ -404,13 +405,13 @@ impl ProxyController { } } -fn convert_to_agent_config(configs: Configs) -> Option> { - let mut agent_configs: Vec = Vec::new(); +fn convert_to_agent_config(configs: Configs) -> Option> { + let mut agent_configs: Vec = Vec::new(); if let Some(config_list) = configs.configs { for config in config_list.into_iter() { match config.contents.and_then(convert_to_config_hashmap) { Some(contents_tmp) => { - let config_tmp = AgentSysconfig { + let config_tmp = Sysconfig { model: config.model.unwrap_or_default(), config_path: config.configpath.unwrap_or_default(), contents: contents_tmp, @@ -445,6 +446,7 @@ fn convert_to_config_hashmap(contents: Vec) -> Option = match pods_api.list(&lp).await { Ok(pods @ ObjectList { .. }) => pods, Err(err) => { - return Err(DrainError::FindTargetPods { + return Err(GetPodListsError { source: err, node_name: node_name.to_string(), }); @@ -85,7 +87,7 @@ async fn get_pods_deleted( } } if filterd_err.len() > 0 { - return Err(error::DrainError::DeletePodsError { + return Err(DeletePodsError { errors: filterd_err, }); } @@ -114,16 +116,27 @@ async fn evict_pod( match eviction_result { Ok(_) => { pod.name(); - event!(Level::INFO, "Successfully evicted Pod '{}'", pod.name_any()); + debug!("Successfully evicted Pod '{}'", pod.name_any()); break; } Err(kube::Error::Api(e)) => { let status_code = StatusCode::from_u16(e.code); match status_code { - Ok(StatusCode::TOO_MANY_REQUESTS) => { - event!( - Level::ERROR, - "Too many requests when creating Eviction for Pod '{}': '{}'. This is likely due to respecting a Pod Disruption Budget. Retrying in {:.2}s.", + Ok(StatusCode::FORBIDDEN) => { + return Err(EvictionErrorNoRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(StatusCode::NOT_FOUND) => { + return Err(EvictionErrorNoRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(StatusCode::INTERNAL_SERVER_ERROR) => { + error!( + "Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to misconfigured PodDisruptionBudgets.", pod.name_any(), e, EVERY_EVICTION_RETRY.as_secs_f64() @@ -131,10 +144,8 @@ async fn evict_pod( sleep(EVERY_EVICTION_RETRY).await; continue; } - Ok(StatusCode::INTERNAL_SERVER_ERROR) => { - event!( - Level::ERROR, - "Error when evicting Pod '{}': '{}'. Check for misconfigured PodDisruptionBudgets. Retrying in {:.2}s.", + Ok(StatusCode::TOO_MANY_REQUESTS) => { + error!("Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to PodDisruptionBugets.", pod.name_any(), e, EVERY_EVICTION_RETRY.as_secs_f64() @@ -142,37 +153,24 @@ async fn evict_pod( sleep(EVERY_EVICTION_RETRY).await; continue; } - Ok(StatusCode::NOT_FOUND) => { - return Err(error::EvictionError::NonRetriableEviction { - source: kube::Error::Api(e.clone()), - pod_name: pod.name_any(), - }); - } - Ok(StatusCode::FORBIDDEN) => { - return Err(error::EvictionError::NonRetriableEviction { - source: kube::Error::Api(e.clone()), - pod_name: pod.name_any(), - }); - } Ok(_) => { - event!( - Level::ERROR, - "Error when evicting Pod '{}': '{}'.", + error!( + "Evict pod {} reported error: '{}'.", pod.name_any(), e ); - return Err(error::EvictionError::RetriableEviction { + return Err(EvictionErrorRetry { source: kube::Error::Api(e.clone()), pod_name: pod.name_any(), }); } Err(_) => { - event!( - Level::ERROR, - "Received invalid response code from Kubernetes API: '{}'", + error!( + "Evict pod {} reported error: '{}'.Received invalid response code from Kubernetes API", + pod.name_any(), e ); - return Err(error::EvictionError::RetriableEviction { + return Err(EvictionErrorRetry { source: kube::Error::Api(e.clone()), pod_name: pod.name_any(), }); @@ -180,8 +178,8 @@ async fn evict_pod( } } Err(e) => { - event!(Level::ERROR, "Eviction failed: '{}'. Retrying...", e); - return Err(error::EvictionError::RetriableEviction { + error!("Evict pod {} reported error: '{}' and will retry", pod.name_any(),e); + return Err(EvictionErrorRetry { source: e, pod_name: pod.name_any(), }); @@ -198,37 +196,28 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e let start_time = Instant::now(); let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); + let response_error_not_found: u16 = 404; loop { match pod_api.get(&pod.name_any()).await { - Err(kube::Error::Api(e)) if e.code == 404 => { - event!(Level::INFO, "Pod {} deleted.", pod.name_any()); - break; - } - Ok(p) if p.uid() != pod.uid() => { - let name = p - .metadata - .name - .clone() - .or_else(|| p.metadata.generate_name.clone()) - .unwrap_or_default(); - event!(Level::INFO, "Pod {} deleted.", name); + let name = (&p).name_any(); + info!("Pod {} deleted.", name); break; } - Ok(_) => { - event!( - Level::DEBUG, - "Pod '{}' not yet deleted. Waiting {}s.", + info!( + "Pod '{}' is not yet deleted. Waiting {}s.", pod.name_any(), EVERY_DELETION_CHECK.as_secs_f64() ); } - + Err(kube::Error::Api(e)) if e.code == response_error_not_found => { + info!("Pod {} is deleted.", pod.name_any()); + break; + } Err(e) => { - event!( - Level::ERROR, - "Could not determine if Pod '{}' has been deleted: '{}'. Waiting {}s.", + error!( + "Get pod {} reported error: '{}', whether pod is deleted cannot be determined, waiting {}s.", pod.name_any(), e, EVERY_DELETION_CHECK.as_secs_f64() @@ -236,7 +225,7 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e } } if start_time.elapsed() > TIMEOUT { - return Err(error::DrainError::WaitForDeletion { + return Err(WaitDeletionError { pod_name: pod.name_any(), max_wait: TIMEOUT, }); @@ -246,12 +235,14 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e } Ok(()) } + fn get_pod_api_with_namespace(client: &kube::Client, pod: &Pod) -> Api { match pod.metadata.namespace.as_ref() { Some(namespace) => Api::namespaced(client.clone(), namespace), None => Api::default_namespaced(client.clone()), } } + trait NameAny { fn name_any(self: &Self) -> String; } @@ -480,7 +471,7 @@ impl PodFilter for CombinedFilter { fn filter(self: &Self, pod: &Pod) -> Box { let mut filter_res = self.deleted_filter.filter(pod); if !filter_res.result { - event!(Level::INFO, filter_res.desc); + info!("{}", filter_res.desc); return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), @@ -489,7 +480,7 @@ impl PodFilter for CombinedFilter { } filter_res = self.daemon_filter.filter(pod); if !filter_res.result { - event!(Level::INFO, filter_res.desc); + info!("{}", filter_res.desc); return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), @@ -498,7 +489,7 @@ impl PodFilter for CombinedFilter { } filter_res = self.mirror_filter.filter(pod); if !filter_res.result { - event!(Level::INFO, filter_res.desc); + info!("{}", filter_res.desc); return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), @@ -507,7 +498,7 @@ impl PodFilter for CombinedFilter { } filter_res = self.local_storage_filter.filter(pod); if !filter_res.result { - event!(Level::INFO, filter_res.desc); + info!("{}", filter_res.desc); return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), @@ -516,7 +507,7 @@ impl PodFilter for CombinedFilter { } filter_res = self.unreplicated_filter.filter(pod); if !filter_res.result { - event!(Level::INFO, filter_res.desc); + info!("{}", filter_res.desc); return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), @@ -592,7 +583,7 @@ impl tokio_retry::Condition for ErrorHandleStrategy { match self { Self::TolerateStrategy => false, Self::RetryStrategy => { - if let error::EvictionError::RetriableEviction { .. } = error { + if let error::EvictionError::EvictionErrorRetry { .. } = error { true } else { false @@ -603,46 +594,36 @@ impl tokio_retry::Condition for ErrorHandleStrategy { } pub mod error { - use snafu::Snafu; + use thiserror::Error; use tokio::time::Duration; - #[derive(Debug, Snafu)] - #[snafu(visibility(pub))] + #[derive(Debug, Error)] pub enum DrainError { - #[snafu(display("Unable to find drainable Pods for Node '{}': '{}'", node_name, source))] - FindTargetPods { + #[error("Get node {} pods list error reported: {}", node_name, source)] + GetPodListsError { source: kube::Error, node_name: String, }, - #[snafu( - display( - "Pod '{}' was not deleted in the time allocated ({:.2}s).", - pod_name, - max_wait.as_secs_f64() - ) - )] - WaitForDeletion { + #[error("Pod '{}' was not deleted in the time allocated ({:.2}s).",pod_name,max_wait.as_secs_f64())] + WaitDeletionError { pod_name: String, max_wait: Duration, }, - DeletePodsError { - errors: Vec, - }, + #[error("")] + DeletePodsError { errors: Vec }, } - #[derive(Debug, Snafu)] - #[snafu(visibility(pub))] + #[derive(Debug, Error)] pub enum EvictionError { - #[snafu(display("Unable to evict Pod '{}': '{}'", pod_name, source))] - RetriableEviction { + #[error("Evict Pod {} error: '{}'", pod_name, source)] + EvictionErrorRetry { source: kube::Error, pod_name: String, }, - #[snafu(display("Unable to evict Pod '{}': '{}'", pod_name, source))] - /// A fatal error occurred while attempting to evict a Pod. This will not be retried. - NonRetriableEviction { + #[error("Evict Pod {} error: '{}'", pod_name, source)] + EvictionErrorNoRetry { source: kube::Error, pod_name: String, }, diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs index e2e06493..090febc6 100644 --- a/KubeOS-Rust/proxy/src/controller/mod.rs +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -10,6 +10,7 @@ * See the Mulan PSL v2 for more details. */ +mod agentclient; mod apiclient; mod controller; mod crd; @@ -17,6 +18,7 @@ mod drain; mod utils; mod values; +pub use agentclient::AgentClient; pub use apiclient::ControllerClient; pub use controller::{error_policy, reconcile, reconciler_error::Error, ProxyController}; pub use crd::OS; diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs index 43610a64..ef53117d 100644 --- a/KubeOS-Rust/proxy/src/main.rs +++ b/KubeOS-Rust/proxy/src/main.rs @@ -20,8 +20,9 @@ use kube::{ }; use log::{error, info}; mod controller; -use cli::client::Client as AgentClient; -use controller::{error_policy, reconcile, ControllerClient, ProxyController, OS, SOCK_PATH}; +use controller::{ + error_policy, reconcile, AgentClient, ControllerClient, ProxyController, OS, SOCK_PATH, +}; const PROXY_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); #[tokio::main] -- Gitee From d6c4896626a4317c59a012eaf0091dd4a006d262 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 28 Dec 2023 16:27:55 +0800 Subject: [PATCH 06/46] feat(os-agent): add disk image type Support using docker to pull upgrade container image. Support using http/https url to download upgrade raw disk image. Signed-off-by: Yuhang Wei --- KubeOS-Rust/Cargo.lock | 849 ++++++------------ KubeOS-Rust/agent/Cargo.toml | 6 +- KubeOS-Rust/agent/src/rpc/agent_impl.rs | 23 +- KubeOS-Rust/cli/src/client.rs | 10 +- KubeOS-Rust/cli/src/method/callable_method.rs | 2 +- KubeOS-Rust/cli/src/method/cleanup.rs | 7 +- KubeOS-Rust/cli/src/method/request.rs | 2 +- KubeOS-Rust/cli/src/method/rollback.rs | 7 +- KubeOS-Rust/cli/src/method/upgrade.rs | 7 +- KubeOS-Rust/manager/Cargo.toml | 2 + KubeOS-Rust/manager/src/api/types.rs | 17 +- KubeOS-Rust/manager/src/sys_mgmt/config.rs | 55 +- .../manager/src/sys_mgmt/containerd_image.rs | 69 +- .../manager/src/sys_mgmt/disk_image.rs | 374 ++++++++ .../manager/src/sys_mgmt/docker_image.rs | 157 ++++ KubeOS-Rust/manager/src/sys_mgmt/mod.rs | 6 +- KubeOS-Rust/manager/src/sys_mgmt/values.rs | 3 +- KubeOS-Rust/manager/src/utils/common.rs | 54 +- .../manager/src/utils/container_image.rs | 88 +- KubeOS-Rust/manager/src/utils/executor.rs | 20 +- KubeOS-Rust/manager/src/utils/partition.rs | 17 +- KubeOS-Rust/proxy/Cargo.toml | 2 +- .../proxy/src/controller/agentclient.rs | 16 +- 23 files changed, 1077 insertions(+), 716 deletions(-) create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 08ad969e..bc0b38b1 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -51,7 +51,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -77,6 +77,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + [[package]] name = "bitflags" version = "1.3.2" @@ -89,6 +95,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "0.2.17" @@ -104,22 +119,6 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "bytes" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - [[package]] name = "bytes" version = "1.5.0" @@ -135,12 +134,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -174,15 +167,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "core-foundation" version = "0.9.3" @@ -200,51 +184,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] -name = "crossbeam-deque" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", - "maybe-uninit", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", -] - -[[package]] -name = "crossbeam-queue" -version = "0.2.3" +name = "cpufeatures" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils", - "maybe-uninit", + "libc", ] [[package]] -name = "crossbeam-utils" -version = "0.7.2" +name = "crypto-common" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", + "generic-array", + "typenum", ] [[package]] @@ -288,7 +243,7 @@ version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "num_cpus", ] @@ -309,13 +264,23 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs-next" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -327,7 +292,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -354,7 +319,7 @@ version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -445,28 +410,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags 1.3.2", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - -[[package]] -name = "futures" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" - [[package]] name = "futures" version = "0.3.29" @@ -556,13 +499,23 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -573,7 +526,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] @@ -597,7 +550,7 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ - "bytes 1.5.0", + "bytes", "fnv", "futures-core", "futures-sink", @@ -605,7 +558,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.14.0", + "tokio", "tokio-util 0.7.2", "tracing", ] @@ -637,7 +590,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.5.0", + "bytes", "fnv", "itoa", ] @@ -648,7 +601,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.5.0", + "bytes", "http", "pin-project-lite", ] @@ -683,7 +636,7 @@ version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ - "bytes 1.5.0", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -695,12 +648,25 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio 1.14.0", + "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -709,7 +675,7 @@ checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ "hyper", "pin-project-lite", - "tokio 1.14.0", + "tokio", "tokio-io-timeout", ] @@ -719,10 +685,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.5.0", + "bytes", "hyper", "native-tls", - "tokio 1.14.0", + "tokio", "tokio-native-tls", ] @@ -781,7 +747,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -795,15 +761,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "ipnet" version = "2.9.0" @@ -862,18 +819,20 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd8d6b3f301ba426b30feca834a2a18d48d5b54e5065496b5c1b05537bee3639" dependencies = [ - "base64", + "base64 0.13.1", "serde", "serde_json", ] [[package]] name = "jsonrpc-core" -version = "15.1.0" +version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0745a6379e3edc893c84ec203589790774e4247420033e71a76d3ab4687991fa" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" dependencies = [ - "futures 0.1.31", + "futures", + "futures-executor", + "futures-util", "log", "serde", "serde_derive", @@ -882,9 +841,9 @@ dependencies = [ [[package]] name = "jsonrpc-derive" -version = "15.1.0" +version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99a847f9ec7bb52149b2786a17c9cb260d6effc6b8eeb8c16b343a487a7563a3" +checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -894,31 +853,34 @@ dependencies = [ [[package]] name = "jsonrpc-ipc-server" -version = "15.1.0" +version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf50e53e4eea8f421a7316c5f63e395f7bc7c4e786a6dc54d76fab6ff7aa7ce7" +checksum = "382bb0206323ca7cda3dcd7e245cea86d37d02457a02a975e3378fb149a48845" dependencies = [ + "futures", "jsonrpc-core", "jsonrpc-server-utils", "log", "parity-tokio-ipc", - "parking_lot 0.10.2", - "tokio-service", + "parking_lot", + "tower-service", ] [[package]] name = "jsonrpc-server-utils" -version = "15.1.0" +version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f1f3990650c033bd8f6bd46deac76d990f9bbfb5f8dc8c4767bf0a00392176" +checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4" dependencies = [ - "bytes 0.4.12", + "bytes", + "futures", "globset", "jsonrpc-core", "lazy_static", "log", - "tokio 0.1.22", - "tokio-codec", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", "unicase", ] @@ -928,8 +890,8 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c" dependencies = [ - "base64", - "bytes 1.5.0", + "base64 0.13.1", + "bytes", "chrono", "http", "percent-encoding", @@ -939,16 +901,6 @@ dependencies = [ "url", ] -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "kube" version = "0.66.0" @@ -968,12 +920,12 @@ version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232db1af3d3680f9289cf0b4db51b2b9fee22550fc65d25869e39b23e0aaa696" dependencies = [ - "base64", - "bytes 1.5.0", + "base64 0.13.1", + "bytes", "chrono", "dirs-next", "either", - "futures 0.3.29", + "futures", "http", "http-body", "hyper", @@ -990,7 +942,7 @@ dependencies = [ "serde_json", "serde_yaml", "thiserror", - "tokio 1.14.0", + "tokio", "tokio-native-tls", "tokio-util 0.6.10", "tower", @@ -1038,16 +990,16 @@ dependencies = [ "backoff", "dashmap", "derivative", - "futures 0.3.29", + "futures", "json-patch", "k8s-openapi", "kube-client", "pin-project", "serde", "serde_json", - "smallvec 1.11.1", + "smallvec", "thiserror", - "tokio 1.14.0", + "tokio", "tokio-util 0.6.10", "tracing", ] @@ -1060,9 +1012,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libredox" @@ -1089,10 +1041,11 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" -version = "0.3.4" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ + "autocfg", "scopeguard", ] @@ -1102,7 +1055,7 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c4dcd960cc540667f619483fc99102f88d6118b87730e24e8fbe8054b7445e4" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1117,32 +1070,19 @@ dependencies = [ "nix", "predicates", "regex", + "reqwest", "serde", "serde_json", + "sha2", "tempfile", ] -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.7.1" @@ -1158,25 +1098,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "mio" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow 0.2.2", - "net2", - "slab", - "winapi 0.2.8", -] - [[package]] name = "mio" version = "0.7.14" @@ -1185,44 +1106,9 @@ checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" dependencies = [ "libc", "log", - "miow 0.3.7", + "miow", "ntapi", - "winapi 0.3.9", -] - -[[package]] -name = "mio-named-pipes" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" -dependencies = [ - "log", - "mio 0.6.23", - "miow 0.3.7", - "winapi 0.3.9", -] - -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio 0.6.23", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", + "winapi", ] [[package]] @@ -1231,7 +1117,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1240,7 +1126,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50e4a1c770583dac7ab5e2f6c139153b783a53a1bbee9729613f193e59828326" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "downcast", "fragile", "lazy_static", @@ -1255,7 +1141,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "proc-macro2", "quote", "syn 1.0.109", @@ -1279,17 +1165,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "net2" -version = "0.2.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", -] - [[package]] name = "nix" version = "0.26.4" @@ -1297,9 +1172,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", - "cfg-if 1.0.0", + "cfg-if", "libc", - "memoffset 0.7.1", + "memoffset", "pin-utils", ] @@ -1315,7 +1190,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1350,7 +1225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" dependencies = [ "bitflags 2.4.0", - "cfg-if 1.0.0", + "cfg-if", "foreign-types", "libc", "once_cell", @@ -1415,71 +1290,41 @@ dependencies = [ [[package]] name = "parity-tokio-ipc" -version = "0.4.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e57fea504fea33f9fbb5f49f378359030e7e026a6ab849bb9e8f0787376f1bf" +checksum = "9981e32fb75e004cc148f5fb70342f393830e0a4aa62e3cc93b50976218d42b6" dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", + "futures", "libc", "log", - "mio-named-pipes", - "miow 0.3.7", "rand 0.7.3", - "tokio 0.1.22", - "tokio-named-pipes", - "tokio-uds", - "winapi 0.3.9", -] - -[[package]] -name = "parking_lot" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" -dependencies = [ - "lock_api", - "parking_lot_core 0.6.3", - "rustc_version", + "tokio", + "winapi", ] [[package]] name = "parking_lot" -version = "0.10.2" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ + "instant", "lock_api", - "parking_lot_core 0.7.3", + "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.6.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ - "cfg-if 0.1.10", - "cloudabi", - "libc", - "redox_syscall 0.1.57", - "rustc_version", - "smallvec 0.6.14", - "winapi 0.3.9", -] - -[[package]] -name = "parking_lot_core" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93f386bb233083c799e6e642a9d73db98c24a5deeb95ffc85bf281255dffc98" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi", + "cfg-if", + "instant", "libc", - "redox_syscall 0.1.57", - "smallvec 1.11.1", - "winapi 0.3.9", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -1488,7 +1333,7 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" dependencies = [ - "base64", + "base64 0.13.1", ] [[package]] @@ -1598,7 +1443,7 @@ dependencies = [ "chrono", "cli", "env_logger", - "futures 0.3.29", + "futures", "h2", "k8s-openapi", "kube", @@ -1612,7 +1457,7 @@ dependencies = [ "socket2", "thiserror", "thread_local", - "tokio 1.14.0", + "tokio", "tokio-retry", ] @@ -1698,9 +1543,12 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.57" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] [[package]] name = "redox_syscall" @@ -1750,12 +1598,12 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.10" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64", - "bytes 1.5.0", + "base64 0.21.5", + "bytes", "encoding_rs", "futures-core", "futures-util", @@ -1763,31 +1611,46 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", - "lazy_static", "log", "mime", + "native-tls", + "once_cell", "percent-encoding", "pin-project-lite", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", - "tokio 1.14.0", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] [[package]] -name = "rustc_version" -version = "0.2.3" +name = "ring" +version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" dependencies = [ - "semver", + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", ] [[package]] @@ -1804,6 +1667,37 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustls" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.5", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.15" @@ -1849,6 +1743,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "secrecy" version = "0.8.0" @@ -1882,21 +1786,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "serde" version = "1.0.188" @@ -1974,6 +1863,17 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1992,15 +1892,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "smallvec" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - [[package]] name = "smallvec" version = "1.11.1" @@ -2014,9 +1905,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "strsim" version = "0.10.0" @@ -2052,7 +1949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ "autocfg", - "cfg-if 1.0.0", + "cfg-if", "fastrand", "redox_syscall 0.3.5", "rustix", @@ -2118,30 +2015,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "tokio" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "mio 0.6.23", - "num_cpus", - "tokio-codec", - "tokio-current-thread", - "tokio-executor", - "tokio-fs", - "tokio-io", - "tokio-reactor", - "tokio-sync", - "tokio-tcp", - "tokio-threadpool", - "tokio-timer", - "tokio-udp", - "tokio-uds", -] - [[package]] name = "tokio" version = "1.14.0" @@ -2149,69 +2022,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" dependencies = [ "autocfg", - "bytes 1.5.0", + "bytes", "libc", "memchr", - "mio 0.7.14", + "mio", "num_cpus", "once_cell", "pin-project-lite", "signal-hook-registry", "tokio-macros", - "winapi 0.3.9", -] - -[[package]] -name = "tokio-codec" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "tokio-io", -] - -[[package]] -name = "tokio-current-thread" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" -dependencies = [ - "futures 0.1.31", - "tokio-executor", -] - -[[package]] -name = "tokio-executor" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" -dependencies = [ - "crossbeam-utils", - "futures 0.1.31", -] - -[[package]] -name = "tokio-fs" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" -dependencies = [ - "futures 0.1.31", - "tokio-io", - "tokio-threadpool", -] - -[[package]] -name = "tokio-io" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", + "winapi", ] [[package]] @@ -2221,7 +2041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ "pin-project-lite", - "tokio 1.14.0", + "tokio", ] [[package]] @@ -2235,19 +2055,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "tokio-named-pipes" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d282d483052288b2308ba5ee795f5673b159c9bdf63c385a05609da782a5eae" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "mio 0.6.23", - "mio-named-pipes", - "tokio 0.1.22", -] - [[package]] name = "tokio-native-tls" version = "0.3.1" @@ -2255,26 +2062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.14.0", -] - -[[package]] -name = "tokio-reactor" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" -dependencies = [ - "crossbeam-utils", - "futures 0.1.31", - "lazy_static", - "log", - "mio 0.6.23", - "num_cpus", - "parking_lot 0.9.0", - "slab", - "tokio-executor", - "tokio-io", - "tokio-sync", + "tokio", ] [[package]] @@ -2285,102 +2073,28 @@ checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" dependencies = [ "pin-project", "rand 0.8.5", - "tokio 1.14.0", -] - -[[package]] -name = "tokio-service" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" -dependencies = [ - "futures 0.1.31", -] - -[[package]] -name = "tokio-sync" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" -dependencies = [ - "fnv", - "futures 0.1.31", -] - -[[package]] -name = "tokio-tcp" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "mio 0.6.23", - "tokio-io", - "tokio-reactor", -] - -[[package]] -name = "tokio-threadpool" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" -dependencies = [ - "crossbeam-deque", - "crossbeam-queue", - "crossbeam-utils", - "futures 0.1.31", - "lazy_static", - "log", - "num_cpus", - "slab", - "tokio-executor", + "tokio", ] [[package]] -name = "tokio-timer" -version = "0.2.13" +name = "tokio-rustls" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "crossbeam-utils", - "futures 0.1.31", - "slab", - "tokio-executor", + "rustls", + "tokio", ] [[package]] -name = "tokio-udp" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", - "mio 0.6.23", - "tokio-codec", - "tokio-io", - "tokio-reactor", -] - -[[package]] -name = "tokio-uds" -version = "0.2.7" +name = "tokio-stream" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "libc", - "log", - "mio 0.6.23", - "mio-uds", - "tokio-codec", - "tokio-io", - "tokio-reactor", + "futures-core", + "pin-project-lite", + "tokio", ] [[package]] @@ -2389,13 +2103,13 @@ version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" dependencies = [ - "bytes 1.5.0", + "bytes", "futures-core", "futures-sink", "log", "pin-project-lite", "slab", - "tokio 1.14.0", + "tokio", ] [[package]] @@ -2404,11 +2118,11 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" dependencies = [ - "bytes 1.5.0", + "bytes", "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.14.0", + "tokio", "tracing", ] @@ -2431,7 +2145,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio 1.14.0", + "tokio", "tokio-util 0.7.2", "tower-layer", "tower-service", @@ -2444,9 +2158,9 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8" dependencies = [ - "base64", + "base64 0.13.1", "bitflags 1.3.2", - "bytes 1.5.0", + "bytes", "futures-core", "futures-util", "http", @@ -2476,7 +2190,7 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -2518,6 +2232,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicase" version = "2.7.0" @@ -2548,6 +2268,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.4.1" @@ -2598,7 +2324,7 @@ version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -2623,7 +2349,7 @@ version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -2669,10 +2395,23 @@ dependencies = [ ] [[package]] -name = "winapi" -version = "0.2.8" +name = "webpki" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] [[package]] name = "winapi" @@ -2684,12 +2423,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -2702,7 +2435,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2792,17 +2525,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", + "winapi", ] [[package]] diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml index fd7d3905..15defa6f 100644 --- a/KubeOS-Rust/agent/Cargo.toml +++ b/KubeOS-Rust/agent/Cargo.toml @@ -8,9 +8,9 @@ license = "MulanPSL-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] manager = { package = "manager", path = "../manager" } -jsonrpc-core = { version = "15.1" } -jsonrpc-derive = { version = "15.1" } -jsonrpc-ipc-server = { version = "15.1" } +jsonrpc-core = { version = "18.0" } +jsonrpc-derive = { version = "18.0" } +jsonrpc-ipc-server = { version = "18.0" } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } log = { version = "= 0.4.15" } diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs index 95a28348..b84b1941 100644 --- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -12,7 +12,7 @@ use std::{sync::Mutex, thread, time::Duration}; -use anyhow::{anyhow, Result}; +use anyhow::{bail, Result}; use log::{debug, error, info}; use nix::{sys::reboot::RebootMode, unistd::sync}; @@ -22,7 +22,10 @@ use super::{ }; use manager::{ api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, - sys_mgmt::{CtrImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH}, + sys_mgmt::{ + CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, + DEFAULT_GRUBENV_PATH, + }, utils::{ clean_env, get_partition_info, switch_boot_menuentry, PreparePath, RealCommandExecutor, }, @@ -72,7 +75,9 @@ impl AgentImpl { let handler: Box> = match req.image_type.as_str() { "containerd" => Box::new(ImageType::Containerd(CtrImageHandler::default())), - _ => return Err(anyhow!("Invalid image type \"{}\"", req.image_type)), + "docker" => Box::new(ImageType::Docker(DockerImageHandler::default())), + "disk" => Box::new(ImageType::Disk(DiskImageHandler::default())), + _ => bail!("Invalid image type \"{}\"", req.image_type), }; let image_manager = handler.download_image(&req)?; @@ -129,7 +134,7 @@ impl AgentImpl { configuration.set_config(config)?; } else { error!("Unknown configuration type: \"{}\"", config_type); - Err(anyhow!("Unknown configuration type: \"{}\"", config_type))?; + bail!("Unknown configuration type: \"{}\"", config_type); } } Ok(Response { @@ -172,7 +177,7 @@ impl AgentImpl { #[cfg(test)] mod test { use super::*; - use manager::api::Sysconfig; + use manager::api::{CertsInfo, Sysconfig}; use std::collections::HashMap; #[test] @@ -217,6 +222,14 @@ mod test { check_sum: "xxx".into(), image_type: "xxx".into(), container_image: "xxx".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { + ca_cert: "".to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, }; let res = agent.prepare_upgrade_impl(req); assert!(res.is_err()); diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs index 71121fe0..d8f68d07 100644 --- a/KubeOS-Rust/cli/src/client.rs +++ b/KubeOS-Rust/cli/src/client.rs @@ -28,10 +28,9 @@ impl<'a> Request<'a> {} impl Client { pub fn new>(socket_path: P) -> Self { - let client = Client { + Client { json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)), - }; - client + } } pub fn build_request<'a>( @@ -39,14 +38,13 @@ impl Client { command: &'a str, params: &'a Vec>, ) -> Request<'a> { - let json_rpc_request = self.json_rpc_client.build_request(command, ¶ms); + let json_rpc_request = self.json_rpc_client.build_request(command, params); let request = Request(json_rpc_request); request } pub fn send_request(&self, request: Request) -> Result { - let response = self.json_rpc_client.send_request(request.0); - response + self.json_rpc_client.send_request(request.0) } } diff --git a/KubeOS-Rust/cli/src/method/callable_method.rs b/KubeOS-Rust/cli/src/method/callable_method.rs index d59ebd62..c46614b4 100644 --- a/KubeOS-Rust/cli/src/method/callable_method.rs +++ b/KubeOS-Rust/cli/src/method/callable_method.rs @@ -21,6 +21,6 @@ pub trait RpcMethod { fn command_params(&self) -> Vec>; fn call(&self, client: &Client) -> Result { let response = request(client, self.command_name(), self.command_params())?; - response.result().map_err(|e| parse_error(e)) + response.result().map_err(parse_error) } } diff --git a/KubeOS-Rust/cli/src/method/cleanup.rs b/KubeOS-Rust/cli/src/method/cleanup.rs index 48a03bc8..f072a107 100644 --- a/KubeOS-Rust/cli/src/method/cleanup.rs +++ b/KubeOS-Rust/cli/src/method/cleanup.rs @@ -15,14 +15,9 @@ use serde_json::value::RawValue; use crate::method::callable_method::RpcMethod; use kubeos_manager::api; +#[derive(Default)] pub struct CleanupMethod {} -impl CleanupMethod { - pub fn new() -> Self { - CleanupMethod {} - } -} - impl RpcMethod for CleanupMethod { type Response = api::Response; fn command_name(&self) -> &'static str { diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs index 4e3dbec6..d4d2a97a 100644 --- a/KubeOS-Rust/cli/src/method/request.rs +++ b/KubeOS-Rust/cli/src/method/request.rs @@ -23,7 +23,7 @@ pub fn request( params: Vec>, ) -> Result { let request = client.build_request(command, ¶ms); - let response = client.send_request(request).map_err(|e| parse_error(e)); + let response = client.send_request(request).map_err(parse_error); debug!("{:#?}", response); response } diff --git a/KubeOS-Rust/cli/src/method/rollback.rs b/KubeOS-Rust/cli/src/method/rollback.rs index 5b9b0fde..0f30a827 100644 --- a/KubeOS-Rust/cli/src/method/rollback.rs +++ b/KubeOS-Rust/cli/src/method/rollback.rs @@ -15,14 +15,9 @@ use serde_json::value::RawValue; use crate::method::callable_method::RpcMethod; use kubeos_manager::api; +#[derive(Default)] pub struct RollbackMethod {} -impl RollbackMethod { - pub fn new() -> Self { - RollbackMethod {} - } -} - impl RpcMethod for RollbackMethod { type Response = api::Response; fn command_name(&self) -> &'static str { diff --git a/KubeOS-Rust/cli/src/method/upgrade.rs b/KubeOS-Rust/cli/src/method/upgrade.rs index 9098e197..ef773bce 100644 --- a/KubeOS-Rust/cli/src/method/upgrade.rs +++ b/KubeOS-Rust/cli/src/method/upgrade.rs @@ -15,14 +15,9 @@ use serde_json::value::RawValue; use crate::method::callable_method::RpcMethod; use kubeos_manager::api; +#[derive(Default)] pub struct UpgradeMethod {} -impl UpgradeMethod { - pub fn new() -> Self { - UpgradeMethod {} - } -} - impl RpcMethod for UpgradeMethod { type Response = api::Response; fn command_name(&self) -> &'static str { diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml index 0b82b17d..a332c9e0 100644 --- a/KubeOS-Rust/manager/Cargo.toml +++ b/KubeOS-Rust/manager/Cargo.toml @@ -18,5 +18,7 @@ log = { version = "0.4" } anyhow = { version = "1.0" } env_logger = { version = "0.9" } lazy_static = { version = "1.4" } +reqwest = { version = "=0.11.18", features = ["rustls-tls", "blocking"] } regex = { version = "1.7.3" } nix = { version = "0.26.2" } +sha2 = { version = "0.10.8" } diff --git a/KubeOS-Rust/manager/src/api/types.rs b/KubeOS-Rust/manager/src/api/types.rs index e21f55bf..28ee97db 100644 --- a/KubeOS-Rust/manager/src/api/types.rs +++ b/KubeOS-Rust/manager/src/api/types.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; use super::agent_status::*; use crate::{ - sys_mgmt::CtrImageHandler, + sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler}, utils::{CommandExecutor, UpgradeImageManager}, }; @@ -26,6 +26,17 @@ pub struct UpgradeRequest { pub check_sum: String, pub image_type: String, pub container_image: String, + pub image_url: String, + pub flag_safe: bool, + pub mtls: bool, + pub certs: CertsInfo, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct CertsInfo { + pub ca_cert: String, + pub client_cert: String, + pub client_key: String, } #[derive(Deserialize, Serialize, Debug)] @@ -53,12 +64,16 @@ pub struct Response { pub enum ImageType { Containerd(CtrImageHandler), + Docker(DockerImageHandler), + Disk(DiskImageHandler), } impl ImageType { pub fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result> { match self { ImageType::Containerd(handler) => handler.download_image(req), + ImageType::Docker(handler) => handler.download_image(req), + ImageType::Disk(handler) => handler.download_image(req), } } } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs index deec26ea..2e4402d4 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -19,13 +19,12 @@ use std::{ string::String, }; -use anyhow::{anyhow, Context, Ok, Result}; +use anyhow::{bail, Context, Result}; use lazy_static::lazy_static; use log::{debug, info, trace, warn}; use regex::Regex; -use super::{api::*, values}; -use crate::utils::*; +use crate::{api::*, sys_mgmt::values, utils::*}; lazy_static! { pub static ref CONFIG_TEMPLATE: HashMap> = { @@ -76,7 +75,7 @@ impl Configuration for KernelSysctl { let proc_path = self.get_proc_path(key); if key_info.operation == "delete" { warn!("Failed to delete kernel.sysctl config with key \"{}\"", key); - } else if key_info.value != "" && key_info.operation == "" { + } else if !key_info.value.is_empty() && key_info.operation.is_empty() { fs::write(&proc_path, format!("{}\n", &key_info.value).as_bytes()).with_context( || format!("Failed to write kernel.sysctl with key: \"{}\"", key), )?; @@ -100,7 +99,7 @@ impl KernelSysctl { } fn get_proc_path(&self, key: &str) -> PathBuf { - let path_str = format!("{}{}", self.proc_path, key.replace(".", "/")); + let path_str = format!("{}{}", self.proc_path, key.replace('.', "/")); Path::new(&path_str).to_path_buf() } } @@ -109,7 +108,7 @@ impl Configuration for KernelSysctlPersist { fn set_config(&self, config: &mut Sysconfig) -> Result<()> { info!("Start set kernel.sysctl.persist"); let mut config_path = &values::DEFAULT_KERNEL_CONFIG_PATH.to_string(); - if config.config_path != "" { + if !config.config_path.is_empty() { config_path = &config.config_path; } debug!("kernel.sysctl.persist config_path: \"{}\"", config_path); @@ -140,14 +139,14 @@ fn get_and_set_configs( for line in io::BufReader::new(f).lines() { let line = line?; // if line is a comment or blank - if line.starts_with("#") || line.starts_with(";") || line.trim().is_empty() { + if line.starts_with('#') || line.starts_with(';') || line.trim().is_empty() { configs_write.push(line); continue; } let config_kv: Vec<&str> = line.splitn(2, '=').map(|s| s.trim()).collect(); // if config_kv is not a key-value pair if config_kv.len() != 2 { - return Err(anyhow!("could not parse sysctl config {}", line)); + bail!("could not parse sysctl config {}", line); } let new_key_info = expect_configs.get(config_kv[0]); let new_config = match new_key_info { @@ -170,7 +169,7 @@ fn write_configs_to_file(config_path: &str, configs: &Vec) -> Result<()> let f = File::create(config_path)?; let mut w = BufWriter::new(f); for line in configs { - if line == "" { + if line.is_empty() { continue; } writeln!(w, "{}", line.as_str())?; @@ -179,17 +178,17 @@ fn write_configs_to_file(config_path: &str, configs: &Vec) -> Result<()> .with_context(|| format!("Failed to flush file {}", config_path))?; w.get_mut() .sync_all() - .with_context(|| format!("Failed to sync"))?; + .with_context(|| "Failed to sync".to_string())?; debug!("Write configuration to file \"{}\" success", config_path); Ok(()) } fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String { let key = config_kv[0]; - if config_kv.len() == 1 && new_config_info.value == "" { + if config_kv.len() == 1 && new_config_info.value.is_empty() { info!("Delete configuration key: \"{}\"", key); return String::from(""); - } else if config_kv.len() == 1 && new_config_info.value != "" { + } else if config_kv.len() == 1 && !new_config_info.value.is_empty() { warn!( "Failed to delete key \"{}\" with inconsistent values \"nil\" and \"{}\"", key, new_config_info.value @@ -210,21 +209,21 @@ fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String fn handle_update_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String { let key = config_kv[0]; - if new_config_info.operation != "" { + if !new_config_info.operation.is_empty() { warn!( "Unknown operation \"{}\", updating key \"{}\" with value \"{}\" by default", new_config_info.operation, key, new_config_info.value ); } - if config_kv.len() == values::ONLY_KEY && new_config_info.value == "" { + if config_kv.len() == values::ONLY_KEY && new_config_info.value.is_empty() { return key.to_string(); } let new_value = new_config_info.value.trim(); - if config_kv.len() == values::ONLY_KEY && new_config_info.value != "" { + if config_kv.len() == values::ONLY_KEY && !new_config_info.value.is_empty() { info!("Update configuration \"{}={}\"", key, new_value); return format!("{}={}", key, new_value); } - if new_config_info.value == "" { + if new_config_info.value.is_empty() { warn!("Failed to update key \"{}\" with \"null\" value", key); return config_kv.join("="); } @@ -242,24 +241,24 @@ fn handle_add_key( warn!("Failed to delete inexistent key: \"{}\"", key); continue; } - if key == "" || key.contains("=") { + if key.is_empty() || key.contains('=') { warn!( "Failed to add \"null\" key or key containing \"=\", key: \"{}\"", key ); continue; } - if config_info.operation != "" { + if !config_info.operation.is_empty() { warn!( "Unknown operation \"{}\", adding key \"{}\" with value \"{}\" by default", config_info.operation, key, config_info.value ); } let (k, v) = (key.trim(), config_info.value.trim()); - if v == "" && is_only_key_valid { + if v.is_empty() && is_only_key_valid { info!("Add configuration \"{}\"", k); configs_write.push(k.to_string()); - } else if v == "" { + } else if v.is_empty() { warn!("Failed to add key \"{}\" with \"null\" value", k); } else { info!("Add configuration \"{}={}\"", k, v); @@ -277,7 +276,7 @@ impl Configuration for GrubCmdline { info!("Start set grub.cmdline.next configuration"); } if !is_file_exist(&self.grub_path) { - return Err(anyhow!("Failed to find grub.cfg file")); + bail!("Failed to find grub.cfg file"); } let config_partition = if cfg!(test) { self.is_cur_partition @@ -337,16 +336,13 @@ fn modify_boot_cfg(expect_configs: &mut HashMap, line: &String) let mut new_configs = vec![" ".to_string()]; let olg_configs: Vec<&str> = line.split(' ').collect(); for old_config in olg_configs { - if old_config == "" { + if old_config.is_empty() { continue; } // At most 2 substrings can be returned to satisfy the case like root=UUID=xxxx - let config = old_config.splitn(2, "=").collect::>(); + let config = old_config.splitn(2, '=').collect::>(); if config.len() != values::ONLY_KEY && config.len() != values::KV_PAIR { - return Err(anyhow!( - "Failed to parse grub.cfg linux line {}", - old_config - )); + bail!("Failed to parse grub.cfg linux line {}", old_config); } let new_key_info = expect_configs.get(config[0]); let new_config = match new_key_info { @@ -560,9 +556,10 @@ mod tests { #[test] fn write_configs_to_file_tests() { init(); - let path = "/home/yuhang/abc.txt"; + let tmp_file = NamedTempFile::new().unwrap(); let configs = vec!["a=1".to_string(), "b=2".to_string()]; - write_configs_to_file(&path.to_string(), &configs).unwrap(); + write_configs_to_file(tmp_file.path().to_str().unwrap(), &configs).unwrap(); + assert_eq!(fs::read(tmp_file.path()).unwrap(), b"a=1\nb=2\n"); } #[test] diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs index 479740e4..e38f22f3 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -15,9 +15,11 @@ use std::{fs, os::unix::fs::PermissionsExt, path::Path}; use anyhow::{anyhow, Result}; use log::{debug, info}; -use super::api::{ImageHandler, UpgradeRequest}; -use crate::sys_mgmt::{IMAGE_PERMISSION, NEED_GB_SIZE, PERSIST_DIR}; -use crate::utils::*; +use crate::{ + api::{ImageHandler, UpgradeRequest}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES, PERSIST_DIR}, + utils::*, +}; pub struct CtrImageHandler { pub paths: PreparePath, @@ -28,7 +30,7 @@ const DEFAULT_NAMESPACE: &str = "k8s.io"; impl ImageHandler for CtrImageHandler { fn download_image(&self, req: &UpgradeRequest) -> Result> { - perpare_env(&self.paths, NEED_GB_SIZE, PERSIST_DIR, IMAGE_PERMISSION)?; + perpare_env(&self.paths, NEED_BYTES, PERSIST_DIR, IMAGE_PERMISSION)?; self.get_image(req)?; self.get_rootfs_archive(req, IMAGE_PERMISSION)?; @@ -60,21 +62,16 @@ impl CtrImageHandler { fn get_image(&self, req: &UpgradeRequest) -> Result<()> { let image_name = &req.container_image; is_valid_image_name(image_name)?; - info!("Start pulling image {}", image_name); - let containerd_command: String; - if is_command_available("crictl", &self.executor) { - containerd_command = "crictl".to_string(); + let cli: String = if is_command_available("crictl", &self.executor) { + "crictl".to_string() } else { - containerd_command = "ctr".to_string(); - } - pull_image(&containerd_command, image_name, &self.executor)?; + "ctr".to_string() + }; + remove_image_if_exist(&cli, image_name, &self.executor)?; + info!("Start pulling image {}", image_name); + pull_image(&cli, image_name, &self.executor)?; info!("Start checking image digest"); - check_oci_image_digest_match( - &containerd_command, - image_name, - &req.check_sum, - &self.executor, - )?; + check_oci_image_digest(&cli, image_name, &req.check_sum, &self.executor)?; Ok(()) } @@ -102,7 +99,7 @@ impl CtrImageHandler { )?; // copy os.tar from mount_path to its partent dir self.copy_file( - &self.paths.mount_path.join(&self.paths.rootfs_file), + self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission, )?; @@ -152,6 +149,7 @@ impl CtrImageHandler { #[cfg(test)] mod tests { use super::*; + use crate::api::CertsInfo; use mockall::mock; use std::io::Write; use std::path::Path; @@ -187,6 +185,14 @@ mod tests { image_type: "containerd".to_string(), container_image: image_name.to_string(), check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { + ca_cert: "".to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, }; // mock is_command_available mock_executor @@ -194,6 +200,17 @@ mod tests { .withf(|cmd, args| cmd == "/bin/sh" && args.contains(&"command -v crictl")) // simplified with a closure .times(1) .returning(|_, _| Ok(())); + // mock remove_image_if_exist + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.contains(&"inspecti")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.contains(&"rmi")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); // mock pull_image mock_executor .expect_run_command() @@ -230,6 +247,14 @@ mod tests { image_type: "containerd".to_string(), container_image: image_name.to_string(), check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { + ca_cert: "".to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, }; // mock check_and_unmount @@ -351,6 +376,14 @@ mod tests { image_type: "containerd".to_string(), container_image: "docker.io/library/busybox:latest".to_string(), check_sum: "".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { + ca_cert: "".to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, }; ctr.download_image(&update_req).unwrap(); let tar_path = "/persist/KubeOS-Update/os.tar"; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs new file mode 100644 index 00000000..f55ae167 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs @@ -0,0 +1,374 @@ +use std::{ + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; + +use anyhow::{bail, Context, Result}; +use log::{debug, info, trace}; +use reqwest::{blocking::Client, Certificate}; +use sha2::{Digest, Sha256}; + +use crate::{ + api::{CertsInfo, ImageHandler, UpgradeRequest}, + sys_mgmt::{CERTS_PATH, IMAGE_PERMISSION, PERSIST_DIR}, + utils::*, +}; + +const BUFFER: u64 = 1024 * 1024 * 10; + +pub struct DiskImageHandler { + pub paths: PreparePath, + pub executor: T, + pub certs_path: String, +} + +impl ImageHandler for DiskImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + self.download(req)?; + self.checksum_match( + self.paths.image_path.to_str().unwrap_or_default(), + &req.check_sum, + )?; + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new( + self.paths.clone(), + next_partition_info, + self.executor.clone(), + ); + Ok(img_manager) + } +} + +impl Default for DiskImageHandler { + fn default() -> Self { + Self { + paths: PreparePath::default(), + executor: RealCommandExecutor {}, + certs_path: CERTS_PATH.to_string(), + } + } +} + +impl DiskImageHandler { + #[cfg(test)] + fn new(paths: PreparePath, executor: T, certs_path: String) -> Self { + Self { + paths, + executor, + certs_path, + } + } + + fn download(&self, req: &UpgradeRequest) -> Result<()> { + let mut resp = self.send_download_request(req)?; + if resp.status() != reqwest::StatusCode::OK { + bail!( + "Failed to download image from {}, status: {}", + req.image_url, + resp.status() + ); + } + debug!( + "Received response body size: {:?}", + resp.content_length().unwrap_or_default() + ); + let need_bytes = resp.content_length().unwrap_or_default() + BUFFER; + + check_disk_size( + i64::try_from(need_bytes) + .with_context(|| "Failed to transform content length from u64 to i64")?, + self.paths + .image_path + .parent() + .unwrap_or_else(|| Path::new(PERSIST_DIR)), + )?; + + let mut out = fs::File::create(&self.paths.image_path)?; + trace!( + "Start to save upgrade image to path {}", + &self.paths.image_path.display() + ); + out.set_permissions(fs::Permissions::from_mode(IMAGE_PERMISSION))?; + let bytes = resp.copy_to(&mut out)?; + info!( + "Download image successfully, upgrade image path: {}, write bytes: {}", + &self.paths.image_path.display(), + bytes + ); + Ok(()) + } + + fn checksum_match(&self, file_path: &str, check_sum: &str) -> Result<()> { + trace!("Start to check checksum"); + let file = fs::read(file_path)?; + let mut hasher = Sha256::new(); + hasher.update(file); + let hash = hasher.finalize(); + // sha256sum -b /persist/update.img + let cal_sum = format!("{:X}", hash); + if cal_sum.to_lowercase() != check_sum.to_lowercase() { + delete_file_or_dir(file_path)?; + bail!("Checksum {} mismatch to {}", cal_sum, check_sum); + } + debug!("Checksum match"); + Ok(()) + } + + fn send_download_request(&self, req: &UpgradeRequest) -> Result { + let client: Client; + + if !req.image_url.starts_with("https://") { + // http request + if !req.flag_safe { + bail!("The upgrade image url is not safe"); + } + info!("Discover http request to: {}", &req.image_url); + client = Client::new(); + } else if req.mtls { + // https mtls request + client = self + .load_ca_client_certs(&req.certs) + .with_context(|| "Failed to load client certificates")?; + info!("Discover https mtls request to: {}", &req.image_url); + } else { + // https request + client = self + .load_ca_certs(&req.certs.ca_cert) + .with_context(|| "Failed to load CA certificates")?; + info!("Discover https request to: {}", &req.image_url); + } + + client + .get(&req.image_url) + .send() + .with_context(|| format!("Failed to fetch from URL: {}", &req.image_url)) + } + + fn load_ca_certs(&self, ca_cert: &str) -> Result { + trace!("Start to load CA certificates"); + self.cert_exist(ca_cert)?; + let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(ca_cert))?)?; + let client = Client::builder().add_root_certificate(ca).build()?; + Ok(client) + } + + fn load_ca_client_certs(&self, certs: &CertsInfo) -> Result { + trace!("Start to load CA and client certificates"); + self.cert_exist(&certs.ca_cert)?; + let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(&certs.ca_cert))?)?; + + self.cert_exist(&certs.client_cert)?; + self.cert_exist(&certs.client_key)?; + let client_cert = std::fs::read(self.get_certs_path(&certs.client_cert))?; + let client_key = std::fs::read(self.get_certs_path(&certs.client_key))?; + let mut client_identity = Vec::new(); + client_identity.extend_from_slice(&client_cert); + client_identity.extend_from_slice(&client_key); + let client_id = reqwest::Identity::from_pem(&client_identity)?; + + let client = Client::builder() + .use_rustls_tls() + .add_root_certificate(ca) + .identity(client_id) + .build()?; + Ok(client) + } + + fn cert_exist(&self, cert_file: &str) -> Result<()> { + if cert_file.is_empty() { + bail!("Please provide the certificate"); + } + if !self.get_certs_path(cert_file).exists() { + bail!("Certificate does not exist: {}", cert_file); + } + Ok(()) + } + + fn get_certs_path(&self, cert: &str) -> PathBuf { + let cert_path = format!("{}{}", self.certs_path, cert); + PathBuf::from(cert_path) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::NamedTempFile; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_certs_path() { + init(); + let handler = DiskImageHandler::::default(); + let certs_path = handler.get_certs_path("ca.pem"); + assert_eq!(certs_path.to_str().unwrap(), "/etc/KubeOS/certs/ca.pem"); + } + + #[test] + fn test_cert_exist() { + init(); + // generate tmp file + let tmp_file = NamedTempFile::new().unwrap(); + let handler = DiskImageHandler::::new( + PreparePath::default(), + RealCommandExecutor {}, + String::new(), + ); + let res = handler.cert_exist(tmp_file.path().to_str().unwrap()); + assert!(res.is_ok()); + + assert!(handler.cert_exist("aaa.pem").is_err()) + } + + #[test] + #[ignore] + fn test_send_download_request() { + init(); + // http + let handler = DiskImageHandler::::default(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "http://localhost:8080/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { + ca_cert: "".to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_ok()); + assert_eq!( + res.unwrap().text().unwrap(), + "This is a test txt file generated by yuhang wei\n" + ); + + // https + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "/home/yuhang/Documents/data/https-nginx/nginx/certs/".to_string(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "https://7.250.142.47:8081/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { + ca_cert: "nginx.crt".to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_ok()); + assert_eq!( + res.unwrap().text().unwrap(), + "This is a test txt file generated by yuhang wei\n" + ); + + // mtls + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "/home/yuhang/Documents/data/cert/".to_string(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "https://7.250.142.47:8082/aaa.txt".to_string(), + flag_safe: true, + mtls: true, + certs: CertsInfo { + ca_cert: "nginx.crt".to_string(), + client_cert: "client.crt".to_string(), + client_key: "client.key".to_string(), + }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_ok()); + assert_eq!( + res.unwrap().text().unwrap(), + "This is a test txt file generated by yuhang wei\n" + ); + } + + #[test] + #[ignore] + fn test_download() { + init(); + let mut handler = DiskImageHandler::::default(); + handler.paths.image_path = + PathBuf::from("/home/yuhang/Documents/KubeOS/KubeOS-Rust/test_download_image"); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "90da5a14e9c06ddb276b06134e90d37098be2830beaa4357205bec7ff1aa1f7c".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "http://localhost:8080/linux-firmware.rpm".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { + ca_cert: "".to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, + }; + let res = handler.download(&req); + assert!(res.is_ok()); + assert_eq!(true, handler.paths.image_path.exists()) + } + + #[test] + #[ignore] + fn test_checksum_match() { + init(); + let mut handler = DiskImageHandler::::default(); + handler.paths.image_path = + PathBuf::from("/home/yuhang/Documents/KubeOS/KubeOS-Rust/test_download_image"); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "90da5a14e9c06ddb276b06134e90d37098be2830beaa4357205bec7ff1aa1f7c".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "http://localhost:8080/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { + ca_cert: "".to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, + }; + assert_eq!(handler.paths.image_path.exists(), true); + handler + .checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum) + .unwrap(); + } + + #[test] + #[ignore] + fn test_load_ca_client_certs() { + init(); + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "/home/yuhang/Documents/data/cert/".to_string(); + let certs = CertsInfo { + ca_cert: "nginx.crt".to_string(), + client_cert: "client.crt".to_string(), + client_key: "client.key".to_string(), + }; + let res = handler.load_ca_client_certs(&certs); + assert!(res.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs new file mode 100644 index 00000000..0708b12d --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -0,0 +1,157 @@ +use anyhow::Result; +use log::{debug, info}; + +use crate::{ + api::{ImageHandler, UpgradeRequest}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES, PERSIST_DIR}, + utils::*, +}; + +pub struct DockerImageHandler { + pub paths: PreparePath, + pub container_name: String, + pub executor: T, +} + +impl ImageHandler for DockerImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + perpare_env(&self.paths, NEED_BYTES, PERSIST_DIR, IMAGE_PERMISSION)?; + self.get_image(req)?; + self.get_rootfs_archive(req)?; + + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new( + self.paths.clone(), + next_partition_info, + self.executor.clone(), + ); + img_manager.create_os_image(IMAGE_PERMISSION) + } +} + +impl Default for DockerImageHandler { + fn default() -> Self { + Self { + paths: PreparePath::default(), + container_name: "kubeos-temp".into(), + executor: RealCommandExecutor {}, + } + } +} + +impl DockerImageHandler { + #[cfg(test)] + fn new(paths: PreparePath, container_name: String, executor: T) -> Self { + Self { + paths, + container_name, + executor, + } + } + + fn get_image(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + is_valid_image_name(image_name)?; + let cli = "docker"; + remove_image_if_exist(cli, image_name, &self.executor)?; + info!("Start pull image {}", image_name); + pull_image(cli, image_name, &self.executor)?; + info!("Start check image digest"); + check_oci_image_digest(cli, image_name, &req.check_sum, &self.executor)?; + Ok(()) + } + + fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + info!("Start get rootfs {}", image_name); + self.check_and_rm_container()?; + debug!("Create container {}", self.container_name); + let container_id = self.executor.run_command_with_output( + "docker", + &["create", "--name", &self.container_name, image_name], + )?; + debug!( + "Copy rootfs from container {} to {}", + container_id, + self.paths.update_path.display() + ); + self.executor.run_command( + "docker", + &[ + "cp", + format!("{}:/{}", container_id, self.paths.rootfs_file).as_str(), + self.paths.update_path.to_str().unwrap(), + ], + )?; + self.check_and_rm_container()?; + Ok(()) + } + + fn check_and_rm_container(&self) -> Result<()> { + let docker_ps_cmd = format!( + "docker ps -a -f=name={} | awk 'NR==2' | awk '{{print $1}}'", + self.container_name + ); + let exist_id = self + .executor + .run_command_with_output("bash", &["-c", &docker_ps_cmd])?; + if !exist_id.is_empty() { + info!( + "Remove container {} {} for cleaning environment", + self.container_name, exist_id + ); + self.executor + .run_command("docker", &["rm", exist_id.as_str()])?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mockall::mock; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_check_and_rm_container() { + init(); + let mut mock_executor = MockCommandExec::new(); + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" + && args.len() == 2 + && args.contains(&"docker ps -a -f=name=test | awk 'NR==2' | awk '{print $1}'") + }) + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111")) + .times(1) + .returning(|_, _| Ok(())); + + let result = DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor) + .check_and_rm_container(); + assert!(result.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/mod.rs b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs index 446d072c..0e06f297 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/mod.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs @@ -10,12 +10,14 @@ * See the Mulan PSL v2 for more details. */ -use super::api; - mod config; mod containerd_image; +mod disk_image; +mod docker_image; mod values; pub use config::*; pub use containerd_image::*; +pub use disk_image::*; +pub use docker_image::*; pub use values::*; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/values.rs b/KubeOS-Rust/manager/src/sys_mgmt/values.rs index 3452b4ad..b107efc3 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/values.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/values.rs @@ -25,6 +25,7 @@ pub const ROOTFS_ARCHIVE: &str = "os.tar"; pub const UPDATE_DIR: &str = "KubeOS-Update"; pub const MOUNT_DIR: &str = "kubeos-update"; pub const OS_IMAGE_NAME: &str = "update.img"; +pub const CERTS_PATH: &str = "/etc/KubeOS/certs/"; pub const DEFAULT_KERNEL_CONFIG_PERM: u32 = 0o644; pub const DEFAULT_GRUB_CFG_PERM: u32 = 0o751; @@ -32,4 +33,4 @@ pub const IMAGE_PERMISSION: u32 = 0o600; pub const ONLY_KEY: usize = 1; pub const KV_PAIR: usize = 2; -pub const NEED_GB_SIZE: i64 = 3; +pub const NEED_BYTES: i64 = 3 * 1024 * 1024 * 1024; diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs index 7bb75a84..8f886bf7 100644 --- a/KubeOS-Rust/manager/src/utils/common.rs +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -17,7 +17,7 @@ use std::{ path::{Path, PathBuf}, }; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, bail, Result}; use log::{debug, info, trace}; use nix::{mount, mount::MntFlags}; @@ -53,12 +53,12 @@ pub fn is_file_exist>(path: P) -> bool { pub fn perpare_env( prepare_path: &PreparePath, - need_gb: i64, + need_bytes: i64, persist_path: &str, permission: u32, ) -> Result<()> { info!("Prepare environment to upgrade"); - check_disk_size(need_gb, persist_path)?; + check_disk_size(need_bytes, persist_path)?; clean_env( &prepare_path.update_path, &prepare_path.mount_path, @@ -71,21 +71,19 @@ pub fn perpare_env( Ok(()) } -pub fn check_disk_size(need_gb: i64, path: &str) -> Result<()> { +pub fn check_disk_size>(need_bytes: i64, path: P) -> Result<()> { trace!("Check if there is enough disk space to upgrade"); - let kb = 1024; - let fs_stat = nix::sys::statfs::statfs(path)?; - let need_disk_size = need_gb * kb * kb * kb; + let fs_stat = nix::sys::statfs::statfs(path.as_ref())?; let available_blocks = i64::try_from(fs_stat.blocks_available())?; let available_space = available_blocks * fs_stat.block_size(); - if available_space < need_disk_size { - return Err(anyhow!("Space is not enough for downloading")); + if available_space < need_bytes { + bail!("Space is not enough for downloading"); } - debug!("There is enough disk space to upgrade"); + info!("There is enough disk space to upgrade"); Ok(()) } -// clean_env will umount the mount path and delete all files in /persist/KubeOS-Update and update.img +// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> where P: AsRef, @@ -94,11 +92,11 @@ where if is_mounted(&mount_path)? { debug!("Umount {}", mount_path.as_ref().display()); if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) { - return Err(anyhow!( + bail!( "Failed to umount {} in clean_env: {}", mount_path.as_ref().display(), errno - )); + ); } } // losetup -D? @@ -234,17 +232,18 @@ mod tests { image_path: PathBuf::from("/tmp/test_prepare_env/update.img"), rootfs_file: "os.tar".to_string(), }; - perpare_env(&paths, 1, "/home", 0o700).unwrap(); + perpare_env(&paths, 1 * 1024 * 1024 * 1024, "/home", 0o700).unwrap(); } #[test] fn test_check_disk_size() { init(); let path = "/home"; - let need_gb = 1; + let gb: i64 = 1 * 1024 * 1024 * 1024; + let need_gb = 1 * gb; let result = check_disk_size(need_gb, path); assert!(result.is_ok()); - let need_gb = 1000; + let need_gb = 10000 * gb; let result = check_disk_size(need_gb, path); assert!(result.is_err()); } @@ -291,14 +290,21 @@ mod tests { let grubenv_path = "/boot/efi/EFI/openEuler/grubenv"; let next_menuentry = "B"; let mut mock = MockCommandExec::new(); - mock.expect_run_command() - .withf(move |name, args| { - name == "grub2-editenv" - && args[0] == grubenv_path - && args[2] == format!("saved_entry={}", next_menuentry).as_str() - }) - .times(1) // Expect it to be called once - .returning(move |_, _| Ok(())); + if get_boot_mode() == "uefi" { + mock.expect_run_command() + .withf(move |name, args| { + name == "grub2-editenv" + && args[0] == grubenv_path + && args[2] == format!("saved_entry={}", next_menuentry).as_str() + }) + .times(1) // Expect it to be called once + .returning(move |_, _| Ok(())); + } else { + mock.expect_run_command() + .withf(move |name, args| name == "grub2-set-default" && args[0] == next_menuentry) + .times(1) // Expect it to be called once + .returning(move |_, _| Ok(())); + } switch_boot_menuentry(&mock, grubenv_path, next_menuentry).unwrap() } diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs index 8c5bfaf7..522e4423 100644 --- a/KubeOS-Rust/manager/src/utils/container_image.rs +++ b/KubeOS-Rust/manager/src/utils/container_image.rs @@ -10,8 +10,8 @@ * See the Mulan PSL v2 for more details. */ -use anyhow::{anyhow, Result}; -use log::{debug, trace}; +use anyhow::{bail, Result}; +use log::{debug, info, trace}; use regex::Regex; use super::executor::CommandExecutor; @@ -20,25 +20,25 @@ pub fn is_valid_image_name(image: &str) -> Result<()> { let pattern = r"^(?P[a-z0-9\-.]+\.[a-z0-9\-]+:?[0-9]*)?/?((?P[a-zA-Z0-9-_]+?)|(?P[a-zA-Z0-9-_]+?)/(?P[a-zA-Z-_]+?))(?P(?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$"; let reg_ex = Regex::new(pattern)?; if !reg_ex.is_match(image) { - return Err(anyhow!("Invalid image name: {}", image)); + bail!("Invalid image name: {}", image); } - trace!("Image name {} is valid", image); + debug!("Image name {} is valid", image); Ok(()) } -pub fn check_oci_image_digest_match( +pub fn check_oci_image_digest( container_runtime: &str, image_name: &str, check_sum: &str, command_executor: &T, ) -> Result<()> { let image_digests = get_oci_image_digest(container_runtime, image_name, command_executor)?; - if image_digests != check_sum { - return Err(anyhow!( + if image_digests.to_lowercase() != check_sum.to_lowercase() { + bail!( "Image digest mismatch, expect {}, got {}", check_sum, image_digests - )); + ); } Ok(()) } @@ -87,17 +87,17 @@ pub fn get_oci_image_digest( trace!("get_oci_image_digest: {}", digest); return Ok(digest.to_string()); } else { - return Err(anyhow!( + bail!( "Failed to get digest from ctr command output: {}", cmd_output - )); + ); } } _ => { - return Err(anyhow!( + bail!( "Container runtime {} cannot be recognized", container_runtime - )); + ); } } @@ -108,16 +108,13 @@ pub fn get_oci_image_digest( let parsed_parts: Vec<&str> = last_part.trim_matches(|c| c == ']').split(':').collect(); // After spliiing by ':', we should get vec like [sha256, digests] if parsed_parts.len() == 2 { - trace!("get_oci_image_digest: {}", parsed_parts[1]); + debug!("get_oci_image_digest: {}", parsed_parts[1]); return Ok(parsed_parts[1].to_string()); // 1 is the index of digests } } } - Err(anyhow!( - "Failed to get digest from command output: {}", - cmd_output - )) + bail!("Failed to get digest from command output: {}", cmd_output) } pub fn pull_image(runtime: &str, image_name: &str, executor: &T) -> Result<()> { @@ -144,10 +141,57 @@ pub fn pull_image(runtime: &str, image_name: &str, executor: executor.run_command("docker", &["pull", image_name])?; } _ => { - return Err(anyhow!( - "Container runtime {} cannot be recognized", - runtime - )); + bail!("Container runtime {} cannot be recognized", runtime); + } + } + Ok(()) +} + +pub fn remove_image_if_exist( + runtime: &str, + image_name: &str, + executor: &T, +) -> Result<()> { + match runtime { + "crictl" => { + if executor + .run_command("crictl", &["inspecti", image_name]) + .is_ok() + { + executor.run_command("crictl", &["rmi", image_name])?; + info!("Remove existing upgrade image: {}", image_name); + } + } + "ctr" => { + let output = executor.run_command_with_output( + "ctr", + &[ + &"-n", + "k8s.io", + "images", + "check", + &format!("name=={}", image_name), + ], + )?; + if !output.is_empty() { + executor.run_command( + "ctr", + &[&"-n", "k8s.io", "images", "rm", image_name, "--sync"], + )?; + info!("Remove existing upgrade image: {}", image_name); + } + } + "docker" => { + if executor + .run_command("docker", &["inspect", image_name]) + .is_ok() + { + executor.run_command("docker", &["rmi", image_name])?; + info!("Remove existing upgrade image: {}", image_name); + } + } + _ => { + bail!("Container runtime {} cannot be recognized", runtime); } } Ok(()) @@ -231,7 +275,7 @@ mod tests { mock.expect_run_command_with_output() .times(1) .returning(|_, _| Ok(command_output.to_string())); - let result = check_oci_image_digest_match(container_runtime, image_name, check_sum, &mock); + let result = check_oci_image_digest(container_runtime, image_name, check_sum, &mock); assert!(result.is_ok()); } diff --git a/KubeOS-Rust/manager/src/utils/executor.rs b/KubeOS-Rust/manager/src/utils/executor.rs index 5c70c8e4..de55a1e8 100644 --- a/KubeOS-Rust/manager/src/utils/executor.rs +++ b/KubeOS-Rust/manager/src/utils/executor.rs @@ -12,8 +12,8 @@ use std::process::Command; -use anyhow::{anyhow, Result}; -use log::trace; +use anyhow::{bail, Result}; +use log::{debug, trace}; pub trait CommandExecutor: Clone { fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; @@ -25,34 +25,36 @@ pub struct RealCommandExecutor {} impl CommandExecutor for RealCommandExecutor { fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()> { + trace!("run_command: {} {:?}", name, args); let output = Command::new(name).args(args).output()?; if !output.status.success() { let error_message = String::from_utf8_lossy(&output.stderr); - return Err(anyhow!( + bail!( "Failed to run command: {} {:?}, stderr: {}", name, args, error_message - )); + ); } - trace!("run_command: {} {:?} done", name, args); + debug!("run_command: {} {:?} done", name, args); Ok(()) } fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result { + trace!("run_command_with_output: {} {:?}", name, args); let output = Command::new(name).args(args).output()?; if !output.status.success() { let error_message = String::from_utf8_lossy(&output.stderr); - return Err(anyhow!( + bail!( "Failed to run command: {} {:?}, stderr: {}", name, args, error_message - )); + ); } let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - trace!("run_command_with_output: {} {:?} done", name, args); - Ok(stdout.trim_end_matches("\n").to_string()) + debug!("run_command_with_output: {} {:?} done", name, args); + Ok(stdout.trim_end_matches('\n').to_string()) } } diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs index 8d59e174..5c0f4818 100644 --- a/KubeOS-Rust/manager/src/utils/partition.rs +++ b/KubeOS-Rust/manager/src/utils/partition.rs @@ -10,7 +10,7 @@ * See the Mulan PSL v2 for more details. */ -use anyhow::{anyhow, Result}; +use anyhow::{bail, Result}; use log::{debug, trace}; use super::executor::CommandExecutor; @@ -38,22 +38,21 @@ pub fn get_partition_info( cur_partition.device = format!("/dev/{}", res[0]).to_string(); cur_partition.fs_type = res[2].to_string(); next_partition.fs_type = res[2].to_string(); - if res[0].contains("2") { + if res[0].contains('2') { + // root directory is mounted on sda2, so sda3 is the next partition cur_partition.menuentry = String::from("A"); next_partition.menuentry = String::from("B"); - next_partition.device = format!("/dev/{}", res[0].replace("2", "3")).to_string(); - } else if res[0].contains("3") { + next_partition.device = format!("/dev/{}", res[0].replace('2', "3")).to_string(); + } else if res[0].contains('3') { + // root directory is mounted on sda3, so sda2 is the next partition cur_partition.menuentry = String::from("B"); next_partition.menuentry = String::from("A"); - next_partition.device = format!("/dev/{}", res[0].replace("3", "2")).to_string(); + next_partition.device = format!("/dev/{}", res[0].replace('3', "2")).to_string(); } } } if cur_partition.device.is_empty() { - return Err(anyhow!( - "Failed to get partition info, lsblk output: {}", - lsblk - )); + bail!("Failed to get partition info, lsblk output: {}", lsblk); } Ok((cur_partition, next_partition)) } diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index a0c48358..474d2397 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -26,6 +26,6 @@ regex = "=1.7.3" chrono = { version = "0.4", default-features = false, features = ["std"] } h2 = "=0.3.16" tokio-retry = "0.3" -reqwest = { version = "=0.11.10", default-features = false, features = [ "json" ] } +reqwest = { version = "=0.11.18", default-features = false, features = [ "json" ] } cli = { version = "0.1.0", path = "../cli" } manager = { version = "0.1.0", path = "../manager" } diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs index 4d954ee8..b099a0e8 100644 --- a/KubeOS-Rust/proxy/src/controller/agentclient.rs +++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs @@ -21,7 +21,8 @@ use cli::{ use agent_call::AgentCallClient; use agent_error::Error; use manager::api::{ - ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest, + CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, + UpgradeRequest, }; use std::collections::HashMap; use std::path::Path; @@ -105,6 +106,15 @@ impl AgentMethod for AgentClient { image_type: upgrade_info.image_type, check_sum: upgrade_info.check_sum, container_image: upgrade_info.container_image, + // TODO: add image_url, flag_safe, mtls, certs + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { + ca_cert: "".to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, }; match agent_call.call_agent( &self.agent_client, @@ -116,14 +126,14 @@ impl AgentMethod for AgentClient { } fn upgrade_method(&self, agent_call: AgentCallClient) -> Result<(), Error> { - match agent_call.call_agent(&self.agent_client, UpgradeMethod::new()) { + match agent_call.call_agent(&self.agent_client, UpgradeMethod::default()) { Ok(_resp) => Ok(()), Err(e) => Err(e), } } fn rollback_method(&self, agent_call: AgentCallClient) -> Result<(), Error> { - match agent_call.call_agent(&self.agent_client, RollbackMethod::new()) { + match agent_call.call_agent(&self.agent_client, RollbackMethod::default()) { Ok(_resp) => Ok(()), Err(e) => Err(e), } -- Gitee From 48cfb40e861b06992b79389a219fd97721d54b24 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 2 Jan 2024 19:23:33 +0800 Subject: [PATCH 07/46] style: format entire KubeOS-Rust project Signed-off-by: Yuhang Wei --- KubeOS-Rust/Cargo.toml | 13 +- KubeOS-Rust/agent/Cargo.toml | 30 +- KubeOS-Rust/agent/src/function.rs | 6 +- KubeOS-Rust/agent/src/main.rs | 17 +- KubeOS-Rust/agent/src/rpc/agent.rs | 3 +- KubeOS-Rust/agent/src/rpc/agent_impl.rs | 74 ++--- KubeOS-Rust/cli/Cargo.toml | 10 +- KubeOS-Rust/cli/src/client.rs | 16 +- KubeOS-Rust/cli/src/method/cleanup.rs | 2 +- KubeOS-Rust/cli/src/method/configure.rs | 2 +- KubeOS-Rust/cli/src/method/prepare_upgrade.rs | 2 +- KubeOS-Rust/cli/src/method/request.rs | 20 +- KubeOS-Rust/cli/src/method/rollback.rs | 2 +- KubeOS-Rust/cli/src/method/upgrade.rs | 2 +- KubeOS-Rust/manager/Cargo.toml | 18 +- KubeOS-Rust/manager/src/sys_mgmt/config.rs | 272 ++++-------------- .../manager/src/sys_mgmt/containerd_image.rs | 141 +++------ .../manager/src/sys_mgmt/disk_image.rs | 119 ++------ .../manager/src/sys_mgmt/docker_image.rs | 54 +--- KubeOS-Rust/manager/src/utils/common.rs | 71 ++--- .../manager/src/utils/container_image.rs | 135 +++------ KubeOS-Rust/manager/src/utils/executor.rs | 25 +- .../manager/src/utils/image_manager.rs | 71 ++--- KubeOS-Rust/manager/src/utils/partition.rs | 26 +- KubeOS-Rust/proxy/Cargo.toml | 36 +-- .../proxy/src/controller/agentclient.rs | 72 ++--- KubeOS-Rust/proxy/src/controller/apiclient.rs | 53 +--- .../proxy/src/controller/controller.rs | 223 +++++--------- KubeOS-Rust/proxy/src/controller/crd.rs | 9 +- KubeOS-Rust/proxy/src/controller/drain.rs | 244 +++++----------- KubeOS-Rust/proxy/src/controller/utils.rs | 65 +++-- KubeOS-Rust/proxy/src/controller/values.rs | 8 +- KubeOS-Rust/proxy/src/main.rs | 15 +- KubeOS-Rust/rustfmt.toml | 11 + 34 files changed, 532 insertions(+), 1335 deletions(-) create mode 100644 KubeOS-Rust/rustfmt.toml diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml index c1299f2d..135028f2 100644 --- a/KubeOS-Rust/Cargo.toml +++ b/KubeOS-Rust/Cargo.toml @@ -1,16 +1,11 @@ [workspace] -members = [ - "manager", - "agent", - "cli", - "proxy", -] +members = ["agent", "cli", "manager", "proxy"] resolver = "2" [profile.release] -opt-level = 's' debug = false -rpath = false debug-assertions = false -overflow-checks = false lto = true +opt-level = 's' +overflow-checks = false +rpath = false diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml index 15defa6f..6db4df45 100644 --- a/KubeOS-Rust/agent/Cargo.toml +++ b/KubeOS-Rust/agent/Cargo.toml @@ -1,20 +1,20 @@ [package] -name = "os-agent" -version = "0.1.0" -edition = "2021" -description = "KubeOS os-agent" -license = "MulanPSL-2.0" +description = "KubeOS os-agent" +edition = "2021" +license = "MulanPSL-2.0" +name = "os-agent" +version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -manager = { package = "manager", path = "../manager" } -jsonrpc-core = { version = "18.0" } -jsonrpc-derive = { version = "18.0" } +anyhow = { version = "1.0" } +env_logger = { version = "0.9" } +jsonrpc-core = { version = "18.0" } +jsonrpc-derive = { version = "18.0" } jsonrpc-ipc-server = { version = "18.0" } -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0" } -log = { version = "= 0.4.15" } -anyhow = { version = "1.0" } -env_logger = { version = "0.9" } -lazy_static = { version = "1.4" } -nix = { version = "0.26.2" } +lazy_static = { version = "1.4" } +log = { version = "= 0.4.15" } +manager = { package = "manager", path = "../manager" } +nix = { version = "0.26.2" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } diff --git a/KubeOS-Rust/agent/src/function.rs b/KubeOS-Rust/agent/src/function.rs index 8fb8a4e3..89775dce 100644 --- a/KubeOS-Rust/agent/src/function.rs +++ b/KubeOS-Rust/agent/src/function.rs @@ -26,11 +26,7 @@ impl RpcFunction { { (f)().map_err(|e| { error!("{:?}", e); - Error { - code: ErrorCode::ServerError(RPC_OP_ERROR), - message: format!("{:?}", e), - data: None, - } + Error { code: ErrorCode::ServerError(RPC_OP_ERROR), message: format!("{:?}", e), data: None } }) } } diff --git a/KubeOS-Rust/agent/src/main.rs b/KubeOS-Rust/agent/src/main.rs index 2201c202..cd95ef07 100644 --- a/KubeOS-Rust/agent/src/main.rs +++ b/KubeOS-Rust/agent/src/main.rs @@ -35,10 +35,7 @@ fn start_and_run(sock_path: &str) { // Create directory for socket if it doesn't exist if let Some(dir_path) = socket_path.parent() { if !dir_path.exists() { - DirBuilder::new() - .mode(0o750) - .create(dir_path) - .expect("Couldn't create directory for socket"); + DirBuilder::new().mode(0o750).create(dir_path).expect("Couldn't create directory for socket"); } } @@ -51,8 +48,7 @@ fn start_and_run(sock_path: &str) { let server = builder.start(sock_path).expect("Couldn't open socket"); let gid = nix::unistd::getgid(); - nix::unistd::chown(socket_path, Some(nix::unistd::ROOT), Some(gid)) - .expect("Couldn't set socket group"); + nix::unistd::chown(socket_path, Some(nix::unistd::ROOT), Some(gid)).expect("Couldn't set socket group"); // Set socket permissions to 0640 let socket_permissions = Permissions::from_mode(0o640); @@ -63,13 +59,8 @@ fn start_and_run(sock_path: &str) { } fn main() { - Builder::from_env(Env::default().default_filter_or("info")) - .target(Target::Stdout) - .init(); + Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init(); - info!( - "os-agent version is: {}", - CARGO_PKG_VERSION.unwrap_or("NOT FOUND") - ); + info!("os-agent version is: {}", CARGO_PKG_VERSION.unwrap_or("NOT FOUND")); start_and_run(SOCK_PATH); } diff --git a/KubeOS-Rust/agent/src/rpc/agent.rs b/KubeOS-Rust/agent/src/rpc/agent.rs index 97eb4566..13775afb 100644 --- a/KubeOS-Rust/agent/src/rpc/agent.rs +++ b/KubeOS-Rust/agent/src/rpc/agent.rs @@ -10,9 +10,10 @@ * See the Mulan PSL v2 for more details. */ -use super::function::{rpc, RpcResult}; use manager::api::{ConfigureRequest, Response, UpgradeRequest}; +use super::function::{rpc, RpcResult}; + #[rpc(server)] pub trait Agent { #[rpc(name = "prepare_upgrade")] diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs index b84b1941..7101d0db 100644 --- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -14,22 +14,17 @@ use std::{sync::Mutex, thread, time::Duration}; use anyhow::{bail, Result}; use log::{debug, error, info}; +use manager::{ + api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, + sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH}, + utils::{clean_env, get_partition_info, switch_boot_menuentry, PreparePath, RealCommandExecutor}, +}; use nix::{sys::reboot::RebootMode, unistd::sync}; use super::{ agent::Agent, function::{RpcFunction, RpcResult}, }; -use manager::{ - api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, - sys_mgmt::{ - CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, - DEFAULT_GRUBENV_PATH, - }, - utils::{ - clean_env, get_partition_info, switch_boot_menuentry, PreparePath, RealCommandExecutor, - }, -}; pub struct AgentImpl { mutex: Mutex<()>, @@ -60,10 +55,7 @@ impl Agent for AgentImpl { impl Default for AgentImpl { fn default() -> Self { - Self { - mutex: Mutex::new(()), - disable_reboot: false, - } + Self { mutex: Mutex::new(()), disable_reboot: false } } } @@ -81,15 +73,10 @@ impl AgentImpl { }; let image_manager = handler.download_image(&req)?; - info!( - "Ready to install image: {:?}", - image_manager.paths.image_path.display() - ); + info!("Ready to install image: {:?}", image_manager.paths.image_path.display()); image_manager.install()?; - Ok(Response { - status: AgentStatus::UpgradeReady, - }) + Ok(Response { status: AgentStatus::UpgradeReady }) } pub fn upgrade_impl(&self) -> Result { @@ -102,14 +89,9 @@ impl AgentImpl { let device = next_partition_info.device.as_str(); let menuentry = next_partition_info.menuentry.as_str(); switch_boot_menuentry(&command_executor, DEFAULT_GRUBENV_PATH, menuentry)?; - info!( - "Switch to boot partition: {}, device: {}", - menuentry, device - ); + info!("Switch to boot partition: {}, device: {}", menuentry, device); self.reboot()?; - Ok(Response { - status: AgentStatus::Upgraded, - }) + Ok(Response { status: AgentStatus::Upgraded }) } pub fn cleanup_impl(&self) -> Result { @@ -117,9 +99,7 @@ impl AgentImpl { info!("Start to cleanup"); let paths = PreparePath::default(); clean_env(paths.update_path, paths.mount_path, paths.image_path)?; - Ok(Response { - status: AgentStatus::CleanedUp, - }) + Ok(Response { status: AgentStatus::CleanedUp }) } pub fn configure_impl(&self, mut req: ConfigureRequest) -> Result { @@ -137,9 +117,7 @@ impl AgentImpl { bail!("Unknown configuration type: \"{}\"", config_type); } } - Ok(Response { - status: AgentStatus::Configured, - }) + Ok(Response { status: AgentStatus::Configured }) } pub fn rollback_impl(&self) -> Result { @@ -152,14 +130,9 @@ impl AgentImpl { manager::sys_mgmt::DEFAULT_GRUBENV_PATH, &next_partition_info.menuentry, )?; - info!( - "Switch to boot partition: {}, device: {}", - next_partition_info.menuentry, next_partition_info.device - ); + info!("Switch to boot partition: {}, device: {}", next_partition_info.menuentry, next_partition_info.device); self.reboot()?; - Ok(Response { - status: AgentStatus::Rollbacked, - }) + Ok(Response { status: AgentStatus::Rollbacked }) } pub fn reboot(&self) -> Result<()> { @@ -176,10 +149,12 @@ impl AgentImpl { #[cfg(test)] mod test { - use super::*; - use manager::api::{CertsInfo, Sysconfig}; use std::collections::HashMap; + use manager::api::{CertsInfo, Sysconfig}; + + use super::*; + #[test] fn configure_impl_tests() { let agent = AgentImpl::default(); @@ -191,12 +166,7 @@ mod test { }], }; let res = agent.configure_impl(req).unwrap(); - assert_eq!( - res, - Response { - status: AgentStatus::Configured, - } - ); + assert_eq!(res, Response { status: AgentStatus::Configured }); let req = ConfigureRequest { configs: vec![Sysconfig { @@ -225,11 +195,7 @@ mod test { image_url: "".to_string(), flag_safe: false, mtls: false, - certs: CertsInfo { - ca_cert: "".to_string(), - client_cert: "".to_string(), - client_key: "".to_string(), - }, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; let res = agent.prepare_upgrade_impl(req); assert!(res.is_err()); diff --git a/KubeOS-Rust/cli/Cargo.toml b/KubeOS-Rust/cli/Cargo.toml index 18ea908a..1c46db36 100644 --- a/KubeOS-Rust/cli/Cargo.toml +++ b/KubeOS-Rust/cli/Cargo.toml @@ -1,15 +1,15 @@ [package] -name = "cli" -version = "0.1.0" -edition = "2021" description = "KubeOS os-agent client" +edition = "2021" license = "MulanPSL-2.0" +name = "cli" +version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -kubeos-manager = { package = "manager", path = "../manager" } +anyhow = { version = "1.0" } jsonrpc = { version = "0.13", features = ["simple_uds"] } +kubeos-manager = { package = "manager", path = "../manager" } log = { version = "0.4" } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } -anyhow = { version = "1.0" } diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs index d8f68d07..ce45cdd7 100644 --- a/KubeOS-Rust/cli/src/client.rs +++ b/KubeOS-Rust/cli/src/client.rs @@ -13,8 +13,7 @@ use std::path::Path; use jsonrpc::{ - simple_uds::UdsTransport, Client as JsonRPCClient, Request as JsonRPCRequest, - Response as JsonRPCResponse, + simple_uds::UdsTransport, Client as JsonRPCClient, Request as JsonRPCRequest, Response as JsonRPCResponse, }; use serde_json::value::RawValue; @@ -28,16 +27,10 @@ impl<'a> Request<'a> {} impl Client { pub fn new>(socket_path: P) -> Self { - Client { - json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)), - } + Client { json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)) } } - pub fn build_request<'a>( - &self, - command: &'a str, - params: &'a Vec>, - ) -> Request<'a> { + pub fn build_request<'a>(&self, command: &'a str, params: &'a Vec>) -> Request<'a> { let json_rpc_request = self.json_rpc_client.build_request(command, params); let request = Request(json_rpc_request); request @@ -50,9 +43,10 @@ impl Client { #[cfg(test)] mod test { + use kubeos_manager::api; + use super::*; use crate::method::{callable_method::RpcMethod, configure::ConfigureMethod}; - use kubeos_manager::api; #[test] #[ignore] diff --git a/KubeOS-Rust/cli/src/method/cleanup.rs b/KubeOS-Rust/cli/src/method/cleanup.rs index f072a107..d1d7dbe2 100644 --- a/KubeOS-Rust/cli/src/method/cleanup.rs +++ b/KubeOS-Rust/cli/src/method/cleanup.rs @@ -10,10 +10,10 @@ * See the Mulan PSL v2 for more details. */ +use kubeos_manager::api; use serde_json::value::RawValue; use crate::method::callable_method::RpcMethod; -use kubeos_manager::api; #[derive(Default)] pub struct CleanupMethod {} diff --git a/KubeOS-Rust/cli/src/method/configure.rs b/KubeOS-Rust/cli/src/method/configure.rs index ddfeb05f..d1371068 100644 --- a/KubeOS-Rust/cli/src/method/configure.rs +++ b/KubeOS-Rust/cli/src/method/configure.rs @@ -10,10 +10,10 @@ * See the Mulan PSL v2 for more details. */ +use kubeos_manager::api; use serde_json::value::{to_raw_value, RawValue}; use crate::method::callable_method::RpcMethod; -use kubeos_manager::api; pub struct ConfigureMethod { req: api::ConfigureRequest, diff --git a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs index dd3157df..91dae793 100644 --- a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs +++ b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs @@ -10,10 +10,10 @@ * See the Mulan PSL v2 for more details. */ +use kubeos_manager::api; use serde_json::value::{to_raw_value, RawValue}; use crate::method::callable_method::RpcMethod; -use kubeos_manager::api; pub struct PrepareUpgradeMethod { req: api::UpgradeRequest, diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs index d4d2a97a..2dc1ffba 100644 --- a/KubeOS-Rust/cli/src/method/request.rs +++ b/KubeOS-Rust/cli/src/method/request.rs @@ -17,11 +17,7 @@ use serde_json::value::RawValue; use crate::client::Client; -pub fn request( - client: &Client, - command: &str, - params: Vec>, -) -> Result { +pub fn request(client: &Client, command: &str, params: Vec>) -> Result { let request = client.build_request(command, ¶ms); let response = client.send_request(request).map_err(parse_error); debug!("{:#?}", response); @@ -33,26 +29,24 @@ pub fn parse_error(error: Error) -> anyhow::Error { Error::Transport(e) => { anyhow!( "Cannot connect to KubeOS os-agent unix socket, {}", - e.source() - .map(|e| e.to_string()) - .unwrap_or_else(|| "Connection timeout".to_string()) + e.source().map(|e| e.to_string()).unwrap_or_else(|| "Connection timeout".to_string()) ) - } + }, Error::Json(e) => { debug!("Json parse error: {:?}", e); anyhow!("Failed to parse response") - } + }, Error::Rpc(ref e) => match e.message == "Method not found" { true => { anyhow!("Method is unimplemented") - } + }, false => { anyhow!("{}", e.message) - } + }, }, _ => { debug!("{:?}", error); anyhow!("Response is invalid") - } + }, } } diff --git a/KubeOS-Rust/cli/src/method/rollback.rs b/KubeOS-Rust/cli/src/method/rollback.rs index 0f30a827..55aa7511 100644 --- a/KubeOS-Rust/cli/src/method/rollback.rs +++ b/KubeOS-Rust/cli/src/method/rollback.rs @@ -10,10 +10,10 @@ * See the Mulan PSL v2 for more details. */ +use kubeos_manager::api; use serde_json::value::RawValue; use crate::method::callable_method::RpcMethod; -use kubeos_manager::api; #[derive(Default)] pub struct RollbackMethod {} diff --git a/KubeOS-Rust/cli/src/method/upgrade.rs b/KubeOS-Rust/cli/src/method/upgrade.rs index ef773bce..a9692ca1 100644 --- a/KubeOS-Rust/cli/src/method/upgrade.rs +++ b/KubeOS-Rust/cli/src/method/upgrade.rs @@ -10,10 +10,10 @@ * See the Mulan PSL v2 for more details. */ +use kubeos_manager::api; use serde_json::value::RawValue; use crate::method::callable_method::RpcMethod; -use kubeos_manager::api; #[derive(Default)] pub struct UpgradeMethod {} diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml index a332c9e0..29ec2f46 100644 --- a/KubeOS-Rust/manager/Cargo.toml +++ b/KubeOS-Rust/manager/Cargo.toml @@ -1,24 +1,24 @@ [package] -name = "manager" -version = "0.1.0" -edition = "2021" description = "KubeOS os-agent manager" +edition = "2021" license = "MulanPSL-2.0" +name = "manager" +version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dev-dependencies] -tempfile = "3.6.0" mockall = { version = "=0.11.3" } predicates = "=2.0.1" +tempfile = "3.6.0" [dependencies] -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0" } -log = { version = "0.4" } anyhow = { version = "1.0" } env_logger = { version = "0.9" } lazy_static = { version = "1.4" } -reqwest = { version = "=0.11.18", features = ["rustls-tls", "blocking"] } -regex = { version = "1.7.3" } +log = { version = "0.4" } nix = { version = "0.26.2" } +regex = { version = "1.7.3" } +reqwest = { version = "=0.11.18", features = ["blocking", "rustls-tls"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } sha2 = { version = "0.10.8" } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs index 2e4402d4..cb5fad1f 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -39,17 +39,13 @@ lazy_static! { ); config_map.insert( values::GRUB_CMDLINE_CURRENT.to_string(), - Box::new(GrubCmdline { - grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), - is_cur_partition: true, - }) as Box, + Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: true }) + as Box, ); config_map.insert( values::GRUB_CMDLINE_NEXT.to_string(), - Box::new(GrubCmdline { - grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), - is_cur_partition: false, - }) as Box, + Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: false }) + as Box, ); config_map }; @@ -76,9 +72,8 @@ impl Configuration for KernelSysctl { if key_info.operation == "delete" { warn!("Failed to delete kernel.sysctl config with key \"{}\"", key); } else if !key_info.value.is_empty() && key_info.operation.is_empty() { - fs::write(&proc_path, format!("{}\n", &key_info.value).as_bytes()).with_context( - || format!("Failed to write kernel.sysctl with key: \"{}\"", key), - )?; + fs::write(&proc_path, format!("{}\n", &key_info.value).as_bytes()) + .with_context(|| format!("Failed to write kernel.sysctl with key: \"{}\"", key))?; info!("Configured kernel.sysctl {}={}", key, key_info.value); } else { warn!( @@ -93,9 +88,7 @@ impl Configuration for KernelSysctl { impl KernelSysctl { fn new(proc_path: &str) -> Self { - Self { - proc_path: String::from(proc_path), - } + Self { proc_path: String::from(proc_path) } } fn get_proc_path(&self, key: &str) -> PathBuf { @@ -130,10 +123,7 @@ fn create_config_file(config_path: &str) -> Result<()> { Ok(()) } -fn get_and_set_configs( - expect_configs: &mut HashMap, - config_path: &str, -) -> Result> { +fn get_and_set_configs(expect_configs: &mut HashMap, config_path: &str) -> Result> { let f = File::open(config_path)?; let mut configs_write = Vec::new(); for line in io::BufReader::new(f).lines() { @@ -150,9 +140,7 @@ fn get_and_set_configs( } let new_key_info = expect_configs.get(config_kv[0]); let new_config = match new_key_info { - Some(new_key_info) if new_key_info.operation == "delete" => { - handle_delete_key(&config_kv, new_key_info) - } + Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config_kv, new_key_info), Some(new_key_info) => handle_update_key(&config_kv, new_key_info), None => config_kv.join("="), }; @@ -174,11 +162,8 @@ fn write_configs_to_file(config_path: &str, configs: &Vec) -> Result<()> } writeln!(w, "{}", line.as_str())?; } - w.flush() - .with_context(|| format!("Failed to flush file {}", config_path))?; - w.get_mut() - .sync_all() - .with_context(|| "Failed to sync".to_string())?; + w.flush().with_context(|| format!("Failed to flush file {}", config_path))?; + w.get_mut().sync_all().with_context(|| "Failed to sync".to_string())?; debug!("Write configuration to file \"{}\" success", config_path); Ok(()) } @@ -189,10 +174,7 @@ fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String info!("Delete configuration key: \"{}\"", key); return String::from(""); } else if config_kv.len() == 1 && !new_config_info.value.is_empty() { - warn!( - "Failed to delete key \"{}\" with inconsistent values \"nil\" and \"{}\"", - key, new_config_info.value - ); + warn!("Failed to delete key \"{}\" with inconsistent values \"nil\" and \"{}\"", key, new_config_info.value); return key.to_string(); } let old_value = config_kv[1]; @@ -231,10 +213,7 @@ fn handle_update_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String format!("{}={}", key, new_value) } -fn handle_add_key( - expect_configs: &HashMap, - is_only_key_valid: bool, -) -> Vec { +fn handle_add_key(expect_configs: &HashMap, is_only_key_valid: bool) -> Vec { let mut configs_write = Vec::new(); for (key, config_info) in expect_configs.iter() { if config_info.operation == "delete" { @@ -242,10 +221,7 @@ fn handle_add_key( continue; } if key.is_empty() || key.contains('=') { - warn!( - "Failed to add \"null\" key or key containing \"=\", key: \"{}\"", - key - ); + warn!("Failed to add \"null\" key or key containing \"=\", key: \"{}\"", key); continue; } if !config_info.operation.is_empty() { @@ -278,15 +254,9 @@ impl Configuration for GrubCmdline { if !is_file_exist(&self.grub_path) { bail!("Failed to find grub.cfg file"); } - let config_partition = if cfg!(test) { - self.is_cur_partition - } else { - self.get_config_partition(RealCommandExecutor {})? - }; - debug!( - "Config_partition: {} (false means partition A, true means partition B)", - config_partition - ); + let config_partition = + if cfg!(test) { self.is_cur_partition } else { self.get_config_partition(RealCommandExecutor {})? }; + debug!("Config_partition: {} (false means partition A, true means partition B)", config_partition); let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition)?; write_configs_to_file(&self.grub_path, &configs)?; Ok(()) @@ -329,10 +299,7 @@ fn get_and_set_grubcfg( } fn modify_boot_cfg(expect_configs: &mut HashMap, line: &String) -> Result { - trace!( - "Match partition that need to be configured, entering modify_boot_cfg, linux line: {}", - line - ); + trace!("Match partition that need to be configured, entering modify_boot_cfg, linux line: {}", line); let mut new_configs = vec![" ".to_string()]; let olg_configs: Vec<&str> = line.split(' ').collect(); for old_config in olg_configs { @@ -346,9 +313,7 @@ fn modify_boot_cfg(expect_configs: &mut HashMap, line: &String) } let new_key_info = expect_configs.get(config[0]); let new_config = match new_key_info { - Some(new_key_info) if new_key_info.operation == "delete" => { - handle_delete_key(&config, new_key_info) - } + Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config, new_key_info), Some(new_key_info) => handle_update_key(&config, new_key_info), None => config.join("="), }; @@ -364,14 +329,14 @@ fn modify_boot_cfg(expect_configs: &mut HashMap, line: &String) #[cfg(test)] mod tests { - use super::*; - use crate::sys_mgmt::{ - GRUB_CMDLINE_CURRENT, GRUB_CMDLINE_NEXT, KERNEL_SYSCTL, KERNEL_SYSCTL_PERSIST, - }; - use mockall::{mock, predicate::*}; use std::fs; + + use mockall::{mock, predicate::*}; use tempfile::{NamedTempFile, TempDir}; + use super::*; + use crate::sys_mgmt::{GRUB_CMDLINE_CURRENT, GRUB_CMDLINE_NEXT, KERNEL_SYSCTL, KERNEL_SYSCTL_PERSIST}; + // Mock the CommandExecutor trait mock! { pub CommandExec{} @@ -395,19 +360,12 @@ mod tests { #[test] fn test_get_config_partition() { init(); - let mut grub_cmdline = GrubCmdline { - grub_path: String::from(""), - is_cur_partition: true, - }; + let mut grub_cmdline = GrubCmdline { grub_path: String::from(""), is_cur_partition: true }; let mut executor = MockCommandExec::new(); // the output shows that current root menuentry is A - let command_output1 = - "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; - executor - .expect_run_command_with_output() - .times(1) - .returning(|_, _| Ok(command_output1.to_string())); + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); let result = grub_cmdline.get_config_partition(executor).unwrap(); // it should return false because the current root menuentry is A and we want to configure current partition @@ -416,12 +374,8 @@ mod tests { let mut executor = MockCommandExec::new(); // the output shows that current root menuentry is A - let command_output1 = - "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; - executor - .expect_run_command_with_output() - .times(1) - .returning(|_, _| Ok(command_output1.to_string())); + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); grub_cmdline.is_cur_partition = false; let result = grub_cmdline.get_config_partition(executor).unwrap(); // it should return true because the current root menuentry is A and we want to configure next partition @@ -436,52 +390,18 @@ mod tests { let kernel_sysctl = KernelSysctl::new(tmp_dir.path().to_str().unwrap()); let config_detail = HashMap::from([ - ( - "a".to_string(), - KeyInfo { - value: "1".to_string(), - operation: "".to_string(), - }, - ), - ( - "b".to_string(), - KeyInfo { - value: "2".to_string(), - operation: "delete".to_string(), - }, - ), - ( - "c".to_string(), - KeyInfo { - value: "3".to_string(), - operation: "add".to_string(), - }, - ), - ( - "d".to_string(), - KeyInfo { - value: "".to_string(), - operation: "".to_string(), - }, - ), - ( - "e".to_string(), - KeyInfo { - value: "".to_string(), - operation: "delete".to_string(), - }, - ), + ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }), + ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }), + ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), ]); - let mut config = Sysconfig { - model: KERNEL_SYSCTL.to_string(), - config_path: String::from(""), - contents: config_detail, - }; + let mut config = + Sysconfig { model: KERNEL_SYSCTL.to_string(), config_path: String::from(""), contents: config_detail }; kernel_sysctl.set_config(&mut config).unwrap(); - let result = - fs::read_to_string(format!("{}{}", tmp_dir.path().to_str().unwrap(), "a")).unwrap(); + let result = fs::read_to_string(format!("{}{}", tmp_dir.path().to_str().unwrap(), "a")).unwrap(); assert_eq!(result, "1\n"); } @@ -495,27 +415,9 @@ mod tests { writeln!(tmp_file, "a=0").unwrap(); let kernel_sysctl_persist = KernelSysctlPersist {}; let config_detail = HashMap::from([ - ( - "a".to_string(), - KeyInfo { - value: "1".to_string(), - operation: "".to_string(), - }, - ), - ( - "b".to_string(), - KeyInfo { - value: "2".to_string(), - operation: "delete".to_string(), - }, - ), - ( - "c".to_string(), - KeyInfo { - value: "3".to_string(), - operation: "add".to_string(), - }, - ), + ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }), + ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }), ]); let mut config = Sysconfig { model: KERNEL_SYSCTL_PERSIST.to_string(), @@ -566,10 +468,8 @@ mod tests { fn test_grub_cmdline() { init(); let mut tmp_file = NamedTempFile::new().unwrap(); - let mut grub_cmdline = GrubCmdline { - grub_path: tmp_file.path().to_str().unwrap().to_string(), - is_cur_partition: true, - }; + let mut grub_cmdline = + GrubCmdline { grub_path: tmp_file.path().to_str().unwrap().to_string(), is_cur_partition: true }; let grub_cfg = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' { load_video set gfxpayload=keep @@ -593,69 +493,15 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri }"; writeln!(tmp_file, "{}", grub_cfg).unwrap(); let config_first_part = HashMap::from([ - ( - "debug".to_string(), - KeyInfo { - value: "".to_string(), - operation: "".to_string(), - }, - ), - ( - "quiet".to_string(), - KeyInfo { - value: "".to_string(), - operation: "delete".to_string(), - }, - ), - ( - "panic".to_string(), - KeyInfo { - value: "5".to_string(), - operation: "".to_string(), - }, - ), - ( - "nomodeset".to_string(), - KeyInfo { - value: "".to_string(), - operation: "update".to_string(), - }, - ), - ( - "oops".to_string(), - KeyInfo { - value: "".to_string(), - operation: "".to_string(), - }, - ), - ( - "".to_string(), - KeyInfo { - value: "test".to_string(), - operation: "".to_string(), - }, - ), - ( - "selinux".to_string(), - KeyInfo { - value: "1".to_string(), - operation: "delete".to_string(), - }, - ), - ( - "acpi".to_string(), - KeyInfo { - value: "off".to_string(), - operation: "delete".to_string(), - }, - ), - ( - "ro".to_string(), - KeyInfo { - value: "1".to_string(), - operation: "".to_string(), - }, - ), + ("debug".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("quiet".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }), + ("nomodeset".to_string(), KeyInfo { value: "".to_string(), operation: "update".to_string() }), + ("oops".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("".to_string(), KeyInfo { value: "test".to_string(), operation: "".to_string() }), + ("selinux".to_string(), KeyInfo { value: "1".to_string(), operation: "delete".to_string() }), + ("acpi".to_string(), KeyInfo { value: "off".to_string(), operation: "delete".to_string() }), + ("ro".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), ]); let mut config = Sysconfig { model: GRUB_CMDLINE_CURRENT.to_string(), @@ -665,20 +511,8 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri grub_cmdline.set_config(&mut config).unwrap(); grub_cmdline.is_cur_partition = false; let config_second = HashMap::from([ - ( - "pci".to_string(), - KeyInfo { - value: "nomis".to_string(), - operation: "".to_string(), - }, - ), - ( - "panic".to_string(), - KeyInfo { - value: "5".to_string(), - operation: "".to_string(), - }, - ), + ("pci".to_string(), KeyInfo { value: "nomis".to_string(), operation: "".to_string() }), + ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }), ]); config.contents = config_second; config.model = GRUB_CMDLINE_NEXT.to_string(); diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs index e38f22f3..6ed3623a 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -35,21 +35,14 @@ impl ImageHandler for CtrImageHandler { self.get_rootfs_archive(req, IMAGE_PERMISSION)?; let (_, next_partition_info) = get_partition_info(&self.executor)?; - let img_manager = UpgradeImageManager::new( - self.paths.clone(), - next_partition_info, - self.executor.clone(), - ); + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); img_manager.create_os_image(IMAGE_PERMISSION) } } impl Default for CtrImageHandler { fn default() -> Self { - Self { - paths: PreparePath::default(), - executor: RealCommandExecutor {}, - } + Self { paths: PreparePath::default(), executor: RealCommandExecutor {} } } } @@ -62,11 +55,8 @@ impl CtrImageHandler { fn get_image(&self, req: &UpgradeRequest) -> Result<()> { let image_name = &req.container_image; is_valid_image_name(image_name)?; - let cli: String = if is_command_available("crictl", &self.executor) { - "crictl".to_string() - } else { - "ctr".to_string() - }; + let cli: String = + if is_command_available("crictl", &self.executor) { "crictl".to_string() } else { "ctr".to_string() }; remove_image_if_exist(&cli, image_name, &self.executor)?; info!("Start pulling image {}", image_name); pull_image(&cli, image_name, &self.executor)?; @@ -77,70 +67,35 @@ impl CtrImageHandler { fn get_rootfs_archive(&self, req: &UpgradeRequest, permission: u32) -> Result<()> { let image_name = &req.container_image; - let mount_path = &self.paths.mount_path.to_str().ok_or_else(|| { - anyhow!( - "Failed to get mount path: {}", - self.paths.mount_path.display() - ) - })?; + let mount_path = &self + .paths + .mount_path + .to_str() + .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?; info!("Start get rootfs {}", image_name); self.check_and_unmount(mount_path)?; - self.executor.run_command( - "ctr", - &[ - "-n", - DEFAULT_NAMESPACE, - "images", - "mount", - "--rw", - image_name, - mount_path, - ], - )?; + self.executor + .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?; // copy os.tar from mount_path to its partent dir - self.copy_file( - self.paths.mount_path.join(&self.paths.rootfs_file), - &self.paths.tar_path, - permission, - )?; + self.copy_file(self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission)?; self.check_and_unmount(mount_path)?; Ok(()) } fn check_and_unmount(&self, mount_path: &str) -> Result<()> { - let ctr_snapshot_cmd = format!( - "ctr -n={} snapshots ls | grep {} | awk '{{print $1}}'", - DEFAULT_NAMESPACE, mount_path - ); - let exist_snapshot = self - .executor - .run_command_with_output("bash", &["-c", &ctr_snapshot_cmd])?; + let ctr_snapshot_cmd = + format!("ctr -n={} snapshots ls | grep {} | awk '{{print $1}}'", DEFAULT_NAMESPACE, mount_path); + let exist_snapshot = self.executor.run_command_with_output("bash", &["-c", &ctr_snapshot_cmd])?; if !exist_snapshot.is_empty() { - self.executor.run_command( - "ctr", - &["-n", DEFAULT_NAMESPACE, "images", "unmount", mount_path], - )?; - self.executor.run_command( - "ctr", - &["-n", DEFAULT_NAMESPACE, "snapshots", "remove", mount_path], - )?; + self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "unmount", mount_path])?; + self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "snapshots", "remove", mount_path])?; } Ok(()) } - fn copy_file, Q: AsRef>( - &self, - src: P, - dst: Q, - permission: u32, - ) -> Result<()> { + fn copy_file, Q: AsRef>(&self, src: P, dst: Q, permission: u32) -> Result<()> { let copied_bytes = fs::copy(src.as_ref(), dst.as_ref())?; - debug!( - "Copy {} to {}, total bytes: {}", - src.as_ref().display(), - dst.as_ref().display(), - copied_bytes - ); + debug!("Copy {} to {}, total bytes: {}", src.as_ref().display(), dst.as_ref().display(), copied_bytes); fs::set_permissions(dst, fs::Permissions::from_mode(permission))?; Ok(()) } @@ -148,14 +103,17 @@ impl CtrImageHandler { #[cfg(test)] mod tests { - use super::*; - use crate::api::CertsInfo; + use std::{ + io::Write, + path::{Path, PathBuf}, + }; + use mockall::mock; - use std::io::Write; - use std::path::Path; - use std::path::PathBuf; use tempfile::NamedTempFile; + use super::*; + use crate::api::CertsInfo; + mock! { pub CommandExec{} impl CommandExecutor for CommandExec { @@ -188,11 +146,7 @@ mod tests { image_url: "".to_string(), flag_safe: false, mtls: false, - certs: CertsInfo { - ca_cert: "".to_string(), - client_cert: "".to_string(), - client_key: "".to_string(), - }, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; // mock is_command_available mock_executor @@ -215,9 +169,7 @@ mod tests { mock_executor .expect_run_command() .withf(|cmd, args| { - cmd == "crictl" - && args.contains(&"pull") - && args.contains(&"docker.io/library/busybox:latest") + cmd == "crictl" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest") }) .times(1) .returning(|_, _| Ok(())); @@ -226,9 +178,7 @@ mod tests { mock_executor .expect_run_command_with_output() .withf(|cmd, args| { - cmd == "crictl" - && args.contains(&"inspecti") - && args.contains(&"{{.status.repoDigests}}") + cmd == "crictl" && args.contains(&"inspecti") && args.contains(&"{{.status.repoDigests}}") }) .times(1) .returning(|_, _| Ok(command_output2.to_string())); @@ -250,11 +200,7 @@ mod tests { image_url: "".to_string(), flag_safe: false, mtls: false, - certs: CertsInfo { - ca_cert: "".to_string(), - client_cert: "".to_string(), - client_key: "".to_string(), - }, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; // mock check_and_unmount @@ -277,13 +223,7 @@ mod tests { // Get the path of the temporary file and the path where it should be copied. let src_dir = tmp_file.path().parent().unwrap(); - let src_file_name = tmp_file - .path() - .file_name() - .unwrap() - .to_str() - .unwrap() - .to_string(); + let src_file_name = tmp_file.path().file_name().unwrap().to_str().unwrap().to_string(); let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file."); let dst_path = dst_file.path().to_path_buf(); @@ -324,8 +264,7 @@ mod tests { assert!(result.is_ok()); let expected_content = "Hello, world!\n"; - let actual_content = - fs::read_to_string(&dst_path).expect("Failed to read destination file."); + let actual_content = fs::read_to_string(&dst_path).expect("Failed to read destination file."); assert_eq!(expected_content, actual_content); // Assert the file permission @@ -357,8 +296,7 @@ mod tests { .times(1) .returning(|_, _| Ok(())); - let result = CtrImageHandler::new(PreparePath::default(), mock_executor) - .check_and_unmount("test_mount_path"); + let result = CtrImageHandler::new(PreparePath::default(), mock_executor).check_and_unmount("test_mount_path"); assert!(result.is_ok()); } @@ -367,10 +305,7 @@ mod tests { #[ignore] fn test_download_image() { init(); - let ctr = CtrImageHandler { - paths: PreparePath::default(), - executor: RealCommandExecutor {}, - }; + let ctr = CtrImageHandler { paths: PreparePath::default(), executor: RealCommandExecutor {} }; let update_req = UpgradeRequest { version: "KubeOS v2".to_string(), image_type: "containerd".to_string(), @@ -379,11 +314,7 @@ mod tests { image_url: "".to_string(), flag_safe: false, mtls: false, - certs: CertsInfo { - ca_cert: "".to_string(), - client_cert: "".to_string(), - client_key: "".to_string(), - }, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; ctr.download_image(&update_req).unwrap(); let tar_path = "/persist/KubeOS-Update/os.tar"; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs index f55ae167..5c4190ab 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs @@ -26,69 +26,40 @@ pub struct DiskImageHandler { impl ImageHandler for DiskImageHandler { fn download_image(&self, req: &UpgradeRequest) -> Result> { self.download(req)?; - self.checksum_match( - self.paths.image_path.to_str().unwrap_or_default(), - &req.check_sum, - )?; + self.checksum_match(self.paths.image_path.to_str().unwrap_or_default(), &req.check_sum)?; let (_, next_partition_info) = get_partition_info(&self.executor)?; - let img_manager = UpgradeImageManager::new( - self.paths.clone(), - next_partition_info, - self.executor.clone(), - ); + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); Ok(img_manager) } } impl Default for DiskImageHandler { fn default() -> Self { - Self { - paths: PreparePath::default(), - executor: RealCommandExecutor {}, - certs_path: CERTS_PATH.to_string(), - } + Self { paths: PreparePath::default(), executor: RealCommandExecutor {}, certs_path: CERTS_PATH.to_string() } } } impl DiskImageHandler { #[cfg(test)] fn new(paths: PreparePath, executor: T, certs_path: String) -> Self { - Self { - paths, - executor, - certs_path, - } + Self { paths, executor, certs_path } } fn download(&self, req: &UpgradeRequest) -> Result<()> { let mut resp = self.send_download_request(req)?; if resp.status() != reqwest::StatusCode::OK { - bail!( - "Failed to download image from {}, status: {}", - req.image_url, - resp.status() - ); + bail!("Failed to download image from {}, status: {}", req.image_url, resp.status()); } - debug!( - "Received response body size: {:?}", - resp.content_length().unwrap_or_default() - ); + debug!("Received response body size: {:?}", resp.content_length().unwrap_or_default()); let need_bytes = resp.content_length().unwrap_or_default() + BUFFER; check_disk_size( - i64::try_from(need_bytes) - .with_context(|| "Failed to transform content length from u64 to i64")?, - self.paths - .image_path - .parent() - .unwrap_or_else(|| Path::new(PERSIST_DIR)), + i64::try_from(need_bytes).with_context(|| "Failed to transform content length from u64 to i64")?, + self.paths.image_path.parent().unwrap_or_else(|| Path::new(PERSIST_DIR)), )?; let mut out = fs::File::create(&self.paths.image_path)?; - trace!( - "Start to save upgrade image to path {}", - &self.paths.image_path.display() - ); + trace!("Start to save upgrade image to path {}", &self.paths.image_path.display()); out.set_permissions(fs::Permissions::from_mode(IMAGE_PERMISSION))?; let bytes = resp.copy_to(&mut out)?; info!( @@ -127,22 +98,15 @@ impl DiskImageHandler { client = Client::new(); } else if req.mtls { // https mtls request - client = self - .load_ca_client_certs(&req.certs) - .with_context(|| "Failed to load client certificates")?; + client = self.load_ca_client_certs(&req.certs).with_context(|| "Failed to load client certificates")?; info!("Discover https mtls request to: {}", &req.image_url); } else { // https request - client = self - .load_ca_certs(&req.certs.ca_cert) - .with_context(|| "Failed to load CA certificates")?; + client = self.load_ca_certs(&req.certs.ca_cert).with_context(|| "Failed to load CA certificates")?; info!("Discover https request to: {}", &req.image_url); } - client - .get(&req.image_url) - .send() - .with_context(|| format!("Failed to fetch from URL: {}", &req.image_url)) + client.get(&req.image_url).send().with_context(|| format!("Failed to fetch from URL: {}", &req.image_url)) } fn load_ca_certs(&self, ca_cert: &str) -> Result { @@ -167,11 +131,7 @@ impl DiskImageHandler { client_identity.extend_from_slice(&client_key); let client_id = reqwest::Identity::from_pem(&client_identity)?; - let client = Client::builder() - .use_rustls_tls() - .add_root_certificate(ca) - .identity(client_id) - .build()?; + let client = Client::builder().use_rustls_tls().add_root_certificate(ca).identity(client_id).build()?; Ok(client) } @@ -193,9 +153,10 @@ impl DiskImageHandler { #[cfg(test)] mod tests { - use super::*; use tempfile::NamedTempFile; + use super::*; + fn init() { let _ = env_logger::builder() .target(env_logger::Target::Stdout) @@ -217,11 +178,8 @@ mod tests { init(); // generate tmp file let tmp_file = NamedTempFile::new().unwrap(); - let handler = DiskImageHandler::::new( - PreparePath::default(), - RealCommandExecutor {}, - String::new(), - ); + let handler = + DiskImageHandler::::new(PreparePath::default(), RealCommandExecutor {}, String::new()); let res = handler.cert_exist(tmp_file.path().to_str().unwrap()); assert!(res.is_ok()); @@ -242,18 +200,11 @@ mod tests { image_url: "http://localhost:8080/aaa.txt".to_string(), flag_safe: true, mtls: false, - certs: CertsInfo { - ca_cert: "".to_string(), - client_cert: "".to_string(), - client_key: "".to_string(), - }, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; let res = handler.send_download_request(&req); assert!(res.is_ok()); - assert_eq!( - res.unwrap().text().unwrap(), - "This is a test txt file generated by yuhang wei\n" - ); + assert_eq!(res.unwrap().text().unwrap(), "This is a test txt file generated by yuhang wei\n"); // https let mut handler = DiskImageHandler::::default(); @@ -274,10 +225,7 @@ mod tests { }; let res = handler.send_download_request(&req); assert!(res.is_ok()); - assert_eq!( - res.unwrap().text().unwrap(), - "This is a test txt file generated by yuhang wei\n" - ); + assert_eq!(res.unwrap().text().unwrap(), "This is a test txt file generated by yuhang wei\n"); // mtls let mut handler = DiskImageHandler::::default(); @@ -298,10 +246,7 @@ mod tests { }; let res = handler.send_download_request(&req); assert!(res.is_ok()); - assert_eq!( - res.unwrap().text().unwrap(), - "This is a test txt file generated by yuhang wei\n" - ); + assert_eq!(res.unwrap().text().unwrap(), "This is a test txt file generated by yuhang wei\n"); } #[test] @@ -309,8 +254,7 @@ mod tests { fn test_download() { init(); let mut handler = DiskImageHandler::::default(); - handler.paths.image_path = - PathBuf::from("/home/yuhang/Documents/KubeOS/KubeOS-Rust/test_download_image"); + handler.paths.image_path = PathBuf::from("/home/yuhang/Documents/KubeOS/KubeOS-Rust/test_download_image"); let req = UpgradeRequest { version: "v2".into(), check_sum: "90da5a14e9c06ddb276b06134e90d37098be2830beaa4357205bec7ff1aa1f7c".into(), @@ -319,11 +263,7 @@ mod tests { image_url: "http://localhost:8080/linux-firmware.rpm".to_string(), flag_safe: true, mtls: false, - certs: CertsInfo { - ca_cert: "".to_string(), - client_cert: "".to_string(), - client_key: "".to_string(), - }, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; let res = handler.download(&req); assert!(res.is_ok()); @@ -335,8 +275,7 @@ mod tests { fn test_checksum_match() { init(); let mut handler = DiskImageHandler::::default(); - handler.paths.image_path = - PathBuf::from("/home/yuhang/Documents/KubeOS/KubeOS-Rust/test_download_image"); + handler.paths.image_path = PathBuf::from("/home/yuhang/Documents/KubeOS/KubeOS-Rust/test_download_image"); let req = UpgradeRequest { version: "v2".into(), check_sum: "90da5a14e9c06ddb276b06134e90d37098be2830beaa4357205bec7ff1aa1f7c".into(), @@ -345,16 +284,10 @@ mod tests { image_url: "http://localhost:8080/aaa.txt".to_string(), flag_safe: true, mtls: false, - certs: CertsInfo { - ca_cert: "".to_string(), - client_cert: "".to_string(), - client_key: "".to_string(), - }, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; assert_eq!(handler.paths.image_path.exists(), true); - handler - .checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum) - .unwrap(); + handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum).unwrap(); } #[test] diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs index 0708b12d..74d3649a 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -20,33 +20,21 @@ impl ImageHandler for DockerImageHandler { self.get_rootfs_archive(req)?; let (_, next_partition_info) = get_partition_info(&self.executor)?; - let img_manager = UpgradeImageManager::new( - self.paths.clone(), - next_partition_info, - self.executor.clone(), - ); + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); img_manager.create_os_image(IMAGE_PERMISSION) } } impl Default for DockerImageHandler { fn default() -> Self { - Self { - paths: PreparePath::default(), - container_name: "kubeos-temp".into(), - executor: RealCommandExecutor {}, - } + Self { paths: PreparePath::default(), container_name: "kubeos-temp".into(), executor: RealCommandExecutor {} } } } impl DockerImageHandler { #[cfg(test)] fn new(paths: PreparePath, container_name: String, executor: T) -> Self { - Self { - paths, - container_name, - executor, - } + Self { paths, container_name, executor } } fn get_image(&self, req: &UpgradeRequest) -> Result<()> { @@ -66,15 +54,9 @@ impl DockerImageHandler { info!("Start get rootfs {}", image_name); self.check_and_rm_container()?; debug!("Create container {}", self.container_name); - let container_id = self.executor.run_command_with_output( - "docker", - &["create", "--name", &self.container_name, image_name], - )?; - debug!( - "Copy rootfs from container {} to {}", - container_id, - self.paths.update_path.display() - ); + let container_id = + self.executor.run_command_with_output("docker", &["create", "--name", &self.container_name, image_name])?; + debug!("Copy rootfs from container {} to {}", container_id, self.paths.update_path.display()); self.executor.run_command( "docker", &[ @@ -88,20 +70,11 @@ impl DockerImageHandler { } fn check_and_rm_container(&self) -> Result<()> { - let docker_ps_cmd = format!( - "docker ps -a -f=name={} | awk 'NR==2' | awk '{{print $1}}'", - self.container_name - ); - let exist_id = self - .executor - .run_command_with_output("bash", &["-c", &docker_ps_cmd])?; + let docker_ps_cmd = format!("docker ps -a -f=name={} | awk 'NR==2' | awk '{{print $1}}'", self.container_name); + let exist_id = self.executor.run_command_with_output("bash", &["-c", &docker_ps_cmd])?; if !exist_id.is_empty() { - info!( - "Remove container {} {} for cleaning environment", - self.container_name, exist_id - ); - self.executor - .run_command("docker", &["rm", exist_id.as_str()])?; + info!("Remove container {} {} for cleaning environment", self.container_name, exist_id); + self.executor.run_command("docker", &["rm", exist_id.as_str()])?; } Ok(()) } @@ -109,9 +82,10 @@ impl DockerImageHandler { #[cfg(test)] mod tests { - use super::*; use mockall::mock; + use super::*; + mock! { pub CommandExec{} impl CommandExecutor for CommandExec { @@ -150,8 +124,8 @@ mod tests { .times(1) .returning(|_, _| Ok(())); - let result = DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor) - .check_and_rm_container(); + let result = + DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor).check_and_rm_container(); assert!(result.is_ok()); } } diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs index 8f886bf7..0fe94b7c 100644 --- a/KubeOS-Rust/manager/src/utils/common.rs +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -12,8 +12,7 @@ use std::{ fs, - os::linux::fs::MetadataExt, - os::unix::fs::DirBuilderExt, + os::{linux::fs::MetadataExt, unix::fs::DirBuilderExt}, path::{Path, PathBuf}, }; @@ -51,23 +50,11 @@ pub fn is_file_exist>(path: P) -> bool { path.as_ref().exists() } -pub fn perpare_env( - prepare_path: &PreparePath, - need_bytes: i64, - persist_path: &str, - permission: u32, -) -> Result<()> { +pub fn perpare_env(prepare_path: &PreparePath, need_bytes: i64, persist_path: &str, permission: u32) -> Result<()> { info!("Prepare environment to upgrade"); check_disk_size(need_bytes, persist_path)?; - clean_env( - &prepare_path.update_path, - &prepare_path.mount_path, - &prepare_path.image_path, - )?; - fs::DirBuilder::new() - .recursive(true) - .mode(permission) - .create(&prepare_path.mount_path)?; + clean_env(&prepare_path.update_path, &prepare_path.mount_path, &prepare_path.image_path)?; + fs::DirBuilder::new().recursive(true).mode(permission).create(&prepare_path.mount_path)?; Ok(()) } @@ -92,11 +79,7 @@ where if is_mounted(&mount_path)? { debug!("Umount {}", mount_path.as_ref().display()); if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) { - bail!( - "Failed to umount {} in clean_env: {}", - mount_path.as_ref().display(), - errno - ); + bail!("Failed to umount {} in clean_env: {}", mount_path.as_ref().display(), errno); } } // losetup -D? @@ -119,18 +102,15 @@ pub fn delete_file_or_dir>(path: P) -> Result<()> { } pub fn is_command_available(command: &str, command_executor: &T) -> bool { - match command_executor.run_command( - "/bin/sh", - &["-c", format!("command -v {}", command).as_str()], - ) { + match command_executor.run_command("/bin/sh", &["-c", format!("command -v {}", command).as_str()]) { Ok(_) => { debug!("command {} is available", command); true - } + }, Err(_) => { debug!("command {} is not available", command); false - } + }, } } @@ -143,12 +123,10 @@ pub fn is_mounted>(mount_path: P) -> Result { let dev = mount_meta.st_dev(); // Get device ID of mountPath's parent directory - let parent = mount_path.as_ref().parent().ok_or_else(|| { - anyhow!( - "Failed to get parent directory of {}", - mount_path.as_ref().display() - ) - })?; + let parent = mount_path + .as_ref() + .parent() + .ok_or_else(|| anyhow!("Failed to get parent directory of {}", mount_path.as_ref().display()))?; let parent_meta = fs::symlink_metadata(parent)?; let dev_parent = parent_meta.st_dev(); Ok(dev != dev_parent) @@ -162,11 +140,7 @@ pub fn switch_boot_menuentry( if get_boot_mode() == "uefi" { command_executor.run_command( "grub2-editenv", - &[ - grub_env_path, - "set", - format!("saved_entry={}", next_menuentry).as_str(), - ], + &[grub_env_path, "set", format!("saved_entry={}", next_menuentry).as_str()], )?; } else { command_executor.run_command("grub2-set-default", &[next_menuentry])?; @@ -175,19 +149,15 @@ pub fn switch_boot_menuentry( } pub fn get_boot_mode() -> String { - if is_file_exist("/sys/firmware/efi") { - "uefi".into() - } else { - "bios".into() - } + if is_file_exist("/sys/firmware/efi") { "uefi".into() } else { "bios".into() } } #[cfg(test)] mod tests { - use super::*; use mockall::{mock, predicate::*}; - use tempfile::NamedTempFile; - use tempfile::TempDir; + use tempfile::{NamedTempFile, TempDir}; + + use super::*; // Mock the CommandExecutor trait mock! { @@ -254,12 +224,7 @@ mod tests { let update_path = "/tmp/test_clean_env"; let mount_path = "/tmp/test_clean_env/kubeos-update"; let image_path = "/tmp/test_clean_env/update.img"; - clean_env( - &update_path.to_string(), - &mount_path.to_string(), - &image_path.to_string(), - ) - .unwrap(); + clean_env(&update_path.to_string(), &mount_path.to_string(), &image_path.to_string()).unwrap(); } #[test] diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs index 522e4423..a54fc193 100644 --- a/KubeOS-Rust/manager/src/utils/container_image.rs +++ b/KubeOS-Rust/manager/src/utils/container_image.rs @@ -34,11 +34,7 @@ pub fn check_oci_image_digest( ) -> Result<()> { let image_digests = get_oci_image_digest(container_runtime, image_name, command_executor)?; if image_digests.to_lowercase() != check_sum.to_lowercase() { - bail!( - "Image digest mismatch, expect {}, got {}", - check_sum, - image_digests - ); + bail!("Image digest mismatch, expect {}, got {}", check_sum, image_digests); } Ok(()) } @@ -53,33 +49,16 @@ pub fn get_oci_image_digest( "crictl" => { cmd_output = executor.run_command_with_output( "crictl", - &[ - "inspecti", - "--output", - "go-template", - "--template", - "{{.status.repoDigests}}", - image_name, - ], + &["inspecti", "--output", "go-template", "--template", "{{.status.repoDigests}}", image_name], )?; - } + }, "docker" => { - cmd_output = executor.run_command_with_output( - "docker", - &["inspect", "--format", "{{.RepoDigests}}", image_name], - )?; - } + cmd_output = + executor.run_command_with_output("docker", &["inspect", "--format", "{{.RepoDigests}}", image_name])?; + }, "ctr" => { - cmd_output = executor.run_command_with_output( - "ctr", - &[ - "-n", - "k8s.io", - "images", - "ls", - &format!("name=={}", image_name), - ], - )?; + cmd_output = executor + .run_command_with_output("ctr", &["-n", "k8s.io", "images", "ls", &format!("name=={}", image_name)])?; // Split by whitespaces, we get vec like [REF TYPE DIGEST SIZE PLATFORMS LABELS x x x x x x] // get the 8th element, and split by ':' to get the digest let fields: Vec<&str> = cmd_output.split_whitespace().collect(); @@ -87,18 +66,12 @@ pub fn get_oci_image_digest( trace!("get_oci_image_digest: {}", digest); return Ok(digest.to_string()); } else { - bail!( - "Failed to get digest from ctr command output: {}", - cmd_output - ); + bail!("Failed to get digest from ctr command output: {}", cmd_output); } - } + }, _ => { - bail!( - "Container runtime {} cannot be recognized", - container_runtime - ); - } + bail!("Container runtime {} cannot be recognized", container_runtime); + }, } // Parse the cmd_output to extract the digest @@ -122,86 +95,60 @@ pub fn pull_image(runtime: &str, image_name: &str, executor: match runtime { "crictl" => { executor.run_command("crictl", &["pull", image_name])?; - } + }, "ctr" => { executor.run_command( "ctr", - &[ - &"-n", - "k8s.io", - "images", - "pull", - "--hosts-dir", - "/etc/containerd/certs.d", - image_name, - ], + &[&"-n", "k8s.io", "images", "pull", "--hosts-dir", "/etc/containerd/certs.d", image_name], )?; - } + }, "docker" => { executor.run_command("docker", &["pull", image_name])?; - } + }, _ => { bail!("Container runtime {} cannot be recognized", runtime); - } + }, } Ok(()) } -pub fn remove_image_if_exist( - runtime: &str, - image_name: &str, - executor: &T, -) -> Result<()> { +pub fn remove_image_if_exist(runtime: &str, image_name: &str, executor: &T) -> Result<()> { match runtime { "crictl" => { - if executor - .run_command("crictl", &["inspecti", image_name]) - .is_ok() - { + if executor.run_command("crictl", &["inspecti", image_name]).is_ok() { executor.run_command("crictl", &["rmi", image_name])?; info!("Remove existing upgrade image: {}", image_name); } - } + }, "ctr" => { let output = executor.run_command_with_output( "ctr", - &[ - &"-n", - "k8s.io", - "images", - "check", - &format!("name=={}", image_name), - ], + &[&"-n", "k8s.io", "images", "check", &format!("name=={}", image_name)], )?; if !output.is_empty() { - executor.run_command( - "ctr", - &[&"-n", "k8s.io", "images", "rm", image_name, "--sync"], - )?; + executor.run_command("ctr", &[&"-n", "k8s.io", "images", "rm", image_name, "--sync"])?; info!("Remove existing upgrade image: {}", image_name); } - } + }, "docker" => { - if executor - .run_command("docker", &["inspect", image_name]) - .is_ok() - { + if executor.run_command("docker", &["inspect", image_name]).is_ok() { executor.run_command("docker", &["rmi", image_name])?; info!("Remove existing upgrade image: {}", image_name); } - } + }, _ => { bail!("Container runtime {} cannot be recognized", runtime); - } + }, } Ok(()) } #[cfg(test)] mod tests { - use super::*; use mockall::{mock, predicate::*}; + use super::*; + // Mock the CommandExecutor trait mock! { pub CommandExec{} @@ -227,14 +174,11 @@ mod tests { init(); let out = is_valid_image_name("nginx").unwrap(); assert_eq!(out, ()); - let out = is_valid_image_name( - "docker.example.com:5000/gmr/alpine@sha256:11111111111111111111111111111111", - ) - .unwrap(); + let out = + is_valid_image_name("docker.example.com:5000/gmr/alpine@sha256:11111111111111111111111111111111").unwrap(); assert_eq!(out, ()); - let out = is_valid_image_name( - "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8", - ); + let out = + is_valid_image_name("sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8"); match out { Ok(_) => assert_eq!(true, false), Err(_) => assert_eq!(true, true), @@ -247,19 +191,16 @@ mod tests { let mut mock = MockCommandExec::new(); let container_runtime = "ctr"; let image_name = "docker.io/nginx:latest"; - let command_output1 = "REF TYPE DIGEST SIZE PLATFORMS LABELS\ndocker.io/nginx:latest text/html sha256:1111 132.5 KIB - -\n"; - mock.expect_run_command_with_output() - .times(1) - .returning(|_, _| Ok(command_output1.to_string())); + let command_output1 = + "REF TYPE DIGEST SIZE PLATFORMS LABELS\ndocker.io/nginx:latest text/html sha256:1111 132.5 KIB - -\n"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); let out1 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); let expect_output = "1111"; assert_eq!(out1, expect_output); let container_runtime = "crictl"; let command_output2 = "[docker.io/nginx@sha256:1111]"; - mock.expect_run_command_with_output() - .times(1) - .returning(|_, _| Ok(command_output2.to_string())); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string())); let out2 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); assert_eq!(out2, expect_output); } @@ -272,9 +213,7 @@ mod tests { let container_runtime = "crictl"; let command_output = "[docker.io/nginx@sha256:1111]"; let check_sum = "1111"; - mock.expect_run_command_with_output() - .times(1) - .returning(|_, _| Ok(command_output.to_string())); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output.to_string())); let result = check_oci_image_digest(container_runtime, image_name, check_sum, &mock); assert!(result.is_ok()); } diff --git a/KubeOS-Rust/manager/src/utils/executor.rs b/KubeOS-Rust/manager/src/utils/executor.rs index de55a1e8..8f4cb25c 100644 --- a/KubeOS-Rust/manager/src/utils/executor.rs +++ b/KubeOS-Rust/manager/src/utils/executor.rs @@ -29,12 +29,7 @@ impl CommandExecutor for RealCommandExecutor { let output = Command::new(name).args(args).output()?; if !output.status.success() { let error_message = String::from_utf8_lossy(&output.stderr); - bail!( - "Failed to run command: {} {:?}, stderr: {}", - name, - args, - error_message - ); + bail!("Failed to run command: {} {:?}, stderr: {}", name, args, error_message); } debug!("run_command: {} {:?} done", name, args); Ok(()) @@ -45,12 +40,7 @@ impl CommandExecutor for RealCommandExecutor { let output = Command::new(name).args(args).output()?; if !output.status.success() { let error_message = String::from_utf8_lossy(&output.stderr); - bail!( - "Failed to run command: {} {:?}, stderr: {}", - name, - args, - error_message - ); + bail!("Failed to run command: {} {:?}, stderr: {}", name, args, error_message); } let stdout = String::from_utf8_lossy(&output.stdout).to_string(); debug!("run_command_with_output: {} {:?} done", name, args); @@ -76,16 +66,11 @@ mod tests { let executor: RealCommandExecutor = RealCommandExecutor {}; // test run_command_with_output - let output = executor - .run_command_with_output("echo", &["hello", "world"]) - .unwrap(); + let output = executor.run_command_with_output("echo", &["hello", "world"]).unwrap(); assert_eq!(output, "hello world"); - let out = executor - .run_command_with_output("sh", &["-c", format!("command -v {}", "cat").as_str()]) - .unwrap(); + let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "cat").as_str()]).unwrap(); assert_eq!(out, "/usr/bin/cat"); - let out = executor - .run_command_with_output("sh", &["-c", format!("command -v {}", "apple").as_str()]); + let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "apple").as_str()]); assert!(out.is_err()); } diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs index ca48712f..e3dddaaf 100644 --- a/KubeOS-Rust/manager/src/utils/image_manager.rs +++ b/KubeOS-Rust/manager/src/utils/image_manager.rs @@ -34,47 +34,26 @@ pub struct UpgradeImageManager { impl UpgradeImageManager { pub fn new(paths: PreparePath, next_partition: PartitionInfo, executor: T) -> Self { - Self { - paths, - next_partition, - executor, - } + Self { paths, next_partition, executor } } fn image_path_str(&self) -> Result<&str> { - self.paths - .image_path - .to_str() - .context("Failed to convert image path to string") + self.paths.image_path.to_str().context("Failed to convert image path to string") } fn mount_path_str(&self) -> Result<&str> { - self.paths - .mount_path - .to_str() - .context("Failed to convert mount path to string") + self.paths.mount_path.to_str().context("Failed to convert mount path to string") } fn tar_path_str(&self) -> Result<&str> { - self.paths - .tar_path - .to_str() - .context("Failed to convert tar path to string") + self.paths.tar_path.to_str().context("Failed to convert tar path to string") } pub fn create_image_file(&self, permission: u32) -> Result<()> { let image_str = self.image_path_str()?; debug!("Create image {}", image_str); - self.executor.run_command( - "dd", - &[ - "if=/dev/zero", - &format!("of={}", image_str), - "bs=2M", - "count=1024", - ], - )?; + self.executor.run_command("dd", &["if=/dev/zero", &format!("of={}", image_str), "bs=2M", "count=1024"])?; fs::set_permissions(&self.paths.image_path, Permissions::from_mode(permission))?; Ok(()) } @@ -84,11 +63,7 @@ impl UpgradeImageManager { debug!("Format image {}", image_str); self.executor.run_command( format!("mkfs.{}", self.next_partition.fs_type).as_str(), - &[ - "-L", - format!("ROOT-{}", self.next_partition.menuentry).as_str(), - image_str, - ], + &["-L", format!("ROOT-{}", self.next_partition.menuentry).as_str(), image_str], )?; Ok(()) } @@ -97,8 +72,7 @@ impl UpgradeImageManager { let image_str = self.image_path_str()?; let mount_str = self.mount_path_str()?; debug!("Mount {} to {}", image_str, mount_str); - self.executor - .run_command("mount", &["-o", "loop", image_str, mount_str])?; + self.executor.run_command("mount", &["-o", "loop", image_str, mount_str])?; Ok(()) } @@ -106,8 +80,7 @@ impl UpgradeImageManager { let tar_str = self.tar_path_str()?; let mount_str = self.mount_path_str()?; debug!("Extract {} to mounted path {}", tar_str, mount_str); - self.executor - .run_command("tar", &["-xvf", tar_str, "-C", mount_str])?; + self.executor.run_command("tar", &["-xvf", tar_str, "-C", mount_str])?; Ok(()) } @@ -117,25 +90,15 @@ impl UpgradeImageManager { self.mount_image()?; self.extract_tar_to_image()?; // Pass empty image_path to clean_env to avoid delete image file - clean_env( - &self.paths.update_path, - &self.paths.mount_path, - &PathBuf::new(), - )?; + clean_env(&self.paths.update_path, &self.paths.mount_path, &PathBuf::new())?; Ok(self) } pub fn install(&self) -> Result<()> { let image_str = self.image_path_str()?; let device = self.next_partition.device.as_str(); - self.executor.run_command( - "dd", - &[ - format!("if={}", image_str).as_str(), - format!("of={}", device).as_str(), - "bs=8M", - ], - )?; + self.executor + .run_command("dd", &[format!("if={}", image_str).as_str(), format!("of={}", device).as_str(), "bs=8M"])?; debug!("Install image {} to {} done", image_str, device); info!( "Device {} is overwritten and unable to rollback to the previous version anymore if the eviction of node fails", @@ -148,11 +111,13 @@ impl UpgradeImageManager { #[cfg(test)] mod tests { - use super::*; - use mockall::{mock, predicate::*}; use std::{fs, io::Write, path::Path}; + + use mockall::{mock, predicate::*}; use tempfile::NamedTempFile; + use super::*; + // Mock the CommandExecutor trait mock! { pub CommandExec{} @@ -227,11 +192,7 @@ mod tests { tar_path: "/tmp/update/image.tar".into(), rootfs_file: "image.tar".into(), }, - PartitionInfo { - device: "/dev/sda3".into(), - fs_type: "ext4".into(), - menuentry: "B".into(), - }, + PartitionInfo { device: "/dev/sda3".into(), fs_type: "ext4".into(), menuentry: "B".into() }, mock, ); diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs index 5c0f4818..0419159b 100644 --- a/KubeOS-Rust/manager/src/utils/partition.rs +++ b/KubeOS-Rust/manager/src/utils/partition.rs @@ -22,9 +22,7 @@ pub struct PartitionInfo { pub fs_type: String, } -pub fn get_partition_info( - executor: &T, -) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> { +pub fn get_partition_info(executor: &T) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> { let lsblk = executor.run_command_with_output("lsblk", &["-lno", "NAME,MOUNTPOINTS,FSTYPE"])?; // After split whitespace, the root directory line should have 3 elements, which are "sda2 / ext4". let mut cur_partition = PartitionInfo::default(); @@ -59,9 +57,10 @@ pub fn get_partition_info( #[cfg(test)] mod tests { - use super::*; use mockall::{mock, predicate::*}; + use super::*; + // Mock the CommandExecutor trait mock! { pub CommandExec{} @@ -85,24 +84,13 @@ mod tests { #[test] fn test_get_partition_info() { init(); - let command_output1 = - "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; let mut mock = MockCommandExec::new(); - mock.expect_run_command_with_output() - .times(1) - .returning(|_, _| Ok(command_output1.to_string())); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); let res = get_partition_info(&mock).unwrap(); let expect_res = ( - PartitionInfo { - device: "/dev/sda2".to_string(), - menuentry: "A".to_string(), - fs_type: "ext4".to_string(), - }, - PartitionInfo { - device: "/dev/sda3".to_string(), - menuentry: "B".to_string(), - fs_type: "ext4".to_string(), - }, + PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() }, + PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() }, ); assert_eq!(res, expect_res); } diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index 474d2397..b96bd1cd 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -1,31 +1,33 @@ [package] +description = "KubeOS os-proxy" +edition = "2021" +license = "MulanPSL-2.0" name = "proxy" version = "0.1.0" -edition = "2021" -description = "KubeOS os-proxy" -license = "MulanPSL-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -kube = { version = "0.66.0", features = ["runtime", "derive"] } -k8s-openapi = { version = "0.13.1", features = ["v1_22"] } -tokio = { version = "=1.14.0", features = ["rt-multi-thread", "macros"] } anyhow = "1.0.44" +async-trait = "0.1" +chrono = { version = "0.4", default-features = false, features = ["std"] } +cli = { version = "0.1.0", path = "../cli" } +env_logger = "0.9.0" futures = "0.3.17" +h2 = "=0.3.16" +k8s-openapi = { version = "0.13.1", features = ["v1_22"] } +kube = { version = "0.66.0", features = ["derive", "runtime"] } +log = "=0.4.15" +manager = { version = "0.1.0", path = "../manager" } +regex = "=1.7.3" +reqwest = { version = "=0.11.18", default-features = false, features = [ + "json", +] } +schemars = "=0.8.10" serde = { version = "1.0.130", features = ["derive"] } serde_json = "1.0.68" -thiserror = "1.0.29" -env_logger = "0.9.0" -schemars = "=0.8.10" socket2 = "=0.4.9" -log = "=0.4.15" +thiserror = "1.0.29" thread_local = "=1.1.4" -async-trait = "0.1" -regex = "=1.7.3" -chrono = { version = "0.4", default-features = false, features = ["std"] } -h2 = "=0.3.16" +tokio = { version = "=1.14.0", features = ["macros", "rt-multi-thread"] } tokio-retry = "0.3" -reqwest = { version = "=0.11.18", default-features = false, features = [ "json" ] } -cli = { version = "0.1.0", path = "../cli" } -manager = { version = "0.1.0", path = "../manager" } diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs index b099a0e8..ddc4f290 100644 --- a/KubeOS-Rust/proxy/src/controller/agentclient.rs +++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs @@ -10,22 +10,18 @@ * See the Mulan PSL v2 for more details. */ +use std::{collections::HashMap, path::Path}; + +use agent_call::AgentCallClient; +use agent_error::Error; use cli::{ client::Client, method::{ - callable_method::RpcMethod, configure::ConfigureMethod, - prepare_upgrade::PrepareUpgradeMethod, rollback::RollbackMethod, upgrade::UpgradeMethod, + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, }, }; - -use agent_call::AgentCallClient; -use agent_error::Error; -use manager::api::{ - CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, - UpgradeRequest, -}; -use std::collections::HashMap; -use std::path::Path; +use manager::api::{CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest}; pub struct UpgradeInfo { pub version: String, @@ -50,18 +46,10 @@ pub struct KeyInfo { } pub trait AgentMethod { - fn prepare_upgrade_method( - &self, - upgrade_info: UpgradeInfo, - agent_call: AgentCallClient, - ) -> Result<(), Error>; + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo, agent_call: AgentCallClient) -> Result<(), Error>; fn upgrade_method(&self, agent_call: AgentCallClient) -> Result<(), Error>; fn rollback_method(&self, agent_call: AgentCallClient) -> Result<(), Error>; - fn configure_method( - &self, - config_info: ConfigInfo, - agent_call: AgentCallClient, - ) -> Result<(), Error>; + fn configure_method(&self, config_info: ConfigInfo, agent_call: AgentCallClient) -> Result<(), Error>; } pub mod agent_call { @@ -70,11 +58,7 @@ pub mod agent_call { pub struct AgentCallClient {} impl AgentCallClient { - pub fn call_agent( - &self, - client: &Client, - method: T, - ) -> Result<(), Error> { + pub fn call_agent(&self, client: &Client, method: T) -> Result<(), Error> { match method.call(client) { Ok(_resp) => Ok(()), Err(e) => Err(Error::AgentError { source: e }), @@ -89,18 +73,12 @@ pub struct AgentClient { impl AgentClient { pub fn new>(socket_path: P) -> Self { - AgentClient { - agent_client: Client::new(socket_path), - } + AgentClient { agent_client: Client::new(socket_path) } } } impl AgentMethod for AgentClient { - fn prepare_upgrade_method( - &self, - upgrade_info: UpgradeInfo, - agent_call: AgentCallClient, - ) -> Result<(), Error> { + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo, agent_call: AgentCallClient) -> Result<(), Error> { let upgrade_request = UpgradeRequest { version: upgrade_info.version, image_type: upgrade_info.image_type, @@ -110,16 +88,9 @@ impl AgentMethod for AgentClient { image_url: "".to_string(), flag_safe: false, mtls: false, - certs: CertsInfo { - ca_cert: "".to_string(), - client_cert: "".to_string(), - client_key: "".to_string(), - }, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; - match agent_call.call_agent( - &self.agent_client, - PrepareUpgradeMethod::new(upgrade_request), - ) { + match agent_call.call_agent(&self.agent_client, PrepareUpgradeMethod::new(upgrade_request)) { Ok(_resp) => Ok(()), Err(e) => Err(e), } @@ -139,21 +110,14 @@ impl AgentMethod for AgentClient { } } - fn configure_method( - &self, - config_info: ConfigInfo, - agent_call: AgentCallClient, - ) -> Result<(), Error> { + fn configure_method(&self, config_info: ConfigInfo, agent_call: AgentCallClient) -> Result<(), Error> { let mut agent_configs: Vec = Vec::new(); for config in config_info.configs { let mut contents_tmp: HashMap = HashMap::new(); for (key, key_info) in config.contents.iter() { contents_tmp.insert( key.to_string(), - AgentKeyInfo { - value: key_info.value.clone(), - operation: key_info.operation.clone(), - }, + AgentKeyInfo { value: key_info.value.clone(), operation: key_info.operation.clone() }, ); } agent_configs.push(AgentSysconfig { @@ -162,9 +126,7 @@ impl AgentMethod for AgentClient { contents: contents_tmp, }) } - let config_request = ConfigureRequest { - configs: agent_configs, - }; + let config_request = ConfigureRequest { configs: agent_configs }; match agent_call.call_agent(&self.agent_client, ConfigureMethod::new(config_request)) { Ok(_resp) => Ok(()), Err(e) => Err(e), diff --git a/KubeOS-Rust/proxy/src/controller/apiclient.rs b/KubeOS-Rust/proxy/src/controller/apiclient.rs index 5ee28004..3afd5a51 100644 --- a/KubeOS-Rust/proxy/src/controller/apiclient.rs +++ b/KubeOS-Rust/proxy/src/controller/apiclient.rs @@ -10,8 +10,8 @@ * See the Mulan PSL v2 for more details. */ -use super::crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}; -use super::values::{LABEL_OSINSTANCE, NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}; +use std::collections::BTreeMap; + use anyhow::Result; use apiclient_error::Error; use async_trait::async_trait; @@ -20,7 +20,11 @@ use kube::{ Client, }; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; + +use super::{ + crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}, + values::{LABEL_OSINSTANCE, NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}, +}; #[derive(Debug, Serialize, Deserialize)] struct OSInstanceSpecPatch { @@ -35,11 +39,7 @@ impl Default for OSInstanceSpecPatch { OSInstanceSpecPatch { api_version: OSINSTANCE_API_VERSION.to_string(), kind: OSINSTANCE_KIND.to_string(), - spec: OSInstanceSpec { - nodestatus: NODE_STATUS_IDLE.to_string(), - sysconfigs: None, - upgradeconfigs: None, - }, + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, } } } @@ -57,10 +57,7 @@ impl Default for OSInstanceStatusPatch { OSInstanceStatusPatch { api_version: OSINSTANCE_API_VERSION.to_string(), kind: OSINSTANCE_KIND.to_string(), - status: Some(OSInstanceStatus { - sysconfigs: None, - upgradeconfigs: None, - }), + status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }), } } } @@ -105,11 +102,7 @@ impl ApplyApi for ControllerClient { labels: Some(labels), ..ObjectMeta::default() }, - spec: OSInstanceSpec { - nodestatus: NODE_STATUS_IDLE.to_string(), - sysconfigs: None, - upgradeconfigs: None, - }, + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, status: None, }; let osi_api = Api::namespaced(self.client.clone(), namespace); @@ -124,17 +117,8 @@ impl ApplyApi for ControllerClient { spec: &OSInstanceSpec, ) -> Result<(), Error> { let osi_api: Api = Api::namespaced(self.client.clone(), namespace); - let osi_spec_patch = OSInstanceSpecPatch { - spec: spec.clone(), - ..Default::default() - }; - osi_api - .patch( - node_name, - &PatchParams::default(), - &Patch::Merge(&osi_spec_patch), - ) - .await?; + let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() }; + osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?; Ok(()) } @@ -145,17 +129,8 @@ impl ApplyApi for ControllerClient { status: &Option, ) -> Result<(), Error> { let osi_api: Api = Api::namespaced(self.client.clone(), namespace); - let osi_status_patch = OSInstanceStatusPatch { - status: status.clone(), - ..Default::default() - }; - osi_api - .patch_status( - node_name, - &PatchParams::default(), - &Patch::Merge(&osi_status_patch), - ) - .await?; + let osi_status_patch = OSInstanceStatusPatch { status: status.clone(), ..Default::default() }; + osi_api.patch_status(node_name, &PatchParams::default(), &Patch::Merge(&osi_status_patch)).await?; Ok(()) } } diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index da2e031e..b23ec187 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -10,19 +10,8 @@ * See the Mulan PSL v2 for more details. */ -use super::{ - agentclient::{ - agent_call::AgentCallClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo, - }, - apiclient::ApplyApi, - crd::{Configs, Content, OSInstance, OS}, - drain::drain_os, - utils::{check_version, get_config_version, ConfigOperation, ConfigType}, - values::{ - LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, - OPERATION_TYPE_UPGRADE, REQUEUE_ERROR, REQUEUE_NORMAL, - }, -}; +use std::{collections::HashMap, env}; + use anyhow::Result; use k8s_openapi::api::core::v1::Node; use kube::{ @@ -33,8 +22,18 @@ use kube::{ }; use log::{debug, error, info}; use reconciler_error::Error; -use std::collections::HashMap; -use std::env; + +use super::{ + agentclient::{agent_call::AgentCallClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, + apiclient::ApplyApi, + crd::{Configs, Content, OSInstance, OS}, + drain::drain_os, + utils::{check_version, get_config_version, ConfigOperation, ConfigType}, + values::{ + LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, + REQUEUE_ERROR, REQUEUE_NORMAL, + }, +}; pub async fn reconcile( os: OS, @@ -44,34 +43,22 @@ pub async fn reconcile( let proxy_controller = ctx.get_ref(); let os_cr = &os; let node_name = env::var("NODE_NAME")?; - let namespace: String = os_cr.namespace().ok_or(Error::MissingObjectKey { - resource: "os".to_string(), - value: "namespace".to_string(), - })?; - proxy_controller - .check_osi_exisit(&namespace, &node_name) - .await?; - let controller_res = proxy_controller - .get_resources(&namespace, &node_name) - .await?; + let namespace: String = os_cr + .namespace() + .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; + proxy_controller.check_osi_exisit(&namespace, &node_name).await?; + let controller_res = proxy_controller.get_resources(&namespace, &node_name).await?; let node = controller_res.node; let mut osinstance = controller_res.osinstance; let node_os_image = &node .status .as_ref() - .ok_or(Error::MissingSubResource { - value: String::from("node.status"), - })? + .ok_or(Error::MissingSubResource { value: String::from("node.status") })? .node_info .as_ref() - .ok_or(Error::MissingSubResource { - value: String::from("node.status.node_info"), - })? + .ok_or(Error::MissingSubResource { value: String::from("node.status.node_info") })? .os_image; - debug!( - "os expected osversion is {},actual osversion is {}", - os_cr.spec.osversion, node_os_image - ); + debug!("os expected osversion is {},actual osversion is {}", os_cr.spec.osversion, node_os_image); if check_version(&os_cr.spec.osversion, &node_os_image) { match ConfigType::SysConfig.check_config_version(&os, &osinstance) { ConfigOperation::Reassign => { @@ -85,7 +72,7 @@ pub async fn reconcile( ) .await?; return Ok(REQUEUE_NORMAL); - } + }, ConfigOperation::UpdateConfig => { debug!("start update config"); osinstance.spec.sysconfigs = os_cr.spec.sysconfigs.clone(); @@ -94,19 +81,12 @@ pub async fn reconcile( .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec) .await?; return Ok(REQUEUE_ERROR); - } - _ => {} + }, + _ => {}, } + proxy_controller.set_config(&mut osinstance, ConfigType::SysConfig).await?; proxy_controller - .set_config(&mut osinstance, ConfigType::SysConfig) - .await?; - proxy_controller - .refresh_node( - node, - osinstance, - &get_config_version(os_cr.spec.sysconfigs.as_ref()), - ConfigType::SysConfig, - ) + .refresh_node(node, osinstance, &get_config_version(os_cr.spec.sysconfigs.as_ref()), ConfigType::SysConfig) .await?; } else { if os_cr.spec.opstype == NODE_STATUS_CONFIG { @@ -124,8 +104,8 @@ pub async fn reconcile( ) .await?; return Ok(REQUEUE_NORMAL); - } - _ => {} + }, + _ => {}, } if node.labels().contains_key(LABEL_UPGRADING) { if osinstance.spec.nodestatus == NODE_STATUS_IDLE { @@ -142,9 +122,7 @@ pub async fn reconcile( .await?; return Ok(REQUEUE_NORMAL); } - proxy_controller - .set_config(&mut osinstance, ConfigType::UpgradeConfig) - .await?; + proxy_controller.set_config(&mut osinstance, ConfigType::UpgradeConfig).await?; proxy_controller.upgrade_node(os_cr, &node).await?; } } @@ -171,11 +149,7 @@ pub struct ProxyController { impl ProxyController { pub fn new(k8s_client: Client, controller_client: T, agent_client: U) -> Self { - ProxyController { - k8s_client, - controller_client, - agent_client, - } + ProxyController { k8s_client, controller_client, agent_client } } } @@ -186,31 +160,22 @@ impl ProxyController { Ok(osi) => { debug!("osinstance is exist {:?}", osi.name()); return Ok(()); - } + }, Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { info!("Create OSInstance {}", node_name); - self.controller_client - .create_osinstance(node_name, namespace) - .await?; + self.controller_client.create_osinstance(node_name, namespace).await?; Ok(()) - } + }, Err(err) => Err(Error::KubeError { source: err }), } } - async fn get_resources( - &self, - namespace: &str, - node_name: &str, - ) -> Result { + async fn get_resources(&self, namespace: &str, node_name: &str) -> Result { let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); let osinstance_cr = osi_api.get(node_name).await?; let node_api: Api = Api::all(self.k8s_client.clone()); let node_cr = node_api.get(node_name).await?; - Ok(ControllerResources { - osinstance: osinstance_cr, - node: node_cr, - }) + Ok(ControllerResources { osinstance: osinstance_cr, node: node_cr }) } async fn refresh_node( @@ -225,9 +190,7 @@ impl ProxyController { let labels = node.labels_mut(); if labels.contains_key(LABEL_UPGRADING) { labels.remove(LABEL_UPGRADING); - node = node_api - .replace(&node.name(), &PostParams::default(), &node) - .await?; + node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; } if let Some(node_spec) = &node.spec { if let Some(node_unschedulable) = node_spec.unschedulable { @@ -237,8 +200,7 @@ impl ProxyController { } } } - self.update_node_status(osinstance, os_config_version, config_type) - .await?; + self.update_node_status(osinstance, os_config_version, config_type).await?; Ok(()) } @@ -252,8 +214,7 @@ impl ProxyController { if osinstance.spec.nodestatus == NODE_STATUS_IDLE { return Ok(()); } - let upgradeconfig_spec_version = - get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + let upgradeconfig_spec_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); let sysconfig_spec_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); let sysconfig_status_version: String; if let Some(osinstance_status) = osinstance.status.as_ref() { @@ -263,65 +224,46 @@ impl ProxyController { } if sysconfig_spec_version == sysconfig_status_version || (config_type == ConfigType::SysConfig && os_config_version != sysconfig_spec_version) - || (config_type == ConfigType::UpgradeConfig - && os_config_version != upgradeconfig_spec_version) + || (config_type == ConfigType::UpgradeConfig && os_config_version != upgradeconfig_spec_version) { let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey { resource: String::from("osinstance"), value: String::from("namespace"), })?; osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); - self.controller_client - .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec) - .await?; + self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; } Ok(()) } - async fn update_osi_status( - &self, - osinstance: &mut OSInstance, - config_type: ConfigType, - ) -> Result<(), Error> { + async fn update_osi_status(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> { debug!("start update_osi_status"); config_type.set_osi_status_config(osinstance); debug!("osinstance status is update to {:?}", osinstance.status); - let namespace = &osinstance.namespace().ok_or(Error::MissingObjectKey { - resource: "osinstance".to_string(), - value: "namespace".to_string(), - })?; - self.controller_client - .update_osinstance_status(&osinstance.name(), &namespace, &osinstance.status) - .await?; + let namespace = &osinstance + .namespace() + .ok_or(Error::MissingObjectKey { resource: "osinstance".to_string(), value: "namespace".to_string() })?; + self.controller_client.update_osinstance_status(&osinstance.name(), &namespace, &osinstance.status).await?; Ok(()) } - async fn set_config( - &self, - osinstance: &mut OSInstance, - config_type: ConfigType, - ) -> Result<(), Error> { + async fn set_config(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> { debug!("start set_config"); let config_info = config_type.check_config_start(osinstance); if config_info.need_config { match config_info.configs.and_then(convert_to_agent_config) { Some(agent_configs) => { let agent_call_client = AgentCallClient::default(); - match self.agent_client.configure_method( - ConfigInfo { - configs: agent_configs, - }, - agent_call_client, - ) { - Ok(_resp) => {} + match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }, agent_call_client) { + Ok(_resp) => {}, Err(e) => { return Err(Error::AgentError { source: e }); - } + }, } - } + }, None => { info!("config is none, no need to config"); - } + }, }; self.update_osi_status(osinstance, config_type).await?; } @@ -339,41 +281,34 @@ impl ProxyController { container_image: os_cr.spec.containerimage.clone(), }; let agent_call_client = AgentCallClient::default(); - match self - .agent_client - .prepare_upgrade_method(upgrade_info, agent_call_client) - { - Ok(_resp) => {} + match self.agent_client.prepare_upgrade_method(upgrade_info, agent_call_client) { + Ok(_resp) => {}, Err(e) => { return Err(Error::AgentError { source: e }); - } + }, } - self.evict_node(&node.name(), os_cr.spec.evictpodforce) - .await?; + self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; let agent_call_client = AgentCallClient::default(); match self.agent_client.upgrade_method(agent_call_client) { - Ok(_resp) => {} + Ok(_resp) => {}, Err(e) => { return Err(Error::AgentError { source: e }); - } + }, } - } + }, OPERATION_TYPE_ROLLBACK => { - self.evict_node(&node.name(), os_cr.spec.evictpodforce) - .await?; + self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; let agent_call_client = AgentCallClient::default(); match self.agent_client.rollback_method(agent_call_client) { - Ok(_resp) => {} + Ok(_resp) => {}, Err(e) => { return Err(Error::AgentError { source: e }); - } + }, } - } + }, _ => { - return Err(Error::OperationError { - value: os_cr.spec.opstype.clone(), - }); - } + return Err(Error::OperationError { value: os_cr.spec.opstype.clone() }); + }, } Ok(()) } @@ -384,12 +319,12 @@ impl ProxyController { node_api.cordon(node_name).await?; info!("Cordon node Successfully{}, start drain nodes", node_name); match self.drain_node(node_name, evict_pod_force).await { - Ok(()) => {} + Ok(()) => {}, Err(e) => { node_api.uncordon(node_name).await?; info!("Drain node {} error, uncordon node successfully", node_name); return Err(e); - } + }, } Ok(()) } @@ -397,9 +332,7 @@ impl ProxyController { async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> { use crate::controller::drain::error::DrainError::*; match drain_os(&self.k8s_client.clone(), node_name, force).await { - Err(DeletePodsError { errors, .. }) => Err(Error::DrainNodeError { - value: errors.join("; "), - }), + Err(DeletePodsError { errors, .. }) => Err(Error::DrainNodeError { value: errors.join("; ") }), _ => Ok(()), } } @@ -417,11 +350,15 @@ fn convert_to_agent_config(configs: Configs) -> Option> { contents: contents_tmp, }; agent_configs.push(config_tmp) - } + }, None => { - info!("model {} which has configpath {} do not has any contents no need to configure",config.model.unwrap_or_default(),config.configpath.unwrap_or_default()); + info!( + "model {} which has configpath {} do not has any contents no need to configure", + config.model.unwrap_or_default(), + config.configpath.unwrap_or_default() + ); continue; - } + }, }; } if agent_configs.len() == 0 { @@ -436,19 +373,17 @@ fn convert_to_agent_config(configs: Configs) -> Option> { fn convert_to_config_hashmap(contents: Vec) -> Option> { let mut contents_tmp: HashMap = HashMap::new(); for content in contents.into_iter() { - let key_info = KeyInfo { - value: content.value.unwrap_or_default(), - operation: content.operation.unwrap_or_default(), - }; + let key_info = + KeyInfo { value: content.value.unwrap_or_default(), operation: content.operation.unwrap_or_default() }; contents_tmp.insert(content.key.unwrap_or_default(), key_info); } return Some(contents_tmp); } pub mod reconciler_error { - use crate::controller::agentclient::agent_error; - use crate::controller::apiclient::apiclient_error; use thiserror::Error; + + use crate::controller::{agentclient::agent_error, apiclient::apiclient_error}; #[derive(Error, Debug)] pub enum Error { #[error("Kubernetes reported error: {source}")] diff --git a/KubeOS-Rust/proxy/src/controller/crd.rs b/KubeOS-Rust/proxy/src/controller/crd.rs index 9f01a964..efec0bd8 100644 --- a/KubeOS-Rust/proxy/src/controller/crd.rs +++ b/KubeOS-Rust/proxy/src/controller/crd.rs @@ -14,14 +14,7 @@ use kube::CustomResource; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; #[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[kube( - group = "upgrade.openeuler.org", - version = "v1alpha1", - kind = "OS", - plural = "os", - singular = "os", - namespaced -)] +#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)] pub struct OSSpec { pub osversion: String, pub maxunavailable: i64, diff --git a/KubeOS-Rust/proxy/src/controller/drain.rs b/KubeOS-Rust/proxy/src/controller/drain.rs index f03284b9..ddc38aed 100644 --- a/KubeOS-Rust/proxy/src/controller/drain.rs +++ b/KubeOS-Rust/proxy/src/controller/drain.rs @@ -10,14 +10,6 @@ * See the Mulan PSL v2 for more details. */ -use self::error::{ - DrainError::{DeletePodsError, GetPodListsError, WaitDeletionError}, - EvictionError::{EvictionErrorNoRetry, EvictionErrorRetry}, -}; -use super::values::{ - EVERY_DELETION_CHECK, EVERY_EVICTION_RETRY, MAX_EVICT_POD_NUM, MAX_RETRIES_TIMES, - RETRY_BASE_DELAY, RETRY_MAX_DELAY, TIMEOUT, -}; use futures::{stream, StreamExt}; use k8s_openapi::api::core::v1::{Pod, PodSpec, PodStatus}; use kube::{ @@ -33,11 +25,16 @@ use tokio_retry::{ RetryIf, }; -pub async fn drain_os( - client: &Client, - node_name: &str, - force: bool, -) -> Result<(), error::DrainError> { +use self::error::{ + DrainError::{DeletePodsError, GetPodListsError, WaitDeletionError}, + EvictionError::{EvictionErrorNoRetry, EvictionErrorRetry}, +}; +use super::values::{ + EVERY_DELETION_CHECK, EVERY_EVICTION_RETRY, MAX_EVICT_POD_NUM, MAX_RETRIES_TIMES, RETRY_BASE_DELAY, + RETRY_MAX_DELAY, TIMEOUT, +}; + +pub async fn drain_os(client: &Client, node_name: &str, force: bool) -> Result<(), error::DrainError> { let pods_list = get_pods_deleted(client, node_name, force).await?; stream::iter(pods_list) @@ -59,19 +56,13 @@ async fn get_pods_deleted( node_name: &str, force: bool, ) -> Result, error::DrainError> { - let lp = ListParams { - field_selector: Some(format!("spec.nodeName={}", node_name)), - ..Default::default() - }; + let lp = ListParams { field_selector: Some(format!("spec.nodeName={}", node_name)), ..Default::default() }; let pods_api: Api = Api::all(client.clone()); let pods: ObjectList = match pods_api.list(&lp).await { Ok(pods @ ObjectList { .. }) => pods, Err(err) => { - return Err(GetPodListsError { - source: err, - node_name: node_name.to_string(), - }); - } + return Err(GetPodListsError { source: err, node_name: node_name.to_string() }); + }, }; let mut filterd_pods_list: Vec = Vec::new(); let mut filterd_err: Vec = Vec::new(); @@ -87,25 +78,16 @@ async fn get_pods_deleted( } } if filterd_err.len() > 0 { - return Err(DeletePodsError { - errors: filterd_err, - }); + return Err(DeletePodsError { errors: filterd_err }); } Ok(filterd_pods_list.into_iter()) } -async fn evict_pod( - k8s_client: &kube::Client, - pod: &Pod, - force: bool, -) -> Result<(), error::EvictionError> { +async fn evict_pod(k8s_client: &kube::Client, pod: &Pod, force: bool) -> Result<(), error::EvictionError> { let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); - let error_handling_strategy = if force { - ErrorHandleStrategy::RetryStrategy - } else { - ErrorHandleStrategy::TolerateStrategy - }; + let error_handling_strategy = + if force { ErrorHandleStrategy::RetryStrategy } else { ErrorHandleStrategy::TolerateStrategy }; RetryIf::spawn( error_handling_strategy.retry_strategy(), @@ -203,18 +185,14 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e let name = (&p).name_any(); info!("Pod {} deleted.", name); break; - } + }, Ok(_) => { - info!( - "Pod '{}' is not yet deleted. Waiting {}s.", - pod.name_any(), - EVERY_DELETION_CHECK.as_secs_f64() - ); - } + info!("Pod '{}' is not yet deleted. Waiting {}s.", pod.name_any(), EVERY_DELETION_CHECK.as_secs_f64()); + }, Err(kube::Error::Api(e)) if e.code == response_error_not_found => { info!("Pod {} is deleted.", pod.name_any()); break; - } + }, Err(e) => { error!( "Get pod {} reported error: '{}', whether pod is deleted cannot be determined, waiting {}s.", @@ -222,13 +200,10 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e e, EVERY_DELETION_CHECK.as_secs_f64() ); - } + }, } if start_time.elapsed() > TIMEOUT { - return Err(WaitDeletionError { - pod_name: pod.name_any(), - max_wait: TIMEOUT, - }); + return Err(WaitDeletionError { pod_name: pod.name_any(), max_wait: TIMEOUT }); } else { sleep(EVERY_DELETION_CHECK).await; } @@ -249,11 +224,7 @@ trait NameAny { impl NameAny for &Pod { fn name_any(self: &Self) -> String { - self.metadata - .name - .clone() - .or_else(|| self.metadata.generate_name.clone()) - .unwrap_or_default() + self.metadata.name.clone().or_else(|| self.metadata.generate_name.clone()).unwrap_or_default() } } trait PodFilter { @@ -264,11 +235,9 @@ struct FinishedOrFailedFilter {} impl PodFilter for FinishedOrFailedFilter { fn filter(self: &Self, pod: &Pod) -> Box { return match pod.status.as_ref() { - Some(PodStatus { - phase: Some(phase), .. - }) if phase == "Failed" || phase == "Succeeded" => { + Some(PodStatus { phase: Some(phase), .. }) if phase == "Failed" || phase == "Succeeded" => { FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) - } + }, _ => FilterResult::create_filter_result(false, "", PodDeleteStatus::Okay), }; } @@ -279,50 +248,31 @@ struct DaemonFilter { } impl PodFilter for DaemonFilter { fn filter(self: &Self, pod: &Pod) -> Box { - if let FilterResult { result: true, .. } = - self.finished_or_failed_filter.filter(pod).as_ref() - { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); } return match pod.metadata.owner_references.as_ref() { Some(owner_references) - if owner_references.iter().any(|reference| { - reference.controller.unwrap_or(false) && reference.kind == "DaemonSet" - }) => + if owner_references + .iter() + .any(|reference| reference.controller.unwrap_or(false) && reference.kind == "DaemonSet") => { if self.force { - let description = format!( - "Ignore Pod '{}': Pod is member of a DaemonSet", - pod.name_any() - ); - Box::new(FilterResult { - result: false, - desc: description, - status: PodDeleteStatus::Warning, - }) + let description = format!("Ignore Pod '{}': Pod is member of a DaemonSet", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Warning }) } else { - let description = format!( - "Cannot drain Pod '{}': Pod is member of a DaemonSet", - pod.name_any() - ); - Box::new(FilterResult { - result: false, - desc: description, - status: PodDeleteStatus::Error, - }) + let description = format!("Cannot drain Pod '{}': Pod is member of a DaemonSet", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) } - } + }, _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } } impl DaemonFilter { fn new(force: bool) -> DaemonFilter { - return DaemonFilter { - finished_or_failed_filter: FinishedOrFailedFilter {}, - force: force, - }; + return DaemonFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force: force }; } } @@ -331,16 +281,9 @@ impl PodFilter for MirrorFilter { fn filter(self: &Self, pod: &Pod) -> Box { return match pod.metadata.annotations.as_ref() { Some(annotations) if annotations.contains_key("kubernetes.io/config.mirror") => { - let description = format!( - "Ignore Pod '{}': Pod is a static Mirror Pod", - pod.name_any() - ); - FilterResult::create_filter_result( - false, - &description.to_string(), - PodDeleteStatus::Warning, - ) - } + let description = format!("Ignore Pod '{}': Pod is a static Mirror Pod", pod.name_any()); + FilterResult::create_filter_result(false, &description.to_string(), PodDeleteStatus::Warning) + }, _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } @@ -352,49 +295,27 @@ struct LocalStorageFilter { } impl PodFilter for LocalStorageFilter { fn filter(self: &Self, pod: &Pod) -> Box { - if let FilterResult { result: true, .. } = - self.finished_or_failed_filter.filter(pod).as_ref() - { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); } return match pod.spec.as_ref() { - Some(PodSpec { - volumes: Some(volumes), - .. - }) if volumes.iter().any(|volume| volume.empty_dir.is_some()) => { + Some(PodSpec { volumes: Some(volumes), .. }) if volumes.iter().any(|volume| volume.empty_dir.is_some()) => { if self.force { - let description = format!( - "Force draining Pod '{}': Pod has local storage", - pod.name_any() - ); - Box::new(FilterResult { - result: true, - desc: description, - status: PodDeleteStatus::Warning, - }) + let description = format!("Force draining Pod '{}': Pod has local storage", pod.name_any()); + Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning }) } else { - let description = format!( - "Cannot drain Pod '{}': Pod has local Storage", - pod.name_any() - ); - Box::new(FilterResult { - result: false, - desc: description, - status: PodDeleteStatus::Error, - }) + let description = format!("Cannot drain Pod '{}': Pod has local Storage", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) } - } + }, _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } } impl LocalStorageFilter { fn new(force: bool) -> LocalStorageFilter { - return LocalStorageFilter { - finished_or_failed_filter: FinishedOrFailedFilter {}, - force: force, - }; + return LocalStorageFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force: force }; } } struct UnreplicatedFilter { @@ -403,9 +324,7 @@ struct UnreplicatedFilter { } impl PodFilter for UnreplicatedFilter { fn filter(self: &Self, pod: &Pod) -> Box { - if let FilterResult { result: true, .. } = - self.finished_or_failed_filter.filter(pod).as_ref() - { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); } @@ -417,27 +336,16 @@ impl PodFilter for UnreplicatedFilter { return if !is_replicated && self.force { let description = format!("Force drain Pod '{}': Pod is unreplicated", pod.name_any()); - Box::new(FilterResult { - result: true, - desc: description, - status: PodDeleteStatus::Warning, - }) + Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning }) } else { let description = format!("Cannot drain Pod '{}': Pod is unreplicated", pod.name_any()); - Box::new(FilterResult { - result: false, - desc: description, - status: PodDeleteStatus::Error, - }) + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) }; } } impl UnreplicatedFilter { fn new(force: bool) -> UnreplicatedFilter { - return UnreplicatedFilter { - finished_or_failed_filter: FinishedOrFailedFilter {}, - force: force, - }; + return UnreplicatedFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force: force }; } } @@ -450,11 +358,10 @@ impl PodFilter for DeletedFilter { return match pod.metadata.deletion_timestamp.as_ref() { Some(time) if time.0.timestamp() != 0 - && now - Duration::from_secs(time.0.timestamp() as u64) - >= self.delete_wait_timeout => + && now - Duration::from_secs(time.0.timestamp() as u64) >= self.delete_wait_timeout => { FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) - } + }, _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } @@ -521,9 +428,7 @@ impl PodFilter for CombinedFilter { impl CombinedFilter { fn new(force: bool) -> CombinedFilter { return CombinedFilter { - deleted_filter: DeletedFilter { - delete_wait_timeout: TIMEOUT, - }, + deleted_filter: DeletedFilter { delete_wait_timeout: TIMEOUT }, daemon_filter: DaemonFilter::new(force), mirror_filter: MirrorFilter {}, local_storage_filter: LocalStorageFilter::new(force), @@ -544,16 +449,8 @@ struct FilterResult { status: PodDeleteStatus, } impl FilterResult { - fn create_filter_result( - result: bool, - desc: &str, - status: PodDeleteStatus, - ) -> Box { - Box::new(FilterResult { - result: result, - desc: desc.to_string(), - status: status, - }) + fn create_filter_result(result: bool, desc: &str, status: PodDeleteStatus) -> Box { + Box::new(FilterResult { result: result, desc: desc.to_string(), status: status }) } } @@ -564,14 +461,13 @@ enum ErrorHandleStrategy { impl ErrorHandleStrategy { fn retry_strategy(&self) -> impl Iterator { - let backoff = ExponentialBackoff::from_millis(RETRY_BASE_DELAY.as_millis() as u64) - .max_delay(RETRY_MAX_DELAY) - .map(jitter); + let backoff = + ExponentialBackoff::from_millis(RETRY_BASE_DELAY.as_millis() as u64).max_delay(RETRY_MAX_DELAY).map(jitter); return match self { Self::TolerateStrategy => { return backoff.take(0); - } + }, Self::RetryStrategy => backoff.take(MAX_RETRIES_TIMES), }; @@ -588,7 +484,7 @@ impl tokio_retry::Condition for ErrorHandleStrategy { } else { false } - } + }, } } } @@ -600,16 +496,10 @@ pub mod error { #[derive(Debug, Error)] pub enum DrainError { #[error("Get node {} pods list error reported: {}", node_name, source)] - GetPodListsError { - source: kube::Error, - node_name: String, - }, + GetPodListsError { source: kube::Error, node_name: String }, #[error("Pod '{}' was not deleted in the time allocated ({:.2}s).",pod_name,max_wait.as_secs_f64())] - WaitDeletionError { - pod_name: String, - max_wait: Duration, - }, + WaitDeletionError { pod_name: String, max_wait: Duration }, #[error("")] DeletePodsError { errors: Vec }, } @@ -617,15 +507,9 @@ pub mod error { #[derive(Debug, Error)] pub enum EvictionError { #[error("Evict Pod {} error: '{}'", pod_name, source)] - EvictionErrorRetry { - source: kube::Error, - pod_name: String, - }, + EvictionErrorRetry { source: kube::Error, pod_name: String }, #[error("Evict Pod {} error: '{}'", pod_name, source)] - EvictionErrorNoRetry { - source: kube::Error, - pod_name: String, - }, + EvictionErrorNoRetry { source: kube::Error, pod_name: String }, } } diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs index 26a0d18a..78502db0 100644 --- a/KubeOS-Rust/proxy/src/controller/utils.rs +++ b/KubeOS-Rust/proxy/src/controller/utils.rs @@ -10,10 +10,13 @@ * See the Mulan PSL v2 for more details. */ -use super::crd::{Configs, OSInstance, OSInstanceStatus, OS}; -use super::values::{NODE_STATUS_CONFIG, NODE_STATUS_IDLE, NODE_STATUS_UPGRADE}; use log::{debug, info}; +use super::{ + crd::{Configs, OSInstance, OSInstanceStatus, OS}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_IDLE, NODE_STATUS_UPGRADE}, +}; + #[derive(PartialEq, Clone, Copy)] pub enum ConfigType { UpgradeConfig, @@ -42,14 +45,18 @@ impl ConfigType { match self { ConfigType::UpgradeConfig => { let os_config_version = get_config_version(os.spec.upgradeconfigs.as_ref()); - let osi_config_version = - get_config_version(osinstance.spec.upgradeconfigs.as_ref()); - debug!("=======os upgradeconfig version is{},osinstance spec upragdeconfig version is{}",os_config_version,osi_config_version); + let osi_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + debug!( + "=======os upgradeconfig version is{},osinstance spec upragdeconfig version is{}", + os_config_version, osi_config_version + ); if !check_version(&os_config_version, &osi_config_version) { - info!("os.spec.upgradeconfig.version is not equal to oninstance.spec.upragdeconfig.version, operation: reassgin upgrade to get newest upgradeconfigs"); + info!( + "os.spec.upgradeconfig.version is not equal to oninstance.spec.upragdeconfig.version, operation: reassgin upgrade to get newest upgradeconfigs" + ); return ConfigOperation::Reassign; } - } + }, ConfigType::SysConfig => { let os_config_version = get_config_version(os.spec.sysconfigs.as_ref()); let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); @@ -59,15 +66,19 @@ impl ConfigType { ); if !check_version(&os_config_version, &osi_config_version) { if node_status == NODE_STATUS_CONFIG { - info!("os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: reassgin config to get newest sysconfigs"); + info!( + "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: reassgin config to get newest sysconfigs" + ); return ConfigOperation::Reassign; } if node_status == NODE_STATUS_UPGRADE { - info!("os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: update osinstance.spec.sysconfig and reconcile"); + info!( + "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: update osinstance.spec.sysconfig and reconcile" + ); return ConfigOperation::UpdateConfig; } } - } + }, }; ConfigOperation::DoNothing } @@ -80,40 +91,30 @@ impl ConfigType { ConfigType::UpgradeConfig => { spec_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); if let Some(osinstance_status) = osinstance.status.as_ref() { - status_config_version = - get_config_version(osinstance_status.upgradeconfigs.as_ref()); + status_config_version = get_config_version(osinstance_status.upgradeconfigs.as_ref()); } else { status_config_version = get_config_version(None); } configs = osinstance.spec.upgradeconfigs.clone(); - } + }, ConfigType::SysConfig => { spec_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); if let Some(osinstance_status) = osinstance.status.as_ref() { - status_config_version = - get_config_version(osinstance_status.sysconfigs.as_ref()); + status_config_version = get_config_version(osinstance_status.sysconfigs.as_ref()); } else { status_config_version = get_config_version(None); } configs = osinstance.spec.sysconfigs.clone(); - } + }, } debug!( "=======osinstance soec config version is {},status config version is {}", spec_config_version, status_config_version ); - if spec_config_version != status_config_version - && osinstance.spec.nodestatus != NODE_STATUS_IDLE - { - return ConfigInfo { - need_config: true, - configs: configs, - }; + if spec_config_version != status_config_version && osinstance.spec.nodestatus != NODE_STATUS_IDLE { + return ConfigInfo { need_config: true, configs: configs }; } - return ConfigInfo { - need_config: false, - configs: None, - }; + return ConfigInfo { need_config: false, configs: None }; } pub fn set_osi_status_config(&self, osinstance: &mut OSInstance) { match self { @@ -126,17 +127,15 @@ impl ConfigType { sysconfigs: None, }) } - } + }, ConfigType::SysConfig => { if let Some(osi_status) = &mut osinstance.status { osi_status.sysconfigs = osinstance.spec.sysconfigs.clone(); } else { - osinstance.status = Some(OSInstanceStatus { - upgradeconfigs: None, - sysconfigs: osinstance.spec.sysconfigs.clone(), - }) + osinstance.status = + Some(OSInstanceStatus { upgradeconfigs: None, sysconfigs: osinstance.spec.sysconfigs.clone() }) } - } + }, } } } diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs index f5d41965..fe43851f 100644 --- a/KubeOS-Rust/proxy/src/controller/values.rs +++ b/KubeOS-Rust/proxy/src/controller/values.rs @@ -28,13 +28,9 @@ pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; -pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { - requeue_after: Some(Duration::from_secs(15)), -}; +pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; -pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { - requeue_after: Some(Duration::from_secs(1)), -}; +pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; pub const MAX_EVICT_POD_NUM: usize = 5; diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs index ef53117d..cd601d0f 100644 --- a/KubeOS-Rust/proxy/src/main.rs +++ b/KubeOS-Rust/proxy/src/main.rs @@ -20,30 +20,23 @@ use kube::{ }; use log::{error, info}; mod controller; -use controller::{ - error_policy, reconcile, AgentClient, ControllerClient, ProxyController, OS, SOCK_PATH, -}; +use controller::{error_policy, reconcile, AgentClient, ControllerClient, ProxyController, OS, SOCK_PATH}; const PROXY_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); #[tokio::main] async fn main() -> Result<()> { - Builder::from_env(Env::default().default_filter_or("info")) - .target(Target::Stdout) - .init(); + Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init(); let client = Client::try_default().await?; let os: Api = Api::all(client.clone()); let controller_client = ControllerClient::new(client.clone()); let agent_client = AgentClient::new(SOCK_PATH); let proxy_controller = ProxyController::new(client, controller_client, agent_client); - info!( - "os-proxy version is {}, start renconcile", - PROXY_VERSION.unwrap_or("Not Found") - ); + info!("os-proxy version is {}, start renconcile", PROXY_VERSION.unwrap_or("Not Found")); Controller::new(os, ListParams::default()) .run(reconcile, error_policy, Context::new(proxy_controller)) .for_each(|res| async move { match res { - Ok(_o) => {} + Ok(_o) => {}, Err(e) => error!("reconcile failed: {}", e.to_string()), } }) diff --git a/KubeOS-Rust/rustfmt.toml b/KubeOS-Rust/rustfmt.toml new file mode 100644 index 00000000..3c565cf5 --- /dev/null +++ b/KubeOS-Rust/rustfmt.toml @@ -0,0 +1,11 @@ +# cargo +nightly fmt +version = "Two" +use_small_heuristics = "MAX" +match_block_trailing_comma = true +newline_style = "Unix" +merge_derives = false +max_width = 120 +group_imports = "StdExternalCrate" +imports_granularity = "Crate" +reorder_imports = true +unstable_features = true \ No newline at end of file -- Gitee From 3b94b0e8a23cabfa6314f1e97bca303257aade17 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 8 Jan 2024 15:28:17 +0800 Subject: [PATCH 08/46] test(os-agent): add docker and disk unit tests Signed-off-by: Yuhang Wei --- KubeOS-Rust/Cargo.lock | 45 +++++ KubeOS-Rust/Cargo.toml | 1 + KubeOS-Rust/agent/src/function.rs | 23 +++ KubeOS-Rust/manager/Cargo.toml | 5 +- KubeOS-Rust/manager/src/api/agent_status.rs | 22 -- .../manager/src/sys_mgmt/containerd_image.rs | 5 +- .../manager/src/sys_mgmt/disk_image.rs | 191 +++++++++++++----- .../manager/src/sys_mgmt/docker_image.rs | 109 +++++++++- KubeOS-Rust/manager/src/utils/common.rs | 21 +- .../manager/src/utils/image_manager.rs | 1 + 10 files changed, 337 insertions(+), 86 deletions(-) diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index bc0b38b1..d9300f60 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -32,6 +32,16 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-trait" version = "0.1.74" @@ -167,6 +177,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -1067,6 +1087,7 @@ dependencies = [ "lazy_static", "log", "mockall", + "mockito", "nix", "predicates", "regex", @@ -1147,6 +1168,24 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "mockito" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80f9fece9bd97ab74339fe19f4bcaf52b76dcc18e5364c7977c1838f76b38de9" +dependencies = [ + "assert-json-diff", + "colored", + "httparse", + "lazy_static", + "log", + "rand 0.8.5", + "regex", + "serde_json", + "serde_urlencoded", + "similar", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1883,6 +1922,12 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" + [[package]] name = "slab" version = "0.4.9" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml index 135028f2..68ee670a 100644 --- a/KubeOS-Rust/Cargo.toml +++ b/KubeOS-Rust/Cargo.toml @@ -8,4 +8,5 @@ debug-assertions = false lto = true opt-level = 's' overflow-checks = false +panic = "unwind" rpath = false diff --git a/KubeOS-Rust/agent/src/function.rs b/KubeOS-Rust/agent/src/function.rs index 89775dce..2c973474 100644 --- a/KubeOS-Rust/agent/src/function.rs +++ b/KubeOS-Rust/agent/src/function.rs @@ -30,3 +30,26 @@ impl RpcFunction { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rpcfunction_call() { + // Define a mock function that returns a result + fn mock_ok_function() -> anyhow::Result { + Ok(42) + } + let result: RpcResult = RpcFunction::call(mock_ok_function); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), 42); + + fn mock_err_function() -> anyhow::Result { + Err(anyhow::anyhow!("error")) + } + let result: RpcResult = RpcFunction::call(mock_err_function); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code, ErrorCode::ServerError(RPC_OP_ERROR)); + } +} diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml index 29ec2f46..40672ccc 100644 --- a/KubeOS-Rust/manager/Cargo.toml +++ b/KubeOS-Rust/manager/Cargo.toml @@ -8,8 +8,9 @@ version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dev-dependencies] mockall = { version = "=0.11.3" } -predicates = "=2.0.1" -tempfile = "3.6.0" +mockito = { version = "0.31.1" } +predicates = { version = "=2.0.1" } +tempfile = { version = "3.6.0" } [dependencies] anyhow = { version = "1.0" } diff --git a/KubeOS-Rust/manager/src/api/agent_status.rs b/KubeOS-Rust/manager/src/api/agent_status.rs index c3152257..e466a50c 100644 --- a/KubeOS-Rust/manager/src/api/agent_status.rs +++ b/KubeOS-Rust/manager/src/api/agent_status.rs @@ -12,14 +12,6 @@ use serde::{Deserialize, Serialize}; -const AGENT_STATUS_UNKNOWN: &str = "UNKNOWN"; -const AGENT_STATUS_NOT_APPLIED: &str = "NOT-APPLIED"; -const AGENT_STATUS_UPGRADEREADY: &str = "UPGRADE-READY"; -const AGENT_STATUS_UPGRADED: &str = "UPGRADED"; -const AGENT_STATUS_ROLLBACKED: &str = "ROLLBACKED"; -const AGENT_STATUS_CONFIGURED: &str = "CONFIGURED"; -const AGENT_STATUS_CLEANEDUP: &str = "CLEANEDUP"; - #[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] pub enum AgentStatus { Unknown, @@ -36,17 +28,3 @@ impl Default for AgentStatus { Self::Unknown } } - -impl std::fmt::Display for AgentStatus { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(match self { - AgentStatus::Unknown => AGENT_STATUS_UNKNOWN, - AgentStatus::NotApplied => AGENT_STATUS_NOT_APPLIED, - AgentStatus::UpgradeReady => AGENT_STATUS_UPGRADEREADY, - AgentStatus::Upgraded => AGENT_STATUS_UPGRADED, - AgentStatus::Rollbacked => AGENT_STATUS_ROLLBACKED, - AgentStatus::Configured => AGENT_STATUS_CONFIGURED, - AgentStatus::CleanedUp => AGENT_STATUS_CLEANEDUP, - }) - } -} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs index 6ed3623a..0b50ad65 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -17,7 +17,7 @@ use log::{debug, info}; use crate::{ api::{ImageHandler, UpgradeRequest}, - sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES, PERSIST_DIR}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES}, utils::*, }; @@ -30,7 +30,7 @@ const DEFAULT_NAMESPACE: &str = "k8s.io"; impl ImageHandler for CtrImageHandler { fn download_image(&self, req: &UpgradeRequest) -> Result> { - perpare_env(&self.paths, NEED_BYTES, PERSIST_DIR, IMAGE_PERMISSION)?; + perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?; self.get_image(req)?; self.get_rootfs_archive(req, IMAGE_PERMISSION)?; @@ -228,6 +228,7 @@ mod tests { let dst_path = dst_file.path().to_path_buf(); let paths = PreparePath { + persist_path: "/tmp".into(), update_path: PathBuf::new(), image_path: PathBuf::new(), mount_path: src_dir.to_path_buf(), diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs index 5c4190ab..4ccb6033 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs @@ -153,6 +153,10 @@ impl DiskImageHandler { #[cfg(test)] mod tests { + use std::io::Write; + + use mockall::mock; + use mockito; use tempfile::NamedTempFile; use super::*; @@ -164,6 +168,16 @@ mod tests { .is_test(true) .try_init(); } + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } #[test] fn test_get_certs_path() { @@ -183,16 +197,43 @@ mod tests { let res = handler.cert_exist(tmp_file.path().to_str().unwrap()); assert!(res.is_ok()); - assert!(handler.cert_exist("aaa.pem").is_err()) + assert!(handler.cert_exist("aaa.pem").is_err()); + assert!(handler.cert_exist("").is_err()) } #[test] - #[ignore] fn test_send_download_request() { init(); + // This is a tmp cert only for KubeOS unit testing. + let tmp_cert = "-----BEGIN CERTIFICATE-----\n\ + MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\ + MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\ + MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\ + bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\ + Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\ + HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\ + c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\ + vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\ + -----END CERTIFICATE-----\n"; + + // This is a tmp private key only for KubeOS unit testing. + let tmp_key = "-----BEGIN PRIVATE KEY-----\n\ + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\ + Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\ + nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\ + -----END PRIVATE KEY-----\n"; + + // Create a temporary file to hold the certificate + let mut cert_file = NamedTempFile::new().unwrap(); + cert_file.write_all(tmp_cert.as_bytes()).unwrap(); + println!("cert_file: {:?}", cert_file.path().to_str().unwrap()); + + // Create a temporary file to hold the private key + let mut key_file = NamedTempFile::new().unwrap(); + key_file.write_all(tmp_key.as_bytes()).unwrap(); // http let handler = DiskImageHandler::::default(); - let req = UpgradeRequest { + let mut req = UpgradeRequest { version: "v2".into(), check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), image_type: "disk".into(), @@ -203,82 +244,65 @@ mod tests { certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; let res = handler.send_download_request(&req); - assert!(res.is_ok()); - assert_eq!(res.unwrap().text().unwrap(), "This is a test txt file generated by yuhang wei\n"); + assert!(res.is_err()); + req.flag_safe = false; + let res = handler.send_download_request(&req); + assert!(res.is_err()); // https let mut handler = DiskImageHandler::::default(); - handler.certs_path = "/home/yuhang/Documents/data/https-nginx/nginx/certs/".to_string(); + handler.certs_path = "/tmp".to_string(); + let tmp_cert_filename = cert_file.path().file_name().unwrap().to_str().unwrap(); let req = UpgradeRequest { version: "v2".into(), check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), image_type: "disk".into(), container_image: "".into(), - image_url: "https://7.250.142.47:8081/aaa.txt".to_string(), + image_url: "https://localhost:8081/aaa.txt".to_string(), flag_safe: true, mtls: false, certs: CertsInfo { - ca_cert: "nginx.crt".to_string(), + ca_cert: tmp_cert_filename.to_string(), client_cert: "".to_string(), client_key: "".to_string(), }, }; let res = handler.send_download_request(&req); - assert!(res.is_ok()); - assert_eq!(res.unwrap().text().unwrap(), "This is a test txt file generated by yuhang wei\n"); + assert!(res.is_err()); // mtls + let tmp_key = NamedTempFile::new().unwrap(); + let tmp_key_filename = tmp_key.path().file_name().unwrap().to_str().unwrap(); let mut handler = DiskImageHandler::::default(); - handler.certs_path = "/home/yuhang/Documents/data/cert/".to_string(); + handler.certs_path = "/tmp".to_string(); let req = UpgradeRequest { version: "v2".into(), check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), image_type: "disk".into(), container_image: "".into(), - image_url: "https://7.250.142.47:8082/aaa.txt".to_string(), + image_url: "https://localhost:8082/aaa.txt".to_string(), flag_safe: true, mtls: true, certs: CertsInfo { - ca_cert: "nginx.crt".to_string(), - client_cert: "client.crt".to_string(), - client_key: "client.key".to_string(), + ca_cert: tmp_cert_filename.to_string(), + client_cert: tmp_cert_filename.to_string(), + client_key: tmp_key_filename.to_string(), }, }; let res = handler.send_download_request(&req); - assert!(res.is_ok()); - assert_eq!(res.unwrap().text().unwrap(), "This is a test txt file generated by yuhang wei\n"); - } - - #[test] - #[ignore] - fn test_download() { - init(); - let mut handler = DiskImageHandler::::default(); - handler.paths.image_path = PathBuf::from("/home/yuhang/Documents/KubeOS/KubeOS-Rust/test_download_image"); - let req = UpgradeRequest { - version: "v2".into(), - check_sum: "90da5a14e9c06ddb276b06134e90d37098be2830beaa4357205bec7ff1aa1f7c".into(), - image_type: "disk".into(), - container_image: "".into(), - image_url: "http://localhost:8080/linux-firmware.rpm".to_string(), - flag_safe: true, - mtls: false, - certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, - }; - let res = handler.download(&req); - assert!(res.is_ok()); - assert_eq!(true, handler.paths.image_path.exists()) + assert!(res.is_err()); } #[test] - #[ignore] fn test_checksum_match() { init(); + let mut tmp_file = NamedTempFile::new().unwrap(); + tmp_file.write(b"This is a test txt file for KubeOS test.\n").unwrap(); let mut handler = DiskImageHandler::::default(); - handler.paths.image_path = PathBuf::from("/home/yuhang/Documents/KubeOS/KubeOS-Rust/test_download_image"); - let req = UpgradeRequest { + handler.paths.image_path = tmp_file.path().to_path_buf(); + let mut req = UpgradeRequest { version: "v2".into(), - check_sum: "90da5a14e9c06ddb276b06134e90d37098be2830beaa4357205bec7ff1aa1f7c".into(), + check_sum: "98ea7aff44631d183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), image_type: "disk".into(), container_image: "".into(), image_url: "http://localhost:8080/aaa.txt".to_string(), @@ -288,20 +312,91 @@ mod tests { }; assert_eq!(handler.paths.image_path.exists(), true); handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum).unwrap(); + + req.check_sum = "1234567890".into(); + let res = handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum); + assert!(res.is_err()); } #[test] - #[ignore] - fn test_load_ca_client_certs() { + fn test_load_certs() { init(); + // This is a tmp cert only for KubeOS unit testing. + let tmp_cert = "-----BEGIN CERTIFICATE-----\n\ + MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\ + MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\ + MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\ + bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\ + Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\ + HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\ + c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\ + vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\ + -----END CERTIFICATE-----\n"; + + // This is a tmp private key only for KubeOS unit testing. + let tmp_key = "-----BEGIN PRIVATE KEY-----\n\ + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\ + Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\ + nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\ + -----END PRIVATE KEY-----\n"; + + // Create a temporary file to hold the certificate + let mut cert_file = NamedTempFile::new().unwrap(); + cert_file.write_all(tmp_cert.as_bytes()).unwrap(); + + // Create a temporary file to hold the private key + let mut key_file = NamedTempFile::new().unwrap(); + key_file.write_all(tmp_key.as_bytes()).unwrap(); + let mut handler = DiskImageHandler::::default(); - handler.certs_path = "/home/yuhang/Documents/data/cert/".to_string(); + handler.certs_path = "".to_string(); let certs = CertsInfo { - ca_cert: "nginx.crt".to_string(), - client_cert: "client.crt".to_string(), - client_key: "client.key".to_string(), + ca_cert: cert_file.path().to_str().unwrap().to_string(), + client_cert: cert_file.path().to_str().unwrap().to_string(), + client_key: key_file.path().to_str().unwrap().to_string(), }; + let res = handler.load_ca_client_certs(&certs); assert!(res.is_ok()); + + let res = handler.load_ca_certs(&certs.ca_cert); + assert!(res.is_ok()); + } + + #[test] + fn test_download_image() { + init(); + let tmp_file = NamedTempFile::new().unwrap(); + + let mock_executor = MockCommandExec::new(); + let mut handler = DiskImageHandler::new(PreparePath::default(), mock_executor, String::new()); + handler.executor.expect_clone().times(1).returning(|| MockCommandExec::new()); + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + handler.executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + handler.paths.image_path = tmp_file.path().to_path_buf(); + assert_eq!(true, handler.paths.image_path.exists()); + + let url = mockito::server_url(); + let upgrade_request = UpgradeRequest { + version: "v2".into(), + check_sum: "98ea7aff44631d183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: format!("{}/test.txt", url), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let _m = mockito::mock("GET", "/test.txt") + .with_status(200) + .with_body("This is a test txt file for KubeOS test.\n") + .create(); + handler.download_image(&upgrade_request).unwrap(); + + assert_eq!(true, handler.paths.image_path.exists()); + assert_eq!( + fs::read(handler.paths.image_path.to_str().unwrap()).unwrap(), + "This is a test txt file for KubeOS test.\n".as_bytes() + ); } } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs index 74d3649a..121e2577 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -1,9 +1,9 @@ use anyhow::Result; -use log::{debug, info}; +use log::{debug, info, trace}; use crate::{ api::{ImageHandler, UpgradeRequest}, - sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES, PERSIST_DIR}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES}, utils::*, }; @@ -15,7 +15,7 @@ pub struct DockerImageHandler { impl ImageHandler for DockerImageHandler { fn download_image(&self, req: &UpgradeRequest) -> Result> { - perpare_env(&self.paths, NEED_BYTES, PERSIST_DIR, IMAGE_PERMISSION)?; + perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?; self.get_image(req)?; self.get_rootfs_archive(req)?; @@ -70,6 +70,7 @@ impl DockerImageHandler { } fn check_and_rm_container(&self) -> Result<()> { + trace!("Check and remove container {}", self.container_name); let docker_ps_cmd = format!("docker ps -a -f=name={} | awk 'NR==2' | awk '{{print $1}}'", self.container_name); let exist_id = self.executor.run_command_with_output("bash", &["-c", &docker_ps_cmd])?; if !exist_id.is_empty() { @@ -85,6 +86,7 @@ mod tests { use mockall::mock; use super::*; + use crate::api::CertsInfo; mock! { pub CommandExec{} @@ -128,4 +130,105 @@ mod tests { DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor).check_and_rm_container(); assert!(result.is_ok()); } + + #[test] + fn test_get_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "docker".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + // mock remove_image_if_exist + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rmi")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock pull_image + mock_executor + .expect_run_command() + .withf(|cmd, args| { + cmd == "docker" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest") + }) + .times(1) + .returning(|_, _| Ok(())); + // mock get_oci_image_digest + let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]"; + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect") && args.contains(&"{{.RepoDigests}}")) + .times(1) + .returning(|_, _| Ok(command_output2.to_string())); + + let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor); + let result = docker.get_image(&req); + assert!(result.is_ok()); + } + + #[test] + fn test_get_rootfs_archive() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "docker".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + // mock check_and_rm_container + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'") + }) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::new())); + // mock get_rootfs_archive + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "docker" && args.contains(&"create")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"cp") && args.contains(&"1111:/os.tar")) + .times(1) + .returning(|_, _| Ok(())); + // mock check_and_rm_container + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'") + }) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111")) + .times(1) + .returning(|_, _| Ok(())); + + let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor); + let result = docker.get_rootfs_archive(&req); + assert!(result.is_ok()); + } } diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs index 0fe94b7c..301a8c86 100644 --- a/KubeOS-Rust/manager/src/utils/common.rs +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -25,18 +25,20 @@ use crate::sys_mgmt::{MOUNT_DIR, OS_IMAGE_NAME, PERSIST_DIR, ROOTFS_ARCHIVE, UPD #[derive(Clone)] pub struct PreparePath { - pub update_path: PathBuf, // update_path: /persist/KubeOS-Update - pub mount_path: PathBuf, // mount_path: /persist/KubeOS-Update/kubeos-update - pub tar_path: PathBuf, // tar_path: /persist/KubeOS-Update/os.tar - pub image_path: PathBuf, // image_path: /persist/update.img - pub rootfs_file: String, // rootfs_file: os.tar + pub persist_path: PathBuf, // persist_path: /persist + pub update_path: PathBuf, // update_path: /persist/KubeOS-Update + pub mount_path: PathBuf, // mount_path: /persist/KubeOS-Update/kubeos-update + pub tar_path: PathBuf, // tar_path: /persist/KubeOS-Update/os.tar + pub image_path: PathBuf, // image_path: /persist/update.img + pub rootfs_file: String, // rootfs_file: os.tar } impl Default for PreparePath { fn default() -> Self { - let update_pathbuf = Path::new(PERSIST_DIR).join(UPDATE_DIR); let persist_dir = Path::new(PERSIST_DIR); + let update_pathbuf = persist_dir.join(UPDATE_DIR); Self { + persist_path: persist_dir.to_path_buf(), update_path: update_pathbuf.clone(), mount_path: update_pathbuf.join(MOUNT_DIR), tar_path: update_pathbuf.join(ROOTFS_ARCHIVE), @@ -50,9 +52,9 @@ pub fn is_file_exist>(path: P) -> bool { path.as_ref().exists() } -pub fn perpare_env(prepare_path: &PreparePath, need_bytes: i64, persist_path: &str, permission: u32) -> Result<()> { +pub fn perpare_env(prepare_path: &PreparePath, need_bytes: i64, permission: u32) -> Result<()> { info!("Prepare environment to upgrade"); - check_disk_size(need_bytes, persist_path)?; + check_disk_size(need_bytes, &prepare_path.persist_path)?; clean_env(&prepare_path.update_path, &prepare_path.mount_path, &prepare_path.image_path)?; fs::DirBuilder::new().recursive(true).mode(permission).create(&prepare_path.mount_path)?; Ok(()) @@ -196,13 +198,14 @@ mod tests { fn test_prepare_env() { init(); let paths = PreparePath { + persist_path: PathBuf::from("/tmp"), update_path: PathBuf::from("/tmp/test_prepare_env"), mount_path: PathBuf::from("/tmp/test_prepare_env/kubeos-update"), tar_path: PathBuf::from("/tmp/test_prepare_env/os.tar"), image_path: PathBuf::from("/tmp/test_prepare_env/update.img"), rootfs_file: "os.tar".to_string(), }; - perpare_env(&paths, 1 * 1024 * 1024 * 1024, "/home", 0o700).unwrap(); + perpare_env(&paths, 1 * 1024 * 1024 * 1024, 0o700).unwrap(); } #[test] diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs index e3dddaaf..dc823232 100644 --- a/KubeOS-Rust/manager/src/utils/image_manager.rs +++ b/KubeOS-Rust/manager/src/utils/image_manager.rs @@ -186,6 +186,7 @@ mod tests { let img_manager = UpgradeImageManager::new( PreparePath { + persist_path: "/tmp".into(), update_path: tmp_dir.into(), image_path: img_path.into(), mount_path: "/tmp/update/mount".into(), -- Gitee From 3cb2a7f808c445406ab042b2178c084cf7af9492 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Tue, 9 Jan 2024 20:13:54 +0800 Subject: [PATCH 09/46] proxy: add unit test and mock files Add the proxy unit test and the apiserver mock file used for the proxy unit test. Signed-off-by: liyuanr --- KubeOS-Rust/Cargo.lock | 67 ++ KubeOS-Rust/proxy/Cargo.toml | 8 + .../proxy/src/controller/agentclient.rs | 15 +- .../proxy/src/controller/apiserver_mock.rs | 607 ++++++++++++++++++ .../proxy/src/controller/controller.rs | 105 ++- KubeOS-Rust/proxy/src/controller/mod.rs | 2 + 6 files changed, 801 insertions(+), 3 deletions(-) create mode 100644 KubeOS-Rust/proxy/src/controller/apiserver_mock.rs diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index d9300f60..19bdc0c7 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -42,6 +42,28 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "async-trait" version = "0.1.74" @@ -1168,6 +1190,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "mockall_double" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ca96e5ac35256ae3e13536edd39b172b88f41615e1d7b653c8ad24524113e8" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "mockito" version = "0.31.1" @@ -1478,16 +1512,21 @@ name = "proxy" version = "0.1.0" dependencies = [ "anyhow", + "assert-json-diff", "async-trait", "chrono", "cli", "env_logger", "futures", "h2", + "http", + "hyper", "k8s-openapi", "kube", "log", "manager", + "mockall", + "mockall_double", "regex", "reqwest", "schemars", @@ -1498,6 +1537,7 @@ dependencies = [ "thread_local", "tokio", "tokio-retry", + "tower-test", ] [[package]] @@ -2142,6 +2182,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-test" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89b3cbabd3ae862100094ae433e1def582cf86451b4e9bf83aa7ac1d8a7d719" +dependencies = [ + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-util" version = "0.6.10" @@ -2229,6 +2282,20 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +[[package]] +name = "tower-test" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4546773ffeab9e4ea02b8872faa49bb616a80a7da66afc2f32688943f97efa7" +dependencies = [ + "futures-util", + "pin-project", + "tokio", + "tokio-test", + "tower-layer", + "tower-service", +] + [[package]] name = "tracing" version = "0.1.35" diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index b96bd1cd..ab98d283 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -31,3 +31,11 @@ thiserror = "1.0.29" thread_local = "=1.1.4" tokio = { version = "=1.14.0", features = ["macros", "rt-multi-thread"] } tokio-retry = "0.3" + +[dev-dependencies] +assert-json-diff = "2.0.2" +http = "0.2.9" +hyper = "0.14.25" +tower-test = "0.4.0" +mockall = { version = "=0.11.3" } +mockall_double = "0.3.0" diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs index ddc4f290..cc3ab079 100644 --- a/KubeOS-Rust/proxy/src/controller/agentclient.rs +++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs @@ -12,7 +12,6 @@ use std::{collections::HashMap, path::Path}; -use agent_call::AgentCallClient; use agent_error::Error; use cli::{ client::Client, @@ -23,6 +22,13 @@ use cli::{ }; use manager::api::{CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest}; +#[cfg(test)] +use mockall::automock; +#[cfg(test)] +use mockall_double::double; +#[cfg_attr(test, double)] +use agent_call::AgentCallClient; + pub struct UpgradeInfo { pub version: String, pub image_type: String, @@ -45,6 +51,7 @@ pub struct KeyInfo { pub operation: String, } +#[cfg_attr(test, automock)] pub trait AgentMethod { fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo, agent_call: AgentCallClient) -> Result<(), Error>; fn upgrade_method(&self, agent_call: AgentCallClient) -> Result<(), Error>; @@ -54,9 +61,13 @@ pub trait AgentMethod { pub mod agent_call { use super::{Client, Error, RpcMethod}; + #[cfg(test)] + use mockall::automock; + #[derive(Default)] pub struct AgentCallClient {} - + + #[cfg_attr(test, automock)] impl AgentCallClient { pub fn call_agent(&self, client: &Client, method: T) -> Result<(), Error> { match method.call(client) { diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs new file mode 100644 index 00000000..a7ae647a --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs @@ -0,0 +1,607 @@ +use self::mock_error::Error; +use crate::controller::{ + apiclient::{ApplyApi, ControllerClient}, + crd::{OSInstance, OSInstanceSpec, OSSpec, OS}, + values::{LABEL_OSINSTANCE, NODE_STATUS_IDLE,LABEL_UPGRADING}, + ProxyController, + +}; +use super::{agentclient::*, values::{NODE_STATUS_UPGRADE, NODE_STATUS_CONFIG}, crd::{Configs, OSInstanceStatus}}; +use anyhow::Result; +use http::{Request, Response}; +use hyper::{body::to_bytes, Body}; +use kube::{api::ObjectMeta, core::{ObjectList, ListMeta}}; +use kube::{Client, Resource, ResourceExt}; +use std::collections::BTreeMap; +use k8s_openapi::api::core::v1::{Node, NodeStatus, NodeSystemInfo, NodeSpec}; +use k8s_openapi::api::core::v1::Pod; + + +type ApiServerHandle = tower_test::mock::Handle, Response>; +pub struct ApiServerVerifier(ApiServerHandle); + +pub enum Testcases { + OSInstanceNotExist(OSInstance), + UpgradeNormal(OSInstance), + UpgradeUpgradeconfigsVersionMismatch(OSInstance), + UpgradeOSInstaceNodestatusConfig(OSInstance), + UpgradeOSInstaceNodestatusIdle(OSInstance), + ConfigNormal(OSInstance), + ConfigVersionMismatchReassign(OSInstance), + ConfigVersionMismatchUpdate(OSInstance), +} + +pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) { + tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("timeout on mock apiserver") + .expect("scenario succeeded") +} + +impl ApiServerVerifier { + pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::OSInstanceNotExist(osi) => { + self.handler_osinstance_get_not_exist(osi.clone()).await.unwrap() + .handler_osinstance_creation(osi.clone()).await.unwrap() + .handler_osinstance_get_exist(osi.clone()).await.unwrap() + .handler_node_get(osi).await + }, + Testcases::UpgradeNormal(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_node_get_with_label(osi.clone()) + .await.unwrap().handler_osinstance_patch_upgradeconfig_v2(osi.clone()) + .await.unwrap().handler_node_cordon(osi.clone()) + .await.unwrap().handler_node_pod_list_get(osi).await + }, + Testcases::UpgradeUpgradeconfigsVersionMismatch(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_node_get_with_label(osi.clone()) + .await.unwrap().handler_node_update_delete_label(osi.clone()) + .await.unwrap().handler_node_uncordon(osi.clone()) + .await.unwrap().handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::UpgradeOSInstaceNodestatusConfig(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_node_get_with_label(osi.clone()) + .await + }, + Testcases::UpgradeOSInstaceNodestatusIdle(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_node_get_with_label(osi.clone()) + .await.unwrap().handler_node_update_delete_label(osi.clone()) + .await.unwrap().handler_node_uncordon(osi) + .await + }, + Testcases::ConfigNormal(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_node_get(osi.clone()) + .await.unwrap().handler_osinstance_patch_sysconfig_v2(osi.clone()) + .await.unwrap().handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::ConfigVersionMismatchReassign(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_node_get(osi.clone()) + .await.unwrap().handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::ConfigVersionMismatchUpdate(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_osinstance_get_exist(osi.clone()) + .await.unwrap().handler_node_get(osi.clone()) + .await.unwrap().handler_osinstance_patch_spec_sysconfig_v2(osi) + .await + }, + + + + }.expect("Case completed without errors"); + }) + } + + async fn handler_osinstance_get_not_exist( + mut self, + osinstance: OSInstance, + ) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", + osinstance.name() + ) + ); + let response_json = serde_json::json!( + { "status": "Failure", "message": "osinstances.upgrade.openeuler.org \"openeuler\" not found", "reason": "NotFound", "code": 404 } + ); + dbg!("handler_osinstance_get_not_exist"); + let response = serde_json::to_vec(&response_json).unwrap(); + send.send_response( + Response::builder() + .status(404) + .body(Body::from(response)) + .unwrap(), + ); + Ok(self) + } + async fn handler_osinstance_get_exist(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", + osinstance.name() + ) + ); + dbg!("handler_osinstance_get_exist"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + async fn handler_osinstance_creation(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::POST); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?") + ); + dbg!("handler_osinstance_creation"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_idle(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?",osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json:serde_json::Value= + serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec:OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_IDLE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_idle"); + osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_upgradeconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?",osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json:serde_json::Value= + serde_json::from_slice(&req_body).expect("valid document from runtime"); + let status_json = body_json.get("status").expect("status object").clone(); + let status:OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!(status.upgradeconfigs.expect("upgradeconfigs is not None").clone(), + osinstance.spec.clone().upgradeconfigs.expect("upgradeconfig is not None")); + + osinstance.status.as_mut().unwrap().upgradeconfigs = osinstance.spec.upgradeconfigs.clone(); + + dbg!("handler_osinstance_patch_upgradeconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?",osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json:serde_json::Value= + serde_json::from_slice(&req_body).expect("valid osinstance"); + let status_json = body_json.get("status").expect("status object").clone(); + let status:OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!(status.sysconfigs.expect("sysconfigs is not None").clone(), + osinstance.spec.clone().sysconfigs.expect("sysconfig is not None")); + + osinstance.status.as_mut().unwrap().sysconfigs = osinstance.spec.sysconfigs.clone(); + + dbg!("handler_osinstance_patch_sysconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_spec_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?",osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json:serde_json::Value= + serde_json::from_slice(&req_body).expect("valid osinstance"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec:OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + + assert_eq!(spec.sysconfigs.expect("upgradeconfigs is not None").clone().version.clone().unwrap(), + String::from("v2")); + + osinstance.spec.sysconfigs.as_mut().unwrap().version = Some(String::from("v2")); + + dbg!("handler_osinstance_patch_spec_sysconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + + async fn handler_node_get(mut self,osinstance: OSInstance) -> Result{ + // return node with name = openeuler, osimage = KubeOS v1，no upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!( + "/api/v1/nodes/{}", + osinstance.name() + ) + ); + let node = Node{ + metadata:ObjectMeta { name:Some(String::from("openeuler")), + ..Default::default()}, + spec:None, + status:Some(NodeStatus{ + node_info:Some(NodeSystemInfo{ + os_image: String::from("KubeOS v1"), + ..Default::default() + }), + ..Default::default() + }) + }; + assert_eq!(node.name(),String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image,String::from("KubeOS v1")); + dbg!("handler_node_get"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_get_with_label(mut self,osinstance: OSInstance) -> Result{ + // return node with name = openeuler, osimage = KubeOS v1，has upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!( + "/api/v1/nodes/{}", + osinstance.name() + ) + ); + let mut node = Node{ + metadata:ObjectMeta { name:Some(String::from("openeuler")), + ..Default::default()}, + spec:None, + status:Some(NodeStatus{ + node_info:Some(NodeSystemInfo{ + os_image: String::from("KubeOS v1"), + ..Default::default() + }), + ..Default::default() + }) + }; + let node_labels = node.labels_mut(); + node_labels.insert(LABEL_UPGRADING.to_string(), "".to_string()); + assert_eq!(node.name(),String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image,String::from("KubeOS v1")); + assert!(node.labels().contains_key(LABEL_UPGRADING)); + dbg!("handler_node_get_with_label"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_update_delete_label(mut self,osinstance: OSInstance) -> Result{ + // return node with name = openeuler, osimage = KubeOS v1，no upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PUT); + assert_eq!( + request.uri().to_string(), + format!( + "/api/v1/nodes/{}?", + osinstance.name() + ) + ); + // check request body has upgrade label + let node = Node{ + metadata:ObjectMeta { name:Some(String::from("openeuler")), + ..Default::default()}, + spec:Some(NodeSpec{ + unschedulable:Some(true), + ..Default::default() + }), + status:Some(NodeStatus{ + node_info:Some(NodeSystemInfo{ + os_image: String::from("KubeOS v1"), + ..Default::default() + }), + ..Default::default() + }) + }; + dbg!("handler_node_update_delete_label"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_cordon(mut self,osinstance: OSInstance) -> Result{ + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!( + "/api/v1/nodes/{}?", + osinstance.name() + ) + ); + assert_eq!(request.extensions().get(),Some(&"cordon")); + let node = Node{ + metadata:ObjectMeta { name:Some(String::from("openeuler")), + ..Default::default()}, + spec:Some(NodeSpec{ + unschedulable:Some(true), + ..Default::default() + }), + status:Some(NodeStatus{ + node_info:Some(NodeSystemInfo{ + os_image: String::from("KubeOS v1"), + ..Default::default() + }), + ..Default::default() + }) + }; + dbg!("handler_node_cordon"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_uncordon(mut self,osinstance: OSInstance) -> Result{ + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!( + "/api/v1/nodes/{}?", + osinstance.name() + ) + ); + assert_eq!(request.extensions().get(),Some(&"cordon")); + let node = Node{ + metadata:ObjectMeta { name:Some(String::from("openeuler")), + ..Default::default()}, + spec:Some(NodeSpec{ + unschedulable:Some(false), + ..Default::default() + }), + status:Some(NodeStatus{ + node_info:Some(NodeSystemInfo{ + os_image: String::from("KubeOS v1"), + ..Default::default() + }), + ..Default::default() + }) + }; + dbg!("handler_node_uncordon"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_pod_list_get(mut self,osinstance: OSInstance) -> Result{ + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!( + "/api/v1/pods?&fieldSelector=spec.nodeName%3D{}", + osinstance.name() + ) + ); + assert_eq!(request.extensions().get(),Some(&"list")); + let pods_list = ObjectList::{ + metadata:ListMeta::default(), + items:vec![] + }; + dbg!("handler_node_pod_list_get"); + let response = serde_json::to_vec(&pods_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } +} + +pub mod mock_error { + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} + +impl ProxyController { + pub fn test() -> (ProxyController, ApiServerVerifier) { + let (mock_service, handle) = tower_test::mock::pair::, Response>(); + let mock_k8s_client = Client::new(mock_service, "default"); + let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); + let mut mock_agent_client: MockAgentMethod = MockAgentMethod::new(); + mock_agent_client.expect_rollback_method().returning(|_x| Ok(())); + mock_agent_client.expect_prepare_upgrade_method().returning(|_x,_y| Ok(())); + mock_agent_client.expect_upgrade_method().returning(|_x| Ok(())); + mock_agent_client.expect_configure_method().returning(|_x,_y| Ok(())); + let proxy_controller: ProxyController = + ProxyController::new(mock_k8s_client, mock_api_client, mock_agent_client); + (proxy_controller, ApiServerVerifier(handle)) + } +} + +impl OSInstance { + pub fn set_osi_default(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: Some(Configs{ + version: Some(String::from("v1")), + configs: None + }), + upgradeconfigs: Some(Configs{ + version: Some(String::from("v1")), + configs: None + }), + }, + status: Some(OSInstanceStatus{ + sysconfigs: Some(Configs{ + version: Some(String::from("v1")), + configs: None + }), + upgradeconfigs: Some(Configs{ + version: Some(String::from("v1")), + configs: None + }), + }), + } + } + + pub fn set_osi_nodestatus_upgrade(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + osinstance + } + + pub fn set_osi_nodestatus_config(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance + } + + pub fn set_osi_upgradecon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.upgradeconfigs.as_mut().unwrap().version = Some(String::from("v2")); + osinstance + } + + pub fn set_osi_nodestatus_upgrade_upgradecon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + osinstance.spec.upgradeconfigs.as_mut().unwrap().version = Some(String::from("v2")); + osinstance + } + + pub fn set_osi_nodestatus_config_syscon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance.spec.sysconfigs.as_mut().unwrap().version = Some(String::from("v2")); + osinstance + } +} + + +impl OS { + pub fn set_os_default() -> Self { + let mut os = OS::new("test", OSSpec::default()); + os.meta_mut().namespace = Some("default".into()); + os + } + + pub fn set_os_osversion_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.opstype = String::from("config"); + os + } + + pub fn set_os_osversion_v2_upgradecon_v2() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.upgradeconfigs = Some(Configs{ + version: Some(String::from("v2")), + configs: None + }); + os + } + + pub fn set_os_syscon_v2_opstype_config()-> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = String::from("config"); + os.spec.sysconfigs = Some(Configs{ + version: Some(String::from("v2")), + configs: None + }); + os + } + +} + +impl Default for OSSpec{ + fn default() -> Self { + OSSpec { + osversion: String::from("KubeOS v1"), + maxunavailable: 2, + checksum: String::from("test"), + imagetype: String::from("containerd"), + containerimage: String::from("test"), + opstype: String::from("upgrade"), + evictpodforce: true, + sysconfigs: Some(Configs{ + version: Some(String::from("v1")), + configs: None + }), + upgradeconfigs: Some(Configs{ + version: Some(String::from("v1")), + configs: None + }) + } + } +} + diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index b23ec187..264d506d 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -24,7 +24,7 @@ use log::{debug, error, info}; use reconciler_error::Error; use super::{ - agentclient::{agent_call::AgentCallClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, + agentclient::{AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, apiclient::ApplyApi, crd::{Configs, Content, OSInstance, OS}, drain::drain_os, @@ -34,6 +34,10 @@ use super::{ REQUEUE_ERROR, REQUEUE_NORMAL, }, }; +#[cfg(test)] +use mockall_double::double; +#[cfg_attr(test, double)] +use super::agentclient::agent_call::AgentCallClient; pub async fn reconcile( os: OS, @@ -423,3 +427,102 @@ pub mod reconciler_error { DrainNodeError { value: String }, } } + +#[cfg(test)] +mod test { + use super::{reconcile,error_policy,ProxyController,OSInstance,OS,Context}; + use crate::controller::ControllerClient; + use crate::controller::{apiserver_mock::{timeout_after_5s,Testcases}, agentclient::MockAgentMethod}; + use std::env; + + #[tokio::test] + async fn test_create_osinstance_with_no_upgrade_or_configuration(){ + let (test_proxy_controller,fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_default(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::OSInstanceNotExist(OSInstance::set_osi_default("openeuler", "default"))); + reconcile(os,context.clone() ).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + #[tokio::test] + async fn test_upgrade_normal(){ + let (test_proxy_controller,fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeNormal(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"))); + reconcile(os,context.clone() ).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_diff_osversion_opstype_config(){ + let (test_proxy_controller,fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeOSInstaceNodestatusConfig(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"))); + let res = reconcile(os,context.clone()).await; + timeout_after_5s(mocksrv).await; + assert!(res.is_err(),"upgrade fails due to opstype=config"); + let err = res.unwrap_err(); + assert!(err.to_string().contains("Expect OS Version is not same with Node OS Version, please upgrade first")); + error_policy(&err, context); + } + + #[tokio::test] + async fn test_upgradeconfigs_version_mismatch(){ + let (test_proxy_controller,fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeUpgradeconfigsVersionMismatch(OSInstance::set_osi_nodestatus_upgrade("openeuler", "default"))); + reconcile(os,context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_upgrade_nodestatus_idle(){ + let (test_proxy_controller,fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeOSInstaceNodestatusIdle(OSInstance::set_osi_upgradecon_v2("openeuler", "default"))); + reconcile(os,context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_config_normal(){ + let (test_proxy_controller,fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigNormal(OSInstance::set_osi_nodestatus_config_syscon_v2("openeuler", "default"))); + reconcile(os,context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_sysconfig_version_mismatch_reassign(){ + let (test_proxy_controller,fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchReassign(OSInstance::set_osi_nodestatus_config("openeuler", "default"))); + reconcile(os,context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_sysconfig_version_mismatch_update(){ + let (test_proxy_controller,fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchUpdate(OSInstance::set_osi_nodestatus_upgrade("openeuler", "default"))); + reconcile(os,context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } +} diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs index 090febc6..b143082e 100644 --- a/KubeOS-Rust/proxy/src/controller/mod.rs +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -13,6 +13,8 @@ mod agentclient; mod apiclient; mod controller; +#[cfg(test)] +mod apiserver_mock; mod crd; mod drain; mod utils; -- Gitee From d7802750a31a7d7bfa8ff9cede44b4466daa8c26 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Tue, 9 Jan 2024 20:15:08 +0800 Subject: [PATCH 10/46] proxy: support upgrade using disk image or docker and format Added the imageurl, flagSafe, mtls, cacert, clientcert, and clientkey parameters required for disk image upgrade or Docker upgrade. And format code. Change the value of mockall_double to 0.2.1 to adapt to rust 1.57. Signed-off-by: liyuanr --- KubeOS-Rust/Cargo.lock | 6 +- KubeOS-Rust/proxy/Cargo.toml | 2 +- .../proxy/src/controller/agentclient.rs | 27 +- .../proxy/src/controller/apiserver_mock.rs | 488 ++++++++---------- .../proxy/src/controller/controller.rs | 115 +++-- KubeOS-Rust/proxy/src/controller/crd.rs | 7 + KubeOS-Rust/proxy/src/controller/mod.rs | 2 +- 7 files changed, 329 insertions(+), 318 deletions(-) diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 19bdc0c7..4b7fc12d 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -1192,14 +1192,14 @@ dependencies = [ [[package]] name = "mockall_double" -version = "0.3.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1ca96e5ac35256ae3e13536edd39b172b88f41615e1d7b653c8ad24524113e8" +checksum = "7dffc15b97456ecc84d2bde8c1df79145e154f45225828c4361f676e1b82acd6" dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.37", + "syn 1.0.109", ] [[package]] diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index ab98d283..9a148e89 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -38,4 +38,4 @@ http = "0.2.9" hyper = "0.14.25" tower-test = "0.4.0" mockall = { version = "=0.11.3" } -mockall_double = "0.3.0" +mockall_double = "0.2.1" diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs index cc3ab079..73489a9b 100644 --- a/KubeOS-Rust/proxy/src/controller/agentclient.rs +++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs @@ -22,18 +22,24 @@ use cli::{ }; use manager::api::{CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest}; +#[cfg_attr(test, double)] +use agent_call::AgentCallClient; #[cfg(test)] use mockall::automock; #[cfg(test)] use mockall_double::double; -#[cfg_attr(test, double)] -use agent_call::AgentCallClient; pub struct UpgradeInfo { pub version: String, pub image_type: String, pub check_sum: String, pub container_image: String, + pub imageurl: String, + pub flagsafe: bool, + pub mtls: bool, + pub cacert: String, + pub clientcert: String, + pub clientkey: String, } pub struct ConfigInfo { @@ -63,10 +69,10 @@ pub mod agent_call { use super::{Client, Error, RpcMethod}; #[cfg(test)] use mockall::automock; - + #[derive(Default)] pub struct AgentCallClient {} - + #[cfg_attr(test, automock)] impl AgentCallClient { pub fn call_agent(&self, client: &Client, method: T) -> Result<(), Error> { @@ -95,11 +101,14 @@ impl AgentMethod for AgentClient { image_type: upgrade_info.image_type, check_sum: upgrade_info.check_sum, container_image: upgrade_info.container_image, - // TODO: add image_url, flag_safe, mtls, certs - image_url: "".to_string(), - flag_safe: false, - mtls: false, - certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + image_url: upgrade_info.imageurl, + flag_safe: upgrade_info.flagsafe, + mtls: upgrade_info.mtls, + certs: CertsInfo { + ca_cert: upgrade_info.cacert, + client_cert: upgrade_info.clientcert, + client_key: upgrade_info.clientkey, + }, }; match agent_call.call_agent(&self.agent_client, PrepareUpgradeMethod::new(upgrade_request)) { Ok(_resp) => Ok(()), diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs index a7ae647a..c46d26a4 100644 --- a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs +++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs @@ -1,21 +1,26 @@ use self::mock_error::Error; +use super::{ + agentclient::*, + crd::{Configs, OSInstanceStatus}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE}, +}; use crate::controller::{ apiclient::{ApplyApi, ControllerClient}, crd::{OSInstance, OSInstanceSpec, OSSpec, OS}, - values::{LABEL_OSINSTANCE, NODE_STATUS_IDLE,LABEL_UPGRADING}, + values::{LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, ProxyController, - }; -use super::{agentclient::*, values::{NODE_STATUS_UPGRADE, NODE_STATUS_CONFIG}, crd::{Configs, OSInstanceStatus}}; use anyhow::Result; use http::{Request, Response}; use hyper::{body::to_bytes, Body}; -use kube::{api::ObjectMeta, core::{ObjectList, ListMeta}}; +use k8s_openapi::api::core::v1::Pod; +use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo}; +use kube::{ + api::ObjectMeta, + core::{ListMeta, ObjectList}, +}; use kube::{Client, Resource, ResourceExt}; use std::collections::BTreeMap; -use k8s_openapi::api::core::v1::{Node, NodeStatus, NodeSystemInfo, NodeSpec}; -use k8s_openapi::api::core::v1::Pod; - type ApiServerHandle = tower_test::mock::Handle, Response>; pub struct ApiServerVerifier(ApiServerHandle); @@ -43,95 +48,142 @@ impl ApiServerVerifier { tokio::spawn(async move { match cases { Testcases::OSInstanceNotExist(osi) => { - self.handler_osinstance_get_not_exist(osi.clone()).await.unwrap() - .handler_osinstance_creation(osi.clone()).await.unwrap() - .handler_osinstance_get_exist(osi.clone()).await.unwrap() - .handler_node_get(osi).await + self.handler_osinstance_get_not_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_creation(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi) + .await }, Testcases::UpgradeNormal(osi) => { self.handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_node_get_with_label(osi.clone()) - .await.unwrap().handler_osinstance_patch_upgradeconfig_v2(osi.clone()) - .await.unwrap().handler_node_cordon(osi.clone()) - .await.unwrap().handler_node_pod_list_get(osi).await + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_upgradeconfig_v2(osi.clone()) + .await + .unwrap() + .handler_node_cordon(osi.clone()) + .await + .unwrap() + .handler_node_pod_list_get(osi) + .await }, Testcases::UpgradeUpgradeconfigsVersionMismatch(osi) => { self.handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_node_get_with_label(osi.clone()) - .await.unwrap().handler_node_update_delete_label(osi.clone()) - .await.unwrap().handler_node_uncordon(osi.clone()) - .await.unwrap().handler_osinstance_patch_nodestatus_idle(osi) - .await + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_node_update_delete_label(osi.clone()) + .await + .unwrap() + .handler_node_uncordon(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await }, Testcases::UpgradeOSInstaceNodestatusConfig(osi) => { self.handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_node_get_with_label(osi.clone()) - .await + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await }, Testcases::UpgradeOSInstaceNodestatusIdle(osi) => { self.handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_node_get_with_label(osi.clone()) - .await.unwrap().handler_node_update_delete_label(osi.clone()) - .await.unwrap().handler_node_uncordon(osi) - .await + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_node_update_delete_label(osi.clone()) + .await + .unwrap() + .handler_node_uncordon(osi) + .await }, Testcases::ConfigNormal(osi) => { self.handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_node_get(osi.clone()) - .await.unwrap().handler_osinstance_patch_sysconfig_v2(osi.clone()) - .await.unwrap().handler_osinstance_patch_nodestatus_idle(osi) - .await + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_sysconfig_v2(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await }, Testcases::ConfigVersionMismatchReassign(osi) => { self.handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_node_get(osi.clone()) - .await.unwrap().handler_osinstance_patch_nodestatus_idle(osi) - .await + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await }, Testcases::ConfigVersionMismatchUpdate(osi) => { self.handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_osinstance_get_exist(osi.clone()) - .await.unwrap().handler_node_get(osi.clone()) - .await.unwrap().handler_osinstance_patch_spec_sysconfig_v2(osi) - .await + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_spec_sysconfig_v2(osi) + .await }, - - - - }.expect("Case completed without errors"); + } + .expect("Case completed without errors"); }) } - async fn handler_osinstance_get_not_exist( - mut self, - osinstance: OSInstance, - ) -> Result { + async fn handler_osinstance_get_not_exist(mut self, osinstance: OSInstance) -> Result { let (request, send) = self.0.next_request().await.expect("service not called"); assert_eq!(request.method(), http::Method::GET); assert_eq!( request.uri().to_string(), - format!( - "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", - osinstance.name() - ) + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name()) ); let response_json = serde_json::json!( { "status": "Failure", "message": "osinstances.upgrade.openeuler.org \"openeuler\" not found", "reason": "NotFound", "code": 404 } ); dbg!("handler_osinstance_get_not_exist"); let response = serde_json::to_vec(&response_json).unwrap(); - send.send_response( - Response::builder() - .status(404) - .body(Body::from(response)) - .unwrap(), - ); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); Ok(self) } async fn handler_osinstance_get_exist(mut self, osinstance: OSInstance) -> Result { @@ -139,10 +191,7 @@ impl ApiServerVerifier { assert_eq!(request.method(), http::Method::GET); assert_eq!( request.uri().to_string(), - format!( - "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", - osinstance.name() - ) + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name()) ); dbg!("handler_osinstance_get_exist"); let response = serde_json::to_vec(&osinstance).unwrap(); @@ -167,16 +216,15 @@ impl ApiServerVerifier { assert_eq!(request.method(), http::Method::PATCH); assert_eq!( request.uri().to_string(), - format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?",osinstance.name()) + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name()) ); let req_body = to_bytes(request.into_body()).await.unwrap(); - let body_json:serde_json::Value= - serde_json::from_slice(&req_body).expect("valid document from runtime"); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); let spec_json = body_json.get("spec").expect("spec object").clone(); - let spec:OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); assert_eq!(spec.nodestatus.clone(), NODE_STATUS_IDLE.to_string()); - + dbg!("handler_osinstance_patch_nodestatus_idle"); osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); let response = serde_json::to_vec(&osinstance).unwrap(); @@ -189,17 +237,21 @@ impl ApiServerVerifier { assert_eq!(request.method(), http::Method::PATCH); assert_eq!( request.uri().to_string(), - format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?",osinstance.name()) + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?", + osinstance.name() + ) ); let req_body = to_bytes(request.into_body()).await.unwrap(); - let body_json:serde_json::Value= - serde_json::from_slice(&req_body).expect("valid document from runtime"); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); let status_json = body_json.get("status").expect("status object").clone(); - let status:OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); - - assert_eq!(status.upgradeconfigs.expect("upgradeconfigs is not None").clone(), - osinstance.spec.clone().upgradeconfigs.expect("upgradeconfig is not None")); + let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!( + status.upgradeconfigs.expect("upgradeconfigs is not None").clone(), + osinstance.spec.clone().upgradeconfigs.expect("upgradeconfig is not None") + ); osinstance.status.as_mut().unwrap().upgradeconfigs = osinstance.spec.upgradeconfigs.clone(); @@ -214,17 +266,21 @@ impl ApiServerVerifier { assert_eq!(request.method(), http::Method::PATCH); assert_eq!( request.uri().to_string(), - format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?",osinstance.name()) + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?", + osinstance.name() + ) ); let req_body = to_bytes(request.into_body()).await.unwrap(); - let body_json:serde_json::Value= - serde_json::from_slice(&req_body).expect("valid osinstance"); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance"); let status_json = body_json.get("status").expect("status object").clone(); - let status:OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); - - assert_eq!(status.sysconfigs.expect("sysconfigs is not None").clone(), - osinstance.spec.clone().sysconfigs.expect("sysconfig is not None")); + let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!( + status.sysconfigs.expect("sysconfigs is not None").clone(), + osinstance.spec.clone().sysconfigs.expect("sysconfig is not None") + ); osinstance.status.as_mut().unwrap().sysconfigs = osinstance.spec.sysconfigs.clone(); @@ -239,17 +295,18 @@ impl ApiServerVerifier { assert_eq!(request.method(), http::Method::PATCH); assert_eq!( request.uri().to_string(), - format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?",osinstance.name()) + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name()) ); let req_body = to_bytes(request.into_body()).await.unwrap(); - let body_json:serde_json::Value= - serde_json::from_slice(&req_body).expect("valid osinstance"); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance"); let spec_json = body_json.get("spec").expect("spec object").clone(); - let spec:OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); - - assert_eq!(spec.sysconfigs.expect("upgradeconfigs is not None").clone().version.clone().unwrap(), - String::from("v2")); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + + assert_eq!( + spec.sysconfigs.expect("upgradeconfigs is not None").clone().version.clone().unwrap(), + String::from("v2") + ); osinstance.spec.sysconfigs.as_mut().unwrap().version = Some(String::from("v2")); @@ -259,65 +316,44 @@ impl ApiServerVerifier { Ok(self) } - - async fn handler_node_get(mut self,osinstance: OSInstance) -> Result{ + async fn handler_node_get(mut self, osinstance: OSInstance) -> Result { // return node with name = openeuler, osimage = KubeOS v1，no upgrade label let (request, send) = self.0.next_request().await.expect("service not called"); assert_eq!(request.method(), http::Method::GET); - assert_eq!( - request.uri().to_string(), - format!( - "/api/v1/nodes/{}", - osinstance.name() - ) - ); - let node = Node{ - metadata:ObjectMeta { name:Some(String::from("openeuler")), - ..Default::default()}, - spec:None, - status:Some(NodeStatus{ - node_info:Some(NodeSystemInfo{ - os_image: String::from("KubeOS v1"), - ..Default::default() - }), + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name())); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), ..Default::default() - }) + }), }; - assert_eq!(node.name(),String::from("openeuler")); - assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image,String::from("KubeOS v1")); + assert_eq!(node.name(), String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1")); dbg!("handler_node_get"); let response = serde_json::to_vec(&node.clone()).unwrap(); send.send_response(Response::builder().body(Body::from(response)).unwrap()); Ok(self) } - async fn handler_node_get_with_label(mut self,osinstance: OSInstance) -> Result{ + async fn handler_node_get_with_label(mut self, osinstance: OSInstance) -> Result { // return node with name = openeuler, osimage = KubeOS v1，has upgrade label let (request, send) = self.0.next_request().await.expect("service not called"); assert_eq!(request.method(), http::Method::GET); - assert_eq!( - request.uri().to_string(), - format!( - "/api/v1/nodes/{}", - osinstance.name() - ) - ); - let mut node = Node{ - metadata:ObjectMeta { name:Some(String::from("openeuler")), - ..Default::default()}, - spec:None, - status:Some(NodeStatus{ - node_info:Some(NodeSystemInfo{ - os_image: String::from("KubeOS v1"), - ..Default::default() - }), + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name())); + let mut node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), ..Default::default() - }) + }), }; let node_labels = node.labels_mut(); node_labels.insert(LABEL_UPGRADING.to_string(), "".to_string()); - assert_eq!(node.name(),String::from("openeuler")); - assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image,String::from("KubeOS v1")); + assert_eq!(node.name(), String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1")); assert!(node.labels().contains_key(LABEL_UPGRADING)); dbg!("handler_node_get_with_label"); let response = serde_json::to_vec(&node.clone()).unwrap(); @@ -325,32 +361,19 @@ impl ApiServerVerifier { Ok(self) } - async fn handler_node_update_delete_label(mut self,osinstance: OSInstance) -> Result{ + async fn handler_node_update_delete_label(mut self, osinstance: OSInstance) -> Result { // return node with name = openeuler, osimage = KubeOS v1，no upgrade label let (request, send) = self.0.next_request().await.expect("service not called"); assert_eq!(request.method(), http::Method::PUT); - assert_eq!( - request.uri().to_string(), - format!( - "/api/v1/nodes/{}?", - osinstance.name() - ) - ); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); // check request body has upgrade label - let node = Node{ - metadata:ObjectMeta { name:Some(String::from("openeuler")), - ..Default::default()}, - spec:Some(NodeSpec{ - unschedulable:Some(true), + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), ..Default::default() }), - status:Some(NodeStatus{ - node_info:Some(NodeSystemInfo{ - os_image: String::from("KubeOS v1"), - ..Default::default() - }), - ..Default::default() - }) }; dbg!("handler_node_update_delete_label"); let response = serde_json::to_vec(&node.clone()).unwrap(); @@ -358,31 +381,18 @@ impl ApiServerVerifier { Ok(self) } - async fn handler_node_cordon(mut self,osinstance: OSInstance) -> Result{ + async fn handler_node_cordon(mut self, osinstance: OSInstance) -> Result { let (request, send) = self.0.next_request().await.expect("service not called"); assert_eq!(request.method(), http::Method::PATCH); - assert_eq!( - request.uri().to_string(), - format!( - "/api/v1/nodes/{}?", - osinstance.name() - ) - ); - assert_eq!(request.extensions().get(),Some(&"cordon")); - let node = Node{ - metadata:ObjectMeta { name:Some(String::from("openeuler")), - ..Default::default()}, - spec:Some(NodeSpec{ - unschedulable:Some(true), + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + assert_eq!(request.extensions().get(), Some(&"cordon")); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), ..Default::default() }), - status:Some(NodeStatus{ - node_info:Some(NodeSystemInfo{ - os_image: String::from("KubeOS v1"), - ..Default::default() - }), - ..Default::default() - }) }; dbg!("handler_node_cordon"); let response = serde_json::to_vec(&node.clone()).unwrap(); @@ -390,31 +400,18 @@ impl ApiServerVerifier { Ok(self) } - async fn handler_node_uncordon(mut self,osinstance: OSInstance) -> Result{ + async fn handler_node_uncordon(mut self, osinstance: OSInstance) -> Result { let (request, send) = self.0.next_request().await.expect("service not called"); assert_eq!(request.method(), http::Method::PATCH); - assert_eq!( - request.uri().to_string(), - format!( - "/api/v1/nodes/{}?", - osinstance.name() - ) - ); - assert_eq!(request.extensions().get(),Some(&"cordon")); - let node = Node{ - metadata:ObjectMeta { name:Some(String::from("openeuler")), - ..Default::default()}, - spec:Some(NodeSpec{ - unschedulable:Some(false), + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + assert_eq!(request.extensions().get(), Some(&"cordon")); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(false), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), ..Default::default() }), - status:Some(NodeStatus{ - node_info:Some(NodeSystemInfo{ - os_image: String::from("KubeOS v1"), - ..Default::default() - }), - ..Default::default() - }) }; dbg!("handler_node_uncordon"); let response = serde_json::to_vec(&node.clone()).unwrap(); @@ -422,21 +419,15 @@ impl ApiServerVerifier { Ok(self) } - async fn handler_node_pod_list_get(mut self,osinstance: OSInstance) -> Result{ + async fn handler_node_pod_list_get(mut self, osinstance: OSInstance) -> Result { let (request, send) = self.0.next_request().await.expect("service not called"); assert_eq!(request.method(), http::Method::GET); assert_eq!( request.uri().to_string(), - format!( - "/api/v1/pods?&fieldSelector=spec.nodeName%3D{}", - osinstance.name() - ) + format!("/api/v1/pods?&fieldSelector=spec.nodeName%3D{}", osinstance.name()) ); - assert_eq!(request.extensions().get(),Some(&"list")); - let pods_list = ObjectList::{ - metadata:ListMeta::default(), - items:vec![] - }; + assert_eq!(request.extensions().get(), Some(&"list")); + let pods_list = ObjectList:: { metadata: ListMeta::default(), items: vec![] }; dbg!("handler_node_pod_list_get"); let response = serde_json::to_vec(&pods_list).unwrap(); send.send_response(Response::builder().body(Body::from(response)).unwrap()); @@ -456,16 +447,16 @@ pub mod mock_error { } } -impl ProxyController { - pub fn test() -> (ProxyController, ApiServerVerifier) { +impl ProxyController { + pub fn test() -> (ProxyController, ApiServerVerifier) { let (mock_service, handle) = tower_test::mock::pair::, Response>(); let mock_k8s_client = Client::new(mock_service, "default"); let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); - let mut mock_agent_client: MockAgentMethod = MockAgentMethod::new(); + let mut mock_agent_client: MockAgentMethod = MockAgentMethod::new(); mock_agent_client.expect_rollback_method().returning(|_x| Ok(())); - mock_agent_client.expect_prepare_upgrade_method().returning(|_x,_y| Ok(())); + mock_agent_client.expect_prepare_upgrade_method().returning(|_x, _y| Ok(())); mock_agent_client.expect_upgrade_method().returning(|_x| Ok(())); - mock_agent_client.expect_configure_method().returning(|_x,_y| Ok(())); + mock_agent_client.expect_configure_method().returning(|_x, _y| Ok(())); let proxy_controller: ProxyController = ProxyController::new(mock_k8s_client, mock_api_client, mock_agent_client); (proxy_controller, ApiServerVerifier(handle)) @@ -486,52 +477,40 @@ impl OSInstance { }, spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), - sysconfigs: Some(Configs{ - version: Some(String::from("v1")), - configs: None - }), - upgradeconfigs: Some(Configs{ - version: Some(String::from("v1")), - configs: None - }), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), }, - status: Some(OSInstanceStatus{ - sysconfigs: Some(Configs{ - version: Some(String::from("v1")), - configs: None - }), - upgradeconfigs: Some(Configs{ - version: Some(String::from("v1")), - configs: None - }), + status: Some(OSInstanceStatus { + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), }), } } pub fn set_osi_nodestatus_upgrade(node_name: &str, namespace: &str) -> Self { // return osinstance with nodestatus = upgrade, upgradeconfig.version=v1, sysconfig.version=v1 - let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); osinstance } pub fn set_osi_nodestatus_config(node_name: &str, namespace: &str) -> Self { // return osinstance with nodestatus = upgrade, upgradeconfig.version=v1, sysconfig.version=v1 - let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); osinstance } pub fn set_osi_upgradecon_v2(node_name: &str, namespace: &str) -> Self { // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 - let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); osinstance.spec.upgradeconfigs.as_mut().unwrap().version = Some(String::from("v2")); osinstance } pub fn set_osi_nodestatus_upgrade_upgradecon_v2(node_name: &str, namespace: &str) -> Self { // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 - let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); osinstance.spec.upgradeconfigs.as_mut().unwrap().version = Some(String::from("v2")); osinstance @@ -539,14 +518,13 @@ impl OSInstance { pub fn set_osi_nodestatus_config_syscon_v2(node_name: &str, namespace: &str) -> Self { // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 - let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); osinstance.spec.sysconfigs.as_mut().unwrap().version = Some(String::from("v2")); osinstance } } - impl OS { pub fn set_os_default() -> Self { let mut os = OS::new("test", OSSpec::default()); @@ -564,44 +542,36 @@ impl OS { pub fn set_os_osversion_v2_upgradecon_v2() -> Self { let mut os = OS::set_os_default(); os.spec.osversion = String::from("KubeOS v2"); - os.spec.upgradeconfigs = Some(Configs{ - version: Some(String::from("v2")), - configs: None - }); + os.spec.upgradeconfigs = Some(Configs { version: Some(String::from("v2")), configs: None }); os } - pub fn set_os_syscon_v2_opstype_config()-> Self { + pub fn set_os_syscon_v2_opstype_config() -> Self { let mut os = OS::set_os_default(); os.spec.opstype = String::from("config"); - os.spec.sysconfigs = Some(Configs{ - version: Some(String::from("v2")), - configs: None - }); + os.spec.sysconfigs = Some(Configs { version: Some(String::from("v2")), configs: None }); os } - } -impl Default for OSSpec{ +impl Default for OSSpec { fn default() -> Self { - OSSpec { - osversion: String::from("KubeOS v1"), - maxunavailable: 2, - checksum: String::from("test"), - imagetype: String::from("containerd"), + OSSpec { + osversion: String::from("KubeOS v1"), + maxunavailable: 2, + checksum: String::from("test"), + imagetype: String::from("containerd"), containerimage: String::from("test"), - opstype: String::from("upgrade"), - evictpodforce: true, - sysconfigs: Some(Configs{ - version: Some(String::from("v1")), - configs: None - }), - upgradeconfigs: Some(Configs{ - version: Some(String::from("v1")), - configs: None - }) + opstype: String::from("upgrade"), + evictpodforce: true, + imageurl: String::from(""), + flagsafe: false, + mtls: false, + cacert: Some(String::from("")), + clientcert: Some(String::from("")), + clientkey: Some(String::from("")), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), } } } - diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index 264d506d..e7ee9f9b 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -23,6 +23,8 @@ use kube::{ use log::{debug, error, info}; use reconciler_error::Error; +#[cfg_attr(test, double)] +use super::agentclient::agent_call::AgentCallClient; use super::{ agentclient::{AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, apiclient::ApplyApi, @@ -36,8 +38,6 @@ use super::{ }; #[cfg(test)] use mockall_double::double; -#[cfg_attr(test, double)] -use super::agentclient::agent_call::AgentCallClient; pub async fn reconcile( os: OS, @@ -283,6 +283,12 @@ impl ProxyController { image_type: os_cr.spec.imagetype.clone(), check_sum: os_cr.spec.checksum.clone(), container_image: os_cr.spec.containerimage.clone(), + flagsafe: os_cr.spec.flagsafe.clone(), + imageurl: os_cr.spec.imageurl.clone(), + mtls: os_cr.spec.mtls.clone(), + cacert: os_cr.spec.cacert.clone().unwrap_or_default(), + clientcert: os_cr.spec.clientcert.clone().unwrap_or_default(), + clientkey: os_cr.spec.clientkey.clone().unwrap_or_default(), }; let agent_call_client = AgentCallClient::default(); match self.agent_client.prepare_upgrade_method(upgrade_info, agent_call_client) { @@ -430,99 +436,118 @@ pub mod reconciler_error { #[cfg(test)] mod test { - use super::{reconcile,error_policy,ProxyController,OSInstance,OS,Context}; + use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS}; use crate::controller::ControllerClient; - use crate::controller::{apiserver_mock::{timeout_after_5s,Testcases}, agentclient::MockAgentMethod}; + use crate::controller::{ + agentclient::MockAgentMethod, + apiserver_mock::{timeout_after_5s, Testcases}, + }; use std::env; #[tokio::test] - async fn test_create_osinstance_with_no_upgrade_or_configuration(){ - let (test_proxy_controller,fakeserver) = ProxyController::::test(); + async fn test_create_osinstance_with_no_upgrade_or_configuration() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_default(); - let context = Context::new(test_proxy_controller); - let mocksrv = fakeserver.run(Testcases::OSInstanceNotExist(OSInstance::set_osi_default("openeuler", "default"))); - reconcile(os,context.clone() ).await.expect("reconciler"); + let context = Context::new(test_proxy_controller); + let mocksrv = + fakeserver.run(Testcases::OSInstanceNotExist(OSInstance::set_osi_default("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); timeout_after_5s(mocksrv).await; } #[tokio::test] - async fn test_upgrade_normal(){ - let (test_proxy_controller,fakeserver) = ProxyController::::test(); + async fn test_upgrade_normal() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_osversion_v2_upgradecon_v2(); - let context = Context::new(test_proxy_controller); - let mocksrv = fakeserver.run(Testcases::UpgradeNormal(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"))); - reconcile(os,context.clone() ).await.expect("reconciler"); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeNormal(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); timeout_after_5s(mocksrv).await; } #[tokio::test] - async fn test_diff_osversion_opstype_config(){ - let (test_proxy_controller,fakeserver) = ProxyController::::test(); + async fn test_diff_osversion_opstype_config() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_osversion_v2_opstype_config(); - let context = Context::new(test_proxy_controller); - let mocksrv = fakeserver.run(Testcases::UpgradeOSInstaceNodestatusConfig(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"))); - let res = reconcile(os,context.clone()).await; + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeOSInstaceNodestatusConfig( + OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"), + )); + let res = reconcile(os, context.clone()).await; timeout_after_5s(mocksrv).await; - assert!(res.is_err(),"upgrade fails due to opstype=config"); + assert!(res.is_err(), "upgrade fails due to opstype=config"); let err = res.unwrap_err(); assert!(err.to_string().contains("Expect OS Version is not same with Node OS Version, please upgrade first")); error_policy(&err, context); } #[tokio::test] - async fn test_upgradeconfigs_version_mismatch(){ - let (test_proxy_controller,fakeserver) = ProxyController::::test(); + async fn test_upgradeconfigs_version_mismatch() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_osversion_v2_upgradecon_v2(); - let context = Context::new(test_proxy_controller); - let mocksrv = fakeserver.run(Testcases::UpgradeUpgradeconfigsVersionMismatch(OSInstance::set_osi_nodestatus_upgrade("openeuler", "default"))); - reconcile(os,context.clone()).await.expect("reconciler"); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeUpgradeconfigsVersionMismatch( + OSInstance::set_osi_nodestatus_upgrade("openeuler", "default"), + )); + reconcile(os, context.clone()).await.expect("reconciler"); timeout_after_5s(mocksrv).await; } #[tokio::test] - async fn test_upgrade_nodestatus_idle(){ - let (test_proxy_controller,fakeserver) = ProxyController::::test(); + async fn test_upgrade_nodestatus_idle() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_osversion_v2_upgradecon_v2(); - let context = Context::new(test_proxy_controller); - let mocksrv = fakeserver.run(Testcases::UpgradeOSInstaceNodestatusIdle(OSInstance::set_osi_upgradecon_v2("openeuler", "default"))); - reconcile(os,context.clone()).await.expect("reconciler"); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::UpgradeOSInstaceNodestatusIdle(OSInstance::set_osi_upgradecon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); timeout_after_5s(mocksrv).await; } #[tokio::test] - async fn test_config_normal(){ - let (test_proxy_controller,fakeserver) = ProxyController::::test(); + async fn test_config_normal() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_syscon_v2_opstype_config(); - let context = Context::new(test_proxy_controller); - let mocksrv = fakeserver.run(Testcases::ConfigNormal(OSInstance::set_osi_nodestatus_config_syscon_v2("openeuler", "default"))); - reconcile(os,context.clone()).await.expect("reconciler"); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::ConfigNormal(OSInstance::set_osi_nodestatus_config_syscon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); timeout_after_5s(mocksrv).await; } #[tokio::test] - async fn test_sysconfig_version_mismatch_reassign(){ - let (test_proxy_controller,fakeserver) = ProxyController::::test(); + async fn test_sysconfig_version_mismatch_reassign() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_syscon_v2_opstype_config(); - let context = Context::new(test_proxy_controller); - let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchReassign(OSInstance::set_osi_nodestatus_config("openeuler", "default"))); - reconcile(os,context.clone()).await.expect("reconciler"); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchReassign(OSInstance::set_osi_nodestatus_config( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); timeout_after_5s(mocksrv).await; } #[tokio::test] - async fn test_sysconfig_version_mismatch_update(){ - let (test_proxy_controller,fakeserver) = ProxyController::::test(); + async fn test_sysconfig_version_mismatch_update() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_syscon_v2_opstype_config(); - let context = Context::new(test_proxy_controller); - let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchUpdate(OSInstance::set_osi_nodestatus_upgrade("openeuler", "default"))); - reconcile(os,context.clone()).await.expect("reconciler"); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchUpdate(OSInstance::set_osi_nodestatus_upgrade( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); timeout_after_5s(mocksrv).await; } } diff --git a/KubeOS-Rust/proxy/src/controller/crd.rs b/KubeOS-Rust/proxy/src/controller/crd.rs index efec0bd8..41f333e8 100644 --- a/KubeOS-Rust/proxy/src/controller/crd.rs +++ b/KubeOS-Rust/proxy/src/controller/crd.rs @@ -23,6 +23,13 @@ pub struct OSSpec { pub containerimage: String, pub opstype: String, pub evictpodforce: bool, + pub imageurl: String, + #[serde(rename = "flagSafe")] + pub flagsafe: bool, + pub mtls: bool, + pub cacert: Option, + pub clientcert: Option, + pub clientkey: Option, pub sysconfigs: Option, pub upgradeconfigs: Option, } diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs index b143082e..384d74b9 100644 --- a/KubeOS-Rust/proxy/src/controller/mod.rs +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -12,9 +12,9 @@ mod agentclient; mod apiclient; -mod controller; #[cfg(test)] mod apiserver_mock; +mod controller; mod crd; mod drain; mod utils; -- Gitee From 84462325a8bff6b3f5db0d08c358c73aae6ff186 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 9 Jan 2024 19:52:46 +0800 Subject: [PATCH 11/46] build: add KubeOS-Rust build in makefile Signed-off-by: Yuhang Wei --- Makefile | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eddf9e6d..d4cd71e3 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,9 @@ GO_BUILD_CGO = CGO_ENABLED=1 \ CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \ ${GO_BUILD} -buildmode=pie -trimpath -tags "seccomp selinux static_build cgo netgo osusergo" -all: proxy operator agent hostshell +RUSTFLAGS := RUSTFLAGS="-C relocation_model=pic -D warnings -W unsafe_code -W rust_2021_incompatible_closure_captures -C link-arg=-s" + +all: proxy operator agent hostshell rust-kubeos # Build binary proxy: @@ -65,6 +67,15 @@ hostshell: ${GO_BUILD_CGO} ${LD_FLAGS} -o bin/hostshell cmd/admin-container/main.go strip bin/hostshell +rust-kubeos: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust + +rust-proxy: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package proxy + +rust-agent: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package os-agent + # Install CRDs into a cluster install: manifests kubectl apply -f confg/crd -- Gitee From 6e62adfa80c33d9b1fc4445487cc15e721db07bc Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Wed, 10 Jan 2024 15:53:26 +0800 Subject: [PATCH 12/46] build(rust os-agent): remove useless dependency Signed-off-by: Yuhang Wei --- KubeOS-Rust/Cargo.lock | 11 ----------- KubeOS-Rust/manager/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 4b7fc12d..20873396 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -199,16 +199,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "colored" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" -dependencies = [ - "lazy_static", - "windows-sys", -] - [[package]] name = "core-foundation" version = "0.9.3" @@ -1209,7 +1199,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80f9fece9bd97ab74339fe19f4bcaf52b76dcc18e5364c7977c1838f76b38de9" dependencies = [ "assert-json-diff", - "colored", "httparse", "lazy_static", "log", diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml index 40672ccc..9431fba9 100644 --- a/KubeOS-Rust/manager/Cargo.toml +++ b/KubeOS-Rust/manager/Cargo.toml @@ -8,7 +8,7 @@ version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dev-dependencies] mockall = { version = "=0.11.3" } -mockito = { version = "0.31.1" } +mockito = { version = "0.31.1", default-features = false } predicates = { version = "=2.0.1" } tempfile = { version = "3.6.0" } -- Gitee From f34fe043179aec4bd2a9270fc25cef3f4377c152 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Thu, 11 Jan 2024 20:25:05 +0800 Subject: [PATCH 13/46] docs:Add the content of the user guide. Modify the format of the user guide, add precautions, and add configuration information. Signed-off-by: liyuanr --- docs/quick-start.md | 399 +++++++++++++++++++++++++++----------------- 1 file changed, 243 insertions(+), 156 deletions(-) diff --git a/docs/quick-start.md b/docs/quick-start.md index 0d4dc4bf..9656fb99 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -1,18 +1,22 @@ # 快速使用指导 -## 编译及部署 +[TOC] -### 编译指导 +## 编译指导 * 编译环境：openEuler Linux x86/AArch64 + * 进行编译需要以下包： * golang(大于等于1.15版本) * make * git + * rust(大于等于1.57版本) + * cargo(大于等于1.57版本) + * openssl-devel ``` shell - sudo yum install golang make git - ``` + sudo yum install golang make git rust cargo openssl-devel + ``` * 使用git获取本项目的源码 @@ -27,76 +31,101 @@ ```shell cd KubeOS - sudo make - ``` - - * proxy及operator容器镜像构建 - * proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕 - * 请用户自行编写Dockerfile来构建镜像，请注意 - * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性 - * 需将operator和proxy拷贝到baseimage上 - * 请确保proxy属主和属组为root，文件权限为500 - * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500 - * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应 - * 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库 - * Dockerfile参考如下, Dockerfile也可以使用多阶段构建: - - ``` dockerfile - FROM your_baseimage - COPY ./bin/proxy /proxy - ENTRYPOINT ["/proxy"] - FROM your_baseimage - COPY --chown=6552:6552 ./bin/operator /operator - ENTRYPOINT ["/operator"] - ``` + sudo make + # 编译生成的二进制在bin目录下，查看二进制 + tree bin + bin + ├── operator + ├── os-agent + ├── proxy + ├── rust + │   ├── ... + │   └── release + │   ├── ... + │   ├── os-agent + │   └── proxy + ``` - ```shell - # 指定proxy的镜像仓库，镜像名及版本 - export IMG_PROXY=your_imageRepository/proxy_imageName:version - # 指定proxy的Dockerfile地址 - export DOCKERFILE_PROXY=your_dockerfile_proxy - # 指定operator的镜像仓库，镜像名及版本 - export IMG_OPERATOR=your_imageRepository/operator_imageName:version - # 指定operator的Dockerfile路径 - export DOCKERFILE_OPERATOR=your_dockerfile_operator - - # 镜像构建 - docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} . - docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} . - # 推送镜像到镜像仓库 - docker push ${IMG_OPERATOR} - docker push ${IMG_PROXY} - ``` + * ```bin/proxy```、```bin/os-agent```为go语言编写的proxy和os-agent，```bin/rust/release/proxy```、```bin/rust/release/os-agent```为rust语言编写的proxy和os-agent，二者功能一致。 + +## 镜像构建指导 + +### proxy及operator镜像构建指导 + +* proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕 + +* 请用户自行编写Dockerfile来构建镜像，请注意 + * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性 + * 需将operator和proxy拷贝到baseimage上 + * 请确保proxy属主和属组为root，文件权限为500 + * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500 + * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应 + +* 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库 + +* Dockerfile参考如下, Dockerfile也可以使用多阶段构建: + + ``` dockerfile + FROM your_baseimage + COPY ./bin/proxy /proxy + ENTRYPOINT ["/proxy"] + FROM your_baseimage + COPY --chown=6552:6552 ./bin/operator /operator + ENTRYPOINT ["/operator"] + ``` -* OS虚拟机镜像制作 - * 制作注意事项 - * 请确保已安装qemu-img，bc，parted，tar，yum，docker - * 容器OS镜像制作需要使用root权限 - * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 - * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式 - * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间 - * 容器镜像制作时不支持用户自定义配置挂载文件 - * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。 - * 请确保os-agent属主和属组为root，建议os-agent文件权限为500 - * 容器OS虚拟机镜像制作 - 进入scripts目录，执行脚本 - - ```shell - cd scripts - bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' - ``` - - * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 - * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： - * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 - * qcow2 格式的系统镜像 system.qcow2。 - * 可用于升级的根文件系统分区镜像 update.img 。 - * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数 - * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) - * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 - * 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 - -### 部署指导 + ```shell + # 指定proxy的镜像仓库，镜像名及版本 + export IMG_PROXY=your_imageRepository/proxy_imageName:version + # 指定proxy的Dockerfile地址 + export DOCKERFILE_PROXY=your_dockerfile_proxy + # 指定operator的镜像仓库，镜像名及版本 + export IMG_OPERATOR=your_imageRepository/operator_imageName:version + # 指定operator的Dockerfile路径 + export DOCKERFILE_OPERATOR=your_dockerfile_operator + + # 镜像构建 + docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} . + docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} . + # 推送镜像到镜像仓库 + docker push ${IMG_OPERATOR} + docker push ${IMG_PROXY} + ``` + +### KubeOS虚拟机镜像制作指导 + +* 制作注意事项 + * 请确保已安装qemu-img，bc，parted，tar，yum，docker + * 容器OS镜像制作需要使用root权限 + * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 + * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式 + * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间 + * 容器镜像制作时不支持用户自定义配置挂载文件 + * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。 + * 请确保os-agent属主和属组为root，建议os-agent文件权限为500 + +* 容器OS虚拟机镜像制作 + 进入scripts目录，执行脚本 + + ```shell + cd scripts + bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' + ``` + + * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 + * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： + * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 + * qcow2 格式的系统镜像 system.qcow2。 + * 可用于升级的根文件系统分区镜像 update.img 。 + * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数 + * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) + * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 + +* 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 + +## 部署指导 + +### os-operator和os-proxy部署指导 * 环境要求 * openEuler Linux x86/AArch64系统 @@ -142,18 +171,35 @@ kubectl get pods -A ``` -### 使用指导 +## 使用指导 #### 注意事项 -* 容器OS升级为所有软件包原子升级，默认不在容器OS内提供单包升级能力。 -* 容器OS升级为双区升级的方式，不支持更多分区数量。 -* 单节点的升级过程的日志可在节点的/var/log/message文件查看。 -* 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。 -* 使用docker镜像升级和mtls双向认证仅支持 openEuler 22.09 及之后的版本 -* 不支持跨大版本升级 - -#### 参数说明 +* 公共注意事项 + * 仅支持虚拟机x86和arm64 UEFI场景。 + * 当前不支持集群节点OS多版本管理，即集群中OS的CR只能为一个。 + * 使用kubectl apply通过YAML创建或更新OS的CR时，不建议并发apply，当并发请求过多时，kube-apiserver会无法处理请求导致失败。 + * 如用户配置了容器镜像仓的证书或密钥，请用户保证证书或密钥文件的权限最小。 +* 升级注意事项 + * 升级为所有软件包原子升级，默认不提供单包升级能力。 + * 升级为双区升级的方式，不支持更多分区数量。 + * 当前暂不支持跨大版本升级。 + * 单节点的升级过程的日志可在节点的 /var/log/messages 文件查看。 + * 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。 + * 节点上containerd如需配置ctr使用的私有镜像，请将配置文件host.toml按照ctr指导放在/etc/containerd/certs.d目录下。 + +* 配置注意事项 + * 用户自行指定配置内容，用户需保证配置内容安全可靠 ，尤其是持久化配置（kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next），KubeOS不对参数有效性进行检验。 + * opstype=config时，若osversion与当前集群节点的OS版本不一致，配置不会进行。 + * 当前仅支持kernel参数临时配置（kernel.sysctl）、持久化配置（kernel.sysctl.persist）和grub cmdline配置（grub.cmdline.current和grub.cmdline.next）。 + * 持久化配置会写入persist持久化分区，升级重启后配置保留；kernel参数临时配置重启后不保留。 + * 配置grub.cmdline.current或grub.cmdline.next时，如为单个参数（非key=value格式参数），请指定key为该参数，value为空。 + * 进行配置删除（operation=delete）时，key=value形式的配置需保证key、value和实际配置一致。 + * 配置不支持回退，如需回退，请修改配置版本和配置内容，重新下发配置。 + * 配置出现错误，节点状态陷入config时，请将配置版本恢复成上一版本并重新下发配置，从而使节点恢复至idel状态。 但是请注意：出现错误前已经配置完成的参数无法恢复。 + * 在配置grub.cmdline.current或grub.cmdline.next时，若需要将已存在的“key=value”格式的参数更新为只有key无value格式，比如将“rd.info=0”更新成rd.info，需要先删除“key=value”，然后在下一次配置时，添加key。不支持直接更新或者更新删除动作在同一次完成。 + +#### OS CR参数说明 在集群中创建类别为OS的定制对象，设置相应字段。类别OS来自于安装和部署章节创建的CRD对象，字段及说明如下： @@ -163,21 +209,21 @@ | 参数 |参数类型 | 参数说明 | 使用说明 | 是否必选 | | -------------- | ------ | ------------------------------------------------------------ | ----- | ---------------- | - | imagetype | string | 使用的升级镜像的类型 | 需为 docker ，containerd ，或者是 disk，其他值无效，且该参数仅在升级场景有效。
**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。|是 | - | opstype | string | 进行的操作，升级,回退或者配置 | 需为 upgrade ，config 或者 rollback ，其他值无效 |是 | - | osversion | string | 用于升级或回退的镜像的OS版本 | 需为 KubeOS version , 例如: KubeOS 1.0.0|是 | - | maxunavailable | int | 同时进行升级或回退的节点数 | maxunavailable值设置为大于实际集群的节点数时也可正常部署，升级或回退时会按照集群内实际节点数进行|是 | - | containerimage | string | 用于升级的容器镜像 | 需要为容器镜像格式：[REPOSITORY/NAME[:TAG@DIGEST]](https://docs.docker.com/engine/reference/commandline/tag/#extended-description)，仅在使用容器镜像升级场景下有效|是 | - | imageurl | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如： 仅在使用磁盘镜像升级场景下有效|是 | + | imagetype | string | 升级镜像的类型 | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。
**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是 | + | opstype | string | 操作类型：升级,回退或者配置 | 仅支持upgrade ，config 或者 rollback |是 | + | osversion | string | 升级/回退的目标版本 | osversion需与节点的目标os版本对应（节点上/etc/os-release中PRETTY_NAME字段或k8s检查到的节点os版本） 例如：KubeOS 1.0.0。 |是 | + | maxunavailable | int | 每批同时进行升级/回退/配置的节点数。 | maxunavailable值大于实际节点数时，取实际节点数进行升级/回退/配置。 |是 | + | containerimage | string | 用于升级的容器镜像 | 仅在imagetype是容器类型时生效，仅支持以下3种格式的容器镜像地址： repository/name repository/name@sha256:xxxx repository/name:tag |是 | + | imageurl | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如： ，仅在使用磁盘镜像升级场景下有效 |是 | | checksum | string | 用于升级的磁盘镜像校验的checksum(SHA-256)值或者是用于升级的容器镜像的digests值 | 仅在升级场景下有效 |是 | | flagSafe | bool | 当imageurl的地址使用http协议表示是否是安全的 | 需为 true 或者 false ，仅在imageurl使用http协议时有效 |是 | | mtls | bool | 用于表示与imageurl连接是否采用https双向认证 | 需为 true 或者 false ，仅在imageurl使用https协议时有效|是 | | cacert | string | https或者https双向认证时使用的根证书文件 | 仅在imageurl使用https协议时有效| imageurl使用https协议时必选 | | clientcert | string | https双向认证时使用的客户端证书文件 | 仅在使用https双向认证时有效|mtls为true时必选 | | clientkey | string | https双向认证时使用的客户端公钥 | 仅在使用https双向认证时有效|mtls为true时必选 | - | evictpodforce | bool | 用于表示升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | - | sysconfigs | / | 需要进行配置的参数值 | 在配置或者升级或者回退机器时有效，在升级或者回退操作之后即机器重启之后起效，详细字段说明请见```配置（Settings）指导```| 可选 | - | upgradeconfigs | / | 需要升级前进行的配置的参数值 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | + | evictpodforce | bool | 升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | + | sysconfigs | / | 配置设置 | 1. “opstype=config”时只进行配置。 2.“opstype=upgrade/rollback”时，代表升级/回退后配置，即在升级/回退重启后进行配置。```配置（Settings）指导``` | “opstype=config”时必选 | + | upgradeconfigs | / | 升级前配置设置 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | #### 升级指导 @@ -271,13 +317,13 @@ sysconfigs: version: edit.os.version configs: - - model: kernel.systcl + - model: kernel.sysctl contents: - key: kernel param key1 value: kernel param value1 - key: kernel param key2 value: kernel param value2 - - model: kernel.systcl.persist + - model: kernel.sysctl.persist configpath: persist file path contents: - key: kernel param key3 @@ -287,7 +333,7 @@ upgradeconfigs: version: 1.0.0 configs: - - model: kernel.systcl + - model: kernel.sysctl contents: - key: kernel param key4 value: kernel param value4 @@ -311,12 +357,13 @@ kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' ``` -* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的 imageurl, osversion, checksum, maxunavailable, flagSafe 或者containerimage字段进行相应修改。 +* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改 #### 配置（Settings）指导 * Settings参数说明: - 以进行配置时的示例yaml为例对配置的参数进行说明，示例yaml如下： + + 基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致： ```yaml apiVersion: upgrade.openeuler.org/v1alpha1 @@ -330,72 +377,97 @@ maxunavailable: edit.node.config.number containerimage: "" evictpodforce: false - imageurl: "" checksum: "" - flagSafe: false - mtls: false sysconfigs: - version: 1.0.0 + version: edit.sysconfigs.version configs: - - model: kernel.systcl - contents: + - model: kernel.sysctl + contents: - key: kernel param key1 value: kernel param value1 - key: kernel param key2 value: kernel param value2 operation: delete - - model: kernel.systcl.persist + - model: kernel.sysctl.persist configpath: persist file path contents: - key: kernel param key3 - value: kernel param value3 + value: kernel param value3 + - model: grub.cmdline.current + contents: + - key: boot param key1 + - key: boot param key2 + value: boot param value2 + - key: boot param key3 + value: boot param value3 + operation: delete + - model: grub.cmdline.next + contents: + - key: boot param key4 + - key: boot param key5 + value: boot param value5 + - key: boot param key6 + value: boot param value6 + operation: delete ``` - * 配置的参数说明如下： - * version: 配置的版本,通过版本差异触发配置，请修改配置后更新 version - * configs: 具体配置内容 - * model: 进行的配置的类型，支持的配置类型请看[Settings 列表](#setting-列表) - * configpath: 如为持久化配置，配置文件路径 - * contents: 配置参数的 key / value 和对参数的操作。 - * key / value: 请看[Settings 列表](#setting-列表)对支持的配置的 key / value的说明。 - * operation: 若不指定operation，则默认为添加或更新。若指定为delete，代表删除目前OS中已配置的参数。 - **注意：** 当operation为delete时，yaml中的key/value必须和OS上想删除参数的key/value**一致**，否则删除失败。 - * upgradeconfigs与sysconfig参数相同，upgradeconfig为升级前进行的配置，仅在升级/回滚场景起效，在升级/回滚操作执行前进行配置，只进行配置或者需要升级/回滚重启后执行配置，使用sysconfigs + 配置的参数说明如下： + + | 参数 | 参数类型 | 参数说明 | 使用说明 | 配置中是否必选 | + | ---------- | -------- | --------------------------- | ------------------------------------------------------------ | ----------------------- | + | version | string | 配置的版本 | 通过version是否相等来判断配置是否触发，version为空（为""或者没有值）时同样进行判断，所以不配置sysconfigs/upgradeconfigs时，继存的version值会被清空并触发配置。 | 是 | + | configs | / | 具体配置内容 | 包含具体配置项列表。 | 是 | + | model | string | 配置的类型 | 支持的配置类型请看附录下的```Settings列表``` | 是 | + | configpath | string | 配置文件路径 | 仅在kernel.sysctl.persist配置类型中生效，请看附录下的```Settings列表```对配置文件路径的说明。 | 否 | + | contents | / | 具体key/value的值及操作类型 | 包含具体配置参数列表。 | 是 | + | key | string | 参数名称 | key不能为空，不能包含"="，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中每种配置类型对key的说明。 | 是 | + | value | string | 参数值 | key=value形式的参数中，value不能为空，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中对每种配置类型对value的说明。 | key=value形式的参数必选 | + | operation | string | 对参数进行的操作 | 仅对kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next类型的参数生效。默认为添加或更新。仅支持配置为delete，代表删除已存在的参数（key=value需完全一致才能删除）。 | 否 | + + + + * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置 + * 使用说明 + * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如上，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml + * 查看配置之前的节点的配置的版本和节点状态（NODESTATUS状态为idle） ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` * 执行命令，在集群中部署cr实例后，节点会根据配置的参数信息进行配置，再次查看节点状态(NODESTATUS变成config) ```shell kubectl apply -f upgrade_v1alpha1_os.yaml - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` * 再次查看节点的配置的版本确认节点是否配置完成(NODESTATUS恢复为idle) ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` -* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 +* 如果后续需要再次配置，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 #### 回退指导 * 回退场景 - * 虚拟机无法正常启动时，需要退回到上一可以启动的版本时进行回退操作，仅支持手动回退容器 OS 。 - * 虚拟机能够正常启动并且进入系统，需要将当前版本退回到老版本时进行回退操作，支持工具回退（类似升级方式）和手动回退，建议使用工具回退。 - * 配置出现错误，节点状态陷入config时，可以回退至上一个配置版本以恢复节点至idle状态。 - **注意**：在配置新版本时，出现错误前已经配置的参数无法回退。 + * 虚拟机无法正常启动时，可在grub启动项页面手动切换启动项，使系统回退至上一版本（即手动回退）。 + * 虚拟机能够正常启动并且进入系统时，支持工具回退和手动回退，建议使用工具回退。 + * 工具回退有两种方式： + 1. rollback模式直接回退至上一版本。 + 2. upgrade模式重新升级至上一版本 * 手动回退指导 - * 手动重启虚拟机，选择第二启动项进行回退，手动回退仅支持回退到本次升级之前的版本。 + + * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。 * 工具回退指导 * 回退至任意版本 - * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 + * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 + * YAML修改完成后执行更新命令，在集群中更新定制对象后，节点会根据配置的字段信息进行回退 ```shell @@ -444,13 +516,13 @@ sysconfigs: version: previous config version configs: - - model: kernel.systcl + - model: kernel.sysctl contents: - key: kernel param key1 value: kernel param value1 - key: kernel param key2 value: kernel param value2 - - model: kernel.systcl.persist + - model: kernel.sysctl.persist configpath: persist file path contents: - key: kernel param key3 @@ -467,23 +539,21 @@ * 查看节点容器 OS 版本(回退OS版本)或节点config版本&节点状态为idle(回退config版本)，确认回退是否成功。 ```shell - kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' - - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` -#### Admin容器 +## Admin容器镜像制作、部署和使用 KubeOS提供一个分离的包含sshd服务和hostshell工具的Admin容器，来帮助管理员在必要情况下登录KubeOS，其中的sshd服务由[sysmaster](https://gitee.com/openeuler/sysmaster)/systemd拉起。Admin容器部署后用户可通过ssh连接到节点的Admin容器，进入Admin容器后执行hostshell命令获取host的root shell。 -##### 部署方法 +### admin容器镜像制作 -以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/everything/aarch64/Packages/)到scripts/admin-container目录下。 +以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-SP1-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/update/aarch64/Packages/)到scripts/admin-container目录下。 -**修改**admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/docker_img)下载。 +修改admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts-sp1可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/docker_img)下载。 ```Dockerfile -FROM openeuler-22.03-lts +FROM openeuler-22.03-lts-sp1 RUN yum -y install openssh-clients util-linux @@ -514,7 +584,9 @@ bash -x kbimg.sh create admin-image -f admin-container/Dockerfile -d your_imageR docker push your_imageRepository/admin_imageName:version ``` -在master节点上部署Admin容器，需要提供ssh公钥来免密登录，**修改**并应用如下示例yaml文件: +### admin容器部署 + +在master节点上部署Admin容器，需要提供ssh公钥来免密登录，修改并应用如下示例yaml文件: ```yaml apiVersion: v1 @@ -583,6 +655,8 @@ spec: control-plane: admin-container-sysmaster ``` +### admin容器使用 + ssh到Admin容器，然后执行hostshell命令进入host root shell, 如： ```shell @@ -590,7 +664,7 @@ ssh -p your-exposed-port root@your.worker.node.ip hostshell ``` -##### hostshell +#### hostshell说明 为了保证KubeOS的轻便性，许多工具或命令没有安装在KubeOS内。因此，用户可以在制作Admin容器时，将期望使用的二进制文件放在容器内的如/usr/bin目录下。hostshell工具在执行时会将容器下的/usr/bin, /usr/sbin, /usr/local/bin, /usr/local/sbin路径添加到host root shell的环境变量。 @@ -605,11 +679,10 @@ hostshell #### kernel Settings -* kenerl.sysctl: 设置内核参数，key/value 表示内核参数的 key/value, 示例如下: - +* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: ```yaml configs: - - model: kernel.systcl + - model: kernel.sysctl contents: - key: user.max_user_namespaces value: 16384 @@ -617,12 +690,10 @@ hostshell value: 0 operation: delete ``` - -* kernel.sysctl.persist: 设置持久化内核参数，key/value 表示内核参数的 key/value, configpath为配置修改/新建的文件路径，如不指定configpath默认修改/etc/sysctl.conf - +* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: ```yaml configs: - - model: kernel.systcl.persist + - model: kernel.sysctl.persist configpath : /etc/persist.conf contents: - key: user.max_user_namespaces @@ -637,22 +708,38 @@ hostshell * grub.cmdline: 设置grub.cfg文件中的内核引导参数，该行参数在grub.cfg文件中类似如下示例： ```shell - linux /boot/vmlinuz root=UUID=5b1aaf5d-5b25-4e4b-a0d3-3d4c8d2e6a6e ro consoleblank=600 console=tty0 console=ttyS0,115200n8 selinux=1 panic=3 - ``` + linux /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + ``` - key/value 表示如上示例中内核引导参数的 key=value。 - **注意：** 当该参数有多个等号，如root=UUID=some-uuid时，配置时的key为第一个等号前的所有字符，value为第一个等号后的所有字符。 - 配置方法示例如下： +* KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置： + + - grub.cmdline.current：对当前分区的启动项参数进行配置。 + - grub.cmdline.next：对下一分区的启动项参数进行配置。 + +* 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。 + +* grub.cmdline.current/next支持“key=value”（value不能为空），也支持单key。若value中有“=”，例如“root=UUID=some-uuid”，key应设置为第一个“=”前的所有字符，value为第一个“=”后的所有字符。 配置方法示例如下： ```yaml configs: - - model: grub.cmdline - contents: - - key: selinux - value: 0 - - key: root - value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 - - key: panic - value: 3 - operation: delete + - model: grub.cmdline.current + contents: + - key: selinux + value: "0" + - key: root + value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 + - key: panic + value: "3" + operation: delete + - key: crash_kexec_post_notifiers + - model: grub.cmdline.next + contents: + - key: selinux + value: "0" + - key: root + value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 + - key: panic + value: "3" + operation: delete + - key: crash_kexec_post_notifiers ``` -- Gitee From 8cd9d4dd70ec0601490d0b1fd997ba4e05aff420 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 16 Jan 2024 17:25:17 +0800 Subject: [PATCH 14/46] Remove cleanup method and related code Signed-off-by: Yuhang Wei --- KubeOS-Rust/agent/src/rpc/agent.rs | 3 --- KubeOS-Rust/agent/src/rpc/agent_impl.rs | 14 +--------- KubeOS-Rust/cli/src/method/cleanup.rs | 29 --------------------- KubeOS-Rust/cli/src/method/mod.rs | 1 - KubeOS-Rust/manager/src/api/agent_status.rs | 11 +------- 5 files changed, 2 insertions(+), 56 deletions(-) delete mode 100644 KubeOS-Rust/cli/src/method/cleanup.rs diff --git a/KubeOS-Rust/agent/src/rpc/agent.rs b/KubeOS-Rust/agent/src/rpc/agent.rs index 13775afb..2496bfb4 100644 --- a/KubeOS-Rust/agent/src/rpc/agent.rs +++ b/KubeOS-Rust/agent/src/rpc/agent.rs @@ -22,9 +22,6 @@ pub trait Agent { #[rpc(name = "upgrade")] fn upgrade(&self) -> RpcResult; - #[rpc(name = "cleanup")] - fn cleanup(&self) -> RpcResult; - #[rpc(name = "configure")] fn configure(&self, req: ConfigureRequest) -> RpcResult; diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs index 7101d0db..bc1eabda 100644 --- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -17,7 +17,7 @@ use log::{debug, error, info}; use manager::{ api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH}, - utils::{clean_env, get_partition_info, switch_boot_menuentry, PreparePath, RealCommandExecutor}, + utils::{get_partition_info, switch_boot_menuentry, RealCommandExecutor}, }; use nix::{sys::reboot::RebootMode, unistd::sync}; @@ -40,10 +40,6 @@ impl Agent for AgentImpl { RpcFunction::call(|| self.upgrade_impl()) } - fn cleanup(&self) -> RpcResult { - RpcFunction::call(|| self.cleanup_impl()) - } - fn configure(&self, req: ConfigureRequest) -> RpcResult { RpcFunction::call(|| self.configure_impl(req)) } @@ -94,14 +90,6 @@ impl AgentImpl { Ok(Response { status: AgentStatus::Upgraded }) } - pub fn cleanup_impl(&self) -> Result { - let _lock = self.mutex.lock().unwrap(); - info!("Start to cleanup"); - let paths = PreparePath::default(); - clean_env(paths.update_path, paths.mount_path, paths.image_path)?; - Ok(Response { status: AgentStatus::CleanedUp }) - } - pub fn configure_impl(&self, mut req: ConfigureRequest) -> Result { let _lock = self.mutex.lock().unwrap(); debug!("Received a 'configure' request: {:?}", req); diff --git a/KubeOS-Rust/cli/src/method/cleanup.rs b/KubeOS-Rust/cli/src/method/cleanup.rs deleted file mode 100644 index d1d7dbe2..00000000 --- a/KubeOS-Rust/cli/src/method/cleanup.rs +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. - * KubeOS is licensed under the Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR - * PURPOSE. - * See the Mulan PSL v2 for more details. - */ - -use kubeos_manager::api; -use serde_json::value::RawValue; - -use crate::method::callable_method::RpcMethod; - -#[derive(Default)] -pub struct CleanupMethod {} - -impl RpcMethod for CleanupMethod { - type Response = api::Response; - fn command_name(&self) -> &'static str { - "cleanup" - } - fn command_params(&self) -> Vec> { - vec![] - } -} diff --git a/KubeOS-Rust/cli/src/method/mod.rs b/KubeOS-Rust/cli/src/method/mod.rs index b04b0fd8..e1f38bcd 100644 --- a/KubeOS-Rust/cli/src/method/mod.rs +++ b/KubeOS-Rust/cli/src/method/mod.rs @@ -11,7 +11,6 @@ */ pub mod callable_method; -pub mod cleanup; pub mod configure; pub mod prepare_upgrade; pub mod request; diff --git a/KubeOS-Rust/manager/src/api/agent_status.rs b/KubeOS-Rust/manager/src/api/agent_status.rs index e466a50c..bb16e6bc 100644 --- a/KubeOS-Rust/manager/src/api/agent_status.rs +++ b/KubeOS-Rust/manager/src/api/agent_status.rs @@ -12,19 +12,10 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)] pub enum AgentStatus { - Unknown, - NotApplied, UpgradeReady, Upgraded, Rollbacked, Configured, - CleanedUp, -} - -impl Default for AgentStatus { - fn default() -> Self { - Self::Unknown - } } -- Gitee From 916ca24576d33dc024944a7ed18aaa39e95753f9 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 18 Jan 2024 11:13:20 +0800 Subject: [PATCH 15/46] test(rust proxy):add drain integration test move drain into a lib for integration test. use kind to deploy a cluster for integration test. Signed-off-by: Yuhang Wei --- KubeOS-Rust/proxy/Cargo.toml | 7 ++ .../proxy/src/controller/controller.rs | 4 +- KubeOS-Rust/proxy/src/controller/mod.rs | 1 - KubeOS-Rust/proxy/src/controller/values.rs | 12 --- .../proxy/src/{controller => }/drain.rs | 12 ++- KubeOS-Rust/proxy/tests/common/mod.rs | 63 +++++++++++ KubeOS-Rust/proxy/tests/drain_test.rs | 41 +++++++ .../proxy/tests/setup/kind-config.yaml | 5 + KubeOS-Rust/proxy/tests/setup/resources.yaml | 102 ++++++++++++++++++ .../proxy/tests/setup/setup_test_env.sh | 81 ++++++++++++++ 10 files changed, 309 insertions(+), 19 deletions(-) rename KubeOS-Rust/proxy/src/{controller => }/drain.rs (97%) create mode 100644 KubeOS-Rust/proxy/tests/common/mod.rs create mode 100644 KubeOS-Rust/proxy/tests/drain_test.rs create mode 100644 KubeOS-Rust/proxy/tests/setup/kind-config.yaml create mode 100644 KubeOS-Rust/proxy/tests/setup/resources.yaml create mode 100644 KubeOS-Rust/proxy/tests/setup/setup_test_env.sh diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index 9a148e89..72eb6b92 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -6,6 +6,13 @@ name = "proxy" version = "0.1.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "drain" +path = "src/drain.rs" + +[[bin]] +name = "proxy" +path = "src/main.rs" [dependencies] anyhow = "1.0.44" diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index e7ee9f9b..b2bb332a 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -13,6 +13,7 @@ use std::{collections::HashMap, env}; use anyhow::Result; +use drain::drain_os; use k8s_openapi::api::core::v1::Node; use kube::{ api::{Api, PostParams}, @@ -29,7 +30,6 @@ use super::{ agentclient::{AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, apiclient::ApplyApi, crd::{Configs, Content, OSInstance, OS}, - drain::drain_os, utils::{check_version, get_config_version, ConfigOperation, ConfigType}, values::{ LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, @@ -340,7 +340,7 @@ impl ProxyController { } async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> { - use crate::controller::drain::error::DrainError::*; + use drain::error::DrainError::*; match drain_os(&self.k8s_client.clone(), node_name, force).await { Err(DeletePodsError { errors, .. }) => Err(Error::DrainNodeError { value: errors.join("; ") }), _ => Ok(()), diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs index 384d74b9..73be45c9 100644 --- a/KubeOS-Rust/proxy/src/controller/mod.rs +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -16,7 +16,6 @@ mod apiclient; mod apiserver_mock; mod controller; mod crd; -mod drain; mod utils; mod values; diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs index fe43851f..dec905a9 100644 --- a/KubeOS-Rust/proxy/src/controller/values.rs +++ b/KubeOS-Rust/proxy/src/controller/values.rs @@ -31,15 +31,3 @@ pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; - -pub const MAX_EVICT_POD_NUM: usize = 5; - -pub const EVERY_EVICTION_RETRY: Duration = Duration::from_secs(5); - -pub const EVERY_DELETION_CHECK: Duration = Duration::from_secs(5); - -pub const TIMEOUT: Duration = Duration::from_secs(u64::MAX); - -pub const RETRY_BASE_DELAY: Duration = Duration::from_millis(100); -pub const RETRY_MAX_DELAY: Duration = Duration::from_secs(20); -pub const MAX_RETRIES_TIMES: usize = 10; diff --git a/KubeOS-Rust/proxy/src/controller/drain.rs b/KubeOS-Rust/proxy/src/drain.rs similarity index 97% rename from KubeOS-Rust/proxy/src/controller/drain.rs rename to KubeOS-Rust/proxy/src/drain.rs index ddc38aed..09cf6625 100644 --- a/KubeOS-Rust/proxy/src/controller/drain.rs +++ b/KubeOS-Rust/proxy/src/drain.rs @@ -29,10 +29,14 @@ use self::error::{ DrainError::{DeletePodsError, GetPodListsError, WaitDeletionError}, EvictionError::{EvictionErrorNoRetry, EvictionErrorRetry}, }; -use super::values::{ - EVERY_DELETION_CHECK, EVERY_EVICTION_RETRY, MAX_EVICT_POD_NUM, MAX_RETRIES_TIMES, RETRY_BASE_DELAY, - RETRY_MAX_DELAY, TIMEOUT, -}; + +pub const MAX_EVICT_POD_NUM: usize = 5; +pub const EVERY_EVICTION_RETRY: Duration = Duration::from_secs(5); +pub const EVERY_DELETION_CHECK: Duration = Duration::from_secs(5); +pub const TIMEOUT: Duration = Duration::from_secs(u64::MAX); +pub const RETRY_BASE_DELAY: Duration = Duration::from_millis(100); +pub const RETRY_MAX_DELAY: Duration = Duration::from_secs(20); +pub const MAX_RETRIES_TIMES: usize = 10; pub async fn drain_os(client: &Client, node_name: &str, force: bool) -> Result<(), error::DrainError> { let pods_list = get_pods_deleted(client, node_name, force).await?; diff --git a/KubeOS-Rust/proxy/tests/common/mod.rs b/KubeOS-Rust/proxy/tests/common/mod.rs new file mode 100644 index 00000000..82577597 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/common/mod.rs @@ -0,0 +1,63 @@ +use std::process::{Command, Stdio}; + +use anyhow::Result; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::ResourceExt, + client::Client, + config::{Config, KubeConfigOptions, Kubeconfig}, + Api, +}; +use manager::utils::{CommandExecutor, RealCommandExecutor}; + +pub const CLUSTER: &str = "kubeos-test"; + +pub fn run_command(cmd: &str, args: &[&str]) -> Result<()> { + let output = Command::new(cmd).args(args).stdout(Stdio::inherit()).stderr(Stdio::inherit()).output()?; + if !output.status.success() { + println!("failed to run command: {} {}\n", cmd, args.join(" ")); + } + Ok(()) +} + +pub async fn setup() -> Result { + // set PATH variable + let path = std::env::var("PATH").unwrap(); + let new_path = format!("{}:{}", path, "../../bin"); + std::env::set_var("PATH", new_path); + + // create cluster + let executor = RealCommandExecutor {}; + println!("Creating cluster"); + run_command("bash", &["./tests/setup/setup_test_env.sh"]).expect("failed to create cluster"); + + // connect to the cluster + let kind_config = executor.run_command_with_output("kind", &["get", "kubeconfig", "-n", CLUSTER]).unwrap(); + let kubeconfig = Kubeconfig::from_yaml(kind_config.as_str()).expect("failed to parse kubeconfig"); + let options = KubeConfigOptions::default(); + let config = Config::from_custom_kubeconfig(kubeconfig, &&options).await.expect("failed to create config"); + let client = Client::try_from(config).expect("failed to create client"); + // list all nodes + let nodes: Api = Api::all(client.clone()); + let node_list = nodes.list(&Default::default()).await.expect("failed to list nodes"); + for n in node_list { + println!("Found Node: {}", n.name()); + } + // check node status + let node = nodes.get("kubeos-test-worker").await.unwrap(); + let status = node.status.unwrap(); + let conditions = status.conditions.unwrap(); + for c in conditions { + if c.type_ == "Ready" { + assert_eq!(c.status, "True"); + } + } + println!("Cluster ready"); + Ok(client) +} + +pub fn clean_env() { + let executor = RealCommandExecutor {}; + println!("Cleaning cluster"); + executor.run_command("kind", &["delete", "clusters", CLUSTER]).expect("failed to clean cluster"); +} diff --git a/KubeOS-Rust/proxy/tests/drain_test.rs b/KubeOS-Rust/proxy/tests/drain_test.rs new file mode 100644 index 00000000..2f4f1501 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/drain_test.rs @@ -0,0 +1,41 @@ +mod common; + +use common::*; +use drain::drain_os; +use k8s_openapi::api::core::v1::{Node, Pod}; +use kube::Api; + +#[tokio::test] +#[ignore = "integration test"] +async fn test_drain() { + let client = setup().await.unwrap(); + // drain node + let nodes: Api = Api::all(client.clone()); + let node_name = "kubeos-test-worker"; + println!("cordon node"); + nodes.cordon(node_name).await.unwrap(); + println!("drain node"); + drain_os(&client, node_name, true).await.unwrap(); + + // assert unschedulable + println!("check node unschedulable"); + let node = nodes.get(node_name).await.unwrap(); + if let Some(spec) = node.spec { + assert_eq!(spec.unschedulable, Some(true)); + } else { + panic!("node spec is none"); + } + // list all pods on kubeos-test-worker node and all pods should belong to daemonset + println!("list all pods on kubeos-test-worker node"); + let pods: Api = Api::all(client.clone()); + let pod_list = pods.list(&Default::default()).await.unwrap(); + // check the pod is from daemonset + for p in pod_list { + if p.spec.unwrap().node_name.unwrap() == node_name { + assert_eq!(p.metadata.owner_references.unwrap()[0].kind, "DaemonSet"); + } + } + nodes.uncordon(node_name).await.unwrap(); + + clean_env() +} diff --git a/KubeOS-Rust/proxy/tests/setup/kind-config.yaml b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml new file mode 100644 index 00000000..0fe29e73 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker \ No newline at end of file diff --git a/KubeOS-Rust/proxy/tests/setup/resources.yaml b/KubeOS-Rust/proxy/tests/setup/resources.yaml new file mode 100644 index 00000000..0e449d5d --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/resources.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: example-daemonset +spec: + selector: + matchLabels: + name: example-daemonset + template: + metadata: + labels: + name: example-daemonset + spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-with-local-storage +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] + volumeMounts: + - mountPath: "/data" + name: local-volume + volumes: + - name: local-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Pod +metadata: + name: standalone-pod +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-deployment +spec: + replicas: 2 + selector: + matchLabels: + app: example + template: + metadata: + labels: + app: example + spec: + containers: + - name: nginx + image: nginx:alpine + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: DoesNotExist + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: example-pdb +spec: + minAvailable: 1 + selector: + matchLabels: + app: example +--- +apiVersion: v1 +kind: Pod +metadata: + name: resource-intensive-pod +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] + resources: + requests: + memory: "256Mi" + cpu: "500m" + limits: + memory: "512Mi" + cpu: "1000m" + diff --git a/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh new file mode 100644 index 00000000..d24d8e01 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# this bash script executes in proxy directory + +set -Eeuxo pipefail + +# Define variables +KIND_VERSION="v0.19.0" +KUBECTL_VERSION="v1.24.15" +KIND_CLUSTER_NAME="kubeos-test" +DOCKER_IMAGES=("busybox:stable" "nginx:alpine" "kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab") +NODE_IMAGE="kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab" +RESOURCE="./tests/setup/resources.yaml" +KIND_CONFIG="./tests/setup/kind-config.yaml" +BIN_PATH="../../bin/" +ARCH=$(uname -m) + +# Install kind and kubectl +install_bins() { + # if bin dir not exist then create + if [ ! -d "${BIN_PATH}" ]; then + mkdir -p "${BIN_PATH}" + fi + if [ ! -f "${BIN_PATH}"kind ]; then + echo "Installing Kind..." + # For AMD64 / x86_64 + if [ "$ARCH" = x86_64 ]; then + # add proxy if you are behind proxy + curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-amd64 + fi + # For ARM64 + if [ "$ARCH" = aarch64 ]; then + curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-arm64 + fi + chmod +x "${BIN_PATH}"kind + fi + if [ ! -f "${BIN_PATH}"kubectl ]; then + echo "Installing kubectl..." + if [ "$ARCH" = x86_64 ]; then + curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + fi + if [ "$ARCH" = aarch64 ]; then + curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/arm64/kubectl" + fi + chmod +x "${BIN_PATH}"kubectl + fi + export PATH=$PATH:"${BIN_PATH}" +} + +# Create Kind Cluster +create_cluster() { + echo "Creating Kind cluster..." + for image in "${DOCKER_IMAGES[@]}"; do + docker pull "$image" + done + kind create cluster --name "${KIND_CLUSTER_NAME}" --config "${KIND_CONFIG}" --image "${NODE_IMAGE}" +} + +# Load Docker image into Kind cluster +load_docker_image() { + echo "Loading Docker image into Kind cluster..." + DOCKER_IMAGE=$(printf "%s " "${DOCKER_IMAGES[@]:0:2}") + kind load docker-image ${DOCKER_IMAGE} --name "${KIND_CLUSTER_NAME}" +} + +# Apply Kubernetes resource files +apply_k8s_resources() { + echo "Applying Kubernetes resources from ${RESOURCE}..." + kubectl apply -f "${RESOURCE}" + echo "Waiting for nodes getting ready..." + sleep 40s +} + +main() { + export no_proxy=localhost,127.0.0.1 + install_bins + create_cluster + load_docker_image + apply_k8s_resources +} + +main -- Gitee From adb73fcebd1d7698a2fffdb791c2e9d08e949d9c Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Wed, 17 Jan 2024 14:39:34 +0800 Subject: [PATCH 16/46] refactor(rust os-agent): fix code check Signed-off-by: Yuhang Wei --- KubeOS-Rust/cli/src/method/request.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs index 2dc1ffba..b4a24aac 100644 --- a/KubeOS-Rust/cli/src/method/request.rs +++ b/KubeOS-Rust/cli/src/method/request.rs @@ -36,13 +36,12 @@ pub fn parse_error(error: Error) -> anyhow::Error { debug!("Json parse error: {:?}", e); anyhow!("Failed to parse response") }, - Error::Rpc(ref e) => match e.message == "Method not found" { - true => { + Error::Rpc(ref e) => { + if e.message == "Method not found" { anyhow!("Method is unimplemented") - }, - false => { + } else { anyhow!("{}", e.message) - }, + } }, _ => { debug!("{:?}", error); -- Gitee From 25a856bf7bfa7f2de32ecb7b4c8d6997a2835f76 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Sat, 25 Nov 2023 15:18:23 +0800 Subject: [PATCH 17/46] fix(agent, proxy): transform log timestamp to human-readable format Originally, the log of controllers is timestamp which is hard to read. Now, transform the log into more human-readable format. Signed-off-by: Yuhang Wei --- cmd/operator/controllers/os_controller_test.go | 3 +-- cmd/operator/controllers/suite_test.go | 12 +++++++++++- cmd/operator/main.go | 13 +++++++++++++ cmd/proxy/controllers/suite_test.go | 11 ++++++++++- cmd/proxy/main.go | 13 +++++++++++++ go.mod | 3 ++- go.sum | 8 ++++++++ 7 files changed, 58 insertions(+), 5 deletions(-) diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index 6cc2760e..8c5d1981 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -23,7 +23,6 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -912,7 +911,7 @@ func Test_getNodes(t *testing.T) { tests := []struct { name string args args - want []corev1.Node + want []v1.Node wantErr bool }{ { diff --git a/cmd/operator/controllers/suite_test.go b/cmd/operator/controllers/suite_test.go index aa6deeae..67fc9e71 100644 --- a/cmd/operator/controllers/suite_test.go +++ b/cmd/operator/controllers/suite_test.go @@ -16,9 +16,13 @@ import ( "context" "path/filepath" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + zaplogfmt "github.com/sykesm/zap-logfmt" + uzap "go.uber.org/zap" + "go.uber.org/zap/zapcore" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" @@ -46,7 +50,13 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + configLog := uzap.NewProductionEncoderConfig() + configLog.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) { + encoder.AppendString(ts.UTC().Format(time.RFC3339Nano)) + } + logfmtEncoder := zaplogfmt.NewEncoder(configLog) + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Encoder(logfmtEncoder))) + ctx, cancel = context.WithCancel(context.TODO()) By("bootstrapping test environment") diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 8249ad2d..6b90b26b 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -14,12 +14,17 @@ package main import ( "os" + "time" + zaplogfmt "github.com/sykesm/zap-logfmt" + uzap "go.uber.org/zap" + "go.uber.org/zap/zapcore" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" upgradev1 "openeuler.org/KubeOS/api/v1alpha1" "openeuler.org/KubeOS/cmd/operator/controllers" @@ -41,6 +46,14 @@ func init() { } func main() { + configLog := uzap.NewProductionEncoderConfig() + configLog.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) { + encoder.AppendString(ts.UTC().Format(time.RFC3339Nano)) + } + logfmtEncoder := zaplogfmt.NewEncoder(configLog) + logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder)) + ctrl.SetLogger(logger) + mgr, err := common.NewControllerManager(setupLog, scheme) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/cmd/proxy/controllers/suite_test.go b/cmd/proxy/controllers/suite_test.go index 00eebbf4..767fe955 100644 --- a/cmd/proxy/controllers/suite_test.go +++ b/cmd/proxy/controllers/suite_test.go @@ -16,9 +16,13 @@ import ( "context" "path/filepath" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + zaplogfmt "github.com/sykesm/zap-logfmt" + uzap "go.uber.org/zap" + "go.uber.org/zap/zapcore" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" @@ -48,7 +52,12 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + configLog := uzap.NewProductionEncoderConfig() + configLog.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) { + encoder.AppendString(ts.UTC().Format(time.RFC3339Nano)) + } + logfmtEncoder := zaplogfmt.NewEncoder(configLog) + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Encoder(logfmtEncoder))) ctx, cancel = context.WithCancel(context.TODO()) By("bootstrapping test environment") diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 3a537d90..e606083a 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -15,12 +15,17 @@ package main import ( "os" "path/filepath" + "time" + zaplogfmt "github.com/sykesm/zap-logfmt" + uzap "go.uber.org/zap" + "go.uber.org/zap/zapcore" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" upgradev1 "openeuler.org/KubeOS/api/v1alpha1" "openeuler.org/KubeOS/cmd/agent/server" @@ -44,6 +49,14 @@ func init() { } func main() { + configLog := uzap.NewProductionEncoderConfig() + configLog.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) { + encoder.AppendString(ts.UTC().Format(time.RFC3339Nano)) + } + logfmtEncoder := zaplogfmt.NewEncoder(configLog) + logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder)) + ctrl.SetLogger(logger) + var err error mgr, err := common.NewControllerManager(setupLog, scheme) if err != nil { diff --git a/go.mod b/go.mod index 057292c8..72ca9782 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,8 @@ require ( github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.20.0 github.com/sirupsen/logrus v1.8.1 + github.com/sykesm/zap-logfmt v0.0.4 + go.uber.org/zap v1.19.1 google.golang.org/grpc v1.49.0 google.golang.org/protobuf v1.28.1 k8s.io/api v0.24.0 @@ -82,7 +84,6 @@ require ( go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.19.1 // indirect golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect diff --git a/go.sum b/go.sum index 6bd1ba1b..325cd88a 100644 --- a/go.sum +++ b/go.sum @@ -516,6 +516,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/sykesm/zap-logfmt v0.0.4 h1:U2WzRvmIWG1wDLCFY3sz8UeEmsdHQjHFNlIdmroVFaI= +github.com/sykesm/zap-logfmt v0.0.4/go.mod h1:AuBd9xQjAe3URrWT1BBDk2v2onAZHkZkWRMiYZXiZWA= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -559,15 +561,19 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= @@ -812,6 +818,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -- Gitee From e1b4b7a7008855920f866d0fa4c16d09f9341baf Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Sat, 25 Nov 2023 15:18:36 +0800 Subject: [PATCH 18/46] build: update vendor use zapfmt to transform timestamp to human-readable format Signed-off-by: Yuhang Wei --- .../github.com/sykesm/zap-logfmt/.gitignore | 1 + .../github.com/sykesm/zap-logfmt/.travis.yml | 12 + vendor/github.com/sykesm/zap-logfmt/LICENSE | 21 + vendor/github.com/sykesm/zap-logfmt/README.md | 76 +++ .../github.com/sykesm/zap-logfmt/encoder.go | 527 ++++++++++++++++++ vendor/modules.txt | 3 + 6 files changed, 640 insertions(+) create mode 100644 vendor/github.com/sykesm/zap-logfmt/.gitignore create mode 100644 vendor/github.com/sykesm/zap-logfmt/.travis.yml create mode 100644 vendor/github.com/sykesm/zap-logfmt/LICENSE create mode 100644 vendor/github.com/sykesm/zap-logfmt/README.md create mode 100644 vendor/github.com/sykesm/zap-logfmt/encoder.go diff --git a/vendor/github.com/sykesm/zap-logfmt/.gitignore b/vendor/github.com/sykesm/zap-logfmt/.gitignore new file mode 100644 index 00000000..7a6353d6 --- /dev/null +++ b/vendor/github.com/sykesm/zap-logfmt/.gitignore @@ -0,0 +1 @@ +.envrc diff --git a/vendor/github.com/sykesm/zap-logfmt/.travis.yml b/vendor/github.com/sykesm/zap-logfmt/.travis.yml new file mode 100644 index 00000000..7ce1f7ab --- /dev/null +++ b/vendor/github.com/sykesm/zap-logfmt/.travis.yml @@ -0,0 +1,12 @@ +language: go + +matrix: + include: + - go: "1.13.x" + install: true + - go: "1.14.x" + install: true + - go: "1.15.x" + install: true + +script: go test -race ./... diff --git a/vendor/github.com/sykesm/zap-logfmt/LICENSE b/vendor/github.com/sykesm/zap-logfmt/LICENSE new file mode 100644 index 00000000..43a1c326 --- /dev/null +++ b/vendor/github.com/sykesm/zap-logfmt/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Jonathan Sternberg +Copyright (c) 2019 Matthew Sykes + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/sykesm/zap-logfmt/README.md b/vendor/github.com/sykesm/zap-logfmt/README.md new file mode 100644 index 00000000..751f2880 --- /dev/null +++ b/vendor/github.com/sykesm/zap-logfmt/README.md @@ -0,0 +1,76 @@ +# Logfmt Encoder + +This package provides a logfmt encoder for [zap][zap]. + +It is a fork of [github.com/jsternberg/zap-logfmt][jsternberg] that improves +the handling of reflected fields and encodes arrays and objects instead of +dropping them from logs. While logging simple fields is preferred for many +reasons, having ugly data is often better than missing data. + +[![Build Status](https://travis-ci.org/sykesm/zap-logfmt.svg?branch=master)](https://travis-ci.org/sykesm/zap-logfmt) +[![GoDoc](https://godoc.org/github.com/sykesm/zap-logfmt?status.svg)](https://godoc.org/github.com/sykesm/zap-logfmt) + +## Usage + +The encoder is easy to configure. Simply create a new core with an instance of +the logfmt encoder and use it with your preferred logging interface. + +```go +package main + +import ( + "os" + + "github.com/sykesm/zap-logfmt" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func main() { + config := zap.NewProductionEncoderConfig() + logger := zap.New(zapcore.NewCore( + zaplogfmt.NewEncoder(config), + os.Stdout, + zapcore.DebugLevel, + )) + logger.Info("Hello World") +} +``` + +## Arrays, Objects, and Reflected Fields + +While it's best to avoid complex data types in log fields, there are times +when they sneak in. When complex fields are included in log records, they will +be encoded, but they won't be very pretty. + +### Arrays + +Arrays are encoded as a comma separated list of values within square brackets. +This format is very similar to JSON encoding. Arrays of simple scalars remain +quite readable but including elements that require quoting will result in very +ugly records. + +### Objects + +Objects are encoded as a space separated list of _key=value_ pairs. Because +this format includes an equals sign, the encoded object will require quoting. +If any value in the object requires quoting, the required escapes will make +the encoded field pretty difficult for humans to read. + +### Channels and Functions + +Channels and functions are encoded as their type and their address. There +aren't many meaningful ways to log channels and functions... + +### Maps and Structs + +Maps and structs are encoded as strings that contain the result of `fmt.Sprint`. + +## Namespaces + +Namespaces are supported. If a namespace is opened, all of the keys will +be prepended with the namespace name. For example, with the namespace +`foo` and the key `bar`, you would get a key of `foo.bar`. + +[zap]: https://github.com/uber-go/zap +[jsternberg]: https://github.com/jsternberg/zap-logfmt diff --git a/vendor/github.com/sykesm/zap-logfmt/encoder.go b/vendor/github.com/sykesm/zap-logfmt/encoder.go new file mode 100644 index 00000000..9e960cc5 --- /dev/null +++ b/vendor/github.com/sykesm/zap-logfmt/encoder.go @@ -0,0 +1,527 @@ +// Package zaplogfmt provides a zap encoder that formats log entries in +// "logfmt" format. +package zaplogfmt + +import ( + "bytes" + "encoding" + "encoding/base64" + "encoding/json" + "fmt" + "math" + "reflect" + "strings" + "sync" + "time" + "unicode/utf8" + + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +var ( + logfmtPool = sync.Pool{ + New: func() interface{} { return &logfmtEncoder{} }, + } + bufferpool = buffer.NewPool() +) + +func getEncoder() *logfmtEncoder { + return logfmtPool.Get().(*logfmtEncoder) +} + +func putEncoder(enc *logfmtEncoder) { + enc.EncoderConfig = nil + enc.buf = nil + enc.namespaces = nil + enc.arrayLiteral = false + logfmtPool.Put(enc) +} + +type logfmtEncoder struct { + *zapcore.EncoderConfig + buf *buffer.Buffer + namespaces []string + arrayLiteral bool +} + +// NewEncoder creates an encoder writes logfmt formatted log entries. +func NewEncoder(cfg zapcore.EncoderConfig) zapcore.Encoder { + return &logfmtEncoder{ + EncoderConfig: &cfg, + buf: bufferpool.Get(), + } +} + +func (enc *logfmtEncoder) AddArray(key string, arr zapcore.ArrayMarshaler) error { + enc.addKey(key) + return enc.AppendArray(arr) +} + +func (enc *logfmtEncoder) AddBinary(key string, value []byte) { + enc.AddString(key, base64.StdEncoding.EncodeToString(value)) +} + +func (enc *logfmtEncoder) AddBool(key string, value bool) { + enc.addKey(key) + enc.AppendBool(value) +} + +func (enc *logfmtEncoder) AddByteString(key string, value []byte) { + enc.addKey(key) + enc.AppendByteString(value) +} + +func (enc *logfmtEncoder) AddComplex64(k string, v complex64) { enc.AddComplex128(k, complex128(v)) } +func (enc *logfmtEncoder) AddComplex128(key string, value complex128) { + enc.addKey(key) + enc.AppendComplex128(value) +} + +func (enc *logfmtEncoder) AddDuration(key string, value time.Duration) { + enc.addKey(key) + enc.AppendDuration(value) +} + +func (enc *logfmtEncoder) AddFloat32(key string, value float32) { + enc.addKey(key) + enc.AppendFloat32(value) +} + +func (enc *logfmtEncoder) AddFloat64(key string, value float64) { + enc.addKey(key) + enc.AppendFloat64(value) +} + +func (enc *logfmtEncoder) AddInt(k string, v int) { enc.AddInt64(k, int64(v)) } +func (enc *logfmtEncoder) AddInt8(k string, v int8) { enc.AddInt64(k, int64(v)) } +func (enc *logfmtEncoder) AddInt32(k string, v int32) { enc.AddInt64(k, int64(v)) } +func (enc *logfmtEncoder) AddInt16(k string, v int16) { enc.AddInt64(k, int64(v)) } +func (enc *logfmtEncoder) AddInt64(key string, value int64) { + enc.addKey(key) + enc.AppendInt64(value) +} + +func (enc *logfmtEncoder) AddObject(key string, obj zapcore.ObjectMarshaler) error { + enc.addKey(key) + return enc.AppendObject(obj) +} + +func (enc *logfmtEncoder) AddReflected(key string, value interface{}) error { + enc.addKey(key) + return enc.AppendReflected(value) +} + +func (enc *logfmtEncoder) AddString(key, value string) { + enc.addKey(key) + enc.AppendString(value) +} + +func (enc *logfmtEncoder) AddTime(key string, value time.Time) { + enc.addKey(key) + enc.AppendTime(value) +} + +func (enc *logfmtEncoder) AddUint(k string, v uint) { enc.AddUint64(k, uint64(v)) } +func (enc *logfmtEncoder) AddUint8(k string, v uint8) { enc.AddUint64(k, uint64(v)) } +func (enc *logfmtEncoder) AddUint32(k string, v uint32) { enc.AddUint64(k, uint64(v)) } +func (enc *logfmtEncoder) AddUint16(k string, v uint16) { enc.AddUint64(k, uint64(v)) } +func (enc *logfmtEncoder) AddUintptr(k string, v uintptr) { enc.AddUint64(k, uint64(v)) } +func (enc *logfmtEncoder) AddUint64(key string, value uint64) { + enc.addKey(key) + enc.AppendUint64(value) +} + +func (enc *logfmtEncoder) AppendArray(arr zapcore.ArrayMarshaler) error { + marshaler := enc.clone() + marshaler.namespaces = nil + marshaler.arrayLiteral = true + + marshaler.buf.AppendByte('[') + err := arr.MarshalLogArray(marshaler) + if err == nil { + marshaler.buf.AppendByte(']') + enc.AppendByteString(marshaler.buf.Bytes()) + } else { + enc.AppendByteString(nil) + } + marshaler.buf.Free() + putEncoder(marshaler) + return err +} + +func (enc *logfmtEncoder) AppendBool(value bool) { + if value { + enc.AppendString("true") + } else { + enc.AppendString("false") + } +} + +func (enc *logfmtEncoder) AppendByteString(value []byte) { + enc.addSeparator() + + needsQuotes := bytes.IndexFunc(value, needsQuotedValueRune) != -1 + if needsQuotes { + enc.buf.AppendByte('"') + } + enc.safeAddByteString(value) + if needsQuotes { + enc.buf.AppendByte('"') + } +} + +func (enc *logfmtEncoder) AppendComplex64(v complex64) { enc.AppendComplex128(complex128(v)) } +func (enc *logfmtEncoder) AppendComplex128(value complex128) { + enc.addSeparator() + + // Cast to a platform-independent, fixed-size type. + r, i := float64(real(value)), float64(imag(value)) + enc.buf.AppendFloat(r, 64) + enc.buf.AppendByte('+') + enc.buf.AppendFloat(i, 64) + enc.buf.AppendByte('i') +} + +func (enc *logfmtEncoder) AppendDuration(value time.Duration) { + cur := enc.buf.Len() + if enc.EncodeDuration != nil { + enc.EncodeDuration(value, enc) + } + if cur == enc.buf.Len() { + enc.AppendInt64(int64(value)) + } +} + +func (enc *logfmtEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), 32) } +func (enc *logfmtEncoder) AppendFloat64(v float64) { enc.appendFloat(v, 64) } +func (enc *logfmtEncoder) appendFloat(val float64, bitSize int) { + enc.addSeparator() + + switch { + case math.IsNaN(val): + enc.buf.AppendString(`NaN`) + case math.IsInf(val, 1): + enc.buf.AppendString(`+Inf`) + case math.IsInf(val, -1): + enc.buf.AppendString(`-Inf`) + default: + enc.buf.AppendFloat(val, bitSize) + } +} + +func (enc *logfmtEncoder) AppendInt(v int) { enc.AppendInt64(int64(v)) } +func (enc *logfmtEncoder) AppendInt8(v int8) { enc.AppendInt64(int64(v)) } +func (enc *logfmtEncoder) AppendInt16(v int16) { enc.AppendInt64(int64(v)) } +func (enc *logfmtEncoder) AppendInt32(v int32) { enc.AppendInt64(int64(v)) } +func (enc *logfmtEncoder) AppendInt64(value int64) { + enc.addSeparator() + enc.buf.AppendInt(value) +} + +func (enc *logfmtEncoder) AppendObject(obj zapcore.ObjectMarshaler) error { + marshaler := enc.clone() + marshaler.namespaces = nil + + err := obj.MarshalLogObject(marshaler) + if err == nil { + enc.AppendByteString(marshaler.buf.Bytes()) + } else { + enc.AppendByteString(nil) + } + marshaler.buf.Free() + putEncoder(marshaler) + return err +} + +func (enc *logfmtEncoder) AppendReflected(value interface{}) error { + switch v := value.(type) { + case nil: + enc.AppendString("null") + case error: + enc.AppendString(v.Error()) + case []byte: + enc.AppendByteString(v) + case fmt.Stringer: + enc.AppendString(v.String()) + case encoding.TextMarshaler: + b, err := v.MarshalText() + if err != nil { + return err + } + enc.AppendString(string(b)) + case json.Marshaler: + b, err := v.MarshalJSON() + if err != nil { + return err + } + enc.AppendString(string(b)) + default: + rvalue := reflect.ValueOf(value) + switch rvalue.Kind() { + case reflect.Bool: + enc.AppendBool(rvalue.Bool()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + enc.AppendInt64(rvalue.Int()) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + enc.AppendUint64(rvalue.Uint()) + case reflect.Float32: + enc.appendFloat(rvalue.Float(), 32) + case reflect.Float64: + enc.AppendFloat64(rvalue.Float()) + case reflect.String: + enc.AppendString(rvalue.String()) + case reflect.Complex64, reflect.Complex128: + enc.AppendComplex128(rvalue.Complex()) + case reflect.Chan, reflect.Func: + enc.AppendString(fmt.Sprintf("%T(%p)", value, value)) + case reflect.Map, reflect.Struct: + enc.AppendString(fmt.Sprint(value)) + case reflect.Array, reflect.Slice: + enc.AppendArray(zapcore.ArrayMarshalerFunc(func(ae zapcore.ArrayEncoder) error { + for i := 0; i < rvalue.Len(); i++ { + ae.AppendReflected(rvalue.Index(i).Interface()) + } + return nil + })) + case reflect.Interface, reflect.Ptr: + return enc.AppendReflected(rvalue.Elem().Interface()) + } + } + return nil +} + +func (enc *logfmtEncoder) AppendString(value string) { + enc.addSeparator() + + needsQuotes := strings.IndexFunc(value, needsQuotedValueRune) != -1 + if needsQuotes { + enc.buf.AppendByte('"') + } + enc.safeAddString(value) + if needsQuotes { + enc.buf.AppendByte('"') + } +} + +func (enc *logfmtEncoder) AppendTime(value time.Time) { + cur := enc.buf.Len() + if enc.EncodeTime != nil { + enc.EncodeTime(value, enc) + } + if cur == enc.buf.Len() { + enc.AppendInt64(value.UnixNano()) + } +} + +func (enc *logfmtEncoder) AppendUint(v uint) { enc.AppendUint64(uint64(v)) } +func (enc *logfmtEncoder) AppendUint8(v uint8) { enc.AppendUint64(uint64(v)) } +func (enc *logfmtEncoder) AppendUint16(v uint16) { enc.AppendUint64(uint64(v)) } +func (enc *logfmtEncoder) AppendUint32(v uint32) { enc.AppendUint64(uint64(v)) } +func (enc *logfmtEncoder) AppendUintptr(v uintptr) { enc.AppendUint64(uint64(v)) } +func (enc *logfmtEncoder) AppendUint64(value uint64) { + enc.addSeparator() + enc.buf.AppendUint(value) +} + +func (enc *logfmtEncoder) Clone() zapcore.Encoder { + clone := enc.clone() + clone.buf.Write(enc.buf.Bytes()) + return clone +} + +func (enc *logfmtEncoder) clone() *logfmtEncoder { + clone := getEncoder() + clone.EncoderConfig = enc.EncoderConfig + clone.buf = bufferpool.Get() + clone.namespaces = enc.namespaces + return clone +} + +func (enc *logfmtEncoder) OpenNamespace(key string) { + key = strings.Map(keyRuneFilter, key) + enc.namespaces = append(enc.namespaces, key) +} + +func (enc *logfmtEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + final := enc.clone() + if final.TimeKey != "" { + final.AddTime(final.TimeKey, ent.Time) + } + if final.LevelKey != "" { + final.addKey(final.LevelKey) + cur := final.buf.Len() + if final.EncodeLevel != nil { + final.EncodeLevel(ent.Level, final) + } + if cur == final.buf.Len() { + // User-supplied EncodeLevel was a no-op. Fall back to strings to keep + // output valid. + final.AppendString(ent.Level.String()) + } + } + if ent.LoggerName != "" && final.NameKey != "" { + final.addKey(final.NameKey) + cur := final.buf.Len() + if final.EncodeName != nil { + final.EncodeName(ent.LoggerName, final) + } + if cur == final.buf.Len() { + // User-supplied EncodeName was a no-op. Fall back to strings to + // keep output valid. + final.AppendString(ent.LoggerName) + } + } + if ent.Caller.Defined && final.CallerKey != "" { + final.addKey(final.CallerKey) + cur := final.buf.Len() + if final.EncodeCaller != nil { + final.EncodeCaller(ent.Caller, final) + } + if cur == final.buf.Len() { + // User-supplied EncodeCaller was a no-op. Fall back to strings to + // keep output valid. + final.AppendString(ent.Caller.String()) + } + } + if final.MessageKey != "" { + final.addKey(enc.MessageKey) + final.AppendString(ent.Message) + } + if enc.buf.Len() > 0 { + if final.buf.Len() > 0 { + final.buf.AppendByte(' ') + } + final.buf.Write(enc.buf.Bytes()) + } + addFields(final, fields) + if ent.Stack != "" && final.StacktraceKey != "" { + final.AddString(final.StacktraceKey, ent.Stack) + } + if final.LineEnding != "" { + final.buf.AppendString(final.LineEnding) + } else { + final.buf.AppendString(zapcore.DefaultLineEnding) + } + + ret := final.buf + putEncoder(final) + return ret, nil +} + +func (enc *logfmtEncoder) addSeparator() { + if !enc.arrayLiteral { + return + } + + last := enc.buf.Len() - 1 + if last >= 0 && enc.buf.Bytes()[last] != '[' { + enc.buf.AppendByte(',') + } +} + +func (enc *logfmtEncoder) addKey(key string) { + key = strings.Map(keyRuneFilter, key) + if enc.buf.Len() > 0 { + enc.buf.AppendByte(' ') + } + for _, ns := range enc.namespaces { + enc.safeAddString(ns) + enc.buf.AppendByte('.') + } + enc.safeAddString(key) + enc.buf.AppendByte('=') +} + +// safeAddString JSON-escapes a string and appends it to the internal buffer. +// Unlike the standard library's encoder, it doesn't attempt to protect the +// user from browser vulnerabilities or JSONP-related problems. +func (enc *logfmtEncoder) safeAddString(s string) { + for i := 0; i < len(s); { + if enc.tryAddRuneSelf(s[i]) { + i++ + continue + } + r, size := utf8.DecodeRuneInString(s[i:]) + if enc.tryAddRuneError(r, size) { + i++ + continue + } + enc.buf.AppendString(s[i : i+size]) + i += size + } +} + +// safeAddByteString is no-alloc equivalent of safeAddString(string(s)) for s []byte. +func (enc *logfmtEncoder) safeAddByteString(s []byte) { + for i := 0; i < len(s); { + if enc.tryAddRuneSelf(s[i]) { + i++ + continue + } + r, size := utf8.DecodeRune(s[i:]) + if enc.tryAddRuneError(r, size) { + i++ + continue + } + enc.buf.Write(s[i : i+size]) + i += size + } +} + +// tryAddRuneSelf appends b if it is valid UTF-8 character represented in a single byte. +func (enc *logfmtEncoder) tryAddRuneSelf(b byte) bool { + if b >= utf8.RuneSelf { + return false + } + if 0x20 <= b && b != '\\' && b != '"' { + enc.buf.AppendByte(b) + return true + } + switch b { + case '\\', '"': + enc.buf.AppendByte('\\') + enc.buf.AppendByte(b) + case '\n': + enc.buf.AppendByte('\\') + enc.buf.AppendByte('n') + case '\r': + enc.buf.AppendByte('\\') + enc.buf.AppendByte('r') + case '\t': + enc.buf.AppendByte('\\') + enc.buf.AppendByte('t') + default: + // Encode bytes < 0x20, except for the escape sequences above. + const _hex = "0123456789abcdef" + enc.buf.AppendString(`\u00`) + enc.buf.AppendByte(_hex[b>>4]) + enc.buf.AppendByte(_hex[b&0xF]) + } + return true +} + +func (enc *logfmtEncoder) tryAddRuneError(r rune, size int) bool { + if r == utf8.RuneError && size == 1 { + enc.buf.AppendString(`\ufffd`) + return true + } + return false +} + +func needsQuotedValueRune(r rune) bool { + return r <= ' ' || r == '=' || r == '"' || r == utf8.RuneError +} + +func keyRuneFilter(r rune) rune { + if needsQuotedValueRune(r) { + return -1 + } + return r +} + +func addFields(enc zapcore.ObjectEncoder, fields []zapcore.Field) { + for i := range fields { + fields[i].AddTo(enc) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c6048c91..67935ef8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -245,6 +245,9 @@ github.com/spf13/pflag ## explicit; go 1.13 github.com/stretchr/testify/assert github.com/stretchr/testify/require +# github.com/sykesm/zap-logfmt v0.0.4 +## explicit; go 1.13 +github.com/sykesm/zap-logfmt # github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca ## explicit github.com/xlab/treeprint -- Gitee From e30fcb3c11ba4290b892e7307976b2c2a64c8fee Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 16 Jan 2024 20:05:26 +0800 Subject: [PATCH 19/46] test(rust os-agent): add os-agent unit tests Signed-off-by: Yuhang Wei --- KubeOS-Rust/agent/src/rpc/agent_impl.rs | 28 +++++---- KubeOS-Rust/cli/src/client.rs | 23 ++----- KubeOS-Rust/cli/src/method/callable_method.rs | 28 +++++++++ KubeOS-Rust/cli/src/method/configure.rs | 31 ++++++++++ KubeOS-Rust/cli/src/method/prepare_upgrade.rs | 37 +++++++++++ KubeOS-Rust/cli/src/method/request.rs | 37 +++++++++++ KubeOS-Rust/cli/src/method/rollback.rs | 13 ++++ KubeOS-Rust/cli/src/method/upgrade.rs | 13 ++++ KubeOS-Rust/manager/src/api/types.rs | 58 +++++++++++++++++ KubeOS-Rust/manager/src/sys_mgmt/config.rs | 62 +++++++++---------- .../manager/src/sys_mgmt/containerd_image.rs | 22 +------ .../manager/src/sys_mgmt/disk_image.rs | 7 ++- .../manager/src/sys_mgmt/docker_image.rs | 4 +- KubeOS-Rust/manager/src/utils/common.rs | 43 ++++++++++--- .../manager/src/utils/container_image.rs | 46 ++++++++++++-- KubeOS-Rust/manager/src/utils/partition.rs | 15 +++++ 16 files changed, 366 insertions(+), 101 deletions(-) diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs index bc1eabda..8aef4140 100644 --- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -13,7 +13,7 @@ use std::{sync::Mutex, thread, time::Duration}; use anyhow::{bail, Result}; -use log::{debug, error, info}; +use log::{debug, info}; use manager::{ api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH}, @@ -101,7 +101,6 @@ impl AgentImpl { debug!("Found configuration type: \"{}\"", config_type); configuration.set_config(config)?; } else { - error!("Unknown configuration type: \"{}\"", config_type); bail!("Unknown configuration type: \"{}\"", config_type); } } @@ -123,7 +122,7 @@ impl AgentImpl { Ok(Response { status: AgentStatus::Rollbacked }) } - pub fn reboot(&self) -> Result<()> { + fn reboot(&self) -> Result<()> { info!("Wait to reboot"); thread::sleep(Duration::from_secs(1)); sync(); @@ -144,7 +143,15 @@ mod test { use super::*; #[test] - fn configure_impl_tests() { + fn test_reboot() { + let mut agent = AgentImpl::default(); + agent.disable_reboot = true; + let res = agent.reboot(); + assert!(res.is_ok()); + } + + #[test] + fn test_configure() { let agent = AgentImpl::default(); let req = ConfigureRequest { configs: vec![Sysconfig { @@ -153,7 +160,7 @@ mod test { contents: HashMap::new(), }], }; - let res = agent.configure_impl(req).unwrap(); + let res = agent.configure(req).unwrap(); assert_eq!(res, Response { status: AgentStatus::Configured }); let req = ConfigureRequest { @@ -163,17 +170,12 @@ mod test { contents: HashMap::new(), }], }; - let res = agent.configure_impl(req); + let res = agent.configure(req); assert!(res.is_err()); } #[test] - fn upgrade_impl_tests() { - let _ = env_logger::builder() - .target(env_logger::Target::Stdout) - .filter_level(log::LevelFilter::Trace) - .is_test(true) - .try_init(); + fn test_prepare_upgrade() { let agent = AgentImpl::default(); let req = UpgradeRequest { version: "v2".into(), @@ -185,7 +187,7 @@ mod test { mtls: false, certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, }; - let res = agent.prepare_upgrade_impl(req); + let res = agent.prepare_upgrade(req); assert!(res.is_err()); } } diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs index ce45cdd7..9765a425 100644 --- a/KubeOS-Rust/cli/src/client.rs +++ b/KubeOS-Rust/cli/src/client.rs @@ -43,27 +43,14 @@ impl Client { #[cfg(test)] mod test { - use kubeos_manager::api; - use super::*; - use crate::method::{callable_method::RpcMethod, configure::ConfigureMethod}; - #[test] - #[ignore] fn test_client() { - let socket_path = "/home/yuhang/os-agent-rust.sock"; + let socket_path = "/tmp/KubeOS-test.sock"; let cli = Client::new(socket_path); - - let configured = api::AgentStatus::Configured; - let resp = api::Response { status: configured }; - let config_request = api::ConfigureRequest { - configs: vec![api::Sysconfig { - model: "kernel.sysctl".into(), - config_path: "".into(), - contents: std::collections::hash_map::HashMap::new(), - }], - }; - let config_resp = ConfigureMethod::new(config_request).call(&cli).unwrap(); - assert_eq!(resp, config_resp); + let command = "example_command"; + let params = vec![]; + let request = cli.send_request(cli.build_request(command, ¶ms)); + assert!(request.is_err()); } } diff --git a/KubeOS-Rust/cli/src/method/callable_method.rs b/KubeOS-Rust/cli/src/method/callable_method.rs index c46614b4..a174b5b8 100644 --- a/KubeOS-Rust/cli/src/method/callable_method.rs +++ b/KubeOS-Rust/cli/src/method/callable_method.rs @@ -24,3 +24,31 @@ pub trait RpcMethod { response.result().map_err(parse_error) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::client; + + #[derive(Default)] + struct DummyMethod; + + impl RpcMethod for DummyMethod { + type Response = String; + + fn command_name(&self) -> &'static str { + "dummy_command" + } + + fn command_params(&self) -> Vec> { + vec![] + } + } + + #[test] + fn test_call() { + let client = client::Client::new("/tmp/KubeOS-test.sock"); + let result = DummyMethod::default().call(&client); + assert!(result.is_err()); + } +} diff --git a/KubeOS-Rust/cli/src/method/configure.rs b/KubeOS-Rust/cli/src/method/configure.rs index d1371068..cca752d0 100644 --- a/KubeOS-Rust/cli/src/method/configure.rs +++ b/KubeOS-Rust/cli/src/method/configure.rs @@ -39,3 +39,34 @@ impl RpcMethod for ConfigureMethod { vec![to_raw_value(&self.req).unwrap()] } } +#[cfg(test)] +mod tests { + use kubeos_manager::api::{ConfigureRequest, Sysconfig}; + + use super::*; + + #[test] + fn test_configure_method() { + let req = ConfigureRequest { configs: vec![] }; + let mut method = ConfigureMethod::new(req); + + // Test set_configure_request method + let new_req = ConfigureRequest { + configs: vec![Sysconfig { + model: "model".to_string(), + config_path: "config_path".to_string(), + contents: Default::default(), + }], + }; + method.set_configure_request(new_req); + + // Test command_name method + assert_eq!(method.command_name(), "configure"); + + // Test command_params method + let expected_params = + "RawValue({\"configs\":[{\"model\":\"model\",\"config_path\":\"config_path\",\"contents\":{}}]})"; + let actual_params = format!("{:?}", method.command_params()[0]); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs index 91dae793..f2034f6b 100644 --- a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs +++ b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs @@ -39,3 +39,40 @@ impl RpcMethod for PrepareUpgradeMethod { vec![to_raw_value(&self.req).unwrap()] } } +#[cfg(test)] +mod tests { + use kubeos_manager::api::{CertsInfo, UpgradeRequest}; + + use super::*; + + #[test] + fn test_prepare_upgrade_method() { + let req = UpgradeRequest { + version: "v1".into(), + check_sum: "".into(), + image_type: "".into(), + container_image: "".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let mut method = PrepareUpgradeMethod::new(req); + let new_req = UpgradeRequest { + version: "v2".into(), + check_sum: "xxx".into(), + image_type: "xxx".into(), + container_image: "xxx".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + method.set_prepare_upgrade_request(new_req); + assert_eq!(method.command_name(), "prepare_upgrade"); + + let expected_params = "RawValue({\"version\":\"v2\",\"check_sum\":\"xxx\",\"image_type\":\"xxx\",\"container_image\":\"xxx\",\"image_url\":\"\",\"flag_safe\":false,\"mtls\":false,\"certs\":{\"ca_cert\":\"\",\"client_cert\":\"\",\"client_key\":\"\"}})"; + let actual_params = format!("{:?}", method.command_params()[0]); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs index 2dc1ffba..ff75afd0 100644 --- a/KubeOS-Rust/cli/src/method/request.rs +++ b/KubeOS-Rust/cli/src/method/request.rs @@ -50,3 +50,40 @@ pub fn parse_error(error: Error) -> anyhow::Error { }, } } + +#[cfg(test)] +mod tests { + use jsonrpc::error::RpcError; + use serde::de::Error as DeError; + + use super::*; + + #[test] + fn test_parse_error() { + // Test Error::Transport + let transport_error = + Error::Transport(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "Connection timeout"))); + let result = parse_error(transport_error); + assert_eq!(result.to_string(), "Cannot connect to KubeOS os-agent unix socket, Connection timeout"); + + // Test Error::Json + let json_error = Error::Json(serde_json::Error::custom("Failed to parse response")); + let result = parse_error(json_error); + assert_eq!(result.to_string(), "Failed to parse response"); + + // Test Error::Rpc with "Method not found" message + let rpc_error = Error::Rpc(RpcError { code: -32601, message: "Method not found".to_string(), data: None }); + let result = parse_error(rpc_error); + assert_eq!(result.to_string(), "Method is unimplemented"); + + // Test Error::Rpc with other message + let rpc_error = Error::Rpc(RpcError { code: -32603, message: "Internal server error".to_string(), data: None }); + let result = parse_error(rpc_error); + assert_eq!(result.to_string(), "Internal server error"); + + // Test other Error variant + let other_error = Error::VersionMismatch; + let result = parse_error(other_error); + assert_eq!(result.to_string(), "Response is invalid"); + } +} diff --git a/KubeOS-Rust/cli/src/method/rollback.rs b/KubeOS-Rust/cli/src/method/rollback.rs index 55aa7511..7945f4b1 100644 --- a/KubeOS-Rust/cli/src/method/rollback.rs +++ b/KubeOS-Rust/cli/src/method/rollback.rs @@ -27,3 +27,16 @@ impl RpcMethod for RollbackMethod { vec![] } } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_rollback_method() { + let method = RollbackMethod::default(); + assert_eq!(method.command_name(), "rollback"); + let expected_params = "[]"; + let actual_params = format!("{:?}", method.command_params()); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/upgrade.rs b/KubeOS-Rust/cli/src/method/upgrade.rs index a9692ca1..f2f94cd5 100644 --- a/KubeOS-Rust/cli/src/method/upgrade.rs +++ b/KubeOS-Rust/cli/src/method/upgrade.rs @@ -27,3 +27,16 @@ impl RpcMethod for UpgradeMethod { vec![] } } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_upgrade_method() { + let method = UpgradeMethod::default(); + assert_eq!(method.command_name(), "upgrade"); + let expected_params = "[]"; + let actual_params = format!("{:?}", method.command_params()); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/manager/src/api/types.rs b/KubeOS-Rust/manager/src/api/types.rs index 28ee97db..98aeaa33 100644 --- a/KubeOS-Rust/manager/src/api/types.rs +++ b/KubeOS-Rust/manager/src/api/types.rs @@ -80,3 +80,61 @@ impl ImageType { pub trait ImageHandler { fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result>; } + +#[cfg(test)] +mod tests { + use anyhow::Result; + use mockall::mock; + + use super::*; + use crate::utils::PreparePath; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + #[test] + fn test_download_image() { + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: "kubeos-temp".to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + let mut mock_executor1 = MockCommandExec::new(); + mock_executor1.expect_run_command().returning(|_, _| Ok(())); + mock_executor1.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let c_handler = CtrImageHandler::new(PreparePath::default(), mock_executor1); + let image_type = ImageType::Containerd(c_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + + let mut mock_executor2 = MockCommandExec::new(); + mock_executor2.expect_run_command().returning(|_, _| Ok(())); + mock_executor2.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let docker_handler = DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor2); + let image_type = ImageType::Docker(docker_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + + let mut mock_executor3 = MockCommandExec::new(); + mock_executor3.expect_run_command().returning(|_, _| Ok(())); + mock_executor3.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let disk_handler = DiskImageHandler::new(PreparePath::default(), mock_executor3, "test".into()); + let image_type = ImageType::Disk(disk_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs index cb5fad1f..48517b4f 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -186,7 +186,7 @@ fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String return config_kv.join("="); } info!("Delete configuration {}={}", key, old_value); - String::from("") + String::new() } fn handle_update_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String { @@ -413,11 +413,20 @@ mod tests { let mut tmp_file = tempfile::NamedTempFile::new().unwrap(); writeln!(tmp_file, "{}", comment).unwrap(); writeln!(tmp_file, "a=0").unwrap(); + writeln!(tmp_file, "d=4").unwrap(); + writeln!(tmp_file, "e=5").unwrap(); + writeln!(tmp_file, "g=7").unwrap(); let kernel_sysctl_persist = KernelSysctlPersist {}; let config_detail = HashMap::from([ ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }), ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }), + ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ("f".to_string(), KeyInfo { value: "".to_string(), operation: "add".to_string() }), + ("g".to_string(), KeyInfo { value: "7".to_string(), operation: "delete".to_string() }), + ("".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }), + ("s=x".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }), ]); let mut config = Sysconfig { model: KERNEL_SYSCTL_PERSIST.to_string(), @@ -426,33 +435,16 @@ mod tests { }; kernel_sysctl_persist.set_config(&mut config).unwrap(); let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); - let expected_res = format!("{}\n{}\n{}\n", comment, "a=1", "c=3"); + let expected_res = format!("{}\n{}\n{}\n{}\n{}\n", comment, "a=1", "d=4", "e=5", "c=3"); assert_eq!(result, expected_res); - - // test config_path is empty - // remember modify DEFAULT_KERNEL_CONFIG_PATH first - // let config_detail = HashMap::from([ - // ( - // "aaa".to_string(), - // KeyInfo { - // value: "3".to_string(), - // operation: "add".to_string(), - // }, - // ), - // ( - // "bbb".to_string(), - // KeyInfo { - // value: "1".to_string(), - // operation: "delete".to_string(), - // }, - // ), - // ]); - // config.config_path = "".to_string(); - // config.contents = config_detail; - // kernel_sysctl_persist.set_config(&mut config).unwrap(); - // let result = fs::read_to_string(crate::sys_mgmt::DEFAULT_KERNEL_CONFIG_PATH).unwrap(); - // let expected_res = format!("{}\n", "aaa=3",); - // assert_eq!(result, expected_res); + let mut config = Sysconfig { + model: KERNEL_SYSCTL_PERSIST.to_string(), + config_path: String::from("/tmp/kubeos-test-kernel-sysctl-persist.txt"), + contents: HashMap::new(), + }; + kernel_sysctl_persist.set_config(&mut config).unwrap(); + assert!(is_file_exist(&config.config_path)); + delete_file_or_dir(&config.config_path).unwrap(); } #[test] @@ -492,7 +484,7 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri initrd /boot/initramfs.img }"; writeln!(tmp_file, "{}", grub_cfg).unwrap(); - let config_first_part = HashMap::from([ + let config_second_part = HashMap::from([ ("debug".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), ("quiet".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }), @@ -506,15 +498,16 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri let mut config = Sysconfig { model: GRUB_CMDLINE_CURRENT.to_string(), config_path: String::new(), - contents: config_first_part, + contents: config_second_part, }; grub_cmdline.set_config(&mut config).unwrap(); grub_cmdline.is_cur_partition = false; - let config_second = HashMap::from([ + let config_first_part = HashMap::from([ ("pci".to_string(), KeyInfo { value: "nomis".to_string(), operation: "".to_string() }), - ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }), + ("quiet".to_string(), KeyInfo { value: "11".to_string(), operation: "delete".to_string() }), + ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "update".to_string() }), ]); - config.contents = config_second; + config.contents = config_first_part; config.model = GRUB_CMDLINE_NEXT.to_string(); grub_cmdline.set_config(&mut config).unwrap(); let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); @@ -540,6 +533,11 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri } "; assert_eq!(result, expected_res); + + // test grub.cfg not exist + grub_cmdline.grub_path = "/tmp/grub-KubeOS-test.cfg".to_string(); + let res = grub_cmdline.set_config(&mut config); + assert!(res.is_err()); } #[test] diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs index 0b50ad65..dd7036f6 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -48,7 +48,7 @@ impl Default for CtrImageHandler { impl CtrImageHandler { #[cfg(test)] - fn new(paths: PreparePath, executor: T) -> Self { + pub fn new(paths: PreparePath, executor: T) -> Self { Self { paths, executor } } @@ -301,24 +301,4 @@ mod tests { assert!(result.is_ok()); } - - #[test] - #[ignore] - fn test_download_image() { - init(); - let ctr = CtrImageHandler { paths: PreparePath::default(), executor: RealCommandExecutor {} }; - let update_req = UpgradeRequest { - version: "KubeOS v2".to_string(), - image_type: "containerd".to_string(), - container_image: "docker.io/library/busybox:latest".to_string(), - check_sum: "".to_string(), - image_url: "".to_string(), - flag_safe: false, - mtls: false, - certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, - }; - ctr.download_image(&update_req).unwrap(); - let tar_path = "/persist/KubeOS-Update/os.tar"; - assert_eq!(true, Path::new(tar_path).exists()); - } } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs index 4ccb6033..a120db82 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs @@ -41,7 +41,7 @@ impl Default for DiskImageHandler { impl DiskImageHandler { #[cfg(test)] - fn new(paths: PreparePath, executor: T, certs_path: String) -> Self { + pub fn new(paths: PreparePath, executor: T, certs_path: String) -> Self { Self { paths, executor, certs_path } } @@ -392,11 +392,14 @@ mod tests { .with_body("This is a test txt file for KubeOS test.\n") .create(); handler.download_image(&upgrade_request).unwrap(); - assert_eq!(true, handler.paths.image_path.exists()); assert_eq!( fs::read(handler.paths.image_path.to_str().unwrap()).unwrap(), "This is a test txt file for KubeOS test.\n".as_bytes() ); + + let _m = mockito::mock("GET", "/test.txt").with_status(404).with_body("Not found").create(); + let res = handler.download_image(&upgrade_request); + assert!(res.is_err()) } } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs index 121e2577..177dfeb1 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -33,7 +33,7 @@ impl Default for DockerImageHandler { impl DockerImageHandler { #[cfg(test)] - fn new(paths: PreparePath, container_name: String, executor: T) -> Self { + pub fn new(paths: PreparePath, container_name: String, executor: T) -> Self { Self { paths, container_name, executor } } @@ -129,6 +129,8 @@ mod tests { let result = DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor).check_and_rm_container(); assert!(result.is_ok()); + + assert_eq!(DockerImageHandler::default().container_name, "kubeos-temp"); } #[test] diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs index 301a8c86..da8c8c30 100644 --- a/KubeOS-Rust/manager/src/utils/common.rs +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -23,14 +23,25 @@ use nix::{mount, mount::MntFlags}; use super::executor::CommandExecutor; use crate::sys_mgmt::{MOUNT_DIR, OS_IMAGE_NAME, PERSIST_DIR, ROOTFS_ARCHIVE, UPDATE_DIR}; +/// * persist_path: /persist +/// +/// * update_path: /persist/KubeOS-Update +/// +/// * mount_path: /persist/KubeOS-Update/kubeos-update +/// +/// * tar_path: /persist/KubeOS-Update/os.tar +/// +/// * image_path: /persist/update.img +/// +/// * rootfs_file: os.tar #[derive(Clone)] pub struct PreparePath { - pub persist_path: PathBuf, // persist_path: /persist - pub update_path: PathBuf, // update_path: /persist/KubeOS-Update - pub mount_path: PathBuf, // mount_path: /persist/KubeOS-Update/kubeos-update - pub tar_path: PathBuf, // tar_path: /persist/KubeOS-Update/os.tar - pub image_path: PathBuf, // image_path: /persist/update.img - pub rootfs_file: String, // rootfs_file: os.tar + pub persist_path: PathBuf, + pub update_path: PathBuf, + pub mount_path: PathBuf, + pub tar_path: PathBuf, + pub image_path: PathBuf, + pub rootfs_file: String, } impl Default for PreparePath { @@ -72,7 +83,7 @@ pub fn check_disk_size>(need_bytes: i64, path: P) -> Result<()> { Ok(()) } -// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img +/// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> where P: AsRef, @@ -160,6 +171,7 @@ mod tests { use tempfile::{NamedTempFile, TempDir}; use super::*; + use crate::utils::RealCommandExecutor; // Mock the CommandExecutor trait mock! { @@ -278,10 +290,23 @@ mod tests { } #[test] - #[ignore] fn test_get_boot_mode() { init(); let boot_mode = get_boot_mode(); - assert!(boot_mode == "uefi"); + let executor = RealCommandExecutor {}; + let res = executor.run_command("ls", &["/sys/firmware/efi"]); + if res.is_ok() { + assert!(boot_mode == "uefi"); + } else { + assert!(boot_mode == "bios"); + } + } + + #[test] + fn test_is_command_available() { + init(); + let executor = RealCommandExecutor {}; + assert_eq!(is_command_available("ls", &executor), true); + assert_eq!(is_command_available("aaaabb", &executor), false); } } diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs index a54fc193..7c3aa022 100644 --- a/KubeOS-Rust/manager/src/utils/container_image.rs +++ b/KubeOS-Rust/manager/src/utils/container_image.rs @@ -197,12 +197,24 @@ mod tests { let out1 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); let expect_output = "1111"; assert_eq!(out1, expect_output); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok("invalid output".to_string())); + let out2 = get_oci_image_digest(container_runtime, image_name, &mock); + assert!(out2.is_err()); let container_runtime = "crictl"; let command_output2 = "[docker.io/nginx@sha256:1111]"; mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string())); - let out2 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); - assert_eq!(out2, expect_output); + let out3 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); + assert_eq!(out3, expect_output); + + let out4 = get_oci_image_digest("invalid", image_name, &mock); + assert!(out4.is_err()); + + let container_runtime = "crictl"; + let command_output3 = "[docker.io/nginx:sha256:1111]"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string())); + let out5 = get_oci_image_digest(container_runtime, image_name, &mock); + assert!(out5.is_err()); } #[test] @@ -211,11 +223,13 @@ mod tests { let mut mock = MockCommandExec::new(); let image_name = "docker.io/nginx:latest"; let container_runtime = "crictl"; - let command_output = "[docker.io/nginx@sha256:1111]"; - let check_sum = "1111"; - mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output.to_string())); + let command_output = "[docker.io/nginx@sha256:1a2b]"; + let check_sum = "1A2B"; + mock.expect_run_command_with_output().times(2).returning(|_, _| Ok(command_output.to_string())); let result = check_oci_image_digest(container_runtime, image_name, check_sum, &mock); assert!(result.is_ok()); + let result = check_oci_image_digest(container_runtime, image_name, "1111", &mock); + assert!(result.is_err()); } #[test] @@ -251,4 +265,26 @@ mod tests { let result = pull_image("aaa", image_name, &mock_executor); assert!(result.is_err()); } + + #[test] + fn test_remove_image_if_exist() { + init(); + let mut mock_executor = MockCommandExec::new(); + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"check")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("something"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"rm")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + let image_name = "docker.io/nginx:latest"; + let res = remove_image_if_exist("ctr", image_name, &mock_executor); + assert!(res.is_ok()); + + let res = remove_image_if_exist("invalid", image_name, &mock_executor); + assert!(res.is_err()); + } } diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs index 0419159b..fcfa2d8b 100644 --- a/KubeOS-Rust/manager/src/utils/partition.rs +++ b/KubeOS-Rust/manager/src/utils/partition.rs @@ -22,6 +22,7 @@ pub struct PartitionInfo { pub fs_type: String, } +/// get_partition_info returns the current partition info and the next partition info. pub fn get_partition_info(executor: &T) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> { let lsblk = executor.run_command_with_output("lsblk", &["-lno", "NAME,MOUNTPOINTS,FSTYPE"])?; // After split whitespace, the root directory line should have 3 elements, which are "sda2 / ext4". @@ -93,5 +94,19 @@ mod tests { PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() }, ); assert_eq!(res, expect_res); + + let command_output2 = "sda\nsda1 /boot/efi vfat\nsda2 ext4\nsda3 / ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string())); + let res = get_partition_info(&mock).unwrap(); + let expect_res = ( + PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() }, + PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() }, + ); + assert_eq!(res, expect_res); + + let command_output3 = ""; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string())); + let res = get_partition_info(&mock); + assert!(res.is_err()); } } -- Gitee From 23b76996928a59e5472a288e40980fc8ba4b1e34 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Wed, 24 Jan 2024 10:39:27 +0800 Subject: [PATCH 20/46] fix: checksum comparison log format Signed-off-by: Yuhang Wei --- KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs index 4ccb6033..f60d714a 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs @@ -72,13 +72,14 @@ impl DiskImageHandler { fn checksum_match(&self, file_path: &str, check_sum: &str) -> Result<()> { trace!("Start to check checksum"); + let check_sum = check_sum.to_ascii_lowercase(); let file = fs::read(file_path)?; let mut hasher = Sha256::new(); hasher.update(file); let hash = hasher.finalize(); // sha256sum -b /persist/update.img - let cal_sum = format!("{:X}", hash); - if cal_sum.to_lowercase() != check_sum.to_lowercase() { + let cal_sum = format!("{:X}", hash).to_ascii_lowercase(); + if cal_sum != check_sum { delete_file_or_dir(file_path)?; bail!("Checksum {} mismatch to {}", cal_sum, check_sum); } @@ -302,7 +303,7 @@ mod tests { handler.paths.image_path = tmp_file.path().to_path_buf(); let mut req = UpgradeRequest { version: "v2".into(), - check_sum: "98ea7aff44631d183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), + check_sum: "98Ea7aff44631D183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), image_type: "disk".into(), container_image: "".into(), image_url: "http://localhost:8080/aaa.txt".to_string(), @@ -313,7 +314,7 @@ mod tests { assert_eq!(handler.paths.image_path.exists(), true); handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum).unwrap(); - req.check_sum = "1234567890".into(); + req.check_sum = "1234567Abc".into(); let res = handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum); assert!(res.is_err()); } -- Gitee From f2c736335868873cd0cb7562a7ba95ee7c19a315 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 25 Jan 2024 11:57:16 +0800 Subject: [PATCH 21/46] fix: check image name is valid regex the regex for checking the validity of the container image image is wrong in case of "IP:PORT@sha256:111" Signed-off-by: Yuhang Wei --- .../manager/src/utils/container_image.rs | 73 ++++++++++++++++--- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs index a54fc193..dc319258 100644 --- a/KubeOS-Rust/manager/src/utils/container_image.rs +++ b/KubeOS-Rust/manager/src/utils/container_image.rs @@ -17,7 +17,7 @@ use regex::Regex; use super::executor::CommandExecutor; pub fn is_valid_image_name(image: &str) -> Result<()> { - let pattern = r"^(?P[a-z0-9\-.]+\.[a-z0-9\-]+:?[0-9]*)?/?((?P[a-zA-Z0-9-_]+?)|(?P[a-zA-Z0-9-_]+?)/(?P[a-zA-Z-_]+?))(?P(?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$"; + let pattern = r"^((?:[\w.-]+)(?::\d+)?/)*(?:[\w.-]+)((?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$"; let reg_ex = Regex::new(pattern)?; if !reg_ex.is_match(image) { bail!("Invalid image name: {}", image); @@ -172,16 +172,67 @@ mod tests { #[test] fn test_is_valid_image_name() { init(); - let out = is_valid_image_name("nginx").unwrap(); - assert_eq!(out, ()); - let out = - is_valid_image_name("docker.example.com:5000/gmr/alpine@sha256:11111111111111111111111111111111").unwrap(); - assert_eq!(out, ()); - let out = - is_valid_image_name("sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8"); - match out { - Ok(_) => assert_eq!(true, false), - Err(_) => assert_eq!(true, true), + let correct_images = vec![ + "alpine", + "alpine:latest", + "localhost/latest", + "library/alpine", + "localhost:1234/test", + "test:1234/blaboon", + "alpine:3.7", + "docker.example.edu/gmr/alpine:3.7", + "docker.example.com:5000/gmr/alpine@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "docker.example.co.uk/gmr/alpine/test2:latest", + "registry.dobby.org/dobby/dobby-servers/arthound:2019-08-08", + "owasp/zap:3.8.0", + "registry.dobby.co/dobby/dobby-servers/github-run:2021-10-04", + "docker.elastic.co/kibana/kibana:7.6.2", + "registry.dobby.org/dobby/dobby-servers/lerphound:latest", + "registry.dobby.org/dobby/dobby-servers/marbletown-poc:2021-03-29", + "marbles/marbles:v0.38.1", + "registry.dobby.org/dobby/dobby-servers/loophole@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "sonatype/nexon:3.30.0", + "prom/node-exporter:v1.1.1", + "sosedoff/pgweb@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "sosedoff/pgweb:latest", + "registry.dobby.org/dobby/dobby-servers/arpeggio:2021-06-01", + "registry.dobby.org/dobby/antique-penguin:release-production", + "dalprodictus/halcon:6.7.5", + "antigua/antigua:v31", + "weblate/weblate:4.7.2-1", + "redis:4.0.01-alpine", + "registry.dobby.com/dobby/dobby-servers/github-run:latest", + "192.168.122.123:5000/kubeos-x86_64:2023-01", + ]; + let wrong_images = vec![ + "alpine;v1.0", + "alpine:latest@sha256:11111111111111111111111111111111", + "alpine|v1.0", + "alpine&v1.0", + "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8", + "192.168.122.123:5000/kubeos-x86_64:2023-01@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + "192.168.122.123:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + "myimage$%^&", + ":myimage", + "/myimage", + "myimage/", + "myimage:", + "myimage@@latest", + "myimage::tag", + "registry.com//myimage:tag", + " myimage", + "myimage ", + "registry.com/:tag", + "myimage:", + "", + ":tag", + "IP:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + ]; + for image in correct_images { + assert!(is_valid_image_name(image).is_ok()); + } + for image in wrong_images { + assert!(is_valid_image_name(image).is_err()); } } -- Gitee From 26823059153bd1b18ecb76a2d94b78946fe6f501 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 23 Jan 2024 15:33:10 +0800 Subject: [PATCH 22/46] Bump version from 1.0.4 to 1.0.5 Signed-off-by: Yuhang Wei --- KubeOS-Rust/Cargo.lock | 8 ++++---- KubeOS-Rust/agent/Cargo.toml | 2 +- KubeOS-Rust/cli/Cargo.toml | 2 +- KubeOS-Rust/manager/Cargo.toml | 2 +- KubeOS-Rust/proxy/Cargo.toml | 6 +++--- VERSION | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 20873396..c44c152d 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -189,7 +189,7 @@ dependencies = [ [[package]] name = "cli" -version = "0.1.0" +version = "1.0.5" dependencies = [ "anyhow", "jsonrpc", @@ -1092,7 +1092,7 @@ dependencies = [ [[package]] name = "manager" -version = "0.1.0" +version = "1.0.5" dependencies = [ "anyhow", "env_logger", @@ -1335,7 +1335,7 @@ dependencies = [ [[package]] name = "os-agent" -version = "0.1.0" +version = "1.0.5" dependencies = [ "anyhow", "env_logger", @@ -1498,7 +1498,7 @@ dependencies = [ [[package]] name = "proxy" -version = "0.1.0" +version = "1.0.5" dependencies = [ "anyhow", "assert-json-diff", diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml index 6db4df45..739bbbc7 100644 --- a/KubeOS-Rust/agent/Cargo.toml +++ b/KubeOS-Rust/agent/Cargo.toml @@ -3,7 +3,7 @@ description = "KubeOS os-agent" edition = "2021" license = "MulanPSL-2.0" name = "os-agent" -version = "0.1.0" +version = "1.0.5" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/KubeOS-Rust/cli/Cargo.toml b/KubeOS-Rust/cli/Cargo.toml index 1c46db36..c3c14c6f 100644 --- a/KubeOS-Rust/cli/Cargo.toml +++ b/KubeOS-Rust/cli/Cargo.toml @@ -3,7 +3,7 @@ description = "KubeOS os-agent client" edition = "2021" license = "MulanPSL-2.0" name = "cli" -version = "0.1.0" +version = "1.0.5" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml index 9431fba9..311a87c1 100644 --- a/KubeOS-Rust/manager/Cargo.toml +++ b/KubeOS-Rust/manager/Cargo.toml @@ -3,7 +3,7 @@ description = "KubeOS os-agent manager" edition = "2021" license = "MulanPSL-2.0" name = "manager" -version = "0.1.0" +version = "1.0.5" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dev-dependencies] diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index 9a148e89..0ef8f8a0 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -3,7 +3,7 @@ description = "KubeOS os-proxy" edition = "2021" license = "MulanPSL-2.0" name = "proxy" -version = "0.1.0" +version = "1.0.5" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -11,14 +11,14 @@ version = "0.1.0" anyhow = "1.0.44" async-trait = "0.1" chrono = { version = "0.4", default-features = false, features = ["std"] } -cli = { version = "0.1.0", path = "../cli" } +cli = { version = "1.0.5", path = "../cli" } env_logger = "0.9.0" futures = "0.3.17" h2 = "=0.3.16" k8s-openapi = { version = "0.13.1", features = ["v1_22"] } kube = { version = "0.66.0", features = ["derive", "runtime"] } log = "=0.4.15" -manager = { version = "0.1.0", path = "../manager" } +manager = { version = "1.0.5", path = "../manager" } regex = "=1.7.3" reqwest = { version = "=0.11.18", default-features = false, features = [ "json", diff --git a/VERSION b/VERSION index ee90284c..90a27f9c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.4 +1.0.5 -- Gitee From 2c90e2e1fae25a8324cf527a71d7501b1d0b7921 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Fri, 26 Jan 2024 10:53:40 +0800 Subject: [PATCH 23/46] perf(crd): improve default display of crd under kubectl get kubectl get os|osinstance doesn't display any valuable information. Adding default printer columns in crd.yaml to display more helpful information such as osversion, config version etc. Signed-off-by: Yuhang Wei --- .../config/crd/upgrade.openeuler.org_os.yaml | 13 ++++++++++++ .../upgrade.openeuler.org_osinstances.yaml | 21 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index 3bb1333d..2dd822e9 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -17,6 +17,19 @@ spec: scope: Namespaced versions: - name: v1alpha1 + additionalPrinterColumns: + - name: OS VERSION + jsonPath: .spec.osversion + type: string + description: The version of OS + - name: SYSCONFIG VERSION + type: string + jsonPath: .spec.sysconfigs.version + description: The version of sysconfig + - name: UPGRADECONFIG VERSION + type: string + jsonPath: .spec.upgradeconfigs.version + description: The version of upgradeconfig schema: openAPIV3Schema: description: OS is a specification for OS in the cluster diff --git a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml index 3fa70c07..df9119b4 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml @@ -17,6 +17,27 @@ spec: scope: Namespaced versions: - name: v1alpha1 + additionalPrinterColumns: + - name: NODESTATUS + type: string + jsonPath: .spec.nodestatus + description: The status of node + - name: SYSCONFIG-VERSION-CURRENT + type: string + jsonPath: .status.sysconfigs.version + description: The current status of sysconfig + - name: SYSCONFIG-VERSION-DESIRED + type: string + jsonPath: .spec.sysconfigs.version + description: The expected version of sysconfig + - name: UPGRADECONFIG-VERSION-CURRENT + type: string + jsonPath: .status.upgradeconfigs.version + description: The current version of upgradeconfig + - name: UPGRADECONFIG-VERSION-DESIRED + type: string + jsonPath: .spec.upgradeconfigs.version + description: The expected version of upgradeconfig schema: openAPIV3Schema: description: OSInstance defines some infomation of a node -- Gitee From fed39fc1ff83f016828d10c8fcbbf26762236dfa Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Wed, 24 Jan 2024 15:49:53 +0800 Subject: [PATCH 24/46] fix: logs content, grammar and format Signed-off-by: Yuhang Wei --- KubeOS-Rust/agent/src/function.rs | 3 ++- KubeOS-Rust/manager/src/sys_mgmt/config.rs | 8 ++++---- KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs | 2 +- KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs | 2 +- KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs | 6 +++--- KubeOS-Rust/manager/src/utils/common.rs | 8 +++----- KubeOS-Rust/manager/src/utils/executor.rs | 7 ++++--- KubeOS-Rust/manager/src/utils/image_manager.rs | 2 +- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/KubeOS-Rust/agent/src/function.rs b/KubeOS-Rust/agent/src/function.rs index 2c973474..9789d95c 100644 --- a/KubeOS-Rust/agent/src/function.rs +++ b/KubeOS-Rust/agent/src/function.rs @@ -25,7 +25,8 @@ impl RpcFunction { F: FnOnce() -> anyhow::Result, { (f)().map_err(|e| { - error!("{:?}", e); + let error_message = format!("{:#}", e); + error!("{}", error_message.replace('\n', " ").replace('\r', "")); Error { code: ErrorCode::ServerError(RPC_OP_ERROR), message: format!("{:?}", e), data: None } }) } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs index 48517b4f..a6297569 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -66,7 +66,7 @@ pub struct GrubCmdline { impl Configuration for KernelSysctl { fn set_config(&self, config: &mut Sysconfig) -> Result<()> { - info!("Start set kernel.sysctl"); + info!("Start setting kernel.sysctl"); for (key, key_info) in config.contents.iter() { let proc_path = self.get_proc_path(key); if key_info.operation == "delete" { @@ -99,7 +99,7 @@ impl KernelSysctl { impl Configuration for KernelSysctlPersist { fn set_config(&self, config: &mut Sysconfig) -> Result<()> { - info!("Start set kernel.sysctl.persist"); + info!("Start setting kernel.sysctl.persist"); let mut config_path = &values::DEFAULT_KERNEL_CONFIG_PATH.to_string(); if !config.config_path.is_empty() { config_path = &config.config_path; @@ -247,9 +247,9 @@ fn handle_add_key(expect_configs: &HashMap, is_only_key_valid: impl Configuration for GrubCmdline { fn set_config(&self, config: &mut Sysconfig) -> Result<()> { if self.is_cur_partition { - info!("Start set grub.cmdline.current configuration"); + info!("Start setting grub.cmdline.current configuration"); } else { - info!("Start set grub.cmdline.next configuration"); + info!("Start setting grub.cmdline.next configuration"); } if !is_file_exist(&self.grub_path) { bail!("Failed to find grub.cfg file"); diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs index dd7036f6..5b0d0b77 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -72,7 +72,7 @@ impl CtrImageHandler { .mount_path .to_str() .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?; - info!("Start get rootfs {}", image_name); + info!("Start getting rootfs {}", image_name); self.check_and_unmount(mount_path)?; self.executor .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs index 7c64bf03..6d836dc4 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs @@ -71,7 +71,7 @@ impl DiskImageHandler { } fn checksum_match(&self, file_path: &str, check_sum: &str) -> Result<()> { - trace!("Start to check checksum"); + info!("Start checking image checksum"); let check_sum = check_sum.to_ascii_lowercase(); let file = fs::read(file_path)?; let mut hasher = Sha256::new(); diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs index 177dfeb1..a8bbee29 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -42,16 +42,16 @@ impl DockerImageHandler { is_valid_image_name(image_name)?; let cli = "docker"; remove_image_if_exist(cli, image_name, &self.executor)?; - info!("Start pull image {}", image_name); + info!("Start pulling image {}", image_name); pull_image(cli, image_name, &self.executor)?; - info!("Start check image digest"); + info!("Start checking image digest"); check_oci_image_digest(cli, image_name, &req.check_sum, &self.executor)?; Ok(()) } fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> { let image_name = &req.container_image; - info!("Start get rootfs {}", image_name); + info!("Start getting rootfs {}", image_name); self.check_and_rm_container()?; debug!("Create container {}", self.container_name); let container_id = diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs index da8c8c30..a6d62a02 100644 --- a/KubeOS-Rust/manager/src/utils/common.rs +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -79,7 +79,6 @@ pub fn check_disk_size>(need_bytes: i64, path: P) -> Result<()> { if available_space < need_bytes { bail!("Space is not enough for downloading"); } - info!("There is enough disk space to upgrade"); Ok(()) } @@ -88,9 +87,8 @@ pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> where P: AsRef, { - info!("Clean up the residual upgrade environment"); if is_mounted(&mount_path)? { - debug!("Umount {}", mount_path.as_ref().display()); + debug!("Umount \"{}\"", mount_path.as_ref().display()); if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) { bail!("Failed to umount {} in clean_env: {}", mount_path.as_ref().display(), errno); } @@ -104,10 +102,10 @@ where pub fn delete_file_or_dir>(path: P) -> Result<()> { if is_file_exist(&path) { if fs::metadata(&path)?.is_file() { - debug!("Delete file {}", path.as_ref().display()); + info!("Delete file \"{}\"", path.as_ref().display()); fs::remove_file(&path)?; } else { - debug!("Delete directory {}", path.as_ref().display()); + info!("Delete directory \"{}\"", path.as_ref().display()); fs::remove_dir_all(&path)?; } } diff --git a/KubeOS-Rust/manager/src/utils/executor.rs b/KubeOS-Rust/manager/src/utils/executor.rs index 8f4cb25c..c87bf2ad 100644 --- a/KubeOS-Rust/manager/src/utils/executor.rs +++ b/KubeOS-Rust/manager/src/utils/executor.rs @@ -28,8 +28,9 @@ impl CommandExecutor for RealCommandExecutor { trace!("run_command: {} {:?}", name, args); let output = Command::new(name).args(args).output()?; if !output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); let error_message = String::from_utf8_lossy(&output.stderr); - bail!("Failed to run command: {} {:?}, stderr: {}", name, args, error_message); + bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message); } debug!("run_command: {} {:?} done", name, args); Ok(()) @@ -38,11 +39,11 @@ impl CommandExecutor for RealCommandExecutor { fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result { trace!("run_command_with_output: {} {:?}", name, args); let output = Command::new(name).args(args).output()?; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); if !output.status.success() { let error_message = String::from_utf8_lossy(&output.stderr); - bail!("Failed to run command: {} {:?}, stderr: {}", name, args, error_message); + bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message); } - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); debug!("run_command_with_output: {} {:?} done", name, args); Ok(stdout.trim_end_matches('\n').to_string()) } diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs index dc823232..90806cf8 100644 --- a/KubeOS-Rust/manager/src/utils/image_manager.rs +++ b/KubeOS-Rust/manager/src/utils/image_manager.rs @@ -89,7 +89,7 @@ impl UpgradeImageManager { self.format_image()?; self.mount_image()?; self.extract_tar_to_image()?; - // Pass empty image_path to clean_env to avoid delete image file + // Pass empty image_path to clean_env but avoid deleting the upgrade image clean_env(&self.paths.update_path, &self.paths.mount_path, &PathBuf::new())?; Ok(self) } -- Gitee From a9d6ff63344023e7ecc7d90904f694714514e19e Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 29 Jan 2024 15:28:29 +0800 Subject: [PATCH 25/46] fix(os-agent):add context when returning error In some cases, the error log only contains "No such file" information. It is hard to identify the origin of the error. Provide more context when propagating error. Signed-off-by: Yuhang Wei --- KubeOS-Rust/manager/src/sys_mgmt/config.rs | 18 ++++++++++++------ .../manager/src/sys_mgmt/containerd_image.rs | 6 +++--- .../manager/src/sys_mgmt/docker_image.rs | 6 +++--- KubeOS-Rust/manager/src/utils/common.rs | 8 ++++---- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs index a6297569..33efdca8 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -105,9 +105,10 @@ impl Configuration for KernelSysctlPersist { config_path = &config.config_path; } debug!("kernel.sysctl.persist config_path: \"{}\"", config_path); - create_config_file(config_path)?; - let configs = get_and_set_configs(&mut config.contents, config_path)?; - write_configs_to_file(config_path, &configs)?; + create_config_file(config_path).with_context(|| format!("Failed to find config path"))?; + let configs = get_and_set_configs(&mut config.contents, config_path) + .with_context(|| format!("Failed to set persist kernel configs"))?; + write_configs_to_file(config_path, &configs).with_context(|| format!("Failed to write configs to file"))?; Ok(()) } } @@ -254,10 +255,15 @@ impl Configuration for GrubCmdline { if !is_file_exist(&self.grub_path) { bail!("Failed to find grub.cfg file"); } - let config_partition = - if cfg!(test) { self.is_cur_partition } else { self.get_config_partition(RealCommandExecutor {})? }; + let config_partition = if cfg!(test) { + self.is_cur_partition + } else { + self.get_config_partition(RealCommandExecutor {}) + .with_context(|| format!("Failed to get config partition"))? + }; debug!("Config_partition: {} (false means partition A, true means partition B)", config_partition); - let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition)?; + let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition) + .with_context(|| format!("Failed to set grub configs"))?; write_configs_to_file(&self.grub_path, &configs)?; Ok(()) } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs index 5b0d0b77..727949b6 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -12,7 +12,7 @@ use std::{fs, os::unix::fs::PermissionsExt, path::Path}; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use log::{debug, info}; use crate::{ @@ -73,12 +73,12 @@ impl CtrImageHandler { .to_str() .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?; info!("Start getting rootfs {}", image_name); - self.check_and_unmount(mount_path)?; + self.check_and_unmount(mount_path).with_context(|| format!("Failed to clean containerd environment"))?; self.executor .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?; // copy os.tar from mount_path to its partent dir self.copy_file(self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission)?; - self.check_and_unmount(mount_path)?; + self.check_and_unmount(mount_path).with_context(|| format!("Failed to clean containerd environment"))?; Ok(()) } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs index a8bbee29..f6971427 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use log::{debug, info, trace}; use crate::{ @@ -52,7 +52,7 @@ impl DockerImageHandler { fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> { let image_name = &req.container_image; info!("Start getting rootfs {}", image_name); - self.check_and_rm_container()?; + self.check_and_rm_container().with_context(|| format!("Failed to remove kubeos-temp container"))?; debug!("Create container {}", self.container_name); let container_id = self.executor.run_command_with_output("docker", &["create", "--name", &self.container_name, image_name])?; @@ -65,7 +65,7 @@ impl DockerImageHandler { self.paths.update_path.to_str().unwrap(), ], )?; - self.check_and_rm_container()?; + self.check_and_rm_container().with_context(|| format!("Failed to remove kubeos-temp container"))?; Ok(()) } diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs index a6d62a02..9baf99e3 100644 --- a/KubeOS-Rust/manager/src/utils/common.rs +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -16,7 +16,7 @@ use std::{ path::{Path, PathBuf}, }; -use anyhow::{anyhow, bail, Result}; +use anyhow::{anyhow, bail, Context, Result}; use log::{debug, info, trace}; use nix::{mount, mount::MntFlags}; @@ -85,7 +85,7 @@ pub fn check_disk_size>(need_bytes: i64, path: P) -> Result<()> { /// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> where - P: AsRef, + P: AsRef + std::fmt::Debug, { if is_mounted(&mount_path)? { debug!("Umount \"{}\"", mount_path.as_ref().display()); @@ -94,8 +94,8 @@ where } } // losetup -D? - delete_file_or_dir(update_path)?; - delete_file_or_dir(image_path)?; + delete_file_or_dir(&update_path).with_context(|| format!("Failed to delete {:?}", update_path))?; + delete_file_or_dir(&image_path).with_context(|| format!("Failed to delete {:?}", image_path))?; Ok(()) } -- Gitee From 2e05a5ce397b1ad3b889e200ee278f9efb5276f8 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Tue, 30 Jan 2024 15:04:13 +0800 Subject: [PATCH 26/46] proxy: Add unit tests and delete useless dependencies from Cargo.toml Add the proxy unit test ,modify the AgentCallClient , AgentClient and ProxyController struct design to make it easier to mock the agent invoking. Delete useless dependencies from Cargo.toml and Cargo.lock Signed-off-by: liyuanr --- KubeOS-Rust/Cargo.lock | 14 -- KubeOS-Rust/proxy/Cargo.toml | 3 +- .../proxy/src/controller/agentclient.rs | 74 ++++------ .../proxy/src/controller/apiserver_mock.rs | 137 +++++++++++++++--- .../proxy/src/controller/controller.rs | 121 ++++++++-------- KubeOS-Rust/proxy/src/controller/mod.rs | 2 +- KubeOS-Rust/proxy/src/controller/utils.rs | 12 +- KubeOS-Rust/proxy/src/drain.rs | 24 +-- KubeOS-Rust/proxy/src/main.rs | 9 +- 9 files changed, 232 insertions(+), 164 deletions(-) diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index c44c152d..2342c7b2 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -1180,18 +1180,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "mockall_double" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dffc15b97456ecc84d2bde8c1df79145e154f45225828c4361f676e1b82acd6" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "mockito" version = "0.31.1" @@ -1503,7 +1491,6 @@ dependencies = [ "anyhow", "assert-json-diff", "async-trait", - "chrono", "cli", "env_logger", "futures", @@ -1515,7 +1502,6 @@ dependencies = [ "log", "manager", "mockall", - "mockall_double", "regex", "reqwest", "schemars", diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index fe657afa..94e3b3c8 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -17,7 +17,6 @@ path = "src/main.rs" [dependencies] anyhow = "1.0.44" async-trait = "0.1" -chrono = { version = "0.4", default-features = false, features = ["std"] } cli = { version = "1.0.5", path = "../cli" } env_logger = "0.9.0" futures = "0.3.17" @@ -45,4 +44,4 @@ http = "0.2.9" hyper = "0.14.25" tower-test = "0.4.0" mockall = { version = "=0.11.3" } -mockall_double = "0.2.1" + diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs index 73489a9b..b833f276 100644 --- a/KubeOS-Rust/proxy/src/controller/agentclient.rs +++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs @@ -22,13 +22,6 @@ use cli::{ }; use manager::api::{CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest}; -#[cfg_attr(test, double)] -use agent_call::AgentCallClient; -#[cfg(test)] -use mockall::automock; -#[cfg(test)] -use mockall_double::double; - pub struct UpgradeInfo { pub version: String, pub image_type: String, @@ -57,45 +50,40 @@ pub struct KeyInfo { pub operation: String, } -#[cfg_attr(test, automock)] pub trait AgentMethod { - fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo, agent_call: AgentCallClient) -> Result<(), Error>; - fn upgrade_method(&self, agent_call: AgentCallClient) -> Result<(), Error>; - fn rollback_method(&self, agent_call: AgentCallClient) -> Result<(), Error>; - fn configure_method(&self, config_info: ConfigInfo, agent_call: AgentCallClient) -> Result<(), Error>; + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error>; + fn upgrade_method(&self) -> Result<(), Error>; + fn rollback_method(&self) -> Result<(), Error>; + fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error>; } - -pub mod agent_call { - use super::{Client, Error, RpcMethod}; - #[cfg(test)] - use mockall::automock; - - #[derive(Default)] - pub struct AgentCallClient {} - - #[cfg_attr(test, automock)] - impl AgentCallClient { - pub fn call_agent(&self, client: &Client, method: T) -> Result<(), Error> { - match method.call(client) { - Ok(_resp) => Ok(()), - Err(e) => Err(Error::AgentError { source: e }), - } - } - } +pub trait AgentCall { + fn call_agent(&self, client: &Client, method: T) -> Result<(), Error>; } -pub struct AgentClient { +pub struct AgentClient { pub agent_client: Client, + pub agent_call_client: T, +} + +impl AgentClient { + pub fn new>(socket_path: P, agent_call_client: T) -> Self { + AgentClient { agent_client: Client::new(socket_path), agent_call_client } + } } -impl AgentClient { - pub fn new>(socket_path: P) -> Self { - AgentClient { agent_client: Client::new(socket_path) } +#[derive(Default)] +pub struct AgentCallClient {} +impl AgentCall for AgentCallClient { + fn call_agent(&self, client: &Client, method: T) -> Result<(), Error> { + match method.call(client) { + Ok(_resp) => Ok(()), + Err(e) => Err(Error::AgentError { source: e }), + } } } -impl AgentMethod for AgentClient { - fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo, agent_call: AgentCallClient) -> Result<(), Error> { +impl AgentMethod for AgentClient { + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error> { let upgrade_request = UpgradeRequest { version: upgrade_info.version, image_type: upgrade_info.image_type, @@ -110,27 +98,27 @@ impl AgentMethod for AgentClient { client_key: upgrade_info.clientkey, }, }; - match agent_call.call_agent(&self.agent_client, PrepareUpgradeMethod::new(upgrade_request)) { + match self.agent_call_client.call_agent(&self.agent_client, PrepareUpgradeMethod::new(upgrade_request)) { Ok(_resp) => Ok(()), Err(e) => Err(e), } } - fn upgrade_method(&self, agent_call: AgentCallClient) -> Result<(), Error> { - match agent_call.call_agent(&self.agent_client, UpgradeMethod::default()) { + fn upgrade_method(&self) -> Result<(), Error> { + match self.agent_call_client.call_agent(&self.agent_client, UpgradeMethod::default()) { Ok(_resp) => Ok(()), Err(e) => Err(e), } } - fn rollback_method(&self, agent_call: AgentCallClient) -> Result<(), Error> { - match agent_call.call_agent(&self.agent_client, RollbackMethod::default()) { + fn rollback_method(&self) -> Result<(), Error> { + match self.agent_call_client.call_agent(&self.agent_client, RollbackMethod::default()) { Ok(_resp) => Ok(()), Err(e) => Err(e), } } - fn configure_method(&self, config_info: ConfigInfo, agent_call: AgentCallClient) -> Result<(), Error> { + fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error> { let mut agent_configs: Vec = Vec::new(); for config in config_info.configs { let mut contents_tmp: HashMap = HashMap::new(); @@ -147,7 +135,7 @@ impl AgentMethod for AgentClient { }) } let config_request = ConfigureRequest { configs: agent_configs }; - match agent_call.call_agent(&self.agent_client, ConfigureMethod::new(config_request)) { + match self.agent_call_client.call_agent(&self.agent_client, ConfigureMethod::new(config_request)) { Ok(_resp) => Ok(()), Err(e) => Err(e), } diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs index c46d26a4..ef5977c1 100644 --- a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs +++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs @@ -2,15 +2,20 @@ use self::mock_error::Error; use super::{ agentclient::*, crd::{Configs, OSInstanceStatus}, - values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK}, }; use crate::controller::{ apiclient::{ApplyApi, ControllerClient}, - crd::{OSInstance, OSInstanceSpec, OSSpec, OS}, + crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS}, values::{LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, ProxyController, }; use anyhow::Result; +use cli::client::Client; +use cli::method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, +}; use http::{Request, Response}; use hyper::{body::to_bytes, Body}; use k8s_openapi::api::core::v1::Pod; @@ -19,7 +24,8 @@ use kube::{ api::ObjectMeta, core::{ListMeta, ObjectList}, }; -use kube::{Client, Resource, ResourceExt}; +use kube::{Client as KubeClient, Resource, ResourceExt}; +use mockall::mock; use std::collections::BTreeMap; type ApiServerHandle = tower_test::mock::Handle, Response>; @@ -34,6 +40,7 @@ pub enum Testcases { ConfigNormal(OSInstance), ConfigVersionMismatchReassign(OSInstance), ConfigVersionMismatchUpdate(OSInstance), + Rollback(OSInstance), } pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) { @@ -59,7 +66,7 @@ impl ApiServerVerifier { .unwrap() .handler_node_get(osi) .await - }, + } Testcases::UpgradeNormal(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -78,7 +85,7 @@ impl ApiServerVerifier { .unwrap() .handler_node_pod_list_get(osi) .await - }, + } Testcases::UpgradeUpgradeconfigsVersionMismatch(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -97,7 +104,7 @@ impl ApiServerVerifier { .unwrap() .handler_osinstance_patch_nodestatus_idle(osi) .await - }, + } Testcases::UpgradeOSInstaceNodestatusConfig(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -107,7 +114,7 @@ impl ApiServerVerifier { .unwrap() .handler_node_get_with_label(osi.clone()) .await - }, + } Testcases::UpgradeOSInstaceNodestatusIdle(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -123,7 +130,7 @@ impl ApiServerVerifier { .unwrap() .handler_node_uncordon(osi) .await - }, + } Testcases::ConfigNormal(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -139,7 +146,7 @@ impl ApiServerVerifier { .unwrap() .handler_osinstance_patch_nodestatus_idle(osi) .await - }, + } Testcases::ConfigVersionMismatchReassign(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -152,7 +159,7 @@ impl ApiServerVerifier { .unwrap() .handler_osinstance_patch_nodestatus_idle(osi) .await - }, + } Testcases::ConfigVersionMismatchUpdate(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -165,7 +172,26 @@ impl ApiServerVerifier { .unwrap() .handler_osinstance_patch_spec_sysconfig_v2(osi) .await - }, + } + Testcases::Rollback(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_upgradeconfig_v2(osi.clone()) + .await + .unwrap() + .handler_node_cordon(osi.clone()) + .await + .unwrap() + .handler_node_pod_list_get(osi) + .await + } } .expect("Case completed without errors"); }) @@ -437,6 +463,7 @@ impl ApiServerVerifier { pub mod mock_error { use thiserror::Error; + #[derive(Error, Debug)] pub enum Error { #[error("Kubernetes reported error: {source}")] @@ -447,17 +474,27 @@ pub mod mock_error { } } -impl ProxyController { - pub fn test() -> (ProxyController, ApiServerVerifier) { +mock! { + pub AgentCallClient{} + impl AgentCall for AgentCallClient{ + fn call_agent(&self, client:&Client, method: T) -> Result<(), agent_error::Error> { + Ok(()) + } + } + +} +impl ProxyController { + pub fn test() -> (ProxyController, ApiServerVerifier) { let (mock_service, handle) = tower_test::mock::pair::, Response>(); - let mock_k8s_client = Client::new(mock_service, "default"); + let mock_k8s_client = KubeClient::new(mock_service, "default"); let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); - let mut mock_agent_client: MockAgentMethod = MockAgentMethod::new(); - mock_agent_client.expect_rollback_method().returning(|_x| Ok(())); - mock_agent_client.expect_prepare_upgrade_method().returning(|_x, _y| Ok(())); - mock_agent_client.expect_upgrade_method().returning(|_x| Ok(())); - mock_agent_client.expect_configure_method().returning(|_x, _y| Ok(())); - let proxy_controller: ProxyController = + let mut mock_agent_call_client = MockAgentCallClient::new(); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + let mock_agent_client = AgentClient::new("test", mock_agent_call_client); + let proxy_controller: ProxyController = ProxyController::new(mock_k8s_client, mock_api_client, mock_agent_client); (proxy_controller, ApiServerVerifier(handle)) } @@ -495,7 +532,7 @@ impl OSInstance { } pub fn set_osi_nodestatus_config(node_name: &str, namespace: &str) -> Self { - // return osinstance with nodestatus = upgrade, upgradeconfig.version=v1, sysconfig.version=v1 + // return osinstance with nodestatus = config, upgradeconfig.version=v1, sysconfig.version=v1 let mut osinstance = OSInstance::set_osi_default(node_name, namespace); osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); osinstance @@ -512,7 +549,18 @@ impl OSInstance { // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 let mut osinstance = OSInstance::set_osi_default(node_name, namespace); osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); - osinstance.spec.upgradeconfigs.as_mut().unwrap().version = Some(String::from("v2")); + osinstance.spec.upgradeconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); osinstance } @@ -520,7 +568,18 @@ impl OSInstance { // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 let mut osinstance = OSInstance::set_osi_default(node_name, namespace); osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); - osinstance.spec.sysconfigs.as_mut().unwrap().version = Some(String::from("v2")); + osinstance.spec.sysconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); osinstance } } @@ -549,7 +608,37 @@ impl OS { pub fn set_os_syscon_v2_opstype_config() -> Self { let mut os = OS::set_os_default(); os.spec.opstype = String::from("config"); - os.spec.sysconfigs = Some(Configs { version: Some(String::from("v2")), configs: None }); + os.spec.sysconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + os + } + + pub fn set_os_rollback_osversion_v2_upgradecon_v2() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string(); + os.spec.upgradeconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); os } } diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index b2bb332a..c21f3044 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -24,10 +24,8 @@ use kube::{ use log::{debug, error, info}; use reconciler_error::Error; -#[cfg_attr(test, double)] -use super::agentclient::agent_call::AgentCallClient; use super::{ - agentclient::{AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, + agentclient::{AgentCall, AgentClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, apiclient::ApplyApi, crd::{Configs, Content, OSInstance, OS}, utils::{check_version, get_config_version, ConfigOperation, ConfigType}, @@ -36,12 +34,10 @@ use super::{ REQUEUE_ERROR, REQUEUE_NORMAL, }, }; -#[cfg(test)] -use mockall_double::double; -pub async fn reconcile( +pub async fn reconcile( os: OS, - ctx: Context>, + ctx: Context>, ) -> Result { debug!("start reconcile"); let proxy_controller = ctx.get_ref(); @@ -76,7 +72,7 @@ pub async fn reconcile( ) .await?; return Ok(REQUEUE_NORMAL); - }, + } ConfigOperation::UpdateConfig => { debug!("start update config"); osinstance.spec.sysconfigs = os_cr.spec.sysconfigs.clone(); @@ -85,8 +81,8 @@ pub async fn reconcile( .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec) .await?; return Ok(REQUEUE_ERROR); - }, - _ => {}, + } + _ => {} } proxy_controller.set_config(&mut osinstance, ConfigType::SysConfig).await?; proxy_controller @@ -108,8 +104,8 @@ pub async fn reconcile( ) .await?; return Ok(REQUEUE_NORMAL); - }, - _ => {}, + } + _ => {} } if node.labels().contains_key(LABEL_UPGRADING) { if osinstance.spec.nodestatus == NODE_STATUS_IDLE { @@ -133,9 +129,9 @@ pub async fn reconcile( Ok(REQUEUE_NORMAL) } -pub fn error_policy( +pub fn error_policy( error: &Error, - _ctx: Context>, + _ctx: Context>, ) -> ReconcilerAction { error!("Reconciliation error:{}", error.to_string()); REQUEUE_ERROR @@ -145,31 +141,31 @@ struct ControllerResources { osinstance: OSInstance, node: Node, } -pub struct ProxyController { +pub struct ProxyController { k8s_client: Client, controller_client: T, - agent_client: U, + agent_client: AgentClient, } -impl ProxyController { - pub fn new(k8s_client: Client, controller_client: T, agent_client: U) -> Self { +impl ProxyController { + pub fn new(k8s_client: Client, controller_client: T, agent_client: AgentClient) -> Self { ProxyController { k8s_client, controller_client, agent_client } } } -impl ProxyController { +impl ProxyController { async fn check_osi_exisit(&self, namespace: &str, node_name: &str) -> Result<(), Error> { let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); match osi_api.get(node_name).await { Ok(osi) => { debug!("osinstance is exist {:?}", osi.name()); return Ok(()); - }, + } Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { info!("Create OSInstance {}", node_name); self.controller_client.create_osinstance(node_name, namespace).await?; Ok(()) - }, + } Err(err) => Err(Error::KubeError { source: err }), } } @@ -257,17 +253,16 @@ impl ProxyController { if config_info.need_config { match config_info.configs.and_then(convert_to_agent_config) { Some(agent_configs) => { - let agent_call_client = AgentCallClient::default(); - match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }, agent_call_client) { - Ok(_resp) => {}, + match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }) { + Ok(_resp) => {} Err(e) => { return Err(Error::AgentError { source: e }); - }, + } } - }, + } None => { - info!("config is none, no need to config"); - }, + info!("config is none, No content can be configured."); + } }; self.update_osi_status(osinstance, config_type).await?; } @@ -290,35 +285,34 @@ impl ProxyController { clientcert: os_cr.spec.clientcert.clone().unwrap_or_default(), clientkey: os_cr.spec.clientkey.clone().unwrap_or_default(), }; - let agent_call_client = AgentCallClient::default(); - match self.agent_client.prepare_upgrade_method(upgrade_info, agent_call_client) { - Ok(_resp) => {}, + + match self.agent_client.prepare_upgrade_method(upgrade_info) { + Ok(_resp) => {} Err(e) => { return Err(Error::AgentError { source: e }); - }, + } } self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; - let agent_call_client = AgentCallClient::default(); - match self.agent_client.upgrade_method(agent_call_client) { - Ok(_resp) => {}, + match self.agent_client.upgrade_method() { + Ok(_resp) => {} Err(e) => { return Err(Error::AgentError { source: e }); - }, + } } - }, + } OPERATION_TYPE_ROLLBACK => { self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; - let agent_call_client = AgentCallClient::default(); - match self.agent_client.rollback_method(agent_call_client) { - Ok(_resp) => {}, + + match self.agent_client.rollback_method() { + Ok(_resp) => {} Err(e) => { return Err(Error::AgentError { source: e }); - }, + } } - }, + } _ => { return Err(Error::OperationError { value: os_cr.spec.opstype.clone() }); - }, + } } Ok(()) } @@ -329,12 +323,12 @@ impl ProxyController { node_api.cordon(node_name).await?; info!("Cordon node Successfully{}, start drain nodes", node_name); match self.drain_node(node_name, evict_pod_force).await { - Ok(()) => {}, + Ok(()) => {} Err(e) => { node_api.uncordon(node_name).await?; info!("Drain node {} error, uncordon node successfully", node_name); return Err(e); - }, + } } Ok(()) } @@ -360,7 +354,7 @@ fn convert_to_agent_config(configs: Configs) -> Option> { contents: contents_tmp, }; agent_configs.push(config_tmp) - }, + } None => { info!( "model {} which has configpath {} do not has any contents no need to configure", @@ -368,7 +362,7 @@ fn convert_to_agent_config(configs: Configs) -> Option> { config.configpath.unwrap_or_default() ); continue; - }, + } }; } if agent_configs.len() == 0 { @@ -437,16 +431,13 @@ pub mod reconciler_error { #[cfg(test)] mod test { use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS}; + use crate::controller::apiserver_mock::{timeout_after_5s, MockAgentCallClient, Testcases}; use crate::controller::ControllerClient; - use crate::controller::{ - agentclient::MockAgentMethod, - apiserver_mock::{timeout_after_5s, Testcases}, - }; use std::env; #[tokio::test] async fn test_create_osinstance_with_no_upgrade_or_configuration() { - let (test_proxy_controller, fakeserver) = ProxyController::::test(); + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_default(); let context = Context::new(test_proxy_controller); @@ -457,7 +448,7 @@ mod test { } #[tokio::test] async fn test_upgrade_normal() { - let (test_proxy_controller, fakeserver) = ProxyController::::test(); + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_osversion_v2_upgradecon_v2(); let context = Context::new(test_proxy_controller); @@ -471,7 +462,7 @@ mod test { #[tokio::test] async fn test_diff_osversion_opstype_config() { - let (test_proxy_controller, fakeserver) = ProxyController::::test(); + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_osversion_v2_opstype_config(); let context = Context::new(test_proxy_controller); @@ -488,7 +479,7 @@ mod test { #[tokio::test] async fn test_upgradeconfigs_version_mismatch() { - let (test_proxy_controller, fakeserver) = ProxyController::::test(); + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_osversion_v2_upgradecon_v2(); let context = Context::new(test_proxy_controller); @@ -501,7 +492,7 @@ mod test { #[tokio::test] async fn test_upgrade_nodestatus_idle() { - let (test_proxy_controller, fakeserver) = ProxyController::::test(); + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_osversion_v2_upgradecon_v2(); let context = Context::new(test_proxy_controller); @@ -513,7 +504,7 @@ mod test { #[tokio::test] async fn test_config_normal() { - let (test_proxy_controller, fakeserver) = ProxyController::::test(); + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_syscon_v2_opstype_config(); let context = Context::new(test_proxy_controller); @@ -525,7 +516,7 @@ mod test { #[tokio::test] async fn test_sysconfig_version_mismatch_reassign() { - let (test_proxy_controller, fakeserver) = ProxyController::::test(); + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_syscon_v2_opstype_config(); let context = Context::new(test_proxy_controller); @@ -539,7 +530,7 @@ mod test { #[tokio::test] async fn test_sysconfig_version_mismatch_update() { - let (test_proxy_controller, fakeserver) = ProxyController::::test(); + let (test_proxy_controller, fakeserver) = ProxyController::::test(); env::set_var("NODE_NAME", "openeuler"); let os = OS::set_os_syscon_v2_opstype_config(); let context = Context::new(test_proxy_controller); @@ -550,4 +541,16 @@ mod test { reconcile(os, context.clone()).await.expect("reconciler"); timeout_after_5s(mocksrv).await; } + + #[tokio::test] + async fn test_rollback() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_rollback_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::Rollback(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } } diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs index 73be45c9..e30c8df7 100644 --- a/KubeOS-Rust/proxy/src/controller/mod.rs +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -19,7 +19,7 @@ mod crd; mod utils; mod values; -pub use agentclient::AgentClient; +pub use agentclient::{AgentCallClient, AgentClient}; pub use apiclient::ControllerClient; pub use controller::{error_policy, reconcile, reconciler_error::Error, ProxyController}; pub use crd::OS; diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs index 78502db0..0f568788 100644 --- a/KubeOS-Rust/proxy/src/controller/utils.rs +++ b/KubeOS-Rust/proxy/src/controller/utils.rs @@ -56,7 +56,7 @@ impl ConfigType { ); return ConfigOperation::Reassign; } - }, + } ConfigType::SysConfig => { let os_config_version = get_config_version(os.spec.sysconfigs.as_ref()); let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); @@ -78,7 +78,7 @@ impl ConfigType { return ConfigOperation::UpdateConfig; } } - }, + } }; ConfigOperation::DoNothing } @@ -96,7 +96,7 @@ impl ConfigType { status_config_version = get_config_version(None); } configs = osinstance.spec.upgradeconfigs.clone(); - }, + } ConfigType::SysConfig => { spec_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); if let Some(osinstance_status) = osinstance.status.as_ref() { @@ -105,7 +105,7 @@ impl ConfigType { status_config_version = get_config_version(None); } configs = osinstance.spec.sysconfigs.clone(); - }, + } } debug!( "=======osinstance soec config version is {},status config version is {}", @@ -127,7 +127,7 @@ impl ConfigType { sysconfigs: None, }) } - }, + } ConfigType::SysConfig => { if let Some(osi_status) = &mut osinstance.status { osi_status.sysconfigs = osinstance.spec.sysconfigs.clone(); @@ -135,7 +135,7 @@ impl ConfigType { osinstance.status = Some(OSInstanceStatus { upgradeconfigs: None, sysconfigs: osinstance.spec.sysconfigs.clone() }) } - }, + } } } } diff --git a/KubeOS-Rust/proxy/src/drain.rs b/KubeOS-Rust/proxy/src/drain.rs index 09cf6625..72836f98 100644 --- a/KubeOS-Rust/proxy/src/drain.rs +++ b/KubeOS-Rust/proxy/src/drain.rs @@ -66,7 +66,7 @@ async fn get_pods_deleted( Ok(pods @ ObjectList { .. }) => pods, Err(err) => { return Err(GetPodListsError { source: err, node_name: node_name.to_string() }); - }, + } }; let mut filterd_pods_list: Vec = Vec::new(); let mut filterd_err: Vec = Vec::new(); @@ -189,14 +189,14 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e let name = (&p).name_any(); info!("Pod {} deleted.", name); break; - }, + } Ok(_) => { info!("Pod '{}' is not yet deleted. Waiting {}s.", pod.name_any(), EVERY_DELETION_CHECK.as_secs_f64()); - }, + } Err(kube::Error::Api(e)) if e.code == response_error_not_found => { info!("Pod {} is deleted.", pod.name_any()); break; - }, + } Err(e) => { error!( "Get pod {} reported error: '{}', whether pod is deleted cannot be determined, waiting {}s.", @@ -204,7 +204,7 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e e, EVERY_DELETION_CHECK.as_secs_f64() ); - }, + } } if start_time.elapsed() > TIMEOUT { return Err(WaitDeletionError { pod_name: pod.name_any(), max_wait: TIMEOUT }); @@ -241,7 +241,7 @@ impl PodFilter for FinishedOrFailedFilter { return match pod.status.as_ref() { Some(PodStatus { phase: Some(phase), .. }) if phase == "Failed" || phase == "Succeeded" => { FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) - }, + } _ => FilterResult::create_filter_result(false, "", PodDeleteStatus::Okay), }; } @@ -269,7 +269,7 @@ impl PodFilter for DaemonFilter { let description = format!("Cannot drain Pod '{}': Pod is member of a DaemonSet", pod.name_any()); Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) } - }, + } _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } @@ -287,7 +287,7 @@ impl PodFilter for MirrorFilter { Some(annotations) if annotations.contains_key("kubernetes.io/config.mirror") => { let description = format!("Ignore Pod '{}': Pod is a static Mirror Pod", pod.name_any()); FilterResult::create_filter_result(false, &description.to_string(), PodDeleteStatus::Warning) - }, + } _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } @@ -312,7 +312,7 @@ impl PodFilter for LocalStorageFilter { let description = format!("Cannot drain Pod '{}': Pod has local Storage", pod.name_any()); Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) } - }, + } _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } @@ -365,7 +365,7 @@ impl PodFilter for DeletedFilter { && now - Duration::from_secs(time.0.timestamp() as u64) >= self.delete_wait_timeout => { FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) - }, + } _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } @@ -471,7 +471,7 @@ impl ErrorHandleStrategy { return match self { Self::TolerateStrategy => { return backoff.take(0); - }, + } Self::RetryStrategy => backoff.take(MAX_RETRIES_TIMES), }; @@ -488,7 +488,7 @@ impl tokio_retry::Condition for ErrorHandleStrategy { } else { false } - }, + } } } } diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs index cd601d0f..ad36b642 100644 --- a/KubeOS-Rust/proxy/src/main.rs +++ b/KubeOS-Rust/proxy/src/main.rs @@ -20,7 +20,9 @@ use kube::{ }; use log::{error, info}; mod controller; -use controller::{error_policy, reconcile, AgentClient, ControllerClient, ProxyController, OS, SOCK_PATH}; +use controller::{ + error_policy, reconcile, AgentCallClient, AgentClient, ControllerClient, ProxyController, OS, SOCK_PATH, +}; const PROXY_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); #[tokio::main] @@ -29,14 +31,15 @@ async fn main() -> Result<()> { let client = Client::try_default().await?; let os: Api = Api::all(client.clone()); let controller_client = ControllerClient::new(client.clone()); - let agent_client = AgentClient::new(SOCK_PATH); + let agent_call_client = AgentCallClient::default(); + let agent_client = AgentClient::new(SOCK_PATH, agent_call_client); let proxy_controller = ProxyController::new(client, controller_client, agent_client); info!("os-proxy version is {}, start renconcile", PROXY_VERSION.unwrap_or("Not Found")); Controller::new(os, ListParams::default()) .run(reconcile, error_policy, Context::new(proxy_controller)) .for_each(|res| async move { match res { - Ok(_o) => {}, + Ok(_o) => {} Err(e) => error!("reconcile failed: {}", e.to_string()), } }) -- Gitee From 1c5fcb965561dd7fb48118ca50952a5323ae93be Mon Sep 17 00:00:00 2001 From: liyuanr Date: Tue, 30 Jan 2024 15:05:20 +0800 Subject: [PATCH 27/46] proxy: fix code review issues 1. Fixed the enumeration naming problem. 2. Resolved the problem of redundant return keywords. 3. Fix unnecessary reference issues. 4. Fix unnecessary matches and replace them with if let. 5. Fix unnecessary copying of bool values. Signed-off-by: liyuanr --- .../proxy/src/controller/controller.rs | 47 +++++++++---------- KubeOS-Rust/proxy/src/controller/utils.rs | 12 ++--- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index c21f3044..ad443802 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -59,7 +59,7 @@ pub async fn reconcile( .ok_or(Error::MissingSubResource { value: String::from("node.status.node_info") })? .os_image; debug!("os expected osversion is {},actual osversion is {}", os_cr.spec.osversion, node_os_image); - if check_version(&os_cr.spec.osversion, &node_os_image) { + if check_version(&os_cr.spec.osversion, node_os_image) { match ConfigType::SysConfig.check_config_version(&os, &osinstance) { ConfigOperation::Reassign => { debug!("start reassign"); @@ -92,8 +92,7 @@ pub async fn reconcile( if os_cr.spec.opstype == NODE_STATUS_CONFIG { return Err(Error::UpgradeBeforeConfig); } - match ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) { - ConfigOperation::Reassign => { + if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) { debug!("start reassign"); proxy_controller .refresh_node( @@ -104,8 +103,6 @@ pub async fn reconcile( ) .await?; return Ok(REQUEUE_NORMAL); - } - _ => {} } if node.labels().contains_key(LABEL_UPGRADING) { if osinstance.spec.nodestatus == NODE_STATUS_IDLE { @@ -159,14 +156,14 @@ impl ProxyController { match osi_api.get(node_name).await { Ok(osi) => { debug!("osinstance is exist {:?}", osi.name()); - return Ok(()); + Ok(()) } Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { info!("Create OSInstance {}", node_name); self.controller_client.create_osinstance(node_name, namespace).await?; Ok(()) } - Err(err) => Err(Error::KubeError { source: err }), + Err(err) => Err(Error::KubeClient { source: err }), } } @@ -243,7 +240,7 @@ impl ProxyController { let namespace = &osinstance .namespace() .ok_or(Error::MissingObjectKey { resource: "osinstance".to_string(), value: "namespace".to_string() })?; - self.controller_client.update_osinstance_status(&osinstance.name(), &namespace, &osinstance.status).await?; + self.controller_client.update_osinstance_status(&osinstance.name(), namespace, &osinstance.status).await?; Ok(()) } @@ -256,7 +253,7 @@ impl ProxyController { match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }) { Ok(_resp) => {} Err(e) => { - return Err(Error::AgentError { source: e }); + return Err(Error::Agent { source: e }); } } } @@ -278,9 +275,9 @@ impl ProxyController { image_type: os_cr.spec.imagetype.clone(), check_sum: os_cr.spec.checksum.clone(), container_image: os_cr.spec.containerimage.clone(), - flagsafe: os_cr.spec.flagsafe.clone(), + flagsafe: os_cr.spec.flagsafe, imageurl: os_cr.spec.imageurl.clone(), - mtls: os_cr.spec.mtls.clone(), + mtls: os_cr.spec.mtls, cacert: os_cr.spec.cacert.clone().unwrap_or_default(), clientcert: os_cr.spec.clientcert.clone().unwrap_or_default(), clientkey: os_cr.spec.clientkey.clone().unwrap_or_default(), @@ -289,14 +286,14 @@ impl ProxyController { match self.agent_client.prepare_upgrade_method(upgrade_info) { Ok(_resp) => {} Err(e) => { - return Err(Error::AgentError { source: e }); + return Err(Error::Agent { source: e }); } } self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; match self.agent_client.upgrade_method() { Ok(_resp) => {} Err(e) => { - return Err(Error::AgentError { source: e }); + return Err(Error::Agent { source: e }); } } } @@ -306,12 +303,12 @@ impl ProxyController { match self.agent_client.rollback_method() { Ok(_resp) => {} Err(e) => { - return Err(Error::AgentError { source: e }); + return Err(Error::Agent { source: e }); } } } _ => { - return Err(Error::OperationError { value: os_cr.spec.opstype.clone() }); + return Err(Error::Operation { value: os_cr.spec.opstype.clone() }); } } Ok(()) @@ -336,7 +333,7 @@ impl ProxyController { async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> { use drain::error::DrainError::*; match drain_os(&self.k8s_client.clone(), node_name, force).await { - Err(DeletePodsError { errors, .. }) => Err(Error::DrainNodeError { value: errors.join("; ") }), + Err(DeletePodsError { errors, .. }) => Err(Error::DrainNode { value: errors.join("; ") }), _ => Ok(()), } } @@ -365,13 +362,13 @@ fn convert_to_agent_config(configs: Configs) -> Option> { } }; } - if agent_configs.len() == 0 { + if agent_configs.is_empty() { info!("no contents in all models, no need to configure"); return None; } return Some(agent_configs); } - return None; + None } fn convert_to_config_hashmap(contents: Vec) -> Option> { @@ -381,7 +378,7 @@ fn convert_to_config_hashmap(contents: Vec) -> Option Date: Tue, 30 Jan 2024 16:05:56 +0800 Subject: [PATCH 28/46] fix: clippy warnings and fmt code fix some clippy warnings and fmt code Signed-off-by: Yuhang Wei --- KubeOS-Rust/agent/src/rpc/agent_impl.rs | 8 +- KubeOS-Rust/cli/src/client.rs | 2 +- KubeOS-Rust/manager/src/sys_mgmt/config.rs | 21 ++--- .../manager/src/sys_mgmt/containerd_image.rs | 9 +- .../manager/src/sys_mgmt/docker_image.rs | 4 +- .../proxy/src/controller/apiserver_mock.rs | 67 +++++++++------ .../proxy/src/controller/controller.rs | 75 ++++++++-------- KubeOS-Rust/proxy/src/controller/mod.rs | 2 +- KubeOS-Rust/proxy/src/controller/utils.rs | 12 +-- KubeOS-Rust/proxy/src/drain.rs | 86 +++++++++---------- KubeOS-Rust/proxy/src/main.rs | 2 +- 11 files changed, 148 insertions(+), 140 deletions(-) diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs index 8aef4140..5f3a3259 100644 --- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -56,7 +56,7 @@ impl Default for AgentImpl { } impl AgentImpl { - pub fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result { + fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result { let _lock = self.mutex.lock().unwrap(); debug!("Received an 'prepare upgrade' request: {:?}", req); info!("Start preparing for upgrading to version: {}", req.version); @@ -75,7 +75,7 @@ impl AgentImpl { Ok(Response { status: AgentStatus::UpgradeReady }) } - pub fn upgrade_impl(&self) -> Result { + fn upgrade_impl(&self) -> Result { let _lock = self.mutex.lock().unwrap(); info!("Start to upgrade"); let command_executor = RealCommandExecutor {}; @@ -90,7 +90,7 @@ impl AgentImpl { Ok(Response { status: AgentStatus::Upgraded }) } - pub fn configure_impl(&self, mut req: ConfigureRequest) -> Result { + fn configure_impl(&self, mut req: ConfigureRequest) -> Result { let _lock = self.mutex.lock().unwrap(); debug!("Received a 'configure' request: {:?}", req); info!("Start to configure"); @@ -107,7 +107,7 @@ impl AgentImpl { Ok(Response { status: AgentStatus::Configured }) } - pub fn rollback_impl(&self) -> Result { + fn rollback_impl(&self) -> Result { let _lock = self.mutex.lock().unwrap(); info!("Start to rollback"); let command_executor = RealCommandExecutor {}; diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs index 9765a425..37518bdc 100644 --- a/KubeOS-Rust/cli/src/client.rs +++ b/KubeOS-Rust/cli/src/client.rs @@ -30,7 +30,7 @@ impl Client { Client { json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)) } } - pub fn build_request<'a>(&self, command: &'a str, params: &'a Vec>) -> Request<'a> { + pub fn build_request<'a>(&self, command: &'a str, params: &'a [Box]) -> Request<'a> { let json_rpc_request = self.json_rpc_client.build_request(command, params); let request = Request(json_rpc_request); request diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs index 33efdca8..138df9da 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/config.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -105,10 +105,10 @@ impl Configuration for KernelSysctlPersist { config_path = &config.config_path; } debug!("kernel.sysctl.persist config_path: \"{}\"", config_path); - create_config_file(config_path).with_context(|| format!("Failed to find config path"))?; + create_config_file(config_path).with_context(|| format!("Failed to find config path \"{}\"", config_path))?; let configs = get_and_set_configs(&mut config.contents, config_path) - .with_context(|| format!("Failed to set persist kernel configs"))?; - write_configs_to_file(config_path, &configs).with_context(|| format!("Failed to write configs to file"))?; + .with_context(|| format!("Failed to set persist kernel configs \"{}\"", config_path))?; + write_configs_to_file(config_path, &configs).with_context(|| "Failed to write configs to file".to_string())?; Ok(()) } } @@ -125,7 +125,7 @@ fn create_config_file(config_path: &str) -> Result<()> { } fn get_and_set_configs(expect_configs: &mut HashMap, config_path: &str) -> Result> { - let f = File::open(config_path)?; + let f = File::open(config_path).with_context(|| format!("Failed to open config path \"{}\"", config_path))?; let mut configs_write = Vec::new(); for line in io::BufReader::new(f).lines() { let line = line?; @@ -169,7 +169,7 @@ fn write_configs_to_file(config_path: &str, configs: &Vec) -> Result<()> Ok(()) } -fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String { +fn handle_delete_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String { let key = config_kv[0]; if config_kv.len() == 1 && new_config_info.value.is_empty() { info!("Delete configuration key: \"{}\"", key); @@ -190,7 +190,7 @@ fn handle_delete_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String String::new() } -fn handle_update_key(config_kv: &Vec<&str>, new_config_info: &KeyInfo) -> String { +fn handle_update_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String { let key = config_kv[0]; if !new_config_info.operation.is_empty() { warn!( @@ -259,12 +259,13 @@ impl Configuration for GrubCmdline { self.is_cur_partition } else { self.get_config_partition(RealCommandExecutor {}) - .with_context(|| format!("Failed to get config partition"))? + .with_context(|| "Failed to get config partition".to_string())? }; debug!("Config_partition: {} (false means partition A, true means partition B)", config_partition); let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition) - .with_context(|| format!("Failed to set grub configs"))?; - write_configs_to_file(&self.grub_path, &configs)?; + .with_context(|| "Failed to set grub configs".to_string())?; + write_configs_to_file(&self.grub_path, &configs) + .with_context(|| "Failed to write configs to file".to_string())?; Ok(()) } } @@ -286,7 +287,7 @@ fn get_and_set_grubcfg( grub_path: &str, config_partition: bool, ) -> Result> { - let f = File::open(grub_path)?; + let f = File::open(grub_path).with_context(|| format!("Failed to open grub.cfg \"{}\"", grub_path))?; let re_find_cur_linux = r"^\s*linux.*root=.*"; let re = Regex::new(re_find_cur_linux)?; let mut configs_write = Vec::new(); diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs index 727949b6..80caf291 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -73,12 +73,12 @@ impl CtrImageHandler { .to_str() .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?; info!("Start getting rootfs {}", image_name); - self.check_and_unmount(mount_path).with_context(|| format!("Failed to clean containerd environment"))?; + self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?; self.executor .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?; // copy os.tar from mount_path to its partent dir self.copy_file(self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission)?; - self.check_and_unmount(mount_path).with_context(|| format!("Failed to clean containerd environment"))?; + self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?; Ok(()) } @@ -103,10 +103,7 @@ impl CtrImageHandler { #[cfg(test)] mod tests { - use std::{ - io::Write, - path::{Path, PathBuf}, - }; + use std::{io::Write, path::PathBuf}; use mockall::mock; use tempfile::NamedTempFile; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs index f6971427..4d97552c 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -52,7 +52,7 @@ impl DockerImageHandler { fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> { let image_name = &req.container_image; info!("Start getting rootfs {}", image_name); - self.check_and_rm_container().with_context(|| format!("Failed to remove kubeos-temp container"))?; + self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?; debug!("Create container {}", self.container_name); let container_id = self.executor.run_command_with_output("docker", &["create", "--name", &self.container_name, image_name])?; @@ -65,7 +65,7 @@ impl DockerImageHandler { self.paths.update_path.to_str().unwrap(), ], )?; - self.check_and_rm_container().with_context(|| format!("Failed to remove kubeos-temp container"))?; + self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?; Ok(()) } diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs index ef5977c1..2b182ca8 100644 --- a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs +++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs @@ -1,3 +1,35 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::BTreeMap; + +use anyhow::Result; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use http::{Request, Response}; +use hyper::{body::to_bytes, Body}; +use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod}; +use kube::{ + api::ObjectMeta, + core::{ListMeta, ObjectList}, + Client as KubeClient, Resource, ResourceExt, +}; +use mockall::mock; + use self::mock_error::Error; use super::{ agentclient::*, @@ -10,23 +42,6 @@ use crate::controller::{ values::{LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, ProxyController, }; -use anyhow::Result; -use cli::client::Client; -use cli::method::{ - callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, - rollback::RollbackMethod, upgrade::UpgradeMethod, -}; -use http::{Request, Response}; -use hyper::{body::to_bytes, Body}; -use k8s_openapi::api::core::v1::Pod; -use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo}; -use kube::{ - api::ObjectMeta, - core::{ListMeta, ObjectList}, -}; -use kube::{Client as KubeClient, Resource, ResourceExt}; -use mockall::mock; -use std::collections::BTreeMap; type ApiServerHandle = tower_test::mock::Handle, Response>; pub struct ApiServerVerifier(ApiServerHandle); @@ -66,7 +81,7 @@ impl ApiServerVerifier { .unwrap() .handler_node_get(osi) .await - } + }, Testcases::UpgradeNormal(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -85,7 +100,7 @@ impl ApiServerVerifier { .unwrap() .handler_node_pod_list_get(osi) .await - } + }, Testcases::UpgradeUpgradeconfigsVersionMismatch(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -104,7 +119,7 @@ impl ApiServerVerifier { .unwrap() .handler_osinstance_patch_nodestatus_idle(osi) .await - } + }, Testcases::UpgradeOSInstaceNodestatusConfig(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -114,7 +129,7 @@ impl ApiServerVerifier { .unwrap() .handler_node_get_with_label(osi.clone()) .await - } + }, Testcases::UpgradeOSInstaceNodestatusIdle(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -130,7 +145,7 @@ impl ApiServerVerifier { .unwrap() .handler_node_uncordon(osi) .await - } + }, Testcases::ConfigNormal(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -146,7 +161,7 @@ impl ApiServerVerifier { .unwrap() .handler_osinstance_patch_nodestatus_idle(osi) .await - } + }, Testcases::ConfigVersionMismatchReassign(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -159,7 +174,7 @@ impl ApiServerVerifier { .unwrap() .handler_osinstance_patch_nodestatus_idle(osi) .await - } + }, Testcases::ConfigVersionMismatchUpdate(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -172,7 +187,7 @@ impl ApiServerVerifier { .unwrap() .handler_osinstance_patch_spec_sysconfig_v2(osi) .await - } + }, Testcases::Rollback(osi) => { self.handler_osinstance_get_exist(osi.clone()) .await @@ -191,7 +206,7 @@ impl ApiServerVerifier { .unwrap() .handler_node_pod_list_get(osi) .await - } + }, } .expect("Case completed without errors"); }) diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index ad443802..80a85d1c 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -72,7 +72,7 @@ pub async fn reconcile( ) .await?; return Ok(REQUEUE_NORMAL); - } + }, ConfigOperation::UpdateConfig => { debug!("start update config"); osinstance.spec.sysconfigs = os_cr.spec.sysconfigs.clone(); @@ -81,8 +81,8 @@ pub async fn reconcile( .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec) .await?; return Ok(REQUEUE_ERROR); - } - _ => {} + }, + _ => {}, } proxy_controller.set_config(&mut osinstance, ConfigType::SysConfig).await?; proxy_controller @@ -92,17 +92,17 @@ pub async fn reconcile( if os_cr.spec.opstype == NODE_STATUS_CONFIG { return Err(Error::UpgradeBeforeConfig); } - if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) { - debug!("start reassign"); - proxy_controller - .refresh_node( - node, - osinstance, - &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), - ConfigType::UpgradeConfig, - ) - .await?; - return Ok(REQUEUE_NORMAL); + if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) { + debug!("start reassign"); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); } if node.labels().contains_key(LABEL_UPGRADING) { if osinstance.spec.nodestatus == NODE_STATUS_IDLE { @@ -157,12 +157,12 @@ impl ProxyController { Ok(osi) => { debug!("osinstance is exist {:?}", osi.name()); Ok(()) - } + }, Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { info!("Create OSInstance {}", node_name); self.controller_client.create_osinstance(node_name, namespace).await?; Ok(()) - } + }, Err(err) => Err(Error::KubeClient { source: err }), } } @@ -251,15 +251,15 @@ impl ProxyController { match config_info.configs.and_then(convert_to_agent_config) { Some(agent_configs) => { match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }) { - Ok(_resp) => {} + Ok(_resp) => {}, Err(e) => { return Err(Error::Agent { source: e }); - } + }, } - } + }, None => { info!("config is none, No content can be configured."); - } + }, }; self.update_osi_status(osinstance, config_type).await?; } @@ -284,32 +284,32 @@ impl ProxyController { }; match self.agent_client.prepare_upgrade_method(upgrade_info) { - Ok(_resp) => {} + Ok(_resp) => {}, Err(e) => { return Err(Error::Agent { source: e }); - } + }, } self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; match self.agent_client.upgrade_method() { - Ok(_resp) => {} + Ok(_resp) => {}, Err(e) => { return Err(Error::Agent { source: e }); - } + }, } - } + }, OPERATION_TYPE_ROLLBACK => { self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; match self.agent_client.rollback_method() { - Ok(_resp) => {} + Ok(_resp) => {}, Err(e) => { return Err(Error::Agent { source: e }); - } + }, } - } + }, _ => { return Err(Error::Operation { value: os_cr.spec.opstype.clone() }); - } + }, } Ok(()) } @@ -320,12 +320,12 @@ impl ProxyController { node_api.cordon(node_name).await?; info!("Cordon node Successfully{}, start drain nodes", node_name); match self.drain_node(node_name, evict_pod_force).await { - Ok(()) => {} + Ok(()) => {}, Err(e) => { node_api.uncordon(node_name).await?; info!("Drain node {} error, uncordon node successfully", node_name); return Err(e); - } + }, } Ok(()) } @@ -351,7 +351,7 @@ fn convert_to_agent_config(configs: Configs) -> Option> { contents: contents_tmp, }; agent_configs.push(config_tmp) - } + }, None => { info!( "model {} which has configpath {} do not has any contents no need to configure", @@ -359,7 +359,7 @@ fn convert_to_agent_config(configs: Configs) -> Option> { config.configpath.unwrap_or_default() ); continue; - } + }, }; } if agent_configs.is_empty() { @@ -427,11 +427,14 @@ pub mod reconciler_error { #[cfg(test)] mod test { - use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS}; - use crate::controller::apiserver_mock::{timeout_after_5s, MockAgentCallClient, Testcases}; - use crate::controller::ControllerClient; use std::env; + use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS}; + use crate::controller::{ + apiserver_mock::{timeout_after_5s, MockAgentCallClient, Testcases}, + ControllerClient, + }; + #[tokio::test] async fn test_create_osinstance_with_no_upgrade_or_configuration() { let (test_proxy_controller, fakeserver) = ProxyController::::test(); diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs index e30c8df7..b8a4e6e5 100644 --- a/KubeOS-Rust/proxy/src/controller/mod.rs +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -21,6 +21,6 @@ mod values; pub use agentclient::{AgentCallClient, AgentClient}; pub use apiclient::ControllerClient; -pub use controller::{error_policy, reconcile, reconciler_error::Error, ProxyController}; +pub use controller::{error_policy, reconcile, ProxyController}; pub use crd::OS; pub use values::SOCK_PATH; diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs index 82d960b4..148ca24d 100644 --- a/KubeOS-Rust/proxy/src/controller/utils.rs +++ b/KubeOS-Rust/proxy/src/controller/utils.rs @@ -56,7 +56,7 @@ impl ConfigType { ); return ConfigOperation::Reassign; } - } + }, ConfigType::SysConfig => { let os_config_version = get_config_version(os.spec.sysconfigs.as_ref()); let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); @@ -78,7 +78,7 @@ impl ConfigType { return ConfigOperation::UpdateConfig; } } - } + }, }; ConfigOperation::DoNothing } @@ -96,7 +96,7 @@ impl ConfigType { status_config_version = get_config_version(None); } configs = osinstance.spec.upgradeconfigs.clone(); - } + }, ConfigType::SysConfig => { spec_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); if let Some(osinstance_status) = osinstance.status.as_ref() { @@ -105,7 +105,7 @@ impl ConfigType { status_config_version = get_config_version(None); } configs = osinstance.spec.sysconfigs.clone(); - } + }, } debug!( "osinstance soec config version is {},status config version is {}", @@ -127,7 +127,7 @@ impl ConfigType { sysconfigs: None, }) } - } + }, ConfigType::SysConfig => { if let Some(osi_status) = &mut osinstance.status { osi_status.sysconfigs = osinstance.spec.sysconfigs.clone(); @@ -135,7 +135,7 @@ impl ConfigType { osinstance.status = Some(OSInstanceStatus { upgradeconfigs: None, sysconfigs: osinstance.spec.sysconfigs.clone() }) } - } + }, } } } diff --git a/KubeOS-Rust/proxy/src/drain.rs b/KubeOS-Rust/proxy/src/drain.rs index 72836f98..64417df3 100644 --- a/KubeOS-Rust/proxy/src/drain.rs +++ b/KubeOS-Rust/proxy/src/drain.rs @@ -66,7 +66,7 @@ async fn get_pods_deleted( Ok(pods @ ObjectList { .. }) => pods, Err(err) => { return Err(GetPodListsError { source: err, node_name: node_name.to_string() }); - } + }, }; let mut filterd_pods_list: Vec = Vec::new(); let mut filterd_err: Vec = Vec::new(); @@ -81,7 +81,7 @@ async fn get_pods_deleted( filterd_pods_list.push(pod); } } - if filterd_err.len() > 0 { + if !filterd_err.is_empty() { return Err(DeletePodsError { errors: filterd_err }); } Ok(filterd_pods_list.into_iter()) @@ -189,14 +189,14 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e let name = (&p).name_any(); info!("Pod {} deleted.", name); break; - } + }, Ok(_) => { info!("Pod '{}' is not yet deleted. Waiting {}s.", pod.name_any(), EVERY_DELETION_CHECK.as_secs_f64()); - } + }, Err(kube::Error::Api(e)) if e.code == response_error_not_found => { info!("Pod {} is deleted.", pod.name_any()); break; - } + }, Err(e) => { error!( "Get pod {} reported error: '{}', whether pod is deleted cannot be determined, waiting {}s.", @@ -204,7 +204,7 @@ async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), e e, EVERY_DELETION_CHECK.as_secs_f64() ); - } + }, } if start_time.elapsed() > TIMEOUT { return Err(WaitDeletionError { pod_name: pod.name_any(), max_wait: TIMEOUT }); @@ -223,25 +223,25 @@ fn get_pod_api_with_namespace(client: &kube::Client, pod: &Pod) -> Api { } trait NameAny { - fn name_any(self: &Self) -> String; + fn name_any(&self) -> String; } impl NameAny for &Pod { - fn name_any(self: &Self) -> String { + fn name_any(&self) -> String { self.metadata.name.clone().or_else(|| self.metadata.generate_name.clone()).unwrap_or_default() } } trait PodFilter { - fn filter(self: &Self, pod: &Pod) -> Box; + fn filter(&self, pod: &Pod) -> Box; } struct FinishedOrFailedFilter {} impl PodFilter for FinishedOrFailedFilter { - fn filter(self: &Self, pod: &Pod) -> Box { + fn filter(&self, pod: &Pod) -> Box { return match pod.status.as_ref() { Some(PodStatus { phase: Some(phase), .. }) if phase == "Failed" || phase == "Succeeded" => { FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) - } + }, _ => FilterResult::create_filter_result(false, "", PodDeleteStatus::Okay), }; } @@ -251,7 +251,7 @@ struct DaemonFilter { force: bool, } impl PodFilter for DaemonFilter { - fn filter(self: &Self, pod: &Pod) -> Box { + fn filter(&self, pod: &Pod) -> Box { if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); } @@ -269,25 +269,25 @@ impl PodFilter for DaemonFilter { let description = format!("Cannot drain Pod '{}': Pod is member of a DaemonSet", pod.name_any()); Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) } - } + }, _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } } impl DaemonFilter { fn new(force: bool) -> DaemonFilter { - return DaemonFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force: force }; + DaemonFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } } } struct MirrorFilter {} impl PodFilter for MirrorFilter { - fn filter(self: &Self, pod: &Pod) -> Box { + fn filter(&self, pod: &Pod) -> Box { return match pod.metadata.annotations.as_ref() { Some(annotations) if annotations.contains_key("kubernetes.io/config.mirror") => { let description = format!("Ignore Pod '{}': Pod is a static Mirror Pod", pod.name_any()); FilterResult::create_filter_result(false, &description.to_string(), PodDeleteStatus::Warning) - } + }, _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } @@ -298,7 +298,7 @@ struct LocalStorageFilter { force: bool, } impl PodFilter for LocalStorageFilter { - fn filter(self: &Self, pod: &Pod) -> Box { + fn filter(&self, pod: &Pod) -> Box { if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); } @@ -312,14 +312,14 @@ impl PodFilter for LocalStorageFilter { let description = format!("Cannot drain Pod '{}': Pod has local Storage", pod.name_any()); Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) } - } + }, _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } } impl LocalStorageFilter { fn new(force: bool) -> LocalStorageFilter { - return LocalStorageFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force: force }; + LocalStorageFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } } } struct UnreplicatedFilter { @@ -327,7 +327,7 @@ struct UnreplicatedFilter { force: bool, } impl PodFilter for UnreplicatedFilter { - fn filter(self: &Self, pod: &Pod) -> Box { + fn filter(&self, pod: &Pod) -> Box { if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); } @@ -338,18 +338,18 @@ impl PodFilter for UnreplicatedFilter { return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); } - return if !is_replicated && self.force { + if !is_replicated && self.force { let description = format!("Force drain Pod '{}': Pod is unreplicated", pod.name_any()); Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning }) } else { let description = format!("Cannot drain Pod '{}': Pod is unreplicated", pod.name_any()); Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) - }; + } } } impl UnreplicatedFilter { fn new(force: bool) -> UnreplicatedFilter { - return UnreplicatedFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force: force }; + UnreplicatedFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } } } @@ -357,7 +357,7 @@ struct DeletedFilter { delete_wait_timeout: Duration, } impl PodFilter for DeletedFilter { - fn filter(self: &Self, pod: &Pod) -> Box { + fn filter(&self, pod: &Pod) -> Box { let now = Instant::now().elapsed(); return match pod.metadata.deletion_timestamp.as_ref() { Some(time) @@ -365,7 +365,7 @@ impl PodFilter for DeletedFilter { && now - Duration::from_secs(time.0.timestamp() as u64) >= self.delete_wait_timeout => { FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) - } + }, _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), }; } @@ -379,14 +379,14 @@ struct CombinedFilter { unreplicated_filter: UnreplicatedFilter, } impl PodFilter for CombinedFilter { - fn filter(self: &Self, pod: &Pod) -> Box { + fn filter(&self, pod: &Pod) -> Box { let mut filter_res = self.deleted_filter.filter(pod); if !filter_res.result { info!("{}", filter_res.desc); return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), - status: filter_res.status.clone(), + status: filter_res.status, }); } filter_res = self.daemon_filter.filter(pod); @@ -395,7 +395,7 @@ impl PodFilter for CombinedFilter { return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), - status: filter_res.status.clone(), + status: filter_res.status, }); } filter_res = self.mirror_filter.filter(pod); @@ -404,7 +404,7 @@ impl PodFilter for CombinedFilter { return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), - status: filter_res.status.clone(), + status: filter_res.status, }); } filter_res = self.local_storage_filter.filter(pod); @@ -413,7 +413,7 @@ impl PodFilter for CombinedFilter { return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), - status: filter_res.status.clone(), + status: filter_res.status, }); } filter_res = self.unreplicated_filter.filter(pod); @@ -422,22 +422,22 @@ impl PodFilter for CombinedFilter { return Box::new(FilterResult { result: filter_res.result, desc: filter_res.desc.clone(), - status: filter_res.status.clone(), + status: filter_res.status, }); } - return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) } } impl CombinedFilter { fn new(force: bool) -> CombinedFilter { - return CombinedFilter { + CombinedFilter { deleted_filter: DeletedFilter { delete_wait_timeout: TIMEOUT }, daemon_filter: DaemonFilter::new(force), mirror_filter: MirrorFilter {}, local_storage_filter: LocalStorageFilter::new(force), unreplicated_filter: UnreplicatedFilter::new(force), - }; + } } } @@ -454,7 +454,7 @@ struct FilterResult { } impl FilterResult { fn create_filter_result(result: bool, desc: &str, status: PodDeleteStatus) -> Box { - Box::new(FilterResult { result: result, desc: desc.to_string(), status: status }) + Box::new(FilterResult { result, desc: desc.to_string(), status }) } } @@ -468,13 +468,11 @@ impl ErrorHandleStrategy { let backoff = ExponentialBackoff::from_millis(RETRY_BASE_DELAY.as_millis() as u64).max_delay(RETRY_MAX_DELAY).map(jitter); - return match self { - Self::TolerateStrategy => { - return backoff.take(0); - } + match self { + Self::TolerateStrategy => backoff.take(0), Self::RetryStrategy => backoff.take(MAX_RETRIES_TIMES), - }; + } } } @@ -482,13 +480,7 @@ impl tokio_retry::Condition for ErrorHandleStrategy { fn should_retry(&mut self, error: &error::EvictionError) -> bool { match self { Self::TolerateStrategy => false, - Self::RetryStrategy => { - if let error::EvictionError::EvictionErrorRetry { .. } = error { - true - } else { - false - } - } + Self::RetryStrategy => matches!(error, error::EvictionError::EvictionErrorRetry { .. }), } } } diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs index ad36b642..5c122ba2 100644 --- a/KubeOS-Rust/proxy/src/main.rs +++ b/KubeOS-Rust/proxy/src/main.rs @@ -39,7 +39,7 @@ async fn main() -> Result<()> { .run(reconcile, error_policy, Context::new(proxy_controller)) .for_each(|res| async move { match res { - Ok(_o) => {} + Ok(_o) => {}, Err(e) => error!("reconcile failed: {}", e.to_string()), } }) -- Gitee From a8f5c5a424a8e406b84bb22337c5f42c30dafdcd Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 5 Feb 2024 10:04:46 +0800 Subject: [PATCH 29/46] Bump tokio to 1.28.0 Signed-off-by: Yuhang Wei --- KubeOS-Rust/Cargo.lock | 42 ++++++++++-------------------------- KubeOS-Rust/proxy/Cargo.toml | 6 ++++-- 2 files changed, 15 insertions(+), 33 deletions(-) diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 2342c7b2..004ef234 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -1133,24 +1133,14 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mio" -version = "0.7.14" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "log", - "miow", - "ntapi", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", ] [[package]] @@ -1234,15 +1224,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", -] - [[package]] name = "num-traits" version = "0.2.17" @@ -2077,21 +2058,20 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.14.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" dependencies = [ "autocfg", "bytes", "libc", - "memchr", "mio", "num_cpus", - "once_cell", "pin-project-lite", "signal-hook-registry", + "socket2", "tokio-macros", - "winapi", + "windows-sys", ] [[package]] @@ -2106,13 +2086,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.37", ] [[package]] diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index 94e3b3c8..429c5fdb 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -35,7 +35,10 @@ serde_json = "1.0.68" socket2 = "=0.4.9" thiserror = "1.0.29" thread_local = "=1.1.4" -tokio = { version = "=1.14.0", features = ["macros", "rt-multi-thread"] } +tokio = { version = "=1.28.0", default-features = false, features = [ + "macros", + "rt-multi-thread", +] } tokio-retry = "0.3" [dev-dependencies] @@ -44,4 +47,3 @@ http = "0.2.9" hyper = "0.14.25" tower-test = "0.4.0" mockall = { version = "=0.11.3" } - -- Gitee From 0b4843d4514cd8b7e653990025d0ecd5e80d56ba Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 20 Feb 2024 10:18:27 +0800 Subject: [PATCH 30/46] fix: mutex locking in agent_impl.rs Signed-off-by: Yuhang Wei --- KubeOS-Rust/agent/src/rpc/agent_impl.rs | 32 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs index 5f3a3259..ab826413 100644 --- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -57,7 +57,10 @@ impl Default for AgentImpl { impl AgentImpl { fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result { - let _lock = self.mutex.lock().unwrap(); + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } debug!("Received an 'prepare upgrade' request: {:?}", req); info!("Start preparing for upgrading to version: {}", req.version); @@ -76,7 +79,10 @@ impl AgentImpl { } fn upgrade_impl(&self) -> Result { - let _lock = self.mutex.lock().unwrap(); + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } info!("Start to upgrade"); let command_executor = RealCommandExecutor {}; let (_, next_partition_info) = get_partition_info(&command_executor)?; @@ -91,7 +97,10 @@ impl AgentImpl { } fn configure_impl(&self, mut req: ConfigureRequest) -> Result { - let _lock = self.mutex.lock().unwrap(); + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } debug!("Received a 'configure' request: {:?}", req); info!("Start to configure"); let config_map = &*CONFIG_TEMPLATE; @@ -108,7 +117,10 @@ impl AgentImpl { } fn rollback_impl(&self) -> Result { - let _lock = self.mutex.lock().unwrap(); + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } info!("Start to rollback"); let command_executor = RealCommandExecutor {}; let (_, next_partition_info) = get_partition_info(&command_executor)?; @@ -172,6 +184,18 @@ mod test { }; let res = agent.configure(req); assert!(res.is_err()); + + // test lock + let _lock = agent.mutex.lock().unwrap(); + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "kernel.sysctl".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req); + assert!(res.is_err()); } #[test] -- Gitee From 3bbb48b9e2569514caa88b9d67aa14d67a48432f Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 20 Feb 2024 10:18:42 +0800 Subject: [PATCH 31/46] fix: partition info retrieval in get_partition_info function Signed-off-by: Yuhang Wei --- KubeOS-Rust/manager/src/utils/partition.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs index fcfa2d8b..799b4b35 100644 --- a/KubeOS-Rust/manager/src/utils/partition.rs +++ b/KubeOS-Rust/manager/src/utils/partition.rs @@ -50,7 +50,7 @@ pub fn get_partition_info(executor: &T) -> Result<(Partition } } } - if cur_partition.device.is_empty() { + if cur_partition.menuentry.is_empty() { bail!("Failed to get partition info, lsblk output: {}", lsblk); } Ok((cur_partition, next_partition)) @@ -108,5 +108,10 @@ mod tests { mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string())); let res = get_partition_info(&mock); assert!(res.is_err()); + + let command_output4 = "sda4 / ext4"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output4.to_string())); + let res = get_partition_info(&mock); + assert!(res.is_err()); } } -- Gitee From f044f68267ad12bd1eab35da57402385f981c1c2 Mon Sep 17 00:00:00 2001 From: iGxnon Date: Tue, 26 Mar 2024 13:45:41 +0800 Subject: [PATCH 32/46] bump rust reqwest to 0.12.2 to support build on riscv Signed-off-by: iGxnon --- KubeOS-Rust/Cargo.lock | 434 +++++++++++++++++++++++++-------- KubeOS-Rust/manager/Cargo.toml | 2 +- KubeOS-Rust/proxy/Cargo.toml | 2 +- 3 files changed, 339 insertions(+), 99 deletions(-) diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 004ef234..4e1619aa 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -184,7 +184,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -367,6 +367,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.3" @@ -375,7 +381,7 @@ checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -587,8 +593,27 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", - "indexmap", + "http 0.2.9", + "indexmap 1.9.3", + "slab", + "tokio", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", + "indexmap 2.2.6", "slab", "tokio", "tokio-util 0.7.2", @@ -601,6 +626,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -627,6 +658,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -634,7 +676,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", - "http", + "http 0.2.9", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -672,31 +737,54 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "http 0.2.9", + "http-body 0.4.5", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" -version = "0.24.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ - "http", - "hyper", + "futures-util", + "http 1.1.0", + "hyper 1.2.0", + "hyper-util", "rustls", + "rustls-pki-types", "tokio", "tokio-rustls", + "tower-service", ] [[package]] @@ -705,7 +793,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.25", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -718,10 +806,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.25", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.2.0", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.2.0", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -770,7 +894,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", ] [[package]] @@ -790,7 +924,7 @@ checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.3", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -925,7 +1059,7 @@ dependencies = [ "base64 0.13.1", "bytes", "chrono", - "http", + "http 0.2.9", "percent-encoding", "serde", "serde-value", @@ -958,11 +1092,11 @@ dependencies = [ "dirs-next", "either", "futures", - "http", - "http-body", - "hyper", + "http 0.2.9", + "http-body 0.4.5", + "hyper 0.14.25", "hyper-timeout", - "hyper-tls", + "hyper-tls 0.5.0", "jsonpath_lib", "k8s-openapi", "kube-core", @@ -990,7 +1124,7 @@ checksum = "de491f8c9ee97117e0b47a629753e939c2392d5d0a40f6928e582a5fba328098" dependencies = [ "chrono", "form_urlencoded", - "http", + "http 0.2.9", "json-patch", "k8s-openapi", "once_cell", @@ -1140,7 +1274,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1475,9 +1609,9 @@ dependencies = [ "cli", "env_logger", "futures", - "h2", - "http", - "hyper", + "h2 0.3.16", + "http 0.2.9", + "hyper 0.14.25", "k8s-openapi", "kube", "log", @@ -1488,7 +1622,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "socket2", + "socket2 0.4.9", "thiserror", "thread_local", "tokio", @@ -1633,21 +1767,24 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.18" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" dependencies = [ "base64 0.21.5", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", "hyper-rustls", - "hyper-tls", + "hyper-tls 0.6.0", + "hyper-util", "ipnet", "js-sys", "log", @@ -1658,9 +1795,12 @@ dependencies = [ "pin-project-lite", "rustls", "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", + "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", @@ -1675,17 +1815,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.20" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", + "getrandom 0.2.10", "libc", - "once_cell", "spin", "untrusted", - "web-sys", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -1699,19 +1839,21 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "rustls" -version = "0.21.1" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c" dependencies = [ "log", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] @@ -1723,13 +1865,20 @@ dependencies = [ "base64 0.21.5", ] +[[package]] +name = "rustls-pki-types" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868e20fada228fefaf6b652e00cc73623d54f8171e7352c18bb281571f2d92da" + [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1745,7 +1894,7 @@ version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1778,16 +1927,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "secrecy" version = "0.8.0" @@ -1868,7 +2007,7 @@ version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" dependencies = [ - "indexmap", + "indexmap 1.9.3", "itoa", "ryu", "serde", @@ -1892,7 +2031,7 @@ version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ - "indexmap", + "indexmap 1.9.3", "ryu", "serde", "yaml-rust", @@ -1935,9 +2074,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" @@ -1949,11 +2088,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "spin" -version = "0.5.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "strsim" @@ -1961,6 +2110,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.109" @@ -1983,6 +2138,33 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.6.0" @@ -1994,7 +2176,7 @@ dependencies = [ "fastrand", "redox_syscall 0.3.5", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2069,9 +2251,9 @@ dependencies = [ "num_cpus", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.4.9", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2118,11 +2300,12 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ "rustls", + "rustls-pki-types", "tokio", ] @@ -2216,8 +2399,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.9", + "http-body 0.4.5", "http-range-header", "pin-project-lite", "tower-layer", @@ -2337,9 +2520,9 @@ dependencies = [ [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" @@ -2461,23 +2644,13 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki-roots" -version = "0.22.6" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" dependencies = [ - "webpki", + "rustls-pki-types", ] [[package]] @@ -2517,7 +2690,7 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -2526,7 +2699,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", ] [[package]] @@ -2535,13 +2717,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -2550,49 +2747,92 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if", + "windows-sys 0.48.0", ] [[package]] diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml index 311a87c1..e694090c 100644 --- a/KubeOS-Rust/manager/Cargo.toml +++ b/KubeOS-Rust/manager/Cargo.toml @@ -19,7 +19,7 @@ lazy_static = { version = "1.4" } log = { version = "0.4" } nix = { version = "0.26.2" } regex = { version = "1.7.3" } -reqwest = { version = "=0.11.18", features = ["blocking", "rustls-tls"] } +reqwest = { version = "=0.12.2", features = ["blocking", "rustls-tls"] } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } sha2 = { version = "0.10.8" } diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index 429c5fdb..3b5d96d1 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -26,7 +26,7 @@ kube = { version = "0.66.0", features = ["derive", "runtime"] } log = "=0.4.15" manager = { version = "1.0.5", path = "../manager" } regex = "=1.7.3" -reqwest = { version = "=0.11.18", default-features = false, features = [ +reqwest = { version = "=0.12.2", default-features = false, features = [ "json", ] } schemars = "=0.8.10" -- Gitee From dbabd83278e84e21a4e3493150add216012ba230 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 17 Jun 2024 10:30:52 +0800 Subject: [PATCH 33/46] Bump kubeos version to 1.0.6 Signed-off-by: Yuhang Wei --- KubeOS-Rust/Cargo.lock | 8 ++++---- KubeOS-Rust/agent/Cargo.toml | 2 +- KubeOS-Rust/cli/Cargo.toml | 2 +- KubeOS-Rust/manager/Cargo.toml | 2 +- KubeOS-Rust/proxy/Cargo.toml | 6 +++--- VERSION | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 4e1619aa..93e3d07d 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -189,7 +189,7 @@ dependencies = [ [[package]] name = "cli" -version = "1.0.5" +version = "1.0.6" dependencies = [ "anyhow", "jsonrpc", @@ -1226,7 +1226,7 @@ dependencies = [ [[package]] name = "manager" -version = "1.0.5" +version = "1.0.6" dependencies = [ "anyhow", "env_logger", @@ -1438,7 +1438,7 @@ dependencies = [ [[package]] name = "os-agent" -version = "1.0.5" +version = "1.0.6" dependencies = [ "anyhow", "env_logger", @@ -1601,7 +1601,7 @@ dependencies = [ [[package]] name = "proxy" -version = "1.0.5" +version = "1.0.6" dependencies = [ "anyhow", "assert-json-diff", diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml index 739bbbc7..83e1b7c0 100644 --- a/KubeOS-Rust/agent/Cargo.toml +++ b/KubeOS-Rust/agent/Cargo.toml @@ -3,7 +3,7 @@ description = "KubeOS os-agent" edition = "2021" license = "MulanPSL-2.0" name = "os-agent" -version = "1.0.5" +version = "1.0.6" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/KubeOS-Rust/cli/Cargo.toml b/KubeOS-Rust/cli/Cargo.toml index c3c14c6f..78d5fd51 100644 --- a/KubeOS-Rust/cli/Cargo.toml +++ b/KubeOS-Rust/cli/Cargo.toml @@ -3,7 +3,7 @@ description = "KubeOS os-agent client" edition = "2021" license = "MulanPSL-2.0" name = "cli" -version = "1.0.5" +version = "1.0.6" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml index e694090c..f60a7c08 100644 --- a/KubeOS-Rust/manager/Cargo.toml +++ b/KubeOS-Rust/manager/Cargo.toml @@ -3,7 +3,7 @@ description = "KubeOS os-agent manager" edition = "2021" license = "MulanPSL-2.0" name = "manager" -version = "1.0.5" +version = "1.0.6" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dev-dependencies] diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml index 3b5d96d1..d804ac77 100644 --- a/KubeOS-Rust/proxy/Cargo.toml +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -3,7 +3,7 @@ description = "KubeOS os-proxy" edition = "2021" license = "MulanPSL-2.0" name = "proxy" -version = "1.0.5" +version = "1.0.6" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] @@ -17,14 +17,14 @@ path = "src/main.rs" [dependencies] anyhow = "1.0.44" async-trait = "0.1" -cli = { version = "1.0.5", path = "../cli" } +cli = { version = "1.0.6", path = "../cli" } env_logger = "0.9.0" futures = "0.3.17" h2 = "=0.3.16" k8s-openapi = { version = "0.13.1", features = ["v1_22"] } kube = { version = "0.66.0", features = ["derive", "runtime"] } log = "=0.4.15" -manager = { version = "1.0.5", path = "../manager" } +manager = { version = "1.0.6", path = "../manager" } regex = "=1.7.3" reqwest = { version = "=0.12.2", default-features = false, features = [ "json", diff --git a/VERSION b/VERSION index 90a27f9c..af0b7ddb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.5 +1.0.6 -- Gitee From e77d86d37c9320547b98b5639dfa328f1b0542fb Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 17 Jun 2024 10:34:37 +0800 Subject: [PATCH 34/46] fix: update bootloader.sh paths for EFI boot Signed-off-by: Yuhang Wei --- scripts/bootloader.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/bootloader.sh b/scripts/bootloader.sh index 75096a38..df4be329 100644 --- a/scripts/bootloader.sh +++ b/scripts/bootloader.sh @@ -19,7 +19,7 @@ function install_grub2_x86 () cp -r /usr/lib/grub/x86_64-efi boot/efi/EFI/openEuler eval "grub2-mkimage -d /usr/lib/grub/x86_64-efi -O x86_64-efi --output=/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" - mkdir -p /boot/EFI/BOOT/ + mkdir -p /boot/efi/EFI/BOOT/ cp -f /boot/efi/EFI/openEuler/grubx64.efi /boot/efi/EFI/BOOT/BOOTX64.EFI fi } @@ -29,7 +29,7 @@ function install_grub2_efi () cp -r /usr/lib/grub/arm64-efi /boot/efi/EFI/openEuler/ eval "grub2-mkimage -d /usr/lib/grub/arm64-efi -O arm64-efi --output=/boot/efi/EFI/openEuler/grubaa64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" - mkdir -p /boot/EFI/BOOT/ + mkdir -p /boot/efi/EFI/BOOT/ cp -f /boot/efi/EFI/openEuler/grubaa64.efi /boot/efi/EFI/BOOT/BOOTAA64.EFI } -- Gitee From 1559b84824331e4c7978b0777d274119cacd82d7 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 12 Aug 2024 19:42:24 +0800 Subject: [PATCH 35/46] docs: update readme and format yaml add kbimg command line explanation and the way to generate root passwd for login KubeOS format yamls Signed-off-by: Yuhang Wei --- .../admin-container/admin-container.yaml | 126 +++++++++--------- docs/example/config/manager/manager.yaml | 58 ++++---- .../config/samples/upgrade_v1alpha1_os.yaml | 68 +++++----- docs/quick-start.md | 122 ++++++++++------- ...66\344\275\234\346\214\207\345\257\274.md" | 71 +++++----- 5 files changed, 238 insertions(+), 207 deletions(-) diff --git a/docs/example/config/admin-container/admin-container.yaml b/docs/example/config/admin-container/admin-container.yaml index b5ec9e03..518b955f 100644 --- a/docs/example/config/admin-container/admin-container.yaml +++ b/docs/example/config/admin-container/admin-container.yaml @@ -1,63 +1,63 @@ -apiVersion: v1 -kind: Secret -metadata: - name: root-secret -data: - ssh-pub-key: your-ssh-pub-key ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: admin-container-sysmaster - namespace: default - labels: - control-plane: admin-container-sysmaster -spec: - selector: - matchLabels: - control-plane: admin-container-sysmaster - replicas: 1 - template: - metadata: - labels: - control-plane: admin-container-sysmaster - spec: - hostPID: true - containers: - - name: admin-container-sysmaster - image: your_imageRepository/admin_imageName:version - imagePullPolicy: Always - securityContext: - privileged: true - ports: - - containerPort: 22 - # sysmaster要求 - env: - - name: container - value: containerd - volumeMounts: - # name 必须与下面的卷名匹配 - - name: secret-volume - # mountPath必须为/etc/secret-volume - mountPath: /etc/secret-volume - readOnly: true - nodeName: your-worker-node-name - volumes: - - name: secret-volume - secret: - # secretName必须与上面指定的Secret的name相同 - secretName: root-secret ---- -apiVersion: v1 -kind: Service -metadata: - name: admin-container-sysmaster - namespace: default -spec: - type: NodePort - ports: - - port: 22 - targetPort: 22 - nodePort: your-exposed-port - selector: - control-plane: admin-container-sysmaster \ No newline at end of file +apiVersion: v1 +kind: Secret +metadata: + name: root-secret +data: + ssh-pub-key: +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin-container-sysmaster + namespace: default + labels: + control-plane: admin-container-sysmaster +spec: + selector: + matchLabels: + control-plane: admin-container-sysmaster + replicas: 1 + template: + metadata: + labels: + control-plane: admin-container-sysmaster + spec: + hostPID: true + containers: + - name: admin-container-sysmaster + image: + imagePullPolicy: Always + securityContext: + privileged: true + ports: + - containerPort: 22 + # sysmaster要求 + env: + - name: container + value: containerd + volumeMounts: + # name 必须与下面的卷名匹配 + - name: secret-volume + # mountPath必须为/etc/secret-volume + mountPath: /etc/secret-volume + readOnly: true + nodeName: + volumes: + - name: secret-volume + secret: + # secretName必须与上面指定的Secret的name相同 + secretName: root-secret +--- +apiVersion: v1 +kind: Service +metadata: + name: admin-container-sysmaster + namespace: default +spec: + type: NodePort + ports: + - port: 22 + targetPort: 22 + nodePort: + selector: + control-plane: admin-container-sysmaster diff --git a/docs/example/config/manager/manager.yaml b/docs/example/config/manager/manager.yaml index 93d15220..f5cf84f9 100644 --- a/docs/example/config/manager/manager.yaml +++ b/docs/example/config/manager/manager.yaml @@ -20,19 +20,19 @@ spec: control-plane: upgrade-proxy spec: containers: - - name: proxy - command: - - /proxy - image: edit.proxy.image.addr - volumeMounts: - - name: upgrade-agent - mountPath: /var/run/os-agent - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName + - name: proxy + command: + - /proxy + image: + volumeMounts: + - name: upgrade-agent + mountPath: /var/run/os-agent + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName volumes: - name: upgrade-agent hostPath: @@ -56,24 +56,26 @@ spec: control-plane: upgrade-operator spec: containers: - - command: - - /operator - image: edit.operator.image.addr - name: operator - securityContext: - allowPrivilegeEscalation: false - runAsUser: 6552 - runAsGroup: 6552 - resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 20Mi + - command: + - /operator + image: + name: operator + securityContext: + allowPrivilegeEscalation: false + runAsUser: 6552 + runAsGroup: 6552 + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi terminationGracePeriodSeconds: 10 nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - key: "node-role.kubernetes.io/master" operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" diff --git a/docs/example/config/samples/upgrade_v1alpha1_os.yaml b/docs/example/config/samples/upgrade_v1alpha1_os.yaml index b33a30b2..d7528831 100644 --- a/docs/example/config/samples/upgrade_v1alpha1_os.yaml +++ b/docs/example/config/samples/upgrade_v1alpha1_os.yaml @@ -1,38 +1,38 @@ apiVersion: upgrade.openeuler.org/v1alpha1 kind: OS metadata: - name: os-sample + name: os-sample spec: - imagetype: docker/containerd/disk - opstype: upgrade/config/rollback - osversion: edit.os.version - maxunavailable: edit.node.upgrade.number - containerimage: "" - evictpodforce: true - imageurl: "" - checksum: image digests - flagSafe: false - mtls: false - sysconfigs: - version: edit.sysconfig.version - configs: - - model: kernel.systcl - contents: - - key: kernel param key1 - value: kernel param value1 - - key: kernel param key2 - value: kernel param value2 - - model: kernel.systcl.persist - configpath: persist file path - contents: - - key: kernel param key3 - value: kernel param value3 - operation: delete - upgradeconfigs: - version: edit.upgradeconfig.version - configs: - - model: kernel.systcl - contents: - - key: kernel param key4 - value: kernel param value4 - operation: delete \ No newline at end of file + imagetype: docker/containerd/disk + opstype: upgrade/config/rollback + osversion: edit.os.version + maxunavailable: edit.node.upgrade.number + containerimage: "" + evictpodforce: true + imageurl: "" + checksum: image digests + flagSafe: false + mtls: false + sysconfigs: + version: edit.sysconfig.version + configs: + - model: kernel.sysctl + contents: + - key: kernel param key1 + value: kernel param value1 + - key: kernel param key2 + value: kernel param value2 + - model: kernel.sysctl.persist + configpath: persist file path + contents: + - key: kernel param key3 + value: kernel param value3 + operation: delete + upgradeconfigs: + version: edit.upgradeconfig.version + configs: + - model: kernel.sysctl + contents: + - key: kernel param key4 + value: kernel param value4 + operation: delete diff --git a/docs/quick-start.md b/docs/quick-start.md index 9656fb99..372c904b 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -10,8 +10,8 @@ * golang(大于等于1.15版本) * make * git - * rust(大于等于1.57版本) - * cargo(大于等于1.57版本) + * rust(大于等于1.64版本) + * cargo(大于等于1.64版本) * openssl-devel ``` shell @@ -65,11 +65,18 @@ * Dockerfile参考如下, Dockerfile也可以使用多阶段构建: + `proxy`容器镜像Dockerfile + ``` dockerfile - FROM your_baseimage + FROM openeuler/openeuler:24.03-lts COPY ./bin/proxy /proxy ENTRYPOINT ["/proxy"] - FROM your_baseimage + ``` + + `operator`容器镜像Dockerfile + + ``` dockerfile + FROM openeuler/openeuler:24.03-lts COPY --chown=6552:6552 ./bin/operator /operator ENTRYPOINT ["/operator"] ``` @@ -112,14 +119,34 @@ bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' ``` - * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 - * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： - * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 - * qcow2 格式的系统镜像 system.qcow2。 - * 可用于升级的根文件系统分区镜像 update.img 。 - * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数 - * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) - * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 + 参数说明如下: + + ```bash + Usage : kbimg create vm-image -p iso-path -v os-version -b os-agent-dir -e os-password + or + kbimg create vm-image -d repository/name:tag + + options: + -p repo path + -v KubeOS version + -b path of os-agent binary + -e os encrypted password + -d docker image like repository/name:tag + -l boot to legacy BIOS mode, if not specify, then UEFI mode + -h,--help show help information + ``` + + * 其中 xxx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 + * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： + * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 + * qcow2 格式的系统镜像 system.qcow2。 + * 可用于升级的根文件系统分区镜像 update.img 。 + * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景。若x86 架构的虚拟机需要使用 legacy 启动模式，需制作镜像时指定-l参数 + * 默认root密码为openEuler12#$ + * 您可通过`openssl passwd -6 -salt $(head -c18 /dev/urandom | openssl base64)`命令生成root密码并通过`-e`参数配置密码 + * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) + * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 + * 详细参数说明请见[《容器OS镜像制作指导》](../docs/user_guide/%E5%AE%B9%E5%99%A8OS%E9%95%9C%E5%83%8F%E5%88%B6%E4%BD%9C%E6%8C%87%E5%AF%BC.md) * 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 @@ -173,7 +200,7 @@ ## 使用指导 -#### 注意事项 +### 注意事项 * 公共注意事项 * 仅支持虚拟机x86和arm64 UEFI场景。 @@ -209,7 +236,7 @@ | 参数 |参数类型 | 参数说明 | 使用说明 | 是否必选 | | -------------- | ------ | ------------------------------------------------------------ | ----- | ---------------- | - | imagetype | string | 升级镜像的类型 | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。
**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是 | + | imagetype | string | 升级镜像的类型 | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是 | | opstype | string | 操作类型：升级,回退或者配置 | 仅支持upgrade ，config 或者 rollback |是 | | osversion | string | 升级/回退的目标版本 | osversion需与节点的目标os版本对应（节点上/etc/os-release中PRETTY_NAME字段或k8s检查到的节点os版本） 例如：KubeOS 1.0.0。 |是 | | maxunavailable | int | 每批同时进行升级/回退/配置的节点数。 | maxunavailable值大于实际节点数时，取实际节点数进行升级/回退/配置。 |是 | @@ -357,13 +384,13 @@ kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' ``` -* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改 +* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改 #### 配置（Settings）指导 * Settings参数说明: - 基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致： + 基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致： ```yaml apiVersion: upgrade.openeuler.org/v1alpha1 @@ -424,9 +451,7 @@ | value | string | 参数值 | key=value形式的参数中，value不能为空，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中对每种配置类型对value的说明。 | key=value形式的参数必选 | | operation | string | 对参数进行的操作 | 仅对kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next类型的参数生效。默认为添加或更新。仅支持配置为delete，代表删除已存在的参数（key=value需完全一致才能删除）。 | 否 | - - - * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置 + * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置 * 使用说明 @@ -463,7 +488,7 @@ 2. upgrade模式重新升级至上一版本 * 手动回退指导 - * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。 + * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。 * 工具回退指导 * 回退至任意版本 * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 @@ -554,19 +579,14 @@ KubeOS提供一个分离的包含sshd服务和hostshell工具的Admin容器， ```Dockerfile FROM openeuler-22.03-lts-sp1 - RUN yum -y install openssh-clients util-linux - ADD ./your-sysmaster.rpm /home RUN rpm -ivh /home/your-sysmaster.rpm - COPY ./hostshell /usr/bin/ COPY ./set-ssh-pub-key.sh /usr/local/bin COPY ./set-ssh-pub-key.service /usr/lib/sysmaster - EXPOSE 22 RUN sed -i 's/sysinit.target/sysinit.target;sshd.service;set-ssh-pub-key.service/g' /usr/lib/sysmaster/basic.target - CMD ["/usr/lib/sysmaster/init"] ``` @@ -594,8 +614,7 @@ kind: Secret metadata: name: root-secret data: - # base64 encode your pub key in one line - ssh-pub-key: your-ssh-pub-key + ssh-pub-key: --- apiVersion: apps/v1 kind: Deployment @@ -603,7 +622,7 @@ metadata: name: admin-container-sysmaster namespace: default labels: - control-plane: admin-container-sysmaster + control-plane: admin-container-sysmaster spec: selector: matchLabels: @@ -616,24 +635,24 @@ spec: spec: hostPID: true containers: - - name: admin-container-sysmaster - image: your_imageRepository/admin_imageName:version - imagePullPolicy: Always - securityContext: - privileged: true - ports: - - containerPort: 22 - # sysmaster要求 - env: - - name: container - value: containerd - volumeMounts: - # name 必须与下面的卷名匹配 - - name: secret-volume - # mountPath必须为/etc/secret-volume - mountPath: /etc/secret-volume - readOnly: true - nodeName: your-worker-node-name + - name: admin-container-sysmaster + image: + imagePullPolicy: Always + securityContext: + privileged: true + ports: + - containerPort: 22 + # sysmaster要求 + env: + - name: container + value: containerd + volumeMounts: + # name 必须与下面的卷名匹配 + - name: secret-volume + # mountPath必须为/etc/secret-volume + mountPath: /etc/secret-volume + readOnly: true + nodeName: volumes: - name: secret-volume secret: @@ -650,9 +669,9 @@ spec: ports: - port: 22 targetPort: 22 - nodePort: your-exposed-port + nodePort: selector: - control-plane: admin-container-sysmaster + control-plane: admin-container-sysmaster ``` ### admin容器使用 @@ -680,6 +699,7 @@ hostshell #### kernel Settings * kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: + ```yaml configs: - model: kernel.sysctl @@ -690,7 +710,9 @@ hostshell value: 0 operation: delete ``` + * kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: + ```yaml configs: - model: kernel.sysctl.persist @@ -713,8 +735,8 @@ hostshell * KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置： - - grub.cmdline.current：对当前分区的启动项参数进行配置。 - - grub.cmdline.next：对下一分区的启动项参数进行配置。 + * grub.cmdline.current：对当前分区的启动项参数进行配置。 + * grub.cmdline.next：对下一分区的启动项参数进行配置。 * 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。 diff --git "a/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" "b/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" index f2823c85..155bc96f 100644 --- "a/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" +++ "b/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" @@ -1,16 +1,16 @@ -# 容器OS镜像制作指导# +# 容器OS镜像制作指导 -## 简介 ## +## 简介 kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制作KubeOS 容器，虚拟机和物理机镜像 -## 命令介绍 ## +## 命令介绍 -### 命令格式 ### +### 命令格式 **bash kbimg.sh** \[ --help | -h \] create \[ COMMANDS \] \[ OPTIONS \] -### 参数说明 ### +### 参数说明 * COMMANDS @@ -20,8 +20,6 @@ kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制 | vm-image | 生成用于部署和升级的虚拟机镜像 | | pxe-image | 生成物理机安装所需的镜像及文件 | - - * OPTIONS | 参数 | 描述 | @@ -34,31 +32,33 @@ kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制 | -l | 如果指定参数，则镜像为legacy引导，不指定默认是UEFI引导 | | -h --help | 查看帮助信息 | +## 使用说明 - -## 使用说明 ## - -#### 注意事项 ### +### 注意事项 * kbimg.sh 执行需要 root 权限 * 当前仅支持 x86和 AArch64 架构使用 * 容器 OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 -### KubeOS OCI 镜像制作 ### +### KubeOS OCI 镜像制作 -#### 注意事项 #### +#### 注意事项 * 制作的 OCI 镜像仅用于后续的虚拟机/物理机镜像制作或升级使用，不支持启动容器 * 使用默认 rpmlist 进行容器OS镜像制作时所需磁盘空间至少为6G，如自已定义 rpmlist 可能会超过6G -#### 使用示例 #### +#### 使用示例 + * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件 + ```shell cd /opt/kubeOS/scripts touch resolv.conf vim resolv.conf ``` + * 制作KubeOS容器镜像 + ``` shell cd /opt/kubeOS/scripts bash kbimg.sh create upgrade-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' -d your_imageRepository/imageName:version @@ -70,25 +70,28 @@ bash kbimg.sh create upgrade-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1 docker images ``` -### KubeOS 虚拟机镜像制作 ### +### KubeOS 虚拟机镜像制作 -#### 注意事项 #### +#### 注意事项 * 如使用 docker 镜像制作请先拉取相应镜像或者先制作docker镜像，并保证 docker 镜像的安全性 * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机 * 容器 OS 目前不支持 x86 架构的虚拟机使用 legacy 启动模式启动 * 使用默认rpmlist进行容器OS镜像制作时所需磁盘空间至少为25G，如自已定义rpmlist可能会超过25G -#### 使用示例 #### +#### 使用示例 * 使用repo源制作 - * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件 + * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件 + ```shell cd /opt/kubeOS/scripts touch resolv.conf vim resolv.conf ``` - * KubeOS虚拟机镜像制作 + + * KubeOS虚拟机镜像制作 + ``` shell cd /opt/kubeOS/scripts bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' @@ -100,15 +103,15 @@ docker images cd /opt/kubeOS/scripts bash kbimg.sh create vm-image -d your_imageRepository/imageName:version ``` -* 结果说明 - 容器 OS 镜像制作完成后，会在 /opt/kubeOS/scripts 目录下生成： - * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。 - * update.img: 用于升级的根文件系统分区镜像 +* 结果说明 + 容器 OS 镜像制作完成后，会在 /opt/kubeOS/scripts 目录下生成： + * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。 + * update.img: 用于升级的根文件系统分区镜像 -### KubeOS 物理机安装所需镜像及文件制作 ### +### KubeOS 物理机安装所需镜像及文件制作 -#### 注意事项 #### +#### 注意事项 * 如使用 docker 镜像制作请先拉取相应镜像或者先制作 docker 镜像，并保证 docker 镜像的安全性 * 制作出来的容器 OS 物理安装所需的镜像目前只能用于 CPU 架构为 x86 和 AArch64 的物理机安装 @@ -116,7 +119,8 @@ docker images * 不支持多个磁盘都安装KubeOS，可能会造成启动失败或挂载紊乱 * 容器OS 目前不支持 x86 架构的物理机使用 legacy 启动模式启动 * 使用默认rpmlist进行镜像制作时所需磁盘空间至少为5G，如自已定义 rpmlist 可能会超过5G -#### 使用示例 #### + +#### 使用示例 * 首先需要修改```00bootup/Global.cfg```的配置，对相关参数进行配置，参数均为必填，ip目前仅支持ipv4，配置示例如下 @@ -138,25 +142,28 @@ docker images ``` * 使用 repo 源制作 - * 如需进行DNS配置，请在```scripts```目录下自定义```resolv.conf```文件 + * 如需进行DNS配置，请在```scripts```目录下自定义```resolv.conf```文件 + ```shell cd /opt/kubeOS/scripts touch resolv.conf vim resolv.conf ``` - * KubeOS物理机安装所需镜像制作 - ``` + + * KubeOS物理机安装所需镜像制作 + + ```shell cd /opt/kubeOS/scripts bash kbimg.sh create pxe-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' ``` * 使用 docker 镜像制作 + ``` shell cd /opt/kubeOS/scripts bash kbimg.sh create pxe-image -d your_imageRepository/imageName:version ``` * 结果说明 - - * initramfs.img: 用于pxe启动用的 initramfs 镜像 - * kubeos.tar: pxe安装所用的 OS + * initramfs.img: 用于pxe启动用的 initramfs 镜像 + * kubeos.tar: pxe安装所用的 OS -- Gitee From 4d8d07df49e44202d252344b25e8b1d366746ecc Mon Sep 17 00:00:00 2001 From: liyuanr Date: Mon, 25 Sep 2023 10:52:29 +0800 Subject: [PATCH 36/46] KubeOS:add nodeselector to OS The nodeselector field is added to the OS to filter the nodes to be upgraded or configured. During nodeselector configuration, only nodes with this label can be upgraded or configured. Signed-off-by: liyuanr --- api/v1alpha1/os_types.go | 2 + cmd/operator/controllers/operation.go | 181 ++++++ cmd/operator/controllers/os_controller.go | 228 +++----- .../controllers/os_controller_test.go | 550 ++++++++++++++---- cmd/proxy/controllers/os_controller.go | 7 + .../config/crd/upgrade.openeuler.org_os.yaml | 2 + docs/quick-start.md | 37 +- pkg/values/values.go | 6 +- 8 files changed, 736 insertions(+), 277 deletions(-) create mode 100644 cmd/operator/controllers/operation.go diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go index f9474b72..d3d636de 100644 --- a/api/v1alpha1/os_types.go +++ b/api/v1alpha1/os_types.go @@ -38,6 +38,8 @@ type OSSpec struct { SysConfigs SysConfigs `json:"sysconfigs"` // +kubebuilder:validation:Optional UpgradeConfigs SysConfigs `json:"upgradeconfigs"` + // +kubebuilder:validation:Optional + NodeSelector string `json:"nodeselector"` } // +kubebuilder:subresource:status diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go new file mode 100644 index 00000000..4b441d18 --- /dev/null +++ b/cmd/operator/controllers/operation.go @@ -0,0 +1,181 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package controllers contains the Reconcile of operator +package controllers + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + upgradev1 "openeuler.org/KubeOS/api/v1alpha1" + "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type operation interface { + newExistRequirement() (labels.Requirement, error) + newNotExistRequirement() (labels.Requirement, error) + updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) + updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error +} + +type upgradeOps struct{} + +func (u upgradeOps) newExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (u upgradeOps) newNotExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) { + var count int + for _, node := range nodes { + if count >= limit { + break + } + osVersionNode := node.Status.NodeInfo.OSImage + if os.Spec.OSVersion != osVersionNode { + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "failed to get osInstance "+node.Name, "skip this node") + return count, err + } + continue + } + if err := u.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + continue + } + count++ + } + } + return count, nil +} + +func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error { + if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { + if err := deepCopySpecConfigs(os, osInstance, values.UpgradeConfigName); err != nil { + return err + } + } + if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { + if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { + return err + } + // exchange "grub.cmdline.current" and "grub.cmdline.next" + for i, config := range osInstance.Spec.SysConfigs.Configs { + if config.Model == "grub.cmdline.current" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" + } + if config.Model == "grub.cmdline.next" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" + } + } + } + osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } + node.Labels[values.LabelUpgrading] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } + return nil +} + +type configOps struct{} + +func (c configOps) newExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.Exists, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelConfiguring) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (c configOps) newNotExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.DoesNotExist, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelConfiguring) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) { + var count int + for _, node := range nodes { + if count >= limit { + break + } + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "failed to get osInstance "+node.Name) + return count, err + } + continue + } + if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { + if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + continue + } + count++ + } + } + return count, nil +} + +func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error { + if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { + return err + } + osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } + node.Labels[values.LabelConfiguring] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } + return nil +} diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index e04d59b1..bd7a70dd 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -65,29 +66,24 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) } ops := os.Spec.OpsType + var opsInsatnce operation switch ops { case "upgrade", "rollback": - limit, err := checkUpgrading(ctx, r, min(os.Spec.MaxUnavailable, nodeNum)) // adjust maxUnavailable if need - if err != nil { - return values.RequeueNow, err - } - if needRequeue, err := assignUpgrade(ctx, r, os, limit, req.Namespace); err != nil { - return values.RequeueNow, err - } else if needRequeue { - return values.Requeue, nil - } + opsInsatnce = upgradeOps{} case "config": - limit, err := checkConfig(ctx, r, min(os.Spec.MaxUnavailable, nodeNum)) - if err != nil { - return values.RequeueNow, err - } - if needRequeue, err := assignConfig(ctx, r, os.Spec.SysConfigs, os.Spec.SysConfigs.Version, limit); err != nil { - return values.RequeueNow, err - } else if needRequeue { - return values.Requeue, nil - } + opsInsatnce = configOps{} default: log.Error(nil, "operation "+ops+" cannot be recognized") + return values.Requeue, nil + } + limit, err := calNodeLimit(ctx, r, opsInsatnce, min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need + if err != nil { + return values.RequeueNow, err + } + if needRequeue, err := assignOperation(ctx, r, os, limit, opsInsatnce); err != nil { + return values.RequeueNow, err + } else if needRequeue { + return values.Requeue, nil } return values.Requeue, nil } @@ -129,30 +125,44 @@ func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLim } } -func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (os upgradev1.OS, - nodeNum int, err error) { - if err = r.Get(ctx, name, &os); err != nil { +func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, + int, error) { + var os upgradev1.OS + if err := r.Get(ctx, name, &os); err != nil { log.Error(err, "unable to fetch OS") - return + return upgradev1.OS{}, 0, err } requirement, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) if err != nil { log.Error(err, "unable to create requirement "+values.LabelMaster) - return + return upgradev1.OS{}, 0, err } - nodesItems, err := getNodes(ctx, r, 0, *requirement) + var requirements []labels.Requirement + requirements = append(requirements, *requirement) + if os.Spec.NodeSelector != "" { + reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Exists, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return upgradev1.OS{}, 0, err + } + requirements = append(requirements, *requirement, *reqSelector) + } + nodesItems, err := getNodes(ctx, r, 0, requirements...) if err != nil { log.Error(err, "get slave nodes fail") - return + return upgradev1.OS{}, 0, err } - nodeNum = len(nodesItems) - return + nodeNum := len(nodesItems) + return os, nodeNum, nil } -func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, - nameSpace string) (bool, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) +func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, + ops operation) (bool, error) { + fmt.Println("start assignOperation") + fmt.Println("ops is ", reflect.TypeOf(ops)) + requirement, err := ops.newNotExistRequirement() + fmt.Println("requirement is ", requirement.String()) if err != nil { log.Error(err, "unable to create requirement "+values.LabelUpgrading) return false, err @@ -162,14 +172,25 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. log.Error(err, "unable to create requirement "+values.LabelMaster) return false, err } + var requirements []labels.Requirement + requirements = append(requirements, requirement, *reqMaster) + if os.Spec.NodeSelector != "" { + reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{os.Spec.NodeSelector}) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return false, err + } + fmt.Println("requirement is ", reqSelector.String()) + requirements = append(requirements, *reqSelector) + } - nodes, err := getNodes(ctx, r, limit+1, *requirement, *reqMaster) // one more to see if all nodes updated + nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated if err != nil { return false, err } - + fmt.Println("nodes has not upgrade/config and has selector ", len(nodes)) // Upgrade OS for selected nodes - count, err := upgradeNodes(ctx, r, &os, nodes, limit) + count, err := ops.updateNodes(ctx, r, &os, nodes, limit) if err != nil { return false, err } @@ -177,90 +198,6 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. return count >= limit, nil } -func upgradeNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - nodes []corev1.Node, limit int) (int, error) { - var count int - for _, node := range nodes { - if count >= limit { - break - } - osVersionNode := node.Status.NodeInfo.OSImage - if os.Spec.OSVersion != osVersionNode { - var osInstance upgradev1.OSInstance - if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { - if err = client.IgnoreNotFound(err); err != nil { - log.Error(err, "failed to get osInstance "+node.Name) - return count, err - } - continue - } - if err := updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { - continue - } - count++ - } - } - return count, nil -} - -func updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - node *corev1.Node, osInstance *upgradev1.OSInstance) error { - if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { - if err := deepCopySpecConfigs(os, osInstance, values.UpgradeConfigName); err != nil { - return err - } - } - if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { - if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { - return err - } - // exchange "grub.cmdline.current" and "grub.cmdline.next" - for i, config := range osInstance.Spec.SysConfigs.Configs { - if config.Model == "grub.cmdline.current" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" - } - if config.Model == "grub.cmdline.next" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" - } - } - } - osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() - if err := r.Update(ctx, osInstance); err != nil { - log.Error(err, "unable to update", "osInstance", osInstance.Name) - return err - } - node.Labels[values.LabelUpgrading] = "" - if err := r.Update(ctx, node); err != nil { - log.Error(err, "unable to label", "node", node.Name) - return err - } - return nil -} - -func assignConfig(ctx context.Context, r common.ReadStatusWriter, sysConfigs upgradev1.SysConfigs, - configVersion string, limit int) (bool, error) { - osInstances, err := getIdleOSInstances(ctx, r, limit+1) // one more to see if all node updated - if err != nil { - return false, err - } - var count = 0 - for _, osInstance := range osInstances { - if count >= limit { - break - } - configVersionNode := osInstance.Spec.SysConfigs.Version - if configVersion != configVersionNode { - count++ - osInstance.Spec.SysConfigs = sysConfigs - osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() - if err = r.Update(ctx, &osInstance); err != nil { - log.Error(err, "unable update osInstance ", "osInstanceName ", osInstance.Name) - } - } - } - return count >= limit, nil -} - func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, reqs ...labels.Requirement) ([]corev1.Node, error) { var nodeList corev1.NodeList @@ -272,48 +209,37 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, return nodeList.Items, nil } -func getIdleOSInstances(ctx context.Context, r common.ReadStatusWriter, limit int) ([]upgradev1.OSInstance, error) { - var osInstanceList upgradev1.OSInstanceList - opt := []client.ListOption{ - client.MatchingFields{values.OsiStatusName: values.NodeStatusIdle.String()}, - &client.ListOptions{Limit: int64(limit)}, - } - if err := r.List(ctx, &osInstanceList, opt...); err != nil { - log.Error(err, "unable to list nodes with requirements") - return nil, err - } - return osInstanceList.Items, nil -} - -func getConfigOSInstances(ctx context.Context, r common.ReadStatusWriter) ([]upgradev1.OSInstance, error) { - var osInstanceList upgradev1.OSInstanceList - if err := r.List(ctx, &osInstanceList, - client.MatchingFields{values.OsiStatusName: values.NodeStatusConfig.String()}); err != nil { - log.Error(err, "unable to list nodes with requirements") - return nil, err - } - return osInstanceList.Items, nil -} - -func checkUpgrading(ctx context.Context, r common.ReadStatusWriter, maxUnavailable int) (int, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) +func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, + ops operation, maxUnavailable int, nodeSelector string) (int, error) { + fmt.Println("start calNodeLimit") + fmt.Println("ops is ", reflect.TypeOf(ops)) + requirement, err := ops.newExistRequirement() if err != nil { log.Error(err, "unable to create requirement "+values.LabelUpgrading) return 0, err } - nodes, err := getNodes(ctx, r, 0, *requirement) - if err != nil { - return 0, err + fmt.Println("requirement is ", requirement.String()) + var requirements []labels.Requirement + requirements = append(requirements, requirement) + if nodeSelector != "" { + reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{nodeSelector}) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return 0, err + } + fmt.Println("requirement is ", reqSelector.String()) + requirements = append(requirements, *reqSelector) } - return maxUnavailable - len(nodes), nil -} - -func checkConfig(ctx context.Context, r common.ReadStatusWriter, maxUnavailable int) (int, error) { - osInstances, err := getConfigOSInstances(ctx, r) + nodes, err := getNodes(ctx, r, 0, requirements...) if err != nil { return 0, err + } - return maxUnavailable - len(osInstances), nil + fmt.Println("nodes has upgrade and selector ", len(nodes)) + for _, n := range nodes { + fmt.Println(" nodes name is ", n.Name) + } + return maxUnavailable - len(nodes), nil } func min(a, b int) int { diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index 8c5d1981..38bf4e61 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -455,6 +455,14 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(configedOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) Expect(configedOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + existingNode1 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode1) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode1.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(true)) configedOSIns2 := &upgradev1.OSInstance{} Eventually(func() bool { @@ -463,6 +471,14 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(configedOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) Expect(configedOSIns2.Spec.SysConfigs.Version).Should(Equal("v2")) + existingNode2 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok = existingNode2.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(true)) }) }) @@ -749,6 +765,419 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) }) }) + + Context("When we want to upgrade node with nodes having NodeSelector label", func() { + It("Should only update node with NodeSelector label", func() { + ctx := context.Background() + // create Node1 + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + "upgrade.openeuler.org/node-selector": "openeuler", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance1 + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "upgrade", + MaxUnavailable: 3, + OSVersion: "KubeOS v2", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + NodeSelector: "openeuler", + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }, + }, + }, + UpgradeConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + {Model: "kernel.sysctl.persist", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }}, + }, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + existingNode1 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode1) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode1.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(true)) + + upgradeOSIns1 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, upgradeOSIns1) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + Expect(upgradeOSIns1.Spec.UpgradeConfigs.Version).Should(Equal("v2")) + Expect(upgradeOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + + existingNode2 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok = existingNode2.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(false)) + + upgradeOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, upgradeOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + Expect(upgradeOSIns2.Spec.UpgradeConfigs.Version).Should(Equal("v1")) + Expect(upgradeOSIns2.Spec.SysConfigs.Version).Should(Equal("v1")) + }) + }) + + Context("When we want to config node with nodes having NodeSelector label", func() { + It("Should only config node with NodeSelector label", func() { + ctx := context.Background() + // create Node1 + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + "upgrade.openeuler.org/node-selector": "openeuler", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance1 + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "config", + MaxUnavailable: 3, + OSVersion: "KubeOS v1", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + NodeSelector: "openeuler", + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.SysConfigs.Version).Should(Equal("v2")) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + existingNode1 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode1) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode1.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(true)) + + upgradeOSIns1 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, upgradeOSIns1) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) + Expect(upgradeOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + + existingNode2 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok = existingNode2.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(false)) + + upgradeOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, upgradeOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + Expect(upgradeOSIns2.Spec.SysConfigs.Version).Should(Equal("v1")) + }) + }) }) func Test_deepCopySpecConfigs(t *testing.T) { @@ -780,127 +1209,6 @@ func Test_deepCopySpecConfigs(t *testing.T) { } } -func Test_getConfigOSInstances(t *testing.T) { - type args struct { - ctx context.Context - r common.ReadStatusWriter - } - tests := []struct { - name string - args args - want []upgradev1.OSInstance - wantErr bool - }{ - { - name: "list error", - args: args{ - ctx: context.Background(), - r: &OSReconciler{}, - }, - want: nil, - wantErr: true, - }, - } - patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ - {Values: gomonkey.Params{fmt.Errorf("list error")}}, - }) - defer patchList.Reset() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := getConfigOSInstances(tt.args.ctx, tt.args.r) - if (err != nil) != tt.wantErr { - t.Errorf("getConfigOSInstances() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getConfigOSInstances() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_checkUpgrading(t *testing.T) { - type args struct { - ctx context.Context - r common.ReadStatusWriter - maxUnavailable int - } - tests := []struct { - name string - args args - want int - wantErr bool - }{ - { - name: "label error", - args: args{ - ctx: context.Background(), - r: &OSReconciler{}, - }, - want: 0, - wantErr: true, - }, - } - patchNewRequirement := gomonkey.ApplyFuncSeq(labels.NewRequirement, []gomonkey.OutputCell{ - {Values: gomonkey.Params{nil, fmt.Errorf("label error")}}, - {Values: gomonkey.Params{nil, nil}}, - }) - defer patchNewRequirement.Reset() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := checkUpgrading(tt.args.ctx, tt.args.r, tt.args.maxUnavailable) - if (err != nil) != tt.wantErr { - t.Errorf("checkUpgrading() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("checkUpgrading() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_getIdleOSInstances(t *testing.T) { - type args struct { - ctx context.Context - r common.ReadStatusWriter - limit int - } - tests := []struct { - name string - args args - want []upgradev1.OSInstance - wantErr bool - }{ - { - name: "list error", - args: args{ - ctx: context.Background(), - r: &OSReconciler{}, - limit: 1, - }, - want: nil, - wantErr: true, - }, - } - patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ - {Values: gomonkey.Params{fmt.Errorf("list error")}}, - }) - defer patchList.Reset() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := getIdleOSInstances(tt.args.ctx, tt.args.r, tt.args.limit) - if (err != nil) != tt.wantErr { - t.Errorf("getIdleOSInstances() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getIdleOSInstances() = %v, want %v", got, tt.want) - } - }) - } -} - func Test_getNodes(t *testing.T) { type args struct { ctx context.Context diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index 0630a98c..b543befc 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -244,6 +244,13 @@ func (r *OSReconciler) refreshNode(ctx context.Context, node *corev1.Node, osIns return err } } + if _, ok := node.Labels[values.LabelConfiguring]; ok { + delete(node.Labels, values.LabelConfiguring) + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to delete label", "node", node.Name) + return err + } + } if node.Spec.Unschedulable { // update done, uncordon the node drainer := &drain.Helper{ Ctx: ctx, diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index 2dd822e9..27ff3273 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -67,6 +67,8 @@ spec: type: integer mtls: type: boolean + nodeselector: + type: string opstype: type: string osversion: diff --git a/docs/quick-start.md b/docs/quick-start.md index 372c904b..af37c224 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -251,7 +251,7 @@ | evictpodforce | bool | 升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | | sysconfigs | / | 配置设置 | 1. “opstype=config”时只进行配置。 2.“opstype=upgrade/rollback”时，代表升级/回退后配置，即在升级/回退重启后进行配置。```配置（Settings）指导``` | “opstype=config”时必选 | | upgradeconfigs | / | 升级前配置设置 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | - + | nodeselector | string | 需要进行升级/配置/回滚操作的节点label | 用于只对具有某些特定label的节点而不是集群所有worker节点进行运维的场景，需要进行运维操作的节点需要包含key为upgrade.openeuler.org/node-selector的label，nodeselector为该label的value值，此参数不配置时，或者配置为""时默认对所有节点进行操作| 可选 | #### 升级指导 * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如下，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml; @@ -320,7 +320,7 @@ mtls: true ``` - * 升级并且进行配置的示例如下， + * 升级并且进行配置的示例如下 * 以节点容器引擎为containerd为例，升级方式对配置无影响，upgradeconfigs在升级前起效，sysconfigs在升级后起效，配置参数说明请见```配置(Settings)指导``` * 升级并且配置时opstype字段需为upgrade * upgradeconfig为升级之前执行的配置，sysconfigs为升级机器重启后执行的配置，用户可按需进行配置 @@ -365,7 +365,37 @@ - key: kernel param key4 value: kernel param value4 ``` + * 只升级部分节点示例如下 + * 以节点容器引擎为containerd为例，升级方式对节点筛选无影响 + * 需要进行升级的节点需包含key为upgrade.openeuler.org/node-selector的label，nodeselector的值为该label的value，即假定nodeselector值为kubeos，则只对包含upgrade.openeuler.org/node-selector=kubeos的label的worker节点进行升级 + * nodeselector对配置和回滚同样有效 + * 节点添加label和label修改命令示例如下： + ``` shell + # 为节点kubeos-node1增加label + kubectl label nodes kubeos-node1 upgrade.openeuler.org/node-selector=kubeos-v1 + # 修改节点kubeos-node1的label + kubectl label --overwrite nodes kubeos-node2 upgrade.openeuler.org/node-selector=kubeos-v2 + ``` + * yaml示例如下： + ```yaml + apiVersion: upgrade.openeuler.org/v1alpha1 + kind: OS + metadata: + name: os-sample + spec: + imagetype: containerd + opstype: upgrade + osversion: edit.os.version + maxunavailable: edit.node.upgrade.number + containerimage: container image like repository/name:tag + evictpodforce: true/false + imageurl: "" + checksum: container image digests + flagSafe: false + mtls: true + nodeselector: edit.node.label.key + ``` * 查看未升级的节点的 OS 版本 ```shell @@ -711,8 +741,7 @@ hostshell operation: delete ``` -* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: - +* kernel.sysctl.persist: 设置持久化内核参数，key/value表示内核参数的key/value，key与value均不能为空且key不能包含“=”， configpath为配置文件路径，支持新建（需保证父目录存在），如不指定configpath默认修改/etc/sysctl.conf，示例如下： ```yaml configs: - model: kernel.sysctl.persist diff --git a/pkg/values/values.go b/pkg/values/values.go index f488ae52..e2c03540 100644 --- a/pkg/values/values.go +++ b/pkg/values/values.go @@ -26,7 +26,11 @@ const ( LabelMaster = "node-role.kubernetes.io/control-plane" // LabelOSinstance is used to select the osinstance with the nodeName by label LabelOSinstance = "upgrade.openeuler.org/osinstance-node" - defaultPeriod = 15 * time.Second + // LabelNodeSelector is used to filter the nodes that need to be upgraded or configured. + LabelNodeSelector = "upgrade.openeuler.org/node-selector" + // LabelConfiguring is the key of the configuring label for nodes + LabelConfiguring = "upgrade.openeuler.org/configuring" + defaultPeriod = 15 * time.Second // OsiStatusName is param name of nodeStatus in osInstance OsiStatusName = "nodestatus" // UpgradeConfigName is param name of UpgradeConfig -- Gitee From 5dda90bb33e8f914bdc32f1f910786a969cffbae Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Fri, 24 Nov 2023 10:54:21 +0800 Subject: [PATCH 37/46] fix(agent, proxy): transform log timestamp to human-readable format Originally, the log of controllers is timestamp which is hard to read. Now, transform the log into more human-readable format. Signed-off-by: Yuhang Wei --- cmd/operator/controllers/operation.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go index 4b441d18..106892c4 100644 --- a/cmd/operator/controllers/operation.go +++ b/cmd/operator/controllers/operation.go @@ -20,10 +20,11 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" - upgradev1 "openeuler.org/KubeOS/api/v1alpha1" "openeuler.org/KubeOS/pkg/common" "openeuler.org/KubeOS/pkg/values" "sigs.k8s.io/controller-runtime/pkg/client" + + upgradev1 "openeuler.org/KubeOS/api/v1alpha1" ) type operation interface { -- Gitee From a9c9847ca824c24acdd1f93d19374f45a38f28d1 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Fri, 24 Nov 2023 10:52:01 +0800 Subject: [PATCH 38/46] operator: delete unnecessary fmt and add printing for upgrades and configurations Delete unnecessary fmts, update osinstance during upgrades and configurations, and add printing of node labels Signed-off-by: liyuanr --- cmd/operator/controllers/operation.go | 6 ++++++ cmd/operator/controllers/os_controller.go | 14 -------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go index 106892c4..9c9c6426 100644 --- a/cmd/operator/controllers/operation.go +++ b/cmd/operator/controllers/operation.go @@ -65,6 +65,7 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, } osVersionNode := node.Status.NodeInfo.OSImage if os.Spec.OSVersion != osVersionNode { + log.Info("Upgrading node " + node.Name) var osInstance upgradev1.OSInstance if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { if err = client.IgnoreNotFound(err); err != nil { @@ -109,11 +110,13 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW log.Error(err, "unable to update", "osInstance", osInstance.Name) return err } + log.Info("Update osinstance spec successfully") node.Labels[values.LabelUpgrading] = "" if err := r.Update(ctx, node); err != nil { log.Error(err, "unable to label", "node", node.Name) return err } + log.Info("Add node upgrading label " + values.LabelUpgrading + " successfully") return nil } @@ -153,6 +156,7 @@ func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o continue } if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { + log.Info("Configuring node " + node.Name) if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { log.Error(err, "failed to update node and osinstance ,skip this node ") continue @@ -173,10 +177,12 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr log.Error(err, "unable to update", "osInstance", osInstance.Name) return err } + log.Info("Update osinstance spec successfully") node.Labels[values.LabelConfiguring] = "" if err := r.Update(ctx, node); err != nil { log.Error(err, "unable to label", "node", node.Name) return err } + log.Info("Add node configuring label " + values.LabelConfiguring + " successfully") return nil } diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index bd7a70dd..2c4ee1f7 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -17,7 +17,6 @@ import ( "context" "encoding/json" "fmt" - "reflect" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -159,10 +158,7 @@ func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.N func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, ops operation) (bool, error) { - fmt.Println("start assignOperation") - fmt.Println("ops is ", reflect.TypeOf(ops)) requirement, err := ops.newNotExistRequirement() - fmt.Println("requirement is ", requirement.String()) if err != nil { log.Error(err, "unable to create requirement "+values.LabelUpgrading) return false, err @@ -180,7 +176,6 @@ func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev log.Error(err, "unable to create requirement "+values.LabelNodeSelector) return false, err } - fmt.Println("requirement is ", reqSelector.String()) requirements = append(requirements, *reqSelector) } @@ -188,7 +183,6 @@ func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev if err != nil { return false, err } - fmt.Println("nodes has not upgrade/config and has selector ", len(nodes)) // Upgrade OS for selected nodes count, err := ops.updateNodes(ctx, r, &os, nodes, limit) if err != nil { @@ -211,14 +205,11 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, ops operation, maxUnavailable int, nodeSelector string) (int, error) { - fmt.Println("start calNodeLimit") - fmt.Println("ops is ", reflect.TypeOf(ops)) requirement, err := ops.newExistRequirement() if err != nil { log.Error(err, "unable to create requirement "+values.LabelUpgrading) return 0, err } - fmt.Println("requirement is ", requirement.String()) var requirements []labels.Requirement requirements = append(requirements, requirement) if nodeSelector != "" { @@ -227,7 +218,6 @@ func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, log.Error(err, "unable to create requirement "+values.LabelNodeSelector) return 0, err } - fmt.Println("requirement is ", reqSelector.String()) requirements = append(requirements, *reqSelector) } nodes, err := getNodes(ctx, r, 0, requirements...) @@ -235,10 +225,6 @@ func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, return 0, err } - fmt.Println("nodes has upgrade and selector ", len(nodes)) - for _, n := range nodes { - fmt.Println(" nodes name is ", n.Name) - } return maxUnavailable - len(nodes), nil } -- Gitee From 9f3e6900d23918907298f8b8646ef6faf8f22e3c Mon Sep 17 00:00:00 2001 From: liyuanr Date: Wed, 14 Aug 2024 21:01:42 +0800 Subject: [PATCH 39/46] feat(os-operator): support setting TimeWindow and TimeInterval The timewindow and timeInterval parameters are added to the OS CRD. The timewindow and timeinterval can be set during upgrade or configuration. Signed-off-by: liyuanr --- api/v1alpha1/os_types.go | 10 ++ cmd/operator/controllers/operation.go | 45 +---- cmd/operator/controllers/os_controller.go | 210 ++++++++++++++-------- cmd/operator/controllers/requirements.go | 85 +++++++++ cmd/operator/controllers/times.go | 100 +++++++++++ 5 files changed, 341 insertions(+), 109 deletions(-) create mode 100644 cmd/operator/controllers/requirements.go create mode 100644 cmd/operator/controllers/times.go diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go index d3d636de..65bb8e2e 100644 --- a/api/v1alpha1/os_types.go +++ b/api/v1alpha1/os_types.go @@ -39,7 +39,12 @@ type OSSpec struct { // +kubebuilder:validation:Optional UpgradeConfigs SysConfigs `json:"upgradeconfigs"` // +kubebuilder:validation:Optional + // +kubebuilder:default:=no-label NodeSelector string `json:"nodeselector"` + // +kubebuilder:validation:Optional + TimeWindow TimeWindow `json:"timewindow"` + // +kubebuilder:validation:Optional + TimeInterval int `json:"timeinterval"` } // +kubebuilder:subresource:status @@ -90,6 +95,11 @@ type Content struct { Operation string `json:"operation"` } +type TimeWindow struct { + StartTime string `json:"starttime"` + EndTime string `json:"endtime"` +} + // +kubebuilder:subresource:status // +kubebuilder:object:root=true diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go index 9c9c6426..2ffcaa2e 100644 --- a/cmd/operator/controllers/operation.go +++ b/cmd/operator/controllers/operation.go @@ -17,8 +17,6 @@ import ( "context" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" "openeuler.org/KubeOS/pkg/common" "openeuler.org/KubeOS/pkg/values" @@ -28,32 +26,19 @@ import ( ) type operation interface { - newExistRequirement() (labels.Requirement, error) - newNotExistRequirement() (labels.Requirement, error) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, nodes []corev1.Node, limit int) (int, error) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, node *corev1.Node, osInstance *upgradev1.OSInstance) error + getOpsLabel() opsLabel } -type upgradeOps struct{} - -func (u upgradeOps) newExistRequirement() (labels.Requirement, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelUpgrading) - return labels.Requirement{}, err - } - return *requirement, nil +type upgradeOps struct { + label opsLabel } -func (u upgradeOps) newNotExistRequirement() (labels.Requirement, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelUpgrading) - return labels.Requirement{}, err - } - return *requirement, nil +func (u upgradeOps) getOpsLabel() opsLabel { + return u.label } func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, @@ -120,24 +105,12 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW return nil } -type configOps struct{} - -func (c configOps) newExistRequirement() (labels.Requirement, error) { - requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.Exists, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelConfiguring) - return labels.Requirement{}, err - } - return *requirement, nil +type configOps struct { + label opsLabel } -func (c configOps) newNotExistRequirement() (labels.Requirement, error) { - requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.DoesNotExist, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelConfiguring) - return labels.Requirement{}, err - } - return *requirement, nil +func (c configOps) getOpsLabel() opsLabel { + return c.label } func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index 2c4ee1f7..690b6458 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -56,7 +57,7 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re // Reconcile compares the actual state with the desired and updates the status of the resources e.g. nodes func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) (ctrl.Result, error) { - os, nodeNum, err := getAndUpdateOS(ctx, r, req.NamespacedName) + os, err := getOSCr(ctx, r, req.NamespacedName) if err != nil { if errors.IsNotFound(err) { return values.NoRequeue, nil @@ -64,18 +65,31 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) return values.RequeueNow, err } + isWithinTimeWindow, err := isWithinTimeWindow(os.Spec.TimeWindow.StartTime, os.Spec.TimeWindow.EndTime) + if err != nil { + return values.RequeueNow, err + } + if !isWithinTimeWindow { + //Todo consider time interval + return values.RequeueNow, nil + } + ops := os.Spec.OpsType var opsInsatnce operation switch ops { case "upgrade", "rollback": - opsInsatnce = upgradeOps{} + opsInsatnce = upgradeOps{label: opsLabel{label: values.LabelUpgrading, op: selection.DoesNotExist}} case "config": - opsInsatnce = configOps{} + opsInsatnce = configOps{label: opsLabel{label: values.LabelConfiguring, op: selection.DoesNotExist}} default: log.Error(nil, "operation "+ops+" cannot be recognized") return values.Requeue, nil } - limit, err := calNodeLimit(ctx, r, opsInsatnce, min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need + nodeNum, err := getNodeNum(ctx, r, os.Spec.NodeSelector) + if err != nil { + return values.RequeueNow, err + } + limit, err := calNodeLimit(ctx, r, opsInsatnce.getOpsLabel(), min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need if err != nil { return values.RequeueNow, err } @@ -84,7 +98,7 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) } else if needRequeue { return values.Requeue, nil } - return values.Requeue, nil + return setTimeInterval(os.Spec.TimeInterval), nil } // SetupWithManager sets up the controller with the Manager. @@ -100,6 +114,28 @@ func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { }); err != nil { return err } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.name", + func(rawObj client.Object) []string { + os, ok := rawObj.(*upgradev1.OS) + if !ok { + log.Error(nil, "failed to convert to osInstance") + return []string{} + } + return []string{os.Name} + }); err != nil { + return err + } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.namespace", + func(rawObj client.Object) []string { + os, ok := rawObj.(*upgradev1.OS) + if !ok { + log.Error(nil, "failed to convert to osInstance") + return []string{} + } + return []string{os.Namespace} + }); err != nil { + return err + } return ctrl.NewControllerManagedBy(mgr). For(&upgradev1.OS{}). Watches(&source.Kind{Type: &corev1.Node{}}, handler.Funcs{DeleteFunc: r.DeleteOSInstance}). @@ -124,72 +160,33 @@ func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLim } } -func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, - int, error) { +func getOSCr(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, error) { var os upgradev1.OS if err := r.Get(ctx, name, &os); err != nil { log.Error(err, "unable to fetch OS") - return upgradev1.OS{}, 0, err + return upgradev1.OS{}, err + } + if err := checkNodeSelector(ctx, r, os); err != nil { + log.Error(err, "nodeselector conficts") + return upgradev1.OS{}, err } + return os, nil +} - requirement, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) +// Get nodes which do not have master label，have nodeselector label or do not have other os cr nodeselector +func getNodeNum(ctx context.Context, r common.ReadStatusWriter, nodeSelector string) (int, error) { + labelList := []labelRequirementCreator{masterLabel{op: selection.DoesNotExist}, nodeSelectorLabel{value: nodeSelector, op: selection.Equals}} + requirements, err := createRequirement(labelList) if err != nil { - log.Error(err, "unable to create requirement "+values.LabelMaster) - return upgradev1.OS{}, 0, err - } - var requirements []labels.Requirement - requirements = append(requirements, *requirement) - if os.Spec.NodeSelector != "" { - reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Exists, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelNodeSelector) - return upgradev1.OS{}, 0, err - } - requirements = append(requirements, *requirement, *reqSelector) + return 0, err } nodesItems, err := getNodes(ctx, r, 0, requirements...) if err != nil { log.Error(err, "get slave nodes fail") - return upgradev1.OS{}, 0, err + return 0, err } nodeNum := len(nodesItems) - return os, nodeNum, nil -} - -func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, - ops operation) (bool, error) { - requirement, err := ops.newNotExistRequirement() - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelUpgrading) - return false, err - } - reqMaster, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelMaster) - return false, err - } - var requirements []labels.Requirement - requirements = append(requirements, requirement, *reqMaster) - if os.Spec.NodeSelector != "" { - reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{os.Spec.NodeSelector}) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelNodeSelector) - return false, err - } - requirements = append(requirements, *reqSelector) - } - - nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated - if err != nil { - return false, err - } - // Upgrade OS for selected nodes - count, err := ops.updateNodes(ctx, r, &os, nodes, limit) - if err != nil { - return false, err - } - - return count >= limit, nil + return nodeNum, nil } func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, @@ -203,31 +200,47 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, return nodeList.Items, nil } +// get now in upgrading and match with nodeselector func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, - ops operation, maxUnavailable int, nodeSelector string) (int, error) { - requirement, err := ops.newExistRequirement() + label opsLabel, maxUnavailable int, nodeSelector string) (int, error) { + label.op = selection.Exists + labelList := []labelRequirementCreator{ + masterLabel{op: selection.DoesNotExist}, + label, + nodeSelectorLabel{value: nodeSelector, op: selection.Equals}} + requirements, err := createRequirement(labelList) if err != nil { - log.Error(err, "unable to create requirement "+values.LabelUpgrading) return 0, err } - var requirements []labels.Requirement - requirements = append(requirements, requirement) - if nodeSelector != "" { - reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{nodeSelector}) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelNodeSelector) - return 0, err - } - requirements = append(requirements, *reqSelector) - } nodes, err := getNodes(ctx, r, 0, requirements...) if err != nil { return 0, err - } return maxUnavailable - len(nodes), nil } +func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, + opsInstance operation) (bool, error) { + opsLabel := opsInstance.getOpsLabel() + opsLabel.op = selection.DoesNotExist + labelList := []labelRequirementCreator{ + masterLabel{op: selection.DoesNotExist}, + opsLabel, + nodeSelectorLabel{value: os.Spec.NodeSelector, op: selection.Equals}} + requirements, err := createRequirement(labelList) + nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated + if err != nil { + return false, err + } + // Upgrade OS for selected nodes + count, err := opsInstance.updateNodes(ctx, r, &os, nodes, limit) + if err != nil { + return false, err + } + + return count >= limit, nil +} + func min(a, b int) int { if a < b { return a @@ -259,3 +272,54 @@ func deepCopySpecConfigs(os *upgradev1.OS, osinstance *upgradev1.OSInstance, con } return nil } + +// Check whether the nodeselector conflicts with other nodeselector in OS CRs. If the nodeselector is empty, return the list of other nodeselectors. +func checkNodeSelector(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS) error { + var osList upgradev1.OSList + if err := r.List(ctx, &osList, &client.ListOptions{}); err != nil { + log.Error(err, "unable to list nodes with requirements") + return err + } + var sameNodeSelectorList []types.NamespacedName + for _, osItem := range osList.Items { + // Exclude current os, controller-runtime not supports multiple indexs as listoptions in current version, + // so cannot list os without current os use List function + if osItem.Name == os.Name && osItem.Namespace == os.Namespace { + continue + } + if os.Spec.NodeSelector == osItem.Spec.NodeSelector { + sameNodeSelectorList = append(sameNodeSelectorList, types.NamespacedName{ + Namespace: osItem.Namespace, + Name: osItem.Name, + }) + } + } + // If a node label corresponds to multiple OS CRs, upgrade or configuration information may conflict. + // As a result, an error is reported and returned when there are one-to-many relationships. + if len(sameNodeSelectorList) > 0 { + errorMessage := sameNodeSelectorList[0].String() + for i := 1; i < len(sameNodeSelectorList); i++ { + errorMessage = errorMessage + " , " + sameNodeSelectorList[i].String() + } + log.Error(nil, "OS CR "+os.Name+" in namespace "+os.Namespace+" has same nodeselector with "+errorMessage) + return fmt.Errorf("OS CR %s in namespace %s has same nodeselector with %s", os.Name, os.Namespace, errorMessage) + } + return nil +} + +func setTimeInterval(timeInterval int) ctrl.Result { + return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(timeInterval) * time.Second} +} + +func createRequirement(labelsList []labelRequirementCreator) ([]labels.Requirement, error) { + var requirements []labels.Requirement + for _, label := range labelsList { + requirement, err := label.createLabelRequirement() + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return []labels.Requirement{}, err + } + requirements = append(requirements, requirement...) + } + return requirements, nil +} diff --git a/cmd/operator/controllers/requirements.go b/cmd/operator/controllers/requirements.go new file mode 100644 index 00000000..5a8f255f --- /dev/null +++ b/cmd/operator/controllers/requirements.go @@ -0,0 +1,85 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package controllers contains the Reconcile of operator +package controllers + +import ( + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "openeuler.org/KubeOS/pkg/values" +) + +const ( + AllNodeSelector = "all-label" + NoNodeSelector = "no-label" +) + +type labelRequirementCreator interface { + createLabelRequirement() ([]labels.Requirement, error) +} + +type masterLabel struct { + op selection.Operator +} + +func (ml masterLabel) createLabelRequirement() ([]labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelMaster, ml.op, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelMaster) + return []labels.Requirement{}, err + } + return []labels.Requirement{*requirement}, nil +} + +type opsLabel struct { + label string + op selection.Operator +} + +func (ol opsLabel) createLabelRequirement() ([]labels.Requirement, error) { + requirement, err := labels.NewRequirement(ol.label, ol.op, nil) + if err != nil { + log.Error(err, "unable to create requirement "+ol.label) + return []labels.Requirement{}, err + } + return []labels.Requirement{*requirement}, nil +} + +type nodeSelectorLabel struct { + value string + op selection.Operator +} + +func (nl nodeSelectorLabel) createLabelRequirement() ([]labels.Requirement, error) { + if nl.value == AllNodeSelector { + return []labels.Requirement{}, nil + } + var requirements []labels.Requirement + // if nodeselector is "", will get the nodes which label value is "",and not have label + if nl.value == NoNodeSelector { + requirement, err := labels.NewRequirement(values.LabelNodeSelector, selection.DoesNotExist, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return []labels.Requirement{}, err + } + requirements = append(requirements, *requirement) + return requirements, nil + } + requirement, err := labels.NewRequirement(values.LabelNodeSelector, nl.op, []string{nl.value}) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return []labels.Requirement{}, err + } + requirements = append(requirements, *requirement) + return requirements, nil +} diff --git a/cmd/operator/controllers/times.go b/cmd/operator/controllers/times.go new file mode 100644 index 00000000..eab3a224 --- /dev/null +++ b/cmd/operator/controllers/times.go @@ -0,0 +1,100 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package controllers contains the Reconcile of operator + +package controllers + +import ( + "fmt" + "regexp" + "time" +) + +const ( + DATE_TIME = "2006-01-02 15:04:05" + TIME_ONLY = "15:04:05" + ExecutionModeSerial = "serial" + ExecutionModeParallel = "parallel" +) + +func isWithinTimeWindow(start, end string) (bool, error) { + if start == "" && end == "" { + return true, nil + } + if start == "" || end == "" { + return false, fmt.Errorf("The start time and end time must be both empty or not empty.") + } + layoutStart, err := checkTimeValid(start) + if err != nil { + return false, err + } + layoutEnd, err := checkTimeValid(end) + if err != nil { + return false, err + } + if layoutStart != layoutEnd { + return false, fmt.Errorf("Start Time needs same time format with End Time") + } + now := time.Now() + if layoutStart == TIME_ONLY { + timeFormat := now.Format(TIME_ONLY) + now, err = time.ParseInLocation(layoutStart, timeFormat, now.Location()) + } + startTime, err := time.ParseInLocation(layoutStart, start, now.Location()) + if err != nil { + return false, err + } + endTime, err := time.ParseInLocation(layoutStart, end, now.Location()) + if err != nil { + return false, err + } + if endTime.Before(startTime) { + endTime = endTime.Add(24 * time.Hour) + fmt.Printf("endtime time add 24 hour is %s\n", endTime.Format(layoutStart)) + if now.Before(startTime) { + now = now.Add(24 * time.Hour) + fmt.Printf("now time add 24 hour is %s\n", now.Format(layoutStart)) + } + + } + return now.After(startTime) && now.Before(endTime), nil +} + +func checkTimeValid(checkTime string) (string, error) { + reDateTime, err := regexp.Compile("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$") + if err != nil { + return "", err + } + reTimeOnly, err := regexp.Compile("^\\d{2}:\\d{2}:\\d{2}$") + if err != nil { + return "", err + } + + if reDateTime.MatchString(checkTime) { + _, err := time.Parse(DATE_TIME, checkTime) + if err != nil { + return "", err + } + return DATE_TIME, nil + + } + if reTimeOnly.MatchString(checkTime) { + _, err := time.Parse(TIME_ONLY, checkTime) + if err != nil { + return "", err + } + return TIME_ONLY, nil + + } + return "", fmt.Errorf("Invalid date format, please use date format YYYY-MM-DD HH:MM:SS, or only HH:MM:SS such as \"2006-01-02 15:04:05\" or \"15:04:05\"") +} -- Gitee From 6b5c8bf60875303e86817b086829b3934550388b Mon Sep 17 00:00:00 2001 From: liyuanr Date: Sat, 17 Aug 2024 19:37:10 +0800 Subject: [PATCH 40/46] feat(os-proxy): add ExcutionMode to os add ExcutionMode to os to support serial during upgrade or configuration. Signed-off-by: liyuanr --- .../proxy/src/controller/controller.rs | 6 + KubeOS-Rust/proxy/src/controller/values.rs | 1 + api/v1alpha1/os_types.go | 10 +- cmd/operator/controllers/operation.go | 141 ++++++++++- cmd/operator/controllers/os_controller.go | 220 +++++++++++------- cmd/operator/controllers/requirements.go | 115 ++++++++- cmd/operator/main.go | 13 +- .../config/crd/upgrade.openeuler.org_os.yaml | 27 +++ pkg/values/values.go | 5 +- 9 files changed, 436 insertions(+), 102 deletions(-) diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index 80a85d1c..baf2f9d8 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -24,6 +24,8 @@ use kube::{ use log::{debug, error, info}; use reconciler_error::Error; +use crate::controller::values::LABEL_CONFIGURING; + use super::{ agentclient::{AgentCall, AgentClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, apiclient::ApplyApi, @@ -188,6 +190,10 @@ impl ProxyController { if labels.contains_key(LABEL_UPGRADING) { labels.remove(LABEL_UPGRADING); node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; + }else if labels.contains_key(LABEL_CONFIGURING) { + labels.remove(LABEL_CONFIGURING); + node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; + } if let Some(node_spec) = &node.spec { if let Some(node_unschedulable) = node_spec.unschedulable { diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs index dec905a9..9066f57d 100644 --- a/KubeOS-Rust/proxy/src/controller/values.rs +++ b/KubeOS-Rust/proxy/src/controller/values.rs @@ -15,6 +15,7 @@ use tokio::time::Duration; pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; +pub const LABEL_CONFIGURING: &str = "upgrade.openeuler.org/configuring"; pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; pub const OSINSTANCE_KIND: &str = "OSInstance"; diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go index 65bb8e2e..6439bc14 100644 --- a/api/v1alpha1/os_types.go +++ b/api/v1alpha1/os_types.go @@ -24,10 +24,12 @@ type OSSpec struct { CheckSum string `json:"checksum"` FlagSafe bool `json:"flagSafe"` MTLS bool `json:"mtls"` + // +kubebuilder:validation:Enum=docker;disk;containerd ImageType string `json:"imagetype"` ContainerImage string `json:"containerimage"` - OpsType string `json:"opstype"` - EvictPodForce bool `json:"evictpodforce"` + // +kubebuilder:validation:Enum=upgrade;config;rollback + OpsType string `json:"opstype"` + EvictPodForce bool `json:"evictpodforce"` // +kubebuilder:validation:Optional CaCert string `json:"cacert"` // +kubebuilder:validation:Optional @@ -45,6 +47,10 @@ type OSSpec struct { TimeWindow TimeWindow `json:"timewindow"` // +kubebuilder:validation:Optional TimeInterval int `json:"timeinterval"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=serial;parallel + // +kubebuilder:default:=parallel + ExecutionMode string `json:"executionmode"` } // +kubebuilder:subresource:status diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go index 2ffcaa2e..dd0b4fec 100644 --- a/cmd/operator/controllers/operation.go +++ b/cmd/operator/controllers/operation.go @@ -15,19 +15,20 @@ package controllers import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "openeuler.org/KubeOS/pkg/common" - "openeuler.org/KubeOS/pkg/values" "sigs.k8s.io/controller-runtime/pkg/client" upgradev1 "openeuler.org/KubeOS/api/v1alpha1" + "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" ) type operation interface { updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - nodes []corev1.Node, limit int) (int, error) + nodes []corev1.Node, limit int) (int, []error) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, node *corev1.Node, osInstance *upgradev1.OSInstance) error getOpsLabel() opsLabel @@ -42,8 +43,9 @@ func (u upgradeOps) getOpsLabel() opsLabel { } func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - nodes []corev1.Node, limit int) (int, error) { - var count int + nodes []corev1.Node, limit int) (int, []error) { + var count = 0 + var errList []error for _, node := range nodes { if count >= limit { break @@ -54,19 +56,49 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, var osInstance upgradev1.OSInstance if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { if err = client.IgnoreNotFound(err); err != nil { - log.Error(err, "failed to get osInstance "+node.Name, "skip this node") - return count, err + log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ + "Restart the reconcile and wait until it is complete.") + return count, []error{err} } + log.Error(err, "failed to get osInstance "+node.Name+"skip this node") + errList = append(errList, err) continue } if err := u.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { log.Error(err, "failed to update node and osinstance ,skip this node ") + errList = append(errList, err) continue } count++ } } + if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { + if errList = deleteSerialLabel(ctx, r, nodes); errList != nil { + log.Error(nil, "failed to delete nodes serial label") + } + } + if len(errList) > 0 { + return count, errList + } return count, nil + +} + +func deleteSerialLabel(ctx context.Context, r common.ReadStatusWriter, nodes []corev1.Node) []error { + var errList []error + for _, node := range nodes { + if _, ok := node.Labels[values.LabelSerial]; ok { + delete(node.Labels, values.LabelSerial) + if err := r.Update(ctx, &node); err != nil { + log.Error(err, "unable to delete serial label ", "node", node.Name+", skip this node") + errList = append(errList, err) + } + } + } + if len(errList) > 0 { + return errList + } + return nil } func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, @@ -114,8 +146,9 @@ func (c configOps) getOpsLabel() opsLabel { } func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - nodes []corev1.Node, limit int) (int, error) { - var count int + nodes []corev1.Node, limit int) (int, []error) { + var count = 0 + var errList []error for _, node := range nodes { if count >= limit { break @@ -123,21 +156,34 @@ func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o var osInstance upgradev1.OSInstance if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { if err = client.IgnoreNotFound(err); err != nil { - log.Error(err, "failed to get osInstance "+node.Name) - return count, err + log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ + "Restart the reconcile and wait until it is complete.") + return count, []error{err} } + log.Error(err, "failed to get osInstance "+node.Name+", skip this node") + errList = append(errList, err) continue } if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { log.Info("Configuring node " + node.Name) if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { log.Error(err, "failed to update node and osinstance ,skip this node ") + errList = append(errList, err) continue } count++ } } - return count, nil + if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { + if errList = deleteSerialLabel(ctx, r, nodes); errList != nil { + log.Error(nil, "failed to delete nodes serial label") + } + } + if len(errList) > 0 { + return count, errList + } + return count, errList + } func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, @@ -159,3 +205,74 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr log.Info("Add node configuring label " + values.LabelConfiguring + " successfully") return nil } + +type serialOps struct { + label opsLabel +} + +func (s serialOps) getOpsLabel() opsLabel { + return s.label +} + +func (s serialOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, []error) { + var count int + var errList []error + for _, node := range nodes { + log.V(1).Info("start add serial label to nodes") + if count >= limit { + break + } + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ + "Restart the reconcile and wait until it is complete.") + return count, []error{err} + } + log.Error(err, "failed to get osInstance "+node.Name+", skip this node") + errList = append(errList, err) + continue + } + switch s.getOpsLabel().label { + case values.LabelUpgrading: + if os.Spec.OSVersion != node.Status.NodeInfo.OSImage { + log.Info("Add Serial Label to node " + node.Name) + if err := s.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + errList = append(errList, err) + continue + } + count++ + } + case values.LabelConfiguring: + if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { + log.Info("Add Serial Label to node " + node.Name) + if err := s.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + errList = append(errList, err) + continue + } + count++ + } + default: + log.Error(nil, "ops "+s.getOpsLabel().label+" cannot be recognized") + return count, []error{fmt.Errorf("ops " + s.getOpsLabel().label + " cannot be recognized")} + } + } + if len(errList) == 0 { + return count, nil + } + return count, errList +} +func (s serialOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error { + log.V(1).Info("start update nodes") + node.Labels[values.LabelSerial] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } + log.Info("Add node serial label " + values.LabelSerial + " successfully") + return nil +} diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index 690b6458..667a265f 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "strconv" "time" corev1 "k8s.io/api/core/v1" @@ -57,6 +58,7 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re // Reconcile compares the actual state with the desired and updates the status of the resources e.g. nodes func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) (ctrl.Result, error) { + log.V(1).Info("start Reconcile of " + req.Name + " with namespace is " + req.Namespace) os, err := getOSCr(ctx, r, req.NamespacedName) if err != nil { if errors.IsNotFound(err) { @@ -64,41 +66,64 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) } return values.RequeueNow, err } - + log.V(1).Info("get os cr name is " + os.Name + ", namespace is " + os.Namespace) isWithinTimeWindow, err := isWithinTimeWindow(os.Spec.TimeWindow.StartTime, os.Spec.TimeWindow.EndTime) if err != nil { return values.RequeueNow, err } if !isWithinTimeWindow { - //Todo consider time interval - return values.RequeueNow, nil + log.V(1).Info("not in time window, the start time is " + os.Spec.TimeWindow.StartTime + " , the start time " + os.Spec.TimeWindow.StartTime) + return values.Requeue, nil } ops := os.Spec.OpsType var opsInsatnce operation switch ops { case "upgrade", "rollback": - opsInsatnce = upgradeOps{label: opsLabel{label: values.LabelUpgrading, op: selection.DoesNotExist}} + opsInsatnce = upgradeOps{ + label: opsLabel{ + label: values.LabelUpgrading, + op: selection.DoesNotExist, + }, + } case "config": - opsInsatnce = configOps{label: opsLabel{label: values.LabelConfiguring, op: selection.DoesNotExist}} + opsInsatnce = configOps{ + label: opsLabel{ + label: values.LabelConfiguring, + op: selection.DoesNotExist, + }, + } default: log.Error(nil, "operation "+ops+" cannot be recognized") return values.Requeue, nil } - nodeNum, err := getNodeNum(ctx, r, os.Spec.NodeSelector) + commonNodesReq, err := newCommonsNodesRequirement(os.Spec.NodeSelector, + selection.Equals).createNodeRequirement(ctx, r) if err != nil { return values.RequeueNow, err } - limit, err := calNodeLimit(ctx, r, opsInsatnce.getOpsLabel(), min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need + allNodes, err := getNodes(ctx, r, 0, commonNodesReq...) if err != nil { return values.RequeueNow, err } - if needRequeue, err := assignOperation(ctx, r, os, limit, opsInsatnce); err != nil { - return values.RequeueNow, err - } else if needRequeue { + log.V(1).Info("get all nodes is " + strconv.Itoa(len(allNodes))) + switch os.Spec.ExecutionMode { + case ExecutionModeParallel: + result, err := excuteParallelOperation(ctx, r, os, opsInsatnce, len(allNodes)) + if err != nil { + return values.RequeueNow, nil + } + return result, nil + case ExecutionModeSerial: + result, err := excuteSerialOperation(ctx, r, os, opsInsatnce, len(allNodes)) + if err != nil { + return values.RequeueNow, err + } + return result, nil + default: + log.Error(nil, "excutionMode "+os.Spec.ExecutionMode+" cannot be recognized") return values.Requeue, nil } - return setTimeInterval(os.Spec.TimeInterval), nil } // SetupWithManager sets up the controller with the Manager. @@ -114,28 +139,28 @@ func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { }); err != nil { return err } - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.name", - func(rawObj client.Object) []string { - os, ok := rawObj.(*upgradev1.OS) - if !ok { - log.Error(nil, "failed to convert to osInstance") - return []string{} - } - return []string{os.Name} - }); err != nil { - return err - } - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.namespace", - func(rawObj client.Object) []string { - os, ok := rawObj.(*upgradev1.OS) - if !ok { - log.Error(nil, "failed to convert to osInstance") - return []string{} - } - return []string{os.Namespace} - }); err != nil { - return err - } + // if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.name", + // func(rawObj client.Object) []string { + // os, ok := rawObj.(*upgradev1.OS) + // if !ok { + // log.Error(nil, "failed to convert to osInstance") + // return []string{} + // } + // return []string{os.Name} + // }); err != nil { + // return err + // } + // if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.namespace", + // func(rawObj client.Object) []string { + // os, ok := rawObj.(*upgradev1.OS) + // if !ok { + // log.Error(nil, "failed to convert to osInstance") + // return []string{} + // } + // return []string{os.Namespace} + // }); err != nil { + // return err + // } return ctrl.NewControllerManagedBy(mgr). For(&upgradev1.OS{}). Watches(&source.Kind{Type: &corev1.Node{}}, handler.Funcs{DeleteFunc: r.DeleteOSInstance}). @@ -173,22 +198,6 @@ func getOSCr(ctx context.Context, r common.ReadStatusWriter, name types.Namespac return os, nil } -// Get nodes which do not have master label，have nodeselector label or do not have other os cr nodeselector -func getNodeNum(ctx context.Context, r common.ReadStatusWriter, nodeSelector string) (int, error) { - labelList := []labelRequirementCreator{masterLabel{op: selection.DoesNotExist}, nodeSelectorLabel{value: nodeSelector, op: selection.Equals}} - requirements, err := createRequirement(labelList) - if err != nil { - return 0, err - } - nodesItems, err := getNodes(ctx, r, 0, requirements...) - if err != nil { - log.Error(err, "get slave nodes fail") - return 0, err - } - nodeNum := len(nodesItems) - return nodeNum, nil -} - func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, reqs ...labels.Requirement) ([]corev1.Node, error) { var nodeList corev1.NodeList @@ -200,45 +209,27 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, return nodeList.Items, nil } -// get now in upgrading and match with nodeselector func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, - label opsLabel, maxUnavailable int, nodeSelector string) (int, error) { - label.op = selection.Exists - labelList := []labelRequirementCreator{ - masterLabel{op: selection.DoesNotExist}, - label, - nodeSelectorLabel{value: nodeSelector, op: selection.Equals}} - requirements, err := createRequirement(labelList) - if err != nil { - return 0, err - } + maxUnavailable int, requirements []labels.Requirement) (int, error) { nodes, err := getNodes(ctx, r, 0, requirements...) if err != nil { return 0, err } return maxUnavailable - len(nodes), nil } - func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, - opsInstance operation) (bool, error) { - opsLabel := opsInstance.getOpsLabel() - opsLabel.op = selection.DoesNotExist - labelList := []labelRequirementCreator{ - masterLabel{op: selection.DoesNotExist}, - opsLabel, - nodeSelectorLabel{value: os.Spec.NodeSelector, op: selection.Equals}} - requirements, err := createRequirement(labelList) + opsInstance operation, requirements []labels.Requirement) (int, error) { nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated if err != nil { - return false, err + return 0, err } + log.V(1).Info("get need nodes is " + strconv.Itoa(len(nodes))) // Upgrade OS for selected nodes - count, err := opsInstance.updateNodes(ctx, r, &os, nodes, limit) - if err != nil { - return false, err + count, errLists := opsInstance.updateNodes(ctx, r, &os, nodes, limit) + if len(errLists) != 0 { + return 0, fmt.Errorf("update nodes and osinstance error") } - - return count >= limit, nil + return count, nil } func min(a, b int) int { @@ -311,15 +302,74 @@ func setTimeInterval(timeInterval int) ctrl.Result { return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(timeInterval) * time.Second} } -func createRequirement(labelsList []labelRequirementCreator) ([]labels.Requirement, error) { - var requirements []labels.Requirement - for _, label := range labelsList { - requirement, err := label.createLabelRequirement() - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelNodeSelector) - return []labels.Requirement{}, err - } - requirements = append(requirements, requirement...) +func excuteParallelOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, + opsInsatnce operation, nodeNum int) (ctrl.Result, error) { + opsLabel := opsInsatnce.getOpsLabel() + opsLabel.op = selection.Exists + opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, + selection.Equals, opsLabel).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + limit, err := calNodeLimit(ctx, r, min(os.Spec.MaxUnavailable, nodeNum), opsNodesReq) // adjust maxUnavailable if need + if err != nil { + return values.RequeueNow, nil + } + opsLabel.op = selection.DoesNotExist + noOpsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, + selection.Equals, opsLabel).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + if _, err := assignOperation(ctx, r, os, limit, opsInsatnce, noOpsNodesReq); err != nil { + return values.RequeueNow, nil } - return requirements, nil + return setTimeInterval(os.Spec.TimeInterval), nil +} + +func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, + opsInsatnce operation, nodeNum int) (ctrl.Result, error) { + opsLabel := opsInsatnce.getOpsLabel() + opsLabel.op = selection.Exists + opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, + selection.Equals, opsLabel).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + opsNodeNum, err := getNodes(ctx, r, 0, opsNodesReq...) + if err != nil { + return values.RequeueNow, nil + } + if len(opsNodeNum) > 0 { + return values.Requeue, nil + } + log.V(1).Info("get opsnodes is " + strconv.Itoa(len(opsNodeNum))) + serialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, + selection.Equals, selection.Exists).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + serialNodeLimit, err := calNodeLimit(ctx, r, min(os.Spec.MaxUnavailable, nodeNum), serialNodesRequirement) + if err != nil { + return values.RequeueNow, nil + } + log.V(1).Info("get serialLimit is " + strconv.Itoa(serialNodeLimit)) + noSerialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, + selection.Equals, selection.DoesNotExist).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + if _, err := assignOperation(ctx, r, os, serialNodeLimit, serialOps{opsInsatnce.getOpsLabel()}, noSerialNodesRequirement); err != nil { + return values.RequeueNow, nil + } + serialLimit := 1 // 1 is the number of operation nodes when excution mode in serial + count, err := assignOperation(ctx, r, os, serialLimit, opsInsatnce, serialNodesRequirement) + if err != nil { + return values.RequeueNow, nil + } + if count > 0 { + return values.Requeue, nil + } + return setTimeInterval(os.Spec.TimeInterval), nil + } diff --git a/cmd/operator/controllers/requirements.go b/cmd/operator/controllers/requirements.go index 5a8f255f..9317b495 100644 --- a/cmd/operator/controllers/requirements.go +++ b/cmd/operator/controllers/requirements.go @@ -14,8 +14,11 @@ package controllers import ( + "context" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + "openeuler.org/KubeOS/pkg/common" "openeuler.org/KubeOS/pkg/values" ) @@ -65,7 +68,7 @@ func (nl nodeSelectorLabel) createLabelRequirement() ([]labels.Requirement, erro return []labels.Requirement{}, nil } var requirements []labels.Requirement - // if nodeselector is "", will get the nodes which label value is "",and not have label + // if nodeselector is "no-label", will get the nodes which not have label if nl.value == NoNodeSelector { requirement, err := labels.NewRequirement(values.LabelNodeSelector, selection.DoesNotExist, nil) if err != nil { @@ -83,3 +86,113 @@ func (nl nodeSelectorLabel) createLabelRequirement() ([]labels.Requirement, erro requirements = append(requirements, *requirement) return requirements, nil } + +type serialLabel struct { + op selection.Operator +} + +func (sl serialLabel) createLabelRequirement() ([]labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelSerial, sl.op, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelSerial) + return []labels.Requirement{}, err + } + return []labels.Requirement{*requirement}, nil +} + +func createRequirement(labelsList []labelRequirementCreator) ([]labels.Requirement, error) { + var requirements []labels.Requirement + for _, label := range labelsList { + requirement, err := label.createLabelRequirement() + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return []labels.Requirement{}, err + } + requirements = append(requirements, requirement...) + } + return requirements, nil +} + +type nodeRequirementCreator interface { + createNodeRequirement(ctx context.Context, r common.ReadStatusWriter, nodeSelector string) ([]labels.Requirement, error) +} + +type commonsNodesRequirement struct { + nodeSelector string + op selection.Operator +} + +func newCommonsNodesRequirement(nodeSelector string, op selection.Operator) commonsNodesRequirement { + return commonsNodesRequirement{ + nodeSelector, + op, + } +} + +func (c commonsNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) { + labelList := []labelRequirementCreator{ + masterLabel{op: selection.DoesNotExist}, + nodeSelectorLabel{value: c.nodeSelector, op: c.op}, + } + requirements, err := createRequirement(labelList) + if err != nil { + return []labels.Requirement{}, err + } + return requirements, nil +} + +type opsNodesRequirement struct { + common commonsNodesRequirement + ops opsLabel +} + +func newopsNodesRequirement(nodeSelector string, nodeSelectorOp selection.Operator, ops opsLabel) opsNodesRequirement { + return opsNodesRequirement{ + common: newCommonsNodesRequirement(nodeSelector, nodeSelectorOp), + ops: ops, + } +} + +func (o opsNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) { + labelList := []labelRequirementCreator{ + o.ops, + } + requirements, err := createRequirement(labelList) + if err != nil { + return []labels.Requirement{}, err + } + commonRequirements, err := o.common.createNodeRequirement(ctx, r) + if err != nil { + return []labels.Requirement{}, err + } + requirements = append(requirements, commonRequirements...) + return requirements, nil +} + +type serialNodesRequirement struct { + common commonsNodesRequirement + serialOp selection.Operator +} + +func newSerialNodesRequirement(nodeSelector string, nodeSelectorOp selection.Operator, serialrOp selection.Operator) serialNodesRequirement { + return serialNodesRequirement{ + common: newCommonsNodesRequirement(nodeSelector, nodeSelectorOp), + serialOp: serialrOp, + } +} + +func (o serialNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) { + labelList := []labelRequirementCreator{ + serialLabel{op: o.serialOp}, + } + requirements, err := createRequirement(labelList) + if err != nil { + return []labels.Requirement{}, err + } + commonRequirements, err := o.common.createNodeRequirement(ctx, r) + if err != nil { + return []labels.Requirement{}, err + } + requirements = append(requirements, commonRequirements...) + return requirements, nil +} diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 6b90b26b..79b23ec7 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -14,6 +14,7 @@ package main import ( "os" + "strings" "time" zaplogfmt "github.com/sykesm/zap-logfmt" @@ -38,6 +39,11 @@ var ( setupLog = ctrl.Log.WithName("setup") ) +const ( + DEBUGLOGLEVEL = "debug" + INFOLOGLEVEL = "info" +) + func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -51,7 +57,12 @@ func main() { encoder.AppendString(ts.UTC().Format(time.RFC3339Nano)) } logfmtEncoder := zaplogfmt.NewEncoder(configLog) - logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder)) + level := zap.Level(zapcore.Level(0)) + goLog := strings.ToLower(os.Getenv("GO_LOG")) + if goLog == DEBUGLOGLEVEL { + level = zap.Level(zapcore.Level(-1)) + } + logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder), level) ctrl.SetLogger(logger) mgr, err := common.NewControllerManager(setupLog, scheme) diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index 27ff3273..f64bad64 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -57,9 +57,19 @@ spec: type: string evictpodforce: type: boolean + executionmode: + default: parallel + enum: + - serial + - parallel + type: string flagSafe: type: boolean imagetype: + enum: + - docker + - disk + - containerd type: string imageurl: type: string @@ -68,8 +78,13 @@ spec: mtls: type: boolean nodeselector: + default: no-label type: string opstype: + enum: + - upgrade + - config + - rollback type: string osversion: type: string @@ -101,6 +116,18 @@ spec: version: type: string type: object + timeinterval: + type: integer + timewindow: + properties: + endtime: + type: string + starttime: + type: string + required: + - endtime + - starttime + type: object upgradeconfigs: description: SysConfigs defines all configurations expected by the user properties: diff --git a/pkg/values/values.go b/pkg/values/values.go index e2c03540..2045bbe6 100644 --- a/pkg/values/values.go +++ b/pkg/values/values.go @@ -30,7 +30,10 @@ const ( LabelNodeSelector = "upgrade.openeuler.org/node-selector" // LabelConfiguring is the key of the configuring label for nodes LabelConfiguring = "upgrade.openeuler.org/configuring" - defaultPeriod = 15 * time.Second + // LabelSerial is the key of the serial label for nodes + LabelSerial = "upgrade.openeuler.org/serial" + + defaultPeriod = 15 * time.Second // OsiStatusName is param name of nodeStatus in osInstance OsiStatusName = "nodestatus" // UpgradeConfigName is param name of UpgradeConfig -- Gitee From d57ff7a993fc74e4f10bc1cc1619d75ec93d2af1 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Wed, 21 Aug 2024 16:52:16 +0800 Subject: [PATCH 41/46] bugfix: fix the problem that proxy will get all os for processing The OS modification triggers the reconcile of the proxy. As a result, the proxy processes the OS upgrade or configuration that does not correspond to the current node. To solve this problem, the namedspace field is added to osinstance.spec. During each upgrade or configuration compelets, the name and namespace of the OS corresponding to osinstance are specified. After the upgrade or configuration is complete, the field is set to None. Signed-off-by: liyuanr --- KubeOS-Rust/proxy/src/controller/apiclient.rs | 4 +- .../proxy/src/controller/apiserver_mock.rs | 9 +++- .../proxy/src/controller/controller.rs | 31 ++++++++----- KubeOS-Rust/proxy/src/controller/crd.rs | 17 +++++++ KubeOS-Rust/proxy/src/controller/values.rs | 3 +- Makefile | 2 +- api/v1alpha1/os_types.go | 11 +++++ cmd/operator/controllers/operation.go | 14 +++--- cmd/operator/controllers/os_controller.go | 44 ++++++------------- cmd/operator/controllers/requirements.go | 5 ++- cmd/operator/controllers/times.go | 24 ++++++---- .../config/crd/upgrade.openeuler.org_os.yaml | 1 + .../upgrade.openeuler.org_osinstances.yaml | 8 ++++ docs/example/config/manager/manager.yaml | 7 +++ pkg/values/values.go | 2 + 15 files changed, 119 insertions(+), 63 deletions(-) diff --git a/KubeOS-Rust/proxy/src/controller/apiclient.rs b/KubeOS-Rust/proxy/src/controller/apiclient.rs index 3afd5a51..85e839f0 100644 --- a/KubeOS-Rust/proxy/src/controller/apiclient.rs +++ b/KubeOS-Rust/proxy/src/controller/apiclient.rs @@ -39,7 +39,7 @@ impl Default for OSInstanceSpecPatch { OSInstanceSpecPatch { api_version: OSINSTANCE_API_VERSION.to_string(), kind: OSINSTANCE_KIND.to_string(), - spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None ,namespacedname:None}, } } } @@ -102,7 +102,7 @@ impl ApplyApi for ControllerClient { labels: Some(labels), ..ObjectMeta::default() }, - spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None ,namespacedname: None}, status: None, }; let osi_api = Api::namespaced(self.client.clone(), namespace); diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs index 2b182ca8..679096d3 100644 --- a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs +++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs @@ -33,7 +33,7 @@ use mockall::mock; use self::mock_error::Error; use super::{ agentclient::*, - crd::{Configs, OSInstanceStatus}, + crd::{Configs, NamespacedName, OSInstanceStatus}, values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK}, }; use crate::controller::{ @@ -531,6 +531,7 @@ impl OSInstance { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + namespacedname:Some(NamespacedName{namespace:String::from("default"),name:String::from("test")}) }, status: Some(OSInstanceStatus { sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), @@ -605,7 +606,7 @@ impl OS { os.meta_mut().namespace = Some("default".into()); os } - + pub fn set_os_osversion_v2_opstype_config() -> Self { let mut os = OS::set_os_default(); os.spec.osversion = String::from("KubeOS v2"); @@ -676,6 +677,10 @@ impl Default for OSSpec { clientkey: Some(String::from("")), sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + nodeselector:None, + timeinterval:None, + timewindow:None, + executionmode:None, } } } diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index baf2f9d8..40405b2d 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -24,16 +24,14 @@ use kube::{ use log::{debug, error, info}; use reconciler_error::Error; -use crate::controller::values::LABEL_CONFIGURING; - use super::{ agentclient::{AgentCall, AgentClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, apiclient::ApplyApi, crd::{Configs, Content, OSInstance, OS}, utils::{check_version, get_config_version, ConfigOperation, ConfigType}, values::{ - LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, - REQUEUE_ERROR, REQUEUE_NORMAL, + LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, + OSINSTANCE_NAMESPACE, REQUEUE_ERROR, REQUEUE_NORMAL,LABEL_CONFIGURING, NO_REQUEUE, }, }; @@ -48,10 +46,20 @@ pub async fn reconcile( let namespace: String = os_cr .namespace() .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; - proxy_controller.check_osi_exisit(&namespace, &node_name).await?; - let controller_res = proxy_controller.get_resources(&namespace, &node_name).await?; + proxy_controller.check_osi_exisit(&node_name).await?; + let controller_res = proxy_controller.get_resources(&node_name).await?; let node = controller_res.node; let mut osinstance = controller_res.osinstance; + if let Some(namespacedname) = osinstance.spec.namespacedname.as_ref(){ + debug!("osinstance correspending os name is {}, namespace is {}",namespacedname.name,namespacedname.namespace); + if !(namespacedname.name == os_cr.name() && namespacedname.namespace == namespace){ + debug!("current os cr name:{}, namespace:{} is not belong to this node",os_cr.name(),namespace); + return Ok(NO_REQUEUE) + } + }else { + return Ok(REQUEUE_NORMAL) + } + let node_os_image = &node .status .as_ref() @@ -153,8 +161,8 @@ impl ProxyController { } impl ProxyController { - async fn check_osi_exisit(&self, namespace: &str, node_name: &str) -> Result<(), Error> { - let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + async fn check_osi_exisit(&self, node_name: &str) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), OSINSTANCE_NAMESPACE); match osi_api.get(node_name).await { Ok(osi) => { debug!("osinstance is exist {:?}", osi.name()); @@ -162,15 +170,15 @@ impl ProxyController { }, Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { info!("Create OSInstance {}", node_name); - self.controller_client.create_osinstance(node_name, namespace).await?; + self.controller_client.create_osinstance(node_name, OSINSTANCE_NAMESPACE).await?; Ok(()) }, Err(err) => Err(Error::KubeClient { source: err }), } } - async fn get_resources(&self, namespace: &str, node_name: &str) -> Result { - let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + async fn get_resources(&self, node_name: &str) -> Result { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), OSINSTANCE_NAMESPACE); let osinstance_cr = osi_api.get(node_name).await?; let node_api: Api = Api::all(self.k8s_client.clone()); let node_cr = node_api.get(node_name).await?; @@ -234,6 +242,7 @@ impl ProxyController { value: String::from("namespace"), })?; osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); + osinstance.spec.namespacedname = None; self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; } Ok(()) diff --git a/KubeOS-Rust/proxy/src/controller/crd.rs b/KubeOS-Rust/proxy/src/controller/crd.rs index 41f333e8..36e14d59 100644 --- a/KubeOS-Rust/proxy/src/controller/crd.rs +++ b/KubeOS-Rust/proxy/src/controller/crd.rs @@ -32,6 +32,11 @@ pub struct OSSpec { pub clientkey: Option, pub sysconfigs: Option, pub upgradeconfigs: Option, + pub nodeselector:Option, + pub timewindow: Option, + pub timeinterval: Option, + pub executionmode:Option, + } #[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] @@ -46,6 +51,7 @@ pub struct OSSpec { )] pub struct OSInstanceSpec { pub nodestatus: String, + pub namespacedname: Option, pub sysconfigs: Option, pub upgradeconfigs: Option, } @@ -75,3 +81,14 @@ pub struct Content { pub value: Option, pub operation: Option, } + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct NamespacedName{ + pub namespace: String, + pub name: String, +} +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct TimeWindow{ + pub starttime:String, + pub endtime:String, +} \ No newline at end of file diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs index 9066f57d..2edb3dcb 100644 --- a/KubeOS-Rust/proxy/src/controller/values.rs +++ b/KubeOS-Rust/proxy/src/controller/values.rs @@ -19,6 +19,7 @@ pub const LABEL_CONFIGURING: &str = "upgrade.openeuler.org/configuring"; pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; pub const OSINSTANCE_KIND: &str = "OSInstance"; +pub const OSINSTANCE_NAMESPACE: &str = "default"; pub const NODE_STATUS_IDLE: &str = "idle"; pub const NODE_STATUS_UPGRADE: &str = "upgrade"; @@ -30,5 +31,5 @@ pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; - pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; +pub const NO_REQUEUE: ReconcilerAction = ReconcilerAction { requeue_after: None }; diff --git a/Makefile b/Makefile index d4cd71e3..7a8db31d 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ generate: controller-gen $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." # Build the docker image -docker-build: operator proxy +docker-build: operator rust-proxy docker build --target operator -t ${IMG_OPERATOR} . docker build --target proxy -t ${IMG_PROXY} . diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go index 6439bc14..bde86d25 100644 --- a/api/v1alpha1/os_types.go +++ b/api/v1alpha1/os_types.go @@ -46,6 +46,7 @@ type OSSpec struct { // +kubebuilder:validation:Optional TimeWindow TimeWindow `json:"timewindow"` // +kubebuilder:validation:Optional + // +kubebuilder:default:=15 TimeInterval int `json:"timeinterval"` // +kubebuilder:validation:Optional // +kubebuilder:validation:Enum=serial;parallel @@ -132,11 +133,21 @@ type OSInstanceSpec struct { // +kubebuilder:validation:Optional NodeStatus string `json:"nodestatus"` // +kubebuilder:validation:Optional + NamespacedName NamespacedName `json:"namespacedname"` + // +kubebuilder:validation:Optional SysConfigs SysConfigs `json:"sysconfigs"` // +kubebuilder:validation:Optional UpgradeConfigs SysConfigs `json:"upgradeconfigs"` } +// NamespacedName defines name and namespace of os corresponding to osinstance +type NamespacedName struct { + // +kubebuilder:validation:Optional + Name string `json:"name"` + // +kubebuilder:validation:Optional + Namespace string `json:"namespace"` +} + // +kubebuilder:object:root=true // OSInstanceList is a list of OSInstance diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go index dd0b4fec..5ac3d6d4 100644 --- a/cmd/operator/controllers/operation.go +++ b/cmd/operator/controllers/operation.go @@ -54,7 +54,7 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, if os.Spec.OSVersion != osVersionNode { log.Info("Upgrading node " + node.Name) var osInstance upgradev1.OSInstance - if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err := r.Get(ctx, types.NamespacedName{Namespace: values.OsiNamespace, Name: node.Name}, &osInstance); err != nil { if err = client.IgnoreNotFound(err); err != nil { log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ "Restart the reconcile and wait until it is complete.") @@ -73,7 +73,7 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, } } if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { - if errList = deleteSerialLabel(ctx, r, nodes); errList != nil { + if errList = deleteSerialLabel(ctx, r, nodes); len(errList) != 0 { log.Error(nil, "failed to delete nodes serial label") } } @@ -123,11 +123,14 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW } } osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() + osInstance.Spec.NamespacedName = upgradev1.NamespacedName{Name: os.Name, Namespace: os.Namespace} + log.V(1).Info("Wait to update osinstance name:" + osInstance.Name + " node name is " + node.Name) if err := r.Update(ctx, osInstance); err != nil { log.Error(err, "unable to update", "osInstance", osInstance.Name) return err } log.Info("Update osinstance spec successfully") + node.Labels[values.LabelUpgrading] = "" if err := r.Update(ctx, node); err != nil { log.Error(err, "unable to label", "node", node.Name) @@ -175,7 +178,7 @@ func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o } } if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { - if errList = deleteSerialLabel(ctx, r, nodes); errList != nil { + if errList = deleteSerialLabel(ctx, r, nodes); len(errList) != 0 { log.Error(nil, "failed to delete nodes serial label") } } @@ -192,11 +195,14 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr return err } osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() + osInstance.Spec.NamespacedName = upgradev1.NamespacedName{Name: os.Name, Namespace: os.Namespace} + log.V(1).Info("Wait to update osinstance name:" + osInstance.Name + " node name is " + node.Name) if err := r.Update(ctx, osInstance); err != nil { log.Error(err, "unable to update", "osInstance", osInstance.Name) return err } log.Info("Update osinstance spec successfully") + node.Labels[values.LabelConfiguring] = "" if err := r.Update(ctx, node); err != nil { log.Error(err, "unable to label", "node", node.Name) @@ -219,7 +225,6 @@ func (s serialOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o var count int var errList []error for _, node := range nodes { - log.V(1).Info("start add serial label to nodes") if count >= limit { break } @@ -267,7 +272,6 @@ func (s serialOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o } func (s serialOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, node *corev1.Node, osInstance *upgradev1.OSInstance) error { - log.V(1).Info("start update nodes") node.Labels[values.LabelSerial] = "" if err := r.Update(ctx, node); err != nil { log.Error(err, "unable to label", "node", node.Name) diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index 667a265f..9e2e8e49 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -66,13 +66,13 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) } return values.RequeueNow, err } - log.V(1).Info("get os cr name is " + os.Name + ", namespace is " + os.Namespace) isWithinTimeWindow, err := isWithinTimeWindow(os.Spec.TimeWindow.StartTime, os.Spec.TimeWindow.EndTime) if err != nil { return values.RequeueNow, err } if !isWithinTimeWindow { - log.V(1).Info("not in time window, the start time is " + os.Spec.TimeWindow.StartTime + " , the start time " + os.Spec.TimeWindow.StartTime) + log.V(1).Info("not in time window, the start time is " + os.Spec.TimeWindow.StartTime + + " , the end time " + os.Spec.TimeWindow.EndTime) return values.Requeue, nil } @@ -106,7 +106,6 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) if err != nil { return values.RequeueNow, err } - log.V(1).Info("get all nodes is " + strconv.Itoa(len(allNodes))) switch os.Spec.ExecutionMode { case ExecutionModeParallel: result, err := excuteParallelOperation(ctx, r, os, opsInsatnce, len(allNodes)) @@ -139,28 +138,6 @@ func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { }); err != nil { return err } - // if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.name", - // func(rawObj client.Object) []string { - // os, ok := rawObj.(*upgradev1.OS) - // if !ok { - // log.Error(nil, "failed to convert to osInstance") - // return []string{} - // } - // return []string{os.Name} - // }); err != nil { - // return err - // } - // if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OS{}, "metadata.namespace", - // func(rawObj client.Object) []string { - // os, ok := rawObj.(*upgradev1.OS) - // if !ok { - // log.Error(nil, "failed to convert to osInstance") - // return []string{} - // } - // return []string{os.Namespace} - // }); err != nil { - // return err - // } return ctrl.NewControllerManagedBy(mgr). For(&upgradev1.OS{}). Watches(&source.Kind{Type: &corev1.Node{}}, handler.Funcs{DeleteFunc: r.DeleteOSInstance}). @@ -219,12 +196,14 @@ func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, } func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, opsInstance operation, requirements []labels.Requirement) (int, error) { + if limit == 0 { + return 0, nil + } nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated if err != nil { return 0, err } - log.V(1).Info("get need nodes is " + strconv.Itoa(len(nodes))) - // Upgrade OS for selected nodes + log.V(1).Info("get wait to check nodes is " + strconv.Itoa(len(nodes))) count, errLists := opsInstance.updateNodes(ctx, r, &os, nodes, limit) if len(errLists) != 0 { return 0, fmt.Errorf("update nodes and osinstance error") @@ -343,7 +322,7 @@ func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os up if len(opsNodeNum) > 0 { return values.Requeue, nil } - log.V(1).Info("get opsnodes is " + strconv.Itoa(len(opsNodeNum))) + serialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, selection.Equals, selection.Exists).createNodeRequirement(ctx, r) if err != nil { @@ -353,15 +332,20 @@ func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os up if err != nil { return values.RequeueNow, nil } - log.V(1).Info("get serialLimit is " + strconv.Itoa(serialNodeLimit)) + noSerialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, selection.Equals, selection.DoesNotExist).createNodeRequirement(ctx, r) if err != nil { return values.RequeueNow, nil } - if _, err := assignOperation(ctx, r, os, serialNodeLimit, serialOps{opsInsatnce.getOpsLabel()}, noSerialNodesRequirement); err != nil { + // add serial label to node + serialOpsInstance := serialOps{ + label: opsInsatnce.getOpsLabel(), + } + if _, err := assignOperation(ctx, r, os, serialNodeLimit, serialOpsInstance, noSerialNodesRequirement); err != nil { return values.RequeueNow, nil } + serialLimit := 1 // 1 is the number of operation nodes when excution mode in serial count, err := assignOperation(ctx, r, os, serialLimit, opsInsatnce, serialNodesRequirement) if err != nil { diff --git a/cmd/operator/controllers/requirements.go b/cmd/operator/controllers/requirements.go index 9317b495..12db8e82 100644 --- a/cmd/operator/controllers/requirements.go +++ b/cmd/operator/controllers/requirements.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + "openeuler.org/KubeOS/pkg/common" "openeuler.org/KubeOS/pkg/values" ) @@ -124,8 +125,8 @@ type commonsNodesRequirement struct { func newCommonsNodesRequirement(nodeSelector string, op selection.Operator) commonsNodesRequirement { return commonsNodesRequirement{ - nodeSelector, - op, + nodeSelector: nodeSelector, + op: op, } } diff --git a/cmd/operator/controllers/times.go b/cmd/operator/controllers/times.go index eab3a224..3a72cce9 100644 --- a/cmd/operator/controllers/times.go +++ b/cmd/operator/controllers/times.go @@ -25,6 +25,7 @@ const ( TIME_ONLY = "15:04:05" ExecutionModeSerial = "serial" ExecutionModeParallel = "parallel" + oneDayTime = 24 * time.Hour ) func isWithinTimeWindow(start, end string) (bool, error) { @@ -32,7 +33,7 @@ func isWithinTimeWindow(start, end string) (bool, error) { return true, nil } if start == "" || end == "" { - return false, fmt.Errorf("The start time and end time must be both empty or not empty.") + return false, fmt.Errorf("invalid TimeWindow: The start time and end time must be both empty or not empty") } layoutStart, err := checkTimeValid(start) if err != nil { @@ -43,13 +44,11 @@ func isWithinTimeWindow(start, end string) (bool, error) { return false, err } if layoutStart != layoutEnd { - return false, fmt.Errorf("Start Time needs same time format with End Time") + return false, fmt.Errorf("invalid TimeWindow: Start Time should have same time format with End Time") } now := time.Now() - if layoutStart == TIME_ONLY { - timeFormat := now.Format(TIME_ONLY) - now, err = time.ParseInLocation(layoutStart, timeFormat, now.Location()) - } + timeFormat := now.Format(layoutStart) + now, err = time.ParseInLocation(layoutStart, timeFormat, now.Location()) startTime, err := time.ParseInLocation(layoutStart, start, now.Location()) if err != nil { return false, err @@ -58,11 +57,18 @@ func isWithinTimeWindow(start, end string) (bool, error) { if err != nil { return false, err } + if endTime.Equal(startTime) { + return false, fmt.Errorf("invalid TimeWindow: start time is equal to end time") + } if endTime.Before(startTime) { - endTime = endTime.Add(24 * time.Hour) + if layoutStart == DATE_TIME { + return false, fmt.Errorf("invalid TimeWindow: Start %s Time is after end time %s", + startTime.Format(layoutStart), endTime.Format(layoutEnd)) + } + endTime = endTime.Add(oneDayTime) fmt.Printf("endtime time add 24 hour is %s\n", endTime.Format(layoutStart)) if now.Before(startTime) { - now = now.Add(24 * time.Hour) + now = now.Add(oneDayTime) fmt.Printf("now time add 24 hour is %s\n", now.Format(layoutStart)) } @@ -96,5 +102,5 @@ func checkTimeValid(checkTime string) (string, error) { return TIME_ONLY, nil } - return "", fmt.Errorf("Invalid date format, please use date format YYYY-MM-DD HH:MM:SS, or only HH:MM:SS such as \"2006-01-02 15:04:05\" or \"15:04:05\"") + return "", fmt.Errorf("invalid TimeWindow: invalid date format, please use date format YYYY-MM-DD HH:MM:SS, or only HH:MM:SS") } diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index f64bad64..8a29e0c3 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -117,6 +117,7 @@ spec: type: string type: object timeinterval: + default: 15 type: integer timewindow: properties: diff --git a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml index df9119b4..47de87e3 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml @@ -53,6 +53,14 @@ spec: spec: description: OSInstanceSpec defines desired state of OS properties: + namespacedname: + description: NamespacedName defines name and namespace of os corresponding to osinstance + properties: + name: + type: string + namespace: + type: string + type: object nodestatus: type: string sysconfigs: diff --git a/docs/example/config/manager/manager.yaml b/docs/example/config/manager/manager.yaml index f5cf84f9..c35d9010 100644 --- a/docs/example/config/manager/manager.yaml +++ b/docs/example/config/manager/manager.yaml @@ -60,6 +60,9 @@ spec: - /operator image: name: operator + volumeMounts: + - name: date-config + mountPath: /etc/localtime securityContext: allowPrivilegeEscalation: false runAsUser: 6552 @@ -71,6 +74,10 @@ spec: requests: cpu: 100m memory: 20Mi + volumes: + - name: date-config + hostPath: + path: /etc/localtime terminationGracePeriodSeconds: 10 nodeSelector: node-role.kubernetes.io/control-plane: "" diff --git a/pkg/values/values.go b/pkg/values/values.go index 2045bbe6..9e213a83 100644 --- a/pkg/values/values.go +++ b/pkg/values/values.go @@ -40,6 +40,8 @@ const ( UpgradeConfigName = "UpgradeConfig" // SysConfigName is param name of SysConfig SysConfigName = "SysConfig" + // OsiNamespace is the namespace of osinstance + OsiNamespace = "default" ) // NodeStatus defines state of nodes -- Gitee From 8f68f9502a19fa786b64c433a6103c2431d142f9 Mon Sep 17 00:00:00 2001 From: Tomoki_sunzj Date: Tue, 10 Sep 2024 10:09:41 +0800 Subject: [PATCH 42/46] feat: add kbimg-rs --- .gitignore | 3 + KubeOS-Rust/Cargo.lock | 218 +- KubeOS-Rust/Cargo.toml | 2 +- KubeOS-Rust/kbimg/Cargo.toml | 16 + KubeOS-Rust/kbimg/kbimg.toml | 87 + KubeOS-Rust/kbimg/src/admin_container.rs | 84 + KubeOS-Rust/kbimg/src/commands.rs | 170 ++ KubeOS-Rust/kbimg/src/docker_img.rs | 74 + KubeOS-Rust/kbimg/src/main.rs | 122 ++ KubeOS-Rust/kbimg/src/repo.rs | 203 ++ KubeOS-Rust/kbimg/src/scripts_gen.rs | 1754 +++++++++++++++++ KubeOS-Rust/kbimg/src/utils.rs | 126 ++ KubeOS-Rust/kbimg/src/values.rs | 42 + Makefile | 11 +- docs/quick-start.md | 2 +- ...66\344\275\234\346\214\207\345\257\274.md" | 316 +++ 16 files changed, 3225 insertions(+), 5 deletions(-) create mode 100644 KubeOS-Rust/kbimg/Cargo.toml create mode 100644 KubeOS-Rust/kbimg/kbimg.toml create mode 100644 KubeOS-Rust/kbimg/src/admin_container.rs create mode 100644 KubeOS-Rust/kbimg/src/commands.rs create mode 100644 KubeOS-Rust/kbimg/src/docker_img.rs create mode 100644 KubeOS-Rust/kbimg/src/main.rs create mode 100644 KubeOS-Rust/kbimg/src/repo.rs create mode 100644 KubeOS-Rust/kbimg/src/scripts_gen.rs create mode 100644 KubeOS-Rust/kbimg/src/utils.rs create mode 100644 KubeOS-Rust/kbimg/src/values.rs diff --git a/.gitignore b/.gitignore index 4d173c5e..0cac753e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ target/ # KubeOS bin /bin + +# scripts-auto +/scripts-auto diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 93e3d07d..5b97095b 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -187,6 +187,45 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "clap" +version = "3.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "cli" version = "1.0.6" @@ -224,6 +263,31 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crypto-common" version = "0.1.6" @@ -632,6 +696,12 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1067,6 +1137,20 @@ dependencies = [ "url", ] +[[package]] +name = "kbimg" +version = "1.0.6" +dependencies = [ + "anyhow", + "clap", + "env_logger", + "log", + "regex", + "serde", + "sysinfo", + "toml 0.7.6", +] + [[package]] name = "kube" version = "0.66.0" @@ -1358,6 +1442,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "num-traits" version = "0.2.17" @@ -1453,6 +1546,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "parity-tokio-ipc" version = "0.9.0" @@ -1587,7 +1686,31 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" dependencies = [ - "toml", + "toml 0.5.11", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", ] [[package]] @@ -1710,6 +1833,26 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2013,6 +2156,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2144,6 +2296,21 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sysinfo" +version = "0.29.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -2194,6 +2361,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + [[package]] name = "thiserror" version = "1.0.50" @@ -2371,6 +2544,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +dependencies = [ + "indexmap 2.2.6", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -2825,6 +3032,15 @@ version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml index 68ee670a..2886023f 100644 --- a/KubeOS-Rust/Cargo.toml +++ b/KubeOS-Rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["agent", "cli", "manager", "proxy"] +members = ["agent", "cli", "kbimg", "manager", "proxy"] resolver = "2" [profile.release] diff --git a/KubeOS-Rust/kbimg/Cargo.toml b/KubeOS-Rust/kbimg/Cargo.toml new file mode 100644 index 00000000..cec12b8d --- /dev/null +++ b/KubeOS-Rust/kbimg/Cargo.toml @@ -0,0 +1,16 @@ +[package] +description = "KubeOS kbimg" +edition = "2021" +license = "MulanPSL-2.0" +name = "kbimg" +version = "1.0.6" + +[dependencies] +anyhow = { version = "1.0" } +clap = { version = "=3.2.23", features = ["derive"] } +env_logger = { version = "0.9" } +log = { version = "= 0.4.15" } +regex = { version = "1.7.3" } +serde = { version = "1.0", features = ["derive"] } +sysinfo = { version = "=0.29.11" } +toml = { version = "=0.7.6" } diff --git a/KubeOS-Rust/kbimg/kbimg.toml b/KubeOS-Rust/kbimg/kbimg.toml new file mode 100644 index 00000000..aaa9b48f --- /dev/null +++ b/KubeOS-Rust/kbimg/kbimg.toml @@ -0,0 +1,87 @@ +[from_repo] +agent_path = "/root/KubeOS/bin/os-agent" +image_type = "vm-repo" +legacy_bios = false +repo_path = "/etc/yum.repos.d/openEuler.repo" +root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0" +version = "v1" +rpmlist = [ + "NetworkManager", + "conntrack-tools", + "containernetworking-plugins", + "coreutils", + "dhcp", + "docker", + "dosfstools", + "dracut", + "ebtables", + "ethtool", + "gawk", + "hwinfo", + "kernel", + "kubernetes-kubeadm", + "kubernetes-kubelet", + "lvm2", + "net-tools", + "openssh-server", + "passwd", + "parted", + "rsyslog", + "socat", + "sudo", + "vi", +] + +# [from_dockerimg] +# docker_img = "" +# image_type = "vm-docker" + +# [admin_container] +# dockerfile = "" +# docker_img = "" + +# [pxe_config] +# rootfs_name = "kubeos.tar" +# disk = "/dev/sda" +# server_ip = "192.168.1.50" +# local_ip = "192.168.1.100" +# route_ip = "192.168.1.1" +# netmask = "255.255.255.0" +# net_name = "eth0" + +# [[users]] +# groups = ["admin"] +# name = "foo" +# passwd = "foo" +# sudo = "ALL=(ALL) ALL" + +# [[users]] +# groups = ["example"] +# name = "bar" +# passwd = "bar" + +# [[copy_files]] +# dst = "/root/ztest2" +# src = "/root/KubeOS/ztest2" + +# [[copy_files]] +# dst = "/root/ztest1" +# src = "/root/KubeOS/ztest1/*" + +# [grub] +# passwd = "foo" + +# [systemd_service] +# name = ["kubelet", "containerd"] + +# [chroot_script] +# path = "../../ztest/myscript.sh" + +# [disk_partition] +# first = 100 +# second = 2500 +# third = 2300 +# img_size = 30 + +# [persist_mkdir] +# name = ["lv_data"] diff --git a/KubeOS-Rust/kbimg/src/admin_container.rs b/KubeOS-Rust/kbimg/src/admin_container.rs new file mode 100644 index 00000000..03196739 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/admin_container.rs @@ -0,0 +1,84 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{create_dir_all, File}, + path::PathBuf, +}; + +use anyhow::bail; + +use crate::{ + commands::AdminContainerInfo, + scripts_gen::*, + utils::{self, set_permissions}, + values::*, + Config, CreateImage, +}; + +impl CreateImage for AdminContainerInfo { + fn prepare(&self, _: &mut Config) -> anyhow::Result<()> { + let dockerfile = &self.dockerfile; + let image_name = &self.docker_img; + verify_admin_input(&dockerfile, &image_name)?; + check_dockerfile_valid(&dockerfile)?; + Ok(()) + } + + fn generate_scripts(&self, _: &Config) -> anyhow::Result { + // admin-container + match create_dir_all(ADMIN_CONTAINER_DIR) { + Ok(_) => { + // Dockerfile + let dockerfile_path = format!("{}/{}", ADMIN_CONTAINER_DIR, ADMIN_DOCKERFILE); + let mut dockerfile = File::create(&dockerfile_path)?; + gen_admin_dockerfile(&mut dockerfile)?; + set_permissions(&dockerfile_path, CONFIG_PERMISSION)?; + // set-ssh-pub-key.service + let set_ssh_pub_key_service_path = format!("{}/{}", ADMIN_CONTAINER_DIR, ADMIN_SET_SSH_PUB_KEY_SERVICE); + let mut set_ssh_pub_key_service = File::create(&set_ssh_pub_key_service_path)?; + gen_set_ssh_pub_key_service(&mut set_ssh_pub_key_service)?; + set_permissions(&set_ssh_pub_key_service_path, CONFIG_PERMISSION)?; + // set-ssh-pub-key.sh + let set_ssh_pub_key_path = format!("{}/{}", ADMIN_CONTAINER_DIR, ADMIN_SET_SSH_PUB_KEY_SH); + let mut set_ssh_pub_key = File::create(&set_ssh_pub_key_path)?; + gen_set_ssh_pub_key(&mut set_ssh_pub_key)?; + set_permissions(&set_ssh_pub_key_path, EXEC_PERMISSION)?; + }, + Err(e) => { + bail!(e); + }, + } + // kbimg.sh + let kbimg_path = format!("{}/{}", SCRIPTS_DIR, KBIMG_SH); + let mut kbimg = File::create(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH))?; + gen_admin_vars(&mut kbimg, &self.docker_img, &self.dockerfile)?; + gen_create_admin_img(&mut kbimg)?; + set_permissions(&kbimg_path, EXEC_PERMISSION)?; + + Ok(PathBuf::from(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH))) + } +} + +fn verify_admin_input(dockerfile: &PathBuf, image_name: &str) -> anyhow::Result<()> { + if !utils::is_valid_param(dockerfile.to_str().unwrap()) { + bail!("params {} is invalid, please check input", dockerfile.to_str().unwrap()); + } + if !utils::is_valid_param(image_name) { + bail!("params {} is invalid, please check input", image_name); + } + Ok(()) +} + +fn check_dockerfile_valid(dockerfile: &PathBuf) -> anyhow::Result<()> { + utils::is_file_valid("admin-container Dockerfile", dockerfile) +} diff --git a/KubeOS-Rust/kbimg/src/commands.rs b/KubeOS-Rust/kbimg/src/commands.rs new file mode 100644 index 00000000..f334f048 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/commands.rs @@ -0,0 +1,170 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::path::PathBuf; + +use clap::{Args, Parser, Subcommand}; +use serde::Deserialize; + +#[derive(Parser)] +#[clap(name = "kbimg")] +#[clap(author, version, about)] +#[clap(long_about = "A tool for creating KubeOS images.")] +pub struct Cli { + /// Path to the detailed configuration toml file + #[clap(short, long, value_parser)] + pub config: Option, + /// Enable debug mode, keep the scripts after execution + #[clap(short, long, action)] + pub debug: bool, + #[clap(subcommand)] + pub commands: Option, +} + +#[derive(Subcommand, Debug, Deserialize)] +pub enum Commands { + /// Create a new container image for upgrading KubeOS + #[clap(name = "upgrade")] + UpgradeImage(RepoInfo), + /// Create a new KubeOS vm image from repo + #[clap(name = "vm-repo")] + VMRepo(RepoInfo), + /// Create a new KubeOS vm image from docker image + #[clap(name = "vm-docker")] + VMDocker(DockerInfo), + /// Create a new KubeOS pxe image from repo + #[clap(name = "pxe-repo")] + PxeRepo(RepoInfo), + /// Create a new KubeOS pxe image from docker image + #[clap(name = "pxe-docker")] + PxeDocker(DockerInfo), + /// Create a KubeOS admin-container image + #[clap(name = "admin-container")] + AdminContainer(AdminContainerInfo), +} + +#[derive(Args, Debug, Deserialize, Clone)] +pub struct RepoInfo { + /// Required: KubeOS version + #[clap(short, long, value_parser)] + pub version: String, + /// Required: Repo path for installing packages + #[clap(short = 'p', long, value_parser)] + pub repo_path: PathBuf, + /// Required: Path to the agent binary + #[clap(short = 'b', long, value_parser)] + pub agent_path: PathBuf, + /// Required: Encrypted password for root user + #[clap(short = 'e', long, value_parser)] + pub root_passwd: String, + /// Required for upgrade + #[clap(short = 'd', long, value_parser)] + pub docker_img: Option, + /// Required: RPM packages + #[clap(short = 'r', long, value_parser)] + pub rpmlist: Vec, + /// Optional: boot mode, default is uefi, enable this flag for legacy bios + #[clap(short, long, value_parser)] + pub legacy_bios: bool, + #[clap(skip)] + pub image_type: String, + #[clap(skip)] + pub arch: Option, +} + +#[derive(Args, Debug, Deserialize, Clone)] +pub struct DockerInfo { + /// Required: Name of the container image + #[clap(short, long, value_parser)] + pub docker_img: String, + #[clap(skip)] + pub image_type: String, +} + +#[derive(Args, Debug, Deserialize, Clone)] +pub struct AdminContainerInfo { + /// Required: Name of the container image + #[clap(short, long, value_parser)] + pub docker_img: String, + /// Required: Path to the Dockerfile + #[clap(short, long, value_parser)] + pub dockerfile: PathBuf, +} + +#[derive(Debug, Deserialize, Default, Clone)] +pub struct Config { + pub from_repo: Option, + pub from_dockerimg: Option, + pub admin_container: Option, + pub users: Option>, + pub copy_files: Option>, + pub grub: Option, + pub systemd_service: Option, + pub chroot_script: Option, + pub disk_partition: Option, + pub persist_mkdir: Option, + pub pxe_config: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct User { + pub name: String, + pub passwd: String, + pub groups: Option>, + pub sudo: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CopyFile { + pub src: String, + pub dst: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Grub { + pub passwd: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct SystemdService { + pub name: Vec, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct ChrootScript { + pub path: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct DiskPartition { + pub first: u32, + pub second: u32, + pub third: u32, + pub img_size: u32, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct PersistMkdir { + pub name: Vec, +} + +// pxe config +#[derive(Debug, Deserialize, Clone)] +pub struct PxeConfig { + pub rootfs_name: String, + pub disk: String, + pub server_ip: String, + pub local_ip: String, + pub route_ip: String, + pub netmask: String, + pub net_name: String, +} diff --git a/KubeOS-Rust/kbimg/src/docker_img.rs b/KubeOS-Rust/kbimg/src/docker_img.rs new file mode 100644 index 00000000..9c5a8c20 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/docker_img.rs @@ -0,0 +1,74 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs::File, path::PathBuf, process::Command, str}; + +use anyhow::bail; + +use crate::{ + commands::DockerInfo, + scripts_gen::*, + utils::{self, set_permissions}, + values::*, + Config, CreateImage, +}; + +impl CreateImage for DockerInfo { + fn prepare(&self, _: &mut Config) -> anyhow::Result<()> { + let image_name = &self.docker_img; + verify_docker_input(&image_name)?; + check_docker_image(&image_name)?; + Ok(()) + } + + fn generate_scripts(&self, config: &Config) -> anyhow::Result { + // kbimg.sh + let kbimg_path = format!("{}/{}", SCRIPTS_DIR, KBIMG_SH); + let mut kbimg = File::create(&kbimg_path)?; + gen_global_vars(&mut kbimg)?; + gen_docker_vars(&mut kbimg, &self.docker_img)?; + gen_global_func(&mut kbimg)?; + gen_create_os_tar_from_docker(&mut kbimg)?; + if self.image_type == "vm-docker" { + // kbimg.sh + gen_init_part(&mut kbimg)?; + gen_create_img(&mut kbimg, false, &config)?; + gen_create_vm_docker_img(&mut kbimg)?; + } else { + // kbimg.sh + gen_create_pxe_docker_img(&mut kbimg)?; + } + set_permissions(&kbimg_path, EXEC_PERMISSION)?; + + Ok(PathBuf::from(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH))) + } +} + +fn verify_docker_input(image_name: &str) -> anyhow::Result<()> { + if !utils::is_valid_param(image_name) { + bail!("params {} is invalid, please check input", image_name); + } + Ok(()) +} + +fn check_docker_image(image_name: &str) -> anyhow::Result<()> { + let output = + Command::new("docker").args(&["images", "-q", image_name]).output().expect("Failed to execute command"); + + if output.status.success() { + let stdout = str::from_utf8(&output.stdout).expect("Invalid UTF-8 output"); + if stdout.trim().is_empty() { + bail!("docker image does NOT exist, please pull {} first.", image_name); + } + } + Ok(()) +} diff --git a/KubeOS-Rust/kbimg/src/main.rs b/KubeOS-Rust/kbimg/src/main.rs new file mode 100644 index 00000000..beb389d5 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/main.rs @@ -0,0 +1,122 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs, path::PathBuf, process::exit}; + +use anyhow::{bail, Result}; +use clap::Parser; +use env_logger::{Builder, Env, Target}; +use log::{debug, error, info}; + +mod admin_container; +mod commands; +mod docker_img; +mod repo; +mod scripts_gen; +mod utils; +mod values; + +use utils::{execute_scripts, get_arch}; +use values::SCRIPTS_DIR; + +use crate::commands::{Cli, Commands, Config}; + +trait CreateImage { + /// validate cmd args, check disk size and other prepare work + fn prepare(&self, config: &mut Config) -> Result<()>; + /// generate scripts for creating image. If debug is enabled, keep the scripts, otherwise execute them + fn generate_scripts(&self, config: &Config) -> Result; +} + +fn process(info: Box, mut config: Config) -> Result<()> { + match fs::create_dir_all(SCRIPTS_DIR) { + Ok(_) => { + info.prepare(&mut config)?; + let path = info.generate_scripts(&config)?; + execute_scripts(path)?; + Ok(()) + }, + Err(e) => bail!(e), + } +} + +fn main() { + let cli = Cli::parse(); + let default_log_level: &str = if cli.debug { "debug" } else { "info" }; + Builder::from_env(Env::default().default_filter_or(default_log_level)).target(Target::Stdout).init(); + match cli.config { + Some(config) => { + info!("Loading config file"); + debug!("Config file path: {:?}", config); + let content = fs::read_to_string(config).unwrap(); + let data: Config = toml::from_str(&content).unwrap(); + debug!("Config: {:?}", data); + let info = if let Some(mut info) = data.from_repo.clone() { + info.arch = Some(get_arch()); + Some(Box::new(info) as Box) + } else if let Some(info) = data.from_dockerimg.clone() { + Some(Box::new(info) as Box) + } else if let Some(info) = data.admin_container.clone() { + Some(Box::new(info) as Box) + } else { + None + }; + if let Some(i) = info { + match process(i, data) { + Ok(_) => { + info!("Image created successfully"); + }, + Err(e) => { + error!("Failed to create image: {:?}", e); + }, + } + } + exit(0); + }, + None => {}, + } + let info = match cli.commands { + Some(Commands::UpgradeImage(mut info)) => { + info.image_type = "upgrade".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::VMRepo(mut info)) => { + info.image_type = "vm-repo".to_string(); + debug!("VMRepo: {:?}", info); + Some(Box::new(info) as Box) + }, + Some(Commands::VMDocker(mut info)) => { + info.image_type = "vm-docker".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::PxeRepo(mut info)) => { + info.image_type = "pxe-repo".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::PxeDocker(mut info)) => { + info.image_type = "pxe-docker".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::AdminContainer(info)) => Some(Box::new(info) as Box), + None => None, + }; + if let Some(i) = info { + match process(i, Config::default()) { + Ok(_) => { + info!("Image created successfully"); + }, + Err(e) => { + error!("Failed to create image: {:?}", e); + }, + } + } +} diff --git a/KubeOS-Rust/kbimg/src/repo.rs b/KubeOS-Rust/kbimg/src/repo.rs new file mode 100644 index 00000000..43aabd3b --- /dev/null +++ b/KubeOS-Rust/kbimg/src/repo.rs @@ -0,0 +1,203 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + env, + fs::{create_dir_all, File}, + path::PathBuf, +}; + +use anyhow::bail; +use sysinfo::{System, SystemExt, DiskExt}; + +use crate::{ + commands::RepoInfo, + scripts_gen::*, + utils::{self, check_pxe_conf_valid, set_permissions}, + values::*, + Config, CreateImage, +}; + +impl CreateImage for RepoInfo { + fn prepare(&self, config: &mut Config) -> anyhow::Result<()> { + verify_repo_input(&self)?; + check_disk_space(&self.image_type)?; + check_repo_file_valid(&self.repo_path)?; + check_agent_file_valid(&self.agent_path)?; + if self.image_type == "pxe-repo" { + if let Some(pxe_config) = &config.pxe_config { + check_pxe_conf_valid(&pxe_config)?; + } else { + bail!("pxe config not found!") + } + } + Ok(()) + } + + fn generate_scripts(&self, config: &Config) -> anyhow::Result { + // rpmlist + let rpmlist_path = format!("{}/{}", SCRIPTS_DIR, RPMLIST); + let mut rpmlist = File::create(&rpmlist_path)?; + gen_rpm_list(&mut rpmlist, &self.rpmlist)?; + set_permissions(&rpmlist_path, CONFIG_PERMISSION)?; + // 00bootup + match create_dir_all(BOOTUP_DIR) { + Ok(_) => { + if let Some(pxe_config) = &config.pxe_config { + let global_cfg_path = format!("{}/{}", BOOTUP_DIR, BOOTUP_GLOBAL_CFG); + let mut global_cfg = File::create(&global_cfg_path)?; + gen_global_cfg(&mut global_cfg, &pxe_config)?; + set_permissions(&global_cfg_path, CONFIG_PERMISSION)?; + } + let module_setup_path = format!("{}/{}", BOOTUP_DIR, BOOTUP_MODULE_SETUP_SH); + let mut module_setup = File::create(&module_setup_path)?; + gen_module_setup(&mut module_setup)?; + set_permissions(&module_setup_path, EXEC_PERMISSION)?; + let mount_path = format!("{}/{}", BOOTUP_DIR, BOOTUP_MOUNT_SH); + let mut mount = File::create(&mount_path)?; + gen_mount(&mut mount)?; + set_permissions(&mount_path, EXEC_PERMISSION)?; + }, + Err(e) => { + bail!(e); + }, + } + // misc-files + match create_dir_all(MISC_FILES_DIR) { + Ok(_) => { + let boot_efi_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_BOOT_EFI_MOUNT); + let mut boot_efi_mount = File::create(&boot_efi_mount_path)?; + gen_boot_efi_mount(&mut boot_efi_mount)?; + set_permissions(&boot_efi_mount_path, CONFIG_PERMISSION)?; + let boot_grub2_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_BOOT_GRUB2_MOUNT); + let mut boot_grub2_mount = File::create(&boot_grub2_mount_path)?; + gen_boot_grub2_mount(&mut boot_grub2_mount)?; + set_permissions(&boot_grub2_mount_path, CONFIG_PERMISSION)?; + let etc_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_ETC_MOUNT); + let mut etc_mount = File::create(&etc_mount_path)?; + gen_etc_mount(&mut etc_mount)?; + set_permissions(&etc_mount_path, CONFIG_PERMISSION)?; + let os_agent_service_path = format!("{}/{}", MISC_FILES_DIR, MISC_OS_AGENT_SERVICE); + let mut os_agent_service = File::create(&os_agent_service_path)?; + gen_os_agent_service(&mut os_agent_service)?; + set_permissions(&os_agent_service_path, CONFIG_PERMISSION)?; + let os_release_path = format!("{}/{}", MISC_FILES_DIR, MISC_OS_RELEASE); + let mut os_release = File::create(&os_release_path)?; + gen_os_release(&mut os_release)?; + set_permissions(&os_release_path, CONFIG_PERMISSION)?; + let persist_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_PERSIST_MOUNT); + let mut persist_mount = File::create(&persist_mount_path)?; + gen_persist_mount(&mut persist_mount)?; + set_permissions(&persist_mount_path, CONFIG_PERMISSION)?; + let var_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_VAR_MOUNT); + let mut var_mount = File::create(&var_mount_path)?; + gen_var_mount(&mut var_mount)?; + set_permissions(&var_mount_path, CONFIG_PERMISSION)?; + }, + Err(e) => { + bail!(e); + }, + } + // grub.cfg + let grub_cfg_path = format!("{}/{}", SCRIPTS_DIR, GRUB_CFG); + let mut grub_cfg = File::create(&grub_cfg_path)?; + gen_grub_cfg(&mut grub_cfg)?; + set_permissions(&grub_cfg_path, CONFIG_PERMISSION)?; + // set_in_chroot.sh + let set_in_chroot_path = format!("{}/{}", SCRIPTS_DIR, SET_IN_CHROOT_SH); + let mut set_in_chroot = File::create(&set_in_chroot_path)?; + gen_set_in_chroot(&mut set_in_chroot, self.legacy_bios, &config)?; + set_permissions(&set_in_chroot_path, EXEC_PERMISSION)?; + // kbimg.sh + let kbimg_path = format!("{}/{}", SCRIPTS_DIR, KBIMG_SH); + let mut kbimg = File::create(&kbimg_path)?; + gen_global_vars(&mut kbimg)?; + gen_repo_vars(&mut kbimg, &self)?; + gen_global_func(&mut kbimg)?; + gen_mount_proc_dev_sys(&mut kbimg)?; + gen_unmount_dir(&mut kbimg)?; + gen_create_os_tar_from_repo(&mut kbimg, &self, &config)?; + if self.image_type == "vm-repo" { + // bootloader.sh + let bootloader_path = format!("{}/{}", SCRIPTS_DIR, BOOTLOADER_SH); + let mut bootloader = File::create(&bootloader_path)?; + gen_bootloader(&mut bootloader, self.arch.as_ref().unwrap(), self.legacy_bios)?; + set_permissions(&bootloader_path, EXEC_PERMISSION)?; + // kbimg.sh + gen_init_part(&mut kbimg)?; + gen_create_img(&mut kbimg, self.legacy_bios, &config)?; + gen_create_vm_repo_img(&mut kbimg)?; + } else if self.image_type == "pxe-repo" { + // kbimg.sh + gen_create_pxe_repo_img(&mut kbimg)?; + } else { + // Dockerfile + let dockerfile_path = format!("{}/{}", SCRIPTS_DIR, DOCKERFILE); + let mut dockerfile = File::create(&dockerfile_path)?; + gen_dockerfile(&mut dockerfile)?; + set_permissions(&dockerfile_path, CONFIG_PERMISSION)?; + // kbimg.sh + gen_create_docker_img(&mut kbimg)?; + } + set_permissions(&kbimg_path, EXEC_PERMISSION)?; + + Ok(PathBuf::from(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH))) + } +} + +fn verify_repo_input(info: &RepoInfo) -> anyhow::Result<()> { + if !utils::is_valid_param(info.repo_path.to_str().unwrap()) { + bail!("params {} is invalid, please check input", info.repo_path.to_str().unwrap()); + } + if !utils::is_valid_param(&info.version) { + bail!("params {} is invalid, please check input", info.version); + } + if !utils::is_valid_param(info.agent_path.to_str().unwrap()) { + bail!("params {} is invalid, please check input", info.agent_path.to_str().unwrap()); + } + if let Some(docker_img) = &info.docker_img { + if !utils::is_valid_param(docker_img) { + bail!("params {} is invalid, please check input", docker_img); + } + } + Ok(()) +} + +fn check_disk_space(image_type: &str) -> anyhow::Result<()> { + let max_size: u64 = match image_type { + "upgrade" => 6, + "vm-repo" => 25, + "pxe-repo" => 5, + _ => bail!("Invalid image type: {}", image_type), + }; + + let current_dir = env::current_dir().expect("Failed to get current directory"); + let root_dir = current_dir.ancestors().last().expect("Failed to get current directory").to_path_buf(); + let mut sys = System::new_all(); + sys.refresh_all(); + for d in sys.disks() { + if d.mount_point() == root_dir { + if d.available_space() < max_size * 1024 * 1024 { + bail!("The available disk space is not enough, at least {}GiB.", max_size); + } + } + } + Ok(()) +} + +fn check_repo_file_valid(repo_path: &PathBuf) -> anyhow::Result<()> { + utils::is_file_valid("REPO file", repo_path) +} + +fn check_agent_file_valid(agent_path: &PathBuf) -> anyhow::Result<()> { + utils::is_file_valid("os-agent binary", agent_path) +} diff --git a/KubeOS-Rust/kbimg/src/scripts_gen.rs b/KubeOS-Rust/kbimg/src/scripts_gen.rs new file mode 100644 index 00000000..7d574888 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/scripts_gen.rs @@ -0,0 +1,1754 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs::File, io::Write, path::PathBuf}; + +use anyhow::{bail, Ok, Result}; + +use crate::{commands::*, values::SCRIPTS_DIR}; + +/* copyright */ +pub(crate) fn gen_copyright(file: &mut File) -> Result<()> { + writeln!( + file, + r#"## Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +# KubeOS is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +## See the Mulan PSL v2 for more details. +"# + )?; + + Ok(()) +} + +/* region: kbimg.sh */ +pub(crate) fn gen_global_vars(file: &mut File) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"set -e + +SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd) +LOCK="${{SCRIPTS_DIR}}"/test.lock +RPM_ROOT="${{SCRIPTS_DIR}}"/rootfs +TMP_MOUNT_PATH="${{SCRIPTS_DIR}}"/mnt +"# + )?; + Ok(()) +} + +pub(crate) fn gen_global_func(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function delete_dir() {{ + local ret=0 + local dir="$1" + unmount_dir "${{dir}}" + ret=$? + if [ "${{ret}}" -eq 0 ]; then + rm -rf "${{dir}}" + return 0 + else + log_error_print "${{dir}} is failed to unmount , can not delete ${{dir}}." + return 1 + fi +}} + +function delete_file() {{ + local file="$1" + if [ ! -e "${{file}}" ]; then + return 0 + fi + + if [ ! -f "${{file}}" ]; then + log_error_print "${{file}} is not a file." + return 1 + fi + + rm -f "${{file}}" + return 0 +}} + +function clean_space() {{ + delete_dir "${{RPM_ROOT}}" + delete_dir "${{TMP_MOUNT_PATH}}" + delete_file "${{SCRIPTS_DIR}}"/os.tar + rm -rf "${{LOCK}}" + delete_file "${{ADMIN_CONTAINER_DIR}}"/hostshell +}} + +function clean_img() {{ + delete_file "${{SCRIPTS_DIR}}"/system.img + delete_file "${{SCRIPTS_DIR}}"/update.img + delete_file "${{SCRIPTS_DIR}}"/initramfs.img + delete_file "${{SCRIPTS_DIR}}"/kubeos.tar +}} + +function file_lock() {{ + local lock_file=$1 + exec {{lock_fd}}>"${{lock_file}}" + flock -xn "${{lock_fd}}" +}} + +function test_lock() {{ + file_lock "${{LOCK}}" + local status=$? + if [ $status -ne 0 ]; then + log_error_print "There is already an generate process running." + exit 203 + fi +}} + +function log_error_print() {{ + local logmsg + logmsg="[ ERROR ] - ""$(date "+%b %d %Y %H:%M:%S")"" $1" + echo "$logmsg" +}} + +function log_info_print() {{ + local logmsg + logmsg="[ INFO ] - ""$(date "+%b %d %Y %H:%M:%S")"" $1" + echo "$logmsg" +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_mount_proc_dev_sys(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function mount_proc_dev_sys() {{ + local tmp_root=$1 + mount -t proc none "${{tmp_root}}"/proc + mount --bind /dev "${{tmp_root}}"/dev + mount --bind /dev/pts "${{tmp_root}}"/dev/pts + mount -t sysfs none "${{tmp_root}}"/sys +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_unmount_dir(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function unmount_dir() {{ + local dir=$1 + + if [ -L "${{dir}}" ] || [ -f "${{dir}}" ]; then + log_error_print "${{dir}} is not a directory, please check it." + return 1 + fi + + if [ ! -d "${{dir}}" ]; then + return 0 + fi + + local real_dir + real_dir=$(readlink -e "${{dir}}") + local mnts + mnts=$(awk '{{print $2}}' < /proc/mounts | grep "^${{real_dir}}" | sort -r) + for m in ${{mnts}}; do + log_info_print "Unmount ${{m}}" + umount -f "${{m}}" || true + done + + return 0 +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_init_part(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function init_part() {{ + local offset + offset=$(fdisk -l "${{SCRIPTS_DIR}}"/system.img | grep "$1" | awk '{{print $2}}') + local sizelimit + sizelimit=$(fdisk -l "${{SCRIPTS_DIR}}"/system.img | grep "$1" | awk '{{print $3}}') + sizelimit=$(echo "($sizelimit - $offset)*512" | bc) + offset=$(echo "${{offset}}*512" | bc) + local loop + loop=$(losetup -f) + losetup -o "${{offset}}" --sizelimit "${{sizelimit}}" "${{loop}}" "${{SCRIPTS_DIR}}"/system.img + if [ "$2" == "BOOT" ];then + mkfs.vfat -n "$2" "${{loop}}" + mount -t vfat "${{loop}}" "$3" + else + mkfs.ext4 -L "$2" "${{loop}}" + mount -t ext4 "${{loop}}" "$3" + rm -rf "$3/lost+found" + fi +}} +"# + )?; + Ok(()) +} + +// repo +pub(crate) fn gen_repo_vars(file: &mut File, info: &RepoInfo) -> Result<()> { + writeln!( + file, + r#"REPO_PATH="{}" +VERSION="{}" +AGENT_PATH="{}" +ROOT_PASSWD='{}' +BOOT_MODE="{}" +"#, + info.repo_path.to_str().unwrap(), + &info.version, + info.agent_path.to_str().unwrap(), + &info.root_passwd, + if info.legacy_bios { "legacy" } else { "efi" } + )?; + if let Some(docker_img) = &info.docker_img { + writeln!(file, "DOCKER_IMG=\"{}\"\n", docker_img)?; + } + Ok(()) +} + +pub(crate) fn gen_prepare_yum(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function prepare_yum() {{ + # init rpmdb + rpm --root "${{RPM_ROOT}}" --initdb + mkdir -p "${{RPM_ROOT}}"{{/etc/yum.repos.d,/persist,/proc,/dev/pts,/sys}} + mount_proc_dev_sys "${{RPM_ROOT}}" + # init yum repo + local iso_repo="${{RPM_ROOT}}"/etc/yum.repos.d/iso.repo + cat "${{REPO_PATH}}" > "$iso_repo" +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_install_packages(file: &mut File, arch: &str, legacy_bios: bool) -> Result<()> { + writeln!( + file, + r#"function install_packages() {{ + prepare_yum "${{REPO_PATH}}" + + echo "install package.." + + local filesize + filesize=$(stat -c "%s" "${{SCRIPTS_DIR}}"/rpmlist) + local maxsize=$((1024*1024)) + if [ "${{filesize}}" -gt "${{maxsize}}" ]; then + echo "please check if rpmlist is too big or something wrong" + exit 7 + fi + + local rpms_name + rpms_name=$(tr "\n" " " < "${{SCRIPTS_DIR}}"/rpmlist) + old_ifs="$IFS" + IFS=' '"# + )?; + + if arch == "x86_64" { + if legacy_bios { + writeln!(file, "\trpms_name+=\" grub2\"")?; + } else { + writeln!(file, "\trpms_name+=\" grub2-efi grub2-tools grub2-efi-x64-modules grub2-pc-modules\"")?; + } + writeln!( + file, + r#" read -ra rpms <<< "${{rpms_name}}" + IFS="$old_ifs" + yum -y --installroot="${{RPM_ROOT}}" install --nogpgcheck --setopt install_weak_deps=False "${{rpms[@]}}""# + )?; + } else if arch == "aarch64" { + writeln!( + file, + r#" read -ra rpms <<< "${{rpms_name}}" + IFS="$old_ifs" + yum -y --installroot="${{RPM_ROOT}}" install --nogpgcheck --setopt install_weak_deps=False "${{rpms[@]}}" grub2-efi grub2-tools grub2-efi-aa64-modules"# + )?; + } + writeln!( + file, + r#" yum -y --installroot="${{RPM_ROOT}}" clean all +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_copy_files(file: &mut File, copy_files: &Vec) -> Result<()> { + writeln!(file, "function copy_files() {{")?; + for copy_file in copy_files { + let dst = format!("{}/rootfs{}", SCRIPTS_DIR, ©_file.dst); + let dst = PathBuf::from(dst); + if !dst.exists() { + writeln!(file, "\tmkdir -p \"${{RPM_ROOT}}{}\"", ©_file.dst)?; + } + let src = PathBuf::from(©_file.src); + if src.is_dir() { + writeln!(file, "\tcp -r {} \"${{RPM_ROOT}}{}\"", ©_file.src, ©_file.dst)?; + } else { + writeln!(file, "\tcp {} \"${{RPM_ROOT}}{}\"", ©_file.src, ©_file.dst)?; + } + } + writeln!(file, "}}\n")?; + Ok(()) +} + +pub(crate) fn gen_grub_config(file: &mut File, legacy_bios: bool, grub: &Grub) -> Result<()> { + writeln!( + file, + r#"function grub_config() {{ + local GRUB_PATH"# + )?; + if legacy_bios { + writeln!(file, "\tGRUB_PATH=\"${{RPM_ROOT}}\"/boot/grub2")?; + } else { + writeln!(file, "\tGRUB_PATH=\"${{RPM_ROOT}}\"/boot/efi/EFI/openEuler")?; + } + if let Some(grub_passwd) = &grub.passwd { + writeln!( + file, + r#" local GRUB_PASSWD + GRUB_PASSWD=$(echo -e "{}\n{}" | grub2-mkpasswd-pbkdf2 | grep PBKDF2 | awk '{{print $7}}') + echo "GRUB2_PASSWD=${{GRUB_PASSWD}}" > "${{GRUB_PATH}}"/user.cfg + chmod 600 "${{GRUB_PATH}}"/user.cfg +}} +"#, + grub_passwd, grub_passwd + )?; + } + Ok(()) +} + +pub(crate) fn gen_chroot_script(file: &mut File, chroot_script: &ChrootScript) -> Result<()> { + let script_path = PathBuf::from(&chroot_script.path); + match script_path.canonicalize() { + core::result::Result::Ok(absolute_path) => { + if let Some(script_name) = absolute_path.file_name() { + writeln!( + file, + r#"function chroot_script() {{ + cp "{}" "${{RPM_ROOT}}" + chroot "${{RPM_ROOT}}" bash /{} +}} +"#, + absolute_path.as_path().to_str().unwrap(), + script_name.to_str().unwrap() + )?; + } + Ok(()) + }, + Err(e) => bail!(e), + } +} + +pub(crate) fn gen_install_misc(file: &mut File, legacy_bios: bool, config: &Config) -> Result<()> { + if let Some(copy_files) = &config.copy_files { + gen_copy_files(file, ©_files)?; + } + if let Some(grub) = &config.grub { + gen_grub_config(file, legacy_bios, &grub)?; + } + if let Some(chroot_script) = &config.chroot_script { + gen_chroot_script(file, &chroot_script)?; + } + + writeln!( + file, + r#"function install_misc() {{ + cp "${{SCRIPTS_DIR}}"/misc-files/*mount "${{SCRIPTS_DIR}}"/misc-files/os-agent.service "${{RPM_ROOT}}"/usr/lib/systemd/system/ + cp "${{SCRIPTS_DIR}}"/misc-files/os-release "${{RPM_ROOT}}"/usr/lib/ + cp "${{AGENT_PATH}}" "${{RPM_ROOT}}"/usr/bin + rm "${{RPM_ROOT}}"/etc/os-release + + cat < "${{RPM_ROOT}}"/usr/lib/os-release +NAME=${{NAME}} +ID=${{NAME}} +EOF + echo "PRETTY_NAME=\"${{NAME}} ${{VERSION}}\"" >> "${{RPM_ROOT}}"/usr/lib/os-release + echo "VERSION_ID=${{VERSION}}" >> "${{RPM_ROOT}}"/usr/lib/os-release + mv "${{RPM_ROOT}}"/boot/vmlinuz* "${{RPM_ROOT}}"/boot/vmlinuz + mv "${{RPM_ROOT}}"/boot/initramfs* "${{RPM_ROOT}}"/boot/initramfs.img"# + )?; + + if legacy_bios { + writeln!( + file, + r#" cp "${{SCRIPTS_DIR}}"/grub.cfg "${{RPM_ROOT}}"/boot/grub2 + sed -i "s/insmod part_gpt/insmod part_msdos/g; \ +s/set root='hd0,gpt2'/set root='hd0,msdos2'/g; \ +s/set root='hd0,gpt3'/set root='hd0,msdos3'/g" \ +"${{RPM_ROOT}}"/boot/grub2/grub.cfg"# + )?; + } else { + writeln!(file, "\tcp \"${{SCRIPTS_DIR}}\"/grub.cfg \"${{RPM_ROOT}}\"/boot/efi/EFI/openEuler")?; + } + + writeln!( + file, + r#" cp -r "${{SCRIPTS_DIR}}"/00bootup "${{RPM_ROOT}}"/usr/lib/dracut/modules.d/ + cp "${{SCRIPTS_DIR}}"/set_in_chroot.sh "${{RPM_ROOT}}" + + # (optional) custom config"# + )?; + + if let Some(_) = &config.copy_files { + writeln!(file, "\tcopy_files")?; + } + if let Some(_) = &config.grub { + writeln!(file, "\tgrub_config")?; + } + if let Some(_) = &config.chroot_script { + writeln!(file, "\tchroot_script")?; + } + + writeln!( + file, + r#" + ROOT_PASSWD="${{ROOT_PASSWD}}" BOOT_MODE="${{BOOT_MODE}}" chroot "${{RPM_ROOT}}" bash /set_in_chroot.sh + rm "${{RPM_ROOT}}/set_in_chroot.sh" +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_os_tar_from_repo(file: &mut File, info: &RepoInfo, config: &Config) -> Result<()> { + gen_prepare_yum(file)?; + gen_install_packages(file, info.arch.as_ref().unwrap(), info.legacy_bios)?; + gen_install_misc(file, info.legacy_bios, config)?; + + writeln!( + file, + r#"function create_os_tar_from_repo() {{ + install_packages + install_misc + unmount_dir "${{RPM_ROOT}}" + tar -C "${{RPM_ROOT}}" -cf "${{SCRIPTS_DIR}}"/os.tar . +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_img(file: &mut File, legacy_bios: bool, config: &Config) -> Result<()> { + let (first, second, third, img_size) = if let Some(disk_partition) = &config.disk_partition { + let first = disk_partition.first; + let second = disk_partition.second; + let third = disk_partition.third; + let img_size = disk_partition.img_size; + if first + second + third + 2100 > img_size * 1024 { + bail!("Image size({}G) is not enough for partitions, please check input", img_size) + } + (first, first + second, first + second + third, img_size) + } else { + (60, 2160, 4260, 20) + }; + + writeln!( + file, + r#"function create_img() {{ + rm -f "${{SCRIPTS_DIR}}"/system.img "${{SCRIPTS_DIR}}/update.img" + qemu-img create "${{SCRIPTS_DIR}}/system.img" {}G"#, + img_size + )?; + + if legacy_bios { + writeln!( + file, + r#" local BOOT_PATH=${{TMP_MOUNT_PATH}}/boot/grub2 + parted "${{SCRIPTS_DIR}}/system.img" -s mklabel msdos + parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 1MiB {}MiB"#, + first + )?; + } else { + writeln!( + file, + r#" local BOOT_PATH=${{TMP_MOUNT_PATH}}/boot/efi + parted "${{SCRIPTS_DIR}}/system.img" -s mklabel gpt + parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary fat32 1MiB {}MiB"#, + first + )?; + } + + writeln!( + file, + r#" parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 {}MiB {}MiB + parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 {}MiB {}MiB + parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 {}MiB 100%"#, + first, second, second, third, third + )?; + + writeln!( + file, + r#" local device + device=$(losetup -f) + losetup "${{device}}" "${{SCRIPTS_DIR}}"/system.img + + mkdir -p "${{TMP_MOUNT_PATH}}" + + init_part "${{SCRIPTS_DIR}}"/system.img2 ROOT-A "${{TMP_MOUNT_PATH}}" + + mkdir -p "${{BOOT_PATH}}" + chmod 755 "${{BOOT_PATH}}""# + )?; + + if legacy_bios { + writeln!( + file, + r#" init_part "${{SCRIPTS_DIR}}"/system.img1 GRUB2 "${{BOOT_PATH}}" + tar -x -C "${{TMP_MOUNT_PATH}}" -f "${{SCRIPTS_DIR}}"/os.tar + sed -i "s/insmod part_gpt/insmod part_msdos/g; \ +s/set root='hd0,gpt2'/set root='hd0,msdos2'/g; \ +s/set root='hd0,gpt3'/set root='hd0,msdos3'/g" \ +"${{TMP_MOUNT_PATH}}"/boot/grub2/grub.cfg"# + )?; + } else { + writeln!( + file, + r#" init_part "${{SCRIPTS_DIR}}"/system.img1 BOOT "${{BOOT_PATH}}" + tar -x -C "${{TMP_MOUNT_PATH}}" -f "${{SCRIPTS_DIR}}"/os.tar"# + )?; + } + + writeln!( + file, + r#" sync + cp "${{SCRIPTS_DIR}}"/bootloader.sh "${{TMP_MOUNT_PATH}}" + mount_proc_dev_sys "${{TMP_MOUNT_PATH}}" + DEVICE="${{device}}" BOOT_MODE="${{BOOT_MODE}}" chroot "${{TMP_MOUNT_PATH}}" bash bootloader.sh + rm -rf "${{TMP_MOUNT_PATH}}"/bootloader.sh + sync + + dd if=/dev/disk/by-label/ROOT-A of="${{SCRIPTS_DIR}}"/update.img bs=8M + sync + unmount_dir "${{TMP_MOUNT_PATH}}" + init_part "${{SCRIPTS_DIR}}"/system.img3 ROOT-B "${{TMP_MOUNT_PATH}}" + umount "${{TMP_MOUNT_PATH}}" + + init_part "${{SCRIPTS_DIR}}"/system.img4 PERSIST "${{TMP_MOUNT_PATH}}" + mkdir "${{TMP_MOUNT_PATH}}"/{{var,etc,etcwork}}"# + )?; + + if let Some(persist_mkdir) = &config.persist_mkdir { + for name in &persist_mkdir.name { + writeln!(file, "\tmkdir \"${{TMP_MOUNT_PATH}}\"/{}", name)?; + } + } + + writeln!( + file, + r#" mkdir -p "${{TMP_MOUNT_PATH}}"/etc/KubeOS/certs + umount "${{TMP_MOUNT_PATH}}" + + losetup -D + parted "${{SCRIPTS_DIR}}"/system.img -- set 1 boot on + qemu-img convert "${{SCRIPTS_DIR}}"/system.img -O qcow2 "${{SCRIPTS_DIR}}"/system.qcow2 +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_vm_repo_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_vm_repo_img() {{ + create_os_tar_from_repo + create_img +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_vm_repo_img"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_pxe_repo_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_pxe_repo_img() {{ + rm -rf "${{SCRIPTS_DIR}}"/initramfs.img "${{SCRIPTS_DIR}}"/kubeos.tar + create_os_tar_from_repo + tar -xvf "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/initramfs.img + mv "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/kubeos.tar +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_pxe_repo_img"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_docker_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_docker_img() {{ + create_os_tar_from_repo + docker build -t "${{DOCKER_IMG}}" -f "${{SCRIPTS_DIR}}"/Dockerfile . +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_docker_img"# + )?; + Ok(()) +} + +// docker +pub(crate) fn gen_docker_vars(file: &mut File, image_name: &str) -> Result<()> { + writeln!( + file, + r#" +IMAGE_NAME="{}" +BOOT_MODE=efi +"#, + image_name + )?; + Ok(()) +} + +pub(crate) fn gen_create_os_tar_from_docker(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_os_tar_from_docker() {{ + container_id=$(docker create "${{DOCKER_IMG}}") + echo "$container_id" + docker cp "$container_id":/os.tar "${{SCRIPTS_DIR}}" + docker rm "$container_id" +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_vm_docker_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_vm_docker_img() {{ + create_os_tar_from_docker + create_img +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_vm_docker_img"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_pxe_docker_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_pxe_docker_img() {{ + rm -rf "${{SCRIPTS_DIR}}"/initramfs.img "${{SCRIPTS_DIR}}"/kubeos.tar + create_os_tar_from_docker + tar -xvf "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/initramfs.img + mv "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/kubeos.tar +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_pxe_docker_img"# + )?; + Ok(()) +} + +// admin +pub(crate) fn gen_admin_vars(file: &mut File, docker_img: &str, dockerfile: &PathBuf) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"set -e + +DOCKER_IMG={} +DOCKERFILE={} +ADMIN_CONTAINER_DIR="${{SCRIPTS_DIR}}"/admin-container +"#, + dockerfile.to_str().unwrap(), + docker_img + )?; + Ok(()) +} + +pub(crate) fn gen_create_admin_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_admin_img() {{ + local kubeos_root_dir=$(dirname $(dirname $(dirname "${{SCRIPTS_DIR}}"))) + cp "${{kubeos_root_dir}}"/bin/hostshell "${{ADMIN_CONTAINER_DIR}}" + docker build -t "${{DOCKER_IMG}}" -f "${{DOCKERFILE}}" "${{ADMIN_CONTAINER_DIR}}" + rm -rf "${{ADMIN_CONTAINER_DIR}}"/hostshell +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_admin_img"# + )?; + Ok(()) +} +/* endregion */ + +/* region: set_in_chroot.sh */ +pub(crate) fn gen_add_users(file: &mut File, users: &Vec) -> Result<()> { + writeln!(file, "# add users")?; + for user in users { + let name = &user.name; + let passwd = &user.passwd; + let groups = match user.groups.clone() { + Some(groups) => groups, + None => vec![name.clone()], + }; + for group in &groups { + writeln!( + file, + r#"if ! getent group "{}" > /dev/null 2>&1; then + groupadd "{}" +fi"#, + group, group + )?; + } + write!(file, "useradd -m -g {}", &groups[0])?; + if groups.len() > 1 { + let additional_groups = &groups[1..].join(","); + write!(file, " -G {}", additional_groups)?; + } + writeln!(file, " -s /bin/bash \"{}\"", &name)?; + writeln!(file, "echo \"{}:{}\" | chpasswd", name, passwd)?; + if let Some(sudo) = &user.sudo { + writeln!( + file, + r#"if visudo -c; then + echo -e "{} {}" | tee -a /etc/sudoers +else + echo "Sudoers file syntax check failed. Please fix the sudoers file manually." + exit 5 +fi"#, + name, sudo + )?; + } + } + Ok(()) +} + +pub(crate) fn gen_systemd_services(file: &mut File, systemd_services: &SystemdService) -> Result<()> { + writeln!(file, "# systemd")?; + for service_name in &systemd_services.name { + writeln!(file, "systemctl enable {}", service_name)?; + } + Ok(()) +} + +pub(crate) fn gen_set_in_chroot(file: &mut File, legacy_bios: bool, config: &Config) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"ln -s /usr/lib/systemd/system/os-agent.service /usr/lib/systemd/system/multi-user.target.wants/os-agent.service +ln -s /usr/lib/systemd/system/kubelet.service /usr/lib/systemd/system/multi-user.target.wants/kubelet.service"# + )?; + if legacy_bios { + writeln!( + file, + "ln -s /usr/lib/systemd/system/boot-grub2.mount /lib/systemd/system/local-fs.target.wants/boot-grub2.mount" + )?; + } else { + writeln!( + file, + "ln -s /usr/lib/systemd/system/boot-efi.mount /lib/systemd/system/local-fs.target.wants/boot-efi.mount" + )?; + } + writeln!(file, r#"ln -s /usr/lib/systemd/system/etc.mount /lib/systemd/system/local-fs.target.wants/etc.mount"#)?; + + if let Some(users) = &config.users { + gen_add_users(file, users)?; + } + if let Some(systemd_services) = &config.systemd_service { + gen_systemd_services(file, systemd_services)?; + } + + writeln!( + file, + r#" +str=$(sed -n '/^root:/p' /etc/shadow | awk -F "root:" '{{print $2}}') +umask 0666 +mv /etc/shadow /etc/shadow_bak +sed -i '/^root:/d' /etc/shadow_bak +echo "root:""${{ROOT_PASSWD}}""${{str:1}}" > /etc/shadow +cat /etc/shadow_bak >> /etc/shadow +rm -rf /etc/shadow_bak + +dracut -f -v --add bootup /initramfs.img --kver "$(ls /lib/modules)" +rm -rf /usr/lib/dracut/modules.d/00bootup"# + )?; + + Ok(()) +} +/* endregion */ + +/* region: bootloader.sh */ +pub(crate) fn gen_bootloader(file: &mut File, arch: &str, legacy_bios: bool) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"set -eu +set -o pipefail +set -x + +function install_grub2 () {{"# + )?; + + if arch == "aarch64" || (arch == "x86_64" && !legacy_bios) { + writeln!( + file, + r#" cp -r /usr/lib/grub/x86_64-efi boot/efi/EFI/openEuler + eval "grub2-mkimage -d /usr/lib/grub/x86_64-efi -O x86_64-efi --output=/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" + + mkdir -p /boot/efi/EFI/BOOT/ + cp -f /boot/efi/EFI/openEuler/grubx64.efi /boot/efi/EFI/BOOT/BOOTX64.EFI +}} +"# + )?; + } else { + writeln!( + file, + r#" GRUBNAME=$(which grub2-install) + echo "Installing GRUB2..." + FORCE_OPT=${{FORCE_OPT:-"--force"}} + TARGET_OPT=${{TARGET_OPT:-"--target=i386-pc"}} + + $GRUBNAME --modules="biosdisk part_msdos" "$FORCE_OPT" "$TARGET_OPT" "$DEVICE" +}} +"# + )?; + } + + writeln!( + file, + r#"install_grub2 +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: rpmlist */ +pub(crate) fn gen_rpm_list(file: &mut File, rpmlist: &Vec) -> Result<()> { + for rpm in rpmlist { + writeln!(file, "{}", rpm)?; + } + Ok(()) +} +/* endregion */ + +/* region: grub.cfg */ +pub(crate) fn gen_grub_cfg(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"set pager=1 + +if [ -f ${{config_directory}}/grubenv ]; then + load_env -f ${{config_directory}}/grubenv +elif [ -s $prefix/grubenv ]; then + load_env +fi +if [ "${{next_entry}}" ] ; then + set default="${{next_entry}}" + set next_entry= + save_env next_entry + set boot_once=true +else + set default="${{saved_entry}}" +fi + +if [ x"${{feature_menuentry_id}}" = xy ]; then + menuentry_id_option="--id" +else + menuentry_id_option="" +fi + +export menuentry_id_option + +if [ "${{prev_saved_entry}}" ]; then + set saved_entry="${{prev_saved_entry}}" + save_env saved_entry + set prev_saved_entry= + save_env prev_saved_entry + set boot_once=true +fi + +function savedefault {{ + if [ -z "${{boot_once}}" ]; then + saved_entry="${{chosen}}" + save_env saved_entry + fi +}} + +function load_video {{ + if [ x$feature_all_video_module = xy ]; then + insmod all_video + else + insmod efi_gop + insmod efi_uga + insmod ieee1275_fb + insmod vbe + insmod vga + insmod video_bochs + insmod video_cirrus + fi +}} + +terminal_output console +if [ x$feature_timeout_style = xy ] ; then + set timeout_style=menu + set timeout=5 +# Fallback normal timeout code in case the timeout_style feature is +# unavailable. +else + set timeout=5 +fi +set superusers="root" +### END /etc/grub.d/00_header ### + +### BEGIN /etc/grub.d/01_users ### +if [ -f ${{prefix}}/user.cfg ]; then + source ${{prefix}}/user.cfg + if [ -n "${{GRUB2_PASSWORD}}" ]; then + set superusers="root" + export superusers + password_pbkdf2 root ${{GRUB2_PASSWORD}} + fi +fi +### END /etc/grub.d/01_users ### + +### BEGIN /etc/grub.d/10_linux ### +menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' {{ + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=/dev/vda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +}} + +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' {{ + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=/dev/vda3 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +}} + +### END /etc/grub.d/10_linux ### + +### BEGIN /etc/grub.d/10_reset_boot_success ### +# Hiding the menu is ok if last boot was ok or if this is a first boot attempt to boot the entry +if [ "${{boot_success}}" = "1" -o "${{boot_indeterminate}}" = "1" ]; then + set menu_hide_ok=1 +else + set menu_hide_ok=0 +fi +# Reset boot_indeterminate after a successful boot +if [ "${{boot_success}}" = "1" ] ; then + set boot_indeterminate=0 +# Avoid boot_indeterminate causing the menu to be hidden more then once +elif [ "${{boot_indeterminate}}" = "1" ]; then + set boot_indeterminate=2 +fi +# Reset boot_success for current boot +set boot_success=0 +save_env boot_success boot_indeterminate +### END /etc/grub.d/10_reset_boot_success ### + +### BEGIN /etc/grub.d/12_menu_auto_hide ### +if [ x$feature_timeout_style = xy ] ; then + if [ "${{menu_show_once}}" ]; then + unset menu_show_once + save_env menu_show_once + set timeout_style=menu + set timeout=60 + elif [ "${{menu_auto_hide}}" -a "${{menu_hide_ok}}" = "1" ]; then + set orig_timeout_style=${{timeout_style}} + set orig_timeout=${{timeout}} + if [ "${{fastboot}}" = "1" ]; then + # timeout_style=menu + timeout=0 avoids the countdown code keypress check + set timeout_style=menu + set timeout=0 + else + set timeout_style=hidden + set timeout=1 + fi + fi +fi +### END /etc/grub.d/12_menu_auto_hide ### + +### BEGIN /etc/grub.d/20_linux_xen ### +### END /etc/grub.d/20_linux_xen ### + +### BEGIN /etc/grub.d/20_ppc_terminfo ### +### END /etc/grub.d/20_ppc_terminfo ### + +### BEGIN /etc/grub.d/30_uefi-firmware ### +### END /etc/grub.d/30_uefi-firmware ### + +### BEGIN /etc/grub.d/40_custom ### +# This file provides an easy way to add custom menu entries. Simply type the +# menu entries you want to add after this comment. Be careful not to change +# the 'exec tail' line above. +### END /etc/grub.d/40_custom ### + +### BEGIN /etc/grub.d/41_custom ### +if [ -f ${{config_directory}}/custom.cfg ]; then + source ${{config_directory}}/custom.cfg +elif [ -z "${{config_directory}}" -a -f $prefix/custom.cfg ]; then + source $prefix/custom.cfg; +fi +### END /etc/grub.d/41_custom ### +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: 00bootup */ +// 00bootup/global.cfg +pub(crate) fn gen_global_cfg(file: &mut File, pxe_config: &PxeConfig) -> Result<()> { + writeln!( + file, + r#"# rootfs file name +rootfs_name={} + +# select the target disk to install kubeOS +disk={} + +# pxe server ip address where stores the rootfs on the http server +server_ip={} +# target machine ip +local_ip={} +# target machine route +route_ip={} +# target machine netmask +netmask={} +# target machine netDevice name +net_name={} +"#, + pxe_config.rootfs_name, + pxe_config.disk, + pxe_config.server_ip, + pxe_config.local_ip, + pxe_config.route_ip, + pxe_config.netmask, + pxe_config.net_name + )?; + Ok(()) +} + +// 00bootup/module-setup.sh +pub(crate) fn gen_module_setup(file: &mut File) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"check() {{ + return 0 +}} + +depends() {{ + echo systemd +}} + +install() {{ + inst_multiple -o grub2-mkimage mkfs.ext4 mkfs.vfat lsblk tar cpio gunzip lspci parted dhclient ifconfig curl hwinfo head tee arch df awk route + inst_hook mount 00 "$moddir/mount.sh" + inst_simple "$moddir/mount.sh" "/mount.sh" + inst_simple "$moddir/Global.cfg" "/Global.cfg" +}} + +installkernel() {{ + hostonly='' + instmods='drivers/ata drivers/nvme drivers/scsi drivers/net fs/fat fs/nls' +}} +"# + )?; + Ok(()) +} + +// 00bootup/mount.sh +pub(crate) fn gen_mount(file: &mut File) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"arch=$(arch) +min_size=8 +log=/install.log + +source ./Global.cfg + +function CheckSpace() {{ + local disk_ava + disk_ava="$(parted -l | grep "${{disk}}" | awk '{{print $3}}')" + if echo "${{disk_ava}}" | grep "[GT]B$"; then + if echo "${{disk_ava}}" | grep GB$; then + disk_ava="$(echo "${{disk_ava}}" | awk -F G '{{print $1}}' | awk -F . '{{print $1}}')" + if [ "${{disk_ava}}" -lt ${{min_size}} ]; then + echo "The available disk space is not enough, at least ${{min_size}}GB." | tee -a ${{log}} + return 1 + fi + fi + else + echo "The available disk space is not enough, at least ${{min_size}}G." | tee -a ${{log}} + return 1 + fi + + return 0 +}} + +function mount_proc_dev_sys() {{ + local tmp_root=$1 + mount -t proc none "${{tmp_root}}/proc" + mount --bind /dev "${{tmp_root}}/dev" + mount --bind /dev/pts "${{tmp_root}}/dev/pts" + mount -t sysfs none "${{tmp_root}}/sys" +}} + +function GetDisk() {{ + mapfile -t disks < <(hwinfo --disk --short 2>&1 | grep -vi "^disk" | awk '{{print $1}}') + if [ ${{#disks[*]}} -gt 0 ]; then + if [ -n "${{disk}}" ] && echo "${{disks[@]}}" | grep -wq "${{disk}}" ; then + echo "${{disk}} exists, start partition" | tee -a ${{log}} + else + echo "disk not exist, please choose correct disk" | tee -a ${{log}} + fi + else + echo "no disk found" | tee -a ${{log}} + return 1 + fi + CheckSpace + local status=$? + if [ $status -ne 0 ]; then + echo "no enough space on ${{disk}}" | tee -a ${{log}} + return 1 + fi + + return 0 +}} + +function PartitionAndFormatting() {{ + echo "Partitioning and formatting disk $disk..." + # partition and format + parted "${{disk}}" -s mklabel gpt >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s mkpart primary fat16 1M 100M >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s mkpart primary ext4 100M 2600M >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s mkpart primary ext4 2600M 5100M >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s mkpart primary ext4 5100M 100% >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s set 1 boot on >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + mkfs.vfat -n "BOOT" "${{disk}}"1 >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "format failed" | tee -a ${{log}} + return 1 + fi + + mkfs.ext4 -L "ROOT-A" "${{disk}}"2 >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "format failed" | tee -a ${{log}} + return 1 + fi + + mkfs.ext4 -L "ROOT-B" "${{disk}}"3 >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "format failed" | tee -a ${{log}} + return 1 + fi + + mkfs.ext4 -L "PERSIST" "${{disk}}"4 >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "format failed" | tee -a ${{log}} + return 1 + fi + + return 0 +}} + +function InitNetwork() {{ + echo "Initializing network..." + mapfile -t netNames < <(ifconfig -a | awk '{{print $1}}' | grep : | grep '^e' | awk -F: '{{print $1}}') + if [ ${{#netNames[*]}} -gt 0 ]; then + if [ -n "${{net_name}}" ] && echo "${{netNames[@]}}" | grep -wq "${{net_name}}" ; then + echo "${{net_name}} exists, start set ip" | tee -a ${{log}} + else + echo "net_name not exist, choose default net" | tee -a ${{log}} + net_name=${{netNames[0]}} + fi + else + echo "no net Device found" | tee -a ${{log}} + return 1 + fi + + ifconfig "${{net_name}}" up + local status=$? + if [ $status -ne 0 ]; then + echo "load net card failed" | tee -a ${{log}} + return 1 + fi + sleep 3 + + ifconfig "${{net_name}}" "${{local_ip}}" netmask "${{netmask}}" >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "ip set failed" | tee -a ${{log}} + return 1 + fi + sleep 3 + + route add default gw "${{route_ip}}" >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "add route failed" | tee -a ${{log}} + return 1 + fi + sleep 3 + return 0 +}} + +function MountRoot() {{ + echo "Mounting rootfs..." + # mount rootfs + mount "${{disk}}"2 /sysroot >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "mount rootfs failed" | tee -a ${{log}} + return 1 + fi + + return 0 +}} + +function MountPersist() {{ + echo "Mounting persist" + mount "${{disk}}"4 /sysroot/persist >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "mount persist failed" | tee -a ${{log}} + return 1 + fi + mkdir /sysroot/persist/{{var,etc,etcwork}} + mkdir -p /sysroot/persist/etc/KubeOS/certs + return 0 +}} + +function MountBoot() {{ + echo "Mounting boot" + mkdir -p /sysroot/boot/efi + mount "${{disk}}"1 /sysroot/boot/efi >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "mount boot failed" | tee -a ${{log}} + return 1 + fi + return 0 +}} + +function GetRootfs() {{ + echo "Downloading rootfs..." + + curl -o /"${{rootfs_name}}" http://"${{server_ip}}"/"${{rootfs_name}}" + if [ ! -e "/${{rootfs_name}}" ]; then + echo "download rootfs failed" | tee -a ${{log}} + return 1 + fi + + tar -xf /"${{rootfs_name}}" -C /sysroot + local status=$? + if [ $status -ne 0 ]; then + echo "decompose rootfs failed" | tee -a ${{log}} + return 1 + fi + + rm -rf "${{rootfs_name:?}}" + mount -o remount,ro "${{disk}}"2 /sysroot >> ${{log}} 2>&1 + return 0 +}} + +function Inst_Grub2_x86() {{ + # copy the files that boot need + cp -r /sysroot/usr/lib/grub/x86_64-efi /sysroot/boot/efi/EFI/openEuler + eval "grub2-mkimage -d /sysroot/usr/lib/grub/x86_64-efi -O x86_64-efi --output=/sysroot/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "grub2-mkimage on x86 failed" | tee -a ${{log}} + return 1 + fi + + mkdir -p /sysroot/boot/efi/EFI/BOOT/ + cp -f /sysroot/boot/efi/EFI/openEuler/grubx64.efi /sysroot/boot/efi/EFI/BOOT/BOOTX64.EFI + + return 0 +}} + +function Inst_Grub2_aarch64() {{ + cp -r /sysroot/usr/lib/grub/arm64-efi /sysroot/boot/efi/EFI/openEuler/ + eval "grub2-mkimage -d /sysroot/usr/lib/grub/arm64-efi -O arm64-efi --output=/sysroot/boot/efi/EFI/openEuler/grubaa64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "grub2-mkimage on aarch64 failed" | tee -a ${{log}} + return 1 + fi + + mkdir -p /sysroot/boot/efi/EFI/BOOT/ + cp -f /sysroot/boot/efi/EFI/openEuler/grubaa64.efi /sysroot/boot/efi/EFI/BOOT/BOOTAA64.EFI + + return 0 +}} + +function SetBoot() {{ + # mount boot + echo "Setting boot" + + if [ "$arch" == "x86_64" ]; then + Inst_Grub2_x86 + local status=$? + if [ $status -ne 0 ]; then + echo "install grub on x86 failed" | tee -a ${{log}} + return 1 + fi + fi + + if [ "$arch" == "aarch64" ]; then + Inst_Grub2_aarch64 + local status=$? + if [ $status -ne 0 ]; then + echo "install grub on aarch64 failed" | tee -a ${{log}} + return 1 + fi + fi + sed -i 's#/dev/sda#'"${{disk}}"'#g' /sysroot/boot/efi/EFI/openEuler/grub.cfg + + return 0 +}} + +function Bootup_Main() {{ + # get disk + echo "Checking disk info..." | tee -a ${{log}} + GetDisk + local status=$? + if [ $status -ne 0 ]; then + echo "Checking disk info failed" | tee -a ${{log}} + return 1 + fi + + # partition and format disk + echo "Partion and formatting..." | tee -a ${{log}} + PartitionAndFormatting + local status=$? + if [ $status -ne 0 ]; then + echo "Partition and formatting disk failed" | tee -a ${{log}} + return 1 + fi + + # init network + echo "Initializing network..." | tee -a ${{log}} + InitNetwork + local status=$? + if [ $status -ne 0 ]; then + echo "Initializing network failed" | tee -a ${{log}} + return 1 + fi + + # mount partitions + + # mount boot + echo "Mounting root..." | tee -a ${{log}} + MountRoot + local status=$? + if [ $status -ne 0 ]; then + echo "Mounting root failed" | tee -a ${{log}} + return 1 + fi + + echo "Mounting boot..." | tee -a ${{log}} + MountBoot + local status=$? + if [ $status -ne 0 ]; then + echo "Mounting boot failed" | tee -a ${{log}} + return 1 + fi + + # download rootfs + echo "Downloading rootfs..." | tee -a ${{log}} + GetRootfs + local status=$? + if [ $status -ne 0 ]; then + echo "Downloading rootfs failed" | tee -a ${{log}} + return 1 + fi + mount_proc_dev_sys /sysroot + # set boot + echo "Setting boot..." | tee -a ${{log}} + SetBoot + local status=$? + if [ $status -ne 0 ]; then + echo "Setting boot failed" | tee -a ${{log}} + return 1 + fi + # mount persist + echo "Mounting persist..." | tee -a ${{log}} + MountPersist + local status=$? + if [ $status -ne 0 ]; then + echo "Mounting persist failed" | tee -a ${{log}} + return 1 + fi + return 0 +}} + +Bootup_Main +ret=$? +if [ ${{ret}} -eq 0 ]; then + echo "kubeOS install success! switch to root" | tee -a ${{log}} + cp ${{log}} /sysroot/persist +else + echo "kubeOS install failed, see install.log" | tee -a ${{log}} +fi + +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: dockerfile */ +pub(crate) fn gen_dockerfile(file: &mut File) -> Result<()> { + writeln!( + file, + r#"FROM scratch +COPY os.tar / +CMD ["/bin/sh"] +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: admin-container */ +// admin-container/dockerfile +pub(crate) fn gen_admin_dockerfile(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"FROM openeuler-22.03-lts +MAINTAINER + +RUN yum -y install openssh-clients util-linux + +ADD ./sysmaster-0.2.3-1.oe2203.aarch64.rpm /home +RUN rpm -ivh /home/sysmaster-0.2.3-1.oe2203.aarch64.rpm + +COPY ./hostshell /usr/bin/ +COPY ./set-ssh-pub-key.sh /usr/local/bin +COPY ./set-ssh-pub-key.service /usr/lib/sysmaster + +EXPOSE 22 +# set sshd.service and set-ssh-pub-key.service pulled up by default +RUN sed -i 's/sysinit.target/sysinit.target;sshd.service;set-ssh-pub-key.service/g' /usr/lib/sysmaster/basic.target + +CMD ["/usr/lib/sysmaster/init"] +"# + )?; + Ok(()) +} + +// admin-container/set-ssh-pub-key.service +pub(crate) fn gen_set_ssh_pub_key_service(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description="set ssh authorized keys according to the secret which is set by user" + +[Service] +ExecStart="/usr/local/bin/set-ssh-pub-key.sh" +"# + )?; + Ok(()) +} + +// admin-container/set-ssh-pub-key.sh +pub(crate) fn gen_set_ssh_pub_key(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"ssh_pub=$(cat /etc/secret-volume/ssh-pub-key) +ssh_dir="/root/.ssh" +authorized_file="$ssh_dir/authorized_keys" + +if [ ! -d "$ssh_dir" ]; then + mkdir "$ssh_dir" + chmod 700 "$ssh_dir" +fi + +if [ ! -f "$authorized_file" ]; then + touch "$authorized_file" + chmod 600 "$authorized_file" +fi + +echo "$ssh_pub" >> "$authorized_file" +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: misc-files */ +// misc-files/boot-efi.mount +pub(crate) fn gen_boot_efi_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=grub2 Dir +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target + +[Mount] +What=/dev/disk/by-label/BOOT +Where=/boot/efi +Type=vfat +Options=defaults + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} + +// misc-files/boot-grub2.mount +pub(crate) fn gen_boot_grub2_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=grub2 Dir +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target + +[Mount] +What=/dev/disk/by-label/GRUB2 +Where=/boot/grub2 +Type=ext4 +Options=defaults + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} + +// misc-files/etc.mount +pub(crate) fn gen_etc_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=etc Dir +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target +Wants=persist.mount +After=persist.mount + +[Mount] +What=overlay +Where=/etc +Type=overlay +Options=upperdir=/persist/etc,lowerdir=/etc,workdir=/persist/etcwork + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} + +// misc-files/os-agent.service +pub(crate) fn gen_os_agent_service(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=Agent For KubeOS + +[Service] +Environment=GOTRACEBACK=crash +ExecStart=/usr/bin/os-agent +KillMode=process +Restart=on-failure + +[Install] +WantedBy=multi-user.target +"# + )?; + Ok(()) +} + +// misc-files/os-release +pub(crate) fn gen_os_release(file: &mut File) -> Result<()> { + writeln!( + file, + r#"NAME=KubeOS +ID=KubeOS +"# + )?; + Ok(()) +} + +// misc-files/persist.mount +pub(crate) fn gen_persist_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=PERSIST Dir (/persist) +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target + +[Mount] +What=/dev/disk/by-label/PERSIST +Where=/persist +Type=ext4 +Options=defaults + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} + +// misc-files/var.mount +pub(crate) fn gen_var_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=var Dir +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target +Wants=persist.mount +After=persist.mount + +[Mount] +What=/persist/var +Where=/var +Type=node +Options=bind + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} +/* endregion */ diff --git a/KubeOS-Rust/kbimg/src/utils.rs b/KubeOS-Rust/kbimg/src/utils.rs new file mode 100644 index 00000000..c7ea5505 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/utils.rs @@ -0,0 +1,126 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs, os::unix::fs::PermissionsExt, path::PathBuf, process::Command}; + +use anyhow::bail; + +use crate::commands::PxeConfig; + +pub(crate) fn execute_scripts(script: PathBuf) -> anyhow::Result<()> { + if !script.exists() { + bail!("Script does not exist: {:?}", script); + } + let status = Command::new("bash").arg(&script).status()?; + if !status.success() { + bail!("Failed to execute script: {}\n", script.display()); + } + Ok(()) +} + +pub(crate) fn set_permissions(path: &str, permission_value: u32) -> anyhow::Result<()> { + let metadata = fs::metadata(path)?; + let mut permissions = metadata.permissions(); + permissions.set_mode(permission_value); + fs::set_permissions(path, permissions)?; + Ok(()) +} + +/// Check if the input parameter is valid +pub(crate) fn is_valid_param + std::fmt::Debug>(param: S) -> bool { + let special_chars = vec!["|", ";", "&", "&&", "||", ">", ">>", "<", ",", "#", "!", "$"]; + !param.as_ref().chars().any(|c| special_chars.contains(&c.to_string().as_str())) +} + +/// Check if the path exists and is indeed a file +pub(crate) fn is_file_valid(msg: &str, path: &PathBuf) -> anyhow::Result<()> { + if !path.exists() { + bail!("{} does not exist: {:?}", msg, path); + } + if !path.is_file() { + bail!("{} exists but is not a file: {:?}", msg, path); + } + Ok(()) +} + +/// Check if addr is valid +pub(crate) fn is_addr_valid(addr: &str) -> bool { + let ip_pattern = regex::Regex::new(r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$").unwrap(); + if !ip_pattern.is_match(addr) { + return false; + } + + for quad in addr.split('.') { + if let Ok(num) = quad.parse::() { + if num <= 255 { + continue; + } + } + return false; + } + + true +} + +/// Check pxe config +pub(crate) fn check_pxe_conf_valid(pxe_config: &PxeConfig) -> anyhow::Result<()> { + if !is_addr_valid(&pxe_config.server_ip) { + bail!("address {} is invalid, please check input", &pxe_config.server_ip) + } + if !is_addr_valid(&pxe_config.local_ip) { + bail!("address {} is invalid, please check input", &pxe_config.local_ip) + } + if !is_addr_valid(&pxe_config.route_ip) { + bail!("address {} is invalid, please check input", &pxe_config.route_ip) + } + if !is_addr_valid(&pxe_config.netmask) { + bail!("address {} is invalid, please check input", &pxe_config.netmask) + } + Ok(()) +} + +/// Get architecture +pub(crate) fn get_arch() -> String { + let output = std::process::Command::new("arch").output().expect("Failed to execute `arch` command"); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_valid_param() { + init(); + assert_eq!(is_valid_param("test"), true); + assert_eq!(is_valid_param("test|test"), false); + assert_eq!(is_valid_param("test;test"), false); + assert_eq!(is_valid_param("test&test"), false); + assert_eq!(is_valid_param("test&&test"), false); + assert_eq!(is_valid_param("test||test"), false); + assert_eq!(is_valid_param("test>test"), false); + assert_eq!(is_valid_param("test>>test"), false); + assert_eq!(is_valid_param("test + +## 配置文件说明 + +* from_repo: 从 repo 创建 OCI 镜像、虚拟机镜像或物理机镜像 + + | 参数 | 描述 | + | --- | --- | + | agent_path | os-agent 二进制的路径 | + | image_type | upgrade: 用于安装和升级的 OCI 镜像格式的 KubeOS 镜像; vm-repo: 用于部署和升级的虚拟机镜像; pxe-repo: 物理机安装所需的镜像及文件 | + | legacy_bios | 镜像为 legacy 引导或 UEFI 引导 | + | repo_path | repo 文件的路径，repo 文件中配置制作镜像所需要的 yum 源 | + | root_passwd | KubeOS 镜像 root 用户密码，加密后的带盐值的密码，可以用 openssl、kiwi 命令生成 | + | version | 制作出来的 KubeOS 镜像的版本 | + | rpmlist | 镜像所需的 rpm 包 | + | docker_img | 生成或者使用的 docker 镜像 | + +* from_docker: 从 docker 镜像创建虚拟机镜像或物理机镜像 + + | 参数 | 描述 | + | --- | --- | + | docker_img | 生成或者使用的 docker 镜像 | + | image_type | vm-docker: 用于部署和升级的虚拟机镜像; pxe-docker: 物理机安装所需的镜像及文件 | + +* admin_container: + + | 参数 | 描述 | + | --- | --- | + | dockerfile | dockerfile 路径 | + | docker_img | 生成或者使用的 docker 镜像 | + +* [OPTIONAL] users: 添加用户 + + | 参数 | 描述 | + | --- | --- | + | groups | [OPTIONAL] 用户组 (第一个为主组，其他为附加组) | + | name | 用户名 | + | passwd | 密码 | + | sudo | [OPTIONAL] 用户是否具有 sudo 权限 | + +* [OPTIONAL] copy_files: 拷贝文件到指定目录 + + | 参数 | 描述 | + | --- | --- | + | dst | 目标目录 | + | src | 源文件路径 | + +* [OPTIONAL] grub: grub配置 + + | 参数 | 描述 | + | --- | --- | + | passwd | [OPTIONAL] grub 密码 | + +* [OPTIONAL] systemd_service: 新增 systemd 服务 + + | 参数 | 描述 | + | --- | --- | + | name | systemd 服务名 | + +* [OPTIONAL] chroot_script: 自定义 chroot 脚本 + + | 参数 | 描述 | + | --- | --- | + | path | 脚本路径 | + +* [OPTIONAL] disk_partition: 自定义分区大小和镜像大小 + + | 参数 | 描述 | + | --- | --- | + | first | 引导分区大小 | + | second | ROOT-A 分区大小 | + | third | ROOT-B 分区大小 | + | img_size | 镜像大小 | + +* [OPTIONAL] persist_mkdir: persist 分区新建目录 + + | 参数 | 描述 | + | --- | --- | + | name | 目录名 | + +## 使用说明 + +#### 注意事项 + +* 新增 systemd 服务需要将对应的 .service 文件或 .mount 文件拷贝至镜像```/usr/lib/systemd/system```目录 + + ```toml + [[copy_files]] + dst = "/usr/lib/systemd/system" + src = ".../containerd.service" + + [systemd_service] + name = ["containerd"] + ``` + + * 如需挂载数据盘，请先自定义```persist-data.mount```文件，并启用```copy_files```和```systemd_service```字段设置启动时挂载，启用```persist_mkdir```字段创建挂载点 + * .mount文件名由挂载点路径生成，将斜杠替换为连字符 + * 请先在磁盘映像文件上创建ext4文件系统 + + ``` + # persist-data.mount + [Unit] + Description=Mount Disk + Documentation=man:systemd.mount(5) + + [Mount] + What=/dev/vdb + Where=/persist/data + Type=ext4 + Options=defaults,noatime + + [Install] + WantedBy=local-fs.target + ``` + + ```toml + [[copy_files]] + dst = "/usr/lib/systemd/system" + src = ".../persist-data.mount" + + [systemd_service] + name = ["persist-data.mount"] + + [persist_mkdir] + name = ["data"] + ``` + + * 如需配置逻辑卷，请先自定义```volume.service```文件，并启用```copy_files```和```systemd_service```设置启动时配置逻辑卷，启用```persist_mkdir```字段创建挂载点 + + ``` + # volume.service + [Unit] + Description=Mount Logical Volume + After=local-fs.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=pvcreate /dev/vdb + ExecStart=pvcreate /dev/vdc + ExecStart=vgcreate my_vg /dev/vdb /dev/vdc + ExecStart=lvcreate -L 15G -n my_lv my_vg + ExecStart=mkfs.ext4 /dev/my_vg/my_lv + ExecStart=mount /dev/my_vg/my_lv /persist/lv_data + + [Install] + WantedBy=local-fs.target + ``` + + ```toml + [[copy_files]] + dst = "/usr/lib/systemd/system" + src = ".../volume.service" + + [systemd_service] + name = ["volume"] + + [persist_mkdir] + name = ["lv_data"] + ``` + +## 使用示例 + +### KubeOS OCI 镜像制作 + +* 如需进行DNS配置，请先自定义```resolv.conf```文件，并启用```copy_files```字段将配置文件拷贝到```/etc```目录 + + ```shell + touch \/resolv.conf + vim \resolv.conf + ``` + + ```toml + [[copy_files]] + dst = "/etc" + src = "" + ``` + +* 制作KubeOS容器镜像 + + ```toml + [from_repo] + agent_path = "/bin/os-agent" + image_type = "upgrade" + legacy_bios = false + repo_path = "xxx.repo" + root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0" + version = "v1" + docker_img = "your_imageRepository/imageName:version" + rpmlist = [ + # your rpms + ] + ``` + +* 制作完成后查看制作出来的KubeOS容器镜像 + ``` shell + docker images + ``` + +### KubeOS 虚拟机镜像制作 + +* 使用repo源制作 + + * 如需进行DNS配置，请先自定义```resolv.conf```文件，并启用**copy_files**字段将配置文件拷贝到```/etc```目录 + + ```shell + touch \/resolv.conf + vim \resolv.conf + ``` + + ```toml + [[copy_files]] + dst = "/etc" + src = "" + ``` + + * KubeOS虚拟机镜像制作 + + ```toml + [from_repo] + agent_path = "/bin/os-agent" + image_type = "vm-repo" + legacy_bios = false + repo_path = "xxx.repo" + root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0" + version = "v1" + rpmlist = [ + # your rpms + ] + ``` + +* 使用docker镜像制作 + + ```toml + [from_dockerimg] + docker_img = "your_imageRepository/imageName:version" + image_type = "vm-docker" + ``` + +* 结果说明 + 容器 OS 镜像制作完成后，会在 ./scripts-auto 目录下生成 + * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。 + * update.img: 用于升级的根文件系统分区镜像 + +### KubeOS 物理机安装所需镜像及文件制作 + +* 首先需要修改```kbimg.toml```中```pxe_config```的配置，对相关参数进行配置，参数均为必填，ip目前仅支持ipv4，配置示例如下 + + ```toml + [pxe_config] + # rootfs file name + rootfs_name = "kubeos.tar" + # select the target disk to install kubeOS + disk = "/dev/sda" + # pxe server ip address where stores the rootfs on the http server + server_ip = "192.168.1.50" + # target machine ip + local_ip = "192.168.1.100" + # target machine route + route_ip = "192.168.1.1" + # target machine netmask + netmask = "255.255.255.0" + # target machine netDevice name + net_name = "eth0" + ``` + +* 使用 repo 源制作 + * 如需进行DNS配置，请先自定义```resolv.conf```文件，并启用```copy_files```字段将配置文件拷贝到```/etc```目录 + + ```shell + touch \/resolv.conf + vim \resolv.conf + ``` + + ```toml + [[copy_files]] + dst = "/etc" + src = "" + ``` + + * KubeOS物理机安装所需镜像制作 + ```toml + [from_repo] + agent_path = "/bin/os-agent" + image_type = "pxe-repo" + legacy_bios = true + repo_path = "xxx.repo" + root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0" + version = "v1" + rpmlist = [ + # your rpms + ] + ``` + +* 使用 docker 镜像制作 + ```toml + [from_dockerimg] + docker_img = "your_imageRepository/imageName:version" + image_type = "vm-docker" + ``` + +* 结果说明 + * initramfs.img: 用于pxe启动用的 initramfs 镜像 + * kubeos.tar: pxe安装所用的 OS -- Gitee From 8fce3e81822b0a5818adfb4ed5112030ed6b957e Mon Sep 17 00:00:00 2001 From: liyuanr Date: Tue, 10 Sep 2024 16:14:21 +0800 Subject: [PATCH 43/46] bugfix (os-operator and proxy): fix the issue that some node configurations are not delivered. When configuring node, the operator updates osinstance and then node. The time of the two updates is affected by the response time of the APIServer. The update time may be different. If the proxy completes the configuration immediately after the osinstance is updated and the node label is updated, the configuration label on the node is not deleted. As a result, the node is skipped during the next configuration. Therefore, the osinstance and node label check is added to the proxy.The configuration is performed only after the operator is updated. In addition, the logs of the operator and proxy are optimized as follows: 1. Fixe an issue where error logs are printed when the values of starttime and endtime are the same. 2. Delete the logs used during development from the time.go file. 3. The log about the successful deletion of the serial label by the operator is added. 4. Add a description before obtaining the logs of the node to be checked. (whether to add serial labels or upgrade/configuration) 5. Logs are added when a node is being upgraded or configuration is returned in serial mode. 6. Some debug logs are added to the operator. Signed-off-by: liyuanr --- .../proxy/src/controller/controller.rs | 26 ++++++++++++++++--- KubeOS-Rust/proxy/src/controller/utils.rs | 6 ++--- KubeOS-Rust/proxy/src/main.rs | 2 +- cmd/operator/controllers/operation.go | 1 + cmd/operator/controllers/os_controller.go | 12 ++++++--- cmd/operator/controllers/times.go | 4 +-- 6 files changed, 37 insertions(+), 14 deletions(-) diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index 40405b2d..787a0e1c 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -57,6 +57,7 @@ pub async fn reconcile( return Ok(NO_REQUEUE) } }else { + debug!("osinstance correspending os name is None, not in upgrading or configuring"); return Ok(REQUEUE_NORMAL) } @@ -68,7 +69,7 @@ pub async fn reconcile( .as_ref() .ok_or(Error::MissingSubResource { value: String::from("node.status.node_info") })? .os_image; - debug!("os expected osversion is {},actual osversion is {}", os_cr.spec.osversion, node_os_image); + debug!("os expected osversion is {}, actual osversion is {}", os_cr.spec.osversion, node_os_image); if check_version(&os_cr.spec.osversion, node_os_image) { match ConfigType::SysConfig.check_config_version(&os, &osinstance) { ConfigOperation::Reassign => { @@ -94,10 +95,26 @@ pub async fn reconcile( }, _ => {}, } + if node.labels().contains_key(LABEL_UPGRADING) || node.labels().contains_key(LABEL_CONFIGURING) { + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + info!( + "node has upgrade/config label , but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin" + ); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } proxy_controller.set_config(&mut osinstance, ConfigType::SysConfig).await?; proxy_controller .refresh_node(node, osinstance, &get_config_version(os_cr.spec.sysconfigs.as_ref()), ConfigType::SysConfig) .await?; + } } else { if os_cr.spec.opstype == NODE_STATUS_CONFIG { return Err(Error::UpgradeBeforeConfig); @@ -117,7 +134,7 @@ pub async fn reconcile( if node.labels().contains_key(LABEL_UPGRADING) { if osinstance.spec.nodestatus == NODE_STATUS_IDLE { info!( - "node has upgrade label ,but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin" + "node has upgrade label , but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin" ); proxy_controller .refresh_node( @@ -196,12 +213,13 @@ impl ProxyController { let node_api: Api = Api::all(self.k8s_client.clone()); let labels = node.labels_mut(); if labels.contains_key(LABEL_UPGRADING) { + debug!("delete label {}", LABEL_UPGRADING); labels.remove(LABEL_UPGRADING); node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; - }else if labels.contains_key(LABEL_CONFIGURING) { + }else if labels.contains_key(LABEL_CONFIGURING){ + debug!("delete label {}", LABEL_CONFIGURING); labels.remove(LABEL_CONFIGURING); node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; - } if let Some(node_spec) = &node.spec { if let Some(node_unschedulable) = node_spec.unschedulable { diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs index 148ca24d..7e7b41d9 100644 --- a/KubeOS-Rust/proxy/src/controller/utils.rs +++ b/KubeOS-Rust/proxy/src/controller/utils.rs @@ -47,7 +47,7 @@ impl ConfigType { let os_config_version = get_config_version(os.spec.upgradeconfigs.as_ref()); let osi_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); debug!( - "os upgradeconfig version is{},osinstance spec upragdeconfig version is{}", + "os upgradeconfig version is {}, osinstance spec upragdeconfig version is {}", os_config_version, osi_config_version ); if !check_version(&os_config_version, &osi_config_version) { @@ -61,7 +61,7 @@ impl ConfigType { let os_config_version = get_config_version(os.spec.sysconfigs.as_ref()); let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); debug!( - "os sysconfig version is{},osinstance spec sysconfig version is{}", + "os sysconfig version is {},osinstance spec sysconfig version is {}", os_config_version, osi_config_version ); if !check_version(&os_config_version, &osi_config_version) { @@ -108,7 +108,7 @@ impl ConfigType { }, } debug!( - "osinstance soec config version is {},status config version is {}", + "osinstance spec config version is {}, status config version is {}", spec_config_version, status_config_version ); if spec_config_version != status_config_version && osinstance.spec.nodestatus != NODE_STATUS_IDLE { diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs index 5c122ba2..c15aebed 100644 --- a/KubeOS-Rust/proxy/src/main.rs +++ b/KubeOS-Rust/proxy/src/main.rs @@ -27,7 +27,7 @@ use controller::{ const PROXY_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); #[tokio::main] async fn main() -> Result<()> { - Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init(); + Builder::from_env(Env::default().default_filter_or("proxy=info")).target(Target::Stdout).init(); let client = Client::try_default().await?; let os: Api = Api::all(client.clone()); let controller_client = ControllerClient::new(client.clone()); diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go index 5ac3d6d4..9f130479 100644 --- a/cmd/operator/controllers/operation.go +++ b/cmd/operator/controllers/operation.go @@ -93,6 +93,7 @@ func deleteSerialLabel(ctx context.Context, r common.ReadStatusWriter, nodes []c log.Error(err, "unable to delete serial label ", "node", node.Name+", skip this node") errList = append(errList, err) } + log.Info("delete node " + node.Name + " serial label " + values.LabelSerial + " successfully") } } if len(errList) > 0 { diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index 9e2e8e49..f9e65b47 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -75,7 +75,6 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) " , the end time " + os.Spec.TimeWindow.EndTime) return values.Requeue, nil } - ops := os.Spec.OpsType var opsInsatnce operation switch ops { @@ -106,6 +105,7 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) if err != nil { return values.RequeueNow, err } + log.V(1).Info("get all nodes num is " + strconv.Itoa(len(allNodes))) switch os.Spec.ExecutionMode { case ExecutionModeParallel: result, err := excuteParallelOperation(ctx, r, os, opsInsatnce, len(allNodes)) @@ -197,6 +197,7 @@ func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, opsInstance operation, requirements []labels.Requirement) (int, error) { if limit == 0 { + log.V(1).Info("limit is 0 , do not need to assign operation") return 0, nil } nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated @@ -283,6 +284,7 @@ func setTimeInterval(timeInterval int) ctrl.Result { func excuteParallelOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, opsInsatnce operation, nodeNum int) (ctrl.Result, error) { + log.V(1).Info("start parallel operation") opsLabel := opsInsatnce.getOpsLabel() opsLabel.op = selection.Exists opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, @@ -294,6 +296,7 @@ func excuteParallelOperation(ctx context.Context, r common.ReadStatusWriter, os if err != nil { return values.RequeueNow, nil } + log.V(1).Info("get limit is " + strconv.Itoa(limit)) opsLabel.op = selection.DoesNotExist noOpsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, selection.Equals, opsLabel).createNodeRequirement(ctx, r) @@ -308,6 +311,7 @@ func excuteParallelOperation(ctx context.Context, r common.ReadStatusWriter, os func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, opsInsatnce operation, nodeNum int) (ctrl.Result, error) { + log.V(1).Info("start serial operation") opsLabel := opsInsatnce.getOpsLabel() opsLabel.op = selection.Exists opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, @@ -320,6 +324,7 @@ func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os up return values.RequeueNow, nil } if len(opsNodeNum) > 0 { + log.V(1).Info("a node is being upgraded or configured. Wait until the node upgrade or configuration is complete.") return values.Requeue, nil } @@ -332,7 +337,7 @@ func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os up if err != nil { return values.RequeueNow, nil } - + log.V(1).Info("get the number of nodes which need to be added serial label num is " + strconv.Itoa(serialNodeLimit)) noSerialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, selection.Equals, selection.DoesNotExist).createNodeRequirement(ctx, r) if err != nil { @@ -342,10 +347,12 @@ func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os up serialOpsInstance := serialOps{ label: opsInsatnce.getOpsLabel(), } + log.V(1).Info("start add serial label to nodes") if _, err := assignOperation(ctx, r, os, serialNodeLimit, serialOpsInstance, noSerialNodesRequirement); err != nil { return values.RequeueNow, nil } + log.V(1).Info("start check nodes needed to be upgrade/configure or not") serialLimit := 1 // 1 is the number of operation nodes when excution mode in serial count, err := assignOperation(ctx, r, os, serialLimit, opsInsatnce, serialNodesRequirement) if err != nil { @@ -355,5 +362,4 @@ func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os up return values.Requeue, nil } return setTimeInterval(os.Spec.TimeInterval), nil - } diff --git a/cmd/operator/controllers/times.go b/cmd/operator/controllers/times.go index 3a72cce9..f651c0e4 100644 --- a/cmd/operator/controllers/times.go +++ b/cmd/operator/controllers/times.go @@ -62,14 +62,12 @@ func isWithinTimeWindow(start, end string) (bool, error) { } if endTime.Before(startTime) { if layoutStart == DATE_TIME { - return false, fmt.Errorf("invalid TimeWindow: Start %s Time is after end time %s", + return false, fmt.Errorf("invalid TimeWindow: start time %s is after end time %s", startTime.Format(layoutStart), endTime.Format(layoutEnd)) } endTime = endTime.Add(oneDayTime) - fmt.Printf("endtime time add 24 hour is %s\n", endTime.Format(layoutStart)) if now.Before(startTime) { now = now.Add(oneDayTime) - fmt.Printf("now time add 24 hour is %s\n", now.Format(layoutStart)) } } -- Gitee From 655ca30a1103ef86cf31e0de44771a610c3a2af3 Mon Sep 17 00:00:00 2001 From: YouMeiYouMaoTai <15335885760@163.com> Date: Thu, 26 Sep 2024 16:28:04 +0800 Subject: [PATCH 44/46] Merge branch 'master' into dev --- .gitignore | 10 + KubeOS-Rust/Cargo.lock | 3067 +++++++++++++++++ KubeOS-Rust/Cargo.toml | 12 + KubeOS-Rust/agent/Cargo.toml | 20 + KubeOS-Rust/agent/src/function.rs | 56 + KubeOS-Rust/agent/src/main.rs | 66 + KubeOS-Rust/agent/src/rpc/agent.rs | 30 + KubeOS-Rust/agent/src/rpc/agent_impl.rs | 217 ++ KubeOS-Rust/agent/src/rpc/mod.rs | 19 + KubeOS-Rust/cli/Cargo.toml | 15 + KubeOS-Rust/cli/src/client.rs | 56 + KubeOS-Rust/cli/src/lib.rs | 14 + KubeOS-Rust/cli/src/method/callable_method.rs | 54 + KubeOS-Rust/cli/src/method/configure.rs | 72 + KubeOS-Rust/cli/src/method/mod.rs | 18 + KubeOS-Rust/cli/src/method/prepare_upgrade.rs | 78 + KubeOS-Rust/cli/src/method/request.rs | 88 + KubeOS-Rust/cli/src/method/rollback.rs | 42 + KubeOS-Rust/cli/src/method/upgrade.rs | 42 + KubeOS-Rust/kbimg/Cargo.toml | 16 + KubeOS-Rust/kbimg/kbimg.toml | 87 + KubeOS-Rust/kbimg/src/admin_container.rs | 84 + KubeOS-Rust/kbimg/src/commands.rs | 170 + KubeOS-Rust/kbimg/src/docker_img.rs | 74 + KubeOS-Rust/kbimg/src/main.rs | 122 + KubeOS-Rust/kbimg/src/repo.rs | 203 ++ KubeOS-Rust/kbimg/src/scripts_gen.rs | 1754 ++++++++++ KubeOS-Rust/kbimg/src/utils.rs | 126 + KubeOS-Rust/kbimg/src/values.rs | 42 + KubeOS-Rust/manager/Cargo.toml | 25 + KubeOS-Rust/manager/src/api/agent_status.rs | 21 + KubeOS-Rust/manager/src/api/mod.rs | 17 + KubeOS-Rust/manager/src/api/types.rs | 140 + KubeOS-Rust/manager/src/lib.rs | 15 + KubeOS-Rust/manager/src/sys_mgmt/config.rs | 558 +++ .../manager/src/sys_mgmt/containerd_image.rs | 301 ++ .../manager/src/sys_mgmt/disk_image.rs | 406 +++ .../manager/src/sys_mgmt/docker_image.rs | 236 ++ KubeOS-Rust/manager/src/sys_mgmt/mod.rs | 23 + KubeOS-Rust/manager/src/sys_mgmt/values.rs | 36 + KubeOS-Rust/manager/src/utils/common.rs | 310 ++ .../manager/src/utils/container_image.rs | 341 ++ KubeOS-Rust/manager/src/utils/executor.rs | 89 + .../manager/src/utils/image_manager.rs | 206 ++ KubeOS-Rust/manager/src/utils/mod.rs | 23 + KubeOS-Rust/manager/src/utils/partition.rs | 117 + KubeOS-Rust/proxy/Cargo.toml | 49 + .../proxy/src/controller/agentclient.rs | 153 + KubeOS-Rust/proxy/src/controller/apiclient.rs | 147 + .../proxy/src/controller/apiserver_mock.rs | 686 ++++ .../proxy/src/controller/controller.rs | 589 ++++ KubeOS-Rust/proxy/src/controller/crd.rs | 94 + KubeOS-Rust/proxy/src/controller/mod.rs | 26 + KubeOS-Rust/proxy/src/controller/utils.rs | 154 + KubeOS-Rust/proxy/src/controller/values.rs | 35 + KubeOS-Rust/proxy/src/drain.rs | 511 +++ KubeOS-Rust/proxy/src/main.rs | 49 + KubeOS-Rust/proxy/tests/common/mod.rs | 63 + KubeOS-Rust/proxy/tests/drain_test.rs | 41 + .../proxy/tests/setup/kind-config.yaml | 5 + KubeOS-Rust/proxy/tests/setup/resources.yaml | 102 + .../proxy/tests/setup/setup_test_env.sh | 81 + KubeOS-Rust/rustfmt.toml | 11 + Makefile | 26 +- VERSION | 2 +- api/v1alpha1/os_types.go | 31 +- cmd/operator/controllers/operation.go | 193 +- cmd/operator/controllers/os_controller.go | 273 +- cmd/operator/controllers/requirements.go | 199 ++ cmd/operator/controllers/times.go | 104 + cmd/operator/main.go | 13 +- .../admin-container/admin-container.yaml | 126 +- .../config/crd/upgrade.openeuler.org_os.yaml | 30 +- .../upgrade.openeuler.org_osinstances.yaml | 16 +- docs/example/config/manager/manager.yaml | 65 +- .../config/samples/upgrade_v1alpha1_os.yaml | 69 +- docs/quick-start.md | 452 ++- ...66\344\275\234\346\214\207\345\257\274.md" | 385 ++- go.mod | 2 - go.sum | 2 - pkg/values/values.go | 7 +- scripts/bootloader.sh | 4 +- vendor/modules.txt | 3 - 83 files changed, 13830 insertions(+), 486 deletions(-) create mode 100644 KubeOS-Rust/Cargo.lock create mode 100644 KubeOS-Rust/Cargo.toml create mode 100644 KubeOS-Rust/agent/Cargo.toml create mode 100644 KubeOS-Rust/agent/src/function.rs create mode 100644 KubeOS-Rust/agent/src/main.rs create mode 100644 KubeOS-Rust/agent/src/rpc/agent.rs create mode 100644 KubeOS-Rust/agent/src/rpc/agent_impl.rs create mode 100644 KubeOS-Rust/agent/src/rpc/mod.rs create mode 100644 KubeOS-Rust/cli/Cargo.toml create mode 100644 KubeOS-Rust/cli/src/client.rs create mode 100644 KubeOS-Rust/cli/src/lib.rs create mode 100644 KubeOS-Rust/cli/src/method/callable_method.rs create mode 100644 KubeOS-Rust/cli/src/method/configure.rs create mode 100644 KubeOS-Rust/cli/src/method/mod.rs create mode 100644 KubeOS-Rust/cli/src/method/prepare_upgrade.rs create mode 100644 KubeOS-Rust/cli/src/method/request.rs create mode 100644 KubeOS-Rust/cli/src/method/rollback.rs create mode 100644 KubeOS-Rust/cli/src/method/upgrade.rs create mode 100644 KubeOS-Rust/kbimg/Cargo.toml create mode 100644 KubeOS-Rust/kbimg/kbimg.toml create mode 100644 KubeOS-Rust/kbimg/src/admin_container.rs create mode 100644 KubeOS-Rust/kbimg/src/commands.rs create mode 100644 KubeOS-Rust/kbimg/src/docker_img.rs create mode 100644 KubeOS-Rust/kbimg/src/main.rs create mode 100644 KubeOS-Rust/kbimg/src/repo.rs create mode 100644 KubeOS-Rust/kbimg/src/scripts_gen.rs create mode 100644 KubeOS-Rust/kbimg/src/utils.rs create mode 100644 KubeOS-Rust/kbimg/src/values.rs create mode 100644 KubeOS-Rust/manager/Cargo.toml create mode 100644 KubeOS-Rust/manager/src/api/agent_status.rs create mode 100644 KubeOS-Rust/manager/src/api/mod.rs create mode 100644 KubeOS-Rust/manager/src/api/types.rs create mode 100644 KubeOS-Rust/manager/src/lib.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/config.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/mod.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/values.rs create mode 100644 KubeOS-Rust/manager/src/utils/common.rs create mode 100644 KubeOS-Rust/manager/src/utils/container_image.rs create mode 100644 KubeOS-Rust/manager/src/utils/executor.rs create mode 100644 KubeOS-Rust/manager/src/utils/image_manager.rs create mode 100644 KubeOS-Rust/manager/src/utils/mod.rs create mode 100644 KubeOS-Rust/manager/src/utils/partition.rs create mode 100644 KubeOS-Rust/proxy/Cargo.toml create mode 100644 KubeOS-Rust/proxy/src/controller/agentclient.rs create mode 100644 KubeOS-Rust/proxy/src/controller/apiclient.rs create mode 100644 KubeOS-Rust/proxy/src/controller/apiserver_mock.rs create mode 100644 KubeOS-Rust/proxy/src/controller/controller.rs create mode 100644 KubeOS-Rust/proxy/src/controller/crd.rs create mode 100644 KubeOS-Rust/proxy/src/controller/mod.rs create mode 100644 KubeOS-Rust/proxy/src/controller/utils.rs create mode 100644 KubeOS-Rust/proxy/src/controller/values.rs create mode 100644 KubeOS-Rust/proxy/src/drain.rs create mode 100644 KubeOS-Rust/proxy/src/main.rs create mode 100644 KubeOS-Rust/proxy/tests/common/mod.rs create mode 100644 KubeOS-Rust/proxy/tests/drain_test.rs create mode 100644 KubeOS-Rust/proxy/tests/setup/kind-config.yaml create mode 100644 KubeOS-Rust/proxy/tests/setup/resources.yaml create mode 100644 KubeOS-Rust/proxy/tests/setup/setup_test_env.sh create mode 100644 KubeOS-Rust/rustfmt.toml create mode 100644 cmd/operator/controllers/requirements.go create mode 100644 cmd/operator/controllers/times.go diff --git a/.gitignore b/.gitignore index 5e56e040..0cac753e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,11 @@ +# vscode settings +.vscode + +# rust dependencies +target/ + +# KubeOS bin /bin + +# scripts-auto +/scripts-auto diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock new file mode 100644 index 00000000..5b97095b --- /dev/null +++ b/KubeOS-Rust/Cargo.lock @@ -0,0 +1,3067 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom 0.2.10", + "instant", + "rand 0.8.5", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.48.5", +] + +[[package]] +name = "clap" +version = "3.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "cli" +version = "1.0.6" +dependencies = [ + "anyhow", + "jsonrpc", + "log", + "manager", + "serde", + "serde_json", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if", + "num_cpus", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "dyn-clone" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "globset" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "h2" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.9", + "indexmap 1.9.3", + "slab", + "tokio", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http 0.2.9", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.9", + "http-body 0.4.5", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.9", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.2.0", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.25", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.25", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.2.0", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.3", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-patch" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce" +dependencies = [ + "serde", + "serde_json", + "treediff", +] + +[[package]] +name = "jsonpath_lib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "jsonrpc" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd8d6b3f301ba426b30feca834a2a18d48d5b54e5065496b5c1b05537bee3639" +dependencies = [ + "base64 0.13.1", + "serde", + "serde_json", +] + +[[package]] +name = "jsonrpc-core" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +dependencies = [ + "futures", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "jsonrpc-derive" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "jsonrpc-ipc-server" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382bb0206323ca7cda3dcd7e245cea86d37d02457a02a975e3378fb149a48845" +dependencies = [ + "futures", + "jsonrpc-core", + "jsonrpc-server-utils", + "log", + "parity-tokio-ipc", + "parking_lot", + "tower-service", +] + +[[package]] +name = "jsonrpc-server-utils" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4" +dependencies = [ + "bytes", + "futures", + "globset", + "jsonrpc-core", + "lazy_static", + "log", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", + "unicase", +] + +[[package]] +name = "k8s-openapi" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c" +dependencies = [ + "base64 0.13.1", + "bytes", + "chrono", + "http 0.2.9", + "percent-encoding", + "serde", + "serde-value", + "serde_json", + "url", +] + +[[package]] +name = "kbimg" +version = "1.0.6" +dependencies = [ + "anyhow", + "clap", + "env_logger", + "log", + "regex", + "serde", + "sysinfo", + "toml 0.7.6", +] + +[[package]] +name = "kube" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4b96944d327b752df4f62f3a31d8694892af06fb585747c0b5e664927823d1a" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-derive", + "kube-runtime", +] + +[[package]] +name = "kube-client" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232db1af3d3680f9289cf0b4db51b2b9fee22550fc65d25869e39b23e0aaa696" +dependencies = [ + "base64 0.13.1", + "bytes", + "chrono", + "dirs-next", + "either", + "futures", + "http 0.2.9", + "http-body 0.4.5", + "hyper 0.14.25", + "hyper-timeout", + "hyper-tls 0.5.0", + "jsonpath_lib", + "k8s-openapi", + "kube-core", + "openssl", + "pem", + "pin-project", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tokio", + "tokio-native-tls", + "tokio-util 0.6.10", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de491f8c9ee97117e0b47a629753e939c2392d5d0a40f6928e582a5fba328098" +dependencies = [ + "chrono", + "form_urlencoded", + "http 0.2.9", + "json-patch", + "k8s-openapi", + "once_cell", + "schemars", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "kube-derive" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbb86bb3607245a67c8ad3a52aff41108f36b0d1e9e3e82ffb5760bfd84b965" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 1.0.109", +] + +[[package]] +name = "kube-runtime" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710729592eb30219b4e84898e91dc991fe09ccafe2c17fec4e45c3426c61abe0" +dependencies = [ + "backoff", + "dashmap", + "derivative", + "futures", + "json-patch", + "k8s-openapi", + "kube-client", + "pin-project", + "serde", + "serde_json", + "smallvec", + "thiserror", + "tokio", + "tokio-util 0.6.10", + "tracing", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" + +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.0", + "libc", + "redox_syscall 0.4.1", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4dcd960cc540667f619483fc99102f88d6118b87730e24e8fbe8054b7445e4" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "manager" +version = "1.0.6" +dependencies = [ + "anyhow", + "env_logger", + "lazy_static", + "log", + "mockall", + "mockito", + "nix", + "predicates", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "tempfile", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", +] + +[[package]] +name = "mockall" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4a1c770583dac7ab5e2f6c139153b783a53a1bbee9729613f193e59828326" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "mockito" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80f9fece9bd97ab74339fe19f4bcaf52b76dcc18e5364c7977c1838f76b38de9" +dependencies = [ + "assert-json-diff", + "httparse", + "lazy_static", + "log", + "rand 0.8.5", + "regex", + "serde_json", + "serde_urlencoded", + "similar", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", + "pin-utils", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.3", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" + +[[package]] +name = "openssl" +version = "0.10.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os-agent" +version = "1.0.6" +dependencies = [ + "anyhow", + "env_logger", + "jsonrpc-core", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "lazy_static", + "log", + "manager", + "nix", + "serde", + "serde_json", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "parity-tokio-ipc" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9981e32fb75e004cc148f5fb70342f393830e0a4aa62e3cc93b50976218d42b6" +dependencies = [ + "futures", + "libc", + "log", + "rand 0.7.3", + "tokio", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "predicates" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc3d91237f5de3bcd9d927e24d03b495adb6135097b001cea7403e2d573d00a9" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da1c2388b1513e1b605fcec39a95e0a9e8ef088f71443ef37099fa9ae6673fcb" + +[[package]] +name = "predicates-tree" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d86de6de25020a36c6d3643a86d9a6a9f552107c0559c60ea03551b5e16c032" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml 0.5.11", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proxy" +version = "1.0.6" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-trait", + "cli", + "env_logger", + "futures", + "h2 0.3.16", + "http 0.2.9", + "hyper 0.14.25", + "k8s-openapi", + "kube", + "log", + "manager", + "mockall", + "regex", + "reqwest", + "schemars", + "serde", + "serde_json", + "socket2 0.4.9", + "thiserror", + "thread_local", + "tokio", + "tokio-retry", + "tower-test", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.10", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom 0.2.10", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "reqwest" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" +dependencies = [ + "base64 0.21.5", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", + "hyper-rustls", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.10", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustls" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.5", +] + +[[package]] +name = "rustls-pki-types" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868e20fada228fefaf6b652e00cc73623d54f8171e7352c18bb281571f2d92da" + +[[package]] +name = "rustls-webpki" +version = "0.102.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "schemars" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1847b767a3d62d95cbf3d8a9f0e421cf57a0d8aa4f411d4b16525afb0284d4ed" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4d7e1b012cb3d9129567661a63755ea4b8a7386d339dc945ae187e403c6743" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "serde", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c4437699b6d34972de58652c68b98cb5b53a4199ab126db8e20ec8ded29a721" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_json" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +dependencies = [ + "indexmap 1.9.3", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap 1.9.3", + "ryu", + "serde", + "yaml-rust", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "similar" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "sysinfo" +version = "0.29.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +dependencies = [ + "autocfg", + "cfg-if", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "termcolor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" + +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.4.9", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-test" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89b3cbabd3ae862100094ae433e1def582cf86451b4e9bf83aa7ac1d8a7d719" +dependencies = [ + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "slab", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "toml" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +dependencies = [ + "indexmap 2.2.6", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tokio-util 0.7.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8" +dependencies = [ + "base64 0.13.1", + "bitflags 1.3.2", + "bytes", + "futures-core", + "futures-util", + "http 0.2.9", + "http-body 0.4.5", + "http-range-header", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tower-test" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4546773ffeab9e4ea02b8872faa49bb616a80a7da66afc2f32688943f97efa7" +dependencies = [ + "futures-util", + "pin-project", + "tokio", + "tokio-test", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tracing" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "treediff" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff" +dependencies = [ + "serde_json", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" + +[[package]] +name = "web-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml new file mode 100644 index 00000000..2886023f --- /dev/null +++ b/KubeOS-Rust/Cargo.toml @@ -0,0 +1,12 @@ +[workspace] +members = ["agent", "cli", "kbimg", "manager", "proxy"] +resolver = "2" + +[profile.release] +debug = false +debug-assertions = false +lto = true +opt-level = 's' +overflow-checks = false +panic = "unwind" +rpath = false diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml new file mode 100644 index 00000000..83e1b7c0 --- /dev/null +++ b/KubeOS-Rust/agent/Cargo.toml @@ -0,0 +1,20 @@ +[package] +description = "KubeOS os-agent" +edition = "2021" +license = "MulanPSL-2.0" +name = "os-agent" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +anyhow = { version = "1.0" } +env_logger = { version = "0.9" } +jsonrpc-core = { version = "18.0" } +jsonrpc-derive = { version = "18.0" } +jsonrpc-ipc-server = { version = "18.0" } +lazy_static = { version = "1.4" } +log = { version = "= 0.4.15" } +manager = { package = "manager", path = "../manager" } +nix = { version = "0.26.2" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } diff --git a/KubeOS-Rust/agent/src/function.rs b/KubeOS-Rust/agent/src/function.rs new file mode 100644 index 00000000..9789d95c --- /dev/null +++ b/KubeOS-Rust/agent/src/function.rs @@ -0,0 +1,56 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub use jsonrpc_core::Result as RpcResult; +use jsonrpc_core::{Error, ErrorCode}; +pub use jsonrpc_derive::rpc; +use log::error; + +const RPC_OP_ERROR: i64 = -1; + +pub struct RpcFunction; + +impl RpcFunction { + pub fn call(f: F) -> RpcResult + where + F: FnOnce() -> anyhow::Result, + { + (f)().map_err(|e| { + let error_message = format!("{:#}", e); + error!("{}", error_message.replace('\n', " ").replace('\r', "")); + Error { code: ErrorCode::ServerError(RPC_OP_ERROR), message: format!("{:?}", e), data: None } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rpcfunction_call() { + // Define a mock function that returns a result + fn mock_ok_function() -> anyhow::Result { + Ok(42) + } + let result: RpcResult = RpcFunction::call(mock_ok_function); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), 42); + + fn mock_err_function() -> anyhow::Result { + Err(anyhow::anyhow!("error")) + } + let result: RpcResult = RpcFunction::call(mock_err_function); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code, ErrorCode::ServerError(RPC_OP_ERROR)); + } +} diff --git a/KubeOS-Rust/agent/src/main.rs b/KubeOS-Rust/agent/src/main.rs new file mode 100644 index 00000000..cd95ef07 --- /dev/null +++ b/KubeOS-Rust/agent/src/main.rs @@ -0,0 +1,66 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{self, DirBuilder, Permissions}, + os::unix::fs::{DirBuilderExt, PermissionsExt}, + path::Path, +}; + +use env_logger::{Builder, Env, Target}; +use jsonrpc_core::{IoHandler, IoHandlerExtension}; +use jsonrpc_ipc_server::ServerBuilder; + +mod function; +mod rpc; + +use log::info; +use rpc::{Agent, AgentImpl}; + +const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; +const CARGO_PKG_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); + +fn start_and_run(sock_path: &str) { + let socket_path = Path::new(sock_path); + + // Create directory for socket if it doesn't exist + if let Some(dir_path) = socket_path.parent() { + if !dir_path.exists() { + DirBuilder::new().mode(0o750).create(dir_path).expect("Couldn't create directory for socket"); + } + } + + // Add RPC methods to IoHandler + let mut io = IoHandler::new(); + AgentImpl::default().to_delegate().augment(&mut io); + + // Build and start server + let builder = ServerBuilder::new(io); + let server = builder.start(sock_path).expect("Couldn't open socket"); + + let gid = nix::unistd::getgid(); + nix::unistd::chown(socket_path, Some(nix::unistd::ROOT), Some(gid)).expect("Couldn't set socket group"); + + // Set socket permissions to 0640 + let socket_permissions = Permissions::from_mode(0o640); + fs::set_permissions(socket_path, socket_permissions).expect("Couldn't set socket permissions"); + + info!("os-agent started, waiting for requests..."); + server.wait(); +} + +fn main() { + Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init(); + + info!("os-agent version is: {}", CARGO_PKG_VERSION.unwrap_or("NOT FOUND")); + start_and_run(SOCK_PATH); +} diff --git a/KubeOS-Rust/agent/src/rpc/agent.rs b/KubeOS-Rust/agent/src/rpc/agent.rs new file mode 100644 index 00000000..2496bfb4 --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/agent.rs @@ -0,0 +1,30 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use manager::api::{ConfigureRequest, Response, UpgradeRequest}; + +use super::function::{rpc, RpcResult}; + +#[rpc(server)] +pub trait Agent { + #[rpc(name = "prepare_upgrade")] + fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult; + + #[rpc(name = "upgrade")] + fn upgrade(&self) -> RpcResult; + + #[rpc(name = "configure")] + fn configure(&self, req: ConfigureRequest) -> RpcResult; + + #[rpc(name = "rollback")] + fn rollback(&self) -> RpcResult; +} diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs new file mode 100644 index 00000000..ab826413 --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -0,0 +1,217 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{sync::Mutex, thread, time::Duration}; + +use anyhow::{bail, Result}; +use log::{debug, info}; +use manager::{ + api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, + sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH}, + utils::{get_partition_info, switch_boot_menuentry, RealCommandExecutor}, +}; +use nix::{sys::reboot::RebootMode, unistd::sync}; + +use super::{ + agent::Agent, + function::{RpcFunction, RpcResult}, +}; + +pub struct AgentImpl { + mutex: Mutex<()>, + disable_reboot: bool, +} + +impl Agent for AgentImpl { + fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult { + RpcFunction::call(|| self.prepare_upgrade_impl(req)) + } + + fn upgrade(&self) -> RpcResult { + RpcFunction::call(|| self.upgrade_impl()) + } + + fn configure(&self, req: ConfigureRequest) -> RpcResult { + RpcFunction::call(|| self.configure_impl(req)) + } + + fn rollback(&self) -> RpcResult { + RpcFunction::call(|| self.rollback_impl()) + } +} + +impl Default for AgentImpl { + fn default() -> Self { + Self { mutex: Mutex::new(()), disable_reboot: false } + } +} + +impl AgentImpl { + fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + debug!("Received an 'prepare upgrade' request: {:?}", req); + info!("Start preparing for upgrading to version: {}", req.version); + + let handler: Box> = match req.image_type.as_str() { + "containerd" => Box::new(ImageType::Containerd(CtrImageHandler::default())), + "docker" => Box::new(ImageType::Docker(DockerImageHandler::default())), + "disk" => Box::new(ImageType::Disk(DiskImageHandler::default())), + _ => bail!("Invalid image type \"{}\"", req.image_type), + }; + + let image_manager = handler.download_image(&req)?; + info!("Ready to install image: {:?}", image_manager.paths.image_path.display()); + image_manager.install()?; + + Ok(Response { status: AgentStatus::UpgradeReady }) + } + + fn upgrade_impl(&self) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + info!("Start to upgrade"); + let command_executor = RealCommandExecutor {}; + let (_, next_partition_info) = get_partition_info(&command_executor)?; + + // based on boot mode use different command to switch boot partition + let device = next_partition_info.device.as_str(); + let menuentry = next_partition_info.menuentry.as_str(); + switch_boot_menuentry(&command_executor, DEFAULT_GRUBENV_PATH, menuentry)?; + info!("Switch to boot partition: {}, device: {}", menuentry, device); + self.reboot()?; + Ok(Response { status: AgentStatus::Upgraded }) + } + + fn configure_impl(&self, mut req: ConfigureRequest) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + debug!("Received a 'configure' request: {:?}", req); + info!("Start to configure"); + let config_map = &*CONFIG_TEMPLATE; + for config in req.configs.iter_mut() { + let config_type = &config.model; + if let Some(configuration) = config_map.get(config_type) { + debug!("Found configuration type: \"{}\"", config_type); + configuration.set_config(config)?; + } else { + bail!("Unknown configuration type: \"{}\"", config_type); + } + } + Ok(Response { status: AgentStatus::Configured }) + } + + fn rollback_impl(&self) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + info!("Start to rollback"); + let command_executor = RealCommandExecutor {}; + let (_, next_partition_info) = get_partition_info(&command_executor)?; + switch_boot_menuentry( + &command_executor, + manager::sys_mgmt::DEFAULT_GRUBENV_PATH, + &next_partition_info.menuentry, + )?; + info!("Switch to boot partition: {}, device: {}", next_partition_info.menuentry, next_partition_info.device); + self.reboot()?; + Ok(Response { status: AgentStatus::Rollbacked }) + } + + fn reboot(&self) -> Result<()> { + info!("Wait to reboot"); + thread::sleep(Duration::from_secs(1)); + sync(); + if self.disable_reboot { + return Ok(()); + } + nix::sys::reboot::reboot(RebootMode::RB_AUTOBOOT)?; + Ok(()) + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + + use manager::api::{CertsInfo, Sysconfig}; + + use super::*; + + #[test] + fn test_reboot() { + let mut agent = AgentImpl::default(); + agent.disable_reboot = true; + let res = agent.reboot(); + assert!(res.is_ok()); + } + + #[test] + fn test_configure() { + let agent = AgentImpl::default(); + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "kernel.sysctl".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req).unwrap(); + assert_eq!(res, Response { status: AgentStatus::Configured }); + + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "invalid".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req); + assert!(res.is_err()); + + // test lock + let _lock = agent.mutex.lock().unwrap(); + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "kernel.sysctl".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req); + assert!(res.is_err()); + } + + #[test] + fn test_prepare_upgrade() { + let agent = AgentImpl::default(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "xxx".into(), + image_type: "xxx".into(), + container_image: "xxx".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let res = agent.prepare_upgrade(req); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/agent/src/rpc/mod.rs b/KubeOS-Rust/agent/src/rpc/mod.rs new file mode 100644 index 00000000..976356be --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/mod.rs @@ -0,0 +1,19 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use super::function; + +mod agent; +mod agent_impl; + +pub use agent::*; +pub use agent_impl::*; diff --git a/KubeOS-Rust/cli/Cargo.toml b/KubeOS-Rust/cli/Cargo.toml new file mode 100644 index 00000000..78d5fd51 --- /dev/null +++ b/KubeOS-Rust/cli/Cargo.toml @@ -0,0 +1,15 @@ +[package] +description = "KubeOS os-agent client" +edition = "2021" +license = "MulanPSL-2.0" +name = "cli" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +anyhow = { version = "1.0" } +jsonrpc = { version = "0.13", features = ["simple_uds"] } +kubeos-manager = { package = "manager", path = "../manager" } +log = { version = "0.4" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs new file mode 100644 index 00000000..37518bdc --- /dev/null +++ b/KubeOS-Rust/cli/src/client.rs @@ -0,0 +1,56 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::path::Path; + +use jsonrpc::{ + simple_uds::UdsTransport, Client as JsonRPCClient, Request as JsonRPCRequest, Response as JsonRPCResponse, +}; +use serde_json::value::RawValue; + +pub struct Client { + json_rpc_client: JsonRPCClient, +} + +pub struct Request<'a>(JsonRPCRequest<'a>); + +impl<'a> Request<'a> {} + +impl Client { + pub fn new>(socket_path: P) -> Self { + Client { json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)) } + } + + pub fn build_request<'a>(&self, command: &'a str, params: &'a [Box]) -> Request<'a> { + let json_rpc_request = self.json_rpc_client.build_request(command, params); + let request = Request(json_rpc_request); + request + } + + pub fn send_request(&self, request: Request) -> Result { + self.json_rpc_client.send_request(request.0) + } +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_client() { + let socket_path = "/tmp/KubeOS-test.sock"; + let cli = Client::new(socket_path); + let command = "example_command"; + let params = vec![]; + let request = cli.send_request(cli.build_request(command, ¶ms)); + assert!(request.is_err()); + } +} diff --git a/KubeOS-Rust/cli/src/lib.rs b/KubeOS-Rust/cli/src/lib.rs new file mode 100644 index 00000000..cd66d72f --- /dev/null +++ b/KubeOS-Rust/cli/src/lib.rs @@ -0,0 +1,14 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod client; +pub mod method; diff --git a/KubeOS-Rust/cli/src/method/callable_method.rs b/KubeOS-Rust/cli/src/method/callable_method.rs new file mode 100644 index 00000000..a174b5b8 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/callable_method.rs @@ -0,0 +1,54 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde_json::value::RawValue; + +use super::request::{parse_error, request}; +use crate::client::Client; + +pub trait RpcMethod { + type Response: serde::de::DeserializeOwned; + fn command_name(&self) -> &'static str; + fn command_params(&self) -> Vec>; + fn call(&self, client: &Client) -> Result { + let response = request(client, self.command_name(), self.command_params())?; + response.result().map_err(parse_error) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::client; + + #[derive(Default)] + struct DummyMethod; + + impl RpcMethod for DummyMethod { + type Response = String; + + fn command_name(&self) -> &'static str { + "dummy_command" + } + + fn command_params(&self) -> Vec> { + vec![] + } + } + + #[test] + fn test_call() { + let client = client::Client::new("/tmp/KubeOS-test.sock"); + let result = DummyMethod::default().call(&client); + assert!(result.is_err()); + } +} diff --git a/KubeOS-Rust/cli/src/method/configure.rs b/KubeOS-Rust/cli/src/method/configure.rs new file mode 100644 index 00000000..cca752d0 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/configure.rs @@ -0,0 +1,72 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::{to_raw_value, RawValue}; + +use crate::method::callable_method::RpcMethod; + +pub struct ConfigureMethod { + req: api::ConfigureRequest, +} + +impl ConfigureMethod { + pub fn new(req: api::ConfigureRequest) -> Self { + ConfigureMethod { req } + } + + pub fn set_configure_request(&mut self, req: api::ConfigureRequest) -> &Self { + self.req = req; + self + } +} + +impl RpcMethod for ConfigureMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "configure" + } + fn command_params(&self) -> Vec> { + vec![to_raw_value(&self.req).unwrap()] + } +} +#[cfg(test)] +mod tests { + use kubeos_manager::api::{ConfigureRequest, Sysconfig}; + + use super::*; + + #[test] + fn test_configure_method() { + let req = ConfigureRequest { configs: vec![] }; + let mut method = ConfigureMethod::new(req); + + // Test set_configure_request method + let new_req = ConfigureRequest { + configs: vec![Sysconfig { + model: "model".to_string(), + config_path: "config_path".to_string(), + contents: Default::default(), + }], + }; + method.set_configure_request(new_req); + + // Test command_name method + assert_eq!(method.command_name(), "configure"); + + // Test command_params method + let expected_params = + "RawValue({\"configs\":[{\"model\":\"model\",\"config_path\":\"config_path\",\"contents\":{}}]})"; + let actual_params = format!("{:?}", method.command_params()[0]); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/mod.rs b/KubeOS-Rust/cli/src/method/mod.rs new file mode 100644 index 00000000..e1f38bcd --- /dev/null +++ b/KubeOS-Rust/cli/src/method/mod.rs @@ -0,0 +1,18 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod callable_method; +pub mod configure; +pub mod prepare_upgrade; +pub mod request; +pub mod rollback; +pub mod upgrade; diff --git a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs new file mode 100644 index 00000000..f2034f6b --- /dev/null +++ b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs @@ -0,0 +1,78 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::{to_raw_value, RawValue}; + +use crate::method::callable_method::RpcMethod; + +pub struct PrepareUpgradeMethod { + req: api::UpgradeRequest, +} + +impl PrepareUpgradeMethod { + pub fn new(req: api::UpgradeRequest) -> Self { + PrepareUpgradeMethod { req } + } + + pub fn set_prepare_upgrade_request(&mut self, req: api::UpgradeRequest) -> &Self { + self.req = req; + self + } +} + +impl RpcMethod for PrepareUpgradeMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "prepare_upgrade" + } + fn command_params(&self) -> Vec> { + vec![to_raw_value(&self.req).unwrap()] + } +} +#[cfg(test)] +mod tests { + use kubeos_manager::api::{CertsInfo, UpgradeRequest}; + + use super::*; + + #[test] + fn test_prepare_upgrade_method() { + let req = UpgradeRequest { + version: "v1".into(), + check_sum: "".into(), + image_type: "".into(), + container_image: "".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let mut method = PrepareUpgradeMethod::new(req); + let new_req = UpgradeRequest { + version: "v2".into(), + check_sum: "xxx".into(), + image_type: "xxx".into(), + container_image: "xxx".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + method.set_prepare_upgrade_request(new_req); + assert_eq!(method.command_name(), "prepare_upgrade"); + + let expected_params = "RawValue({\"version\":\"v2\",\"check_sum\":\"xxx\",\"image_type\":\"xxx\",\"container_image\":\"xxx\",\"image_url\":\"\",\"flag_safe\":false,\"mtls\":false,\"certs\":{\"ca_cert\":\"\",\"client_cert\":\"\",\"client_key\":\"\"}})"; + let actual_params = format!("{:?}", method.command_params()[0]); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs new file mode 100644 index 00000000..581aa639 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/request.rs @@ -0,0 +1,88 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::anyhow; +use jsonrpc::{Error, Response}; +use log::debug; +use serde_json::value::RawValue; + +use crate::client::Client; + +pub fn request(client: &Client, command: &str, params: Vec>) -> Result { + let request = client.build_request(command, ¶ms); + let response = client.send_request(request).map_err(parse_error); + debug!("{:#?}", response); + response +} + +pub fn parse_error(error: Error) -> anyhow::Error { + match error { + Error::Transport(e) => { + anyhow!( + "Cannot connect to KubeOS os-agent unix socket, {}", + e.source().map(|e| e.to_string()).unwrap_or_else(|| "Connection timeout".to_string()) + ) + }, + Error::Json(e) => { + debug!("Json parse error: {:?}", e); + anyhow!("Failed to parse response") + }, + Error::Rpc(ref e) => { + if e.message == "Method not found" { + anyhow!("Method is unimplemented") + } else { + anyhow!("{}", e.message) + } + }, + _ => { + debug!("{:?}", error); + anyhow!("Response is invalid") + }, + } +} + +#[cfg(test)] +mod tests { + use jsonrpc::error::RpcError; + use serde::de::Error as DeError; + + use super::*; + + #[test] + fn test_parse_error() { + // Test Error::Transport + let transport_error = + Error::Transport(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "Connection timeout"))); + let result = parse_error(transport_error); + assert_eq!(result.to_string(), "Cannot connect to KubeOS os-agent unix socket, Connection timeout"); + + // Test Error::Json + let json_error = Error::Json(serde_json::Error::custom("Failed to parse response")); + let result = parse_error(json_error); + assert_eq!(result.to_string(), "Failed to parse response"); + + // Test Error::Rpc with "Method not found" message + let rpc_error = Error::Rpc(RpcError { code: -32601, message: "Method not found".to_string(), data: None }); + let result = parse_error(rpc_error); + assert_eq!(result.to_string(), "Method is unimplemented"); + + // Test Error::Rpc with other message + let rpc_error = Error::Rpc(RpcError { code: -32603, message: "Internal server error".to_string(), data: None }); + let result = parse_error(rpc_error); + assert_eq!(result.to_string(), "Internal server error"); + + // Test other Error variant + let other_error = Error::VersionMismatch; + let result = parse_error(other_error); + assert_eq!(result.to_string(), "Response is invalid"); + } +} diff --git a/KubeOS-Rust/cli/src/method/rollback.rs b/KubeOS-Rust/cli/src/method/rollback.rs new file mode 100644 index 00000000..7945f4b1 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/rollback.rs @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::RawValue; + +use crate::method::callable_method::RpcMethod; + +#[derive(Default)] +pub struct RollbackMethod {} + +impl RpcMethod for RollbackMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "rollback" + } + fn command_params(&self) -> Vec> { + vec![] + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_rollback_method() { + let method = RollbackMethod::default(); + assert_eq!(method.command_name(), "rollback"); + let expected_params = "[]"; + let actual_params = format!("{:?}", method.command_params()); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/upgrade.rs b/KubeOS-Rust/cli/src/method/upgrade.rs new file mode 100644 index 00000000..f2f94cd5 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/upgrade.rs @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::RawValue; + +use crate::method::callable_method::RpcMethod; + +#[derive(Default)] +pub struct UpgradeMethod {} + +impl RpcMethod for UpgradeMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "upgrade" + } + fn command_params(&self) -> Vec> { + vec![] + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_upgrade_method() { + let method = UpgradeMethod::default(); + assert_eq!(method.command_name(), "upgrade"); + let expected_params = "[]"; + let actual_params = format!("{:?}", method.command_params()); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/kbimg/Cargo.toml b/KubeOS-Rust/kbimg/Cargo.toml new file mode 100644 index 00000000..cec12b8d --- /dev/null +++ b/KubeOS-Rust/kbimg/Cargo.toml @@ -0,0 +1,16 @@ +[package] +description = "KubeOS kbimg" +edition = "2021" +license = "MulanPSL-2.0" +name = "kbimg" +version = "1.0.6" + +[dependencies] +anyhow = { version = "1.0" } +clap = { version = "=3.2.23", features = ["derive"] } +env_logger = { version = "0.9" } +log = { version = "= 0.4.15" } +regex = { version = "1.7.3" } +serde = { version = "1.0", features = ["derive"] } +sysinfo = { version = "=0.29.11" } +toml = { version = "=0.7.6" } diff --git a/KubeOS-Rust/kbimg/kbimg.toml b/KubeOS-Rust/kbimg/kbimg.toml new file mode 100644 index 00000000..aaa9b48f --- /dev/null +++ b/KubeOS-Rust/kbimg/kbimg.toml @@ -0,0 +1,87 @@ +[from_repo] +agent_path = "/root/KubeOS/bin/os-agent" +image_type = "vm-repo" +legacy_bios = false +repo_path = "/etc/yum.repos.d/openEuler.repo" +root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0" +version = "v1" +rpmlist = [ + "NetworkManager", + "conntrack-tools", + "containernetworking-plugins", + "coreutils", + "dhcp", + "docker", + "dosfstools", + "dracut", + "ebtables", + "ethtool", + "gawk", + "hwinfo", + "kernel", + "kubernetes-kubeadm", + "kubernetes-kubelet", + "lvm2", + "net-tools", + "openssh-server", + "passwd", + "parted", + "rsyslog", + "socat", + "sudo", + "vi", +] + +# [from_dockerimg] +# docker_img = "" +# image_type = "vm-docker" + +# [admin_container] +# dockerfile = "" +# docker_img = "" + +# [pxe_config] +# rootfs_name = "kubeos.tar" +# disk = "/dev/sda" +# server_ip = "192.168.1.50" +# local_ip = "192.168.1.100" +# route_ip = "192.168.1.1" +# netmask = "255.255.255.0" +# net_name = "eth0" + +# [[users]] +# groups = ["admin"] +# name = "foo" +# passwd = "foo" +# sudo = "ALL=(ALL) ALL" + +# [[users]] +# groups = ["example"] +# name = "bar" +# passwd = "bar" + +# [[copy_files]] +# dst = "/root/ztest2" +# src = "/root/KubeOS/ztest2" + +# [[copy_files]] +# dst = "/root/ztest1" +# src = "/root/KubeOS/ztest1/*" + +# [grub] +# passwd = "foo" + +# [systemd_service] +# name = ["kubelet", "containerd"] + +# [chroot_script] +# path = "../../ztest/myscript.sh" + +# [disk_partition] +# first = 100 +# second = 2500 +# third = 2300 +# img_size = 30 + +# [persist_mkdir] +# name = ["lv_data"] diff --git a/KubeOS-Rust/kbimg/src/admin_container.rs b/KubeOS-Rust/kbimg/src/admin_container.rs new file mode 100644 index 00000000..03196739 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/admin_container.rs @@ -0,0 +1,84 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{create_dir_all, File}, + path::PathBuf, +}; + +use anyhow::bail; + +use crate::{ + commands::AdminContainerInfo, + scripts_gen::*, + utils::{self, set_permissions}, + values::*, + Config, CreateImage, +}; + +impl CreateImage for AdminContainerInfo { + fn prepare(&self, _: &mut Config) -> anyhow::Result<()> { + let dockerfile = &self.dockerfile; + let image_name = &self.docker_img; + verify_admin_input(&dockerfile, &image_name)?; + check_dockerfile_valid(&dockerfile)?; + Ok(()) + } + + fn generate_scripts(&self, _: &Config) -> anyhow::Result { + // admin-container + match create_dir_all(ADMIN_CONTAINER_DIR) { + Ok(_) => { + // Dockerfile + let dockerfile_path = format!("{}/{}", ADMIN_CONTAINER_DIR, ADMIN_DOCKERFILE); + let mut dockerfile = File::create(&dockerfile_path)?; + gen_admin_dockerfile(&mut dockerfile)?; + set_permissions(&dockerfile_path, CONFIG_PERMISSION)?; + // set-ssh-pub-key.service + let set_ssh_pub_key_service_path = format!("{}/{}", ADMIN_CONTAINER_DIR, ADMIN_SET_SSH_PUB_KEY_SERVICE); + let mut set_ssh_pub_key_service = File::create(&set_ssh_pub_key_service_path)?; + gen_set_ssh_pub_key_service(&mut set_ssh_pub_key_service)?; + set_permissions(&set_ssh_pub_key_service_path, CONFIG_PERMISSION)?; + // set-ssh-pub-key.sh + let set_ssh_pub_key_path = format!("{}/{}", ADMIN_CONTAINER_DIR, ADMIN_SET_SSH_PUB_KEY_SH); + let mut set_ssh_pub_key = File::create(&set_ssh_pub_key_path)?; + gen_set_ssh_pub_key(&mut set_ssh_pub_key)?; + set_permissions(&set_ssh_pub_key_path, EXEC_PERMISSION)?; + }, + Err(e) => { + bail!(e); + }, + } + // kbimg.sh + let kbimg_path = format!("{}/{}", SCRIPTS_DIR, KBIMG_SH); + let mut kbimg = File::create(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH))?; + gen_admin_vars(&mut kbimg, &self.docker_img, &self.dockerfile)?; + gen_create_admin_img(&mut kbimg)?; + set_permissions(&kbimg_path, EXEC_PERMISSION)?; + + Ok(PathBuf::from(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH))) + } +} + +fn verify_admin_input(dockerfile: &PathBuf, image_name: &str) -> anyhow::Result<()> { + if !utils::is_valid_param(dockerfile.to_str().unwrap()) { + bail!("params {} is invalid, please check input", dockerfile.to_str().unwrap()); + } + if !utils::is_valid_param(image_name) { + bail!("params {} is invalid, please check input", image_name); + } + Ok(()) +} + +fn check_dockerfile_valid(dockerfile: &PathBuf) -> anyhow::Result<()> { + utils::is_file_valid("admin-container Dockerfile", dockerfile) +} diff --git a/KubeOS-Rust/kbimg/src/commands.rs b/KubeOS-Rust/kbimg/src/commands.rs new file mode 100644 index 00000000..f334f048 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/commands.rs @@ -0,0 +1,170 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::path::PathBuf; + +use clap::{Args, Parser, Subcommand}; +use serde::Deserialize; + +#[derive(Parser)] +#[clap(name = "kbimg")] +#[clap(author, version, about)] +#[clap(long_about = "A tool for creating KubeOS images.")] +pub struct Cli { + /// Path to the detailed configuration toml file + #[clap(short, long, value_parser)] + pub config: Option, + /// Enable debug mode, keep the scripts after execution + #[clap(short, long, action)] + pub debug: bool, + #[clap(subcommand)] + pub commands: Option, +} + +#[derive(Subcommand, Debug, Deserialize)] +pub enum Commands { + /// Create a new container image for upgrading KubeOS + #[clap(name = "upgrade")] + UpgradeImage(RepoInfo), + /// Create a new KubeOS vm image from repo + #[clap(name = "vm-repo")] + VMRepo(RepoInfo), + /// Create a new KubeOS vm image from docker image + #[clap(name = "vm-docker")] + VMDocker(DockerInfo), + /// Create a new KubeOS pxe image from repo + #[clap(name = "pxe-repo")] + PxeRepo(RepoInfo), + /// Create a new KubeOS pxe image from docker image + #[clap(name = "pxe-docker")] + PxeDocker(DockerInfo), + /// Create a KubeOS admin-container image + #[clap(name = "admin-container")] + AdminContainer(AdminContainerInfo), +} + +#[derive(Args, Debug, Deserialize, Clone)] +pub struct RepoInfo { + /// Required: KubeOS version + #[clap(short, long, value_parser)] + pub version: String, + /// Required: Repo path for installing packages + #[clap(short = 'p', long, value_parser)] + pub repo_path: PathBuf, + /// Required: Path to the agent binary + #[clap(short = 'b', long, value_parser)] + pub agent_path: PathBuf, + /// Required: Encrypted password for root user + #[clap(short = 'e', long, value_parser)] + pub root_passwd: String, + /// Required for upgrade + #[clap(short = 'd', long, value_parser)] + pub docker_img: Option, + /// Required: RPM packages + #[clap(short = 'r', long, value_parser)] + pub rpmlist: Vec, + /// Optional: boot mode, default is uefi, enable this flag for legacy bios + #[clap(short, long, value_parser)] + pub legacy_bios: bool, + #[clap(skip)] + pub image_type: String, + #[clap(skip)] + pub arch: Option, +} + +#[derive(Args, Debug, Deserialize, Clone)] +pub struct DockerInfo { + /// Required: Name of the container image + #[clap(short, long, value_parser)] + pub docker_img: String, + #[clap(skip)] + pub image_type: String, +} + +#[derive(Args, Debug, Deserialize, Clone)] +pub struct AdminContainerInfo { + /// Required: Name of the container image + #[clap(short, long, value_parser)] + pub docker_img: String, + /// Required: Path to the Dockerfile + #[clap(short, long, value_parser)] + pub dockerfile: PathBuf, +} + +#[derive(Debug, Deserialize, Default, Clone)] +pub struct Config { + pub from_repo: Option, + pub from_dockerimg: Option, + pub admin_container: Option, + pub users: Option>, + pub copy_files: Option>, + pub grub: Option, + pub systemd_service: Option, + pub chroot_script: Option, + pub disk_partition: Option, + pub persist_mkdir: Option, + pub pxe_config: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct User { + pub name: String, + pub passwd: String, + pub groups: Option>, + pub sudo: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CopyFile { + pub src: String, + pub dst: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Grub { + pub passwd: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct SystemdService { + pub name: Vec, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct ChrootScript { + pub path: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct DiskPartition { + pub first: u32, + pub second: u32, + pub third: u32, + pub img_size: u32, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct PersistMkdir { + pub name: Vec, +} + +// pxe config +#[derive(Debug, Deserialize, Clone)] +pub struct PxeConfig { + pub rootfs_name: String, + pub disk: String, + pub server_ip: String, + pub local_ip: String, + pub route_ip: String, + pub netmask: String, + pub net_name: String, +} diff --git a/KubeOS-Rust/kbimg/src/docker_img.rs b/KubeOS-Rust/kbimg/src/docker_img.rs new file mode 100644 index 00000000..9c5a8c20 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/docker_img.rs @@ -0,0 +1,74 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs::File, path::PathBuf, process::Command, str}; + +use anyhow::bail; + +use crate::{ + commands::DockerInfo, + scripts_gen::*, + utils::{self, set_permissions}, + values::*, + Config, CreateImage, +}; + +impl CreateImage for DockerInfo { + fn prepare(&self, _: &mut Config) -> anyhow::Result<()> { + let image_name = &self.docker_img; + verify_docker_input(&image_name)?; + check_docker_image(&image_name)?; + Ok(()) + } + + fn generate_scripts(&self, config: &Config) -> anyhow::Result { + // kbimg.sh + let kbimg_path = format!("{}/{}", SCRIPTS_DIR, KBIMG_SH); + let mut kbimg = File::create(&kbimg_path)?; + gen_global_vars(&mut kbimg)?; + gen_docker_vars(&mut kbimg, &self.docker_img)?; + gen_global_func(&mut kbimg)?; + gen_create_os_tar_from_docker(&mut kbimg)?; + if self.image_type == "vm-docker" { + // kbimg.sh + gen_init_part(&mut kbimg)?; + gen_create_img(&mut kbimg, false, &config)?; + gen_create_vm_docker_img(&mut kbimg)?; + } else { + // kbimg.sh + gen_create_pxe_docker_img(&mut kbimg)?; + } + set_permissions(&kbimg_path, EXEC_PERMISSION)?; + + Ok(PathBuf::from(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH))) + } +} + +fn verify_docker_input(image_name: &str) -> anyhow::Result<()> { + if !utils::is_valid_param(image_name) { + bail!("params {} is invalid, please check input", image_name); + } + Ok(()) +} + +fn check_docker_image(image_name: &str) -> anyhow::Result<()> { + let output = + Command::new("docker").args(&["images", "-q", image_name]).output().expect("Failed to execute command"); + + if output.status.success() { + let stdout = str::from_utf8(&output.stdout).expect("Invalid UTF-8 output"); + if stdout.trim().is_empty() { + bail!("docker image does NOT exist, please pull {} first.", image_name); + } + } + Ok(()) +} diff --git a/KubeOS-Rust/kbimg/src/main.rs b/KubeOS-Rust/kbimg/src/main.rs new file mode 100644 index 00000000..beb389d5 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/main.rs @@ -0,0 +1,122 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs, path::PathBuf, process::exit}; + +use anyhow::{bail, Result}; +use clap::Parser; +use env_logger::{Builder, Env, Target}; +use log::{debug, error, info}; + +mod admin_container; +mod commands; +mod docker_img; +mod repo; +mod scripts_gen; +mod utils; +mod values; + +use utils::{execute_scripts, get_arch}; +use values::SCRIPTS_DIR; + +use crate::commands::{Cli, Commands, Config}; + +trait CreateImage { + /// validate cmd args, check disk size and other prepare work + fn prepare(&self, config: &mut Config) -> Result<()>; + /// generate scripts for creating image. If debug is enabled, keep the scripts, otherwise execute them + fn generate_scripts(&self, config: &Config) -> Result; +} + +fn process(info: Box, mut config: Config) -> Result<()> { + match fs::create_dir_all(SCRIPTS_DIR) { + Ok(_) => { + info.prepare(&mut config)?; + let path = info.generate_scripts(&config)?; + execute_scripts(path)?; + Ok(()) + }, + Err(e) => bail!(e), + } +} + +fn main() { + let cli = Cli::parse(); + let default_log_level: &str = if cli.debug { "debug" } else { "info" }; + Builder::from_env(Env::default().default_filter_or(default_log_level)).target(Target::Stdout).init(); + match cli.config { + Some(config) => { + info!("Loading config file"); + debug!("Config file path: {:?}", config); + let content = fs::read_to_string(config).unwrap(); + let data: Config = toml::from_str(&content).unwrap(); + debug!("Config: {:?}", data); + let info = if let Some(mut info) = data.from_repo.clone() { + info.arch = Some(get_arch()); + Some(Box::new(info) as Box) + } else if let Some(info) = data.from_dockerimg.clone() { + Some(Box::new(info) as Box) + } else if let Some(info) = data.admin_container.clone() { + Some(Box::new(info) as Box) + } else { + None + }; + if let Some(i) = info { + match process(i, data) { + Ok(_) => { + info!("Image created successfully"); + }, + Err(e) => { + error!("Failed to create image: {:?}", e); + }, + } + } + exit(0); + }, + None => {}, + } + let info = match cli.commands { + Some(Commands::UpgradeImage(mut info)) => { + info.image_type = "upgrade".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::VMRepo(mut info)) => { + info.image_type = "vm-repo".to_string(); + debug!("VMRepo: {:?}", info); + Some(Box::new(info) as Box) + }, + Some(Commands::VMDocker(mut info)) => { + info.image_type = "vm-docker".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::PxeRepo(mut info)) => { + info.image_type = "pxe-repo".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::PxeDocker(mut info)) => { + info.image_type = "pxe-docker".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::AdminContainer(info)) => Some(Box::new(info) as Box), + None => None, + }; + if let Some(i) = info { + match process(i, Config::default()) { + Ok(_) => { + info!("Image created successfully"); + }, + Err(e) => { + error!("Failed to create image: {:?}", e); + }, + } + } +} diff --git a/KubeOS-Rust/kbimg/src/repo.rs b/KubeOS-Rust/kbimg/src/repo.rs new file mode 100644 index 00000000..43aabd3b --- /dev/null +++ b/KubeOS-Rust/kbimg/src/repo.rs @@ -0,0 +1,203 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + env, + fs::{create_dir_all, File}, + path::PathBuf, +}; + +use anyhow::bail; +use sysinfo::{System, SystemExt, DiskExt}; + +use crate::{ + commands::RepoInfo, + scripts_gen::*, + utils::{self, check_pxe_conf_valid, set_permissions}, + values::*, + Config, CreateImage, +}; + +impl CreateImage for RepoInfo { + fn prepare(&self, config: &mut Config) -> anyhow::Result<()> { + verify_repo_input(&self)?; + check_disk_space(&self.image_type)?; + check_repo_file_valid(&self.repo_path)?; + check_agent_file_valid(&self.agent_path)?; + if self.image_type == "pxe-repo" { + if let Some(pxe_config) = &config.pxe_config { + check_pxe_conf_valid(&pxe_config)?; + } else { + bail!("pxe config not found!") + } + } + Ok(()) + } + + fn generate_scripts(&self, config: &Config) -> anyhow::Result { + // rpmlist + let rpmlist_path = format!("{}/{}", SCRIPTS_DIR, RPMLIST); + let mut rpmlist = File::create(&rpmlist_path)?; + gen_rpm_list(&mut rpmlist, &self.rpmlist)?; + set_permissions(&rpmlist_path, CONFIG_PERMISSION)?; + // 00bootup + match create_dir_all(BOOTUP_DIR) { + Ok(_) => { + if let Some(pxe_config) = &config.pxe_config { + let global_cfg_path = format!("{}/{}", BOOTUP_DIR, BOOTUP_GLOBAL_CFG); + let mut global_cfg = File::create(&global_cfg_path)?; + gen_global_cfg(&mut global_cfg, &pxe_config)?; + set_permissions(&global_cfg_path, CONFIG_PERMISSION)?; + } + let module_setup_path = format!("{}/{}", BOOTUP_DIR, BOOTUP_MODULE_SETUP_SH); + let mut module_setup = File::create(&module_setup_path)?; + gen_module_setup(&mut module_setup)?; + set_permissions(&module_setup_path, EXEC_PERMISSION)?; + let mount_path = format!("{}/{}", BOOTUP_DIR, BOOTUP_MOUNT_SH); + let mut mount = File::create(&mount_path)?; + gen_mount(&mut mount)?; + set_permissions(&mount_path, EXEC_PERMISSION)?; + }, + Err(e) => { + bail!(e); + }, + } + // misc-files + match create_dir_all(MISC_FILES_DIR) { + Ok(_) => { + let boot_efi_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_BOOT_EFI_MOUNT); + let mut boot_efi_mount = File::create(&boot_efi_mount_path)?; + gen_boot_efi_mount(&mut boot_efi_mount)?; + set_permissions(&boot_efi_mount_path, CONFIG_PERMISSION)?; + let boot_grub2_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_BOOT_GRUB2_MOUNT); + let mut boot_grub2_mount = File::create(&boot_grub2_mount_path)?; + gen_boot_grub2_mount(&mut boot_grub2_mount)?; + set_permissions(&boot_grub2_mount_path, CONFIG_PERMISSION)?; + let etc_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_ETC_MOUNT); + let mut etc_mount = File::create(&etc_mount_path)?; + gen_etc_mount(&mut etc_mount)?; + set_permissions(&etc_mount_path, CONFIG_PERMISSION)?; + let os_agent_service_path = format!("{}/{}", MISC_FILES_DIR, MISC_OS_AGENT_SERVICE); + let mut os_agent_service = File::create(&os_agent_service_path)?; + gen_os_agent_service(&mut os_agent_service)?; + set_permissions(&os_agent_service_path, CONFIG_PERMISSION)?; + let os_release_path = format!("{}/{}", MISC_FILES_DIR, MISC_OS_RELEASE); + let mut os_release = File::create(&os_release_path)?; + gen_os_release(&mut os_release)?; + set_permissions(&os_release_path, CONFIG_PERMISSION)?; + let persist_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_PERSIST_MOUNT); + let mut persist_mount = File::create(&persist_mount_path)?; + gen_persist_mount(&mut persist_mount)?; + set_permissions(&persist_mount_path, CONFIG_PERMISSION)?; + let var_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_VAR_MOUNT); + let mut var_mount = File::create(&var_mount_path)?; + gen_var_mount(&mut var_mount)?; + set_permissions(&var_mount_path, CONFIG_PERMISSION)?; + }, + Err(e) => { + bail!(e); + }, + } + // grub.cfg + let grub_cfg_path = format!("{}/{}", SCRIPTS_DIR, GRUB_CFG); + let mut grub_cfg = File::create(&grub_cfg_path)?; + gen_grub_cfg(&mut grub_cfg)?; + set_permissions(&grub_cfg_path, CONFIG_PERMISSION)?; + // set_in_chroot.sh + let set_in_chroot_path = format!("{}/{}", SCRIPTS_DIR, SET_IN_CHROOT_SH); + let mut set_in_chroot = File::create(&set_in_chroot_path)?; + gen_set_in_chroot(&mut set_in_chroot, self.legacy_bios, &config)?; + set_permissions(&set_in_chroot_path, EXEC_PERMISSION)?; + // kbimg.sh + let kbimg_path = format!("{}/{}", SCRIPTS_DIR, KBIMG_SH); + let mut kbimg = File::create(&kbimg_path)?; + gen_global_vars(&mut kbimg)?; + gen_repo_vars(&mut kbimg, &self)?; + gen_global_func(&mut kbimg)?; + gen_mount_proc_dev_sys(&mut kbimg)?; + gen_unmount_dir(&mut kbimg)?; + gen_create_os_tar_from_repo(&mut kbimg, &self, &config)?; + if self.image_type == "vm-repo" { + // bootloader.sh + let bootloader_path = format!("{}/{}", SCRIPTS_DIR, BOOTLOADER_SH); + let mut bootloader = File::create(&bootloader_path)?; + gen_bootloader(&mut bootloader, self.arch.as_ref().unwrap(), self.legacy_bios)?; + set_permissions(&bootloader_path, EXEC_PERMISSION)?; + // kbimg.sh + gen_init_part(&mut kbimg)?; + gen_create_img(&mut kbimg, self.legacy_bios, &config)?; + gen_create_vm_repo_img(&mut kbimg)?; + } else if self.image_type == "pxe-repo" { + // kbimg.sh + gen_create_pxe_repo_img(&mut kbimg)?; + } else { + // Dockerfile + let dockerfile_path = format!("{}/{}", SCRIPTS_DIR, DOCKERFILE); + let mut dockerfile = File::create(&dockerfile_path)?; + gen_dockerfile(&mut dockerfile)?; + set_permissions(&dockerfile_path, CONFIG_PERMISSION)?; + // kbimg.sh + gen_create_docker_img(&mut kbimg)?; + } + set_permissions(&kbimg_path, EXEC_PERMISSION)?; + + Ok(PathBuf::from(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH))) + } +} + +fn verify_repo_input(info: &RepoInfo) -> anyhow::Result<()> { + if !utils::is_valid_param(info.repo_path.to_str().unwrap()) { + bail!("params {} is invalid, please check input", info.repo_path.to_str().unwrap()); + } + if !utils::is_valid_param(&info.version) { + bail!("params {} is invalid, please check input", info.version); + } + if !utils::is_valid_param(info.agent_path.to_str().unwrap()) { + bail!("params {} is invalid, please check input", info.agent_path.to_str().unwrap()); + } + if let Some(docker_img) = &info.docker_img { + if !utils::is_valid_param(docker_img) { + bail!("params {} is invalid, please check input", docker_img); + } + } + Ok(()) +} + +fn check_disk_space(image_type: &str) -> anyhow::Result<()> { + let max_size: u64 = match image_type { + "upgrade" => 6, + "vm-repo" => 25, + "pxe-repo" => 5, + _ => bail!("Invalid image type: {}", image_type), + }; + + let current_dir = env::current_dir().expect("Failed to get current directory"); + let root_dir = current_dir.ancestors().last().expect("Failed to get current directory").to_path_buf(); + let mut sys = System::new_all(); + sys.refresh_all(); + for d in sys.disks() { + if d.mount_point() == root_dir { + if d.available_space() < max_size * 1024 * 1024 { + bail!("The available disk space is not enough, at least {}GiB.", max_size); + } + } + } + Ok(()) +} + +fn check_repo_file_valid(repo_path: &PathBuf) -> anyhow::Result<()> { + utils::is_file_valid("REPO file", repo_path) +} + +fn check_agent_file_valid(agent_path: &PathBuf) -> anyhow::Result<()> { + utils::is_file_valid("os-agent binary", agent_path) +} diff --git a/KubeOS-Rust/kbimg/src/scripts_gen.rs b/KubeOS-Rust/kbimg/src/scripts_gen.rs new file mode 100644 index 00000000..7d574888 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/scripts_gen.rs @@ -0,0 +1,1754 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs::File, io::Write, path::PathBuf}; + +use anyhow::{bail, Ok, Result}; + +use crate::{commands::*, values::SCRIPTS_DIR}; + +/* copyright */ +pub(crate) fn gen_copyright(file: &mut File) -> Result<()> { + writeln!( + file, + r#"## Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +# KubeOS is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +## See the Mulan PSL v2 for more details. +"# + )?; + + Ok(()) +} + +/* region: kbimg.sh */ +pub(crate) fn gen_global_vars(file: &mut File) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"set -e + +SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd) +LOCK="${{SCRIPTS_DIR}}"/test.lock +RPM_ROOT="${{SCRIPTS_DIR}}"/rootfs +TMP_MOUNT_PATH="${{SCRIPTS_DIR}}"/mnt +"# + )?; + Ok(()) +} + +pub(crate) fn gen_global_func(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function delete_dir() {{ + local ret=0 + local dir="$1" + unmount_dir "${{dir}}" + ret=$? + if [ "${{ret}}" -eq 0 ]; then + rm -rf "${{dir}}" + return 0 + else + log_error_print "${{dir}} is failed to unmount , can not delete ${{dir}}." + return 1 + fi +}} + +function delete_file() {{ + local file="$1" + if [ ! -e "${{file}}" ]; then + return 0 + fi + + if [ ! -f "${{file}}" ]; then + log_error_print "${{file}} is not a file." + return 1 + fi + + rm -f "${{file}}" + return 0 +}} + +function clean_space() {{ + delete_dir "${{RPM_ROOT}}" + delete_dir "${{TMP_MOUNT_PATH}}" + delete_file "${{SCRIPTS_DIR}}"/os.tar + rm -rf "${{LOCK}}" + delete_file "${{ADMIN_CONTAINER_DIR}}"/hostshell +}} + +function clean_img() {{ + delete_file "${{SCRIPTS_DIR}}"/system.img + delete_file "${{SCRIPTS_DIR}}"/update.img + delete_file "${{SCRIPTS_DIR}}"/initramfs.img + delete_file "${{SCRIPTS_DIR}}"/kubeos.tar +}} + +function file_lock() {{ + local lock_file=$1 + exec {{lock_fd}}>"${{lock_file}}" + flock -xn "${{lock_fd}}" +}} + +function test_lock() {{ + file_lock "${{LOCK}}" + local status=$? + if [ $status -ne 0 ]; then + log_error_print "There is already an generate process running." + exit 203 + fi +}} + +function log_error_print() {{ + local logmsg + logmsg="[ ERROR ] - ""$(date "+%b %d %Y %H:%M:%S")"" $1" + echo "$logmsg" +}} + +function log_info_print() {{ + local logmsg + logmsg="[ INFO ] - ""$(date "+%b %d %Y %H:%M:%S")"" $1" + echo "$logmsg" +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_mount_proc_dev_sys(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function mount_proc_dev_sys() {{ + local tmp_root=$1 + mount -t proc none "${{tmp_root}}"/proc + mount --bind /dev "${{tmp_root}}"/dev + mount --bind /dev/pts "${{tmp_root}}"/dev/pts + mount -t sysfs none "${{tmp_root}}"/sys +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_unmount_dir(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function unmount_dir() {{ + local dir=$1 + + if [ -L "${{dir}}" ] || [ -f "${{dir}}" ]; then + log_error_print "${{dir}} is not a directory, please check it." + return 1 + fi + + if [ ! -d "${{dir}}" ]; then + return 0 + fi + + local real_dir + real_dir=$(readlink -e "${{dir}}") + local mnts + mnts=$(awk '{{print $2}}' < /proc/mounts | grep "^${{real_dir}}" | sort -r) + for m in ${{mnts}}; do + log_info_print "Unmount ${{m}}" + umount -f "${{m}}" || true + done + + return 0 +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_init_part(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function init_part() {{ + local offset + offset=$(fdisk -l "${{SCRIPTS_DIR}}"/system.img | grep "$1" | awk '{{print $2}}') + local sizelimit + sizelimit=$(fdisk -l "${{SCRIPTS_DIR}}"/system.img | grep "$1" | awk '{{print $3}}') + sizelimit=$(echo "($sizelimit - $offset)*512" | bc) + offset=$(echo "${{offset}}*512" | bc) + local loop + loop=$(losetup -f) + losetup -o "${{offset}}" --sizelimit "${{sizelimit}}" "${{loop}}" "${{SCRIPTS_DIR}}"/system.img + if [ "$2" == "BOOT" ];then + mkfs.vfat -n "$2" "${{loop}}" + mount -t vfat "${{loop}}" "$3" + else + mkfs.ext4 -L "$2" "${{loop}}" + mount -t ext4 "${{loop}}" "$3" + rm -rf "$3/lost+found" + fi +}} +"# + )?; + Ok(()) +} + +// repo +pub(crate) fn gen_repo_vars(file: &mut File, info: &RepoInfo) -> Result<()> { + writeln!( + file, + r#"REPO_PATH="{}" +VERSION="{}" +AGENT_PATH="{}" +ROOT_PASSWD='{}' +BOOT_MODE="{}" +"#, + info.repo_path.to_str().unwrap(), + &info.version, + info.agent_path.to_str().unwrap(), + &info.root_passwd, + if info.legacy_bios { "legacy" } else { "efi" } + )?; + if let Some(docker_img) = &info.docker_img { + writeln!(file, "DOCKER_IMG=\"{}\"\n", docker_img)?; + } + Ok(()) +} + +pub(crate) fn gen_prepare_yum(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function prepare_yum() {{ + # init rpmdb + rpm --root "${{RPM_ROOT}}" --initdb + mkdir -p "${{RPM_ROOT}}"{{/etc/yum.repos.d,/persist,/proc,/dev/pts,/sys}} + mount_proc_dev_sys "${{RPM_ROOT}}" + # init yum repo + local iso_repo="${{RPM_ROOT}}"/etc/yum.repos.d/iso.repo + cat "${{REPO_PATH}}" > "$iso_repo" +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_install_packages(file: &mut File, arch: &str, legacy_bios: bool) -> Result<()> { + writeln!( + file, + r#"function install_packages() {{ + prepare_yum "${{REPO_PATH}}" + + echo "install package.." + + local filesize + filesize=$(stat -c "%s" "${{SCRIPTS_DIR}}"/rpmlist) + local maxsize=$((1024*1024)) + if [ "${{filesize}}" -gt "${{maxsize}}" ]; then + echo "please check if rpmlist is too big or something wrong" + exit 7 + fi + + local rpms_name + rpms_name=$(tr "\n" " " < "${{SCRIPTS_DIR}}"/rpmlist) + old_ifs="$IFS" + IFS=' '"# + )?; + + if arch == "x86_64" { + if legacy_bios { + writeln!(file, "\trpms_name+=\" grub2\"")?; + } else { + writeln!(file, "\trpms_name+=\" grub2-efi grub2-tools grub2-efi-x64-modules grub2-pc-modules\"")?; + } + writeln!( + file, + r#" read -ra rpms <<< "${{rpms_name}}" + IFS="$old_ifs" + yum -y --installroot="${{RPM_ROOT}}" install --nogpgcheck --setopt install_weak_deps=False "${{rpms[@]}}""# + )?; + } else if arch == "aarch64" { + writeln!( + file, + r#" read -ra rpms <<< "${{rpms_name}}" + IFS="$old_ifs" + yum -y --installroot="${{RPM_ROOT}}" install --nogpgcheck --setopt install_weak_deps=False "${{rpms[@]}}" grub2-efi grub2-tools grub2-efi-aa64-modules"# + )?; + } + writeln!( + file, + r#" yum -y --installroot="${{RPM_ROOT}}" clean all +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_copy_files(file: &mut File, copy_files: &Vec) -> Result<()> { + writeln!(file, "function copy_files() {{")?; + for copy_file in copy_files { + let dst = format!("{}/rootfs{}", SCRIPTS_DIR, ©_file.dst); + let dst = PathBuf::from(dst); + if !dst.exists() { + writeln!(file, "\tmkdir -p \"${{RPM_ROOT}}{}\"", ©_file.dst)?; + } + let src = PathBuf::from(©_file.src); + if src.is_dir() { + writeln!(file, "\tcp -r {} \"${{RPM_ROOT}}{}\"", ©_file.src, ©_file.dst)?; + } else { + writeln!(file, "\tcp {} \"${{RPM_ROOT}}{}\"", ©_file.src, ©_file.dst)?; + } + } + writeln!(file, "}}\n")?; + Ok(()) +} + +pub(crate) fn gen_grub_config(file: &mut File, legacy_bios: bool, grub: &Grub) -> Result<()> { + writeln!( + file, + r#"function grub_config() {{ + local GRUB_PATH"# + )?; + if legacy_bios { + writeln!(file, "\tGRUB_PATH=\"${{RPM_ROOT}}\"/boot/grub2")?; + } else { + writeln!(file, "\tGRUB_PATH=\"${{RPM_ROOT}}\"/boot/efi/EFI/openEuler")?; + } + if let Some(grub_passwd) = &grub.passwd { + writeln!( + file, + r#" local GRUB_PASSWD + GRUB_PASSWD=$(echo -e "{}\n{}" | grub2-mkpasswd-pbkdf2 | grep PBKDF2 | awk '{{print $7}}') + echo "GRUB2_PASSWD=${{GRUB_PASSWD}}" > "${{GRUB_PATH}}"/user.cfg + chmod 600 "${{GRUB_PATH}}"/user.cfg +}} +"#, + grub_passwd, grub_passwd + )?; + } + Ok(()) +} + +pub(crate) fn gen_chroot_script(file: &mut File, chroot_script: &ChrootScript) -> Result<()> { + let script_path = PathBuf::from(&chroot_script.path); + match script_path.canonicalize() { + core::result::Result::Ok(absolute_path) => { + if let Some(script_name) = absolute_path.file_name() { + writeln!( + file, + r#"function chroot_script() {{ + cp "{}" "${{RPM_ROOT}}" + chroot "${{RPM_ROOT}}" bash /{} +}} +"#, + absolute_path.as_path().to_str().unwrap(), + script_name.to_str().unwrap() + )?; + } + Ok(()) + }, + Err(e) => bail!(e), + } +} + +pub(crate) fn gen_install_misc(file: &mut File, legacy_bios: bool, config: &Config) -> Result<()> { + if let Some(copy_files) = &config.copy_files { + gen_copy_files(file, ©_files)?; + } + if let Some(grub) = &config.grub { + gen_grub_config(file, legacy_bios, &grub)?; + } + if let Some(chroot_script) = &config.chroot_script { + gen_chroot_script(file, &chroot_script)?; + } + + writeln!( + file, + r#"function install_misc() {{ + cp "${{SCRIPTS_DIR}}"/misc-files/*mount "${{SCRIPTS_DIR}}"/misc-files/os-agent.service "${{RPM_ROOT}}"/usr/lib/systemd/system/ + cp "${{SCRIPTS_DIR}}"/misc-files/os-release "${{RPM_ROOT}}"/usr/lib/ + cp "${{AGENT_PATH}}" "${{RPM_ROOT}}"/usr/bin + rm "${{RPM_ROOT}}"/etc/os-release + + cat < "${{RPM_ROOT}}"/usr/lib/os-release +NAME=${{NAME}} +ID=${{NAME}} +EOF + echo "PRETTY_NAME=\"${{NAME}} ${{VERSION}}\"" >> "${{RPM_ROOT}}"/usr/lib/os-release + echo "VERSION_ID=${{VERSION}}" >> "${{RPM_ROOT}}"/usr/lib/os-release + mv "${{RPM_ROOT}}"/boot/vmlinuz* "${{RPM_ROOT}}"/boot/vmlinuz + mv "${{RPM_ROOT}}"/boot/initramfs* "${{RPM_ROOT}}"/boot/initramfs.img"# + )?; + + if legacy_bios { + writeln!( + file, + r#" cp "${{SCRIPTS_DIR}}"/grub.cfg "${{RPM_ROOT}}"/boot/grub2 + sed -i "s/insmod part_gpt/insmod part_msdos/g; \ +s/set root='hd0,gpt2'/set root='hd0,msdos2'/g; \ +s/set root='hd0,gpt3'/set root='hd0,msdos3'/g" \ +"${{RPM_ROOT}}"/boot/grub2/grub.cfg"# + )?; + } else { + writeln!(file, "\tcp \"${{SCRIPTS_DIR}}\"/grub.cfg \"${{RPM_ROOT}}\"/boot/efi/EFI/openEuler")?; + } + + writeln!( + file, + r#" cp -r "${{SCRIPTS_DIR}}"/00bootup "${{RPM_ROOT}}"/usr/lib/dracut/modules.d/ + cp "${{SCRIPTS_DIR}}"/set_in_chroot.sh "${{RPM_ROOT}}" + + # (optional) custom config"# + )?; + + if let Some(_) = &config.copy_files { + writeln!(file, "\tcopy_files")?; + } + if let Some(_) = &config.grub { + writeln!(file, "\tgrub_config")?; + } + if let Some(_) = &config.chroot_script { + writeln!(file, "\tchroot_script")?; + } + + writeln!( + file, + r#" + ROOT_PASSWD="${{ROOT_PASSWD}}" BOOT_MODE="${{BOOT_MODE}}" chroot "${{RPM_ROOT}}" bash /set_in_chroot.sh + rm "${{RPM_ROOT}}/set_in_chroot.sh" +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_os_tar_from_repo(file: &mut File, info: &RepoInfo, config: &Config) -> Result<()> { + gen_prepare_yum(file)?; + gen_install_packages(file, info.arch.as_ref().unwrap(), info.legacy_bios)?; + gen_install_misc(file, info.legacy_bios, config)?; + + writeln!( + file, + r#"function create_os_tar_from_repo() {{ + install_packages + install_misc + unmount_dir "${{RPM_ROOT}}" + tar -C "${{RPM_ROOT}}" -cf "${{SCRIPTS_DIR}}"/os.tar . +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_img(file: &mut File, legacy_bios: bool, config: &Config) -> Result<()> { + let (first, second, third, img_size) = if let Some(disk_partition) = &config.disk_partition { + let first = disk_partition.first; + let second = disk_partition.second; + let third = disk_partition.third; + let img_size = disk_partition.img_size; + if first + second + third + 2100 > img_size * 1024 { + bail!("Image size({}G) is not enough for partitions, please check input", img_size) + } + (first, first + second, first + second + third, img_size) + } else { + (60, 2160, 4260, 20) + }; + + writeln!( + file, + r#"function create_img() {{ + rm -f "${{SCRIPTS_DIR}}"/system.img "${{SCRIPTS_DIR}}/update.img" + qemu-img create "${{SCRIPTS_DIR}}/system.img" {}G"#, + img_size + )?; + + if legacy_bios { + writeln!( + file, + r#" local BOOT_PATH=${{TMP_MOUNT_PATH}}/boot/grub2 + parted "${{SCRIPTS_DIR}}/system.img" -s mklabel msdos + parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 1MiB {}MiB"#, + first + )?; + } else { + writeln!( + file, + r#" local BOOT_PATH=${{TMP_MOUNT_PATH}}/boot/efi + parted "${{SCRIPTS_DIR}}/system.img" -s mklabel gpt + parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary fat32 1MiB {}MiB"#, + first + )?; + } + + writeln!( + file, + r#" parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 {}MiB {}MiB + parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 {}MiB {}MiB + parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 {}MiB 100%"#, + first, second, second, third, third + )?; + + writeln!( + file, + r#" local device + device=$(losetup -f) + losetup "${{device}}" "${{SCRIPTS_DIR}}"/system.img + + mkdir -p "${{TMP_MOUNT_PATH}}" + + init_part "${{SCRIPTS_DIR}}"/system.img2 ROOT-A "${{TMP_MOUNT_PATH}}" + + mkdir -p "${{BOOT_PATH}}" + chmod 755 "${{BOOT_PATH}}""# + )?; + + if legacy_bios { + writeln!( + file, + r#" init_part "${{SCRIPTS_DIR}}"/system.img1 GRUB2 "${{BOOT_PATH}}" + tar -x -C "${{TMP_MOUNT_PATH}}" -f "${{SCRIPTS_DIR}}"/os.tar + sed -i "s/insmod part_gpt/insmod part_msdos/g; \ +s/set root='hd0,gpt2'/set root='hd0,msdos2'/g; \ +s/set root='hd0,gpt3'/set root='hd0,msdos3'/g" \ +"${{TMP_MOUNT_PATH}}"/boot/grub2/grub.cfg"# + )?; + } else { + writeln!( + file, + r#" init_part "${{SCRIPTS_DIR}}"/system.img1 BOOT "${{BOOT_PATH}}" + tar -x -C "${{TMP_MOUNT_PATH}}" -f "${{SCRIPTS_DIR}}"/os.tar"# + )?; + } + + writeln!( + file, + r#" sync + cp "${{SCRIPTS_DIR}}"/bootloader.sh "${{TMP_MOUNT_PATH}}" + mount_proc_dev_sys "${{TMP_MOUNT_PATH}}" + DEVICE="${{device}}" BOOT_MODE="${{BOOT_MODE}}" chroot "${{TMP_MOUNT_PATH}}" bash bootloader.sh + rm -rf "${{TMP_MOUNT_PATH}}"/bootloader.sh + sync + + dd if=/dev/disk/by-label/ROOT-A of="${{SCRIPTS_DIR}}"/update.img bs=8M + sync + unmount_dir "${{TMP_MOUNT_PATH}}" + init_part "${{SCRIPTS_DIR}}"/system.img3 ROOT-B "${{TMP_MOUNT_PATH}}" + umount "${{TMP_MOUNT_PATH}}" + + init_part "${{SCRIPTS_DIR}}"/system.img4 PERSIST "${{TMP_MOUNT_PATH}}" + mkdir "${{TMP_MOUNT_PATH}}"/{{var,etc,etcwork}}"# + )?; + + if let Some(persist_mkdir) = &config.persist_mkdir { + for name in &persist_mkdir.name { + writeln!(file, "\tmkdir \"${{TMP_MOUNT_PATH}}\"/{}", name)?; + } + } + + writeln!( + file, + r#" mkdir -p "${{TMP_MOUNT_PATH}}"/etc/KubeOS/certs + umount "${{TMP_MOUNT_PATH}}" + + losetup -D + parted "${{SCRIPTS_DIR}}"/system.img -- set 1 boot on + qemu-img convert "${{SCRIPTS_DIR}}"/system.img -O qcow2 "${{SCRIPTS_DIR}}"/system.qcow2 +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_vm_repo_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_vm_repo_img() {{ + create_os_tar_from_repo + create_img +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_vm_repo_img"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_pxe_repo_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_pxe_repo_img() {{ + rm -rf "${{SCRIPTS_DIR}}"/initramfs.img "${{SCRIPTS_DIR}}"/kubeos.tar + create_os_tar_from_repo + tar -xvf "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/initramfs.img + mv "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/kubeos.tar +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_pxe_repo_img"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_docker_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_docker_img() {{ + create_os_tar_from_repo + docker build -t "${{DOCKER_IMG}}" -f "${{SCRIPTS_DIR}}"/Dockerfile . +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_docker_img"# + )?; + Ok(()) +} + +// docker +pub(crate) fn gen_docker_vars(file: &mut File, image_name: &str) -> Result<()> { + writeln!( + file, + r#" +IMAGE_NAME="{}" +BOOT_MODE=efi +"#, + image_name + )?; + Ok(()) +} + +pub(crate) fn gen_create_os_tar_from_docker(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_os_tar_from_docker() {{ + container_id=$(docker create "${{DOCKER_IMG}}") + echo "$container_id" + docker cp "$container_id":/os.tar "${{SCRIPTS_DIR}}" + docker rm "$container_id" +}} +"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_vm_docker_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_vm_docker_img() {{ + create_os_tar_from_docker + create_img +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_vm_docker_img"# + )?; + Ok(()) +} + +pub(crate) fn gen_create_pxe_docker_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_pxe_docker_img() {{ + rm -rf "${{SCRIPTS_DIR}}"/initramfs.img "${{SCRIPTS_DIR}}"/kubeos.tar + create_os_tar_from_docker + tar -xvf "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/initramfs.img + mv "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/kubeos.tar +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_pxe_docker_img"# + )?; + Ok(()) +} + +// admin +pub(crate) fn gen_admin_vars(file: &mut File, docker_img: &str, dockerfile: &PathBuf) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"set -e + +DOCKER_IMG={} +DOCKERFILE={} +ADMIN_CONTAINER_DIR="${{SCRIPTS_DIR}}"/admin-container +"#, + dockerfile.to_str().unwrap(), + docker_img + )?; + Ok(()) +} + +pub(crate) fn gen_create_admin_img(file: &mut File) -> Result<()> { + writeln!( + file, + r#"function create_admin_img() {{ + local kubeos_root_dir=$(dirname $(dirname $(dirname "${{SCRIPTS_DIR}}"))) + cp "${{kubeos_root_dir}}"/bin/hostshell "${{ADMIN_CONTAINER_DIR}}" + docker build -t "${{DOCKER_IMG}}" -f "${{DOCKERFILE}}" "${{ADMIN_CONTAINER_DIR}}" + rm -rf "${{ADMIN_CONTAINER_DIR}}"/hostshell +}} + +test_lock +trap clean_space EXIT +trap clean_img ERR + +create_admin_img"# + )?; + Ok(()) +} +/* endregion */ + +/* region: set_in_chroot.sh */ +pub(crate) fn gen_add_users(file: &mut File, users: &Vec) -> Result<()> { + writeln!(file, "# add users")?; + for user in users { + let name = &user.name; + let passwd = &user.passwd; + let groups = match user.groups.clone() { + Some(groups) => groups, + None => vec![name.clone()], + }; + for group in &groups { + writeln!( + file, + r#"if ! getent group "{}" > /dev/null 2>&1; then + groupadd "{}" +fi"#, + group, group + )?; + } + write!(file, "useradd -m -g {}", &groups[0])?; + if groups.len() > 1 { + let additional_groups = &groups[1..].join(","); + write!(file, " -G {}", additional_groups)?; + } + writeln!(file, " -s /bin/bash \"{}\"", &name)?; + writeln!(file, "echo \"{}:{}\" | chpasswd", name, passwd)?; + if let Some(sudo) = &user.sudo { + writeln!( + file, + r#"if visudo -c; then + echo -e "{} {}" | tee -a /etc/sudoers +else + echo "Sudoers file syntax check failed. Please fix the sudoers file manually." + exit 5 +fi"#, + name, sudo + )?; + } + } + Ok(()) +} + +pub(crate) fn gen_systemd_services(file: &mut File, systemd_services: &SystemdService) -> Result<()> { + writeln!(file, "# systemd")?; + for service_name in &systemd_services.name { + writeln!(file, "systemctl enable {}", service_name)?; + } + Ok(()) +} + +pub(crate) fn gen_set_in_chroot(file: &mut File, legacy_bios: bool, config: &Config) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"ln -s /usr/lib/systemd/system/os-agent.service /usr/lib/systemd/system/multi-user.target.wants/os-agent.service +ln -s /usr/lib/systemd/system/kubelet.service /usr/lib/systemd/system/multi-user.target.wants/kubelet.service"# + )?; + if legacy_bios { + writeln!( + file, + "ln -s /usr/lib/systemd/system/boot-grub2.mount /lib/systemd/system/local-fs.target.wants/boot-grub2.mount" + )?; + } else { + writeln!( + file, + "ln -s /usr/lib/systemd/system/boot-efi.mount /lib/systemd/system/local-fs.target.wants/boot-efi.mount" + )?; + } + writeln!(file, r#"ln -s /usr/lib/systemd/system/etc.mount /lib/systemd/system/local-fs.target.wants/etc.mount"#)?; + + if let Some(users) = &config.users { + gen_add_users(file, users)?; + } + if let Some(systemd_services) = &config.systemd_service { + gen_systemd_services(file, systemd_services)?; + } + + writeln!( + file, + r#" +str=$(sed -n '/^root:/p' /etc/shadow | awk -F "root:" '{{print $2}}') +umask 0666 +mv /etc/shadow /etc/shadow_bak +sed -i '/^root:/d' /etc/shadow_bak +echo "root:""${{ROOT_PASSWD}}""${{str:1}}" > /etc/shadow +cat /etc/shadow_bak >> /etc/shadow +rm -rf /etc/shadow_bak + +dracut -f -v --add bootup /initramfs.img --kver "$(ls /lib/modules)" +rm -rf /usr/lib/dracut/modules.d/00bootup"# + )?; + + Ok(()) +} +/* endregion */ + +/* region: bootloader.sh */ +pub(crate) fn gen_bootloader(file: &mut File, arch: &str, legacy_bios: bool) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"set -eu +set -o pipefail +set -x + +function install_grub2 () {{"# + )?; + + if arch == "aarch64" || (arch == "x86_64" && !legacy_bios) { + writeln!( + file, + r#" cp -r /usr/lib/grub/x86_64-efi boot/efi/EFI/openEuler + eval "grub2-mkimage -d /usr/lib/grub/x86_64-efi -O x86_64-efi --output=/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" + + mkdir -p /boot/efi/EFI/BOOT/ + cp -f /boot/efi/EFI/openEuler/grubx64.efi /boot/efi/EFI/BOOT/BOOTX64.EFI +}} +"# + )?; + } else { + writeln!( + file, + r#" GRUBNAME=$(which grub2-install) + echo "Installing GRUB2..." + FORCE_OPT=${{FORCE_OPT:-"--force"}} + TARGET_OPT=${{TARGET_OPT:-"--target=i386-pc"}} + + $GRUBNAME --modules="biosdisk part_msdos" "$FORCE_OPT" "$TARGET_OPT" "$DEVICE" +}} +"# + )?; + } + + writeln!( + file, + r#"install_grub2 +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: rpmlist */ +pub(crate) fn gen_rpm_list(file: &mut File, rpmlist: &Vec) -> Result<()> { + for rpm in rpmlist { + writeln!(file, "{}", rpm)?; + } + Ok(()) +} +/* endregion */ + +/* region: grub.cfg */ +pub(crate) fn gen_grub_cfg(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"set pager=1 + +if [ -f ${{config_directory}}/grubenv ]; then + load_env -f ${{config_directory}}/grubenv +elif [ -s $prefix/grubenv ]; then + load_env +fi +if [ "${{next_entry}}" ] ; then + set default="${{next_entry}}" + set next_entry= + save_env next_entry + set boot_once=true +else + set default="${{saved_entry}}" +fi + +if [ x"${{feature_menuentry_id}}" = xy ]; then + menuentry_id_option="--id" +else + menuentry_id_option="" +fi + +export menuentry_id_option + +if [ "${{prev_saved_entry}}" ]; then + set saved_entry="${{prev_saved_entry}}" + save_env saved_entry + set prev_saved_entry= + save_env prev_saved_entry + set boot_once=true +fi + +function savedefault {{ + if [ -z "${{boot_once}}" ]; then + saved_entry="${{chosen}}" + save_env saved_entry + fi +}} + +function load_video {{ + if [ x$feature_all_video_module = xy ]; then + insmod all_video + else + insmod efi_gop + insmod efi_uga + insmod ieee1275_fb + insmod vbe + insmod vga + insmod video_bochs + insmod video_cirrus + fi +}} + +terminal_output console +if [ x$feature_timeout_style = xy ] ; then + set timeout_style=menu + set timeout=5 +# Fallback normal timeout code in case the timeout_style feature is +# unavailable. +else + set timeout=5 +fi +set superusers="root" +### END /etc/grub.d/00_header ### + +### BEGIN /etc/grub.d/01_users ### +if [ -f ${{prefix}}/user.cfg ]; then + source ${{prefix}}/user.cfg + if [ -n "${{GRUB2_PASSWORD}}" ]; then + set superusers="root" + export superusers + password_pbkdf2 root ${{GRUB2_PASSWORD}} + fi +fi +### END /etc/grub.d/01_users ### + +### BEGIN /etc/grub.d/10_linux ### +menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' {{ + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=/dev/vda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +}} + +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' {{ + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=/dev/vda3 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +}} + +### END /etc/grub.d/10_linux ### + +### BEGIN /etc/grub.d/10_reset_boot_success ### +# Hiding the menu is ok if last boot was ok or if this is a first boot attempt to boot the entry +if [ "${{boot_success}}" = "1" -o "${{boot_indeterminate}}" = "1" ]; then + set menu_hide_ok=1 +else + set menu_hide_ok=0 +fi +# Reset boot_indeterminate after a successful boot +if [ "${{boot_success}}" = "1" ] ; then + set boot_indeterminate=0 +# Avoid boot_indeterminate causing the menu to be hidden more then once +elif [ "${{boot_indeterminate}}" = "1" ]; then + set boot_indeterminate=2 +fi +# Reset boot_success for current boot +set boot_success=0 +save_env boot_success boot_indeterminate +### END /etc/grub.d/10_reset_boot_success ### + +### BEGIN /etc/grub.d/12_menu_auto_hide ### +if [ x$feature_timeout_style = xy ] ; then + if [ "${{menu_show_once}}" ]; then + unset menu_show_once + save_env menu_show_once + set timeout_style=menu + set timeout=60 + elif [ "${{menu_auto_hide}}" -a "${{menu_hide_ok}}" = "1" ]; then + set orig_timeout_style=${{timeout_style}} + set orig_timeout=${{timeout}} + if [ "${{fastboot}}" = "1" ]; then + # timeout_style=menu + timeout=0 avoids the countdown code keypress check + set timeout_style=menu + set timeout=0 + else + set timeout_style=hidden + set timeout=1 + fi + fi +fi +### END /etc/grub.d/12_menu_auto_hide ### + +### BEGIN /etc/grub.d/20_linux_xen ### +### END /etc/grub.d/20_linux_xen ### + +### BEGIN /etc/grub.d/20_ppc_terminfo ### +### END /etc/grub.d/20_ppc_terminfo ### + +### BEGIN /etc/grub.d/30_uefi-firmware ### +### END /etc/grub.d/30_uefi-firmware ### + +### BEGIN /etc/grub.d/40_custom ### +# This file provides an easy way to add custom menu entries. Simply type the +# menu entries you want to add after this comment. Be careful not to change +# the 'exec tail' line above. +### END /etc/grub.d/40_custom ### + +### BEGIN /etc/grub.d/41_custom ### +if [ -f ${{config_directory}}/custom.cfg ]; then + source ${{config_directory}}/custom.cfg +elif [ -z "${{config_directory}}" -a -f $prefix/custom.cfg ]; then + source $prefix/custom.cfg; +fi +### END /etc/grub.d/41_custom ### +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: 00bootup */ +// 00bootup/global.cfg +pub(crate) fn gen_global_cfg(file: &mut File, pxe_config: &PxeConfig) -> Result<()> { + writeln!( + file, + r#"# rootfs file name +rootfs_name={} + +# select the target disk to install kubeOS +disk={} + +# pxe server ip address where stores the rootfs on the http server +server_ip={} +# target machine ip +local_ip={} +# target machine route +route_ip={} +# target machine netmask +netmask={} +# target machine netDevice name +net_name={} +"#, + pxe_config.rootfs_name, + pxe_config.disk, + pxe_config.server_ip, + pxe_config.local_ip, + pxe_config.route_ip, + pxe_config.netmask, + pxe_config.net_name + )?; + Ok(()) +} + +// 00bootup/module-setup.sh +pub(crate) fn gen_module_setup(file: &mut File) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"check() {{ + return 0 +}} + +depends() {{ + echo systemd +}} + +install() {{ + inst_multiple -o grub2-mkimage mkfs.ext4 mkfs.vfat lsblk tar cpio gunzip lspci parted dhclient ifconfig curl hwinfo head tee arch df awk route + inst_hook mount 00 "$moddir/mount.sh" + inst_simple "$moddir/mount.sh" "/mount.sh" + inst_simple "$moddir/Global.cfg" "/Global.cfg" +}} + +installkernel() {{ + hostonly='' + instmods='drivers/ata drivers/nvme drivers/scsi drivers/net fs/fat fs/nls' +}} +"# + )?; + Ok(()) +} + +// 00bootup/mount.sh +pub(crate) fn gen_mount(file: &mut File) -> Result<()> { + writeln!(file, "#!/bin/bash")?; + gen_copyright(file)?; + + writeln!( + file, + r#"arch=$(arch) +min_size=8 +log=/install.log + +source ./Global.cfg + +function CheckSpace() {{ + local disk_ava + disk_ava="$(parted -l | grep "${{disk}}" | awk '{{print $3}}')" + if echo "${{disk_ava}}" | grep "[GT]B$"; then + if echo "${{disk_ava}}" | grep GB$; then + disk_ava="$(echo "${{disk_ava}}" | awk -F G '{{print $1}}' | awk -F . '{{print $1}}')" + if [ "${{disk_ava}}" -lt ${{min_size}} ]; then + echo "The available disk space is not enough, at least ${{min_size}}GB." | tee -a ${{log}} + return 1 + fi + fi + else + echo "The available disk space is not enough, at least ${{min_size}}G." | tee -a ${{log}} + return 1 + fi + + return 0 +}} + +function mount_proc_dev_sys() {{ + local tmp_root=$1 + mount -t proc none "${{tmp_root}}/proc" + mount --bind /dev "${{tmp_root}}/dev" + mount --bind /dev/pts "${{tmp_root}}/dev/pts" + mount -t sysfs none "${{tmp_root}}/sys" +}} + +function GetDisk() {{ + mapfile -t disks < <(hwinfo --disk --short 2>&1 | grep -vi "^disk" | awk '{{print $1}}') + if [ ${{#disks[*]}} -gt 0 ]; then + if [ -n "${{disk}}" ] && echo "${{disks[@]}}" | grep -wq "${{disk}}" ; then + echo "${{disk}} exists, start partition" | tee -a ${{log}} + else + echo "disk not exist, please choose correct disk" | tee -a ${{log}} + fi + else + echo "no disk found" | tee -a ${{log}} + return 1 + fi + CheckSpace + local status=$? + if [ $status -ne 0 ]; then + echo "no enough space on ${{disk}}" | tee -a ${{log}} + return 1 + fi + + return 0 +}} + +function PartitionAndFormatting() {{ + echo "Partitioning and formatting disk $disk..." + # partition and format + parted "${{disk}}" -s mklabel gpt >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s mkpart primary fat16 1M 100M >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s mkpart primary ext4 100M 2600M >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s mkpart primary ext4 2600M 5100M >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s mkpart primary ext4 5100M 100% >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + parted "${{disk}}" -s set 1 boot on >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "partition failed" | tee -a ${{log}} + return 1 + fi + + mkfs.vfat -n "BOOT" "${{disk}}"1 >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "format failed" | tee -a ${{log}} + return 1 + fi + + mkfs.ext4 -L "ROOT-A" "${{disk}}"2 >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "format failed" | tee -a ${{log}} + return 1 + fi + + mkfs.ext4 -L "ROOT-B" "${{disk}}"3 >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "format failed" | tee -a ${{log}} + return 1 + fi + + mkfs.ext4 -L "PERSIST" "${{disk}}"4 >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "format failed" | tee -a ${{log}} + return 1 + fi + + return 0 +}} + +function InitNetwork() {{ + echo "Initializing network..." + mapfile -t netNames < <(ifconfig -a | awk '{{print $1}}' | grep : | grep '^e' | awk -F: '{{print $1}}') + if [ ${{#netNames[*]}} -gt 0 ]; then + if [ -n "${{net_name}}" ] && echo "${{netNames[@]}}" | grep -wq "${{net_name}}" ; then + echo "${{net_name}} exists, start set ip" | tee -a ${{log}} + else + echo "net_name not exist, choose default net" | tee -a ${{log}} + net_name=${{netNames[0]}} + fi + else + echo "no net Device found" | tee -a ${{log}} + return 1 + fi + + ifconfig "${{net_name}}" up + local status=$? + if [ $status -ne 0 ]; then + echo "load net card failed" | tee -a ${{log}} + return 1 + fi + sleep 3 + + ifconfig "${{net_name}}" "${{local_ip}}" netmask "${{netmask}}" >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "ip set failed" | tee -a ${{log}} + return 1 + fi + sleep 3 + + route add default gw "${{route_ip}}" >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "add route failed" | tee -a ${{log}} + return 1 + fi + sleep 3 + return 0 +}} + +function MountRoot() {{ + echo "Mounting rootfs..." + # mount rootfs + mount "${{disk}}"2 /sysroot >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "mount rootfs failed" | tee -a ${{log}} + return 1 + fi + + return 0 +}} + +function MountPersist() {{ + echo "Mounting persist" + mount "${{disk}}"4 /sysroot/persist >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "mount persist failed" | tee -a ${{log}} + return 1 + fi + mkdir /sysroot/persist/{{var,etc,etcwork}} + mkdir -p /sysroot/persist/etc/KubeOS/certs + return 0 +}} + +function MountBoot() {{ + echo "Mounting boot" + mkdir -p /sysroot/boot/efi + mount "${{disk}}"1 /sysroot/boot/efi >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "mount boot failed" | tee -a ${{log}} + return 1 + fi + return 0 +}} + +function GetRootfs() {{ + echo "Downloading rootfs..." + + curl -o /"${{rootfs_name}}" http://"${{server_ip}}"/"${{rootfs_name}}" + if [ ! -e "/${{rootfs_name}}" ]; then + echo "download rootfs failed" | tee -a ${{log}} + return 1 + fi + + tar -xf /"${{rootfs_name}}" -C /sysroot + local status=$? + if [ $status -ne 0 ]; then + echo "decompose rootfs failed" | tee -a ${{log}} + return 1 + fi + + rm -rf "${{rootfs_name:?}}" + mount -o remount,ro "${{disk}}"2 /sysroot >> ${{log}} 2>&1 + return 0 +}} + +function Inst_Grub2_x86() {{ + # copy the files that boot need + cp -r /sysroot/usr/lib/grub/x86_64-efi /sysroot/boot/efi/EFI/openEuler + eval "grub2-mkimage -d /sysroot/usr/lib/grub/x86_64-efi -O x86_64-efi --output=/sysroot/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "grub2-mkimage on x86 failed" | tee -a ${{log}} + return 1 + fi + + mkdir -p /sysroot/boot/efi/EFI/BOOT/ + cp -f /sysroot/boot/efi/EFI/openEuler/grubx64.efi /sysroot/boot/efi/EFI/BOOT/BOOTX64.EFI + + return 0 +}} + +function Inst_Grub2_aarch64() {{ + cp -r /sysroot/usr/lib/grub/arm64-efi /sysroot/boot/efi/EFI/openEuler/ + eval "grub2-mkimage -d /sysroot/usr/lib/grub/arm64-efi -O arm64-efi --output=/sysroot/boot/efi/EFI/openEuler/grubaa64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" >> ${{log}} 2>&1 + local status=$? + if [ $status -ne 0 ]; then + echo "grub2-mkimage on aarch64 failed" | tee -a ${{log}} + return 1 + fi + + mkdir -p /sysroot/boot/efi/EFI/BOOT/ + cp -f /sysroot/boot/efi/EFI/openEuler/grubaa64.efi /sysroot/boot/efi/EFI/BOOT/BOOTAA64.EFI + + return 0 +}} + +function SetBoot() {{ + # mount boot + echo "Setting boot" + + if [ "$arch" == "x86_64" ]; then + Inst_Grub2_x86 + local status=$? + if [ $status -ne 0 ]; then + echo "install grub on x86 failed" | tee -a ${{log}} + return 1 + fi + fi + + if [ "$arch" == "aarch64" ]; then + Inst_Grub2_aarch64 + local status=$? + if [ $status -ne 0 ]; then + echo "install grub on aarch64 failed" | tee -a ${{log}} + return 1 + fi + fi + sed -i 's#/dev/sda#'"${{disk}}"'#g' /sysroot/boot/efi/EFI/openEuler/grub.cfg + + return 0 +}} + +function Bootup_Main() {{ + # get disk + echo "Checking disk info..." | tee -a ${{log}} + GetDisk + local status=$? + if [ $status -ne 0 ]; then + echo "Checking disk info failed" | tee -a ${{log}} + return 1 + fi + + # partition and format disk + echo "Partion and formatting..." | tee -a ${{log}} + PartitionAndFormatting + local status=$? + if [ $status -ne 0 ]; then + echo "Partition and formatting disk failed" | tee -a ${{log}} + return 1 + fi + + # init network + echo "Initializing network..." | tee -a ${{log}} + InitNetwork + local status=$? + if [ $status -ne 0 ]; then + echo "Initializing network failed" | tee -a ${{log}} + return 1 + fi + + # mount partitions + + # mount boot + echo "Mounting root..." | tee -a ${{log}} + MountRoot + local status=$? + if [ $status -ne 0 ]; then + echo "Mounting root failed" | tee -a ${{log}} + return 1 + fi + + echo "Mounting boot..." | tee -a ${{log}} + MountBoot + local status=$? + if [ $status -ne 0 ]; then + echo "Mounting boot failed" | tee -a ${{log}} + return 1 + fi + + # download rootfs + echo "Downloading rootfs..." | tee -a ${{log}} + GetRootfs + local status=$? + if [ $status -ne 0 ]; then + echo "Downloading rootfs failed" | tee -a ${{log}} + return 1 + fi + mount_proc_dev_sys /sysroot + # set boot + echo "Setting boot..." | tee -a ${{log}} + SetBoot + local status=$? + if [ $status -ne 0 ]; then + echo "Setting boot failed" | tee -a ${{log}} + return 1 + fi + # mount persist + echo "Mounting persist..." | tee -a ${{log}} + MountPersist + local status=$? + if [ $status -ne 0 ]; then + echo "Mounting persist failed" | tee -a ${{log}} + return 1 + fi + return 0 +}} + +Bootup_Main +ret=$? +if [ ${{ret}} -eq 0 ]; then + echo "kubeOS install success! switch to root" | tee -a ${{log}} + cp ${{log}} /sysroot/persist +else + echo "kubeOS install failed, see install.log" | tee -a ${{log}} +fi + +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: dockerfile */ +pub(crate) fn gen_dockerfile(file: &mut File) -> Result<()> { + writeln!( + file, + r#"FROM scratch +COPY os.tar / +CMD ["/bin/sh"] +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: admin-container */ +// admin-container/dockerfile +pub(crate) fn gen_admin_dockerfile(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"FROM openeuler-22.03-lts +MAINTAINER + +RUN yum -y install openssh-clients util-linux + +ADD ./sysmaster-0.2.3-1.oe2203.aarch64.rpm /home +RUN rpm -ivh /home/sysmaster-0.2.3-1.oe2203.aarch64.rpm + +COPY ./hostshell /usr/bin/ +COPY ./set-ssh-pub-key.sh /usr/local/bin +COPY ./set-ssh-pub-key.service /usr/lib/sysmaster + +EXPOSE 22 +# set sshd.service and set-ssh-pub-key.service pulled up by default +RUN sed -i 's/sysinit.target/sysinit.target;sshd.service;set-ssh-pub-key.service/g' /usr/lib/sysmaster/basic.target + +CMD ["/usr/lib/sysmaster/init"] +"# + )?; + Ok(()) +} + +// admin-container/set-ssh-pub-key.service +pub(crate) fn gen_set_ssh_pub_key_service(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description="set ssh authorized keys according to the secret which is set by user" + +[Service] +ExecStart="/usr/local/bin/set-ssh-pub-key.sh" +"# + )?; + Ok(()) +} + +// admin-container/set-ssh-pub-key.sh +pub(crate) fn gen_set_ssh_pub_key(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"ssh_pub=$(cat /etc/secret-volume/ssh-pub-key) +ssh_dir="/root/.ssh" +authorized_file="$ssh_dir/authorized_keys" + +if [ ! -d "$ssh_dir" ]; then + mkdir "$ssh_dir" + chmod 700 "$ssh_dir" +fi + +if [ ! -f "$authorized_file" ]; then + touch "$authorized_file" + chmod 600 "$authorized_file" +fi + +echo "$ssh_pub" >> "$authorized_file" +"# + )?; + Ok(()) +} +/* endregion */ + +/* region: misc-files */ +// misc-files/boot-efi.mount +pub(crate) fn gen_boot_efi_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=grub2 Dir +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target + +[Mount] +What=/dev/disk/by-label/BOOT +Where=/boot/efi +Type=vfat +Options=defaults + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} + +// misc-files/boot-grub2.mount +pub(crate) fn gen_boot_grub2_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=grub2 Dir +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target + +[Mount] +What=/dev/disk/by-label/GRUB2 +Where=/boot/grub2 +Type=ext4 +Options=defaults + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} + +// misc-files/etc.mount +pub(crate) fn gen_etc_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=etc Dir +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target +Wants=persist.mount +After=persist.mount + +[Mount] +What=overlay +Where=/etc +Type=overlay +Options=upperdir=/persist/etc,lowerdir=/etc,workdir=/persist/etcwork + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} + +// misc-files/os-agent.service +pub(crate) fn gen_os_agent_service(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=Agent For KubeOS + +[Service] +Environment=GOTRACEBACK=crash +ExecStart=/usr/bin/os-agent +KillMode=process +Restart=on-failure + +[Install] +WantedBy=multi-user.target +"# + )?; + Ok(()) +} + +// misc-files/os-release +pub(crate) fn gen_os_release(file: &mut File) -> Result<()> { + writeln!( + file, + r#"NAME=KubeOS +ID=KubeOS +"# + )?; + Ok(()) +} + +// misc-files/persist.mount +pub(crate) fn gen_persist_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=PERSIST Dir (/persist) +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target + +[Mount] +What=/dev/disk/by-label/PERSIST +Where=/persist +Type=ext4 +Options=defaults + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} + +// misc-files/var.mount +pub(crate) fn gen_var_mount(file: &mut File) -> Result<()> { + gen_copyright(file)?; + + writeln!( + file, + r#"[Unit] +Description=var Dir +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target umount.target +Wants=persist.mount +After=persist.mount + +[Mount] +What=/persist/var +Where=/var +Type=node +Options=bind + +[Install] +WantedBy=local-fs.target +"# + )?; + Ok(()) +} +/* endregion */ diff --git a/KubeOS-Rust/kbimg/src/utils.rs b/KubeOS-Rust/kbimg/src/utils.rs new file mode 100644 index 00000000..c7ea5505 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/utils.rs @@ -0,0 +1,126 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs, os::unix::fs::PermissionsExt, path::PathBuf, process::Command}; + +use anyhow::bail; + +use crate::commands::PxeConfig; + +pub(crate) fn execute_scripts(script: PathBuf) -> anyhow::Result<()> { + if !script.exists() { + bail!("Script does not exist: {:?}", script); + } + let status = Command::new("bash").arg(&script).status()?; + if !status.success() { + bail!("Failed to execute script: {}\n", script.display()); + } + Ok(()) +} + +pub(crate) fn set_permissions(path: &str, permission_value: u32) -> anyhow::Result<()> { + let metadata = fs::metadata(path)?; + let mut permissions = metadata.permissions(); + permissions.set_mode(permission_value); + fs::set_permissions(path, permissions)?; + Ok(()) +} + +/// Check if the input parameter is valid +pub(crate) fn is_valid_param + std::fmt::Debug>(param: S) -> bool { + let special_chars = vec!["|", ";", "&", "&&", "||", ">", ">>", "<", ",", "#", "!", "$"]; + !param.as_ref().chars().any(|c| special_chars.contains(&c.to_string().as_str())) +} + +/// Check if the path exists and is indeed a file +pub(crate) fn is_file_valid(msg: &str, path: &PathBuf) -> anyhow::Result<()> { + if !path.exists() { + bail!("{} does not exist: {:?}", msg, path); + } + if !path.is_file() { + bail!("{} exists but is not a file: {:?}", msg, path); + } + Ok(()) +} + +/// Check if addr is valid +pub(crate) fn is_addr_valid(addr: &str) -> bool { + let ip_pattern = regex::Regex::new(r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$").unwrap(); + if !ip_pattern.is_match(addr) { + return false; + } + + for quad in addr.split('.') { + if let Ok(num) = quad.parse::() { + if num <= 255 { + continue; + } + } + return false; + } + + true +} + +/// Check pxe config +pub(crate) fn check_pxe_conf_valid(pxe_config: &PxeConfig) -> anyhow::Result<()> { + if !is_addr_valid(&pxe_config.server_ip) { + bail!("address {} is invalid, please check input", &pxe_config.server_ip) + } + if !is_addr_valid(&pxe_config.local_ip) { + bail!("address {} is invalid, please check input", &pxe_config.local_ip) + } + if !is_addr_valid(&pxe_config.route_ip) { + bail!("address {} is invalid, please check input", &pxe_config.route_ip) + } + if !is_addr_valid(&pxe_config.netmask) { + bail!("address {} is invalid, please check input", &pxe_config.netmask) + } + Ok(()) +} + +/// Get architecture +pub(crate) fn get_arch() -> String { + let output = std::process::Command::new("arch").output().expect("Failed to execute `arch` command"); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_valid_param() { + init(); + assert_eq!(is_valid_param("test"), true); + assert_eq!(is_valid_param("test|test"), false); + assert_eq!(is_valid_param("test;test"), false); + assert_eq!(is_valid_param("test&test"), false); + assert_eq!(is_valid_param("test&&test"), false); + assert_eq!(is_valid_param("test||test"), false); + assert_eq!(is_valid_param("test>test"), false); + assert_eq!(is_valid_param("test>>test"), false); + assert_eq!(is_valid_param("test, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct ConfigureRequest { + pub configs: Vec, +} + +#[derive(Deserialize, Serialize, Debug, PartialEq)] +pub struct Response { + pub status: AgentStatus, +} + +pub enum ImageType { + Containerd(CtrImageHandler), + Docker(DockerImageHandler), + Disk(DiskImageHandler), +} + +impl ImageType { + pub fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result> { + match self { + ImageType::Containerd(handler) => handler.download_image(req), + ImageType::Docker(handler) => handler.download_image(req), + ImageType::Disk(handler) => handler.download_image(req), + } + } +} +pub trait ImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result>; +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use mockall::mock; + + use super::*; + use crate::utils::PreparePath; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + #[test] + fn test_download_image() { + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: "kubeos-temp".to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + let mut mock_executor1 = MockCommandExec::new(); + mock_executor1.expect_run_command().returning(|_, _| Ok(())); + mock_executor1.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let c_handler = CtrImageHandler::new(PreparePath::default(), mock_executor1); + let image_type = ImageType::Containerd(c_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + + let mut mock_executor2 = MockCommandExec::new(); + mock_executor2.expect_run_command().returning(|_, _| Ok(())); + mock_executor2.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let docker_handler = DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor2); + let image_type = ImageType::Docker(docker_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + + let mut mock_executor3 = MockCommandExec::new(); + mock_executor3.expect_run_command().returning(|_, _| Ok(())); + mock_executor3.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let disk_handler = DiskImageHandler::new(PreparePath::default(), mock_executor3, "test".into()); + let image_type = ImageType::Disk(disk_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + } +} diff --git a/KubeOS-Rust/manager/src/lib.rs b/KubeOS-Rust/manager/src/lib.rs new file mode 100644 index 00000000..b45cab99 --- /dev/null +++ b/KubeOS-Rust/manager/src/lib.rs @@ -0,0 +1,15 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod api; +pub mod sys_mgmt; +pub mod utils; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs new file mode 100644 index 00000000..138df9da --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -0,0 +1,558 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + collections::HashMap, + fs::{self, File}, + io::{self, BufRead, BufWriter, Write}, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + string::String, +}; + +use anyhow::{bail, Context, Result}; +use lazy_static::lazy_static; +use log::{debug, info, trace, warn}; +use regex::Regex; + +use crate::{api::*, sys_mgmt::values, utils::*}; + +lazy_static! { + pub static ref CONFIG_TEMPLATE: HashMap> = { + let mut config_map = HashMap::new(); + config_map.insert( + values::KERNEL_SYSCTL.to_string(), + Box::new(KernelSysctl::new(values::DEFAULT_PROC_PATH)) as Box, + ); + config_map.insert( + values::KERNEL_SYSCTL_PERSIST.to_string(), + Box::new(KernelSysctlPersist) as Box, + ); + config_map.insert( + values::GRUB_CMDLINE_CURRENT.to_string(), + Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: true }) + as Box, + ); + config_map.insert( + values::GRUB_CMDLINE_NEXT.to_string(), + Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: false }) + as Box, + ); + config_map + }; +} + +pub trait Configuration { + fn set_config(&self, config: &mut Sysconfig) -> Result<()>; +} + +pub struct KernelSysctl { + pub proc_path: String, +} +pub struct KernelSysctlPersist; +pub struct GrubCmdline { + pub grub_path: String, + pub is_cur_partition: bool, +} + +impl Configuration for KernelSysctl { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + info!("Start setting kernel.sysctl"); + for (key, key_info) in config.contents.iter() { + let proc_path = self.get_proc_path(key); + if key_info.operation == "delete" { + warn!("Failed to delete kernel.sysctl config with key \"{}\"", key); + } else if !key_info.value.is_empty() && key_info.operation.is_empty() { + fs::write(&proc_path, format!("{}\n", &key_info.value).as_bytes()) + .with_context(|| format!("Failed to write kernel.sysctl with key: \"{}\"", key))?; + info!("Configured kernel.sysctl {}={}", key, key_info.value); + } else { + warn!( + "Failed to parse kernel.sysctl, key: \"{}\", value: \"{}\", operation: \"{}\"", + key, key_info.value, key_info.operation + ); + } + } + Ok(()) + } +} + +impl KernelSysctl { + fn new(proc_path: &str) -> Self { + Self { proc_path: String::from(proc_path) } + } + + fn get_proc_path(&self, key: &str) -> PathBuf { + let path_str = format!("{}{}", self.proc_path, key.replace('.', "/")); + Path::new(&path_str).to_path_buf() + } +} + +impl Configuration for KernelSysctlPersist { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + info!("Start setting kernel.sysctl.persist"); + let mut config_path = &values::DEFAULT_KERNEL_CONFIG_PATH.to_string(); + if !config.config_path.is_empty() { + config_path = &config.config_path; + } + debug!("kernel.sysctl.persist config_path: \"{}\"", config_path); + create_config_file(config_path).with_context(|| format!("Failed to find config path \"{}\"", config_path))?; + let configs = get_and_set_configs(&mut config.contents, config_path) + .with_context(|| format!("Failed to set persist kernel configs \"{}\"", config_path))?; + write_configs_to_file(config_path, &configs).with_context(|| "Failed to write configs to file".to_string())?; + Ok(()) + } +} + +fn create_config_file(config_path: &str) -> Result<()> { + if !is_file_exist(config_path) { + let f = fs::File::create(config_path)?; + let metadata = f.metadata()?; + let mut permissions = metadata.permissions(); + permissions.set_mode(values::DEFAULT_KERNEL_CONFIG_PERM); + debug!("Create file {} with permission 0644", config_path); + } + Ok(()) +} + +fn get_and_set_configs(expect_configs: &mut HashMap, config_path: &str) -> Result> { + let f = File::open(config_path).with_context(|| format!("Failed to open config path \"{}\"", config_path))?; + let mut configs_write = Vec::new(); + for line in io::BufReader::new(f).lines() { + let line = line?; + // if line is a comment or blank + if line.starts_with('#') || line.starts_with(';') || line.trim().is_empty() { + configs_write.push(line); + continue; + } + let config_kv: Vec<&str> = line.splitn(2, '=').map(|s| s.trim()).collect(); + // if config_kv is not a key-value pair + if config_kv.len() != 2 { + bail!("could not parse sysctl config {}", line); + } + let new_key_info = expect_configs.get(config_kv[0]); + let new_config = match new_key_info { + Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config_kv, new_key_info), + Some(new_key_info) => handle_update_key(&config_kv, new_key_info), + None => config_kv.join("="), + }; + configs_write.push(new_config); + expect_configs.remove(config_kv[0]); + } + let new_config = handle_add_key(expect_configs, false); + configs_write.extend(new_config); + Ok(configs_write) +} + +fn write_configs_to_file(config_path: &str, configs: &Vec) -> Result<()> { + info!("Write configuration to file \"{}\"", config_path); + let f = File::create(config_path)?; + let mut w = BufWriter::new(f); + for line in configs { + if line.is_empty() { + continue; + } + writeln!(w, "{}", line.as_str())?; + } + w.flush().with_context(|| format!("Failed to flush file {}", config_path))?; + w.get_mut().sync_all().with_context(|| "Failed to sync".to_string())?; + debug!("Write configuration to file \"{}\" success", config_path); + Ok(()) +} + +fn handle_delete_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String { + let key = config_kv[0]; + if config_kv.len() == 1 && new_config_info.value.is_empty() { + info!("Delete configuration key: \"{}\"", key); + return String::from(""); + } else if config_kv.len() == 1 && !new_config_info.value.is_empty() { + warn!("Failed to delete key \"{}\" with inconsistent values \"nil\" and \"{}\"", key, new_config_info.value); + return key.to_string(); + } + let old_value = config_kv[1]; + if old_value != new_config_info.value { + warn!( + "Failed to delete key \"{}\" with inconsistent values \"{}\" and \"{}\"", + key, old_value, new_config_info.value + ); + return config_kv.join("="); + } + info!("Delete configuration {}={}", key, old_value); + String::new() +} + +fn handle_update_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String { + let key = config_kv[0]; + if !new_config_info.operation.is_empty() { + warn!( + "Unknown operation \"{}\", updating key \"{}\" with value \"{}\" by default", + new_config_info.operation, key, new_config_info.value + ); + } + if config_kv.len() == values::ONLY_KEY && new_config_info.value.is_empty() { + return key.to_string(); + } + let new_value = new_config_info.value.trim(); + if config_kv.len() == values::ONLY_KEY && !new_config_info.value.is_empty() { + info!("Update configuration \"{}={}\"", key, new_value); + return format!("{}={}", key, new_value); + } + if new_config_info.value.is_empty() { + warn!("Failed to update key \"{}\" with \"null\" value", key); + return config_kv.join("="); + } + info!("Update configuration \"{}={}\"", key, new_value); + format!("{}={}", key, new_value) +} + +fn handle_add_key(expect_configs: &HashMap, is_only_key_valid: bool) -> Vec { + let mut configs_write = Vec::new(); + for (key, config_info) in expect_configs.iter() { + if config_info.operation == "delete" { + warn!("Failed to delete inexistent key: \"{}\"", key); + continue; + } + if key.is_empty() || key.contains('=') { + warn!("Failed to add \"null\" key or key containing \"=\", key: \"{}\"", key); + continue; + } + if !config_info.operation.is_empty() { + warn!( + "Unknown operation \"{}\", adding key \"{}\" with value \"{}\" by default", + config_info.operation, key, config_info.value + ); + } + let (k, v) = (key.trim(), config_info.value.trim()); + if v.is_empty() && is_only_key_valid { + info!("Add configuration \"{}\"", k); + configs_write.push(k.to_string()); + } else if v.is_empty() { + warn!("Failed to add key \"{}\" with \"null\" value", k); + } else { + info!("Add configuration \"{}={}\"", k, v); + configs_write.push(format!("{}={}", k, v)); + } + } + configs_write +} + +impl Configuration for GrubCmdline { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + if self.is_cur_partition { + info!("Start setting grub.cmdline.current configuration"); + } else { + info!("Start setting grub.cmdline.next configuration"); + } + if !is_file_exist(&self.grub_path) { + bail!("Failed to find grub.cfg file"); + } + let config_partition = if cfg!(test) { + self.is_cur_partition + } else { + self.get_config_partition(RealCommandExecutor {}) + .with_context(|| "Failed to get config partition".to_string())? + }; + debug!("Config_partition: {} (false means partition A, true means partition B)", config_partition); + let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition) + .with_context(|| "Failed to set grub configs".to_string())?; + write_configs_to_file(&self.grub_path, &configs) + .with_context(|| "Failed to write configs to file".to_string())?; + Ok(()) + } +} + +impl GrubCmdline { + // get_config_partition returns false if the menuentry to be configured is A, true for menuentry B + fn get_config_partition(&self, executor: T) -> Result { + let (_, next_partition) = get_partition_info(&executor)?; + let mut flag = false; + if next_partition.menuentry == "B" { + flag = true + } + Ok(self.is_cur_partition != flag) + } +} + +fn get_and_set_grubcfg( + expect_configs: &mut HashMap, + grub_path: &str, + config_partition: bool, +) -> Result> { + let f = File::open(grub_path).with_context(|| format!("Failed to open grub.cfg \"{}\"", grub_path))?; + let re_find_cur_linux = r"^\s*linux.*root=.*"; + let re = Regex::new(re_find_cur_linux)?; + let mut configs_write = Vec::new(); + let mut match_config_partition = false; + for line in io::BufReader::new(f).lines() { + let mut line = line?; + if re.is_match(&line) { + if match_config_partition == config_partition { + line = modify_boot_cfg(expect_configs, &line)?; + } + match_config_partition = true; + } + configs_write.push(line); + } + Ok(configs_write) +} + +fn modify_boot_cfg(expect_configs: &mut HashMap, line: &String) -> Result { + trace!("Match partition that need to be configured, entering modify_boot_cfg, linux line: {}", line); + let mut new_configs = vec![" ".to_string()]; + let olg_configs: Vec<&str> = line.split(' ').collect(); + for old_config in olg_configs { + if old_config.is_empty() { + continue; + } + // At most 2 substrings can be returned to satisfy the case like root=UUID=xxxx + let config = old_config.splitn(2, '=').collect::>(); + if config.len() != values::ONLY_KEY && config.len() != values::KV_PAIR { + bail!("Failed to parse grub.cfg linux line {}", old_config); + } + let new_key_info = expect_configs.get(config[0]); + let new_config = match new_key_info { + Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config, new_key_info), + Some(new_key_info) => handle_update_key(&config, new_key_info), + None => config.join("="), + }; + if !new_config.is_empty() { + new_configs.push(new_config); + } + expect_configs.remove(config[0]); + } + let new_config = handle_add_key(expect_configs, true); + new_configs.extend(new_config); + Ok(new_configs.join(" ")) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use mockall::{mock, predicate::*}; + use tempfile::{NamedTempFile, TempDir}; + + use super::*; + use crate::sys_mgmt::{GRUB_CMDLINE_CURRENT, GRUB_CMDLINE_NEXT, KERNEL_SYSCTL, KERNEL_SYSCTL_PERSIST}; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_config_partition() { + init(); + let mut grub_cmdline = GrubCmdline { grub_path: String::from(""), is_cur_partition: true }; + let mut executor = MockCommandExec::new(); + + // the output shows that current root menuentry is A + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + + let result = grub_cmdline.get_config_partition(executor).unwrap(); + // it should return false because the current root menuentry is A and we want to configure current partition + assert_eq!(result, false); + + let mut executor = MockCommandExec::new(); + + // the output shows that current root menuentry is A + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + grub_cmdline.is_cur_partition = false; + let result = grub_cmdline.get_config_partition(executor).unwrap(); + // it should return true because the current root menuentry is A and we want to configure next partition + assert_eq!(result, true); + } + + #[test] + fn test_kernel_sysctl() { + init(); + let tmp_dir = TempDir::new().unwrap(); + assert_eq!(tmp_dir.path().exists(), true); + let kernel_sysctl = KernelSysctl::new(tmp_dir.path().to_str().unwrap()); + + let config_detail = HashMap::from([ + ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }), + ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }), + ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ]); + + let mut config = + Sysconfig { model: KERNEL_SYSCTL.to_string(), config_path: String::from(""), contents: config_detail }; + kernel_sysctl.set_config(&mut config).unwrap(); + + let result = fs::read_to_string(format!("{}{}", tmp_dir.path().to_str().unwrap(), "a")).unwrap(); + assert_eq!(result, "1\n"); + } + + #[test] + fn test_kernel_sysctl_persist() { + init(); + let comment = r"# This file is managed by KubeOS for unit testing."; + // create a tmp file with comment + let mut tmp_file = tempfile::NamedTempFile::new().unwrap(); + writeln!(tmp_file, "{}", comment).unwrap(); + writeln!(tmp_file, "a=0").unwrap(); + writeln!(tmp_file, "d=4").unwrap(); + writeln!(tmp_file, "e=5").unwrap(); + writeln!(tmp_file, "g=7").unwrap(); + let kernel_sysctl_persist = KernelSysctlPersist {}; + let config_detail = HashMap::from([ + ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }), + ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }), + ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ("f".to_string(), KeyInfo { value: "".to_string(), operation: "add".to_string() }), + ("g".to_string(), KeyInfo { value: "7".to_string(), operation: "delete".to_string() }), + ("".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }), + ("s=x".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }), + ]); + let mut config = Sysconfig { + model: KERNEL_SYSCTL_PERSIST.to_string(), + config_path: String::from(tmp_file.path().to_str().unwrap()), + contents: config_detail, + }; + kernel_sysctl_persist.set_config(&mut config).unwrap(); + let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); + let expected_res = format!("{}\n{}\n{}\n{}\n{}\n", comment, "a=1", "d=4", "e=5", "c=3"); + assert_eq!(result, expected_res); + let mut config = Sysconfig { + model: KERNEL_SYSCTL_PERSIST.to_string(), + config_path: String::from("/tmp/kubeos-test-kernel-sysctl-persist.txt"), + contents: HashMap::new(), + }; + kernel_sysctl_persist.set_config(&mut config).unwrap(); + assert!(is_file_exist(&config.config_path)); + delete_file_or_dir(&config.config_path).unwrap(); + } + + #[test] + fn write_configs_to_file_tests() { + init(); + let tmp_file = NamedTempFile::new().unwrap(); + let configs = vec!["a=1".to_string(), "b=2".to_string()]; + write_configs_to_file(tmp_file.path().to_str().unwrap(), &configs).unwrap(); + assert_eq!(fs::read(tmp_file.path()).unwrap(), b"a=1\nb=2\n"); + } + + #[test] + fn test_grub_cmdline() { + init(); + let mut tmp_file = NamedTempFile::new().unwrap(); + let mut grub_cmdline = + GrubCmdline { grub_path: tmp_file.path().to_str().unwrap().to_string(), is_cur_partition: true }; + let grub_cfg = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +} + +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=UUID=2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +}"; + writeln!(tmp_file, "{}", grub_cfg).unwrap(); + let config_second_part = HashMap::from([ + ("debug".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("quiet".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }), + ("nomodeset".to_string(), KeyInfo { value: "".to_string(), operation: "update".to_string() }), + ("oops".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("".to_string(), KeyInfo { value: "test".to_string(), operation: "".to_string() }), + ("selinux".to_string(), KeyInfo { value: "1".to_string(), operation: "delete".to_string() }), + ("acpi".to_string(), KeyInfo { value: "off".to_string(), operation: "delete".to_string() }), + ("ro".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ]); + let mut config = Sysconfig { + model: GRUB_CMDLINE_CURRENT.to_string(), + config_path: String::new(), + contents: config_second_part, + }; + grub_cmdline.set_config(&mut config).unwrap(); + grub_cmdline.is_cur_partition = false; + let config_first_part = HashMap::from([ + ("pci".to_string(), KeyInfo { value: "nomis".to_string(), operation: "".to_string() }), + ("quiet".to_string(), KeyInfo { value: "11".to_string(), operation: "delete".to_string() }), + ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "update".to_string() }), + ]); + config.contents = config_first_part; + config.model = GRUB_CMDLINE_NEXT.to_string(); + grub_cmdline.set_config(&mut config).unwrap(); + let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); + let expected_res = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 pci=nomis + initrd /boot/initramfs.img +} +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=UUID=2 ro=1 rootfstype=ext4 nomodeset oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 debug + initrd /boot/initramfs.img +} +"; + assert_eq!(result, expected_res); + + // test grub.cfg not exist + grub_cmdline.grub_path = "/tmp/grub-KubeOS-test.cfg".to_string(); + let res = grub_cmdline.set_config(&mut config); + assert!(res.is_err()); + } + + #[test] + fn test_create_config_file() { + init(); + let tmp_file = "/tmp/kubeos-test-create-config-file.txt"; + create_config_file(&tmp_file).unwrap(); + assert!(is_file_exist(&tmp_file)); + fs::remove_file(tmp_file).unwrap(); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs new file mode 100644 index 00000000..80caf291 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -0,0 +1,301 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs, os::unix::fs::PermissionsExt, path::Path}; + +use anyhow::{anyhow, Context, Result}; +use log::{debug, info}; + +use crate::{ + api::{ImageHandler, UpgradeRequest}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES}, + utils::*, +}; + +pub struct CtrImageHandler { + pub paths: PreparePath, + pub executor: T, +} + +const DEFAULT_NAMESPACE: &str = "k8s.io"; + +impl ImageHandler for CtrImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?; + self.get_image(req)?; + self.get_rootfs_archive(req, IMAGE_PERMISSION)?; + + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); + img_manager.create_os_image(IMAGE_PERMISSION) + } +} + +impl Default for CtrImageHandler { + fn default() -> Self { + Self { paths: PreparePath::default(), executor: RealCommandExecutor {} } + } +} + +impl CtrImageHandler { + #[cfg(test)] + pub fn new(paths: PreparePath, executor: T) -> Self { + Self { paths, executor } + } + + fn get_image(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + is_valid_image_name(image_name)?; + let cli: String = + if is_command_available("crictl", &self.executor) { "crictl".to_string() } else { "ctr".to_string() }; + remove_image_if_exist(&cli, image_name, &self.executor)?; + info!("Start pulling image {}", image_name); + pull_image(&cli, image_name, &self.executor)?; + info!("Start checking image digest"); + check_oci_image_digest(&cli, image_name, &req.check_sum, &self.executor)?; + Ok(()) + } + + fn get_rootfs_archive(&self, req: &UpgradeRequest, permission: u32) -> Result<()> { + let image_name = &req.container_image; + let mount_path = &self + .paths + .mount_path + .to_str() + .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?; + info!("Start getting rootfs {}", image_name); + self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?; + self.executor + .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?; + // copy os.tar from mount_path to its partent dir + self.copy_file(self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission)?; + self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?; + Ok(()) + } + + fn check_and_unmount(&self, mount_path: &str) -> Result<()> { + let ctr_snapshot_cmd = + format!("ctr -n={} snapshots ls | grep {} | awk '{{print $1}}'", DEFAULT_NAMESPACE, mount_path); + let exist_snapshot = self.executor.run_command_with_output("bash", &["-c", &ctr_snapshot_cmd])?; + if !exist_snapshot.is_empty() { + self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "unmount", mount_path])?; + self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "snapshots", "remove", mount_path])?; + } + Ok(()) + } + + fn copy_file, Q: AsRef>(&self, src: P, dst: Q, permission: u32) -> Result<()> { + let copied_bytes = fs::copy(src.as_ref(), dst.as_ref())?; + debug!("Copy {} to {}, total bytes: {}", src.as_ref().display(), dst.as_ref().display(), copied_bytes); + fs::set_permissions(dst, fs::Permissions::from_mode(permission))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{io::Write, path::PathBuf}; + + use mockall::mock; + use tempfile::NamedTempFile; + + use super::*; + use crate::api::CertsInfo; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + // mock is_command_available + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "/bin/sh" && args.contains(&"command -v crictl")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock remove_image_if_exist + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.contains(&"inspecti")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.contains(&"rmi")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock pull_image + mock_executor + .expect_run_command() + .withf(|cmd, args| { + cmd == "crictl" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest") + }) + .times(1) + .returning(|_, _| Ok(())); + // mock get_oci_image_digest + let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]"; + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "crictl" && args.contains(&"inspecti") && args.contains(&"{{.status.repoDigests}}") + }) + .times(1) + .returning(|_, _| Ok(command_output2.to_string())); + let ctr = CtrImageHandler::new(PreparePath::default(), mock_executor); + let result = ctr.get_image(&req); + assert!(result.is_ok()); + } + + #[test] + fn test_get_rootfs_archive() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + // mock check_and_unmount + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure + .times(1) + .returning(|_, _| Ok("".to_string())); + + // mock ctr mount rw + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[4] == "--rw") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + // create temp file for copy + let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file."); + writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file."); + + // Get the path of the temporary file and the path where it should be copied. + let src_dir = tmp_file.path().parent().unwrap(); + let src_file_name = tmp_file.path().file_name().unwrap().to_str().unwrap().to_string(); + let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file."); + let dst_path = dst_file.path().to_path_buf(); + + let paths = PreparePath { + persist_path: "/tmp".into(), + update_path: PathBuf::new(), + image_path: PathBuf::new(), + mount_path: src_dir.to_path_buf(), + rootfs_file: src_file_name.clone(), + tar_path: dst_path.clone(), + }; + + // mock check_and_unmount + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure + .times(1) + .returning(|_, _| Ok("".to_string())); + + let ctr = CtrImageHandler::new(paths, mock_executor); + let result = ctr.get_rootfs_archive(&req, IMAGE_PERMISSION); + assert!(result.is_ok()); + } + + #[test] + fn test_copy_file() { + // Setup: Create a temporary file and write some data to it. + let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file."); + writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file."); + + // Get the path of the temporary file and the path where it should be copied. + let src_path = tmp_file.path().to_str().unwrap().to_string(); + let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file."); + let dst_path = dst_file.path().to_str().unwrap().to_string(); + + let ctr = CtrImageHandler::default(); + let result = ctr.copy_file(&src_path, &dst_path, IMAGE_PERMISSION); + + assert!(result.is_ok()); + + let expected_content = "Hello, world!\n"; + let actual_content = fs::read_to_string(&dst_path).expect("Failed to read destination file."); + assert_eq!(expected_content, actual_content); + + // Assert the file permission + let metadata = fs::metadata(&dst_path).expect("Failed to read destination file."); + let expected_permission = 0o100600; + assert_eq!(metadata.permissions().mode(), expected_permission); + } + + #[test] + fn test_check_and_unmount() { + let mut mock_executor = MockCommandExec::new(); + + // When `run_command_with_output` is called with "bash" and the specific args, it will return Ok("snapshot_exists"). + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") + .times(1) + .returning(|_, _| Ok("snapshot_exists".to_string())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"images")) + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"snapshots")) + .times(1) + .returning(|_, _| Ok(())); + + let result = CtrImageHandler::new(PreparePath::default(), mock_executor).check_and_unmount("test_mount_path"); + + assert!(result.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs new file mode 100644 index 00000000..6d836dc4 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs @@ -0,0 +1,406 @@ +use std::{ + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; + +use anyhow::{bail, Context, Result}; +use log::{debug, info, trace}; +use reqwest::{blocking::Client, Certificate}; +use sha2::{Digest, Sha256}; + +use crate::{ + api::{CertsInfo, ImageHandler, UpgradeRequest}, + sys_mgmt::{CERTS_PATH, IMAGE_PERMISSION, PERSIST_DIR}, + utils::*, +}; + +const BUFFER: u64 = 1024 * 1024 * 10; + +pub struct DiskImageHandler { + pub paths: PreparePath, + pub executor: T, + pub certs_path: String, +} + +impl ImageHandler for DiskImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + self.download(req)?; + self.checksum_match(self.paths.image_path.to_str().unwrap_or_default(), &req.check_sum)?; + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); + Ok(img_manager) + } +} + +impl Default for DiskImageHandler { + fn default() -> Self { + Self { paths: PreparePath::default(), executor: RealCommandExecutor {}, certs_path: CERTS_PATH.to_string() } + } +} + +impl DiskImageHandler { + #[cfg(test)] + pub fn new(paths: PreparePath, executor: T, certs_path: String) -> Self { + Self { paths, executor, certs_path } + } + + fn download(&self, req: &UpgradeRequest) -> Result<()> { + let mut resp = self.send_download_request(req)?; + if resp.status() != reqwest::StatusCode::OK { + bail!("Failed to download image from {}, status: {}", req.image_url, resp.status()); + } + debug!("Received response body size: {:?}", resp.content_length().unwrap_or_default()); + let need_bytes = resp.content_length().unwrap_or_default() + BUFFER; + + check_disk_size( + i64::try_from(need_bytes).with_context(|| "Failed to transform content length from u64 to i64")?, + self.paths.image_path.parent().unwrap_or_else(|| Path::new(PERSIST_DIR)), + )?; + + let mut out = fs::File::create(&self.paths.image_path)?; + trace!("Start to save upgrade image to path {}", &self.paths.image_path.display()); + out.set_permissions(fs::Permissions::from_mode(IMAGE_PERMISSION))?; + let bytes = resp.copy_to(&mut out)?; + info!( + "Download image successfully, upgrade image path: {}, write bytes: {}", + &self.paths.image_path.display(), + bytes + ); + Ok(()) + } + + fn checksum_match(&self, file_path: &str, check_sum: &str) -> Result<()> { + info!("Start checking image checksum"); + let check_sum = check_sum.to_ascii_lowercase(); + let file = fs::read(file_path)?; + let mut hasher = Sha256::new(); + hasher.update(file); + let hash = hasher.finalize(); + // sha256sum -b /persist/update.img + let cal_sum = format!("{:X}", hash).to_ascii_lowercase(); + if cal_sum != check_sum { + delete_file_or_dir(file_path)?; + bail!("Checksum {} mismatch to {}", cal_sum, check_sum); + } + debug!("Checksum match"); + Ok(()) + } + + fn send_download_request(&self, req: &UpgradeRequest) -> Result { + let client: Client; + + if !req.image_url.starts_with("https://") { + // http request + if !req.flag_safe { + bail!("The upgrade image url is not safe"); + } + info!("Discover http request to: {}", &req.image_url); + client = Client::new(); + } else if req.mtls { + // https mtls request + client = self.load_ca_client_certs(&req.certs).with_context(|| "Failed to load client certificates")?; + info!("Discover https mtls request to: {}", &req.image_url); + } else { + // https request + client = self.load_ca_certs(&req.certs.ca_cert).with_context(|| "Failed to load CA certificates")?; + info!("Discover https request to: {}", &req.image_url); + } + + client.get(&req.image_url).send().with_context(|| format!("Failed to fetch from URL: {}", &req.image_url)) + } + + fn load_ca_certs(&self, ca_cert: &str) -> Result { + trace!("Start to load CA certificates"); + self.cert_exist(ca_cert)?; + let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(ca_cert))?)?; + let client = Client::builder().add_root_certificate(ca).build()?; + Ok(client) + } + + fn load_ca_client_certs(&self, certs: &CertsInfo) -> Result { + trace!("Start to load CA and client certificates"); + self.cert_exist(&certs.ca_cert)?; + let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(&certs.ca_cert))?)?; + + self.cert_exist(&certs.client_cert)?; + self.cert_exist(&certs.client_key)?; + let client_cert = std::fs::read(self.get_certs_path(&certs.client_cert))?; + let client_key = std::fs::read(self.get_certs_path(&certs.client_key))?; + let mut client_identity = Vec::new(); + client_identity.extend_from_slice(&client_cert); + client_identity.extend_from_slice(&client_key); + let client_id = reqwest::Identity::from_pem(&client_identity)?; + + let client = Client::builder().use_rustls_tls().add_root_certificate(ca).identity(client_id).build()?; + Ok(client) + } + + fn cert_exist(&self, cert_file: &str) -> Result<()> { + if cert_file.is_empty() { + bail!("Please provide the certificate"); + } + if !self.get_certs_path(cert_file).exists() { + bail!("Certificate does not exist: {}", cert_file); + } + Ok(()) + } + + fn get_certs_path(&self, cert: &str) -> PathBuf { + let cert_path = format!("{}{}", self.certs_path, cert); + PathBuf::from(cert_path) + } +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use mockall::mock; + use mockito; + use tempfile::NamedTempFile; + + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + #[test] + fn test_get_certs_path() { + init(); + let handler = DiskImageHandler::::default(); + let certs_path = handler.get_certs_path("ca.pem"); + assert_eq!(certs_path.to_str().unwrap(), "/etc/KubeOS/certs/ca.pem"); + } + + #[test] + fn test_cert_exist() { + init(); + // generate tmp file + let tmp_file = NamedTempFile::new().unwrap(); + let handler = + DiskImageHandler::::new(PreparePath::default(), RealCommandExecutor {}, String::new()); + let res = handler.cert_exist(tmp_file.path().to_str().unwrap()); + assert!(res.is_ok()); + + assert!(handler.cert_exist("aaa.pem").is_err()); + assert!(handler.cert_exist("").is_err()) + } + + #[test] + fn test_send_download_request() { + init(); + // This is a tmp cert only for KubeOS unit testing. + let tmp_cert = "-----BEGIN CERTIFICATE-----\n\ + MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\ + MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\ + MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\ + bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\ + Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\ + HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\ + c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\ + vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\ + -----END CERTIFICATE-----\n"; + + // This is a tmp private key only for KubeOS unit testing. + let tmp_key = "-----BEGIN PRIVATE KEY-----\n\ + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\ + Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\ + nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\ + -----END PRIVATE KEY-----\n"; + + // Create a temporary file to hold the certificate + let mut cert_file = NamedTempFile::new().unwrap(); + cert_file.write_all(tmp_cert.as_bytes()).unwrap(); + println!("cert_file: {:?}", cert_file.path().to_str().unwrap()); + + // Create a temporary file to hold the private key + let mut key_file = NamedTempFile::new().unwrap(); + key_file.write_all(tmp_key.as_bytes()).unwrap(); + // http + let handler = DiskImageHandler::::default(); + let mut req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "http://localhost:8080/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + req.flag_safe = false; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + + // https + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "/tmp".to_string(); + let tmp_cert_filename = cert_file.path().file_name().unwrap().to_str().unwrap(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "https://localhost:8081/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { + ca_cert: tmp_cert_filename.to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + + // mtls + let tmp_key = NamedTempFile::new().unwrap(); + let tmp_key_filename = tmp_key.path().file_name().unwrap().to_str().unwrap(); + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "/tmp".to_string(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "https://localhost:8082/aaa.txt".to_string(), + flag_safe: true, + mtls: true, + certs: CertsInfo { + ca_cert: tmp_cert_filename.to_string(), + client_cert: tmp_cert_filename.to_string(), + client_key: tmp_key_filename.to_string(), + }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + } + + #[test] + fn test_checksum_match() { + init(); + let mut tmp_file = NamedTempFile::new().unwrap(); + tmp_file.write(b"This is a test txt file for KubeOS test.\n").unwrap(); + let mut handler = DiskImageHandler::::default(); + handler.paths.image_path = tmp_file.path().to_path_buf(); + let mut req = UpgradeRequest { + version: "v2".into(), + check_sum: "98Ea7aff44631D183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "http://localhost:8080/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + assert_eq!(handler.paths.image_path.exists(), true); + handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum).unwrap(); + + req.check_sum = "1234567Abc".into(); + let res = handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum); + assert!(res.is_err()); + } + + #[test] + fn test_load_certs() { + init(); + // This is a tmp cert only for KubeOS unit testing. + let tmp_cert = "-----BEGIN CERTIFICATE-----\n\ + MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\ + MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\ + MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\ + bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\ + Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\ + HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\ + c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\ + vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\ + -----END CERTIFICATE-----\n"; + + // This is a tmp private key only for KubeOS unit testing. + let tmp_key = "-----BEGIN PRIVATE KEY-----\n\ + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\ + Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\ + nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\ + -----END PRIVATE KEY-----\n"; + + // Create a temporary file to hold the certificate + let mut cert_file = NamedTempFile::new().unwrap(); + cert_file.write_all(tmp_cert.as_bytes()).unwrap(); + + // Create a temporary file to hold the private key + let mut key_file = NamedTempFile::new().unwrap(); + key_file.write_all(tmp_key.as_bytes()).unwrap(); + + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "".to_string(); + let certs = CertsInfo { + ca_cert: cert_file.path().to_str().unwrap().to_string(), + client_cert: cert_file.path().to_str().unwrap().to_string(), + client_key: key_file.path().to_str().unwrap().to_string(), + }; + + let res = handler.load_ca_client_certs(&certs); + assert!(res.is_ok()); + + let res = handler.load_ca_certs(&certs.ca_cert); + assert!(res.is_ok()); + } + + #[test] + fn test_download_image() { + init(); + let tmp_file = NamedTempFile::new().unwrap(); + + let mock_executor = MockCommandExec::new(); + let mut handler = DiskImageHandler::new(PreparePath::default(), mock_executor, String::new()); + handler.executor.expect_clone().times(1).returning(|| MockCommandExec::new()); + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + handler.executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + handler.paths.image_path = tmp_file.path().to_path_buf(); + assert_eq!(true, handler.paths.image_path.exists()); + + let url = mockito::server_url(); + let upgrade_request = UpgradeRequest { + version: "v2".into(), + check_sum: "98ea7aff44631d183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: format!("{}/test.txt", url), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let _m = mockito::mock("GET", "/test.txt") + .with_status(200) + .with_body("This is a test txt file for KubeOS test.\n") + .create(); + handler.download_image(&upgrade_request).unwrap(); + assert_eq!(true, handler.paths.image_path.exists()); + assert_eq!( + fs::read(handler.paths.image_path.to_str().unwrap()).unwrap(), + "This is a test txt file for KubeOS test.\n".as_bytes() + ); + + let _m = mockito::mock("GET", "/test.txt").with_status(404).with_body("Not found").create(); + let res = handler.download_image(&upgrade_request); + assert!(res.is_err()) + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs new file mode 100644 index 00000000..4d97552c --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -0,0 +1,236 @@ +use anyhow::{Context, Result}; +use log::{debug, info, trace}; + +use crate::{ + api::{ImageHandler, UpgradeRequest}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES}, + utils::*, +}; + +pub struct DockerImageHandler { + pub paths: PreparePath, + pub container_name: String, + pub executor: T, +} + +impl ImageHandler for DockerImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?; + self.get_image(req)?; + self.get_rootfs_archive(req)?; + + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); + img_manager.create_os_image(IMAGE_PERMISSION) + } +} + +impl Default for DockerImageHandler { + fn default() -> Self { + Self { paths: PreparePath::default(), container_name: "kubeos-temp".into(), executor: RealCommandExecutor {} } + } +} + +impl DockerImageHandler { + #[cfg(test)] + pub fn new(paths: PreparePath, container_name: String, executor: T) -> Self { + Self { paths, container_name, executor } + } + + fn get_image(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + is_valid_image_name(image_name)?; + let cli = "docker"; + remove_image_if_exist(cli, image_name, &self.executor)?; + info!("Start pulling image {}", image_name); + pull_image(cli, image_name, &self.executor)?; + info!("Start checking image digest"); + check_oci_image_digest(cli, image_name, &req.check_sum, &self.executor)?; + Ok(()) + } + + fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + info!("Start getting rootfs {}", image_name); + self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?; + debug!("Create container {}", self.container_name); + let container_id = + self.executor.run_command_with_output("docker", &["create", "--name", &self.container_name, image_name])?; + debug!("Copy rootfs from container {} to {}", container_id, self.paths.update_path.display()); + self.executor.run_command( + "docker", + &[ + "cp", + format!("{}:/{}", container_id, self.paths.rootfs_file).as_str(), + self.paths.update_path.to_str().unwrap(), + ], + )?; + self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?; + Ok(()) + } + + fn check_and_rm_container(&self) -> Result<()> { + trace!("Check and remove container {}", self.container_name); + let docker_ps_cmd = format!("docker ps -a -f=name={} | awk 'NR==2' | awk '{{print $1}}'", self.container_name); + let exist_id = self.executor.run_command_with_output("bash", &["-c", &docker_ps_cmd])?; + if !exist_id.is_empty() { + info!("Remove container {} {} for cleaning environment", self.container_name, exist_id); + self.executor.run_command("docker", &["rm", exist_id.as_str()])?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use mockall::mock; + + use super::*; + use crate::api::CertsInfo; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_check_and_rm_container() { + init(); + let mut mock_executor = MockCommandExec::new(); + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" + && args.len() == 2 + && args.contains(&"docker ps -a -f=name=test | awk 'NR==2' | awk '{print $1}'") + }) + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111")) + .times(1) + .returning(|_, _| Ok(())); + + let result = + DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor).check_and_rm_container(); + assert!(result.is_ok()); + + assert_eq!(DockerImageHandler::default().container_name, "kubeos-temp"); + } + + #[test] + fn test_get_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "docker".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + // mock remove_image_if_exist + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rmi")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock pull_image + mock_executor + .expect_run_command() + .withf(|cmd, args| { + cmd == "docker" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest") + }) + .times(1) + .returning(|_, _| Ok(())); + // mock get_oci_image_digest + let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]"; + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect") && args.contains(&"{{.RepoDigests}}")) + .times(1) + .returning(|_, _| Ok(command_output2.to_string())); + + let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor); + let result = docker.get_image(&req); + assert!(result.is_ok()); + } + + #[test] + fn test_get_rootfs_archive() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "docker".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + // mock check_and_rm_container + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'") + }) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::new())); + // mock get_rootfs_archive + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "docker" && args.contains(&"create")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"cp") && args.contains(&"1111:/os.tar")) + .times(1) + .returning(|_, _| Ok(())); + // mock check_and_rm_container + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'") + }) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111")) + .times(1) + .returning(|_, _| Ok(())); + + let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor); + let result = docker.get_rootfs_archive(&req); + assert!(result.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/mod.rs b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs new file mode 100644 index 00000000..0e06f297 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs @@ -0,0 +1,23 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod config; +mod containerd_image; +mod disk_image; +mod docker_image; +mod values; + +pub use config::*; +pub use containerd_image::*; +pub use disk_image::*; +pub use docker_image::*; +pub use values::*; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/values.rs b/KubeOS-Rust/manager/src/sys_mgmt/values.rs new file mode 100644 index 00000000..b107efc3 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/values.rs @@ -0,0 +1,36 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub const KERNEL_SYSCTL: &str = "kernel.sysctl"; +pub const KERNEL_SYSCTL_PERSIST: &str = "kernel.sysctl.persist"; +pub const GRUB_CMDLINE_CURRENT: &str = "grub.cmdline.current"; +pub const GRUB_CMDLINE_NEXT: &str = "grub.cmdline.next"; + +pub const DEFAULT_PROC_PATH: &str = "/proc/sys/"; +pub const DEFAULT_KERNEL_CONFIG_PATH: &str = "/etc/sysctl.conf"; +pub const DEFAULT_GRUB_CFG_PATH: &str = "/boot/efi/EFI/openEuler/grub.cfg"; +pub const DEFAULT_GRUBENV_PATH: &str = "/boot/efi/EFI/openEuler/grubenv"; + +pub const PERSIST_DIR: &str = "/persist"; +pub const ROOTFS_ARCHIVE: &str = "os.tar"; +pub const UPDATE_DIR: &str = "KubeOS-Update"; +pub const MOUNT_DIR: &str = "kubeos-update"; +pub const OS_IMAGE_NAME: &str = "update.img"; +pub const CERTS_PATH: &str = "/etc/KubeOS/certs/"; + +pub const DEFAULT_KERNEL_CONFIG_PERM: u32 = 0o644; +pub const DEFAULT_GRUB_CFG_PERM: u32 = 0o751; +pub const IMAGE_PERMISSION: u32 = 0o600; + +pub const ONLY_KEY: usize = 1; +pub const KV_PAIR: usize = 2; +pub const NEED_BYTES: i64 = 3 * 1024 * 1024 * 1024; diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs new file mode 100644 index 00000000..9baf99e3 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -0,0 +1,310 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs, + os::{linux::fs::MetadataExt, unix::fs::DirBuilderExt}, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, bail, Context, Result}; +use log::{debug, info, trace}; +use nix::{mount, mount::MntFlags}; + +use super::executor::CommandExecutor; +use crate::sys_mgmt::{MOUNT_DIR, OS_IMAGE_NAME, PERSIST_DIR, ROOTFS_ARCHIVE, UPDATE_DIR}; + +/// * persist_path: /persist +/// +/// * update_path: /persist/KubeOS-Update +/// +/// * mount_path: /persist/KubeOS-Update/kubeos-update +/// +/// * tar_path: /persist/KubeOS-Update/os.tar +/// +/// * image_path: /persist/update.img +/// +/// * rootfs_file: os.tar +#[derive(Clone)] +pub struct PreparePath { + pub persist_path: PathBuf, + pub update_path: PathBuf, + pub mount_path: PathBuf, + pub tar_path: PathBuf, + pub image_path: PathBuf, + pub rootfs_file: String, +} + +impl Default for PreparePath { + fn default() -> Self { + let persist_dir = Path::new(PERSIST_DIR); + let update_pathbuf = persist_dir.join(UPDATE_DIR); + Self { + persist_path: persist_dir.to_path_buf(), + update_path: update_pathbuf.clone(), + mount_path: update_pathbuf.join(MOUNT_DIR), + tar_path: update_pathbuf.join(ROOTFS_ARCHIVE), + image_path: persist_dir.join(OS_IMAGE_NAME), + rootfs_file: ROOTFS_ARCHIVE.to_string(), + } + } +} + +pub fn is_file_exist>(path: P) -> bool { + path.as_ref().exists() +} + +pub fn perpare_env(prepare_path: &PreparePath, need_bytes: i64, permission: u32) -> Result<()> { + info!("Prepare environment to upgrade"); + check_disk_size(need_bytes, &prepare_path.persist_path)?; + clean_env(&prepare_path.update_path, &prepare_path.mount_path, &prepare_path.image_path)?; + fs::DirBuilder::new().recursive(true).mode(permission).create(&prepare_path.mount_path)?; + Ok(()) +} + +pub fn check_disk_size>(need_bytes: i64, path: P) -> Result<()> { + trace!("Check if there is enough disk space to upgrade"); + let fs_stat = nix::sys::statfs::statfs(path.as_ref())?; + let available_blocks = i64::try_from(fs_stat.blocks_available())?; + let available_space = available_blocks * fs_stat.block_size(); + if available_space < need_bytes { + bail!("Space is not enough for downloading"); + } + Ok(()) +} + +/// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img +pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> +where + P: AsRef + std::fmt::Debug, +{ + if is_mounted(&mount_path)? { + debug!("Umount \"{}\"", mount_path.as_ref().display()); + if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) { + bail!("Failed to umount {} in clean_env: {}", mount_path.as_ref().display(), errno); + } + } + // losetup -D? + delete_file_or_dir(&update_path).with_context(|| format!("Failed to delete {:?}", update_path))?; + delete_file_or_dir(&image_path).with_context(|| format!("Failed to delete {:?}", image_path))?; + Ok(()) +} + +pub fn delete_file_or_dir>(path: P) -> Result<()> { + if is_file_exist(&path) { + if fs::metadata(&path)?.is_file() { + info!("Delete file \"{}\"", path.as_ref().display()); + fs::remove_file(&path)?; + } else { + info!("Delete directory \"{}\"", path.as_ref().display()); + fs::remove_dir_all(&path)?; + } + } + Ok(()) +} + +pub fn is_command_available(command: &str, command_executor: &T) -> bool { + match command_executor.run_command("/bin/sh", &["-c", format!("command -v {}", command).as_str()]) { + Ok(_) => { + debug!("command {} is available", command); + true + }, + Err(_) => { + debug!("command {} is not available", command); + false + }, + } +} + +pub fn is_mounted>(mount_path: P) -> Result { + if !is_file_exist(&mount_path) { + return Ok(false); + } + // Get device ID of mountPath + let mount_meta = fs::symlink_metadata(&mount_path)?; + let dev = mount_meta.st_dev(); + + // Get device ID of mountPath's parent directory + let parent = mount_path + .as_ref() + .parent() + .ok_or_else(|| anyhow!("Failed to get parent directory of {}", mount_path.as_ref().display()))?; + let parent_meta = fs::symlink_metadata(parent)?; + let dev_parent = parent_meta.st_dev(); + Ok(dev != dev_parent) +} + +pub fn switch_boot_menuentry( + command_executor: &T, + grub_env_path: &str, + next_menuentry: &str, +) -> Result<()> { + if get_boot_mode() == "uefi" { + command_executor.run_command( + "grub2-editenv", + &[grub_env_path, "set", format!("saved_entry={}", next_menuentry).as_str()], + )?; + } else { + command_executor.run_command("grub2-set-default", &[next_menuentry])?; + } + Ok(()) +} + +pub fn get_boot_mode() -> String { + if is_file_exist("/sys/firmware/efi") { "uefi".into() } else { "bios".into() } +} + +#[cfg(test)] +mod tests { + use mockall::{mock, predicate::*}; + use tempfile::{NamedTempFile, TempDir}; + + use super::*; + use crate::utils::RealCommandExecutor; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_file_exist() { + init(); + let path = "/tmp/test_is_file_exist"; + assert_eq!(is_file_exist(path), false); + + let file = NamedTempFile::new().unwrap(); + assert_eq!(is_file_exist(file.path().to_str().unwrap()), true); + + let tmp_dir = TempDir::new().unwrap(); + assert_eq!(is_file_exist(tmp_dir.path().to_str().unwrap()), true); + } + + #[test] + fn test_prepare_env() { + init(); + let paths = PreparePath { + persist_path: PathBuf::from("/tmp"), + update_path: PathBuf::from("/tmp/test_prepare_env"), + mount_path: PathBuf::from("/tmp/test_prepare_env/kubeos-update"), + tar_path: PathBuf::from("/tmp/test_prepare_env/os.tar"), + image_path: PathBuf::from("/tmp/test_prepare_env/update.img"), + rootfs_file: "os.tar".to_string(), + }; + perpare_env(&paths, 1 * 1024 * 1024 * 1024, 0o700).unwrap(); + } + + #[test] + fn test_check_disk_size() { + init(); + let path = "/home"; + let gb: i64 = 1 * 1024 * 1024 * 1024; + let need_gb = 1 * gb; + let result = check_disk_size(need_gb, path); + assert!(result.is_ok()); + let need_gb = 10000 * gb; + let result = check_disk_size(need_gb, path); + assert!(result.is_err()); + } + + #[test] + fn test_clean_env() { + init(); + let update_path = "/tmp/test_clean_env"; + let mount_path = "/tmp/test_clean_env/kubeos-update"; + let image_path = "/tmp/test_clean_env/update.img"; + clean_env(&update_path.to_string(), &mount_path.to_string(), &image_path.to_string()).unwrap(); + } + + #[test] + fn test_delete_file_or_dir() { + init(); + let path = "/tmp/test_delete_file"; + fs::File::create(path).unwrap(); + assert_eq!(Path::new(path).exists(), true); + delete_file_or_dir(&path.to_string()).unwrap(); + assert_eq!(Path::new(path).exists(), false); + + let path = "/tmp/test_dir"; + fs::create_dir(path).unwrap(); + assert_eq!(Path::new(path).exists(), true); + delete_file_or_dir(&path.to_string()).unwrap(); + assert_eq!(Path::new(path).exists(), false); + + let path = "/tmp/nonexist"; + delete_file_or_dir(path).unwrap(); + + let path = PathBuf::new(); + delete_file_or_dir(path).unwrap(); + } + + #[test] + fn test_switch_boot_menuentry() { + init(); + let grubenv_path = "/boot/efi/EFI/openEuler/grubenv"; + let next_menuentry = "B"; + let mut mock = MockCommandExec::new(); + if get_boot_mode() == "uefi" { + mock.expect_run_command() + .withf(move |name, args| { + name == "grub2-editenv" + && args[0] == grubenv_path + && args[2] == format!("saved_entry={}", next_menuentry).as_str() + }) + .times(1) // Expect it to be called once + .returning(move |_, _| Ok(())); + } else { + mock.expect_run_command() + .withf(move |name, args| name == "grub2-set-default" && args[0] == next_menuentry) + .times(1) // Expect it to be called once + .returning(move |_, _| Ok(())); + } + + switch_boot_menuentry(&mock, grubenv_path, next_menuentry).unwrap() + } + + #[test] + fn test_get_boot_mode() { + init(); + let boot_mode = get_boot_mode(); + let executor = RealCommandExecutor {}; + let res = executor.run_command("ls", &["/sys/firmware/efi"]); + if res.is_ok() { + assert!(boot_mode == "uefi"); + } else { + assert!(boot_mode == "bios"); + } + } + + #[test] + fn test_is_command_available() { + init(); + let executor = RealCommandExecutor {}; + assert_eq!(is_command_available("ls", &executor), true); + assert_eq!(is_command_available("aaaabb", &executor), false); + } +} diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs new file mode 100644 index 00000000..d84c6853 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/container_image.rs @@ -0,0 +1,341 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::{bail, Result}; +use log::{debug, info, trace}; +use regex::Regex; + +use super::executor::CommandExecutor; + +pub fn is_valid_image_name(image: &str) -> Result<()> { + let pattern = r"^((?:[\w.-]+)(?::\d+)?/)*(?:[\w.-]+)((?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$"; + let reg_ex = Regex::new(pattern)?; + if !reg_ex.is_match(image) { + bail!("Invalid image name: {}", image); + } + debug!("Image name {} is valid", image); + Ok(()) +} + +pub fn check_oci_image_digest( + container_runtime: &str, + image_name: &str, + check_sum: &str, + command_executor: &T, +) -> Result<()> { + let image_digests = get_oci_image_digest(container_runtime, image_name, command_executor)?; + if image_digests.to_lowercase() != check_sum.to_lowercase() { + bail!("Image digest mismatch, expect {}, got {}", check_sum, image_digests); + } + Ok(()) +} + +pub fn get_oci_image_digest( + container_runtime: &str, + image_name: &str, + executor: &T, +) -> Result { + let cmd_output: String; + match container_runtime { + "crictl" => { + cmd_output = executor.run_command_with_output( + "crictl", + &["inspecti", "--output", "go-template", "--template", "{{.status.repoDigests}}", image_name], + )?; + }, + "docker" => { + cmd_output = + executor.run_command_with_output("docker", &["inspect", "--format", "{{.RepoDigests}}", image_name])?; + }, + "ctr" => { + cmd_output = executor + .run_command_with_output("ctr", &["-n", "k8s.io", "images", "ls", &format!("name=={}", image_name)])?; + // Split by whitespaces, we get vec like [REF TYPE DIGEST SIZE PLATFORMS LABELS x x x x x x] + // get the 8th element, and split by ':' to get the digest + let fields: Vec<&str> = cmd_output.split_whitespace().collect(); + if let Some(digest) = fields.get(8).and_then(|field| field.split(':').nth(1)) { + trace!("get_oci_image_digest: {}", digest); + return Ok(digest.to_string()); + } else { + bail!("Failed to get digest from ctr command output: {}", cmd_output); + } + }, + _ => { + bail!("Container runtime {} cannot be recognized", container_runtime); + }, + } + + // Parse the cmd_output to extract the digest + let parts: Vec<&str> = cmd_output.split('@').collect(); + if let Some(last_part) = parts.last() { + if last_part.starts_with("sha256") { + let parsed_parts: Vec<&str> = last_part.trim_matches(|c| c == ']').split(':').collect(); + // After spliiing by ':', we should get vec like [sha256, digests] + if parsed_parts.len() == 2 { + debug!("get_oci_image_digest: {}", parsed_parts[1]); + return Ok(parsed_parts[1].to_string()); // 1 is the index of digests + } + } + } + + bail!("Failed to get digest from command output: {}", cmd_output) +} + +pub fn pull_image(runtime: &str, image_name: &str, executor: &T) -> Result<()> { + debug!("Pull image {}", image_name); + match runtime { + "crictl" => { + executor.run_command("crictl", &["pull", image_name])?; + }, + "ctr" => { + executor.run_command( + "ctr", + &[&"-n", "k8s.io", "images", "pull", "--hosts-dir", "/etc/containerd/certs.d", image_name], + )?; + }, + "docker" => { + executor.run_command("docker", &["pull", image_name])?; + }, + _ => { + bail!("Container runtime {} cannot be recognized", runtime); + }, + } + Ok(()) +} + +pub fn remove_image_if_exist(runtime: &str, image_name: &str, executor: &T) -> Result<()> { + match runtime { + "crictl" => { + if executor.run_command("crictl", &["inspecti", image_name]).is_ok() { + executor.run_command("crictl", &["rmi", image_name])?; + info!("Remove existing upgrade image: {}", image_name); + } + }, + "ctr" => { + let output = executor.run_command_with_output( + "ctr", + &[&"-n", "k8s.io", "images", "check", &format!("name=={}", image_name)], + )?; + if !output.is_empty() { + executor.run_command("ctr", &[&"-n", "k8s.io", "images", "rm", image_name, "--sync"])?; + info!("Remove existing upgrade image: {}", image_name); + } + }, + "docker" => { + if executor.run_command("docker", &["inspect", image_name]).is_ok() { + executor.run_command("docker", &["rmi", image_name])?; + info!("Remove existing upgrade image: {}", image_name); + } + }, + _ => { + bail!("Container runtime {} cannot be recognized", runtime); + }, + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use mockall::{mock, predicate::*}; + + use super::*; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_valid_image_name() { + init(); + let correct_images = vec![ + "alpine", + "alpine:latest", + "localhost/latest", + "library/alpine", + "localhost:1234/test", + "test:1234/blaboon", + "alpine:3.7", + "docker.example.edu/gmr/alpine:3.7", + "docker.example.com:5000/gmr/alpine@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "docker.example.co.uk/gmr/alpine/test2:latest", + "registry.dobby.org/dobby/dobby-servers/arthound:2019-08-08", + "owasp/zap:3.8.0", + "registry.dobby.co/dobby/dobby-servers/github-run:2021-10-04", + "docker.elastic.co/kibana/kibana:7.6.2", + "registry.dobby.org/dobby/dobby-servers/lerphound:latest", + "registry.dobby.org/dobby/dobby-servers/marbletown-poc:2021-03-29", + "marbles/marbles:v0.38.1", + "registry.dobby.org/dobby/dobby-servers/loophole@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "sonatype/nexon:3.30.0", + "prom/node-exporter:v1.1.1", + "sosedoff/pgweb@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "sosedoff/pgweb:latest", + "registry.dobby.org/dobby/dobby-servers/arpeggio:2021-06-01", + "registry.dobby.org/dobby/antique-penguin:release-production", + "dalprodictus/halcon:6.7.5", + "antigua/antigua:v31", + "weblate/weblate:4.7.2-1", + "redis:4.0.01-alpine", + "registry.dobby.com/dobby/dobby-servers/github-run:latest", + "192.168.122.123:5000/kubeos-x86_64:2023-01", + ]; + let wrong_images = vec![ + "alpine;v1.0", + "alpine:latest@sha256:11111111111111111111111111111111", + "alpine|v1.0", + "alpine&v1.0", + "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8", + "192.168.122.123:5000/kubeos-x86_64:2023-01@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + "192.168.122.123:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + "myimage$%^&", + ":myimage", + "/myimage", + "myimage/", + "myimage:", + "myimage@@latest", + "myimage::tag", + "registry.com//myimage:tag", + " myimage", + "myimage ", + "registry.com/:tag", + "myimage:", + "", + ":tag", + "IP:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + ]; + for image in correct_images { + assert!(is_valid_image_name(image).is_ok()); + } + for image in wrong_images { + assert!(is_valid_image_name(image).is_err()); + } + } + + #[test] + fn test_get_oci_image_digest() { + init(); + let mut mock = MockCommandExec::new(); + let container_runtime = "ctr"; + let image_name = "docker.io/nginx:latest"; + let command_output1 = + "REF TYPE DIGEST SIZE PLATFORMS LABELS\ndocker.io/nginx:latest text/html sha256:1111 132.5 KIB - -\n"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + let out1 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); + let expect_output = "1111"; + assert_eq!(out1, expect_output); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok("invalid output".to_string())); + let out2 = get_oci_image_digest(container_runtime, image_name, &mock); + assert!(out2.is_err()); + + let container_runtime = "crictl"; + let command_output2 = "[docker.io/nginx@sha256:1111]"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string())); + let out3 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); + assert_eq!(out3, expect_output); + + let out4 = get_oci_image_digest("invalid", image_name, &mock); + assert!(out4.is_err()); + + let container_runtime = "crictl"; + let command_output3 = "[docker.io/nginx:sha256:1111]"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string())); + let out5 = get_oci_image_digest(container_runtime, image_name, &mock); + assert!(out5.is_err()); + } + + #[test] + fn test_check_oci_image_digest_match() { + init(); + let mut mock = MockCommandExec::new(); + let image_name = "docker.io/nginx:latest"; + let container_runtime = "crictl"; + let command_output = "[docker.io/nginx@sha256:1a2b]"; + let check_sum = "1A2B"; + mock.expect_run_command_with_output().times(2).returning(|_, _| Ok(command_output.to_string())); + let result = check_oci_image_digest(container_runtime, image_name, check_sum, &mock); + assert!(result.is_ok()); + let result = check_oci_image_digest(container_runtime, image_name, "1111", &mock); + assert!(result.is_err()); + } + + #[test] + fn test_pull_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.len() == 2 && args[0] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[3] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.len() == 2 && args[0] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + let image_name = "docker.io/nginx:latest"; + let result = pull_image("crictl", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("ctr", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("docker", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("aaa", image_name, &mock_executor); + assert!(result.is_err()); + } + + #[test] + fn test_remove_image_if_exist() { + init(); + let mut mock_executor = MockCommandExec::new(); + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"check")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("something"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"rm")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + let image_name = "docker.io/nginx:latest"; + let res = remove_image_if_exist("ctr", image_name, &mock_executor); + assert!(res.is_ok()); + + let res = remove_image_if_exist("invalid", image_name, &mock_executor); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/manager/src/utils/executor.rs b/KubeOS-Rust/manager/src/utils/executor.rs new file mode 100644 index 00000000..c87bf2ad --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/executor.rs @@ -0,0 +1,89 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::process::Command; + +use anyhow::{bail, Result}; +use log::{debug, trace}; + +pub trait CommandExecutor: Clone { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; +} + +#[derive(Clone)] +pub struct RealCommandExecutor {} + +impl CommandExecutor for RealCommandExecutor { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()> { + trace!("run_command: {} {:?}", name, args); + let output = Command::new(name).args(args).output()?; + if !output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); + let error_message = String::from_utf8_lossy(&output.stderr); + bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message); + } + debug!("run_command: {} {:?} done", name, args); + Ok(()) + } + + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result { + trace!("run_command_with_output: {} {:?}", name, args); + let output = Command::new(name).args(args).output()?; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + if !output.status.success() { + let error_message = String::from_utf8_lossy(&output.stderr); + bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message); + } + debug!("run_command_with_output: {} {:?} done", name, args); + Ok(stdout.trim_end_matches('\n').to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_run_command_with_output() { + init(); + let executor: RealCommandExecutor = RealCommandExecutor {}; + + // test run_command_with_output + let output = executor.run_command_with_output("echo", &["hello", "world"]).unwrap(); + assert_eq!(output, "hello world"); + let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "cat").as_str()]).unwrap(); + assert_eq!(out, "/usr/bin/cat"); + let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "apple").as_str()]); + assert!(out.is_err()); + } + + #[test] + fn test_run_command() { + init(); + let executor: RealCommandExecutor = RealCommandExecutor {}; + // test run_command + let out = executor.run_command("sh", &["-c", format!("command -v {}", "apple").as_str()]); + assert!(out.is_err()); + + let out = executor.run_command("sh", &["-c", format!("command -v {}", "cat").as_str()]); + assert!(out.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs new file mode 100644 index 00000000..90806cf8 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/image_manager.rs @@ -0,0 +1,206 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{self, Permissions}, + os::unix::fs::PermissionsExt, + path::PathBuf, +}; + +use anyhow::{Context, Result}; +use log::{debug, info}; + +use super::{ + clean_env, + common::{delete_file_or_dir, PreparePath}, + executor::CommandExecutor, + partition::PartitionInfo, +}; + +pub struct UpgradeImageManager { + pub paths: PreparePath, + pub next_partition: PartitionInfo, + pub executor: T, +} + +impl UpgradeImageManager { + pub fn new(paths: PreparePath, next_partition: PartitionInfo, executor: T) -> Self { + Self { paths, next_partition, executor } + } + + fn image_path_str(&self) -> Result<&str> { + self.paths.image_path.to_str().context("Failed to convert image path to string") + } + + fn mount_path_str(&self) -> Result<&str> { + self.paths.mount_path.to_str().context("Failed to convert mount path to string") + } + + fn tar_path_str(&self) -> Result<&str> { + self.paths.tar_path.to_str().context("Failed to convert tar path to string") + } + + pub fn create_image_file(&self, permission: u32) -> Result<()> { + let image_str = self.image_path_str()?; + + debug!("Create image {}", image_str); + self.executor.run_command("dd", &["if=/dev/zero", &format!("of={}", image_str), "bs=2M", "count=1024"])?; + fs::set_permissions(&self.paths.image_path, Permissions::from_mode(permission))?; + Ok(()) + } + + pub fn format_image(&self) -> Result<()> { + let image_str = self.image_path_str()?; + debug!("Format image {}", image_str); + self.executor.run_command( + format!("mkfs.{}", self.next_partition.fs_type).as_str(), + &["-L", format!("ROOT-{}", self.next_partition.menuentry).as_str(), image_str], + )?; + Ok(()) + } + + pub fn mount_image(&self) -> Result<()> { + let image_str = self.image_path_str()?; + let mount_str = self.mount_path_str()?; + debug!("Mount {} to {}", image_str, mount_str); + self.executor.run_command("mount", &["-o", "loop", image_str, mount_str])?; + Ok(()) + } + + pub fn extract_tar_to_image(&self) -> Result<()> { + let tar_str = self.tar_path_str()?; + let mount_str = self.mount_path_str()?; + debug!("Extract {} to mounted path {}", tar_str, mount_str); + self.executor.run_command("tar", &["-xvf", tar_str, "-C", mount_str])?; + Ok(()) + } + + pub fn create_os_image(self, permission: u32) -> Result { + self.create_image_file(permission)?; + self.format_image()?; + self.mount_image()?; + self.extract_tar_to_image()?; + // Pass empty image_path to clean_env but avoid deleting the upgrade image + clean_env(&self.paths.update_path, &self.paths.mount_path, &PathBuf::new())?; + Ok(self) + } + + pub fn install(&self) -> Result<()> { + let image_str = self.image_path_str()?; + let device = self.next_partition.device.as_str(); + self.executor + .run_command("dd", &[format!("if={}", image_str).as_str(), format!("of={}", device).as_str(), "bs=8M"])?; + debug!("Install image {} to {} done", image_str, device); + info!( + "Device {} is overwritten and unable to rollback to the previous version anymore if the eviction of node fails", + device + ); + delete_file_or_dir(image_str)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{fs, io::Write, path::Path}; + + use mockall::{mock, predicate::*}; + use tempfile::NamedTempFile; + + use super::*; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_update_image_manager() { + init(); + // create a dir in tmp dir + let tmp_dir = "/tmp/test_update_image_manager"; + let img_path = format!("{}/test_image", tmp_dir); + let mut temp_file = NamedTempFile::new().unwrap(); + write!(temp_file, "test content").unwrap(); // Writing s + fs::create_dir(tmp_dir).unwrap(); + let clone_img_path = img_path.clone(); + + let mut mock = MockCommandExec::new(); + //mock create_image_file + mock.expect_run_command() + .withf(|name, args| name == "dd" && args[0] == "if=/dev/zero") + .times(1) // Expect it to be called once + .returning(move |_, _| { + // simulate 'dd' by copying the contents of the temporary file + std::fs::copy(temp_file.path(), &clone_img_path).unwrap(); + Ok(()) + }); + + //mock format_image + mock.expect_run_command() + .withf(|name, args| name == "mkfs.ext4" && args[1] == "ROOT-B") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock mount_image + mock.expect_run_command() + .withf(|name, _| name == "mount") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock extract_tar_to_image + mock.expect_run_command() + .withf(|name, args| name == "tar" && args[0] == "-xvf") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock install->dd + mock.expect_run_command() + .withf(|name, _| name == "dd") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + let img_manager = UpgradeImageManager::new( + PreparePath { + persist_path: "/tmp".into(), + update_path: tmp_dir.into(), + image_path: img_path.into(), + mount_path: "/tmp/update/mount".into(), + tar_path: "/tmp/update/image.tar".into(), + rootfs_file: "image.tar".into(), + }, + PartitionInfo { device: "/dev/sda3".into(), fs_type: "ext4".into(), menuentry: "B".into() }, + mock, + ); + + let img_manager = img_manager.create_os_image(0o755).unwrap(); + let result = img_manager.install(); + assert!(result.is_ok()); + + assert_eq!(Path::new(&tmp_dir).exists(), false); + } +} diff --git a/KubeOS-Rust/manager/src/utils/mod.rs b/KubeOS-Rust/manager/src/utils/mod.rs new file mode 100644 index 00000000..caf406e3 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/mod.rs @@ -0,0 +1,23 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod common; +mod container_image; +mod executor; +mod image_manager; +mod partition; + +pub use common::*; +pub use container_image::*; +pub use executor::*; +pub use image_manager::*; +pub use partition::*; diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs new file mode 100644 index 00000000..799b4b35 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/partition.rs @@ -0,0 +1,117 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::{bail, Result}; +use log::{debug, trace}; + +use super::executor::CommandExecutor; + +#[derive(PartialEq, Debug, Default)] +pub struct PartitionInfo { + pub device: String, + pub menuentry: String, + pub fs_type: String, +} + +/// get_partition_info returns the current partition info and the next partition info. +pub fn get_partition_info(executor: &T) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> { + let lsblk = executor.run_command_with_output("lsblk", &["-lno", "NAME,MOUNTPOINTS,FSTYPE"])?; + // After split whitespace, the root directory line should have 3 elements, which are "sda2 / ext4". + let mut cur_partition = PartitionInfo::default(); + let mut next_partition = PartitionInfo::default(); + let splitted_len = 3; + trace!("get_partition_info lsblk command output:\n{}", lsblk); + for line in lsblk.lines() { + let res: Vec<&str> = line.split_whitespace().collect(); + if res.len() == splitted_len && res[1] == "/" { + debug!("root directory line: device={}, fs_type={}", res[0], res[2]); + cur_partition.device = format!("/dev/{}", res[0]).to_string(); + cur_partition.fs_type = res[2].to_string(); + next_partition.fs_type = res[2].to_string(); + if res[0].contains('2') { + // root directory is mounted on sda2, so sda3 is the next partition + cur_partition.menuentry = String::from("A"); + next_partition.menuentry = String::from("B"); + next_partition.device = format!("/dev/{}", res[0].replace('2', "3")).to_string(); + } else if res[0].contains('3') { + // root directory is mounted on sda3, so sda2 is the next partition + cur_partition.menuentry = String::from("B"); + next_partition.menuentry = String::from("A"); + next_partition.device = format!("/dev/{}", res[0].replace('3', "2")).to_string(); + } + } + } + if cur_partition.menuentry.is_empty() { + bail!("Failed to get partition info, lsblk output: {}", lsblk); + } + Ok((cur_partition, next_partition)) +} + +#[cfg(test)] +mod tests { + use mockall::{mock, predicate::*}; + + use super::*; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_partition_info() { + init(); + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + let mut mock = MockCommandExec::new(); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + let res = get_partition_info(&mock).unwrap(); + let expect_res = ( + PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() }, + PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() }, + ); + assert_eq!(res, expect_res); + + let command_output2 = "sda\nsda1 /boot/efi vfat\nsda2 ext4\nsda3 / ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string())); + let res = get_partition_info(&mock).unwrap(); + let expect_res = ( + PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() }, + PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() }, + ); + assert_eq!(res, expect_res); + + let command_output3 = ""; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string())); + let res = get_partition_info(&mock); + assert!(res.is_err()); + + let command_output4 = "sda4 / ext4"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output4.to_string())); + let res = get_partition_info(&mock); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml new file mode 100644 index 00000000..d804ac77 --- /dev/null +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -0,0 +1,49 @@ +[package] +description = "KubeOS os-proxy" +edition = "2021" +license = "MulanPSL-2.0" +name = "proxy" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "drain" +path = "src/drain.rs" + +[[bin]] +name = "proxy" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.44" +async-trait = "0.1" +cli = { version = "1.0.6", path = "../cli" } +env_logger = "0.9.0" +futures = "0.3.17" +h2 = "=0.3.16" +k8s-openapi = { version = "0.13.1", features = ["v1_22"] } +kube = { version = "0.66.0", features = ["derive", "runtime"] } +log = "=0.4.15" +manager = { version = "1.0.6", path = "../manager" } +regex = "=1.7.3" +reqwest = { version = "=0.12.2", default-features = false, features = [ + "json", +] } +schemars = "=0.8.10" +serde = { version = "1.0.130", features = ["derive"] } +serde_json = "1.0.68" +socket2 = "=0.4.9" +thiserror = "1.0.29" +thread_local = "=1.1.4" +tokio = { version = "=1.28.0", default-features = false, features = [ + "macros", + "rt-multi-thread", +] } +tokio-retry = "0.3" + +[dev-dependencies] +assert-json-diff = "2.0.2" +http = "0.2.9" +hyper = "0.14.25" +tower-test = "0.4.0" +mockall = { version = "=0.11.3" } diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs new file mode 100644 index 00000000..b833f276 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs @@ -0,0 +1,153 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{collections::HashMap, path::Path}; + +use agent_error::Error; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use manager::api::{CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest}; + +pub struct UpgradeInfo { + pub version: String, + pub image_type: String, + pub check_sum: String, + pub container_image: String, + pub imageurl: String, + pub flagsafe: bool, + pub mtls: bool, + pub cacert: String, + pub clientcert: String, + pub clientkey: String, +} + +pub struct ConfigInfo { + pub configs: Vec, +} + +pub struct Sysconfig { + pub model: String, + pub config_path: String, + pub contents: HashMap, +} + +pub struct KeyInfo { + pub value: String, + pub operation: String, +} + +pub trait AgentMethod { + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error>; + fn upgrade_method(&self) -> Result<(), Error>; + fn rollback_method(&self) -> Result<(), Error>; + fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error>; +} +pub trait AgentCall { + fn call_agent(&self, client: &Client, method: T) -> Result<(), Error>; +} + +pub struct AgentClient { + pub agent_client: Client, + pub agent_call_client: T, +} + +impl AgentClient { + pub fn new>(socket_path: P, agent_call_client: T) -> Self { + AgentClient { agent_client: Client::new(socket_path), agent_call_client } + } +} + +#[derive(Default)] +pub struct AgentCallClient {} +impl AgentCall for AgentCallClient { + fn call_agent(&self, client: &Client, method: T) -> Result<(), Error> { + match method.call(client) { + Ok(_resp) => Ok(()), + Err(e) => Err(Error::AgentError { source: e }), + } + } +} + +impl AgentMethod for AgentClient { + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error> { + let upgrade_request = UpgradeRequest { + version: upgrade_info.version, + image_type: upgrade_info.image_type, + check_sum: upgrade_info.check_sum, + container_image: upgrade_info.container_image, + image_url: upgrade_info.imageurl, + flag_safe: upgrade_info.flagsafe, + mtls: upgrade_info.mtls, + certs: CertsInfo { + ca_cert: upgrade_info.cacert, + client_cert: upgrade_info.clientcert, + client_key: upgrade_info.clientkey, + }, + }; + match self.agent_call_client.call_agent(&self.agent_client, PrepareUpgradeMethod::new(upgrade_request)) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn upgrade_method(&self) -> Result<(), Error> { + match self.agent_call_client.call_agent(&self.agent_client, UpgradeMethod::default()) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn rollback_method(&self) -> Result<(), Error> { + match self.agent_call_client.call_agent(&self.agent_client, RollbackMethod::default()) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error> { + let mut agent_configs: Vec = Vec::new(); + for config in config_info.configs { + let mut contents_tmp: HashMap = HashMap::new(); + for (key, key_info) in config.contents.iter() { + contents_tmp.insert( + key.to_string(), + AgentKeyInfo { value: key_info.value.clone(), operation: key_info.operation.clone() }, + ); + } + agent_configs.push(AgentSysconfig { + model: config.model, + config_path: config.config_path, + contents: contents_tmp, + }) + } + let config_request = ConfigureRequest { configs: agent_configs }; + match self.agent_call_client.call_agent(&self.agent_client, ConfigureMethod::new(config_request)) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } +} + +pub mod agent_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("{source}")] + AgentError { source: anyhow::Error }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/apiclient.rs b/KubeOS-Rust/proxy/src/controller/apiclient.rs new file mode 100644 index 00000000..85e839f0 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/apiclient.rs @@ -0,0 +1,147 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::BTreeMap; + +use anyhow::Result; +use apiclient_error::Error; +use async_trait::async_trait; +use kube::{ + api::{Api, ObjectMeta, Patch, PatchParams, PostParams}, + Client, +}; +use serde::{Deserialize, Serialize}; + +use super::{ + crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}, + values::{LABEL_OSINSTANCE, NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}, +}; + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceSpecPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + spec: OSInstanceSpec, +} + +impl Default for OSInstanceSpecPatch { + fn default() -> Self { + OSInstanceSpecPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None ,namespacedname:None}, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceStatusPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + status: Option, +} + +impl Default for OSInstanceStatusPatch { + fn default() -> Self { + OSInstanceStatusPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }), + } + } +} + +#[derive(Clone)] +pub struct ControllerClient { + pub client: Client, +} + +impl ControllerClient { + pub fn new(client: Client) -> Self { + ControllerClient { client } + } +} + +#[async_trait] +pub trait ApplyApi: Clone + Sized + Send + Sync { + async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error>; + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error>; + async fn update_osinstance_status( + &self, + node_name: &str, + namespace: &str, + status: &Option, + ) -> Result<(), Error>; +} + +#[async_trait] +impl ApplyApi for ControllerClient { + async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error> { + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + let osinstance = OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None ,namespacedname: None}, + status: None, + }; + let osi_api = Api::namespaced(self.client.clone(), namespace); + osi_api.create(&PostParams::default(), &osinstance).await?; + Ok(()) + } + + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() }; + osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?; + Ok(()) + } + + async fn update_osinstance_status( + &self, + node_name: &str, + namespace: &str, + status: &Option, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_status_patch = OSInstanceStatusPatch { status: status.clone(), ..Default::default() }; + osi_api.patch_status(node_name, &PatchParams::default(), &Patch::Merge(&osi_status_patch)).await?; + Ok(()) + } +} +pub mod apiclient_error { + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs new file mode 100644 index 00000000..679096d3 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs @@ -0,0 +1,686 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::BTreeMap; + +use anyhow::Result; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use http::{Request, Response}; +use hyper::{body::to_bytes, Body}; +use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod}; +use kube::{ + api::ObjectMeta, + core::{ListMeta, ObjectList}, + Client as KubeClient, Resource, ResourceExt, +}; +use mockall::mock; + +use self::mock_error::Error; +use super::{ + agentclient::*, + crd::{Configs, NamespacedName, OSInstanceStatus}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK}, +}; +use crate::controller::{ + apiclient::{ApplyApi, ControllerClient}, + crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS}, + values::{LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, + ProxyController, +}; + +type ApiServerHandle = tower_test::mock::Handle, Response>; +pub struct ApiServerVerifier(ApiServerHandle); + +pub enum Testcases { + OSInstanceNotExist(OSInstance), + UpgradeNormal(OSInstance), + UpgradeUpgradeconfigsVersionMismatch(OSInstance), + UpgradeOSInstaceNodestatusConfig(OSInstance), + UpgradeOSInstaceNodestatusIdle(OSInstance), + ConfigNormal(OSInstance), + ConfigVersionMismatchReassign(OSInstance), + ConfigVersionMismatchUpdate(OSInstance), + Rollback(OSInstance), +} + +pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) { + tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("timeout on mock apiserver") + .expect("scenario succeeded") +} + +impl ApiServerVerifier { + pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::OSInstanceNotExist(osi) => { + self.handler_osinstance_get_not_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_creation(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi) + .await + }, + Testcases::UpgradeNormal(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_upgradeconfig_v2(osi.clone()) + .await + .unwrap() + .handler_node_cordon(osi.clone()) + .await + .unwrap() + .handler_node_pod_list_get(osi) + .await + }, + Testcases::UpgradeUpgradeconfigsVersionMismatch(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_node_update_delete_label(osi.clone()) + .await + .unwrap() + .handler_node_uncordon(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::UpgradeOSInstaceNodestatusConfig(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + }, + Testcases::UpgradeOSInstaceNodestatusIdle(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_node_update_delete_label(osi.clone()) + .await + .unwrap() + .handler_node_uncordon(osi) + .await + }, + Testcases::ConfigNormal(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_sysconfig_v2(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::ConfigVersionMismatchReassign(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::ConfigVersionMismatchUpdate(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_spec_sysconfig_v2(osi) + .await + }, + Testcases::Rollback(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_upgradeconfig_v2(osi.clone()) + .await + .unwrap() + .handler_node_cordon(osi.clone()) + .await + .unwrap() + .handler_node_pod_list_get(osi) + .await + }, + } + .expect("Case completed without errors"); + }) + } + + async fn handler_osinstance_get_not_exist(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name()) + ); + let response_json = serde_json::json!( + { "status": "Failure", "message": "osinstances.upgrade.openeuler.org \"openeuler\" not found", "reason": "NotFound", "code": 404 } + ); + dbg!("handler_osinstance_get_not_exist"); + let response = serde_json::to_vec(&response_json).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + Ok(self) + } + async fn handler_osinstance_get_exist(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name()) + ); + dbg!("handler_osinstance_get_exist"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + async fn handler_osinstance_creation(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::POST); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?") + ); + dbg!("handler_osinstance_creation"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_idle(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_IDLE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_idle"); + osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_upgradeconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?", + osinstance.name() + ) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let status_json = body_json.get("status").expect("status object").clone(); + let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!( + status.upgradeconfigs.expect("upgradeconfigs is not None").clone(), + osinstance.spec.clone().upgradeconfigs.expect("upgradeconfig is not None") + ); + + osinstance.status.as_mut().unwrap().upgradeconfigs = osinstance.spec.upgradeconfigs.clone(); + + dbg!("handler_osinstance_patch_upgradeconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?", + osinstance.name() + ) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance"); + let status_json = body_json.get("status").expect("status object").clone(); + let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!( + status.sysconfigs.expect("sysconfigs is not None").clone(), + osinstance.spec.clone().sysconfigs.expect("sysconfig is not None") + ); + + osinstance.status.as_mut().unwrap().sysconfigs = osinstance.spec.sysconfigs.clone(); + + dbg!("handler_osinstance_patch_sysconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_spec_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + + assert_eq!( + spec.sysconfigs.expect("upgradeconfigs is not None").clone().version.clone().unwrap(), + String::from("v2") + ); + + osinstance.spec.sysconfigs.as_mut().unwrap().version = Some(String::from("v2")); + + dbg!("handler_osinstance_patch_spec_sysconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_get(mut self, osinstance: OSInstance) -> Result { + // return node with name = openeuler, osimage = KubeOS v1，no upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name())); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + assert_eq!(node.name(), String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1")); + dbg!("handler_node_get"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_get_with_label(mut self, osinstance: OSInstance) -> Result { + // return node with name = openeuler, osimage = KubeOS v1，has upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name())); + let mut node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + let node_labels = node.labels_mut(); + node_labels.insert(LABEL_UPGRADING.to_string(), "".to_string()); + assert_eq!(node.name(), String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1")); + assert!(node.labels().contains_key(LABEL_UPGRADING)); + dbg!("handler_node_get_with_label"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_update_delete_label(mut self, osinstance: OSInstance) -> Result { + // return node with name = openeuler, osimage = KubeOS v1，no upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PUT); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + // check request body has upgrade label + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + dbg!("handler_node_update_delete_label"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_cordon(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + assert_eq!(request.extensions().get(), Some(&"cordon")); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + dbg!("handler_node_cordon"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_uncordon(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + assert_eq!(request.extensions().get(), Some(&"cordon")); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(false), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + dbg!("handler_node_uncordon"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_pod_list_get(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!("/api/v1/pods?&fieldSelector=spec.nodeName%3D{}", osinstance.name()) + ); + assert_eq!(request.extensions().get(), Some(&"list")); + let pods_list = ObjectList:: { metadata: ListMeta::default(), items: vec![] }; + dbg!("handler_node_pod_list_get"); + let response = serde_json::to_vec(&pods_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } +} + +pub mod mock_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} + +mock! { + pub AgentCallClient{} + impl AgentCall for AgentCallClient{ + fn call_agent(&self, client:&Client, method: T) -> Result<(), agent_error::Error> { + Ok(()) + } + } + +} +impl ProxyController { + pub fn test() -> (ProxyController, ApiServerVerifier) { + let (mock_service, handle) = tower_test::mock::pair::, Response>(); + let mock_k8s_client = KubeClient::new(mock_service, "default"); + let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); + let mut mock_agent_call_client = MockAgentCallClient::new(); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + let mock_agent_client = AgentClient::new("test", mock_agent_call_client); + let proxy_controller: ProxyController = + ProxyController::new(mock_k8s_client, mock_api_client, mock_agent_client); + (proxy_controller, ApiServerVerifier(handle)) + } +} + +impl OSInstance { + pub fn set_osi_default(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + namespacedname:Some(NamespacedName{namespace:String::from("default"),name:String::from("test")}) + }, + status: Some(OSInstanceStatus { + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }), + } + } + + pub fn set_osi_nodestatus_upgrade(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + osinstance + } + + pub fn set_osi_nodestatus_config(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = config, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance + } + + pub fn set_osi_upgradecon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.upgradeconfigs.as_mut().unwrap().version = Some(String::from("v2")); + osinstance + } + + pub fn set_osi_nodestatus_upgrade_upgradecon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + osinstance.spec.upgradeconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + osinstance + } + + pub fn set_osi_nodestatus_config_syscon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance.spec.sysconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + osinstance + } +} + +impl OS { + pub fn set_os_default() -> Self { + let mut os = OS::new("test", OSSpec::default()); + os.meta_mut().namespace = Some("default".into()); + os + } + + pub fn set_os_osversion_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.opstype = String::from("config"); + os + } + + pub fn set_os_osversion_v2_upgradecon_v2() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.upgradeconfigs = Some(Configs { version: Some(String::from("v2")), configs: None }); + os + } + + pub fn set_os_syscon_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = String::from("config"); + os.spec.sysconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + os + } + + pub fn set_os_rollback_osversion_v2_upgradecon_v2() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string(); + os.spec.upgradeconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + os + } +} + +impl Default for OSSpec { + fn default() -> Self { + OSSpec { + osversion: String::from("KubeOS v1"), + maxunavailable: 2, + checksum: String::from("test"), + imagetype: String::from("containerd"), + containerimage: String::from("test"), + opstype: String::from("upgrade"), + evictpodforce: true, + imageurl: String::from(""), + flagsafe: false, + mtls: false, + cacert: Some(String::from("")), + clientcert: Some(String::from("")), + clientkey: Some(String::from("")), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + nodeselector:None, + timeinterval:None, + timewindow:None, + executionmode:None, + } + } +} diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs new file mode 100644 index 00000000..787a0e1c --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -0,0 +1,589 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{collections::HashMap, env}; + +use anyhow::Result; +use drain::drain_os; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::{Api, PostParams}, + core::ErrorResponse, + runtime::controller::{Context, ReconcilerAction}, + Client, ResourceExt, +}; +use log::{debug, error, info}; +use reconciler_error::Error; + +use super::{ + agentclient::{AgentCall, AgentClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, + apiclient::ApplyApi, + crd::{Configs, Content, OSInstance, OS}, + utils::{check_version, get_config_version, ConfigOperation, ConfigType}, + values::{ + LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, + OSINSTANCE_NAMESPACE, REQUEUE_ERROR, REQUEUE_NORMAL,LABEL_CONFIGURING, NO_REQUEUE, + }, +}; + +pub async fn reconcile( + os: OS, + ctx: Context>, +) -> Result { + debug!("start reconcile"); + let proxy_controller = ctx.get_ref(); + let os_cr = &os; + let node_name = env::var("NODE_NAME")?; + let namespace: String = os_cr + .namespace() + .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; + proxy_controller.check_osi_exisit(&node_name).await?; + let controller_res = proxy_controller.get_resources(&node_name).await?; + let node = controller_res.node; + let mut osinstance = controller_res.osinstance; + if let Some(namespacedname) = osinstance.spec.namespacedname.as_ref(){ + debug!("osinstance correspending os name is {}, namespace is {}",namespacedname.name,namespacedname.namespace); + if !(namespacedname.name == os_cr.name() && namespacedname.namespace == namespace){ + debug!("current os cr name:{}, namespace:{} is not belong to this node",os_cr.name(),namespace); + return Ok(NO_REQUEUE) + } + }else { + debug!("osinstance correspending os name is None, not in upgrading or configuring"); + return Ok(REQUEUE_NORMAL) + } + + let node_os_image = &node + .status + .as_ref() + .ok_or(Error::MissingSubResource { value: String::from("node.status") })? + .node_info + .as_ref() + .ok_or(Error::MissingSubResource { value: String::from("node.status.node_info") })? + .os_image; + debug!("os expected osversion is {}, actual osversion is {}", os_cr.spec.osversion, node_os_image); + if check_version(&os_cr.spec.osversion, node_os_image) { + match ConfigType::SysConfig.check_config_version(&os, &osinstance) { + ConfigOperation::Reassign => { + debug!("start reassign"); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.sysconfigs.as_ref()), + ConfigType::SysConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + }, + ConfigOperation::UpdateConfig => { + debug!("start update config"); + osinstance.spec.sysconfigs = os_cr.spec.sysconfigs.clone(); + proxy_controller + .controller_client + .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec) + .await?; + return Ok(REQUEUE_ERROR); + }, + _ => {}, + } + if node.labels().contains_key(LABEL_UPGRADING) || node.labels().contains_key(LABEL_CONFIGURING) { + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + info!( + "node has upgrade/config label , but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin" + ); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + proxy_controller.set_config(&mut osinstance, ConfigType::SysConfig).await?; + proxy_controller + .refresh_node(node, osinstance, &get_config_version(os_cr.spec.sysconfigs.as_ref()), ConfigType::SysConfig) + .await?; + } + } else { + if os_cr.spec.opstype == NODE_STATUS_CONFIG { + return Err(Error::UpgradeBeforeConfig); + } + if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) { + debug!("start reassign"); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + if node.labels().contains_key(LABEL_UPGRADING) { + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + info!( + "node has upgrade label , but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin" + ); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + proxy_controller.set_config(&mut osinstance, ConfigType::UpgradeConfig).await?; + proxy_controller.upgrade_node(os_cr, &node).await?; + } + } + Ok(REQUEUE_NORMAL) +} + +pub fn error_policy( + error: &Error, + _ctx: Context>, +) -> ReconcilerAction { + error!("Reconciliation error:{}", error.to_string()); + REQUEUE_ERROR +} + +struct ControllerResources { + osinstance: OSInstance, + node: Node, +} +pub struct ProxyController { + k8s_client: Client, + controller_client: T, + agent_client: AgentClient, +} + +impl ProxyController { + pub fn new(k8s_client: Client, controller_client: T, agent_client: AgentClient) -> Self { + ProxyController { k8s_client, controller_client, agent_client } + } +} + +impl ProxyController { + async fn check_osi_exisit(&self, node_name: &str) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), OSINSTANCE_NAMESPACE); + match osi_api.get(node_name).await { + Ok(osi) => { + debug!("osinstance is exist {:?}", osi.name()); + Ok(()) + }, + Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { + info!("Create OSInstance {}", node_name); + self.controller_client.create_osinstance(node_name, OSINSTANCE_NAMESPACE).await?; + Ok(()) + }, + Err(err) => Err(Error::KubeClient { source: err }), + } + } + + async fn get_resources(&self, node_name: &str) -> Result { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), OSINSTANCE_NAMESPACE); + let osinstance_cr = osi_api.get(node_name).await?; + let node_api: Api = Api::all(self.k8s_client.clone()); + let node_cr = node_api.get(node_name).await?; + Ok(ControllerResources { osinstance: osinstance_cr, node: node_cr }) + } + + async fn refresh_node( + &self, + mut node: Node, + osinstance: OSInstance, + os_config_version: &str, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start refresh_node"); + let node_api: Api = Api::all(self.k8s_client.clone()); + let labels = node.labels_mut(); + if labels.contains_key(LABEL_UPGRADING) { + debug!("delete label {}", LABEL_UPGRADING); + labels.remove(LABEL_UPGRADING); + node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; + }else if labels.contains_key(LABEL_CONFIGURING){ + debug!("delete label {}", LABEL_CONFIGURING); + labels.remove(LABEL_CONFIGURING); + node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; + } + if let Some(node_spec) = &node.spec { + if let Some(node_unschedulable) = node_spec.unschedulable { + if node_unschedulable { + node_api.uncordon(&node.name()).await?; + info!("Uncordon successfully node{}", node.name()); + } + } + } + self.update_node_status(osinstance, os_config_version, config_type).await?; + Ok(()) + } + + async fn update_node_status( + &self, + mut osinstance: OSInstance, + os_config_version: &str, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start update_node_status"); + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + return Ok(()); + } + let upgradeconfig_spec_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + let sysconfig_spec_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + let sysconfig_status_version: String; + if let Some(osinstance_status) = osinstance.status.as_ref() { + sysconfig_status_version = get_config_version(osinstance_status.sysconfigs.as_ref()); + } else { + sysconfig_status_version = get_config_version(None); + } + if sysconfig_spec_version == sysconfig_status_version + || (config_type == ConfigType::SysConfig && os_config_version != sysconfig_spec_version) + || (config_type == ConfigType::UpgradeConfig && os_config_version != upgradeconfig_spec_version) + { + let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); + osinstance.spec.namespacedname = None; + self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; + } + Ok(()) + } + + async fn update_osi_status(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> { + debug!("start update_osi_status"); + config_type.set_osi_status_config(osinstance); + debug!("osinstance status is update to {:?}", osinstance.status); + let namespace = &osinstance + .namespace() + .ok_or(Error::MissingObjectKey { resource: "osinstance".to_string(), value: "namespace".to_string() })?; + self.controller_client.update_osinstance_status(&osinstance.name(), namespace, &osinstance.status).await?; + Ok(()) + } + + async fn set_config(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> { + debug!("start set_config"); + let config_info = config_type.check_config_start(osinstance); + if config_info.need_config { + match config_info.configs.and_then(convert_to_agent_config) { + Some(agent_configs) => { + match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }) { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + }, + None => { + info!("config is none, No content can be configured."); + }, + }; + self.update_osi_status(osinstance, config_type).await?; + } + Ok(()) + } + + async fn upgrade_node(&self, os_cr: &OS, node: &Node) -> Result<(), Error> { + debug!("start upgrade node"); + match os_cr.spec.opstype.as_str() { + OPERATION_TYPE_UPGRADE => { + let upgrade_info = UpgradeInfo { + version: os_cr.spec.osversion.clone(), + image_type: os_cr.spec.imagetype.clone(), + check_sum: os_cr.spec.checksum.clone(), + container_image: os_cr.spec.containerimage.clone(), + flagsafe: os_cr.spec.flagsafe, + imageurl: os_cr.spec.imageurl.clone(), + mtls: os_cr.spec.mtls, + cacert: os_cr.spec.cacert.clone().unwrap_or_default(), + clientcert: os_cr.spec.clientcert.clone().unwrap_or_default(), + clientkey: os_cr.spec.clientkey.clone().unwrap_or_default(), + }; + + match self.agent_client.prepare_upgrade_method(upgrade_info) { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; + match self.agent_client.upgrade_method() { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + }, + OPERATION_TYPE_ROLLBACK => { + self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; + + match self.agent_client.rollback_method() { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + }, + _ => { + return Err(Error::Operation { value: os_cr.spec.opstype.clone() }); + }, + } + Ok(()) + } + + async fn evict_node(&self, node_name: &str, evict_pod_force: bool) -> Result<(), Error> { + debug!("start evict_node"); + let node_api = Api::all(self.k8s_client.clone()); + node_api.cordon(node_name).await?; + info!("Cordon node Successfully{}, start drain nodes", node_name); + match self.drain_node(node_name, evict_pod_force).await { + Ok(()) => {}, + Err(e) => { + node_api.uncordon(node_name).await?; + info!("Drain node {} error, uncordon node successfully", node_name); + return Err(e); + }, + } + Ok(()) + } + + async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> { + use drain::error::DrainError::*; + match drain_os(&self.k8s_client.clone(), node_name, force).await { + Err(DeletePodsError { errors, .. }) => Err(Error::DrainNode { value: errors.join("; ") }), + _ => Ok(()), + } + } +} + +fn convert_to_agent_config(configs: Configs) -> Option> { + let mut agent_configs: Vec = Vec::new(); + if let Some(config_list) = configs.configs { + for config in config_list.into_iter() { + match config.contents.and_then(convert_to_config_hashmap) { + Some(contents_tmp) => { + let config_tmp = Sysconfig { + model: config.model.unwrap_or_default(), + config_path: config.configpath.unwrap_or_default(), + contents: contents_tmp, + }; + agent_configs.push(config_tmp) + }, + None => { + info!( + "model {} which has configpath {} do not has any contents no need to configure", + config.model.unwrap_or_default(), + config.configpath.unwrap_or_default() + ); + continue; + }, + }; + } + if agent_configs.is_empty() { + info!("no contents in all models, no need to configure"); + return None; + } + return Some(agent_configs); + } + None +} + +fn convert_to_config_hashmap(contents: Vec) -> Option> { + let mut contents_tmp: HashMap = HashMap::new(); + for content in contents.into_iter() { + let key_info = + KeyInfo { value: content.value.unwrap_or_default(), operation: content.operation.unwrap_or_default() }; + contents_tmp.insert(content.key.unwrap_or_default(), key_info); + } + Some(contents_tmp) +} + +pub mod reconciler_error { + use thiserror::Error; + + use crate::controller::{agentclient::agent_error, apiclient::apiclient_error}; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeClient { + #[from] + source: kube::Error, + }, + + #[error("Create/Patch OSInstance reported error: {source}")] + ApplyApi { + #[from] + source: apiclient_error::Error, + }, + + #[error("Cannot get environment NODE_NAME, error: {source}")] + Env { + #[from] + source: std::env::VarError, + }, + + #[error("{}.metadata.{} is not exist", resource, value)] + MissingObjectKey { resource: String, value: String }, + + #[error("Cannot get {}, {} is None", value, value)] + MissingSubResource { value: String }, + + #[error("operation {} cannot be recognized", value)] + Operation { value: String }, + + #[error("Expect OS Version is not same with Node OS Version, please upgrade first")] + UpgradeBeforeConfig, + + #[error("os-agent reported error:{source}")] + Agent { source: agent_error::Error }, + + #[error("Error when drain node, error reported: {}", value)] + DrainNode { value: String }, + } +} + +#[cfg(test)] +mod test { + use std::env; + + use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS}; + use crate::controller::{ + apiserver_mock::{timeout_after_5s, MockAgentCallClient, Testcases}, + ControllerClient, + }; + + #[tokio::test] + async fn test_create_osinstance_with_no_upgrade_or_configuration() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_default(); + let context = Context::new(test_proxy_controller); + let mocksrv = + fakeserver.run(Testcases::OSInstanceNotExist(OSInstance::set_osi_default("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + #[tokio::test] + async fn test_upgrade_normal() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeNormal(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_diff_osversion_opstype_config() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeOSInstaceNodestatusConfig( + OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"), + )); + let res = reconcile(os, context.clone()).await; + timeout_after_5s(mocksrv).await; + assert!(res.is_err(), "upgrade fails due to opstype=config"); + let err = res.unwrap_err(); + assert!(err.to_string().contains("Expect OS Version is not same with Node OS Version, please upgrade first")); + error_policy(&err, context); + } + + #[tokio::test] + async fn test_upgradeconfigs_version_mismatch() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeUpgradeconfigsVersionMismatch( + OSInstance::set_osi_nodestatus_upgrade("openeuler", "default"), + )); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_upgrade_nodestatus_idle() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::UpgradeOSInstaceNodestatusIdle(OSInstance::set_osi_upgradecon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_config_normal() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::ConfigNormal(OSInstance::set_osi_nodestatus_config_syscon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_sysconfig_version_mismatch_reassign() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchReassign(OSInstance::set_osi_nodestatus_config( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_sysconfig_version_mismatch_update() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchUpdate(OSInstance::set_osi_nodestatus_upgrade( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_rollback() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_rollback_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::Rollback(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } +} diff --git a/KubeOS-Rust/proxy/src/controller/crd.rs b/KubeOS-Rust/proxy/src/controller/crd.rs new file mode 100644 index 00000000..36e14d59 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/crd.rs @@ -0,0 +1,94 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)] +pub struct OSSpec { + pub osversion: String, + pub maxunavailable: i64, + pub checksum: String, + pub imagetype: String, + pub containerimage: String, + pub opstype: String, + pub evictpodforce: bool, + pub imageurl: String, + #[serde(rename = "flagSafe")] + pub flagsafe: bool, + pub mtls: bool, + pub cacert: Option, + pub clientcert: Option, + pub clientkey: Option, + pub sysconfigs: Option, + pub upgradeconfigs: Option, + pub nodeselector:Option, + pub timewindow: Option, + pub timeinterval: Option, + pub executionmode:Option, + +} + +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube( + group = "upgrade.openeuler.org", + version = "v1alpha1", + kind = "OSInstance", + plural = "osinstances", + singular = "osinstance", + status = "OSInstanceStatus", + namespaced +)] +pub struct OSInstanceSpec { + pub nodestatus: String, + pub namespacedname: Option, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct OSInstanceStatus { + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Configs { + pub version: Option, + pub configs: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Config { + pub model: Option, + pub configpath: Option, + pub contents: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Content { + pub key: Option, + pub value: Option, + pub operation: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct NamespacedName{ + pub namespace: String, + pub name: String, +} +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct TimeWindow{ + pub starttime:String, + pub endtime:String, +} \ No newline at end of file diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs new file mode 100644 index 00000000..b8a4e6e5 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -0,0 +1,26 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod agentclient; +mod apiclient; +#[cfg(test)] +mod apiserver_mock; +mod controller; +mod crd; +mod utils; +mod values; + +pub use agentclient::{AgentCallClient, AgentClient}; +pub use apiclient::ControllerClient; +pub use controller::{error_policy, reconcile, ProxyController}; +pub use crd::OS; +pub use values::SOCK_PATH; diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs new file mode 100644 index 00000000..7e7b41d9 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/utils.rs @@ -0,0 +1,154 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use log::{debug, info}; + +use super::{ + crd::{Configs, OSInstance, OSInstanceStatus, OS}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_IDLE, NODE_STATUS_UPGRADE}, +}; + +#[derive(PartialEq, Clone, Copy)] +pub enum ConfigType { + UpgradeConfig, + SysConfig, +} + +pub enum ConfigOperation { + DoNothing, + Reassign, + UpdateConfig, +} + +pub struct ConfigInfo { + pub need_config: bool, + pub configs: Option, +} + +impl ConfigType { + pub fn check_config_version(&self, os: &OS, osinstance: &OSInstance) -> ConfigOperation { + debug!("start check_config_version"); + let node_status = &osinstance.spec.nodestatus; + if node_status == NODE_STATUS_IDLE { + debug!("node status is idle"); + return ConfigOperation::DoNothing; + }; + match self { + ConfigType::UpgradeConfig => { + let os_config_version = get_config_version(os.spec.upgradeconfigs.as_ref()); + let osi_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + debug!( + "os upgradeconfig version is {}, osinstance spec upragdeconfig version is {}", + os_config_version, osi_config_version + ); + if !check_version(&os_config_version, &osi_config_version) { + info!( + "os.spec.upgradeconfig.version is not equal to oninstance.spec.upragdeconfig.version, operation: reassgin upgrade to get newest upgradeconfigs" + ); + return ConfigOperation::Reassign; + } + }, + ConfigType::SysConfig => { + let os_config_version = get_config_version(os.spec.sysconfigs.as_ref()); + let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + debug!( + "os sysconfig version is {},osinstance spec sysconfig version is {}", + os_config_version, osi_config_version + ); + if !check_version(&os_config_version, &osi_config_version) { + if node_status == NODE_STATUS_CONFIG { + info!( + "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: reassgin config to get newest sysconfigs" + ); + return ConfigOperation::Reassign; + } + if node_status == NODE_STATUS_UPGRADE { + info!( + "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: update osinstance.spec.sysconfig and reconcile" + ); + return ConfigOperation::UpdateConfig; + } + } + }, + }; + ConfigOperation::DoNothing + } + pub fn check_config_start(&self, osinstance: &OSInstance) -> ConfigInfo { + debug!("start check_config_start"); + let spec_config_version: String; + let status_config_version: String; + let configs: Option; + match self { + ConfigType::UpgradeConfig => { + spec_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + if let Some(osinstance_status) = osinstance.status.as_ref() { + status_config_version = get_config_version(osinstance_status.upgradeconfigs.as_ref()); + } else { + status_config_version = get_config_version(None); + } + configs = osinstance.spec.upgradeconfigs.clone(); + }, + ConfigType::SysConfig => { + spec_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + if let Some(osinstance_status) = osinstance.status.as_ref() { + status_config_version = get_config_version(osinstance_status.sysconfigs.as_ref()); + } else { + status_config_version = get_config_version(None); + } + configs = osinstance.spec.sysconfigs.clone(); + }, + } + debug!( + "osinstance spec config version is {}, status config version is {}", + spec_config_version, status_config_version + ); + if spec_config_version != status_config_version && osinstance.spec.nodestatus != NODE_STATUS_IDLE { + return ConfigInfo { need_config: true, configs }; + } + ConfigInfo { need_config: false, configs: None } + } + pub fn set_osi_status_config(&self, osinstance: &mut OSInstance) { + match self { + ConfigType::UpgradeConfig => { + if let Some(osi_status) = &mut osinstance.status { + osi_status.upgradeconfigs = osinstance.spec.upgradeconfigs.clone(); + } else { + osinstance.status = Some(OSInstanceStatus { + upgradeconfigs: osinstance.spec.upgradeconfigs.clone(), + sysconfigs: None, + }) + } + }, + ConfigType::SysConfig => { + if let Some(osi_status) = &mut osinstance.status { + osi_status.sysconfigs = osinstance.spec.sysconfigs.clone(); + } else { + osinstance.status = + Some(OSInstanceStatus { upgradeconfigs: None, sysconfigs: osinstance.spec.sysconfigs.clone() }) + } + }, + } + } +} + +pub fn check_version(version_a: &str, version_b: &str) -> bool { + version_a.eq(version_b) +} + +pub fn get_config_version(configs: Option<&Configs>) -> String { + if let Some(configs) = configs { + if let Some(version) = configs.version.as_ref() { + return version.to_string(); + } + }; + String::from("") +} diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs new file mode 100644 index 00000000..2edb3dcb --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/values.rs @@ -0,0 +1,35 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::runtime::controller::ReconcilerAction; +use tokio::time::Duration; + +pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; +pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; +pub const LABEL_CONFIGURING: &str = "upgrade.openeuler.org/configuring"; + +pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; +pub const OSINSTANCE_KIND: &str = "OSInstance"; +pub const OSINSTANCE_NAMESPACE: &str = "default"; + +pub const NODE_STATUS_IDLE: &str = "idle"; +pub const NODE_STATUS_UPGRADE: &str = "upgrade"; +pub const NODE_STATUS_CONFIG: &str = "config"; + +pub const OPERATION_TYPE_UPGRADE: &str = "upgrade"; +pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; + +pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; + +pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; +pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; +pub const NO_REQUEUE: ReconcilerAction = ReconcilerAction { requeue_after: None }; diff --git a/KubeOS-Rust/proxy/src/drain.rs b/KubeOS-Rust/proxy/src/drain.rs new file mode 100644 index 00000000..64417df3 --- /dev/null +++ b/KubeOS-Rust/proxy/src/drain.rs @@ -0,0 +1,511 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use futures::{stream, StreamExt}; +use k8s_openapi::api::core::v1::{Pod, PodSpec, PodStatus}; +use kube::{ + api::{EvictParams, ListParams}, + core::ObjectList, + Api, Client, ResourceExt, +}; +use log::{debug, error, info}; +use reqwest::StatusCode; +use tokio::time::{sleep, Duration, Instant}; +use tokio_retry::{ + strategy::{jitter, ExponentialBackoff}, + RetryIf, +}; + +use self::error::{ + DrainError::{DeletePodsError, GetPodListsError, WaitDeletionError}, + EvictionError::{EvictionErrorNoRetry, EvictionErrorRetry}, +}; + +pub const MAX_EVICT_POD_NUM: usize = 5; +pub const EVERY_EVICTION_RETRY: Duration = Duration::from_secs(5); +pub const EVERY_DELETION_CHECK: Duration = Duration::from_secs(5); +pub const TIMEOUT: Duration = Duration::from_secs(u64::MAX); +pub const RETRY_BASE_DELAY: Duration = Duration::from_millis(100); +pub const RETRY_MAX_DELAY: Duration = Duration::from_secs(20); +pub const MAX_RETRIES_TIMES: usize = 10; + +pub async fn drain_os(client: &Client, node_name: &str, force: bool) -> Result<(), error::DrainError> { + let pods_list = get_pods_deleted(client, node_name, force).await?; + + stream::iter(pods_list) + .for_each_concurrent(MAX_EVICT_POD_NUM, move |pod| { + let k8s_client = client.clone(); + async move { + if evict_pod(&k8s_client, &pod, force).await.is_ok() { + wait_for_deletion(&k8s_client, &pod).await.ok(); + } + } + }) + .await; + + Ok(()) +} + +async fn get_pods_deleted( + client: &Client, + node_name: &str, + force: bool, +) -> Result, error::DrainError> { + let lp = ListParams { field_selector: Some(format!("spec.nodeName={}", node_name)), ..Default::default() }; + let pods_api: Api = Api::all(client.clone()); + let pods: ObjectList = match pods_api.list(&lp).await { + Ok(pods @ ObjectList { .. }) => pods, + Err(err) => { + return Err(GetPodListsError { source: err, node_name: node_name.to_string() }); + }, + }; + let mut filterd_pods_list: Vec = Vec::new(); + let mut filterd_err: Vec = Vec::new(); + let pod_filter = CombinedFilter::new(force); + for pod in pods.into_iter() { + let filter_result = pod_filter.filter(&pod); + if filter_result.status == PodDeleteStatus::Error { + filterd_err.push(filter_result.desc); + continue; + } + if filter_result.result { + filterd_pods_list.push(pod); + } + } + if !filterd_err.is_empty() { + return Err(DeletePodsError { errors: filterd_err }); + } + Ok(filterd_pods_list.into_iter()) +} + +async fn evict_pod(k8s_client: &kube::Client, pod: &Pod, force: bool) -> Result<(), error::EvictionError> { + let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); + + let error_handling_strategy = + if force { ErrorHandleStrategy::RetryStrategy } else { ErrorHandleStrategy::TolerateStrategy }; + + RetryIf::spawn( + error_handling_strategy.retry_strategy(), + || async { + loop { + let eviction_result = pod_api.evict(&pod.name_any(), &EvictParams::default()).await; + + match eviction_result { + Ok(_) => { + pod.name(); + debug!("Successfully evicted Pod '{}'", pod.name_any()); + break; + } + Err(kube::Error::Api(e)) => { + let status_code = StatusCode::from_u16(e.code); + match status_code { + Ok(StatusCode::FORBIDDEN) => { + return Err(EvictionErrorNoRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(StatusCode::NOT_FOUND) => { + return Err(EvictionErrorNoRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(StatusCode::INTERNAL_SERVER_ERROR) => { + error!( + "Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to misconfigured PodDisruptionBudgets.", + pod.name_any(), + e, + EVERY_EVICTION_RETRY.as_secs_f64() + ); + sleep(EVERY_EVICTION_RETRY).await; + continue; + } + Ok(StatusCode::TOO_MANY_REQUESTS) => { + error!("Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to PodDisruptionBugets.", + pod.name_any(), + e, + EVERY_EVICTION_RETRY.as_secs_f64() + ); + sleep(EVERY_EVICTION_RETRY).await; + continue; + } + Ok(_) => { + error!( + "Evict pod {} reported error: '{}'.", + pod.name_any(), + e + ); + return Err(EvictionErrorRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Err(_) => { + error!( + "Evict pod {} reported error: '{}'.Received invalid response code from Kubernetes API", + pod.name_any(), + e + ); + return Err(EvictionErrorRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + } + } + Err(e) => { + error!("Evict pod {} reported error: '{}' and will retry", pod.name_any(),e); + return Err(EvictionErrorRetry { + source: e, + pod_name: pod.name_any(), + }); + } + } + } + Ok(()) + }, + error_handling_strategy + ).await +} + +async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), error::DrainError> { + let start_time = Instant::now(); + + let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); + let response_error_not_found: u16 = 404; + loop { + match pod_api.get(&pod.name_any()).await { + Ok(p) if p.uid() != pod.uid() => { + let name = (&p).name_any(); + info!("Pod {} deleted.", name); + break; + }, + Ok(_) => { + info!("Pod '{}' is not yet deleted. Waiting {}s.", pod.name_any(), EVERY_DELETION_CHECK.as_secs_f64()); + }, + Err(kube::Error::Api(e)) if e.code == response_error_not_found => { + info!("Pod {} is deleted.", pod.name_any()); + break; + }, + Err(e) => { + error!( + "Get pod {} reported error: '{}', whether pod is deleted cannot be determined, waiting {}s.", + pod.name_any(), + e, + EVERY_DELETION_CHECK.as_secs_f64() + ); + }, + } + if start_time.elapsed() > TIMEOUT { + return Err(WaitDeletionError { pod_name: pod.name_any(), max_wait: TIMEOUT }); + } else { + sleep(EVERY_DELETION_CHECK).await; + } + } + Ok(()) +} + +fn get_pod_api_with_namespace(client: &kube::Client, pod: &Pod) -> Api { + match pod.metadata.namespace.as_ref() { + Some(namespace) => Api::namespaced(client.clone(), namespace), + None => Api::default_namespaced(client.clone()), + } +} + +trait NameAny { + fn name_any(&self) -> String; +} + +impl NameAny for &Pod { + fn name_any(&self) -> String { + self.metadata.name.clone().or_else(|| self.metadata.generate_name.clone()).unwrap_or_default() + } +} +trait PodFilter { + fn filter(&self, pod: &Pod) -> Box; +} + +struct FinishedOrFailedFilter {} +impl PodFilter for FinishedOrFailedFilter { + fn filter(&self, pod: &Pod) -> Box { + return match pod.status.as_ref() { + Some(PodStatus { phase: Some(phase), .. }) if phase == "Failed" || phase == "Succeeded" => { + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + }, + _ => FilterResult::create_filter_result(false, "", PodDeleteStatus::Okay), + }; + } +} +struct DaemonFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for DaemonFilter { + fn filter(&self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + return match pod.metadata.owner_references.as_ref() { + Some(owner_references) + if owner_references + .iter() + .any(|reference| reference.controller.unwrap_or(false) && reference.kind == "DaemonSet") => + { + if self.force { + let description = format!("Ignore Pod '{}': Pod is member of a DaemonSet", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Warning }) + } else { + let description = format!("Cannot drain Pod '{}': Pod is member of a DaemonSet", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) + } + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} +impl DaemonFilter { + fn new(force: bool) -> DaemonFilter { + DaemonFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } + } +} + +struct MirrorFilter {} +impl PodFilter for MirrorFilter { + fn filter(&self, pod: &Pod) -> Box { + return match pod.metadata.annotations.as_ref() { + Some(annotations) if annotations.contains_key("kubernetes.io/config.mirror") => { + let description = format!("Ignore Pod '{}': Pod is a static Mirror Pod", pod.name_any()); + FilterResult::create_filter_result(false, &description.to_string(), PodDeleteStatus::Warning) + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} + +struct LocalStorageFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for LocalStorageFilter { + fn filter(&self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + return match pod.spec.as_ref() { + Some(PodSpec { volumes: Some(volumes), .. }) if volumes.iter().any(|volume| volume.empty_dir.is_some()) => { + if self.force { + let description = format!("Force draining Pod '{}': Pod has local storage", pod.name_any()); + Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning }) + } else { + let description = format!("Cannot drain Pod '{}': Pod has local Storage", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) + } + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} +impl LocalStorageFilter { + fn new(force: bool) -> LocalStorageFilter { + LocalStorageFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } + } +} +struct UnreplicatedFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for UnreplicatedFilter { + fn filter(&self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + let is_replicated = pod.metadata.owner_references.is_some(); + + if is_replicated { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + if !is_replicated && self.force { + let description = format!("Force drain Pod '{}': Pod is unreplicated", pod.name_any()); + Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning }) + } else { + let description = format!("Cannot drain Pod '{}': Pod is unreplicated", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) + } + } +} +impl UnreplicatedFilter { + fn new(force: bool) -> UnreplicatedFilter { + UnreplicatedFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } + } +} + +struct DeletedFilter { + delete_wait_timeout: Duration, +} +impl PodFilter for DeletedFilter { + fn filter(&self, pod: &Pod) -> Box { + let now = Instant::now().elapsed(); + return match pod.metadata.deletion_timestamp.as_ref() { + Some(time) + if time.0.timestamp() != 0 + && now - Duration::from_secs(time.0.timestamp() as u64) >= self.delete_wait_timeout => + { + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} + +struct CombinedFilter { + deleted_filter: DeletedFilter, + daemon_filter: DaemonFilter, + mirror_filter: MirrorFilter, + local_storage_filter: LocalStorageFilter, + unreplicated_filter: UnreplicatedFilter, +} +impl PodFilter for CombinedFilter { + fn filter(&self, pod: &Pod) -> Box { + let mut filter_res = self.deleted_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.daemon_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.mirror_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.local_storage_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.unreplicated_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + } +} +impl CombinedFilter { + fn new(force: bool) -> CombinedFilter { + CombinedFilter { + deleted_filter: DeletedFilter { delete_wait_timeout: TIMEOUT }, + daemon_filter: DaemonFilter::new(force), + mirror_filter: MirrorFilter {}, + local_storage_filter: LocalStorageFilter::new(force), + unreplicated_filter: UnreplicatedFilter::new(force), + } + } +} + +#[derive(PartialEq, Clone, Copy)] +enum PodDeleteStatus { + Okay, + Warning, + Error, +} +struct FilterResult { + result: bool, + desc: String, + status: PodDeleteStatus, +} +impl FilterResult { + fn create_filter_result(result: bool, desc: &str, status: PodDeleteStatus) -> Box { + Box::new(FilterResult { result, desc: desc.to_string(), status }) + } +} + +enum ErrorHandleStrategy { + RetryStrategy, + TolerateStrategy, +} + +impl ErrorHandleStrategy { + fn retry_strategy(&self) -> impl Iterator { + let backoff = + ExponentialBackoff::from_millis(RETRY_BASE_DELAY.as_millis() as u64).max_delay(RETRY_MAX_DELAY).map(jitter); + + match self { + Self::TolerateStrategy => backoff.take(0), + + Self::RetryStrategy => backoff.take(MAX_RETRIES_TIMES), + } + } +} + +impl tokio_retry::Condition for ErrorHandleStrategy { + fn should_retry(&mut self, error: &error::EvictionError) -> bool { + match self { + Self::TolerateStrategy => false, + Self::RetryStrategy => matches!(error, error::EvictionError::EvictionErrorRetry { .. }), + } + } +} + +pub mod error { + use thiserror::Error; + use tokio::time::Duration; + + #[derive(Debug, Error)] + pub enum DrainError { + #[error("Get node {} pods list error reported: {}", node_name, source)] + GetPodListsError { source: kube::Error, node_name: String }, + + #[error("Pod '{}' was not deleted in the time allocated ({:.2}s).",pod_name,max_wait.as_secs_f64())] + WaitDeletionError { pod_name: String, max_wait: Duration }, + #[error("")] + DeletePodsError { errors: Vec }, + } + + #[derive(Debug, Error)] + pub enum EvictionError { + #[error("Evict Pod {} error: '{}'", pod_name, source)] + EvictionErrorRetry { source: kube::Error, pod_name: String }, + + #[error("Evict Pod {} error: '{}'", pod_name, source)] + EvictionErrorNoRetry { source: kube::Error, pod_name: String }, + } +} diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs new file mode 100644 index 00000000..c15aebed --- /dev/null +++ b/KubeOS-Rust/proxy/src/main.rs @@ -0,0 +1,49 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::Result; +use env_logger::{Builder, Env, Target}; +use futures::StreamExt; +use kube::{ + api::{Api, ListParams}, + client::Client, + runtime::controller::{Context, Controller}, +}; +use log::{error, info}; +mod controller; +use controller::{ + error_policy, reconcile, AgentCallClient, AgentClient, ControllerClient, ProxyController, OS, SOCK_PATH, +}; + +const PROXY_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); +#[tokio::main] +async fn main() -> Result<()> { + Builder::from_env(Env::default().default_filter_or("proxy=info")).target(Target::Stdout).init(); + let client = Client::try_default().await?; + let os: Api = Api::all(client.clone()); + let controller_client = ControllerClient::new(client.clone()); + let agent_call_client = AgentCallClient::default(); + let agent_client = AgentClient::new(SOCK_PATH, agent_call_client); + let proxy_controller = ProxyController::new(client, controller_client, agent_client); + info!("os-proxy version is {}, start renconcile", PROXY_VERSION.unwrap_or("Not Found")); + Controller::new(os, ListParams::default()) + .run(reconcile, error_policy, Context::new(proxy_controller)) + .for_each(|res| async move { + match res { + Ok(_o) => {}, + Err(e) => error!("reconcile failed: {}", e.to_string()), + } + }) + .await; + info!("os-proxy terminated"); + Ok(()) +} diff --git a/KubeOS-Rust/proxy/tests/common/mod.rs b/KubeOS-Rust/proxy/tests/common/mod.rs new file mode 100644 index 00000000..82577597 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/common/mod.rs @@ -0,0 +1,63 @@ +use std::process::{Command, Stdio}; + +use anyhow::Result; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::ResourceExt, + client::Client, + config::{Config, KubeConfigOptions, Kubeconfig}, + Api, +}; +use manager::utils::{CommandExecutor, RealCommandExecutor}; + +pub const CLUSTER: &str = "kubeos-test"; + +pub fn run_command(cmd: &str, args: &[&str]) -> Result<()> { + let output = Command::new(cmd).args(args).stdout(Stdio::inherit()).stderr(Stdio::inherit()).output()?; + if !output.status.success() { + println!("failed to run command: {} {}\n", cmd, args.join(" ")); + } + Ok(()) +} + +pub async fn setup() -> Result { + // set PATH variable + let path = std::env::var("PATH").unwrap(); + let new_path = format!("{}:{}", path, "../../bin"); + std::env::set_var("PATH", new_path); + + // create cluster + let executor = RealCommandExecutor {}; + println!("Creating cluster"); + run_command("bash", &["./tests/setup/setup_test_env.sh"]).expect("failed to create cluster"); + + // connect to the cluster + let kind_config = executor.run_command_with_output("kind", &["get", "kubeconfig", "-n", CLUSTER]).unwrap(); + let kubeconfig = Kubeconfig::from_yaml(kind_config.as_str()).expect("failed to parse kubeconfig"); + let options = KubeConfigOptions::default(); + let config = Config::from_custom_kubeconfig(kubeconfig, &&options).await.expect("failed to create config"); + let client = Client::try_from(config).expect("failed to create client"); + // list all nodes + let nodes: Api = Api::all(client.clone()); + let node_list = nodes.list(&Default::default()).await.expect("failed to list nodes"); + for n in node_list { + println!("Found Node: {}", n.name()); + } + // check node status + let node = nodes.get("kubeos-test-worker").await.unwrap(); + let status = node.status.unwrap(); + let conditions = status.conditions.unwrap(); + for c in conditions { + if c.type_ == "Ready" { + assert_eq!(c.status, "True"); + } + } + println!("Cluster ready"); + Ok(client) +} + +pub fn clean_env() { + let executor = RealCommandExecutor {}; + println!("Cleaning cluster"); + executor.run_command("kind", &["delete", "clusters", CLUSTER]).expect("failed to clean cluster"); +} diff --git a/KubeOS-Rust/proxy/tests/drain_test.rs b/KubeOS-Rust/proxy/tests/drain_test.rs new file mode 100644 index 00000000..2f4f1501 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/drain_test.rs @@ -0,0 +1,41 @@ +mod common; + +use common::*; +use drain::drain_os; +use k8s_openapi::api::core::v1::{Node, Pod}; +use kube::Api; + +#[tokio::test] +#[ignore = "integration test"] +async fn test_drain() { + let client = setup().await.unwrap(); + // drain node + let nodes: Api = Api::all(client.clone()); + let node_name = "kubeos-test-worker"; + println!("cordon node"); + nodes.cordon(node_name).await.unwrap(); + println!("drain node"); + drain_os(&client, node_name, true).await.unwrap(); + + // assert unschedulable + println!("check node unschedulable"); + let node = nodes.get(node_name).await.unwrap(); + if let Some(spec) = node.spec { + assert_eq!(spec.unschedulable, Some(true)); + } else { + panic!("node spec is none"); + } + // list all pods on kubeos-test-worker node and all pods should belong to daemonset + println!("list all pods on kubeos-test-worker node"); + let pods: Api = Api::all(client.clone()); + let pod_list = pods.list(&Default::default()).await.unwrap(); + // check the pod is from daemonset + for p in pod_list { + if p.spec.unwrap().node_name.unwrap() == node_name { + assert_eq!(p.metadata.owner_references.unwrap()[0].kind, "DaemonSet"); + } + } + nodes.uncordon(node_name).await.unwrap(); + + clean_env() +} diff --git a/KubeOS-Rust/proxy/tests/setup/kind-config.yaml b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml new file mode 100644 index 00000000..0fe29e73 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker \ No newline at end of file diff --git a/KubeOS-Rust/proxy/tests/setup/resources.yaml b/KubeOS-Rust/proxy/tests/setup/resources.yaml new file mode 100644 index 00000000..0e449d5d --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/resources.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: example-daemonset +spec: + selector: + matchLabels: + name: example-daemonset + template: + metadata: + labels: + name: example-daemonset + spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-with-local-storage +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] + volumeMounts: + - mountPath: "/data" + name: local-volume + volumes: + - name: local-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Pod +metadata: + name: standalone-pod +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-deployment +spec: + replicas: 2 + selector: + matchLabels: + app: example + template: + metadata: + labels: + app: example + spec: + containers: + - name: nginx + image: nginx:alpine + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: DoesNotExist + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: example-pdb +spec: + minAvailable: 1 + selector: + matchLabels: + app: example +--- +apiVersion: v1 +kind: Pod +metadata: + name: resource-intensive-pod +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] + resources: + requests: + memory: "256Mi" + cpu: "500m" + limits: + memory: "512Mi" + cpu: "1000m" + diff --git a/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh new file mode 100644 index 00000000..d24d8e01 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# this bash script executes in proxy directory + +set -Eeuxo pipefail + +# Define variables +KIND_VERSION="v0.19.0" +KUBECTL_VERSION="v1.24.15" +KIND_CLUSTER_NAME="kubeos-test" +DOCKER_IMAGES=("busybox:stable" "nginx:alpine" "kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab") +NODE_IMAGE="kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab" +RESOURCE="./tests/setup/resources.yaml" +KIND_CONFIG="./tests/setup/kind-config.yaml" +BIN_PATH="../../bin/" +ARCH=$(uname -m) + +# Install kind and kubectl +install_bins() { + # if bin dir not exist then create + if [ ! -d "${BIN_PATH}" ]; then + mkdir -p "${BIN_PATH}" + fi + if [ ! -f "${BIN_PATH}"kind ]; then + echo "Installing Kind..." + # For AMD64 / x86_64 + if [ "$ARCH" = x86_64 ]; then + # add proxy if you are behind proxy + curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-amd64 + fi + # For ARM64 + if [ "$ARCH" = aarch64 ]; then + curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-arm64 + fi + chmod +x "${BIN_PATH}"kind + fi + if [ ! -f "${BIN_PATH}"kubectl ]; then + echo "Installing kubectl..." + if [ "$ARCH" = x86_64 ]; then + curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + fi + if [ "$ARCH" = aarch64 ]; then + curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/arm64/kubectl" + fi + chmod +x "${BIN_PATH}"kubectl + fi + export PATH=$PATH:"${BIN_PATH}" +} + +# Create Kind Cluster +create_cluster() { + echo "Creating Kind cluster..." + for image in "${DOCKER_IMAGES[@]}"; do + docker pull "$image" + done + kind create cluster --name "${KIND_CLUSTER_NAME}" --config "${KIND_CONFIG}" --image "${NODE_IMAGE}" +} + +# Load Docker image into Kind cluster +load_docker_image() { + echo "Loading Docker image into Kind cluster..." + DOCKER_IMAGE=$(printf "%s " "${DOCKER_IMAGES[@]:0:2}") + kind load docker-image ${DOCKER_IMAGE} --name "${KIND_CLUSTER_NAME}" +} + +# Apply Kubernetes resource files +apply_k8s_resources() { + echo "Applying Kubernetes resources from ${RESOURCE}..." + kubectl apply -f "${RESOURCE}" + echo "Waiting for nodes getting ready..." + sleep 40s +} + +main() { + export no_proxy=localhost,127.0.0.1 + install_bins + create_cluster + load_docker_image + apply_k8s_resources +} + +main diff --git a/KubeOS-Rust/rustfmt.toml b/KubeOS-Rust/rustfmt.toml new file mode 100644 index 00000000..3c565cf5 --- /dev/null +++ b/KubeOS-Rust/rustfmt.toml @@ -0,0 +1,11 @@ +# cargo +nightly fmt +version = "Two" +use_small_heuristics = "MAX" +match_block_trailing_comma = true +newline_style = "Unix" +merge_derives = false +max_width = 120 +group_imports = "StdExternalCrate" +imports_granularity = "Crate" +reorder_imports = true +unstable_features = true \ No newline at end of file diff --git a/Makefile b/Makefile index 634a3bcf..a712e872 100644 --- a/Makefile +++ b/Makefile @@ -46,11 +46,13 @@ GO_BUILD_CGO = CGO_ENABLED=1 \ CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \ ${GO_BUILD} -buildmode=pie -trimpath -tags "seccomp selinux static_build cgo netgo osusergo" -all: proxy operator agent hostshell +RUSTFLAGS := RUSTFLAGS="-C relocation_model=pic -D warnings -W unsafe_code -W rust_2021_incompatible_closure_captures -C link-arg=-s" + +all: proxy operator agent hostshell rust-kubeos # Build binary proxy: - ${GO_BUILD_CGO} ${LD_FLAGS} -o bin/proxy cmd/proxy/main.go + ${GO_BUILD_CGO} ${LD_FLAGS} -o bin/proxy cmd/proxy/main.go strip bin/proxy operator: @@ -69,9 +71,25 @@ hostshell: ${GO_BUILD_CGO} ${LD_FLAGS} -o bin/hostshell cmd/admin-container/main.go strip bin/hostshell +rust-kubeos: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust + +rust-proxy: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package proxy + +rust-agent: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package os-agent + +rust-kbimg: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package kbimg + +# clean binary +clean: + rm -rf bin + # Install CRDs into a cluster install: manifests - kubectl apply -f confg/crd + kubectl apply -f config/crd # Uninstall CRDs from a cluster uninstall: manifests @@ -104,7 +122,7 @@ generate: controller-gen $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." # Build the docker image -docker-build: operator proxy +docker-build: operator rust-proxy docker build --target operator -t ${IMG_OPERATOR} . docker build --target proxy -t ${IMG_PROXY} . diff --git a/VERSION b/VERSION index ee90284c..af0b7ddb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.4 +1.0.6 diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go index d3d636de..bde86d25 100644 --- a/api/v1alpha1/os_types.go +++ b/api/v1alpha1/os_types.go @@ -24,10 +24,12 @@ type OSSpec struct { CheckSum string `json:"checksum"` FlagSafe bool `json:"flagSafe"` MTLS bool `json:"mtls"` + // +kubebuilder:validation:Enum=docker;disk;containerd ImageType string `json:"imagetype"` ContainerImage string `json:"containerimage"` - OpsType string `json:"opstype"` - EvictPodForce bool `json:"evictpodforce"` + // +kubebuilder:validation:Enum=upgrade;config;rollback + OpsType string `json:"opstype"` + EvictPodForce bool `json:"evictpodforce"` // +kubebuilder:validation:Optional CaCert string `json:"cacert"` // +kubebuilder:validation:Optional @@ -39,7 +41,17 @@ type OSSpec struct { // +kubebuilder:validation:Optional UpgradeConfigs SysConfigs `json:"upgradeconfigs"` // +kubebuilder:validation:Optional + // +kubebuilder:default:=no-label NodeSelector string `json:"nodeselector"` + // +kubebuilder:validation:Optional + TimeWindow TimeWindow `json:"timewindow"` + // +kubebuilder:validation:Optional + // +kubebuilder:default:=15 + TimeInterval int `json:"timeinterval"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=serial;parallel + // +kubebuilder:default:=parallel + ExecutionMode string `json:"executionmode"` } // +kubebuilder:subresource:status @@ -90,6 +102,11 @@ type Content struct { Operation string `json:"operation"` } +type TimeWindow struct { + StartTime string `json:"starttime"` + EndTime string `json:"endtime"` +} + // +kubebuilder:subresource:status // +kubebuilder:object:root=true @@ -116,11 +133,21 @@ type OSInstanceSpec struct { // +kubebuilder:validation:Optional NodeStatus string `json:"nodestatus"` // +kubebuilder:validation:Optional + NamespacedName NamespacedName `json:"namespacedname"` + // +kubebuilder:validation:Optional SysConfigs SysConfigs `json:"sysconfigs"` // +kubebuilder:validation:Optional UpgradeConfigs SysConfigs `json:"upgradeconfigs"` } +// NamespacedName defines name and namespace of os corresponding to osinstance +type NamespacedName struct { + // +kubebuilder:validation:Optional + Name string `json:"name"` + // +kubebuilder:validation:Optional + Namespace string `json:"namespace"` +} + // +kubebuilder:object:root=true // OSInstanceList is a list of OSInstance diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go index 9c9c6426..9f130479 100644 --- a/cmd/operator/controllers/operation.go +++ b/cmd/operator/controllers/operation.go @@ -15,50 +15,37 @@ package controllers import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" - "openeuler.org/KubeOS/pkg/common" - "openeuler.org/KubeOS/pkg/values" "sigs.k8s.io/controller-runtime/pkg/client" upgradev1 "openeuler.org/KubeOS/api/v1alpha1" + "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" ) type operation interface { - newExistRequirement() (labels.Requirement, error) - newNotExistRequirement() (labels.Requirement, error) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - nodes []corev1.Node, limit int) (int, error) + nodes []corev1.Node, limit int) (int, []error) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, node *corev1.Node, osInstance *upgradev1.OSInstance) error + getOpsLabel() opsLabel } -type upgradeOps struct{} - -func (u upgradeOps) newExistRequirement() (labels.Requirement, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelUpgrading) - return labels.Requirement{}, err - } - return *requirement, nil +type upgradeOps struct { + label opsLabel } -func (u upgradeOps) newNotExistRequirement() (labels.Requirement, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelUpgrading) - return labels.Requirement{}, err - } - return *requirement, nil +func (u upgradeOps) getOpsLabel() opsLabel { + return u.label } func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - nodes []corev1.Node, limit int) (int, error) { - var count int + nodes []corev1.Node, limit int) (int, []error) { + var count = 0 + var errList []error for _, node := range nodes { if count >= limit { break @@ -67,21 +54,52 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, if os.Spec.OSVersion != osVersionNode { log.Info("Upgrading node " + node.Name) var osInstance upgradev1.OSInstance - if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err := r.Get(ctx, types.NamespacedName{Namespace: values.OsiNamespace, Name: node.Name}, &osInstance); err != nil { if err = client.IgnoreNotFound(err); err != nil { - log.Error(err, "failed to get osInstance "+node.Name, "skip this node") - return count, err + log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ + "Restart the reconcile and wait until it is complete.") + return count, []error{err} } + log.Error(err, "failed to get osInstance "+node.Name+"skip this node") + errList = append(errList, err) continue } if err := u.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { log.Error(err, "failed to update node and osinstance ,skip this node ") + errList = append(errList, err) continue } count++ } } + if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { + if errList = deleteSerialLabel(ctx, r, nodes); len(errList) != 0 { + log.Error(nil, "failed to delete nodes serial label") + } + } + if len(errList) > 0 { + return count, errList + } return count, nil + +} + +func deleteSerialLabel(ctx context.Context, r common.ReadStatusWriter, nodes []corev1.Node) []error { + var errList []error + for _, node := range nodes { + if _, ok := node.Labels[values.LabelSerial]; ok { + delete(node.Labels, values.LabelSerial) + if err := r.Update(ctx, &node); err != nil { + log.Error(err, "unable to delete serial label ", "node", node.Name+", skip this node") + errList = append(errList, err) + } + log.Info("delete node " + node.Name + " serial label " + values.LabelSerial + " successfully") + } + } + if len(errList) > 0 { + return errList + } + return nil } func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, @@ -106,11 +124,14 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW } } osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() + osInstance.Spec.NamespacedName = upgradev1.NamespacedName{Name: os.Name, Namespace: os.Namespace} + log.V(1).Info("Wait to update osinstance name:" + osInstance.Name + " node name is " + node.Name) if err := r.Update(ctx, osInstance); err != nil { log.Error(err, "unable to update", "osInstance", osInstance.Name) return err } log.Info("Update osinstance spec successfully") + node.Labels[values.LabelUpgrading] = "" if err := r.Update(ctx, node); err != nil { log.Error(err, "unable to label", "node", node.Name) @@ -120,29 +141,18 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW return nil } -type configOps struct{} - -func (c configOps) newExistRequirement() (labels.Requirement, error) { - requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.Exists, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelConfiguring) - return labels.Requirement{}, err - } - return *requirement, nil +type configOps struct { + label opsLabel } -func (c configOps) newNotExistRequirement() (labels.Requirement, error) { - requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.DoesNotExist, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelConfiguring) - return labels.Requirement{}, err - } - return *requirement, nil +func (c configOps) getOpsLabel() opsLabel { + return c.label } func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - nodes []corev1.Node, limit int) (int, error) { - var count int + nodes []corev1.Node, limit int) (int, []error) { + var count = 0 + var errList []error for _, node := range nodes { if count >= limit { break @@ -150,21 +160,34 @@ func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o var osInstance upgradev1.OSInstance if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { if err = client.IgnoreNotFound(err); err != nil { - log.Error(err, "failed to get osInstance "+node.Name) - return count, err + log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ + "Restart the reconcile and wait until it is complete.") + return count, []error{err} } + log.Error(err, "failed to get osInstance "+node.Name+", skip this node") + errList = append(errList, err) continue } if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { log.Info("Configuring node " + node.Name) if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { log.Error(err, "failed to update node and osinstance ,skip this node ") + errList = append(errList, err) continue } count++ } } - return count, nil + if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial { + if errList = deleteSerialLabel(ctx, r, nodes); len(errList) != 0 { + log.Error(nil, "failed to delete nodes serial label") + } + } + if len(errList) > 0 { + return count, errList + } + return count, errList + } func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, @@ -173,11 +196,14 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr return err } osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() + osInstance.Spec.NamespacedName = upgradev1.NamespacedName{Name: os.Name, Namespace: os.Namespace} + log.V(1).Info("Wait to update osinstance name:" + osInstance.Name + " node name is " + node.Name) if err := r.Update(ctx, osInstance); err != nil { log.Error(err, "unable to update", "osInstance", osInstance.Name) return err } log.Info("Update osinstance spec successfully") + node.Labels[values.LabelConfiguring] = "" if err := r.Update(ctx, node); err != nil { log.Error(err, "unable to label", "node", node.Name) @@ -186,3 +212,72 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr log.Info("Add node configuring label " + values.LabelConfiguring + " successfully") return nil } + +type serialOps struct { + label opsLabel +} + +func (s serialOps) getOpsLabel() opsLabel { + return s.label +} + +func (s serialOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, []error) { + var count int + var errList []error + for _, node := range nodes { + if count >= limit { + break + } + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+ + "Restart the reconcile and wait until it is complete.") + return count, []error{err} + } + log.Error(err, "failed to get osInstance "+node.Name+", skip this node") + errList = append(errList, err) + continue + } + switch s.getOpsLabel().label { + case values.LabelUpgrading: + if os.Spec.OSVersion != node.Status.NodeInfo.OSImage { + log.Info("Add Serial Label to node " + node.Name) + if err := s.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + errList = append(errList, err) + continue + } + count++ + } + case values.LabelConfiguring: + if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { + log.Info("Add Serial Label to node " + node.Name) + if err := s.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + errList = append(errList, err) + continue + } + count++ + } + default: + log.Error(nil, "ops "+s.getOpsLabel().label+" cannot be recognized") + return count, []error{fmt.Errorf("ops " + s.getOpsLabel().label + " cannot be recognized")} + } + } + if len(errList) == 0 { + return count, nil + } + return count, errList +} +func (s serialOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error { + node.Labels[values.LabelSerial] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } + log.Info("Add node serial label " + values.LabelSerial + " successfully") + return nil +} diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index 2c4ee1f7..adcae87d 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -17,6 +17,8 @@ import ( "context" "encoding/json" "fmt" + "strconv" + "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -56,35 +58,71 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re // Reconcile compares the actual state with the desired and updates the status of the resources e.g. nodes func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) (ctrl.Result, error) { - os, nodeNum, err := getAndUpdateOS(ctx, r, req.NamespacedName) + log.V(1).Info("start Reconcile of " + req.Name + " with namespace is " + req.Namespace) + os, err := getOSCr(ctx, r, req.NamespacedName) if err != nil { if errors.IsNotFound(err) { return values.NoRequeue, nil } return values.RequeueNow, err } - + isWithinTimeWindow, err := isWithinTimeWindow(os.Spec.TimeWindow.StartTime, os.Spec.TimeWindow.EndTime) + if err != nil { + return values.RequeueNow, err + } + if !isWithinTimeWindow { + log.V(1).Info("not in time window, the start time is " + os.Spec.TimeWindow.StartTime + + " , the end time " + os.Spec.TimeWindow.EndTime) + return values.Requeue, nil + } ops := os.Spec.OpsType var opsInsatnce operation switch ops { case "upgrade", "rollback": - opsInsatnce = upgradeOps{} + opsInsatnce = upgradeOps{ + label: opsLabel{ + label: values.LabelUpgrading, + op: selection.DoesNotExist, + }, + } case "config": - opsInsatnce = configOps{} + opsInsatnce = configOps{ + label: opsLabel{ + label: values.LabelConfiguring, + op: selection.DoesNotExist, + }, + } default: log.Error(nil, "operation "+ops+" cannot be recognized") return values.Requeue, nil } - limit, err := calNodeLimit(ctx, r, opsInsatnce, min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need + commonNodesReq, err := newCommonsNodesRequirement(os.Spec.NodeSelector, + selection.Equals).createNodeRequirement(ctx, r) if err != nil { return values.RequeueNow, err } - if needRequeue, err := assignOperation(ctx, r, os, limit, opsInsatnce); err != nil { + allNodes, err := getNodes(ctx, r, 0, commonNodesReq...) + if err != nil { return values.RequeueNow, err - } else if needRequeue { + } + log.V(1).Info("get all nodes num is " + strconv.Itoa(len(allNodes))) + switch os.Spec.ExecutionMode { + case ExecutionModeParallel: + result, err := excuteParallelOperation(ctx, r, os, opsInsatnce, len(allNodes)) + if err != nil { + return values.RequeueNow, nil + } + return result, nil + case ExecutionModeSerial: + result, err := excuteSerialOperation(ctx, r, os, opsInsatnce, len(allNodes)) + if err != nil { + return values.RequeueNow, err + } + return result, nil + default: + log.Error(nil, "excutionMode "+os.Spec.ExecutionMode+" cannot be recognized") return values.Requeue, nil } - return values.Requeue, nil } // SetupWithManager sets up the controller with the Manager. @@ -124,72 +162,17 @@ func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLim } } -func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, - int, error) { +func getOSCr(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, error) { var os upgradev1.OS if err := r.Get(ctx, name, &os); err != nil { log.Error(err, "unable to fetch OS") - return upgradev1.OS{}, 0, err - } - - requirement, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelMaster) - return upgradev1.OS{}, 0, err - } - var requirements []labels.Requirement - requirements = append(requirements, *requirement) - if os.Spec.NodeSelector != "" { - reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Exists, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelNodeSelector) - return upgradev1.OS{}, 0, err - } - requirements = append(requirements, *requirement, *reqSelector) - } - nodesItems, err := getNodes(ctx, r, 0, requirements...) - if err != nil { - log.Error(err, "get slave nodes fail") - return upgradev1.OS{}, 0, err - } - nodeNum := len(nodesItems) - return os, nodeNum, nil -} - -func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, - ops operation) (bool, error) { - requirement, err := ops.newNotExistRequirement() - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelUpgrading) - return false, err - } - reqMaster, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelMaster) - return false, err - } - var requirements []labels.Requirement - requirements = append(requirements, requirement, *reqMaster) - if os.Spec.NodeSelector != "" { - reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{os.Spec.NodeSelector}) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelNodeSelector) - return false, err - } - requirements = append(requirements, *reqSelector) - } - - nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated - if err != nil { - return false, err + return upgradev1.OS{}, err } - // Upgrade OS for selected nodes - count, err := ops.updateNodes(ctx, r, &os, nodes, limit) - if err != nil { - return false, err + if err := checkNodeSelector(ctx, r, os); err != nil { + log.Error(err, "nodeselector conficts") + return upgradev1.OS{}, err } - - return count >= limit, nil + return os, nil } func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, @@ -204,28 +187,30 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, } func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, - ops operation, maxUnavailable int, nodeSelector string) (int, error) { - requirement, err := ops.newExistRequirement() + maxUnavailable int, requirements []labels.Requirement) (int, error) { + nodes, err := getNodes(ctx, r, 0, requirements...) if err != nil { - log.Error(err, "unable to create requirement "+values.LabelUpgrading) return 0, err + } - var requirements []labels.Requirement - requirements = append(requirements, requirement) - if nodeSelector != "" { - reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{nodeSelector}) - if err != nil { - log.Error(err, "unable to create requirement "+values.LabelNodeSelector) - return 0, err - } - requirements = append(requirements, *reqSelector) + return maxUnavailable - len(nodes), nil +} +func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, + opsInstance operation, requirements []labels.Requirement) (int, error) { + if limit == 0 { + log.V(1).Info("limit is 0 , do not need to assign operation") + return 0, nil } - nodes, err := getNodes(ctx, r, 0, requirements...) + nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated if err != nil { return 0, err - } - return maxUnavailable - len(nodes), nil + log.V(1).Info("get wait to check nodes is " + strconv.Itoa(len(nodes))) + count, errLists := opsInstance.updateNodes(ctx, r, &os, nodes, limit) + if len(errLists) != 0 { + return 0, fmt.Errorf("update nodes and osinstance error") + } + return count, nil } func min(a, b int) int { @@ -259,3 +244,123 @@ func deepCopySpecConfigs(os *upgradev1.OS, osinstance *upgradev1.OSInstance, con } return nil } + +// Check whether the nodeselector conflicts with other nodeselector in OS CRs. If the nodeselector is empty, return the list of other nodeselectors. +func checkNodeSelector(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS) error { + var osList upgradev1.OSList + if err := r.List(ctx, &osList, &client.ListOptions{}); err != nil { + log.Error(err, "unable to list nodes with requirements") + return err + } + var sameNodeSelectorList []types.NamespacedName + for _, osItem := range osList.Items { + // Exclude current os, controller-runtime not supports multiple indexs as listoptions in current version, + // so cannot list os without current os use List function + if osItem.Name == os.Name && osItem.Namespace == os.Namespace { + continue + } + if os.Spec.NodeSelector == osItem.Spec.NodeSelector { + sameNodeSelectorList = append(sameNodeSelectorList, types.NamespacedName{ + Namespace: osItem.Namespace, + Name: osItem.Name, + }) + } + } + // If a node label corresponds to multiple OS CRs, upgrade or configuration information may conflict. + // As a result, an error is reported and returned when there are one-to-many relationships. + if len(sameNodeSelectorList) > 0 { + errorMessage := sameNodeSelectorList[0].String() + for i := 1; i < len(sameNodeSelectorList); i++ { + errorMessage = errorMessage + " , " + sameNodeSelectorList[i].String() + } + log.Error(nil, "OS CR "+os.Name+" in namespace "+os.Namespace+" has same nodeselector with "+errorMessage) + return fmt.Errorf("OS CR %s in namespace %s has same nodeselector with %s", os.Name, os.Namespace, errorMessage) + } + return nil +} + +func setTimeInterval(timeInterval int) ctrl.Result { + return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(timeInterval) * time.Second} +} + +func excuteParallelOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, + opsInsatnce operation, nodeNum int) (ctrl.Result, error) { + log.V(1).Info("start parallel operation") + opsLabel := opsInsatnce.getOpsLabel() + opsLabel.op = selection.Exists + opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, + selection.Equals, opsLabel).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + limit, err := calNodeLimit(ctx, r, min(os.Spec.MaxUnavailable, nodeNum), opsNodesReq) // adjust maxUnavailable if need + if err != nil { + return values.RequeueNow, nil + } + log.V(1).Info("get limit is " + strconv.Itoa(limit)) + opsLabel.op = selection.DoesNotExist + noOpsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, + selection.Equals, opsLabel).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + if _, err := assignOperation(ctx, r, os, limit, opsInsatnce, noOpsNodesReq); err != nil { + return values.RequeueNow, nil + } + return setTimeInterval(os.Spec.TimeInterval), nil +} + +func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, + opsInsatnce operation, nodeNum int) (ctrl.Result, error) { + log.V(1).Info("start serial operation") + opsLabel := opsInsatnce.getOpsLabel() + opsLabel.op = selection.Exists + opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector, + selection.Equals, opsLabel).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + opsNodeNum, err := getNodes(ctx, r, 0, opsNodesReq...) + if err != nil { + return values.RequeueNow, nil + } + if len(opsNodeNum) > 0 { + log.V(1).Info("a node is being upgraded or configured. Wait until the node upgrade or configuration is complete.") + return values.Requeue, nil + } + + serialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, + selection.Equals, selection.Exists).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + serialNodeLimit, err := calNodeLimit(ctx, r, min(os.Spec.MaxUnavailable, nodeNum), serialNodesRequirement) + if err != nil { + return values.RequeueNow, nil + } + log.V(1).Info("get the number of nodes which need to be added serial label num is " + strconv.Itoa(serialNodeLimit)) + noSerialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector, + selection.Equals, selection.DoesNotExist).createNodeRequirement(ctx, r) + if err != nil { + return values.RequeueNow, nil + } + // add serial label to node + serialOpsInstance := serialOps{ + label: opsInsatnce.getOpsLabel(), + } + log.V(1).Info("start add serial label to nodes") + if _, err := assignOperation(ctx, r, os, serialNodeLimit, serialOpsInstance, noSerialNodesRequirement); err != nil { + return values.RequeueNow, nil + } + + log.V(1).Info("start check nodes needed to be upgrade/configure or not") + serialLimit := 1 // 1 is the number of operation nodes when excution mode in serial + count, err := assignOperation(ctx, r, os, serialLimit, opsInsatnce, serialNodesRequirement) + if err != nil { + return values.RequeueNow, nil + } + if count > 0 { + return values.Requeue, nil + } + return setTimeInterval(os.Spec.TimeInterval), nil +} diff --git a/cmd/operator/controllers/requirements.go b/cmd/operator/controllers/requirements.go new file mode 100644 index 00000000..12db8e82 --- /dev/null +++ b/cmd/operator/controllers/requirements.go @@ -0,0 +1,199 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package controllers contains the Reconcile of operator +package controllers + +import ( + "context" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + + "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" +) + +const ( + AllNodeSelector = "all-label" + NoNodeSelector = "no-label" +) + +type labelRequirementCreator interface { + createLabelRequirement() ([]labels.Requirement, error) +} + +type masterLabel struct { + op selection.Operator +} + +func (ml masterLabel) createLabelRequirement() ([]labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelMaster, ml.op, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelMaster) + return []labels.Requirement{}, err + } + return []labels.Requirement{*requirement}, nil +} + +type opsLabel struct { + label string + op selection.Operator +} + +func (ol opsLabel) createLabelRequirement() ([]labels.Requirement, error) { + requirement, err := labels.NewRequirement(ol.label, ol.op, nil) + if err != nil { + log.Error(err, "unable to create requirement "+ol.label) + return []labels.Requirement{}, err + } + return []labels.Requirement{*requirement}, nil +} + +type nodeSelectorLabel struct { + value string + op selection.Operator +} + +func (nl nodeSelectorLabel) createLabelRequirement() ([]labels.Requirement, error) { + if nl.value == AllNodeSelector { + return []labels.Requirement{}, nil + } + var requirements []labels.Requirement + // if nodeselector is "no-label", will get the nodes which not have label + if nl.value == NoNodeSelector { + requirement, err := labels.NewRequirement(values.LabelNodeSelector, selection.DoesNotExist, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return []labels.Requirement{}, err + } + requirements = append(requirements, *requirement) + return requirements, nil + } + requirement, err := labels.NewRequirement(values.LabelNodeSelector, nl.op, []string{nl.value}) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return []labels.Requirement{}, err + } + requirements = append(requirements, *requirement) + return requirements, nil +} + +type serialLabel struct { + op selection.Operator +} + +func (sl serialLabel) createLabelRequirement() ([]labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelSerial, sl.op, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelSerial) + return []labels.Requirement{}, err + } + return []labels.Requirement{*requirement}, nil +} + +func createRequirement(labelsList []labelRequirementCreator) ([]labels.Requirement, error) { + var requirements []labels.Requirement + for _, label := range labelsList { + requirement, err := label.createLabelRequirement() + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return []labels.Requirement{}, err + } + requirements = append(requirements, requirement...) + } + return requirements, nil +} + +type nodeRequirementCreator interface { + createNodeRequirement(ctx context.Context, r common.ReadStatusWriter, nodeSelector string) ([]labels.Requirement, error) +} + +type commonsNodesRequirement struct { + nodeSelector string + op selection.Operator +} + +func newCommonsNodesRequirement(nodeSelector string, op selection.Operator) commonsNodesRequirement { + return commonsNodesRequirement{ + nodeSelector: nodeSelector, + op: op, + } +} + +func (c commonsNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) { + labelList := []labelRequirementCreator{ + masterLabel{op: selection.DoesNotExist}, + nodeSelectorLabel{value: c.nodeSelector, op: c.op}, + } + requirements, err := createRequirement(labelList) + if err != nil { + return []labels.Requirement{}, err + } + return requirements, nil +} + +type opsNodesRequirement struct { + common commonsNodesRequirement + ops opsLabel +} + +func newopsNodesRequirement(nodeSelector string, nodeSelectorOp selection.Operator, ops opsLabel) opsNodesRequirement { + return opsNodesRequirement{ + common: newCommonsNodesRequirement(nodeSelector, nodeSelectorOp), + ops: ops, + } +} + +func (o opsNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) { + labelList := []labelRequirementCreator{ + o.ops, + } + requirements, err := createRequirement(labelList) + if err != nil { + return []labels.Requirement{}, err + } + commonRequirements, err := o.common.createNodeRequirement(ctx, r) + if err != nil { + return []labels.Requirement{}, err + } + requirements = append(requirements, commonRequirements...) + return requirements, nil +} + +type serialNodesRequirement struct { + common commonsNodesRequirement + serialOp selection.Operator +} + +func newSerialNodesRequirement(nodeSelector string, nodeSelectorOp selection.Operator, serialrOp selection.Operator) serialNodesRequirement { + return serialNodesRequirement{ + common: newCommonsNodesRequirement(nodeSelector, nodeSelectorOp), + serialOp: serialrOp, + } +} + +func (o serialNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) { + labelList := []labelRequirementCreator{ + serialLabel{op: o.serialOp}, + } + requirements, err := createRequirement(labelList) + if err != nil { + return []labels.Requirement{}, err + } + commonRequirements, err := o.common.createNodeRequirement(ctx, r) + if err != nil { + return []labels.Requirement{}, err + } + requirements = append(requirements, commonRequirements...) + return requirements, nil +} diff --git a/cmd/operator/controllers/times.go b/cmd/operator/controllers/times.go new file mode 100644 index 00000000..f651c0e4 --- /dev/null +++ b/cmd/operator/controllers/times.go @@ -0,0 +1,104 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package controllers contains the Reconcile of operator + +package controllers + +import ( + "fmt" + "regexp" + "time" +) + +const ( + DATE_TIME = "2006-01-02 15:04:05" + TIME_ONLY = "15:04:05" + ExecutionModeSerial = "serial" + ExecutionModeParallel = "parallel" + oneDayTime = 24 * time.Hour +) + +func isWithinTimeWindow(start, end string) (bool, error) { + if start == "" && end == "" { + return true, nil + } + if start == "" || end == "" { + return false, fmt.Errorf("invalid TimeWindow: The start time and end time must be both empty or not empty") + } + layoutStart, err := checkTimeValid(start) + if err != nil { + return false, err + } + layoutEnd, err := checkTimeValid(end) + if err != nil { + return false, err + } + if layoutStart != layoutEnd { + return false, fmt.Errorf("invalid TimeWindow: Start Time should have same time format with End Time") + } + now := time.Now() + timeFormat := now.Format(layoutStart) + now, err = time.ParseInLocation(layoutStart, timeFormat, now.Location()) + startTime, err := time.ParseInLocation(layoutStart, start, now.Location()) + if err != nil { + return false, err + } + endTime, err := time.ParseInLocation(layoutStart, end, now.Location()) + if err != nil { + return false, err + } + if endTime.Equal(startTime) { + return false, fmt.Errorf("invalid TimeWindow: start time is equal to end time") + } + if endTime.Before(startTime) { + if layoutStart == DATE_TIME { + return false, fmt.Errorf("invalid TimeWindow: start time %s is after end time %s", + startTime.Format(layoutStart), endTime.Format(layoutEnd)) + } + endTime = endTime.Add(oneDayTime) + if now.Before(startTime) { + now = now.Add(oneDayTime) + } + + } + return now.After(startTime) && now.Before(endTime), nil +} + +func checkTimeValid(checkTime string) (string, error) { + reDateTime, err := regexp.Compile("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$") + if err != nil { + return "", err + } + reTimeOnly, err := regexp.Compile("^\\d{2}:\\d{2}:\\d{2}$") + if err != nil { + return "", err + } + + if reDateTime.MatchString(checkTime) { + _, err := time.Parse(DATE_TIME, checkTime) + if err != nil { + return "", err + } + return DATE_TIME, nil + + } + if reTimeOnly.MatchString(checkTime) { + _, err := time.Parse(TIME_ONLY, checkTime) + if err != nil { + return "", err + } + return TIME_ONLY, nil + + } + return "", fmt.Errorf("invalid TimeWindow: invalid date format, please use date format YYYY-MM-DD HH:MM:SS, or only HH:MM:SS") +} diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 6b90b26b..79b23ec7 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -14,6 +14,7 @@ package main import ( "os" + "strings" "time" zaplogfmt "github.com/sykesm/zap-logfmt" @@ -38,6 +39,11 @@ var ( setupLog = ctrl.Log.WithName("setup") ) +const ( + DEBUGLOGLEVEL = "debug" + INFOLOGLEVEL = "info" +) + func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -51,7 +57,12 @@ func main() { encoder.AppendString(ts.UTC().Format(time.RFC3339Nano)) } logfmtEncoder := zaplogfmt.NewEncoder(configLog) - logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder)) + level := zap.Level(zapcore.Level(0)) + goLog := strings.ToLower(os.Getenv("GO_LOG")) + if goLog == DEBUGLOGLEVEL { + level = zap.Level(zapcore.Level(-1)) + } + logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder), level) ctrl.SetLogger(logger) mgr, err := common.NewControllerManager(setupLog, scheme) diff --git a/docs/example/config/admin-container/admin-container.yaml b/docs/example/config/admin-container/admin-container.yaml index b5ec9e03..518b955f 100644 --- a/docs/example/config/admin-container/admin-container.yaml +++ b/docs/example/config/admin-container/admin-container.yaml @@ -1,63 +1,63 @@ -apiVersion: v1 -kind: Secret -metadata: - name: root-secret -data: - ssh-pub-key: your-ssh-pub-key ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: admin-container-sysmaster - namespace: default - labels: - control-plane: admin-container-sysmaster -spec: - selector: - matchLabels: - control-plane: admin-container-sysmaster - replicas: 1 - template: - metadata: - labels: - control-plane: admin-container-sysmaster - spec: - hostPID: true - containers: - - name: admin-container-sysmaster - image: your_imageRepository/admin_imageName:version - imagePullPolicy: Always - securityContext: - privileged: true - ports: - - containerPort: 22 - # sysmaster要求 - env: - - name: container - value: containerd - volumeMounts: - # name 必须与下面的卷名匹配 - - name: secret-volume - # mountPath必须为/etc/secret-volume - mountPath: /etc/secret-volume - readOnly: true - nodeName: your-worker-node-name - volumes: - - name: secret-volume - secret: - # secretName必须与上面指定的Secret的name相同 - secretName: root-secret ---- -apiVersion: v1 -kind: Service -metadata: - name: admin-container-sysmaster - namespace: default -spec: - type: NodePort - ports: - - port: 22 - targetPort: 22 - nodePort: your-exposed-port - selector: - control-plane: admin-container-sysmaster \ No newline at end of file +apiVersion: v1 +kind: Secret +metadata: + name: root-secret +data: + ssh-pub-key: +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin-container-sysmaster + namespace: default + labels: + control-plane: admin-container-sysmaster +spec: + selector: + matchLabels: + control-plane: admin-container-sysmaster + replicas: 1 + template: + metadata: + labels: + control-plane: admin-container-sysmaster + spec: + hostPID: true + containers: + - name: admin-container-sysmaster + image: + imagePullPolicy: Always + securityContext: + privileged: true + ports: + - containerPort: 22 + # sysmaster要求 + env: + - name: container + value: containerd + volumeMounts: + # name 必须与下面的卷名匹配 + - name: secret-volume + # mountPath必须为/etc/secret-volume + mountPath: /etc/secret-volume + readOnly: true + nodeName: + volumes: + - name: secret-volume + secret: + # secretName必须与上面指定的Secret的name相同 + secretName: root-secret +--- +apiVersion: v1 +kind: Service +metadata: + name: admin-container-sysmaster + namespace: default +spec: + type: NodePort + ports: + - port: 22 + targetPort: 22 + nodePort: + selector: + control-plane: admin-container-sysmaster diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index 98ef1f56..8a29e0c3 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -18,7 +18,7 @@ spec: versions: - name: v1alpha1 additionalPrinterColumns: - - name: OSVersion + - name: OS VERSION jsonPath: .spec.osversion type: string description: The version of OS @@ -57,9 +57,19 @@ spec: type: string evictpodforce: type: boolean + executionmode: + default: parallel + enum: + - serial + - parallel + type: string flagSafe: type: boolean imagetype: + enum: + - docker + - disk + - containerd type: string imageurl: type: string @@ -68,8 +78,13 @@ spec: mtls: type: boolean nodeselector: + default: no-label type: string opstype: + enum: + - upgrade + - config + - rollback type: string osversion: type: string @@ -101,6 +116,19 @@ spec: version: type: string type: object + timeinterval: + default: 15 + type: integer + timewindow: + properties: + endtime: + type: string + starttime: + type: string + required: + - endtime + - starttime + type: object upgradeconfigs: description: SysConfigs defines all configurations expected by the user properties: diff --git a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml index a7bad3f0..47de87e3 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml @@ -22,19 +22,19 @@ spec: type: string jsonPath: .spec.nodestatus description: The status of node - - name: SYSCONFIG STATUS + - name: SYSCONFIG-VERSION-CURRENT type: string jsonPath: .status.sysconfigs.version description: The current status of sysconfig - - name: SYSCONFIG SPEC + - name: SYSCONFIG-VERSION-DESIRED type: string jsonPath: .spec.sysconfigs.version description: The expected version of sysconfig - - name: UPGRADECONFIG STATUS + - name: UPGRADECONFIG-VERSION-CURRENT type: string jsonPath: .status.upgradeconfigs.version description: The current version of upgradeconfig - - name: UPGRADECONFIG SPEC + - name: UPGRADECONFIG-VERSION-DESIRED type: string jsonPath: .spec.upgradeconfigs.version description: The expected version of upgradeconfig @@ -53,6 +53,14 @@ spec: spec: description: OSInstanceSpec defines desired state of OS properties: + namespacedname: + description: NamespacedName defines name and namespace of os corresponding to osinstance + properties: + name: + type: string + namespace: + type: string + type: object nodestatus: type: string sysconfigs: diff --git a/docs/example/config/manager/manager.yaml b/docs/example/config/manager/manager.yaml index 93d15220..c35d9010 100644 --- a/docs/example/config/manager/manager.yaml +++ b/docs/example/config/manager/manager.yaml @@ -20,19 +20,19 @@ spec: control-plane: upgrade-proxy spec: containers: - - name: proxy - command: - - /proxy - image: edit.proxy.image.addr - volumeMounts: - - name: upgrade-agent - mountPath: /var/run/os-agent - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName + - name: proxy + command: + - /proxy + image: + volumeMounts: + - name: upgrade-agent + mountPath: /var/run/os-agent + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName volumes: - name: upgrade-agent hostPath: @@ -56,24 +56,33 @@ spec: control-plane: upgrade-operator spec: containers: - - command: - - /operator - image: edit.operator.image.addr - name: operator - securityContext: - allowPrivilegeEscalation: false - runAsUser: 6552 - runAsGroup: 6552 - resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 20Mi + - command: + - /operator + image: + name: operator + volumeMounts: + - name: date-config + mountPath: /etc/localtime + securityContext: + allowPrivilegeEscalation: false + runAsUser: 6552 + runAsGroup: 6552 + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + volumes: + - name: date-config + hostPath: + path: /etc/localtime terminationGracePeriodSeconds: 10 nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: - key: "node-role.kubernetes.io/master" operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" diff --git a/docs/example/config/samples/upgrade_v1alpha1_os.yaml b/docs/example/config/samples/upgrade_v1alpha1_os.yaml index ca3da1b2..d7528831 100644 --- a/docs/example/config/samples/upgrade_v1alpha1_os.yaml +++ b/docs/example/config/samples/upgrade_v1alpha1_os.yaml @@ -1,39 +1,38 @@ apiVersion: upgrade.openeuler.org/v1alpha1 kind: OS metadata: - name: os-sample + name: os-sample spec: - imagetype: docker/containerd/disk - opstype: upgrade/config/rollback - osversion: edit.os.version - maxunavailable: edit.node.upgrade.number - containerimage: "" - evictpodforce: true - imageurl: "" - checksum: image digests - flagSafe: false - mtls: false - nodeselector: edit.nodes.label - sysconfigs: - version: edit.sysconfig.version - configs: - - model: kernel.systcl - contents: - - key: kernel param key1 - value: kernel param value1 - - key: kernel param key2 - value: kernel param value2 - - model: kernel.systcl.persist - configpath: persist file path - contents: - - key: kernel param key3 - value: kernel param value3 - operation: delete - upgradeconfigs: - version: edit.upgradeconfig.version - configs: - - model: kernel.systcl - contents: - - key: kernel param key4 - value: kernel param value4 - operation: delete + imagetype: docker/containerd/disk + opstype: upgrade/config/rollback + osversion: edit.os.version + maxunavailable: edit.node.upgrade.number + containerimage: "" + evictpodforce: true + imageurl: "" + checksum: image digests + flagSafe: false + mtls: false + sysconfigs: + version: edit.sysconfig.version + configs: + - model: kernel.sysctl + contents: + - key: kernel param key1 + value: kernel param value1 + - key: kernel param key2 + value: kernel param value2 + - model: kernel.sysctl.persist + configpath: persist file path + contents: + - key: kernel param key3 + value: kernel param value3 + operation: delete + upgradeconfigs: + version: edit.upgradeconfig.version + configs: + - model: kernel.sysctl + contents: + - key: kernel param key4 + value: kernel param value4 + operation: delete diff --git a/docs/quick-start.md b/docs/quick-start.md index 13bb08d9..1d59048e 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -1,18 +1,22 @@ # 快速使用指导 -## 编译及部署 +[TOC] -### 编译指导 +## 编译指导 * 编译环境：openEuler Linux x86/AArch64 + * 进行编译需要以下包： * golang(大于等于1.15版本) * make * git + * rust(大于等于1.64版本) + * cargo(大于等于1.64版本) + * openssl-devel ``` shell - sudo yum install golang make git - ``` + sudo yum install golang make git rust cargo openssl-devel + ``` * 使用git获取本项目的源码 @@ -27,77 +31,128 @@ ```shell cd KubeOS - sudo make - ``` - - * proxy及operator容器镜像构建 - * proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕 - * 请用户自行编写Dockerfile来构建镜像，请注意 - * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性 - * 需将operator和proxy拷贝到baseimage上 - * 请确保proxy属主和属组为root，文件权限为500 - * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500 - * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应 - * 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库 - * Dockerfile参考如下, Dockerfile也可以使用多阶段构建: - - ``` dockerfile - FROM your_baseimage - COPY ./bin/proxy /proxy - ENTRYPOINT ["/proxy"] - FROM your_baseimage - COPY --chown=6552:6552 ./bin/operator /operator - ENTRYPOINT ["/operator"] - ``` + sudo make + # 编译生成的二进制在bin目录下，查看二进制 + tree bin + bin + ├── operator + ├── os-agent + ├── proxy + ├── rust + │   ├── ... + │   └── release + │   ├── ... + │   ├── os-agent + │   └── proxy + ``` - ```shell - # 指定proxy的镜像仓库，镜像名及版本 - export IMG_PROXY=your_imageRepository/proxy_imageName:version - # 指定proxy的Dockerfile地址 - export DOCKERFILE_PROXY=your_dockerfile_proxy - # 指定operator的镜像仓库，镜像名及版本 - export IMG_OPERATOR=your_imageRepository/operator_imageName:version - # 指定operator的Dockerfile路径 - export DOCKERFILE_OPERATOR=your_dockerfile_operator - - # 镜像构建 - docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} . - docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} . - # 推送镜像到镜像仓库 - docker push ${IMG_OPERATOR} - docker push ${IMG_PROXY} - ``` + * ```bin/proxy```、```bin/os-agent```为go语言编写的proxy和os-agent，```bin/rust/release/proxy```、```bin/rust/release/os-agent```为rust语言编写的proxy和os-agent，二者功能一致。 + +## 镜像构建指导 + +### proxy及operator镜像构建指导 + +* proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕 + +* 请用户自行编写Dockerfile来构建镜像，请注意 + * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性 + * 需将operator和proxy拷贝到baseimage上 + * 请确保proxy属主和属组为root，文件权限为500 + * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500 + * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应 + +* 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库 + +* Dockerfile参考如下, Dockerfile也可以使用多阶段构建: + + `proxy`容器镜像Dockerfile + + ``` dockerfile + FROM openeuler/openeuler:24.03-lts + COPY ./bin/proxy /proxy + ENTRYPOINT ["/proxy"] + ``` + + `operator`容器镜像Dockerfile + + ``` dockerfile + FROM openeuler/openeuler:24.03-lts + COPY --chown=6552:6552 ./bin/operator /operator + ENTRYPOINT ["/operator"] + ``` -* OS虚拟机镜像制作 - * 制作注意事项 - * 请确保已安装qemu-img，bc，parted，tar，yum，docker, dosfstools - * 容器OS镜像制作需要使用root权限 - * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 - * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式 - * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间 - * 容器镜像制作时不支持用户自定义配置挂载文件 - * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。 - * 请确保os-agent属主和属组为root，建议os-agent文件权限为500 - * 容器OS虚拟机镜像制作 - 进入scripts目录，执行脚本 - - ```shell - cd scripts - bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' - ``` - - * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 - * 示例命令使用的密码为```openEuler12#$``` - * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： - * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 - * qcow2 格式的系统镜像 system.qcow2。 - * 可用于升级的根文件系统分区镜像 update.img 。 - * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数 - * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) - * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 - * 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 - -### 部署指导 + ```shell + # 指定proxy的镜像仓库，镜像名及版本 + export IMG_PROXY=your_imageRepository/proxy_imageName:version + # 指定proxy的Dockerfile地址 + export DOCKERFILE_PROXY=your_dockerfile_proxy + # 指定operator的镜像仓库，镜像名及版本 + export IMG_OPERATOR=your_imageRepository/operator_imageName:version + # 指定operator的Dockerfile路径 + export DOCKERFILE_OPERATOR=your_dockerfile_operator + + # 镜像构建 + docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} . + docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} . + # 推送镜像到镜像仓库 + docker push ${IMG_OPERATOR} + docker push ${IMG_PROXY} + ``` + +### KubeOS虚拟机镜像制作指导 + +* 制作注意事项 + * 请确保已安装qemu-img，bc，parted，tar，yum，docker，dosfstools + * 容器OS镜像制作需要使用root权限 + * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 + * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式 + * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间 + * 容器镜像制作时不支持用户自定义配置挂载文件 + * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。 + * 请确保os-agent属主和属组为root，建议os-agent文件权限为500 + +* 容器OS虚拟机镜像制作 + 进入scripts目录，执行脚本 + + ```shell + cd scripts + bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' + ``` + + 参数说明如下: + + ```bash + Usage : kbimg create vm-image -p iso-path -v os-version -b os-agent-dir -e os-password + or + kbimg create vm-image -d repository/name:tag + + options: + -p repo path + -v KubeOS version + -b path of os-agent binary + -e os encrypted password + -d docker image like repository/name:tag + -l boot to legacy BIOS mode, if not specify, then UEFI mode + -h,--help show help information + ``` + + * 其中 xxx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 + * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： + * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 + * qcow2 格式的系统镜像 system.qcow2。 + * 可用于升级的根文件系统分区镜像 update.img 。 + * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景。若x86 架构的虚拟机需要使用 legacy 启动模式，需制作镜像时指定-l参数 + * 默认root密码为openEuler12#$ + * 您可通过`openssl passwd -6 -salt $(head -c18 /dev/urandom | openssl base64)`命令生成root密码并通过`-e`参数配置密码 + * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) + * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 + * 详细参数说明请见[《容器OS镜像制作指导》](../docs/user_guide/%E5%AE%B9%E5%99%A8OS%E9%95%9C%E5%83%8F%E5%88%B6%E4%BD%9C%E6%8C%87%E5%AF%BC.md) + +* 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 + +## 部署指导 + +### os-operator和os-proxy部署指导 * 环境要求 * openEuler Linux x86/AArch64系统 @@ -143,18 +198,35 @@ kubectl get pods -A ``` -### 使用指导 - -#### 注意事项 - -* 容器OS升级为所有软件包原子升级，默认不在容器OS内提供单包升级能力。 -* 容器OS升级为双区升级的方式，不支持更多分区数量。 -* 单节点的升级过程的日志可在节点的/var/log/message文件查看。 -* 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。 -* 使用docker镜像升级和mtls双向认证仅支持 openEuler 22.09 及之后的版本 -* 不支持跨大版本升级 - -#### 参数说明 +## 使用指导 + +### 注意事项 + +* 公共注意事项 + * 仅支持虚拟机x86和arm64 UEFI场景。 + * 当前不支持集群节点OS多版本管理，即集群中OS的CR只能为一个。 + * 使用kubectl apply通过YAML创建或更新OS的CR时，不建议并发apply，当并发请求过多时，kube-apiserver会无法处理请求导致失败。 + * 如用户配置了容器镜像仓的证书或密钥，请用户保证证书或密钥文件的权限最小。 +* 升级注意事项 + * 升级为所有软件包原子升级，默认不提供单包升级能力。 + * 升级为双区升级的方式，不支持更多分区数量。 + * 当前暂不支持跨大版本升级。 + * 单节点的升级过程的日志可在节点的 /var/log/messages 文件查看。 + * 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。 + * 节点上containerd如需配置ctr使用的私有镜像，请将配置文件host.toml按照ctr指导放在/etc/containerd/certs.d目录下。 + +* 配置注意事项 + * 用户自行指定配置内容，用户需保证配置内容安全可靠 ，尤其是持久化配置（kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next），KubeOS不对参数有效性进行检验。 + * opstype=config时，若osversion与当前集群节点的OS版本不一致，配置不会进行。 + * 当前仅支持kernel参数临时配置（kernel.sysctl）、持久化配置（kernel.sysctl.persist）和grub cmdline配置（grub.cmdline.current和grub.cmdline.next）。 + * 持久化配置会写入persist持久化分区，升级重启后配置保留；kernel参数临时配置重启后不保留。 + * 配置grub.cmdline.current或grub.cmdline.next时，如为单个参数（非key=value格式参数），请指定key为该参数，value为空。 + * 进行配置删除（operation=delete）时，key=value形式的配置需保证key、value和实际配置一致。 + * 配置不支持回退，如需回退，请修改配置版本和配置内容，重新下发配置。 + * 配置出现错误，节点状态陷入config时，请将配置版本恢复成上一版本并重新下发配置，从而使节点恢复至idel状态。 但是请注意：出现错误前已经配置完成的参数无法恢复。 + * 在配置grub.cmdline.current或grub.cmdline.next时，若需要将已存在的“key=value”格式的参数更新为只有key无value格式，比如将“rd.info=0”更新成rd.info，需要先删除“key=value”，然后在下一次配置时，添加key。不支持直接更新或者更新删除动作在同一次完成。 + +#### OS CR参数说明 在集群中创建类别为OS的定制对象，设置相应字段。类别OS来自于安装和部署章节创建的CRD对象，字段及说明如下： @@ -164,23 +236,22 @@ | 参数 |参数类型 | 参数说明 | 使用说明 | 是否必选 | | -------------- | ------ | ------------------------------------------------------------ | ----- | ---------------- | - | imagetype | string | 使用的升级镜像的类型 | 需为 docker ，containerd ，或者是 disk，其他值无效，且该参数仅在升级场景有效。
**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。|是 | - | opstype | string | 进行的操作，升级,回退或者配置 | 需为 upgrade ，config 或者 rollback ，其他值无效 |是 | - | osversion | string | 用于升级或回退的镜像的OS版本 | 需为 KubeOS version , 例如: KubeOS 1.0.0|是 | - | maxunavailable | int | 同时进行升级或回退的节点数 | maxunavailable值设置为大于实际集群的节点数时也可正常部署，升级或回退时会按照集群内实际节点数进行|是 | - | containerimage | string | 用于升级的容器镜像 | 需要为容器镜像格式：[REPOSITORY/NAME[:TAG@DIGEST]](https://docs.docker.com/engine/reference/commandline/tag/#extended-description)，仅在使用容器镜像升级场景下有效|是 | - | imageurl | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如： 仅在使用磁盘镜像升级场景下有效|是 | + | imagetype | string | 升级镜像的类型 | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是 | + | opstype | string | 操作类型：升级,回退或者配置 | 仅支持upgrade ，config 或者 rollback |是 | + | osversion | string | 升级/回退的目标版本 | osversion需与节点的目标os版本对应（节点上/etc/os-release中PRETTY_NAME字段或k8s检查到的节点os版本） 例如：KubeOS 1.0.0。 |是 | + | maxunavailable | int | 每批同时进行升级/回退/配置的节点数。 | maxunavailable值大于实际节点数时，取实际节点数进行升级/回退/配置。 |是 | + | containerimage | string | 用于升级的容器镜像 | 仅在imagetype是容器类型时生效，仅支持以下3种格式的容器镜像地址： repository/name repository/name@sha256:xxxx repository/name:tag |是 | + | imageurl | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如： ，仅在使用磁盘镜像升级场景下有效 |是 | | checksum | string | 用于升级的磁盘镜像校验的checksum(SHA-256)值或者是用于升级的容器镜像的digests值 | 仅在升级场景下有效 |是 | | flagSafe | bool | 当imageurl的地址使用http协议表示是否是安全的 | 需为 true 或者 false ，仅在imageurl使用http协议时有效 |是 | | mtls | bool | 用于表示与imageurl连接是否采用https双向认证 | 需为 true 或者 false ，仅在imageurl使用https协议时有效|是 | | cacert | string | https或者https双向认证时使用的根证书文件 | 仅在imageurl使用https协议时有效| imageurl使用https协议时必选 | | clientcert | string | https双向认证时使用的客户端证书文件 | 仅在使用https双向认证时有效|mtls为true时必选 | | clientkey | string | https双向认证时使用的客户端公钥 | 仅在使用https双向认证时有效|mtls为true时必选 | - | evictpodforce | bool | 用于表示升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | + | evictpodforce | bool | 升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | + | sysconfigs | / | 配置设置 | 1. “opstype=config”时只进行配置。 2.“opstype=upgrade/rollback”时，代表升级/回退后配置，即在升级/回退重启后进行配置。```配置（Settings）指导``` | “opstype=config”时必选 | + | upgradeconfigs | / | 升级前配置设置 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | | nodeselector | string | 需要进行升级/配置/回滚操作的节点label | 用于只对具有某些特定label的节点而不是集群所有worker节点进行运维的场景，需要进行运维操作的节点需要包含key为upgrade.openeuler.org/node-selector的label，nodeselector为该label的value值，此参数不配置时，或者配置为""时默认对所有节点进行操作| 可选 | - | sysconfigs | / | 需要进行配置的参数值 | 在配置或者升级或者回退机器时有效，在升级或者回退操作之后即机器重启之后起效，详细字段说明请见```配置（Settings）指导```| 可选 | - | upgradeconfigs | / | 需要升级前进行的配置的参数值 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | - #### 升级指导 * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如下，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml; @@ -343,12 +414,13 @@ kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' ``` -* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的 imageurl, osversion, checksum, maxunavailable, flagSafe 或者containerimage字段进行相应修改。 +* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改 #### 配置（Settings）指导 * Settings参数说明: - 以进行配置时的示例yaml为例对配置的参数进行说明，示例yaml如下： + + 基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致： ```yaml apiVersion: upgrade.openeuler.org/v1alpha1 @@ -362,15 +434,12 @@ maxunavailable: edit.node.config.number containerimage: "" evictpodforce: false - imageurl: "" checksum: "" - flagSafe: false - mtls: false sysconfigs: - version: 1.0.0 + version: edit.sysconfigs.version configs: - model: kernel.sysctl - contents: + contents: - key: kernel param key1 value: kernel param value1 - key: kernel param key2 @@ -380,54 +449,80 @@ configpath: persist file path contents: - key: kernel param key3 - value: kernel param value3 + value: kernel param value3 + - model: grub.cmdline.current + contents: + - key: boot param key1 + - key: boot param key2 + value: boot param value2 + - key: boot param key3 + value: boot param value3 + operation: delete + - model: grub.cmdline.next + contents: + - key: boot param key4 + - key: boot param key5 + value: boot param value5 + - key: boot param key6 + value: boot param value6 + operation: delete ``` - * 配置的参数说明如下： - * version: 配置的版本,通过版本差异触发配置，请修改配置后更新 version - * configs: 具体配置内容 - * model: 进行的配置的类型，支持的配置类型请看[Settings 列表](#setting-列表) - * configpath: 如为持久化配置，配置文件路径 - * contents: 配置参数的 key / value 和对参数的操作。 - * key / value: 请看[Settings 列表](#setting-列表)对支持的配置的 key / value的说明。 - * operation: 若不指定operation，则默认为添加或更新。若指定为delete，代表删除目前OS中已配置的参数。 - **注意：** 当operation为delete时，yaml中的key/value必须和OS上想删除参数的key/value**一致**，否则删除失败。 - * upgradeconfigs与sysconfig参数相同，upgradeconfig为升级前进行的配置，仅在升级/回滚场景起效，在升级/回滚操作执行前进行配置，只进行配置或者需要升级/回滚重启后执行配置，使用sysconfigs + 配置的参数说明如下： + + | 参数 | 参数类型 | 参数说明 | 使用说明 | 配置中是否必选 | + | ---------- | -------- | --------------------------- | ------------------------------------------------------------ | ----------------------- | + | version | string | 配置的版本 | 通过version是否相等来判断配置是否触发，version为空（为""或者没有值）时同样进行判断，所以不配置sysconfigs/upgradeconfigs时，继存的version值会被清空并触发配置。 | 是 | + | configs | / | 具体配置内容 | 包含具体配置项列表。 | 是 | + | model | string | 配置的类型 | 支持的配置类型请看附录下的```Settings列表``` | 是 | + | configpath | string | 配置文件路径 | 仅在kernel.sysctl.persist配置类型中生效，请看附录下的```Settings列表```对配置文件路径的说明。 | 否 | + | contents | / | 具体key/value的值及操作类型 | 包含具体配置参数列表。 | 是 | + | key | string | 参数名称 | key不能为空，不能包含"="，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中每种配置类型对key的说明。 | 是 | + | value | string | 参数值 | key=value形式的参数中，value不能为空，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中对每种配置类型对value的说明。 | key=value形式的参数必选 | + | operation | string | 对参数进行的操作 | 仅对kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next类型的参数生效。默认为添加或更新。仅支持配置为delete，代表删除已存在的参数（key=value需完全一致才能删除）。 | 否 | + + * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置 + * 使用说明 + * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如上，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml + * 查看配置之前的节点的配置的版本和节点状态（NODESTATUS状态为idle） ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` * 执行命令，在集群中部署cr实例后，节点会根据配置的参数信息进行配置，再次查看节点状态(NODESTATUS变成config) ```shell kubectl apply -f upgrade_v1alpha1_os.yaml - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` * 再次查看节点的配置的版本确认节点是否配置完成(NODESTATUS恢复为idle) ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` -* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 +* 如果后续需要再次配置，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 #### 回退指导 * 回退场景 - * 虚拟机无法正常启动时，需要退回到上一可以启动的版本时进行回退操作，仅支持手动回退容器 OS 。 - * 虚拟机能够正常启动并且进入系统，需要将当前版本退回到老版本时进行回退操作，支持工具回退（类似升级方式）和手动回退，建议使用工具回退。 - * 配置出现错误，节点状态陷入config时，可以回退至上一个配置版本以恢复节点至idle状态。 - **注意**：在配置新版本时，出现错误前已经配置的参数无法回退。 + * 虚拟机无法正常启动时，可在grub启动项页面手动切换启动项，使系统回退至上一版本（即手动回退）。 + * 虚拟机能够正常启动并且进入系统时，支持工具回退和手动回退，建议使用工具回退。 + * 工具回退有两种方式： + 1. rollback模式直接回退至上一版本。 + 2. upgrade模式重新升级至上一版本 * 手动回退指导 - * 手动重启虚拟机，选择第二启动项进行回退，手动回退仅支持回退到本次升级之前的版本。 + + * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。 * 工具回退指导 * 回退至任意版本 - * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 + * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 + * YAML修改完成后执行更新命令，在集群中更新定制对象后，节点会根据配置的字段信息进行回退 ```shell @@ -499,36 +594,29 @@ * 查看节点容器 OS 版本(回退OS版本)或节点config版本&节点状态为idle(回退config版本)，确认回退是否成功。 ```shell - kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' - - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` -#### Admin容器 +## Admin容器镜像制作、部署和使用 KubeOS提供一个分离的包含sshd服务和hostshell工具的Admin容器，来帮助管理员在必要情况下登录KubeOS，其中的sshd服务由[sysmaster](https://gitee.com/openeuler/sysmaster)/systemd拉起。Admin容器部署后用户可通过ssh连接到节点的Admin容器，进入Admin容器后执行hostshell命令获取host的root shell。 -##### 部署方法 +### admin容器镜像制作 -以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/everything/aarch64/Packages/)到scripts/admin-container目录下。 +以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-SP1-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/update/aarch64/Packages/)到scripts/admin-container目录下。 -**修改**admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/docker_img)下载。 +修改admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts-sp1可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/docker_img)下载。 ```Dockerfile -FROM openeuler-22.03-lts - +FROM openeuler-22.03-lts-sp1 RUN yum -y install openssh-clients util-linux - ADD ./your-sysmaster.rpm /home RUN rpm -ivh /home/your-sysmaster.rpm - COPY ./hostshell /usr/bin/ COPY ./set-ssh-pub-key.sh /usr/local/bin COPY ./set-ssh-pub-key.service /usr/lib/sysmaster - EXPOSE 22 RUN sed -i 's/sysinit.target/sysinit.target;sshd.service;set-ssh-pub-key.service/g' /usr/lib/sysmaster/basic.target - CMD ["/usr/lib/sysmaster/init"] ``` @@ -546,7 +634,9 @@ bash -x kbimg.sh create admin-image -f admin-container/Dockerfile -d your_imageR docker push your_imageRepository/admin_imageName:version ``` -在master节点上部署Admin容器，需要提供ssh公钥来免密登录，**修改**并应用如下示例yaml文件: +### admin容器部署 + +在master节点上部署Admin容器，需要提供ssh公钥来免密登录，修改并应用如下示例yaml文件: ```yaml apiVersion: v1 @@ -554,8 +644,7 @@ kind: Secret metadata: name: root-secret data: - # base64 encode your pub key in one line - ssh-pub-key: your-ssh-pub-key + ssh-pub-key: --- apiVersion: apps/v1 kind: Deployment @@ -563,7 +652,7 @@ metadata: name: admin-container-sysmaster namespace: default labels: - control-plane: admin-container-sysmaster + control-plane: admin-container-sysmaster spec: selector: matchLabels: @@ -576,24 +665,24 @@ spec: spec: hostPID: true containers: - - name: admin-container-sysmaster - image: your_imageRepository/admin_imageName:version - imagePullPolicy: Always - securityContext: - privileged: true - ports: - - containerPort: 22 - # sysmaster要求 - env: - - name: container - value: containerd - volumeMounts: - # name 必须与下面的卷名匹配 - - name: secret-volume - # mountPath必须为/etc/secret-volume - mountPath: /etc/secret-volume - readOnly: true - nodeName: your-worker-node-name + - name: admin-container-sysmaster + image: + imagePullPolicy: Always + securityContext: + privileged: true + ports: + - containerPort: 22 + # sysmaster要求 + env: + - name: container + value: containerd + volumeMounts: + # name 必须与下面的卷名匹配 + - name: secret-volume + # mountPath必须为/etc/secret-volume + mountPath: /etc/secret-volume + readOnly: true + nodeName: volumes: - name: secret-volume secret: @@ -610,11 +699,13 @@ spec: ports: - port: 22 targetPort: 22 - nodePort: your-exposed-port + nodePort: selector: - control-plane: admin-container-sysmaster + control-plane: admin-container-sysmaster ``` +### admin容器使用 + ssh到Admin容器，然后执行hostshell命令进入host root shell, 如： ```shell @@ -622,7 +713,7 @@ ssh -p your-exposed-port root@your.worker.node.ip hostshell ``` -##### hostshell +#### hostshell说明 为了保证KubeOS的轻便性，许多工具或命令没有安装在KubeOS内。因此，用户可以在制作Admin容器时，将期望使用的二进制文件放在容器内的如/usr/bin目录下。hostshell工具在执行时会将容器下的/usr/bin, /usr/sbin, /usr/local/bin, /usr/local/sbin路径添加到host root shell的环境变量。 @@ -637,7 +728,7 @@ hostshell #### kernel Settings -* kenerl.sysctl: 临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）, 示例如下: +* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: ```yaml configs: @@ -668,23 +759,38 @@ hostshell * grub.cmdline: 设置grub.cfg文件中的内核引导参数，该行参数在grub.cfg文件中类似如下示例： ```shell - linux /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 - ``` + linux /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + ``` * KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置： + * grub.cmdline.current：对当前分区的启动项参数进行配置。 * grub.cmdline.next：对下一分区的启动项参数进行配置。 + * 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。 + * grub.cmdline.current/next支持“key=value”（value不能为空），也支持单key。若value中有“=”，例如“root=UUID=some-uuid”，key应设置为第一个“=”前的所有字符，value为第一个“=”后的所有字符。 配置方法示例如下： + ```yaml configs: - - model: grub.cmdline.current - contents: - - key: selinux - value: 0 - - key: root - value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 - - key: panic - value: 3 - operation: delete + - model: grub.cmdline.current + contents: + - key: selinux + value: "0" + - key: root + value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 + - key: panic + value: "3" + operation: delete + - key: crash_kexec_post_notifiers + - model: grub.cmdline.next + contents: + - key: selinux + value: "0" + - key: root + value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 + - key: panic + value: "3" + operation: delete + - key: crash_kexec_post_notifiers ``` diff --git "a/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" "b/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" index f2823c85..d54cc837 100644 --- "a/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" +++ "b/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" @@ -1,16 +1,16 @@ -# 容器OS镜像制作指导# +# 容器OS镜像制作指导 -## 简介 ## +## 简介 kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制作KubeOS 容器，虚拟机和物理机镜像 -## 命令介绍 ## +## 命令介绍 -### 命令格式 ### +### 命令格式 **bash kbimg.sh** \[ --help | -h \] create \[ COMMANDS \] \[ OPTIONS \] -### 参数说明 ### +### 参数说明 * COMMANDS @@ -20,8 +20,6 @@ kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制 | vm-image | 生成用于部署和升级的虚拟机镜像 | | pxe-image | 生成物理机安装所需的镜像及文件 | - - * OPTIONS | 参数 | 描述 | @@ -34,31 +32,33 @@ kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制 | -l | 如果指定参数，则镜像为legacy引导，不指定默认是UEFI引导 | | -h --help | 查看帮助信息 | +## 使用说明 - -## 使用说明 ## - -#### 注意事项 ### +### 注意事项 * kbimg.sh 执行需要 root 权限 * 当前仅支持 x86和 AArch64 架构使用 * 容器 OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 -### KubeOS OCI 镜像制作 ### +### KubeOS OCI 镜像制作 -#### 注意事项 #### +#### 注意事项 * 制作的 OCI 镜像仅用于后续的虚拟机/物理机镜像制作或升级使用，不支持启动容器 * 使用默认 rpmlist 进行容器OS镜像制作时所需磁盘空间至少为6G，如自已定义 rpmlist 可能会超过6G -#### 使用示例 #### +#### 使用示例 + * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件 + ```shell cd /opt/kubeOS/scripts touch resolv.conf vim resolv.conf ``` + * 制作KubeOS容器镜像 + ``` shell cd /opt/kubeOS/scripts bash kbimg.sh create upgrade-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' -d your_imageRepository/imageName:version @@ -70,25 +70,28 @@ bash kbimg.sh create upgrade-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1 docker images ``` -### KubeOS 虚拟机镜像制作 ### +### KubeOS 虚拟机镜像制作 -#### 注意事项 #### +#### 注意事项 * 如使用 docker 镜像制作请先拉取相应镜像或者先制作docker镜像，并保证 docker 镜像的安全性 * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机 * 容器 OS 目前不支持 x86 架构的虚拟机使用 legacy 启动模式启动 * 使用默认rpmlist进行容器OS镜像制作时所需磁盘空间至少为25G，如自已定义rpmlist可能会超过25G -#### 使用示例 #### +#### 使用示例 * 使用repo源制作 - * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件 + * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件 + ```shell cd /opt/kubeOS/scripts touch resolv.conf vim resolv.conf ``` - * KubeOS虚拟机镜像制作 + + * KubeOS虚拟机镜像制作 + ``` shell cd /opt/kubeOS/scripts bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' @@ -100,15 +103,15 @@ docker images cd /opt/kubeOS/scripts bash kbimg.sh create vm-image -d your_imageRepository/imageName:version ``` -* 结果说明 - 容器 OS 镜像制作完成后，会在 /opt/kubeOS/scripts 目录下生成： - * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。 - * update.img: 用于升级的根文件系统分区镜像 +* 结果说明 + 容器 OS 镜像制作完成后，会在 /opt/kubeOS/scripts 目录下生成： + * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。 + * update.img: 用于升级的根文件系统分区镜像 -### KubeOS 物理机安装所需镜像及文件制作 ### +### KubeOS 物理机安装所需镜像及文件制作 -#### 注意事项 #### +#### 注意事项 * 如使用 docker 镜像制作请先拉取相应镜像或者先制作 docker 镜像，并保证 docker 镜像的安全性 * 制作出来的容器 OS 物理安装所需的镜像目前只能用于 CPU 架构为 x86 和 AArch64 的物理机安装 @@ -116,7 +119,8 @@ docker images * 不支持多个磁盘都安装KubeOS，可能会造成启动失败或挂载紊乱 * 容器OS 目前不支持 x86 架构的物理机使用 legacy 启动模式启动 * 使用默认rpmlist进行镜像制作时所需磁盘空间至少为5G，如自已定义 rpmlist 可能会超过5G -#### 使用示例 #### + +#### 使用示例 * 首先需要修改```00bootup/Global.cfg```的配置，对相关参数进行配置，参数均为必填，ip目前仅支持ipv4，配置示例如下 @@ -138,25 +142,344 @@ docker images ``` * 使用 repo 源制作 - * 如需进行DNS配置，请在```scripts```目录下自定义```resolv.conf```文件 + * 如需进行DNS配置，请在```scripts```目录下自定义```resolv.conf```文件 + ```shell cd /opt/kubeOS/scripts touch resolv.conf vim resolv.conf ``` - * KubeOS物理机安装所需镜像制作 - ``` + + * KubeOS物理机安装所需镜像制作 + + ```shell cd /opt/kubeOS/scripts bash kbimg.sh create pxe-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' ``` * 使用 docker 镜像制作 + ``` shell cd /opt/kubeOS/scripts bash kbimg.sh create pxe-image -d your_imageRepository/imageName:version ``` * 结果说明 + * initramfs.img: 用于pxe启动用的 initramfs 镜像 + * kubeos.tar: pxe安装所用的 OS + + +# KubeOS-Rust 镜像制作说明 + +## 简介 + +KubeOS 虚拟机镜像制作的 Rust 二进制版本 + +## 命令介绍 + +### 命令格式 + +**.../kbimg** \[ --config | -c \] \ + +## 配置文件说明 + +* from_repo: 从 repo 创建 OCI 镜像、虚拟机镜像或物理机镜像 + + | 参数 | 描述 | + | --- | --- | + | agent_path | os-agent 二进制的路径 | + | image_type | upgrade: 用于安装和升级的 OCI 镜像格式的 KubeOS 镜像; vm-repo: 用于部署和升级的虚拟机镜像; pxe-repo: 物理机安装所需的镜像及文件 | + | legacy_bios | 镜像为 legacy 引导或 UEFI 引导 | + | repo_path | repo 文件的路径，repo 文件中配置制作镜像所需要的 yum 源 | + | root_passwd | KubeOS 镜像 root 用户密码，加密后的带盐值的密码，可以用 openssl、kiwi 命令生成 | + | version | 制作出来的 KubeOS 镜像的版本 | + | rpmlist | 镜像所需的 rpm 包 | + | docker_img | 生成或者使用的 docker 镜像 | + +* from_docker: 从 docker 镜像创建虚拟机镜像或物理机镜像 + + | 参数 | 描述 | + | --- | --- | + | docker_img | 生成或者使用的 docker 镜像 | + | image_type | vm-docker: 用于部署和升级的虚拟机镜像; pxe-docker: 物理机安装所需的镜像及文件 | + +* admin_container: + + | 参数 | 描述 | + | --- | --- | + | dockerfile | dockerfile 路径 | + | docker_img | 生成或者使用的 docker 镜像 | + +* [OPTIONAL] users: 添加用户 + + | 参数 | 描述 | + | --- | --- | + | groups | [OPTIONAL] 用户组 (第一个为主组，其他为附加组) | + | name | 用户名 | + | passwd | 密码 | + | sudo | [OPTIONAL] 用户是否具有 sudo 权限 | + +* [OPTIONAL] copy_files: 拷贝文件到指定目录 + + | 参数 | 描述 | + | --- | --- | + | dst | 目标目录 | + | src | 源文件路径 | + +* [OPTIONAL] grub: grub配置 + + | 参数 | 描述 | + | --- | --- | + | passwd | [OPTIONAL] grub 密码 | + +* [OPTIONAL] systemd_service: 新增 systemd 服务 + + | 参数 | 描述 | + | --- | --- | + | name | systemd 服务名 | + +* [OPTIONAL] chroot_script: 自定义 chroot 脚本 + + | 参数 | 描述 | + | --- | --- | + | path | 脚本路径 | + +* [OPTIONAL] disk_partition: 自定义分区大小和镜像大小 + + | 参数 | 描述 | + | --- | --- | + | first | 引导分区大小 | + | second | ROOT-A 分区大小 | + | third | ROOT-B 分区大小 | + | img_size | 镜像大小 | + +* [OPTIONAL] persist_mkdir: persist 分区新建目录 + + | 参数 | 描述 | + | --- | --- | + | name | 目录名 | + +## 使用说明 + +#### 注意事项 + +* 新增 systemd 服务需要将对应的 .service 文件或 .mount 文件拷贝至镜像```/usr/lib/systemd/system```目录 + + ```toml + [[copy_files]] + dst = "/usr/lib/systemd/system" + src = ".../containerd.service" + + [systemd_service] + name = ["containerd"] + ``` + + * 如需挂载数据盘，请先自定义```persist-data.mount```文件，并启用```copy_files```和```systemd_service```字段设置启动时挂载，启用```persist_mkdir```字段创建挂载点 + * .mount文件名由挂载点路径生成，将斜杠替换为连字符 + * 请先在磁盘映像文件上创建ext4文件系统 - * initramfs.img: 用于pxe启动用的 initramfs 镜像 - * kubeos.tar: pxe安装所用的 OS + ``` + # persist-data.mount + [Unit] + Description=Mount Disk + Documentation=man:systemd.mount(5) + + [Mount] + What=/dev/vdb + Where=/persist/data + Type=ext4 + Options=defaults,noatime + + [Install] + WantedBy=local-fs.target + ``` + + ```toml + [[copy_files]] + dst = "/usr/lib/systemd/system" + src = ".../persist-data.mount" + + [systemd_service] + name = ["persist-data.mount"] + + [persist_mkdir] + name = ["data"] + ``` + + * 如需配置逻辑卷，请先自定义```volume.service```文件，并启用```copy_files```和```systemd_service```设置启动时配置逻辑卷，启用```persist_mkdir```字段创建挂载点 + + ``` + # volume.service + [Unit] + Description=Mount Logical Volume + After=local-fs.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=pvcreate /dev/vdb + ExecStart=pvcreate /dev/vdc + ExecStart=vgcreate my_vg /dev/vdb /dev/vdc + ExecStart=lvcreate -L 15G -n my_lv my_vg + ExecStart=mkfs.ext4 /dev/my_vg/my_lv + ExecStart=mount /dev/my_vg/my_lv /persist/lv_data + + [Install] + WantedBy=local-fs.target + ``` + + ```toml + [[copy_files]] + dst = "/usr/lib/systemd/system" + src = ".../volume.service" + + [systemd_service] + name = ["volume"] + + [persist_mkdir] + name = ["lv_data"] + ``` + +## 使用示例 + +### KubeOS OCI 镜像制作 + +* 如需进行DNS配置，请先自定义```resolv.conf```文件，并启用```copy_files```字段将配置文件拷贝到```/etc```目录 + + ```shell + touch \/resolv.conf + vim \resolv.conf + ``` + + ```toml + [[copy_files]] + dst = "/etc" + src = "" + ``` + +* 制作KubeOS容器镜像 + + ```toml + [from_repo] + agent_path = "/bin/os-agent" + image_type = "upgrade" + legacy_bios = false + repo_path = "xxx.repo" + root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0" + version = "v1" + docker_img = "your_imageRepository/imageName:version" + rpmlist = [ + # your rpms + ] + ``` + +* 制作完成后查看制作出来的KubeOS容器镜像 + ``` shell + docker images + ``` + +### KubeOS 虚拟机镜像制作 + +* 使用repo源制作 + + * 如需进行DNS配置，请先自定义```resolv.conf```文件，并启用**copy_files**字段将配置文件拷贝到```/etc```目录 + + ```shell + touch \/resolv.conf + vim \resolv.conf + ``` + + ```toml + [[copy_files]] + dst = "/etc" + src = "" + ``` + + * KubeOS虚拟机镜像制作 + + ```toml + [from_repo] + agent_path = "/bin/os-agent" + image_type = "vm-repo" + legacy_bios = false + repo_path = "xxx.repo" + root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0" + version = "v1" + rpmlist = [ + # your rpms + ] + ``` + +* 使用docker镜像制作 + + ```toml + [from_dockerimg] + docker_img = "your_imageRepository/imageName:version" + image_type = "vm-docker" + ``` + +* 结果说明 + 容器 OS 镜像制作完成后，会在 ./scripts-auto 目录下生成 + * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。 + * update.img: 用于升级的根文件系统分区镜像 + +### KubeOS 物理机安装所需镜像及文件制作 + +* 首先需要修改```kbimg.toml```中```pxe_config```的配置，对相关参数进行配置，参数均为必填，ip目前仅支持ipv4，配置示例如下 + + ```toml + [pxe_config] + # rootfs file name + rootfs_name = "kubeos.tar" + # select the target disk to install kubeOS + disk = "/dev/sda" + # pxe server ip address where stores the rootfs on the http server + server_ip = "192.168.1.50" + # target machine ip + local_ip = "192.168.1.100" + # target machine route + route_ip = "192.168.1.1" + # target machine netmask + netmask = "255.255.255.0" + # target machine netDevice name + net_name = "eth0" + ``` + +* 使用 repo 源制作 + * 如需进行DNS配置，请先自定义```resolv.conf```文件，并启用```copy_files```字段将配置文件拷贝到```/etc```目录 + + ```shell + touch \/resolv.conf + vim \resolv.conf + ``` + + ```toml + [[copy_files]] + dst = "/etc" + src = "" + ``` + + * KubeOS物理机安装所需镜像制作 + ```toml + [from_repo] + agent_path = "/bin/os-agent" + image_type = "pxe-repo" + legacy_bios = true + repo_path = "xxx.repo" + root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0" + version = "v1" + rpmlist = [ + # your rpms + ] + ``` + +* 使用 docker 镜像制作 + ```toml + [from_dockerimg] + docker_img = "your_imageRepository/imageName:version" + image_type = "vm-docker" + ``` + +* 结果说明 + * initramfs.img: 用于pxe启动用的 initramfs 镜像 + * kubeos.tar: pxe安装所用的 OS diff --git a/go.mod b/go.mod index 69a86dd7..255344ce 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,6 @@ require ( github.com/onsi/gomega v1.20.0 github.com/shirou/gopsutil/v3 v3.23.7 github.com/sirupsen/logrus v1.8.1 - github.com/spf13/cobra v1.4.0 - github.com/spf13/viper v1.8.1 github.com/sykesm/zap-logfmt v0.0.4 go.uber.org/zap v1.19.1 google.golang.org/grpc v1.49.0 diff --git a/go.sum b/go.sum index 33c37080..e9e7e6ab 100644 --- a/go.sum +++ b/go.sum @@ -554,8 +554,6 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/sykesm/zap-logfmt v0.0.4 h1:U2WzRvmIWG1wDLCFY3sz8UeEmsdHQjHFNlIdmroVFaI= github.com/sykesm/zap-logfmt v0.0.4/go.mod h1:AuBd9xQjAe3URrWT1BBDk2v2onAZHkZkWRMiYZXiZWA= -github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/pkg/values/values.go b/pkg/values/values.go index e2c03540..9e213a83 100644 --- a/pkg/values/values.go +++ b/pkg/values/values.go @@ -30,13 +30,18 @@ const ( LabelNodeSelector = "upgrade.openeuler.org/node-selector" // LabelConfiguring is the key of the configuring label for nodes LabelConfiguring = "upgrade.openeuler.org/configuring" - defaultPeriod = 15 * time.Second + // LabelSerial is the key of the serial label for nodes + LabelSerial = "upgrade.openeuler.org/serial" + + defaultPeriod = 15 * time.Second // OsiStatusName is param name of nodeStatus in osInstance OsiStatusName = "nodestatus" // UpgradeConfigName is param name of UpgradeConfig UpgradeConfigName = "UpgradeConfig" // SysConfigName is param name of SysConfig SysConfigName = "SysConfig" + // OsiNamespace is the namespace of osinstance + OsiNamespace = "default" ) // NodeStatus defines state of nodes diff --git a/scripts/bootloader.sh b/scripts/bootloader.sh index 75096a38..df4be329 100644 --- a/scripts/bootloader.sh +++ b/scripts/bootloader.sh @@ -19,7 +19,7 @@ function install_grub2_x86 () cp -r /usr/lib/grub/x86_64-efi boot/efi/EFI/openEuler eval "grub2-mkimage -d /usr/lib/grub/x86_64-efi -O x86_64-efi --output=/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" - mkdir -p /boot/EFI/BOOT/ + mkdir -p /boot/efi/EFI/BOOT/ cp -f /boot/efi/EFI/openEuler/grubx64.efi /boot/efi/EFI/BOOT/BOOTX64.EFI fi } @@ -29,7 +29,7 @@ function install_grub2_efi () cp -r /usr/lib/grub/arm64-efi /boot/efi/EFI/openEuler/ eval "grub2-mkimage -d /usr/lib/grub/arm64-efi -O arm64-efi --output=/boot/efi/EFI/openEuler/grubaa64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" - mkdir -p /boot/EFI/BOOT/ + mkdir -p /boot/efi/EFI/BOOT/ cp -f /boot/efi/EFI/openEuler/grubaa64.efi /boot/efi/EFI/BOOT/BOOTAA64.EFI } diff --git a/vendor/modules.txt b/vendor/modules.txt index 5b9304ea..ee0fd673 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -295,9 +295,6 @@ github.com/spf13/viper ## explicit; go 1.20 github.com/stretchr/testify/assert github.com/stretchr/testify/require -# github.com/subosito/gotenv v1.2.0 -## explicit -github.com/subosito/gotenv # github.com/sykesm/zap-logfmt v0.0.4 ## explicit; go 1.13 github.com/sykesm/zap-logfmt -- Gitee From b8a5b7aa499507f708743d66e88f0fcb6424e54b Mon Sep 17 00:00:00 2001 From: YouMeiYouMaoTai <15335885760@163.com> Date: Thu, 26 Sep 2024 16:46:41 +0800 Subject: [PATCH 45/46] feat:the rust version of the os-operator --- KubeOS-Rust/Cargo.lock | 31 + KubeOS-Rust/Cargo.toml | 2 +- KubeOS-Rust/operator/Cargo.toml | 47 + .../operator/src/controller/apiclient.rs | 110 ++ .../operator/src/controller/apiserver_mock.rs | 995 ++++++++++++++++++ .../operator/src/controller/controller.rs | 696 ++++++++++++ KubeOS-Rust/operator/src/controller/crd.rs | 79 ++ KubeOS-Rust/operator/src/controller/mod.rs | 24 + KubeOS-Rust/operator/src/controller/values.rs | 43 + KubeOS-Rust/operator/src/main.rs | 74 ++ Makefile | 3 + go.mod | 2 + go.sum | 2 + vendor/modules.txt | 3 + 14 files changed, 2110 insertions(+), 1 deletion(-) create mode 100644 KubeOS-Rust/operator/Cargo.toml create mode 100644 KubeOS-Rust/operator/src/controller/apiclient.rs create mode 100644 KubeOS-Rust/operator/src/controller/apiserver_mock.rs create mode 100644 KubeOS-Rust/operator/src/controller/controller.rs create mode 100644 KubeOS-Rust/operator/src/controller/crd.rs create mode 100644 KubeOS-Rust/operator/src/controller/mod.rs create mode 100644 KubeOS-Rust/operator/src/controller/values.rs create mode 100644 KubeOS-Rust/operator/src/main.rs diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 5b97095b..89e56ec2 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -1520,6 +1520,37 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "operator" +version = "1.0.6" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-trait", + "cli", + "env_logger", + "futures", + "h2 0.3.16", + "http 0.2.9", + "hyper 0.14.25", + "k8s-openapi", + "kube", + "log", + "manager", + "mockall", + "regex", + "reqwest", + "schemars", + "serde", + "serde_json", + "socket2 0.4.9", + "thiserror", + "thread_local", + "tokio", + "tokio-retry", + "tower-test", +] + [[package]] name = "ordered-float" version = "2.10.1" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml index 2886023f..93379823 100644 --- a/KubeOS-Rust/Cargo.toml +++ b/KubeOS-Rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["agent", "cli", "kbimg", "manager", "proxy"] +members = ["agent", "cli", "kbimg", "manager", "operator", "proxy"] resolver = "2" [profile.release] diff --git a/KubeOS-Rust/operator/Cargo.toml b/KubeOS-Rust/operator/Cargo.toml new file mode 100644 index 00000000..91cca265 --- /dev/null +++ b/KubeOS-Rust/operator/Cargo.toml @@ -0,0 +1,47 @@ +[package] +description = "KubeOS os-operator" +edition = "2021" +license = "MulanPSL-2.0" +name = "operator" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +name = "operator" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.44" +async-trait = "0.1" +cli = { version = "1.0.6", path = "../cli" } +env_logger = "0.9.0" +futures = "0.3.17" +h2 = "=0.3.16" +k8s-openapi = { version = "0.13.1", features = ["v1_22"] } +kube = { version = "0.66.0", features = ["derive", "runtime"] } +log = "=0.4.15" +manager = { version = "1.0.6", path = "../manager" } +regex = "=1.7.3" +reqwest = { version = "=0.12.2", default-features = false, features = [ + "json", +] } +schemars = "=0.8.10" +serde = { version = "1.0.130", features = ["derive"] } +serde_json = "1.0.68" +socket2 = "=0.4.9" +thiserror = "1.0.29" +thread_local = "=1.1.4" +tokio = { version = "=1.28.0", default-features = false, features = [ + "macros", + "rt-multi-thread", +] } +tokio-retry = "0.3" + +[dev-dependencies] +assert-json-diff = "2.0.2" +http = "0.2.9" +hyper = "0.14.25" +tower-test = "0.4.0" +mockall = { version = "=0.11.3" } +regex = "1" diff --git a/KubeOS-Rust/operator/src/controller/apiclient.rs b/KubeOS-Rust/operator/src/controller/apiclient.rs new file mode 100644 index 00000000..3642b475 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/apiclient.rs @@ -0,0 +1,110 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + +use anyhow::Result; +use apiclient_error::Error; +use async_trait::async_trait; +use kube::{ + api::{Api, Patch, PatchParams}, + Client, +}; +use serde::{Deserialize, Serialize}; +use super::{ + crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}, + values::{NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}, +}; + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceSpecPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + spec: OSInstanceSpec, +} + +impl Default for OSInstanceSpecPatch { + fn default() -> Self { + OSInstanceSpecPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceStatusPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + status: Option, +} + +impl Default for OSInstanceStatusPatch { + fn default() -> Self { + OSInstanceStatusPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }), + } + } +} + +#[derive(Clone)] +pub struct ControllerClient { + pub client: Client, +} + +impl ControllerClient { + pub fn new(client: Client) -> Self { + ControllerClient { client } + } +} + +#[async_trait] +pub trait ApplyApi: Clone + Sized + Send + Sync { + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error>; +} + +#[async_trait] +impl ApplyApi for ControllerClient { + + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() }; + osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?; + Ok(()) + } + +} +pub mod apiclient_error { + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} diff --git a/KubeOS-Rust/operator/src/controller/apiserver_mock.rs b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs new file mode 100644 index 00000000..21abd405 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs @@ -0,0 +1,995 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{borrow::BorrowMut, cell::{RefCell, RefMut}, clone, collections::BTreeMap, default}; +use regex::Regex; +use anyhow::Result; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use http::{status, Request, Response}; +use hyper::{body::to_bytes, Body}; +use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod}; +use kube::{ + api::ObjectMeta, + core::{ErrorResponse, ListMeta, ObjectList}, + Client as KubeClient, Resource, ResourceExt, +}; +use log::debug; +use mockall::mock; +use serde_json::json; + +use self::mock_error::Error; +use super::{ + crd::{Configs, OSInstanceStatus}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_CONFIG}, +}; +use crate::controller::{ + apiclient::{ApplyApi, ControllerClient}, + crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS}, + values::{LABEL_MASTER, LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, + OperatorController, +}; + +type ApiServerHandle = tower_test::mock::Handle, Response>; +pub struct ApiServerVerifier(ApiServerHandle); + + +#[derive(Clone, Debug, Default)] +pub struct K8sResources{ + pub node_list: Vec, + pub osi_list: Vec, +} + +pub enum Testcases { + Rollback(K8sResources), + ConfigNormal(K8sResources), + SkipNoOsiNode(K8sResources), + ExchangeCurrentAndNext(K8sResources), + GetConfigOSInstances(String), + CheckUpgrading(String), + GetIdleOSInstances(String), + +} + +pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) { + tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("timeout on mock apiserver") + .expect("scenario succeeded") +} + +impl ApiServerVerifier { + pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::Rollback(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + }, + Testcases::ConfigNormal(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_config_osi_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_idle_osi_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_patch_spec_config(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_spec_config(k8s_resc.clone()) + .await + }, + Testcases::SkipNoOsiNode(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + }, + Testcases::ExchangeCurrentAndNext(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + }, + _ => { + Err(Error::ArgumentError) + } + } + .expect("Case completed without errors"); + }) + } + + pub fn test_function(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::GetConfigOSInstances(error) => { + self.handler_config_osi_list_get_error(error) + .await + }, + Testcases::CheckUpgrading(error) => { + self.handler_upgrading_node_list_get_error(error) + .await + }, + Testcases::GetIdleOSInstances(error) => { + self.handler_idle_osi_list_get_error(error) + .await + }, + _ => { + Err(Error::ArgumentError) + } + } + .expect("Case completed without errors"); + }) + } + + // 获取所有的 worker 节点，对应于reconcile的第一个 get_nodes 函数 + async fn handler_worker_node_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=%21node-role.kubernetes.io%2Fcontrol-plane&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中所有的 worker 节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if !node.labels().contains_key(LABEL_MASTER){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_worker_node_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 upgrading 的节点 + async fn handler_upgrading_node_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中标签为正在升级的节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if node.labels().contains_key(LABEL_UPGRADING){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_upgrading_node_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取所有的非 upgrading 的 worker 节点 + async fn handler_worker_and_no_upgrade_noding_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + + let remove_limit = |input: &str| -> String { + let re = Regex::new(r"limit=\d+").unwrap(); + re.replace_all(input, "").to_string() + }; + + assert_eq!( + remove_limit(request.uri().to_string().as_str()), + "/api/v1/nodes?&labelSelector=%21upgrade.openeuler.org%2Fupgrading%2C%21node-role.kubernetes.io%2Fcontrol-plane&"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中所有的非 upgrading 的 worker 节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if !node.labels().contains_key(LABEL_UPGRADING) && !node.labels().contains_key(LABEL_MASTER){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_worker_and_no_upgrade_noding_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_osinstance_get_by_node_name(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_get_by_node_name: req_node_name: {:?}", req_node_name); + + let response = serde_json::to_vec(&osinstance).unwrap(); + dbg!("handler_osinstance_get_by_node_name"); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_upgrade(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_nodestatus_upgrade: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_upgrade"); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_exchange(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_nodestatus_exchange: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + + let sysconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + + let upgradeconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + + assert_eq!(spec.sysconfigs.clone(), sysconfigs); + assert_eq!(spec.upgradeconfigs.clone(), upgradeconfigs); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_exchange"); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + // 通过节点名称获取对应节点 + async fn handler_replace_node_by_name(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PUT); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get node and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut node = Node{..Default::default()}; + let mut boolean_get_node = false; + for node_iter in k8s_resc.node_list.clone() { + if node_iter.name() == req_node_name { + boolean_get_node = true; + node = node_iter.clone(); + break; + } + } + assert!(boolean_get_node); + assert_eq!(request.extensions().get(), Some(&"replace")); + + println!("handler_replace_node_by_name: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let metadata_json = body_json.get("metadata").expect("metadata object").clone(); + let metadata: ObjectMeta = serde_json::from_value(metadata_json).expect("valid metadata"); + assert!(metadata.labels.unwrap().contains_key(LABEL_UPGRADING)); + + // 修改 node 并传出 + node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string()); + + dbg!("handler_replace_node_by_name"); + let response = serde_json::to_vec(&node).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 config 的节点上的 osi + async fn handler_config_osi_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出 + let mut osis = vec![]; + for osi in k8s_resc.osi_list.clone() { + if osi.spec.nodestatus == NODE_STATUS_CONFIG{ + osis.push(osi.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: osis, + }; + + dbg!("handler_config_osi_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 config 的节点上的 osi + async fn handler_idle_osi_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出 + let mut osis = vec![]; + for osi in k8s_resc.osi_list.clone() { + if osi.spec.nodestatus == NODE_STATUS_IDLE{ + osis.push(osi.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: osis, + }; + + dbg!("handler_idle_osi_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_osinstance_patch_spec_config(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_osi_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_osi_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_spec_config: req_osi_name: {:?}", req_osi_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_CONFIG.to_string()); + + let sysconfig = Some( + Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl")), + configpath: Some(String::from("")), + contents: + Some(vec![ + Content { + key: Some(String::from("key1")), + value: Some(String::from("a")), + operation: Some(String::from("")), + }, + Content { + key: Some(String::from("key2")), + value: Some(String::from("b")), + operation: Some(String::from("")), + }, + ]), + }]), + } + ); + assert_eq!( + spec.sysconfigs.clone(), + sysconfig + ); + + dbg!("handler_osinstance_patch_spec_config"); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance.spec.sysconfigs = sysconfig; + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_config_osi_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_config_osi_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "NotFound".to_string(), + code: 404, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_upgrading_node_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_upgrading_node_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "Invalid".to_string(), + code: 400, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(400).body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_idle_osi_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_idle_osi_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "NotFound".to_string(), + code: 404, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + + Ok(self) + } + +} + +pub mod mock_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + + #[error("Parameters other than expected were entered")] + ArgumentError, + } +} + + +impl OperatorController { + pub fn test() -> (OperatorController, ApiServerVerifier) { + let (mock_service, handle) = tower_test::mock::pair::, Response>(); + let mock_k8s_client = KubeClient::new(mock_service, "default"); + let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); + let operator_controller: OperatorController = + OperatorController::new(mock_k8s_client, mock_api_client); + (operator_controller, ApiServerVerifier(handle)) + } +} + +impl OSInstance { + pub fn set_osi_default(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }, + status: Some(OSInstanceStatus { + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }), + } + } +} + +impl OS { + pub fn set_os_default() -> Self { + let mut os = OS::new("test", OSSpec::default()); + os.meta_mut().namespace = Some("default".into()); + os + } + + pub fn set_os_rollback_osversion_v1_upgradecon_v1() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string(); + os + } + + pub fn set_os_syscon_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = OPERATION_TYPE_CONFIG.to_string(); + os.spec.sysconfigs = Some( + Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("key1")), + value: Some(String::from("a")), + operation: Some(String::from("")), + }, + Content { + key: Some(String::from("key2")), + value: Some(String::from("b")), + operation: Some(String::from("")), + }, + ]), + }]), + } + ); + os + } + + pub fn set_os_skip_osversion_v2_upgradecon_v1() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os + } + + pub fn set_os_exchange_current_and_next() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + let sysconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + os.spec.sysconfigs = sysconfigs.clone(); + os.spec.upgradeconfigs = sysconfigs.clone(); + + os + } + +} + +impl K8sResources { + pub fn set_rollback_nodes_v2_and_osi_v1() -> Self { + // 创建 node1 和 node2 + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v2".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let node2 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node2".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v2".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default"); + let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default"); + + let node_list = Vec::from([node1, node2]); + let osi_list = Vec::from([osi1, osi2]); + + K8sResources{ + node_list, + osi_list + } + } + + pub fn set_nodes_v1_and_osi_v1() -> Self { + // 创建 node1 和 node2 + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let node2 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node2".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default"); + let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default"); + + let node_list = Vec::from([node1, node2]); + let osi_list = Vec::from([osi1, osi2]); + + K8sResources{ + node_list, + osi_list + } + } + + pub fn set_skip_nodes_and_osi() -> Self { + // 创建 node1 并且不设置 osi + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let node_list = Vec::from([node1]); + let osi_list = Vec::new(); + + K8sResources{ + node_list, + osi_list + } + } + +} + +impl Default for OSSpec { + fn default() -> Self { + OSSpec { + osversion: String::from("KubeOS v1"), + maxunavailable: 3, + checksum: String::from("test"), + imagetype: String::from("containerd"), + containerimage: String::from("test"), + opstype: String::from("upgrade"), + evictpodforce: true, + imageurl: String::from(""), + flagsafe: true, + mtls: false, + cacert: Some(String::from("")), + clientcert: Some(String::from("")), + clientkey: Some(String::from("")), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + } + } +} diff --git a/KubeOS-Rust/operator/src/controller/controller.rs b/KubeOS-Rust/operator/src/controller/controller.rs new file mode 100644 index 00000000..2beb8945 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/controller.rs @@ -0,0 +1,696 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +* KubeOS is licensed under the Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +* PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + + +use anyhow::Result; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::{Api, ListParams, ObjectList, PostParams}, + core::ErrorResponse, + runtime::controller::{Context, ReconcilerAction}, + Client, ResourceExt, +}; +use log::{debug, error}; +use reconciler_error::Error; + +use crate::controller::values::NODE_STATUS_UPGRADE; + +use super::{ + apiclient::ApplyApi, + crd::{Configs, OSInstance, OS}, + values::{ + LABEL_MASTER, LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, + NO_REQUEUE, OPERATION_TYPE_CONFIG, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, + REQUEUE_ERROR, REQUEUE_NORMAL, SYS_CONFIG_NAME, UPGRADE_CONFIG_NAME + }, +}; + +#[derive(Clone)] +pub struct OperatorController { + k8s_client: Client, + controller_client: T, +} + +impl OperatorController { + pub fn new(k8s_client: Client, controller_client: T) -> Self { + OperatorController { + k8s_client, + controller_client, + } + } + + // async fn reconcile( + // &self, + // os: OS, + // ctx: Context>, + // ) -> Result { + // // k8s_client 变量不是 Option 类型，而是直接的 Client 类型，Rust 会确保不为空，不用检查 + + // // Kube-rs库中的Context类型已经处理了上下文的管理，也不需要初始化空的上下文，proxy组件的rust版本也没初始化 + + // // 调用外部的Reconcile函数 + // reconcile(os, ctx).await + // } + + // 获取 worker 节点数 + async fn get_and_update_os(&self, _namespace: &str) -> Result { + + // 创建一个筛选标签的 String 数组，只找 worker 节点 + let reqs = vec![ + format!("!{}", LABEL_MASTER), + ]; + + // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量 + let nodes_items = self.get_nodes( 0, reqs).await?; + + Ok(nodes_items.items.len() as i64) + } + + // 获取 worker 节点，传入 reqs 为 String 数组，表示多个筛选条件 + async fn get_nodes(&self, limit: i64, reqs: Vec) -> Result, Error> { + + let nodes_api: Api = Api::all(self.k8s_client.clone()); + + // 将多个标签筛选器组合成一个字符串，用逗号分隔 + let label_selector = reqs.join(","); + + // 设置 ListParams + let list_params = ListParams::default() + .labels(&label_selector) + .limit(limit as u32); + + let nodes = match nodes_api.list(&list_params).await { + Ok(nodes) => nodes, + Err(e) => { + log::error!("{:?} unable to list nodes with requirements", e); + return Err(Error::KubeClient { source: e }); + }, + }; + + Ok(nodes) + } + + // 获取可以进行升级操作的最大节点数量 + async fn check_upgrading(&self, _namespace: &str, max_unavailable: i64) -> Result { + + // 设置筛选标签，选择标签为正在升级的节点 + let reqs = vec![ + LABEL_UPGRADING.to_string(), + ]; + + // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量 + let nodes_items = self.get_nodes( 0, reqs).await?; + + Ok(max_unavailable - nodes_items.items.len() as i64) + } + + // 为指定数量的节点进行升级操作 + async fn assign_upgrade(&self, os: &OS, limit: i64, namespace: &str) -> Result { + + // 创建筛选标签，只找 worker 节点，并且标签 LabelUpgrading 不存在 + let reqs = vec![ + format!("!{}", LABEL_UPGRADING), + format!("!{}", LABEL_MASTER), + ]; + + // 获取符合标签要求的节点 + let mut nodes_items = self.get_nodes( limit + 1, reqs).await?; + + // 对选定的节点进行升级，返回升级成功的数量和可能的错误 + let count = self.upgrade_nodes(os, &mut nodes_items, limit, namespace).await?; + + Ok(count >= limit) + } + + async fn upgrade_nodes(&self, os: &OS, nodes: &mut ObjectList, limit: i64, namespace: &str) -> Result { + + let mut count = 0; + + for node in nodes.iter_mut() { + // 如果已经达到升级限制，则退出循环 + if count >= limit { + break + } + + let os_version_node = node.status.clone().unwrap().node_info.unwrap().os_image; + + debug!("node name: {}, os_version_node: {}, os_version: {}", node.name(), os_version_node, os.spec.osversion); + + // 检查 os 对象中的操作系统版本是否与节点的操作系统版本不同 + if os_version_node != os.spec.osversion { + + // 尝试获取该节点上的 os 实例 + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + match osi_api.get(&node.name().clone()).await { + Ok(mut osi) => { + // info!("osinstance is exist {:?}", osi.name()); + + debug!("osinstance is exist: \n {:?} \n", osi); + + match self.update_node_and_osins(os, node, &mut osi).await { + Ok(_) => { + count += 1; + }, + Err(_) => { + continue; + }, + } + }, + Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { + debug!("failed to get osInstance {}", &node.name().clone()); + + return Err(Error::KubeClient { + source: kube::Error::Api(ErrorResponse { + reason, + status: "".to_string(), + message: "".to_string(), + code: 0 + })}); + }, + Err(_) => continue, + } + } + + } + + Ok(count) + } + + // 升级节点以及节点上的 OSinstance + async fn update_node_and_osins(&self, os: &OS, node: &mut Node, osinstance: &mut OSInstance, ) -> Result<(), Error> { + debug!("start update_node_and_OSins"); + + // 检查os实例中的升级配置版本与os对象中的升级配置版本是否匹配。osi 字段未初始化时直接进行拷贝 + let mut copy_sign = true; + if let Some(upgradeconfigs) = osinstance.spec.upgradeconfigs.clone() { + if let Some(version) = upgradeconfigs.version { + if version == os.spec.upgradeconfigs.clone().unwrap().version.unwrap() { + copy_sign = false; + } + } + } + if copy_sign { + self.deep_copy_spec_configs(os, osinstance, UPGRADE_CONFIG_NAME.to_string()).await?; + assert!(osinstance.spec.upgradeconfigs.is_some()); + } + + copy_sign = true; + // 检查os实例中的系统配置版本与os对象中的系统配置版本是否匹配。osi 字段未初始化时直接进行拷贝 + if let Some(sysconfigs) = osinstance.spec.sysconfigs.clone() { + if let Some(version) = sysconfigs.version { + if version == os.spec.sysconfigs.clone().unwrap().version.unwrap() { + copy_sign = false; + } + } + } + if copy_sign { + self.deep_copy_spec_configs(os, osinstance, SYS_CONFIG_NAME.to_string()).await?; + assert!(osinstance.spec.sysconfigs.is_some()); + if let Some(sysconfigs) = osinstance.spec.sysconfigs.as_mut() { + if let Some(configs) = &mut sysconfigs.configs { + for config in configs { + if config.model.clone().unwrap() == "grub.cmdline.current" { + config.model = Some("grub.cmdline.next".to_string()); + } + else if config.model.clone().unwrap() == "grub.cmdline.next" { + config.model = Some("grub.cmdline.current".to_string()); + } + } + } + } + } + + // 更新os实例中的状态为升级完成状态 + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + + // 把对 osinstance 的更改从内存更新到 k8s 集群 + let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; + + node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string()); + + // 把对 node 的更改从内存更新到 k8s 集群 + let node_api: Api = Api::all(self.k8s_client.clone()); + node_api.replace(&node.name(), &PostParams::default(), &node).await?; + + Ok(()) + } + + // 深拷贝 + async fn deep_copy_spec_configs(&self, os: &OS, os_instance: &mut OSInstance, config_type: String) -> Result<(), Error> { + + match config_type.as_str() { + UPGRADE_CONFIG_NAME =>{ + + if let Ok(data) = serde_json::to_vec(&os.spec.upgradeconfigs){ + + if let Ok(upgradeconfigs) = serde_json::from_slice(&data) { + os_instance.spec.upgradeconfigs = Some(upgradeconfigs); + }else { + debug!("{} Deserialization failure", config_type); + return Err(Error::Operation { value: "Deserialization".to_string()}); + } + } + else { + debug!("{} Serialization failure", config_type); + return Err(Error::Operation { value: "Serialization".to_string()}); + } + + }, + SYS_CONFIG_NAME => { + + if let Ok(data) = serde_json::to_vec(&os.spec.sysconfigs){ + + if let Ok(sysconfigs) = serde_json::from_slice(&data) { + os_instance.spec.sysconfigs = Some(sysconfigs); + }else { + debug!("{} Deserialization failure", config_type); + return Err(Error::Operation { value: "Deserialization".to_string()}); + } + } + else { + debug!("{} Serialization failure", config_type); + return Err(Error::Operation { value: "Serialization".to_string()}); + } + + }, + _ => { + debug!("configType {} cannot be recognized", config_type); + return Err(Error::Operation { value: config_type.clone() }); + }, + } + + Ok(()) + } + + // 获取可以进行配置操作的节点数量，返回的是 instance 的列表 + async fn check_config(&self, namespace: &str, max_unavailable: i64) -> Result { + let osinstances = self.get_config_osinstances(namespace).await?; + + Ok(max_unavailable - osinstances.len() as i64) + } + + // 获取所在节点状态为配置的 osinstance列表 + async fn get_config_osinstances(&self, namespace: &str) -> Result, Error> { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + // 获取所有 OSInstance 资源 + let all_osinstances = osi_api.list(&ListParams::default()).await?; + + // 在客户端进行过滤，节点状态为 NODE_STATUS_CONFIG + let osinstances: Vec = all_osinstances + .items + .into_iter() + .filter(|osi| osi.spec.nodestatus == NODE_STATUS_CONFIG) + .collect(); + + debug!("config_osi count = {:?}", osinstances.len()); + + Ok(osinstances) + } + + // 为指定数量的节点进行配置操作 + async fn assign_config(&self, _os: &OS, sysconfigs: Configs, config_version: String, limit: i64, namespace: &str) -> Result { + + debug!("start assign_config"); + + let mut osinstances = self.get_idle_os_instances(namespace, limit + 1).await?; + + let mut count = 0; + // 遍历 osi 列表 + for osi in osinstances.iter_mut() { + if count > limit { + break; + } + + let mut config_sign = true; + if let Some(sysconfigs) = osi.spec.sysconfigs.clone() { + if let Some(version) = sysconfigs.version { + debug!("node name: {:?}, config_version_node: {:?}, config_version: {:?}", osi.name(), version, config_version); + if version == config_version { + config_sign = false; + } + } + } + + // 如果版本不同或 osi 未初始化，则将新的配置信息更新到实例中，并将节点状态标记为“配置完成”。 + if config_sign { + count += 1; + osi.spec.sysconfigs = Some(sysconfigs.clone()); + osi.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + + // 把对 osinstance 的更改从内存更新到 k8s 集群 + let namespace = osi.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + self.controller_client.update_osinstance_spec(&osi.name(), &namespace, &osi.spec).await?; + } + } + + Ok(count >= limit) + } + + // 获取所在节点状态为空闲的 osinstance列表 + async fn get_idle_os_instances(&self, namespace: &str, limit: i64) -> Result, Error> { + + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + // 获取所有 OSInstance 资源 + let all_osinstances: ObjectList = osi_api.list(&ListParams::default().limit(limit as u32)).await?; + + // 在客户端进行过滤，节点状态为 NODE_STATUS_IDLE + let osinstances: Vec = all_osinstances + .items + .into_iter() + .filter(|osi| osi.spec.nodestatus == NODE_STATUS_IDLE) + .collect(); + + Ok(osinstances) + } + + +} + +// 调用的函数的具体逻辑需要进一步完成 +pub async fn reconcile( + os: OS, + ctx: Context>, +) -> Result { + + // 初始化 operator_controller 和 os，从环境变量获取NODE_NAME + debug!("start reconcile"); + let operator_controller = ctx.get_ref(); + let os_cr: &OS = &os; + + // 从 os_cr 中获取命名空间，如果命名空间不存在则返回错误 + let namespace: String = os_cr + .namespace() + .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; + + debug!("namespace : {:?}", namespace); + + // 获取 worker 节点数 + let node_num = match operator_controller.get_and_update_os(&namespace).await { + Ok(node_num) => node_num, + Err(Error::KubeClient { source: kube::Error::Api(ErrorResponse { reason, .. })}) if &reason == "NotFound" => { + return Ok(NO_REQUEUE); + }, + Err(_) => return Ok(REQUEUE_ERROR), + }; + + debug!("node_num : {:?}", node_num); + + let opstype = os_cr.spec.opstype.clone(); + let ops = opstype.as_str(); + + debug!("opstype: {}", ops); + + match ops { + // 如果是升级或者回滚 + OPERATION_TYPE_UPGRADE | OPERATION_TYPE_ROLLBACK =>{ + debug!("start upgrade OR rollback"); + + // 获取可以进行升级操作的最大节点数量 + let limit = operator_controller.check_upgrading(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?; + + debug!("limit: {}", limit); + + // 为指定数量的节点进行升级操作 + let need_requeue = operator_controller.assign_upgrade(os_cr, limit, &namespace).await?; + if need_requeue { + return Ok(REQUEUE_NORMAL); + } + }, + // 配置操作 + OPERATION_TYPE_CONFIG =>{ + debug!("start config"); + + // 检查待配置的节点数量 + let limit = operator_controller.check_config(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?; + + debug!("limit: {}", limit); + + // 指派配置任务给节点 + let sys_configs = os_cr.spec.sysconfigs.clone().unwrap(); + let version = os_cr.spec.sysconfigs.clone().unwrap().version.unwrap(); + let need_requeue = operator_controller.assign_config(os_cr, sys_configs, version, limit, &namespace).await?; + + if need_requeue { + return Ok(REQUEUE_NORMAL); + } + }, + _ =>{ + log::error!("operation {} cannot be recognized", ops); + } + } + return Ok(REQUEUE_NORMAL); +} + +pub fn error_policy( + error: &Error, + _ctx: Context>, +) -> ReconcilerAction { + error!("Reconciliation error: {}", error.to_string()); + REQUEUE_ERROR +} + +pub mod reconciler_error { + use thiserror::Error; + + use crate::controller::{apiclient::apiclient_error}; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeClient { + #[from] + source: kube::Error, + }, + + #[error("Create/Patch OSInstance reported error: {source}")] + ApplyApi { + #[from] + source: apiclient_error::Error, + }, + + #[error("Cannot get environment NODE_NAME, error: {source}")] + Env { + #[from] + source: std::env::VarError, + }, + + #[error("{}.metadata.{} is not exist", resource, value)] + MissingObjectKey { resource: String, value: String }, + + // #[error("Cannot get {}, {} is None", value, value)] + // MissingSubResource { value: String }, + + #[error("operation {} cannot be recognized", value)] + Operation { value: String }, + + // #[error("Expect OS Version is not same with Node OS Version, please upgrade first")] + // UpgradeBeforeConfig, + + // #[error("Error when drain node, error reported: {}", value)] + // DrainNode { value: String }, + } +} + +#[cfg(test)] +mod test { + use std::{borrow::Borrow, cell::RefCell, env}; + + use serde::de; + + use super::{error_policy, reconcile, reconciler_error::Error, Context, OSInstance, OperatorController, OS}; + use crate::controller::{ + apiserver_mock::{timeout_after_5s, K8sResources, Testcases}, + ControllerClient, + }; + + #[tokio::test] + async fn test_rollback() { + env::set_var("RUST_LOG", "info"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_rollback_osversion_v1_upgradecon_v1(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::Rollback(K8sResources::set_rollback_nodes_v2_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_config_normal() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::ConfigNormal(K8sResources::set_nodes_v1_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_skip_no_osi_node() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_skip_osversion_v2_upgradecon_v1(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::SkipNoOsiNode(K8sResources::set_skip_nodes_and_osi())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_exchange_current_and_next() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_exchange_current_and_next(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::ExchangeCurrentAndNext(K8sResources::set_nodes_v1_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_deep_copy_spec_configs() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let deep_copy_result = test_operator_controller.clone().deep_copy_spec_configs(&OS::set_os_default(), &mut OSInstance::set_osi_default("", ""), "test".to_string()).await; + + assert!(deep_copy_result.is_err()); + + if let Err(err) = deep_copy_result { + assert_eq!("operation test cannot be recognized".to_string(), err.borrow().to_string()); + } + } + + #[tokio::test] + async fn test_get_config_osinstances() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + let expected_error = "list error".to_string(); + + fakeserver.test_function(Testcases::GetConfigOSInstances(expected_error.clone())); + + // 执行测试 + let result = test_operator_controller.get_config_osinstances("default").await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!(expected_error, error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + + #[tokio::test] + async fn test_check_upgrading() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + fakeserver.test_function(Testcases::CheckUpgrading("label error".to_string())); + + // 执行测试 + let result = test_operator_controller.check_upgrading("default", 2).await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!("label error", error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + + + #[tokio::test] + async fn test_get_idle_osinstances() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + let expected_error = "list error".to_string(); + + fakeserver.test_function(Testcases::GetIdleOSInstances(expected_error.clone())); + + // 执行测试 + let result = test_operator_controller.get_idle_os_instances("default", 3).await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!(expected_error, error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + +} diff --git a/KubeOS-Rust/operator/src/controller/crd.rs b/KubeOS-Rust/operator/src/controller/crd.rs new file mode 100644 index 00000000..1fa63035 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/crd.rs @@ -0,0 +1,79 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)] +pub struct OSSpec { + pub osversion: String, + pub maxunavailable: i64, + pub checksum: String, + pub imagetype: String, + pub containerimage: String, + pub opstype: String, + pub evictpodforce: bool, + pub imageurl: String, + #[serde(rename = "flagSafe")] + pub flagsafe: bool, + pub mtls: bool, + pub cacert: Option, + pub clientcert: Option, + pub clientkey: Option, + #[serde(rename = "sysconfigs")] + pub sysconfigs: Option, + #[serde(rename = "upgradeconfigs")] + pub upgradeconfigs: Option, +} + +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube( + group = "upgrade.openeuler.org", + version = "v1alpha1", + kind = "OSInstance", + plural = "osinstances", + singular = "osinstance", + status = "OSInstanceStatus", + namespaced +)] +pub struct OSInstanceSpec { + pub nodestatus: String, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct OSInstanceStatus { + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Configs { + pub version: Option, + pub configs: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Config { + pub model: Option, + pub configpath: Option, + pub contents: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Content { + pub key: Option, + pub value: Option, + pub operation: Option, +} diff --git a/KubeOS-Rust/operator/src/controller/mod.rs b/KubeOS-Rust/operator/src/controller/mod.rs new file mode 100644 index 00000000..468ebfdb --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/mod.rs @@ -0,0 +1,24 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + + mod apiclient; + #[cfg(test)] + mod apiserver_mock; + mod controller; + mod crd; + mod values; + +pub use apiclient::ControllerClient; +pub use controller::{error_policy, reconcile, OperatorController}; +pub use crd::OS; + \ No newline at end of file diff --git a/KubeOS-Rust/operator/src/controller/values.rs b/KubeOS-Rust/operator/src/controller/values.rs new file mode 100644 index 00000000..267270a9 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/values.rs @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::runtime::controller::ReconcilerAction; +use tokio::time::Duration; + +#[cfg(test)] +pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; + +pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; + +pub const LABEL_MASTER: &str = "node-role.kubernetes.io/control-plane"; + +pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; +pub const OSINSTANCE_KIND: &str = "OSInstance"; +// pub const OSI_STATUS_NAME: &str = "nodestatus"; + +pub const UPGRADE_CONFIG_NAME: &str = "UpgradeConfig"; +pub const SYS_CONFIG_NAME: &str = "SysConfig"; + +pub const NODE_STATUS_IDLE: &str = "idle"; +pub const NODE_STATUS_UPGRADE: &str = "upgrade"; +pub const NODE_STATUS_CONFIG: &str = "config"; + +pub const OPERATION_TYPE_UPGRADE: &str = "upgrade"; +pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; +pub const OPERATION_TYPE_CONFIG: &str = "config"; + + +pub const NO_REQUEUE: ReconcilerAction = ReconcilerAction { requeue_after: None }; + +pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; + +pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; diff --git a/KubeOS-Rust/operator/src/main.rs b/KubeOS-Rust/operator/src/main.rs new file mode 100644 index 00000000..51c91618 --- /dev/null +++ b/KubeOS-Rust/operator/src/main.rs @@ -0,0 +1,74 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + use anyhow::Result; + use env_logger::{Builder, Env, Target}; + use futures::StreamExt; + use kube::{ + api::{Api, ListParams}, + client::Client, + runtime::controller::{Context, Controller}, + }; + use log::{error, info}; + // use std::sync::Arc; + use tokio::signal; + + mod controller; + use controller::{ + error_policy, reconcile, ControllerClient, OperatorController, OS, + }; + + const OPERATOR_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); + + + #[tokio::main] + async fn main() -> Result<()> { + // 初始化日志记录器，默认日志级别为info，输出到标准输出 + Builder::from_env(Env::default().default_filter_or("debug")).target(Target::Stdout).init(); + + // 创建一个默认的Kubernetes客户端 + let client = Client::try_default().await?; + + // 创建一个用于操作OS资源的API对象，自定义的CR + let os: Api = Api::all(client.clone()); + + // 创建一个控制器客户端 + let controller_client = ControllerClient::new(client.clone()); + // let controller_client = Arc::new(ControllerClient::new(client.clone())); + + // 创建一个OSReconciler实例 + let os_reconciler = OperatorController::new(client.clone(), controller_client.clone()); + + // 记录操作符版本和启动信息 + info!( + "os-operator version is {}, starting operator manager", + OPERATOR_VERSION.unwrap_or("Not Found") + ); + + // 创建一个新的控制器并运行，处理reconcile和error_policy，并记录错误信息 + Controller::new(os, ListParams::default()) + .run(reconcile, error_policy, Context::new(os_reconciler)) + .for_each(|res| async move { + match res { + Ok(_) => {} + Err(e) => error!("reconcile failed: {}", e.to_string()), + } + }) + .await; + + // 等待终止信号 + signal::ctrl_c().await?; + info!("os-operator terminated"); + + Ok(()) + } + \ No newline at end of file diff --git a/Makefile b/Makefile index a712e872..34e27aaa 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,9 @@ rust-proxy: rust-agent: cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package os-agent +rust-operator: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package operator + rust-kbimg: cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package kbimg diff --git a/go.mod b/go.mod index 255344ce..69a86dd7 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/onsi/gomega v1.20.0 github.com/shirou/gopsutil/v3 v3.23.7 github.com/sirupsen/logrus v1.8.1 + github.com/spf13/cobra v1.4.0 + github.com/spf13/viper v1.8.1 github.com/sykesm/zap-logfmt v0.0.4 go.uber.org/zap v1.19.1 google.golang.org/grpc v1.49.0 diff --git a/go.sum b/go.sum index e9e7e6ab..33c37080 100644 --- a/go.sum +++ b/go.sum @@ -554,6 +554,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/sykesm/zap-logfmt v0.0.4 h1:U2WzRvmIWG1wDLCFY3sz8UeEmsdHQjHFNlIdmroVFaI= github.com/sykesm/zap-logfmt v0.0.4/go.mod h1:AuBd9xQjAe3URrWT1BBDk2v2onAZHkZkWRMiYZXiZWA= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/vendor/modules.txt b/vendor/modules.txt index ee0fd673..5b9304ea 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -295,6 +295,9 @@ github.com/spf13/viper ## explicit; go 1.20 github.com/stretchr/testify/assert github.com/stretchr/testify/require +# github.com/subosito/gotenv v1.2.0 +## explicit +github.com/subosito/gotenv # github.com/sykesm/zap-logfmt v0.0.4 ## explicit; go 1.13 github.com/sykesm/zap-logfmt -- Gitee From 47a0e0b084ac888c48464ea2b192dd5f9f35e77b Mon Sep 17 00:00:00 2001 From: YouMeiYouMaoTai <15335885760@163.com> Date: Fri, 27 Sep 2024 11:15:52 +0800 Subject: [PATCH 46/46] feat:the rust version of the os-operator --- KubeOS-Rust/Cargo.lock | 31 + KubeOS-Rust/Cargo.toml | 2 +- KubeOS-Rust/operator/Cargo.toml | 47 + .../operator/src/controller/apiclient.rs | 110 ++ .../operator/src/controller/apiserver_mock.rs | 995 ++++++++++++++++++ .../operator/src/controller/controller.rs | 696 ++++++++++++ KubeOS-Rust/operator/src/controller/crd.rs | 79 ++ KubeOS-Rust/operator/src/controller/mod.rs | 24 + KubeOS-Rust/operator/src/controller/values.rs | 43 + KubeOS-Rust/operator/src/main.rs | 74 ++ Makefile | 3 + 11 files changed, 2103 insertions(+), 1 deletion(-) create mode 100644 KubeOS-Rust/operator/Cargo.toml create mode 100644 KubeOS-Rust/operator/src/controller/apiclient.rs create mode 100644 KubeOS-Rust/operator/src/controller/apiserver_mock.rs create mode 100644 KubeOS-Rust/operator/src/controller/controller.rs create mode 100644 KubeOS-Rust/operator/src/controller/crd.rs create mode 100644 KubeOS-Rust/operator/src/controller/mod.rs create mode 100644 KubeOS-Rust/operator/src/controller/values.rs create mode 100644 KubeOS-Rust/operator/src/main.rs diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 5b97095b..89e56ec2 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -1520,6 +1520,37 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "operator" +version = "1.0.6" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-trait", + "cli", + "env_logger", + "futures", + "h2 0.3.16", + "http 0.2.9", + "hyper 0.14.25", + "k8s-openapi", + "kube", + "log", + "manager", + "mockall", + "regex", + "reqwest", + "schemars", + "serde", + "serde_json", + "socket2 0.4.9", + "thiserror", + "thread_local", + "tokio", + "tokio-retry", + "tower-test", +] + [[package]] name = "ordered-float" version = "2.10.1" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml index 2886023f..93379823 100644 --- a/KubeOS-Rust/Cargo.toml +++ b/KubeOS-Rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["agent", "cli", "kbimg", "manager", "proxy"] +members = ["agent", "cli", "kbimg", "manager", "operator", "proxy"] resolver = "2" [profile.release] diff --git a/KubeOS-Rust/operator/Cargo.toml b/KubeOS-Rust/operator/Cargo.toml new file mode 100644 index 00000000..91cca265 --- /dev/null +++ b/KubeOS-Rust/operator/Cargo.toml @@ -0,0 +1,47 @@ +[package] +description = "KubeOS os-operator" +edition = "2021" +license = "MulanPSL-2.0" +name = "operator" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +name = "operator" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.44" +async-trait = "0.1" +cli = { version = "1.0.6", path = "../cli" } +env_logger = "0.9.0" +futures = "0.3.17" +h2 = "=0.3.16" +k8s-openapi = { version = "0.13.1", features = ["v1_22"] } +kube = { version = "0.66.0", features = ["derive", "runtime"] } +log = "=0.4.15" +manager = { version = "1.0.6", path = "../manager" } +regex = "=1.7.3" +reqwest = { version = "=0.12.2", default-features = false, features = [ + "json", +] } +schemars = "=0.8.10" +serde = { version = "1.0.130", features = ["derive"] } +serde_json = "1.0.68" +socket2 = "=0.4.9" +thiserror = "1.0.29" +thread_local = "=1.1.4" +tokio = { version = "=1.28.0", default-features = false, features = [ + "macros", + "rt-multi-thread", +] } +tokio-retry = "0.3" + +[dev-dependencies] +assert-json-diff = "2.0.2" +http = "0.2.9" +hyper = "0.14.25" +tower-test = "0.4.0" +mockall = { version = "=0.11.3" } +regex = "1" diff --git a/KubeOS-Rust/operator/src/controller/apiclient.rs b/KubeOS-Rust/operator/src/controller/apiclient.rs new file mode 100644 index 00000000..3642b475 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/apiclient.rs @@ -0,0 +1,110 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + +use anyhow::Result; +use apiclient_error::Error; +use async_trait::async_trait; +use kube::{ + api::{Api, Patch, PatchParams}, + Client, +}; +use serde::{Deserialize, Serialize}; +use super::{ + crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}, + values::{NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}, +}; + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceSpecPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + spec: OSInstanceSpec, +} + +impl Default for OSInstanceSpecPatch { + fn default() -> Self { + OSInstanceSpecPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceStatusPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + status: Option, +} + +impl Default for OSInstanceStatusPatch { + fn default() -> Self { + OSInstanceStatusPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }), + } + } +} + +#[derive(Clone)] +pub struct ControllerClient { + pub client: Client, +} + +impl ControllerClient { + pub fn new(client: Client) -> Self { + ControllerClient { client } + } +} + +#[async_trait] +pub trait ApplyApi: Clone + Sized + Send + Sync { + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error>; +} + +#[async_trait] +impl ApplyApi for ControllerClient { + + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() }; + osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?; + Ok(()) + } + +} +pub mod apiclient_error { + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} diff --git a/KubeOS-Rust/operator/src/controller/apiserver_mock.rs b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs new file mode 100644 index 00000000..21abd405 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs @@ -0,0 +1,995 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{borrow::BorrowMut, cell::{RefCell, RefMut}, clone, collections::BTreeMap, default}; +use regex::Regex; +use anyhow::Result; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use http::{status, Request, Response}; +use hyper::{body::to_bytes, Body}; +use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod}; +use kube::{ + api::ObjectMeta, + core::{ErrorResponse, ListMeta, ObjectList}, + Client as KubeClient, Resource, ResourceExt, +}; +use log::debug; +use mockall::mock; +use serde_json::json; + +use self::mock_error::Error; +use super::{ + crd::{Configs, OSInstanceStatus}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_CONFIG}, +}; +use crate::controller::{ + apiclient::{ApplyApi, ControllerClient}, + crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS}, + values::{LABEL_MASTER, LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, + OperatorController, +}; + +type ApiServerHandle = tower_test::mock::Handle, Response>; +pub struct ApiServerVerifier(ApiServerHandle); + + +#[derive(Clone, Debug, Default)] +pub struct K8sResources{ + pub node_list: Vec, + pub osi_list: Vec, +} + +pub enum Testcases { + Rollback(K8sResources), + ConfigNormal(K8sResources), + SkipNoOsiNode(K8sResources), + ExchangeCurrentAndNext(K8sResources), + GetConfigOSInstances(String), + CheckUpgrading(String), + GetIdleOSInstances(String), + +} + +pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) { + tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("timeout on mock apiserver") + .expect("scenario succeeded") +} + +impl ApiServerVerifier { + pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::Rollback(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + }, + Testcases::ConfigNormal(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_config_osi_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_idle_osi_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_patch_spec_config(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_spec_config(k8s_resc.clone()) + .await + }, + Testcases::SkipNoOsiNode(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + }, + Testcases::ExchangeCurrentAndNext(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + }, + _ => { + Err(Error::ArgumentError) + } + } + .expect("Case completed without errors"); + }) + } + + pub fn test_function(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::GetConfigOSInstances(error) => { + self.handler_config_osi_list_get_error(error) + .await + }, + Testcases::CheckUpgrading(error) => { + self.handler_upgrading_node_list_get_error(error) + .await + }, + Testcases::GetIdleOSInstances(error) => { + self.handler_idle_osi_list_get_error(error) + .await + }, + _ => { + Err(Error::ArgumentError) + } + } + .expect("Case completed without errors"); + }) + } + + // 获取所有的 worker 节点，对应于reconcile的第一个 get_nodes 函数 + async fn handler_worker_node_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=%21node-role.kubernetes.io%2Fcontrol-plane&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中所有的 worker 节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if !node.labels().contains_key(LABEL_MASTER){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_worker_node_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 upgrading 的节点 + async fn handler_upgrading_node_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中标签为正在升级的节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if node.labels().contains_key(LABEL_UPGRADING){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_upgrading_node_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取所有的非 upgrading 的 worker 节点 + async fn handler_worker_and_no_upgrade_noding_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + + let remove_limit = |input: &str| -> String { + let re = Regex::new(r"limit=\d+").unwrap(); + re.replace_all(input, "").to_string() + }; + + assert_eq!( + remove_limit(request.uri().to_string().as_str()), + "/api/v1/nodes?&labelSelector=%21upgrade.openeuler.org%2Fupgrading%2C%21node-role.kubernetes.io%2Fcontrol-plane&"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中所有的非 upgrading 的 worker 节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if !node.labels().contains_key(LABEL_UPGRADING) && !node.labels().contains_key(LABEL_MASTER){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_worker_and_no_upgrade_noding_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_osinstance_get_by_node_name(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_get_by_node_name: req_node_name: {:?}", req_node_name); + + let response = serde_json::to_vec(&osinstance).unwrap(); + dbg!("handler_osinstance_get_by_node_name"); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_upgrade(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_nodestatus_upgrade: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_upgrade"); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_exchange(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_nodestatus_exchange: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + + let sysconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + + let upgradeconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + + assert_eq!(spec.sysconfigs.clone(), sysconfigs); + assert_eq!(spec.upgradeconfigs.clone(), upgradeconfigs); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_exchange"); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + // 通过节点名称获取对应节点 + async fn handler_replace_node_by_name(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PUT); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get node and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut node = Node{..Default::default()}; + let mut boolean_get_node = false; + for node_iter in k8s_resc.node_list.clone() { + if node_iter.name() == req_node_name { + boolean_get_node = true; + node = node_iter.clone(); + break; + } + } + assert!(boolean_get_node); + assert_eq!(request.extensions().get(), Some(&"replace")); + + println!("handler_replace_node_by_name: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let metadata_json = body_json.get("metadata").expect("metadata object").clone(); + let metadata: ObjectMeta = serde_json::from_value(metadata_json).expect("valid metadata"); + assert!(metadata.labels.unwrap().contains_key(LABEL_UPGRADING)); + + // 修改 node 并传出 + node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string()); + + dbg!("handler_replace_node_by_name"); + let response = serde_json::to_vec(&node).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 config 的节点上的 osi + async fn handler_config_osi_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出 + let mut osis = vec![]; + for osi in k8s_resc.osi_list.clone() { + if osi.spec.nodestatus == NODE_STATUS_CONFIG{ + osis.push(osi.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: osis, + }; + + dbg!("handler_config_osi_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 config 的节点上的 osi + async fn handler_idle_osi_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出 + let mut osis = vec![]; + for osi in k8s_resc.osi_list.clone() { + if osi.spec.nodestatus == NODE_STATUS_IDLE{ + osis.push(osi.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: osis, + }; + + dbg!("handler_idle_osi_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_osinstance_patch_spec_config(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_osi_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_osi_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_spec_config: req_osi_name: {:?}", req_osi_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_CONFIG.to_string()); + + let sysconfig = Some( + Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl")), + configpath: Some(String::from("")), + contents: + Some(vec![ + Content { + key: Some(String::from("key1")), + value: Some(String::from("a")), + operation: Some(String::from("")), + }, + Content { + key: Some(String::from("key2")), + value: Some(String::from("b")), + operation: Some(String::from("")), + }, + ]), + }]), + } + ); + assert_eq!( + spec.sysconfigs.clone(), + sysconfig + ); + + dbg!("handler_osinstance_patch_spec_config"); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance.spec.sysconfigs = sysconfig; + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_config_osi_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_config_osi_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "NotFound".to_string(), + code: 404, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_upgrading_node_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_upgrading_node_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "Invalid".to_string(), + code: 400, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(400).body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_idle_osi_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_idle_osi_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "NotFound".to_string(), + code: 404, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + + Ok(self) + } + +} + +pub mod mock_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + + #[error("Parameters other than expected were entered")] + ArgumentError, + } +} + + +impl OperatorController { + pub fn test() -> (OperatorController, ApiServerVerifier) { + let (mock_service, handle) = tower_test::mock::pair::, Response>(); + let mock_k8s_client = KubeClient::new(mock_service, "default"); + let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); + let operator_controller: OperatorController = + OperatorController::new(mock_k8s_client, mock_api_client); + (operator_controller, ApiServerVerifier(handle)) + } +} + +impl OSInstance { + pub fn set_osi_default(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }, + status: Some(OSInstanceStatus { + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }), + } + } +} + +impl OS { + pub fn set_os_default() -> Self { + let mut os = OS::new("test", OSSpec::default()); + os.meta_mut().namespace = Some("default".into()); + os + } + + pub fn set_os_rollback_osversion_v1_upgradecon_v1() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string(); + os + } + + pub fn set_os_syscon_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = OPERATION_TYPE_CONFIG.to_string(); + os.spec.sysconfigs = Some( + Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("key1")), + value: Some(String::from("a")), + operation: Some(String::from("")), + }, + Content { + key: Some(String::from("key2")), + value: Some(String::from("b")), + operation: Some(String::from("")), + }, + ]), + }]), + } + ); + os + } + + pub fn set_os_skip_osversion_v2_upgradecon_v1() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os + } + + pub fn set_os_exchange_current_and_next() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + let sysconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + os.spec.sysconfigs = sysconfigs.clone(); + os.spec.upgradeconfigs = sysconfigs.clone(); + + os + } + +} + +impl K8sResources { + pub fn set_rollback_nodes_v2_and_osi_v1() -> Self { + // 创建 node1 和 node2 + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v2".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let node2 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node2".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v2".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default"); + let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default"); + + let node_list = Vec::from([node1, node2]); + let osi_list = Vec::from([osi1, osi2]); + + K8sResources{ + node_list, + osi_list + } + } + + pub fn set_nodes_v1_and_osi_v1() -> Self { + // 创建 node1 和 node2 + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let node2 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node2".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default"); + let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default"); + + let node_list = Vec::from([node1, node2]); + let osi_list = Vec::from([osi1, osi2]); + + K8sResources{ + node_list, + osi_list + } + } + + pub fn set_skip_nodes_and_osi() -> Self { + // 创建 node1 并且不设置 osi + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let node_list = Vec::from([node1]); + let osi_list = Vec::new(); + + K8sResources{ + node_list, + osi_list + } + } + +} + +impl Default for OSSpec { + fn default() -> Self { + OSSpec { + osversion: String::from("KubeOS v1"), + maxunavailable: 3, + checksum: String::from("test"), + imagetype: String::from("containerd"), + containerimage: String::from("test"), + opstype: String::from("upgrade"), + evictpodforce: true, + imageurl: String::from(""), + flagsafe: true, + mtls: false, + cacert: Some(String::from("")), + clientcert: Some(String::from("")), + clientkey: Some(String::from("")), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + } + } +} diff --git a/KubeOS-Rust/operator/src/controller/controller.rs b/KubeOS-Rust/operator/src/controller/controller.rs new file mode 100644 index 00000000..2beb8945 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/controller.rs @@ -0,0 +1,696 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +* KubeOS is licensed under the Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +* PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + + +use anyhow::Result; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::{Api, ListParams, ObjectList, PostParams}, + core::ErrorResponse, + runtime::controller::{Context, ReconcilerAction}, + Client, ResourceExt, +}; +use log::{debug, error}; +use reconciler_error::Error; + +use crate::controller::values::NODE_STATUS_UPGRADE; + +use super::{ + apiclient::ApplyApi, + crd::{Configs, OSInstance, OS}, + values::{ + LABEL_MASTER, LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, + NO_REQUEUE, OPERATION_TYPE_CONFIG, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, + REQUEUE_ERROR, REQUEUE_NORMAL, SYS_CONFIG_NAME, UPGRADE_CONFIG_NAME + }, +}; + +#[derive(Clone)] +pub struct OperatorController { + k8s_client: Client, + controller_client: T, +} + +impl OperatorController { + pub fn new(k8s_client: Client, controller_client: T) -> Self { + OperatorController { + k8s_client, + controller_client, + } + } + + // async fn reconcile( + // &self, + // os: OS, + // ctx: Context>, + // ) -> Result { + // // k8s_client 变量不是 Option 类型，而是直接的 Client 类型，Rust 会确保不为空，不用检查 + + // // Kube-rs库中的Context类型已经处理了上下文的管理，也不需要初始化空的上下文，proxy组件的rust版本也没初始化 + + // // 调用外部的Reconcile函数 + // reconcile(os, ctx).await + // } + + // 获取 worker 节点数 + async fn get_and_update_os(&self, _namespace: &str) -> Result { + + // 创建一个筛选标签的 String 数组，只找 worker 节点 + let reqs = vec![ + format!("!{}", LABEL_MASTER), + ]; + + // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量 + let nodes_items = self.get_nodes( 0, reqs).await?; + + Ok(nodes_items.items.len() as i64) + } + + // 获取 worker 节点，传入 reqs 为 String 数组，表示多个筛选条件 + async fn get_nodes(&self, limit: i64, reqs: Vec) -> Result, Error> { + + let nodes_api: Api = Api::all(self.k8s_client.clone()); + + // 将多个标签筛选器组合成一个字符串，用逗号分隔 + let label_selector = reqs.join(","); + + // 设置 ListParams + let list_params = ListParams::default() + .labels(&label_selector) + .limit(limit as u32); + + let nodes = match nodes_api.list(&list_params).await { + Ok(nodes) => nodes, + Err(e) => { + log::error!("{:?} unable to list nodes with requirements", e); + return Err(Error::KubeClient { source: e }); + }, + }; + + Ok(nodes) + } + + // 获取可以进行升级操作的最大节点数量 + async fn check_upgrading(&self, _namespace: &str, max_unavailable: i64) -> Result { + + // 设置筛选标签，选择标签为正在升级的节点 + let reqs = vec![ + LABEL_UPGRADING.to_string(), + ]; + + // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量 + let nodes_items = self.get_nodes( 0, reqs).await?; + + Ok(max_unavailable - nodes_items.items.len() as i64) + } + + // 为指定数量的节点进行升级操作 + async fn assign_upgrade(&self, os: &OS, limit: i64, namespace: &str) -> Result { + + // 创建筛选标签，只找 worker 节点，并且标签 LabelUpgrading 不存在 + let reqs = vec![ + format!("!{}", LABEL_UPGRADING), + format!("!{}", LABEL_MASTER), + ]; + + // 获取符合标签要求的节点 + let mut nodes_items = self.get_nodes( limit + 1, reqs).await?; + + // 对选定的节点进行升级，返回升级成功的数量和可能的错误 + let count = self.upgrade_nodes(os, &mut nodes_items, limit, namespace).await?; + + Ok(count >= limit) + } + + async fn upgrade_nodes(&self, os: &OS, nodes: &mut ObjectList, limit: i64, namespace: &str) -> Result { + + let mut count = 0; + + for node in nodes.iter_mut() { + // 如果已经达到升级限制，则退出循环 + if count >= limit { + break + } + + let os_version_node = node.status.clone().unwrap().node_info.unwrap().os_image; + + debug!("node name: {}, os_version_node: {}, os_version: {}", node.name(), os_version_node, os.spec.osversion); + + // 检查 os 对象中的操作系统版本是否与节点的操作系统版本不同 + if os_version_node != os.spec.osversion { + + // 尝试获取该节点上的 os 实例 + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + match osi_api.get(&node.name().clone()).await { + Ok(mut osi) => { + // info!("osinstance is exist {:?}", osi.name()); + + debug!("osinstance is exist: \n {:?} \n", osi); + + match self.update_node_and_osins(os, node, &mut osi).await { + Ok(_) => { + count += 1; + }, + Err(_) => { + continue; + }, + } + }, + Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { + debug!("failed to get osInstance {}", &node.name().clone()); + + return Err(Error::KubeClient { + source: kube::Error::Api(ErrorResponse { + reason, + status: "".to_string(), + message: "".to_string(), + code: 0 + })}); + }, + Err(_) => continue, + } + } + + } + + Ok(count) + } + + // 升级节点以及节点上的 OSinstance + async fn update_node_and_osins(&self, os: &OS, node: &mut Node, osinstance: &mut OSInstance, ) -> Result<(), Error> { + debug!("start update_node_and_OSins"); + + // 检查os实例中的升级配置版本与os对象中的升级配置版本是否匹配。osi 字段未初始化时直接进行拷贝 + let mut copy_sign = true; + if let Some(upgradeconfigs) = osinstance.spec.upgradeconfigs.clone() { + if let Some(version) = upgradeconfigs.version { + if version == os.spec.upgradeconfigs.clone().unwrap().version.unwrap() { + copy_sign = false; + } + } + } + if copy_sign { + self.deep_copy_spec_configs(os, osinstance, UPGRADE_CONFIG_NAME.to_string()).await?; + assert!(osinstance.spec.upgradeconfigs.is_some()); + } + + copy_sign = true; + // 检查os实例中的系统配置版本与os对象中的系统配置版本是否匹配。osi 字段未初始化时直接进行拷贝 + if let Some(sysconfigs) = osinstance.spec.sysconfigs.clone() { + if let Some(version) = sysconfigs.version { + if version == os.spec.sysconfigs.clone().unwrap().version.unwrap() { + copy_sign = false; + } + } + } + if copy_sign { + self.deep_copy_spec_configs(os, osinstance, SYS_CONFIG_NAME.to_string()).await?; + assert!(osinstance.spec.sysconfigs.is_some()); + if let Some(sysconfigs) = osinstance.spec.sysconfigs.as_mut() { + if let Some(configs) = &mut sysconfigs.configs { + for config in configs { + if config.model.clone().unwrap() == "grub.cmdline.current" { + config.model = Some("grub.cmdline.next".to_string()); + } + else if config.model.clone().unwrap() == "grub.cmdline.next" { + config.model = Some("grub.cmdline.current".to_string()); + } + } + } + } + } + + // 更新os实例中的状态为升级完成状态 + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + + // 把对 osinstance 的更改从内存更新到 k8s 集群 + let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; + + node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string()); + + // 把对 node 的更改从内存更新到 k8s 集群 + let node_api: Api = Api::all(self.k8s_client.clone()); + node_api.replace(&node.name(), &PostParams::default(), &node).await?; + + Ok(()) + } + + // 深拷贝 + async fn deep_copy_spec_configs(&self, os: &OS, os_instance: &mut OSInstance, config_type: String) -> Result<(), Error> { + + match config_type.as_str() { + UPGRADE_CONFIG_NAME =>{ + + if let Ok(data) = serde_json::to_vec(&os.spec.upgradeconfigs){ + + if let Ok(upgradeconfigs) = serde_json::from_slice(&data) { + os_instance.spec.upgradeconfigs = Some(upgradeconfigs); + }else { + debug!("{} Deserialization failure", config_type); + return Err(Error::Operation { value: "Deserialization".to_string()}); + } + } + else { + debug!("{} Serialization failure", config_type); + return Err(Error::Operation { value: "Serialization".to_string()}); + } + + }, + SYS_CONFIG_NAME => { + + if let Ok(data) = serde_json::to_vec(&os.spec.sysconfigs){ + + if let Ok(sysconfigs) = serde_json::from_slice(&data) { + os_instance.spec.sysconfigs = Some(sysconfigs); + }else { + debug!("{} Deserialization failure", config_type); + return Err(Error::Operation { value: "Deserialization".to_string()}); + } + } + else { + debug!("{} Serialization failure", config_type); + return Err(Error::Operation { value: "Serialization".to_string()}); + } + + }, + _ => { + debug!("configType {} cannot be recognized", config_type); + return Err(Error::Operation { value: config_type.clone() }); + }, + } + + Ok(()) + } + + // 获取可以进行配置操作的节点数量，返回的是 instance 的列表 + async fn check_config(&self, namespace: &str, max_unavailable: i64) -> Result { + let osinstances = self.get_config_osinstances(namespace).await?; + + Ok(max_unavailable - osinstances.len() as i64) + } + + // 获取所在节点状态为配置的 osinstance列表 + async fn get_config_osinstances(&self, namespace: &str) -> Result, Error> { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + // 获取所有 OSInstance 资源 + let all_osinstances = osi_api.list(&ListParams::default()).await?; + + // 在客户端进行过滤，节点状态为 NODE_STATUS_CONFIG + let osinstances: Vec = all_osinstances + .items + .into_iter() + .filter(|osi| osi.spec.nodestatus == NODE_STATUS_CONFIG) + .collect(); + + debug!("config_osi count = {:?}", osinstances.len()); + + Ok(osinstances) + } + + // 为指定数量的节点进行配置操作 + async fn assign_config(&self, _os: &OS, sysconfigs: Configs, config_version: String, limit: i64, namespace: &str) -> Result { + + debug!("start assign_config"); + + let mut osinstances = self.get_idle_os_instances(namespace, limit + 1).await?; + + let mut count = 0; + // 遍历 osi 列表 + for osi in osinstances.iter_mut() { + if count > limit { + break; + } + + let mut config_sign = true; + if let Some(sysconfigs) = osi.spec.sysconfigs.clone() { + if let Some(version) = sysconfigs.version { + debug!("node name: {:?}, config_version_node: {:?}, config_version: {:?}", osi.name(), version, config_version); + if version == config_version { + config_sign = false; + } + } + } + + // 如果版本不同或 osi 未初始化，则将新的配置信息更新到实例中，并将节点状态标记为“配置完成”。 + if config_sign { + count += 1; + osi.spec.sysconfigs = Some(sysconfigs.clone()); + osi.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + + // 把对 osinstance 的更改从内存更新到 k8s 集群 + let namespace = osi.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + self.controller_client.update_osinstance_spec(&osi.name(), &namespace, &osi.spec).await?; + } + } + + Ok(count >= limit) + } + + // 获取所在节点状态为空闲的 osinstance列表 + async fn get_idle_os_instances(&self, namespace: &str, limit: i64) -> Result, Error> { + + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + // 获取所有 OSInstance 资源 + let all_osinstances: ObjectList = osi_api.list(&ListParams::default().limit(limit as u32)).await?; + + // 在客户端进行过滤，节点状态为 NODE_STATUS_IDLE + let osinstances: Vec = all_osinstances + .items + .into_iter() + .filter(|osi| osi.spec.nodestatus == NODE_STATUS_IDLE) + .collect(); + + Ok(osinstances) + } + + +} + +// 调用的函数的具体逻辑需要进一步完成 +pub async fn reconcile( + os: OS, + ctx: Context>, +) -> Result { + + // 初始化 operator_controller 和 os，从环境变量获取NODE_NAME + debug!("start reconcile"); + let operator_controller = ctx.get_ref(); + let os_cr: &OS = &os; + + // 从 os_cr 中获取命名空间，如果命名空间不存在则返回错误 + let namespace: String = os_cr + .namespace() + .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; + + debug!("namespace : {:?}", namespace); + + // 获取 worker 节点数 + let node_num = match operator_controller.get_and_update_os(&namespace).await { + Ok(node_num) => node_num, + Err(Error::KubeClient { source: kube::Error::Api(ErrorResponse { reason, .. })}) if &reason == "NotFound" => { + return Ok(NO_REQUEUE); + }, + Err(_) => return Ok(REQUEUE_ERROR), + }; + + debug!("node_num : {:?}", node_num); + + let opstype = os_cr.spec.opstype.clone(); + let ops = opstype.as_str(); + + debug!("opstype: {}", ops); + + match ops { + // 如果是升级或者回滚 + OPERATION_TYPE_UPGRADE | OPERATION_TYPE_ROLLBACK =>{ + debug!("start upgrade OR rollback"); + + // 获取可以进行升级操作的最大节点数量 + let limit = operator_controller.check_upgrading(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?; + + debug!("limit: {}", limit); + + // 为指定数量的节点进行升级操作 + let need_requeue = operator_controller.assign_upgrade(os_cr, limit, &namespace).await?; + if need_requeue { + return Ok(REQUEUE_NORMAL); + } + }, + // 配置操作 + OPERATION_TYPE_CONFIG =>{ + debug!("start config"); + + // 检查待配置的节点数量 + let limit = operator_controller.check_config(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?; + + debug!("limit: {}", limit); + + // 指派配置任务给节点 + let sys_configs = os_cr.spec.sysconfigs.clone().unwrap(); + let version = os_cr.spec.sysconfigs.clone().unwrap().version.unwrap(); + let need_requeue = operator_controller.assign_config(os_cr, sys_configs, version, limit, &namespace).await?; + + if need_requeue { + return Ok(REQUEUE_NORMAL); + } + }, + _ =>{ + log::error!("operation {} cannot be recognized", ops); + } + } + return Ok(REQUEUE_NORMAL); +} + +pub fn error_policy( + error: &Error, + _ctx: Context>, +) -> ReconcilerAction { + error!("Reconciliation error: {}", error.to_string()); + REQUEUE_ERROR +} + +pub mod reconciler_error { + use thiserror::Error; + + use crate::controller::{apiclient::apiclient_error}; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeClient { + #[from] + source: kube::Error, + }, + + #[error("Create/Patch OSInstance reported error: {source}")] + ApplyApi { + #[from] + source: apiclient_error::Error, + }, + + #[error("Cannot get environment NODE_NAME, error: {source}")] + Env { + #[from] + source: std::env::VarError, + }, + + #[error("{}.metadata.{} is not exist", resource, value)] + MissingObjectKey { resource: String, value: String }, + + // #[error("Cannot get {}, {} is None", value, value)] + // MissingSubResource { value: String }, + + #[error("operation {} cannot be recognized", value)] + Operation { value: String }, + + // #[error("Expect OS Version is not same with Node OS Version, please upgrade first")] + // UpgradeBeforeConfig, + + // #[error("Error when drain node, error reported: {}", value)] + // DrainNode { value: String }, + } +} + +#[cfg(test)] +mod test { + use std::{borrow::Borrow, cell::RefCell, env}; + + use serde::de; + + use super::{error_policy, reconcile, reconciler_error::Error, Context, OSInstance, OperatorController, OS}; + use crate::controller::{ + apiserver_mock::{timeout_after_5s, K8sResources, Testcases}, + ControllerClient, + }; + + #[tokio::test] + async fn test_rollback() { + env::set_var("RUST_LOG", "info"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_rollback_osversion_v1_upgradecon_v1(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::Rollback(K8sResources::set_rollback_nodes_v2_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_config_normal() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::ConfigNormal(K8sResources::set_nodes_v1_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_skip_no_osi_node() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_skip_osversion_v2_upgradecon_v1(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::SkipNoOsiNode(K8sResources::set_skip_nodes_and_osi())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_exchange_current_and_next() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_exchange_current_and_next(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::ExchangeCurrentAndNext(K8sResources::set_nodes_v1_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_deep_copy_spec_configs() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let deep_copy_result = test_operator_controller.clone().deep_copy_spec_configs(&OS::set_os_default(), &mut OSInstance::set_osi_default("", ""), "test".to_string()).await; + + assert!(deep_copy_result.is_err()); + + if let Err(err) = deep_copy_result { + assert_eq!("operation test cannot be recognized".to_string(), err.borrow().to_string()); + } + } + + #[tokio::test] + async fn test_get_config_osinstances() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + let expected_error = "list error".to_string(); + + fakeserver.test_function(Testcases::GetConfigOSInstances(expected_error.clone())); + + // 执行测试 + let result = test_operator_controller.get_config_osinstances("default").await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!(expected_error, error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + + #[tokio::test] + async fn test_check_upgrading() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + fakeserver.test_function(Testcases::CheckUpgrading("label error".to_string())); + + // 执行测试 + let result = test_operator_controller.check_upgrading("default", 2).await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!("label error", error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + + + #[tokio::test] + async fn test_get_idle_osinstances() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + let expected_error = "list error".to_string(); + + fakeserver.test_function(Testcases::GetIdleOSInstances(expected_error.clone())); + + // 执行测试 + let result = test_operator_controller.get_idle_os_instances("default", 3).await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!(expected_error, error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + +} diff --git a/KubeOS-Rust/operator/src/controller/crd.rs b/KubeOS-Rust/operator/src/controller/crd.rs new file mode 100644 index 00000000..1fa63035 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/crd.rs @@ -0,0 +1,79 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)] +pub struct OSSpec { + pub osversion: String, + pub maxunavailable: i64, + pub checksum: String, + pub imagetype: String, + pub containerimage: String, + pub opstype: String, + pub evictpodforce: bool, + pub imageurl: String, + #[serde(rename = "flagSafe")] + pub flagsafe: bool, + pub mtls: bool, + pub cacert: Option, + pub clientcert: Option, + pub clientkey: Option, + #[serde(rename = "sysconfigs")] + pub sysconfigs: Option, + #[serde(rename = "upgradeconfigs")] + pub upgradeconfigs: Option, +} + +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube( + group = "upgrade.openeuler.org", + version = "v1alpha1", + kind = "OSInstance", + plural = "osinstances", + singular = "osinstance", + status = "OSInstanceStatus", + namespaced +)] +pub struct OSInstanceSpec { + pub nodestatus: String, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct OSInstanceStatus { + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Configs { + pub version: Option, + pub configs: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Config { + pub model: Option, + pub configpath: Option, + pub contents: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Content { + pub key: Option, + pub value: Option, + pub operation: Option, +} diff --git a/KubeOS-Rust/operator/src/controller/mod.rs b/KubeOS-Rust/operator/src/controller/mod.rs new file mode 100644 index 00000000..468ebfdb --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/mod.rs @@ -0,0 +1,24 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + + mod apiclient; + #[cfg(test)] + mod apiserver_mock; + mod controller; + mod crd; + mod values; + +pub use apiclient::ControllerClient; +pub use controller::{error_policy, reconcile, OperatorController}; +pub use crd::OS; + \ No newline at end of file diff --git a/KubeOS-Rust/operator/src/controller/values.rs b/KubeOS-Rust/operator/src/controller/values.rs new file mode 100644 index 00000000..267270a9 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/values.rs @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::runtime::controller::ReconcilerAction; +use tokio::time::Duration; + +#[cfg(test)] +pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; + +pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; + +pub const LABEL_MASTER: &str = "node-role.kubernetes.io/control-plane"; + +pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; +pub const OSINSTANCE_KIND: &str = "OSInstance"; +// pub const OSI_STATUS_NAME: &str = "nodestatus"; + +pub const UPGRADE_CONFIG_NAME: &str = "UpgradeConfig"; +pub const SYS_CONFIG_NAME: &str = "SysConfig"; + +pub const NODE_STATUS_IDLE: &str = "idle"; +pub const NODE_STATUS_UPGRADE: &str = "upgrade"; +pub const NODE_STATUS_CONFIG: &str = "config"; + +pub const OPERATION_TYPE_UPGRADE: &str = "upgrade"; +pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; +pub const OPERATION_TYPE_CONFIG: &str = "config"; + + +pub const NO_REQUEUE: ReconcilerAction = ReconcilerAction { requeue_after: None }; + +pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; + +pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; diff --git a/KubeOS-Rust/operator/src/main.rs b/KubeOS-Rust/operator/src/main.rs new file mode 100644 index 00000000..51c91618 --- /dev/null +++ b/KubeOS-Rust/operator/src/main.rs @@ -0,0 +1,74 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + use anyhow::Result; + use env_logger::{Builder, Env, Target}; + use futures::StreamExt; + use kube::{ + api::{Api, ListParams}, + client::Client, + runtime::controller::{Context, Controller}, + }; + use log::{error, info}; + // use std::sync::Arc; + use tokio::signal; + + mod controller; + use controller::{ + error_policy, reconcile, ControllerClient, OperatorController, OS, + }; + + const OPERATOR_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); + + + #[tokio::main] + async fn main() -> Result<()> { + // 初始化日志记录器，默认日志级别为info，输出到标准输出 + Builder::from_env(Env::default().default_filter_or("debug")).target(Target::Stdout).init(); + + // 创建一个默认的Kubernetes客户端 + let client = Client::try_default().await?; + + // 创建一个用于操作OS资源的API对象，自定义的CR + let os: Api = Api::all(client.clone()); + + // 创建一个控制器客户端 + let controller_client = ControllerClient::new(client.clone()); + // let controller_client = Arc::new(ControllerClient::new(client.clone())); + + // 创建一个OSReconciler实例 + let os_reconciler = OperatorController::new(client.clone(), controller_client.clone()); + + // 记录操作符版本和启动信息 + info!( + "os-operator version is {}, starting operator manager", + OPERATOR_VERSION.unwrap_or("Not Found") + ); + + // 创建一个新的控制器并运行，处理reconcile和error_policy，并记录错误信息 + Controller::new(os, ListParams::default()) + .run(reconcile, error_policy, Context::new(os_reconciler)) + .for_each(|res| async move { + match res { + Ok(_) => {} + Err(e) => error!("reconcile failed: {}", e.to_string()), + } + }) + .await; + + // 等待终止信号 + signal::ctrl_c().await?; + info!("os-operator terminated"); + + Ok(()) + } + \ No newline at end of file diff --git a/Makefile b/Makefile index a712e872..34e27aaa 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,9 @@ rust-proxy: rust-agent: cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package os-agent +rust-operator: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package operator + rust-kbimg: cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package kbimg -- Gitee