From f5097d1e0e90e1a30c777333a557d65d0c5bc1f0 Mon Sep 17 00:00:00 2001 From: YouMeiYouMaoTai <15335885760@163.com> Date: Wed, 25 Sep 2024 16:15:32 +0800 Subject: [PATCH 1/2] Merge branch 'master' into dev; Feat:The rust version of the os-operator --- .gitignore | 7 + KubeOS-Rust/Cargo.lock | 2882 +++++++++++++++++ KubeOS-Rust/Cargo.toml | 12 + KubeOS-Rust/agent/Cargo.toml | 20 + KubeOS-Rust/agent/src/function.rs | 56 + KubeOS-Rust/agent/src/main.rs | 66 + KubeOS-Rust/agent/src/rpc/agent.rs | 30 + KubeOS-Rust/agent/src/rpc/agent_impl.rs | 217 ++ KubeOS-Rust/agent/src/rpc/mod.rs | 19 + KubeOS-Rust/cli/Cargo.toml | 15 + KubeOS-Rust/cli/src/client.rs | 56 + KubeOS-Rust/cli/src/lib.rs | 14 + KubeOS-Rust/cli/src/method/callable_method.rs | 54 + KubeOS-Rust/cli/src/method/configure.rs | 72 + KubeOS-Rust/cli/src/method/mod.rs | 18 + KubeOS-Rust/cli/src/method/prepare_upgrade.rs | 78 + KubeOS-Rust/cli/src/method/request.rs | 88 + KubeOS-Rust/cli/src/method/rollback.rs | 42 + KubeOS-Rust/cli/src/method/upgrade.rs | 42 + KubeOS-Rust/manager/Cargo.toml | 25 + KubeOS-Rust/manager/src/api/agent_status.rs | 21 + KubeOS-Rust/manager/src/api/mod.rs | 17 + KubeOS-Rust/manager/src/api/types.rs | 140 + KubeOS-Rust/manager/src/lib.rs | 15 + KubeOS-Rust/manager/src/sys_mgmt/config.rs | 558 ++++ .../manager/src/sys_mgmt/containerd_image.rs | 301 ++ .../manager/src/sys_mgmt/disk_image.rs | 406 +++ .../manager/src/sys_mgmt/docker_image.rs | 236 ++ KubeOS-Rust/manager/src/sys_mgmt/mod.rs | 23 + KubeOS-Rust/manager/src/sys_mgmt/values.rs | 36 + KubeOS-Rust/manager/src/utils/common.rs | 310 ++ .../manager/src/utils/container_image.rs | 341 ++ KubeOS-Rust/manager/src/utils/executor.rs | 89 + .../manager/src/utils/image_manager.rs | 206 ++ KubeOS-Rust/manager/src/utils/mod.rs | 23 + KubeOS-Rust/manager/src/utils/partition.rs | 117 + KubeOS-Rust/operator/Cargo.toml | 47 + .../operator/src/controller/apiclient.rs | 110 + .../operator/src/controller/apiserver_mock.rs | 995 ++++++ .../operator/src/controller/controller.rs | 696 ++++ KubeOS-Rust/operator/src/controller/crd.rs | 79 + KubeOS-Rust/operator/src/controller/mod.rs | 24 + KubeOS-Rust/operator/src/controller/values.rs | 43 + KubeOS-Rust/operator/src/main.rs | 74 + KubeOS-Rust/proxy/Cargo.toml | 49 + .../proxy/src/controller/agentclient.rs | 153 + KubeOS-Rust/proxy/src/controller/apiclient.rs | 147 + .../proxy/src/controller/apiserver_mock.rs | 681 ++++ .../proxy/src/controller/controller.rs | 556 ++++ KubeOS-Rust/proxy/src/controller/crd.rs | 77 + KubeOS-Rust/proxy/src/controller/mod.rs | 26 + KubeOS-Rust/proxy/src/controller/utils.rs | 154 + KubeOS-Rust/proxy/src/controller/values.rs | 33 + KubeOS-Rust/proxy/src/drain.rs | 511 +++ KubeOS-Rust/proxy/src/main.rs | 49 + KubeOS-Rust/proxy/tests/common/mod.rs | 63 + KubeOS-Rust/proxy/tests/drain_test.rs | 41 + .../proxy/tests/setup/kind-config.yaml | 5 + KubeOS-Rust/proxy/tests/setup/resources.yaml | 102 + .../proxy/tests/setup/setup_test_env.sh | 81 + KubeOS-Rust/rustfmt.toml | 11 + Makefile | 13 +- VERSION | 2 +- .../config/crd/upgrade.openeuler.org_os.yaml | 2 +- .../upgrade.openeuler.org_osinstances.yaml | 8 +- docs/quick-start.md | 377 ++- scripts/bootloader.sh | 4 +- 67 files changed, 11710 insertions(+), 155 deletions(-) create mode 100644 KubeOS-Rust/Cargo.lock create mode 100644 KubeOS-Rust/Cargo.toml create mode 100644 KubeOS-Rust/agent/Cargo.toml create mode 100644 KubeOS-Rust/agent/src/function.rs create mode 100644 KubeOS-Rust/agent/src/main.rs create mode 100644 KubeOS-Rust/agent/src/rpc/agent.rs create mode 100644 KubeOS-Rust/agent/src/rpc/agent_impl.rs create mode 100644 KubeOS-Rust/agent/src/rpc/mod.rs create mode 100644 KubeOS-Rust/cli/Cargo.toml create mode 100644 KubeOS-Rust/cli/src/client.rs create mode 100644 KubeOS-Rust/cli/src/lib.rs create mode 100644 KubeOS-Rust/cli/src/method/callable_method.rs create mode 100644 KubeOS-Rust/cli/src/method/configure.rs create mode 100644 KubeOS-Rust/cli/src/method/mod.rs create mode 100644 KubeOS-Rust/cli/src/method/prepare_upgrade.rs create mode 100644 KubeOS-Rust/cli/src/method/request.rs create mode 100644 KubeOS-Rust/cli/src/method/rollback.rs create mode 100644 KubeOS-Rust/cli/src/method/upgrade.rs create mode 100644 KubeOS-Rust/manager/Cargo.toml create mode 100644 KubeOS-Rust/manager/src/api/agent_status.rs create mode 100644 KubeOS-Rust/manager/src/api/mod.rs create mode 100644 KubeOS-Rust/manager/src/api/types.rs create mode 100644 KubeOS-Rust/manager/src/lib.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/config.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/mod.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/values.rs create mode 100644 KubeOS-Rust/manager/src/utils/common.rs create mode 100644 KubeOS-Rust/manager/src/utils/container_image.rs create mode 100644 KubeOS-Rust/manager/src/utils/executor.rs create mode 100644 KubeOS-Rust/manager/src/utils/image_manager.rs create mode 100644 KubeOS-Rust/manager/src/utils/mod.rs create mode 100644 KubeOS-Rust/manager/src/utils/partition.rs create mode 100644 KubeOS-Rust/operator/Cargo.toml create mode 100644 KubeOS-Rust/operator/src/controller/apiclient.rs create mode 100644 KubeOS-Rust/operator/src/controller/apiserver_mock.rs create mode 100644 KubeOS-Rust/operator/src/controller/controller.rs create mode 100644 KubeOS-Rust/operator/src/controller/crd.rs create mode 100644 KubeOS-Rust/operator/src/controller/mod.rs create mode 100644 KubeOS-Rust/operator/src/controller/values.rs create mode 100644 KubeOS-Rust/operator/src/main.rs create mode 100644 KubeOS-Rust/proxy/Cargo.toml create mode 100644 KubeOS-Rust/proxy/src/controller/agentclient.rs create mode 100644 KubeOS-Rust/proxy/src/controller/apiclient.rs create mode 100644 KubeOS-Rust/proxy/src/controller/apiserver_mock.rs create mode 100644 KubeOS-Rust/proxy/src/controller/controller.rs create mode 100644 KubeOS-Rust/proxy/src/controller/crd.rs create mode 100644 KubeOS-Rust/proxy/src/controller/mod.rs create mode 100644 KubeOS-Rust/proxy/src/controller/utils.rs create mode 100644 KubeOS-Rust/proxy/src/controller/values.rs create mode 100644 KubeOS-Rust/proxy/src/drain.rs create mode 100644 KubeOS-Rust/proxy/src/main.rs create mode 100644 KubeOS-Rust/proxy/tests/common/mod.rs create mode 100644 KubeOS-Rust/proxy/tests/drain_test.rs create mode 100644 KubeOS-Rust/proxy/tests/setup/kind-config.yaml create mode 100644 KubeOS-Rust/proxy/tests/setup/resources.yaml create mode 100644 KubeOS-Rust/proxy/tests/setup/setup_test_env.sh create mode 100644 KubeOS-Rust/rustfmt.toml diff --git a/.gitignore b/.gitignore index 5e56e040..4d173c5e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,8 @@ +# vscode settings +.vscode + +# rust dependencies +target/ + +# KubeOS bin /bin diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock new file mode 100644 index 00000000..ad175d9c --- /dev/null +++ b/KubeOS-Rust/Cargo.lock @@ -0,0 +1,2882 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom 0.2.10", + "instant", + "rand 0.8.5", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.48.5", +] + +[[package]] +name = "cli" +version = "1.0.6" +dependencies = [ + "anyhow", + "jsonrpc", + "log", + "manager", + "serde", + "serde_json", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if", + "num_cpus", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "dyn-clone" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "globset" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "h2" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.9", + "indexmap 1.9.3", + "slab", + "tokio", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http 0.2.9", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.9", + "http-body 0.4.5", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.9", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.2.0", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.25", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.25", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.2.0", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.3", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-patch" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce" +dependencies = [ + "serde", + "serde_json", + "treediff", +] + +[[package]] +name = "jsonpath_lib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "jsonrpc" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd8d6b3f301ba426b30feca834a2a18d48d5b54e5065496b5c1b05537bee3639" +dependencies = [ + "base64 0.13.1", + "serde", + "serde_json", +] + +[[package]] +name = "jsonrpc-core" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +dependencies = [ + "futures", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "jsonrpc-derive" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "jsonrpc-ipc-server" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382bb0206323ca7cda3dcd7e245cea86d37d02457a02a975e3378fb149a48845" +dependencies = [ + "futures", + "jsonrpc-core", + "jsonrpc-server-utils", + "log", + "parity-tokio-ipc", + "parking_lot", + "tower-service", +] + +[[package]] +name = "jsonrpc-server-utils" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4" +dependencies = [ + "bytes", + "futures", + "globset", + "jsonrpc-core", + "lazy_static", + "log", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", + "unicase", +] + +[[package]] +name = "k8s-openapi" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c" +dependencies = [ + "base64 0.13.1", + "bytes", + "chrono", + "http 0.2.9", + "percent-encoding", + "serde", + "serde-value", + "serde_json", + "url", +] + +[[package]] +name = "kube" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4b96944d327b752df4f62f3a31d8694892af06fb585747c0b5e664927823d1a" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-derive", + "kube-runtime", +] + +[[package]] +name = "kube-client" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232db1af3d3680f9289cf0b4db51b2b9fee22550fc65d25869e39b23e0aaa696" +dependencies = [ + "base64 0.13.1", + "bytes", + "chrono", + "dirs-next", + "either", + "futures", + "http 0.2.9", + "http-body 0.4.5", + "hyper 0.14.25", + "hyper-timeout", + "hyper-tls 0.5.0", + "jsonpath_lib", + "k8s-openapi", + "kube-core", + "openssl", + "pem", + "pin-project", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tokio", + "tokio-native-tls", + "tokio-util 0.6.10", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de491f8c9ee97117e0b47a629753e939c2392d5d0a40f6928e582a5fba328098" +dependencies = [ + "chrono", + "form_urlencoded", + "http 0.2.9", + "json-patch", + "k8s-openapi", + "once_cell", + "schemars", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "kube-derive" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbb86bb3607245a67c8ad3a52aff41108f36b0d1e9e3e82ffb5760bfd84b965" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 1.0.109", +] + +[[package]] +name = "kube-runtime" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710729592eb30219b4e84898e91dc991fe09ccafe2c17fec4e45c3426c61abe0" +dependencies = [ + "backoff", + "dashmap", + "derivative", + "futures", + "json-patch", + "k8s-openapi", + "kube-client", + "pin-project", + "serde", + "serde_json", + "smallvec", + "thiserror", + "tokio", + "tokio-util 0.6.10", + "tracing", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" + +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.0", + "libc", + "redox_syscall 0.4.1", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4dcd960cc540667f619483fc99102f88d6118b87730e24e8fbe8054b7445e4" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "manager" +version = "1.0.6" +dependencies = [ + "anyhow", + "env_logger", + "lazy_static", + "log", + "mockall", + "mockito", + "nix", + "predicates", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "tempfile", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", +] + +[[package]] +name = "mockall" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4a1c770583dac7ab5e2f6c139153b783a53a1bbee9729613f193e59828326" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "mockito" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80f9fece9bd97ab74339fe19f4bcaf52b76dcc18e5364c7977c1838f76b38de9" +dependencies = [ + "assert-json-diff", + "httparse", + "lazy_static", + "log", + "rand 0.8.5", + "regex", + "serde_json", + "serde_urlencoded", + "similar", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", + "pin-utils", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.3", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" + +[[package]] +name = "openssl" +version = "0.10.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "operator" +version = "1.0.6" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-trait", + "cli", + "env_logger", + "futures", + "h2 0.3.16", + "http 0.2.9", + "hyper 0.14.25", + "k8s-openapi", + "kube", + "log", + "manager", + "mockall", + "regex", + "reqwest", + "schemars", + "serde", + "serde_json", + "socket2 0.4.9", + "thiserror", + "thread_local", + "tokio", + "tokio-retry", + "tower-test", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os-agent" +version = "1.0.6" +dependencies = [ + "anyhow", + "env_logger", + "jsonrpc-core", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "lazy_static", + "log", + "manager", + "nix", + "serde", + "serde_json", +] + +[[package]] +name = "parity-tokio-ipc" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9981e32fb75e004cc148f5fb70342f393830e0a4aa62e3cc93b50976218d42b6" +dependencies = [ + "futures", + "libc", + "log", + "rand 0.7.3", + "tokio", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "predicates" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc3d91237f5de3bcd9d927e24d03b495adb6135097b001cea7403e2d573d00a9" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da1c2388b1513e1b605fcec39a95e0a9e8ef088f71443ef37099fa9ae6673fcb" + +[[package]] +name = "predicates-tree" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d86de6de25020a36c6d3643a86d9a6a9f552107c0559c60ea03551b5e16c032" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proxy" +version = "1.0.6" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-trait", + "cli", + "env_logger", + "futures", + "h2 0.3.16", + "http 0.2.9", + "hyper 0.14.25", + "k8s-openapi", + "kube", + "log", + "manager", + "mockall", + "regex", + "reqwest", + "schemars", + "serde", + "serde_json", + "socket2 0.4.9", + "thiserror", + "thread_local", + "tokio", + "tokio-retry", + "tower-test", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.10", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom 0.2.10", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "reqwest" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" +dependencies = [ + "base64 0.21.5", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", + "hyper-rustls", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.10", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustls" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.5", +] + +[[package]] +name = "rustls-pki-types" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868e20fada228fefaf6b652e00cc73623d54f8171e7352c18bb281571f2d92da" + +[[package]] +name = "rustls-webpki" +version = "0.102.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "schemars" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1847b767a3d62d95cbf3d8a9f0e421cf57a0d8aa4f411d4b16525afb0284d4ed" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4d7e1b012cb3d9129567661a63755ea4b8a7386d339dc945ae187e403c6743" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "serde", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c4437699b6d34972de58652c68b98cb5b53a4199ab126db8e20ec8ded29a721" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_json" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +dependencies = [ + "indexmap 1.9.3", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap 1.9.3", + "ryu", + "serde", + "yaml-rust", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "similar" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +dependencies = [ + "autocfg", + "cfg-if", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "termcolor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.4.9", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-test" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89b3cbabd3ae862100094ae433e1def582cf86451b4e9bf83aa7ac1d8a7d719" +dependencies = [ + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "slab", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tokio-util 0.7.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8" +dependencies = [ + "base64 0.13.1", + "bitflags 1.3.2", + "bytes", + "futures-core", + "futures-util", + "http 0.2.9", + "http-body 0.4.5", + "http-range-header", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tower-test" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4546773ffeab9e4ea02b8872faa49bb616a80a7da66afc2f32688943f97efa7" +dependencies = [ + "futures-util", + "pin-project", + "tokio", + "tokio-test", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tracing" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "treediff" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff" +dependencies = [ + "serde_json", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" + +[[package]] +name = "web-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml new file mode 100644 index 00000000..01e05b11 --- /dev/null +++ b/KubeOS-Rust/Cargo.toml @@ -0,0 +1,12 @@ +[workspace] +members = ["agent", "cli", "manager", "operator", "proxy"] +resolver = "2" + +[profile.release] +debug = false +debug-assertions = false +lto = true +opt-level = 's' +overflow-checks = false +panic = "unwind" +rpath = false diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml new file mode 100644 index 00000000..83e1b7c0 --- /dev/null +++ b/KubeOS-Rust/agent/Cargo.toml @@ -0,0 +1,20 @@ +[package] +description = "KubeOS os-agent" +edition = "2021" +license = "MulanPSL-2.0" +name = "os-agent" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +anyhow = { version = "1.0" } +env_logger = { version = "0.9" } +jsonrpc-core = { version = "18.0" } +jsonrpc-derive = { version = "18.0" } +jsonrpc-ipc-server = { version = "18.0" } +lazy_static = { version = "1.4" } +log = { version = "= 0.4.15" } +manager = { package = "manager", path = "../manager" } +nix = { version = "0.26.2" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } diff --git a/KubeOS-Rust/agent/src/function.rs b/KubeOS-Rust/agent/src/function.rs new file mode 100644 index 00000000..9789d95c --- /dev/null +++ b/KubeOS-Rust/agent/src/function.rs @@ -0,0 +1,56 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub use jsonrpc_core::Result as RpcResult; +use jsonrpc_core::{Error, ErrorCode}; +pub use jsonrpc_derive::rpc; +use log::error; + +const RPC_OP_ERROR: i64 = -1; + +pub struct RpcFunction; + +impl RpcFunction { + pub fn call(f: F) -> RpcResult + where + F: FnOnce() -> anyhow::Result, + { + (f)().map_err(|e| { + let error_message = format!("{:#}", e); + error!("{}", error_message.replace('\n', " ").replace('\r', "")); + Error { code: ErrorCode::ServerError(RPC_OP_ERROR), message: format!("{:?}", e), data: None } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rpcfunction_call() { + // Define a mock function that returns a result + fn mock_ok_function() -> anyhow::Result { + Ok(42) + } + let result: RpcResult = RpcFunction::call(mock_ok_function); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), 42); + + fn mock_err_function() -> anyhow::Result { + Err(anyhow::anyhow!("error")) + } + let result: RpcResult = RpcFunction::call(mock_err_function); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code, ErrorCode::ServerError(RPC_OP_ERROR)); + } +} diff --git a/KubeOS-Rust/agent/src/main.rs b/KubeOS-Rust/agent/src/main.rs new file mode 100644 index 00000000..cd95ef07 --- /dev/null +++ b/KubeOS-Rust/agent/src/main.rs @@ -0,0 +1,66 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{self, DirBuilder, Permissions}, + os::unix::fs::{DirBuilderExt, PermissionsExt}, + path::Path, +}; + +use env_logger::{Builder, Env, Target}; +use jsonrpc_core::{IoHandler, IoHandlerExtension}; +use jsonrpc_ipc_server::ServerBuilder; + +mod function; +mod rpc; + +use log::info; +use rpc::{Agent, AgentImpl}; + +const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; +const CARGO_PKG_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); + +fn start_and_run(sock_path: &str) { + let socket_path = Path::new(sock_path); + + // Create directory for socket if it doesn't exist + if let Some(dir_path) = socket_path.parent() { + if !dir_path.exists() { + DirBuilder::new().mode(0o750).create(dir_path).expect("Couldn't create directory for socket"); + } + } + + // Add RPC methods to IoHandler + let mut io = IoHandler::new(); + AgentImpl::default().to_delegate().augment(&mut io); + + // Build and start server + let builder = ServerBuilder::new(io); + let server = builder.start(sock_path).expect("Couldn't open socket"); + + let gid = nix::unistd::getgid(); + nix::unistd::chown(socket_path, Some(nix::unistd::ROOT), Some(gid)).expect("Couldn't set socket group"); + + // Set socket permissions to 0640 + let socket_permissions = Permissions::from_mode(0o640); + fs::set_permissions(socket_path, socket_permissions).expect("Couldn't set socket permissions"); + + info!("os-agent started, waiting for requests..."); + server.wait(); +} + +fn main() { + Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init(); + + info!("os-agent version is: {}", CARGO_PKG_VERSION.unwrap_or("NOT FOUND")); + start_and_run(SOCK_PATH); +} diff --git a/KubeOS-Rust/agent/src/rpc/agent.rs b/KubeOS-Rust/agent/src/rpc/agent.rs new file mode 100644 index 00000000..2496bfb4 --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/agent.rs @@ -0,0 +1,30 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use manager::api::{ConfigureRequest, Response, UpgradeRequest}; + +use super::function::{rpc, RpcResult}; + +#[rpc(server)] +pub trait Agent { + #[rpc(name = "prepare_upgrade")] + fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult; + + #[rpc(name = "upgrade")] + fn upgrade(&self) -> RpcResult; + + #[rpc(name = "configure")] + fn configure(&self, req: ConfigureRequest) -> RpcResult; + + #[rpc(name = "rollback")] + fn rollback(&self) -> RpcResult; +} diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs new file mode 100644 index 00000000..ab826413 --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -0,0 +1,217 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{sync::Mutex, thread, time::Duration}; + +use anyhow::{bail, Result}; +use log::{debug, info}; +use manager::{ + api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, + sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH}, + utils::{get_partition_info, switch_boot_menuentry, RealCommandExecutor}, +}; +use nix::{sys::reboot::RebootMode, unistd::sync}; + +use super::{ + agent::Agent, + function::{RpcFunction, RpcResult}, +}; + +pub struct AgentImpl { + mutex: Mutex<()>, + disable_reboot: bool, +} + +impl Agent for AgentImpl { + fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult { + RpcFunction::call(|| self.prepare_upgrade_impl(req)) + } + + fn upgrade(&self) -> RpcResult { + RpcFunction::call(|| self.upgrade_impl()) + } + + fn configure(&self, req: ConfigureRequest) -> RpcResult { + RpcFunction::call(|| self.configure_impl(req)) + } + + fn rollback(&self) -> RpcResult { + RpcFunction::call(|| self.rollback_impl()) + } +} + +impl Default for AgentImpl { + fn default() -> Self { + Self { mutex: Mutex::new(()), disable_reboot: false } + } +} + +impl AgentImpl { + fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + debug!("Received an 'prepare upgrade' request: {:?}", req); + info!("Start preparing for upgrading to version: {}", req.version); + + let handler: Box> = match req.image_type.as_str() { + "containerd" => Box::new(ImageType::Containerd(CtrImageHandler::default())), + "docker" => Box::new(ImageType::Docker(DockerImageHandler::default())), + "disk" => Box::new(ImageType::Disk(DiskImageHandler::default())), + _ => bail!("Invalid image type \"{}\"", req.image_type), + }; + + let image_manager = handler.download_image(&req)?; + info!("Ready to install image: {:?}", image_manager.paths.image_path.display()); + image_manager.install()?; + + Ok(Response { status: AgentStatus::UpgradeReady }) + } + + fn upgrade_impl(&self) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + info!("Start to upgrade"); + let command_executor = RealCommandExecutor {}; + let (_, next_partition_info) = get_partition_info(&command_executor)?; + + // based on boot mode use different command to switch boot partition + let device = next_partition_info.device.as_str(); + let menuentry = next_partition_info.menuentry.as_str(); + switch_boot_menuentry(&command_executor, DEFAULT_GRUBENV_PATH, menuentry)?; + info!("Switch to boot partition: {}, device: {}", menuentry, device); + self.reboot()?; + Ok(Response { status: AgentStatus::Upgraded }) + } + + fn configure_impl(&self, mut req: ConfigureRequest) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + debug!("Received a 'configure' request: {:?}", req); + info!("Start to configure"); + let config_map = &*CONFIG_TEMPLATE; + for config in req.configs.iter_mut() { + let config_type = &config.model; + if let Some(configuration) = config_map.get(config_type) { + debug!("Found configuration type: \"{}\"", config_type); + configuration.set_config(config)?; + } else { + bail!("Unknown configuration type: \"{}\"", config_type); + } + } + Ok(Response { status: AgentStatus::Configured }) + } + + fn rollback_impl(&self) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + info!("Start to rollback"); + let command_executor = RealCommandExecutor {}; + let (_, next_partition_info) = get_partition_info(&command_executor)?; + switch_boot_menuentry( + &command_executor, + manager::sys_mgmt::DEFAULT_GRUBENV_PATH, + &next_partition_info.menuentry, + )?; + info!("Switch to boot partition: {}, device: {}", next_partition_info.menuentry, next_partition_info.device); + self.reboot()?; + Ok(Response { status: AgentStatus::Rollbacked }) + } + + fn reboot(&self) -> Result<()> { + info!("Wait to reboot"); + thread::sleep(Duration::from_secs(1)); + sync(); + if self.disable_reboot { + return Ok(()); + } + nix::sys::reboot::reboot(RebootMode::RB_AUTOBOOT)?; + Ok(()) + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + + use manager::api::{CertsInfo, Sysconfig}; + + use super::*; + + #[test] + fn test_reboot() { + let mut agent = AgentImpl::default(); + agent.disable_reboot = true; + let res = agent.reboot(); + assert!(res.is_ok()); + } + + #[test] + fn test_configure() { + let agent = AgentImpl::default(); + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "kernel.sysctl".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req).unwrap(); + assert_eq!(res, Response { status: AgentStatus::Configured }); + + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "invalid".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req); + assert!(res.is_err()); + + // test lock + let _lock = agent.mutex.lock().unwrap(); + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "kernel.sysctl".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req); + assert!(res.is_err()); + } + + #[test] + fn test_prepare_upgrade() { + let agent = AgentImpl::default(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "xxx".into(), + image_type: "xxx".into(), + container_image: "xxx".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let res = agent.prepare_upgrade(req); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/agent/src/rpc/mod.rs b/KubeOS-Rust/agent/src/rpc/mod.rs new file mode 100644 index 00000000..976356be --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/mod.rs @@ -0,0 +1,19 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use super::function; + +mod agent; +mod agent_impl; + +pub use agent::*; +pub use agent_impl::*; diff --git a/KubeOS-Rust/cli/Cargo.toml b/KubeOS-Rust/cli/Cargo.toml new file mode 100644 index 00000000..78d5fd51 --- /dev/null +++ b/KubeOS-Rust/cli/Cargo.toml @@ -0,0 +1,15 @@ +[package] +description = "KubeOS os-agent client" +edition = "2021" +license = "MulanPSL-2.0" +name = "cli" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +anyhow = { version = "1.0" } +jsonrpc = { version = "0.13", features = ["simple_uds"] } +kubeos-manager = { package = "manager", path = "../manager" } +log = { version = "0.4" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs new file mode 100644 index 00000000..37518bdc --- /dev/null +++ b/KubeOS-Rust/cli/src/client.rs @@ -0,0 +1,56 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::path::Path; + +use jsonrpc::{ + simple_uds::UdsTransport, Client as JsonRPCClient, Request as JsonRPCRequest, Response as JsonRPCResponse, +}; +use serde_json::value::RawValue; + +pub struct Client { + json_rpc_client: JsonRPCClient, +} + +pub struct Request<'a>(JsonRPCRequest<'a>); + +impl<'a> Request<'a> {} + +impl Client { + pub fn new>(socket_path: P) -> Self { + Client { json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)) } + } + + pub fn build_request<'a>(&self, command: &'a str, params: &'a [Box]) -> Request<'a> { + let json_rpc_request = self.json_rpc_client.build_request(command, params); + let request = Request(json_rpc_request); + request + } + + pub fn send_request(&self, request: Request) -> Result { + self.json_rpc_client.send_request(request.0) + } +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_client() { + let socket_path = "/tmp/KubeOS-test.sock"; + let cli = Client::new(socket_path); + let command = "example_command"; + let params = vec![]; + let request = cli.send_request(cli.build_request(command, ¶ms)); + assert!(request.is_err()); + } +} diff --git a/KubeOS-Rust/cli/src/lib.rs b/KubeOS-Rust/cli/src/lib.rs new file mode 100644 index 00000000..cd66d72f --- /dev/null +++ b/KubeOS-Rust/cli/src/lib.rs @@ -0,0 +1,14 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod client; +pub mod method; diff --git a/KubeOS-Rust/cli/src/method/callable_method.rs b/KubeOS-Rust/cli/src/method/callable_method.rs new file mode 100644 index 00000000..a174b5b8 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/callable_method.rs @@ -0,0 +1,54 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde_json::value::RawValue; + +use super::request::{parse_error, request}; +use crate::client::Client; + +pub trait RpcMethod { + type Response: serde::de::DeserializeOwned; + fn command_name(&self) -> &'static str; + fn command_params(&self) -> Vec>; + fn call(&self, client: &Client) -> Result { + let response = request(client, self.command_name(), self.command_params())?; + response.result().map_err(parse_error) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::client; + + #[derive(Default)] + struct DummyMethod; + + impl RpcMethod for DummyMethod { + type Response = String; + + fn command_name(&self) -> &'static str { + "dummy_command" + } + + fn command_params(&self) -> Vec> { + vec![] + } + } + + #[test] + fn test_call() { + let client = client::Client::new("/tmp/KubeOS-test.sock"); + let result = DummyMethod::default().call(&client); + assert!(result.is_err()); + } +} diff --git a/KubeOS-Rust/cli/src/method/configure.rs b/KubeOS-Rust/cli/src/method/configure.rs new file mode 100644 index 00000000..cca752d0 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/configure.rs @@ -0,0 +1,72 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::{to_raw_value, RawValue}; + +use crate::method::callable_method::RpcMethod; + +pub struct ConfigureMethod { + req: api::ConfigureRequest, +} + +impl ConfigureMethod { + pub fn new(req: api::ConfigureRequest) -> Self { + ConfigureMethod { req } + } + + pub fn set_configure_request(&mut self, req: api::ConfigureRequest) -> &Self { + self.req = req; + self + } +} + +impl RpcMethod for ConfigureMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "configure" + } + fn command_params(&self) -> Vec> { + vec![to_raw_value(&self.req).unwrap()] + } +} +#[cfg(test)] +mod tests { + use kubeos_manager::api::{ConfigureRequest, Sysconfig}; + + use super::*; + + #[test] + fn test_configure_method() { + let req = ConfigureRequest { configs: vec![] }; + let mut method = ConfigureMethod::new(req); + + // Test set_configure_request method + let new_req = ConfigureRequest { + configs: vec![Sysconfig { + model: "model".to_string(), + config_path: "config_path".to_string(), + contents: Default::default(), + }], + }; + method.set_configure_request(new_req); + + // Test command_name method + assert_eq!(method.command_name(), "configure"); + + // Test command_params method + let expected_params = + "RawValue({\"configs\":[{\"model\":\"model\",\"config_path\":\"config_path\",\"contents\":{}}]})"; + let actual_params = format!("{:?}", method.command_params()[0]); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/mod.rs b/KubeOS-Rust/cli/src/method/mod.rs new file mode 100644 index 00000000..e1f38bcd --- /dev/null +++ b/KubeOS-Rust/cli/src/method/mod.rs @@ -0,0 +1,18 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod callable_method; +pub mod configure; +pub mod prepare_upgrade; +pub mod request; +pub mod rollback; +pub mod upgrade; diff --git a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs new file mode 100644 index 00000000..f2034f6b --- /dev/null +++ b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs @@ -0,0 +1,78 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::{to_raw_value, RawValue}; + +use crate::method::callable_method::RpcMethod; + +pub struct PrepareUpgradeMethod { + req: api::UpgradeRequest, +} + +impl PrepareUpgradeMethod { + pub fn new(req: api::UpgradeRequest) -> Self { + PrepareUpgradeMethod { req } + } + + pub fn set_prepare_upgrade_request(&mut self, req: api::UpgradeRequest) -> &Self { + self.req = req; + self + } +} + +impl RpcMethod for PrepareUpgradeMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "prepare_upgrade" + } + fn command_params(&self) -> Vec> { + vec![to_raw_value(&self.req).unwrap()] + } +} +#[cfg(test)] +mod tests { + use kubeos_manager::api::{CertsInfo, UpgradeRequest}; + + use super::*; + + #[test] + fn test_prepare_upgrade_method() { + let req = UpgradeRequest { + version: "v1".into(), + check_sum: "".into(), + image_type: "".into(), + container_image: "".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let mut method = PrepareUpgradeMethod::new(req); + let new_req = UpgradeRequest { + version: "v2".into(), + check_sum: "xxx".into(), + image_type: "xxx".into(), + container_image: "xxx".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + method.set_prepare_upgrade_request(new_req); + assert_eq!(method.command_name(), "prepare_upgrade"); + + let expected_params = "RawValue({\"version\":\"v2\",\"check_sum\":\"xxx\",\"image_type\":\"xxx\",\"container_image\":\"xxx\",\"image_url\":\"\",\"flag_safe\":false,\"mtls\":false,\"certs\":{\"ca_cert\":\"\",\"client_cert\":\"\",\"client_key\":\"\"}})"; + let actual_params = format!("{:?}", method.command_params()[0]); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs new file mode 100644 index 00000000..581aa639 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/request.rs @@ -0,0 +1,88 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::anyhow; +use jsonrpc::{Error, Response}; +use log::debug; +use serde_json::value::RawValue; + +use crate::client::Client; + +pub fn request(client: &Client, command: &str, params: Vec>) -> Result { + let request = client.build_request(command, ¶ms); + let response = client.send_request(request).map_err(parse_error); + debug!("{:#?}", response); + response +} + +pub fn parse_error(error: Error) -> anyhow::Error { + match error { + Error::Transport(e) => { + anyhow!( + "Cannot connect to KubeOS os-agent unix socket, {}", + e.source().map(|e| e.to_string()).unwrap_or_else(|| "Connection timeout".to_string()) + ) + }, + Error::Json(e) => { + debug!("Json parse error: {:?}", e); + anyhow!("Failed to parse response") + }, + Error::Rpc(ref e) => { + if e.message == "Method not found" { + anyhow!("Method is unimplemented") + } else { + anyhow!("{}", e.message) + } + }, + _ => { + debug!("{:?}", error); + anyhow!("Response is invalid") + }, + } +} + +#[cfg(test)] +mod tests { + use jsonrpc::error::RpcError; + use serde::de::Error as DeError; + + use super::*; + + #[test] + fn test_parse_error() { + // Test Error::Transport + let transport_error = + Error::Transport(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "Connection timeout"))); + let result = parse_error(transport_error); + assert_eq!(result.to_string(), "Cannot connect to KubeOS os-agent unix socket, Connection timeout"); + + // Test Error::Json + let json_error = Error::Json(serde_json::Error::custom("Failed to parse response")); + let result = parse_error(json_error); + assert_eq!(result.to_string(), "Failed to parse response"); + + // Test Error::Rpc with "Method not found" message + let rpc_error = Error::Rpc(RpcError { code: -32601, message: "Method not found".to_string(), data: None }); + let result = parse_error(rpc_error); + assert_eq!(result.to_string(), "Method is unimplemented"); + + // Test Error::Rpc with other message + let rpc_error = Error::Rpc(RpcError { code: -32603, message: "Internal server error".to_string(), data: None }); + let result = parse_error(rpc_error); + assert_eq!(result.to_string(), "Internal server error"); + + // Test other Error variant + let other_error = Error::VersionMismatch; + let result = parse_error(other_error); + assert_eq!(result.to_string(), "Response is invalid"); + } +} diff --git a/KubeOS-Rust/cli/src/method/rollback.rs b/KubeOS-Rust/cli/src/method/rollback.rs new file mode 100644 index 00000000..7945f4b1 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/rollback.rs @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::RawValue; + +use crate::method::callable_method::RpcMethod; + +#[derive(Default)] +pub struct RollbackMethod {} + +impl RpcMethod for RollbackMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "rollback" + } + fn command_params(&self) -> Vec> { + vec![] + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_rollback_method() { + let method = RollbackMethod::default(); + assert_eq!(method.command_name(), "rollback"); + let expected_params = "[]"; + let actual_params = format!("{:?}", method.command_params()); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/upgrade.rs b/KubeOS-Rust/cli/src/method/upgrade.rs new file mode 100644 index 00000000..f2f94cd5 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/upgrade.rs @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::RawValue; + +use crate::method::callable_method::RpcMethod; + +#[derive(Default)] +pub struct UpgradeMethod {} + +impl RpcMethod for UpgradeMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "upgrade" + } + fn command_params(&self) -> Vec> { + vec![] + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_upgrade_method() { + let method = UpgradeMethod::default(); + assert_eq!(method.command_name(), "upgrade"); + let expected_params = "[]"; + let actual_params = format!("{:?}", method.command_params()); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml new file mode 100644 index 00000000..f60a7c08 --- /dev/null +++ b/KubeOS-Rust/manager/Cargo.toml @@ -0,0 +1,25 @@ +[package] +description = "KubeOS os-agent manager" +edition = "2021" +license = "MulanPSL-2.0" +name = "manager" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dev-dependencies] +mockall = { version = "=0.11.3" } +mockito = { version = "0.31.1", default-features = false } +predicates = { version = "=2.0.1" } +tempfile = { version = "3.6.0" } + +[dependencies] +anyhow = { version = "1.0" } +env_logger = { version = "0.9" } +lazy_static = { version = "1.4" } +log = { version = "0.4" } +nix = { version = "0.26.2" } +regex = { version = "1.7.3" } +reqwest = { version = "=0.12.2", features = ["blocking", "rustls-tls"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } +sha2 = { version = "0.10.8" } diff --git a/KubeOS-Rust/manager/src/api/agent_status.rs b/KubeOS-Rust/manager/src/api/agent_status.rs new file mode 100644 index 00000000..bb16e6bc --- /dev/null +++ b/KubeOS-Rust/manager/src/api/agent_status.rs @@ -0,0 +1,21 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)] +pub enum AgentStatus { + UpgradeReady, + Upgraded, + Rollbacked, + Configured, +} diff --git a/KubeOS-Rust/manager/src/api/mod.rs b/KubeOS-Rust/manager/src/api/mod.rs new file mode 100644 index 00000000..01c9df1a --- /dev/null +++ b/KubeOS-Rust/manager/src/api/mod.rs @@ -0,0 +1,17 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod agent_status; +mod types; + +pub use agent_status::*; +pub use types::*; diff --git a/KubeOS-Rust/manager/src/api/types.rs b/KubeOS-Rust/manager/src/api/types.rs new file mode 100644 index 00000000..98aeaa33 --- /dev/null +++ b/KubeOS-Rust/manager/src/api/types.rs @@ -0,0 +1,140 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::agent_status::*; +use crate::{ + sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler}, + utils::{CommandExecutor, UpgradeImageManager}, +}; + +#[derive(Deserialize, Serialize, Debug)] +pub struct UpgradeRequest { + pub version: String, + pub check_sum: String, + pub image_type: String, + pub container_image: String, + pub image_url: String, + pub flag_safe: bool, + pub mtls: bool, + pub certs: CertsInfo, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct CertsInfo { + pub ca_cert: String, + pub client_cert: String, + pub client_key: String, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct KeyInfo { + pub value: String, + pub operation: String, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct Sysconfig { + pub model: String, + pub config_path: String, + pub contents: HashMap, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct ConfigureRequest { + pub configs: Vec, +} + +#[derive(Deserialize, Serialize, Debug, PartialEq)] +pub struct Response { + pub status: AgentStatus, +} + +pub enum ImageType { + Containerd(CtrImageHandler), + Docker(DockerImageHandler), + Disk(DiskImageHandler), +} + +impl ImageType { + pub fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result> { + match self { + ImageType::Containerd(handler) => handler.download_image(req), + ImageType::Docker(handler) => handler.download_image(req), + ImageType::Disk(handler) => handler.download_image(req), + } + } +} +pub trait ImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result>; +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use mockall::mock; + + use super::*; + use crate::utils::PreparePath; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + #[test] + fn test_download_image() { + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: "kubeos-temp".to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + let mut mock_executor1 = MockCommandExec::new(); + mock_executor1.expect_run_command().returning(|_, _| Ok(())); + mock_executor1.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let c_handler = CtrImageHandler::new(PreparePath::default(), mock_executor1); + let image_type = ImageType::Containerd(c_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + + let mut mock_executor2 = MockCommandExec::new(); + mock_executor2.expect_run_command().returning(|_, _| Ok(())); + mock_executor2.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let docker_handler = DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor2); + let image_type = ImageType::Docker(docker_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + + let mut mock_executor3 = MockCommandExec::new(); + mock_executor3.expect_run_command().returning(|_, _| Ok(())); + mock_executor3.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let disk_handler = DiskImageHandler::new(PreparePath::default(), mock_executor3, "test".into()); + let image_type = ImageType::Disk(disk_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + } +} diff --git a/KubeOS-Rust/manager/src/lib.rs b/KubeOS-Rust/manager/src/lib.rs new file mode 100644 index 00000000..b45cab99 --- /dev/null +++ b/KubeOS-Rust/manager/src/lib.rs @@ -0,0 +1,15 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod api; +pub mod sys_mgmt; +pub mod utils; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs new file mode 100644 index 00000000..138df9da --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -0,0 +1,558 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + collections::HashMap, + fs::{self, File}, + io::{self, BufRead, BufWriter, Write}, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + string::String, +}; + +use anyhow::{bail, Context, Result}; +use lazy_static::lazy_static; +use log::{debug, info, trace, warn}; +use regex::Regex; + +use crate::{api::*, sys_mgmt::values, utils::*}; + +lazy_static! { + pub static ref CONFIG_TEMPLATE: HashMap> = { + let mut config_map = HashMap::new(); + config_map.insert( + values::KERNEL_SYSCTL.to_string(), + Box::new(KernelSysctl::new(values::DEFAULT_PROC_PATH)) as Box, + ); + config_map.insert( + values::KERNEL_SYSCTL_PERSIST.to_string(), + Box::new(KernelSysctlPersist) as Box, + ); + config_map.insert( + values::GRUB_CMDLINE_CURRENT.to_string(), + Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: true }) + as Box, + ); + config_map.insert( + values::GRUB_CMDLINE_NEXT.to_string(), + Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: false }) + as Box, + ); + config_map + }; +} + +pub trait Configuration { + fn set_config(&self, config: &mut Sysconfig) -> Result<()>; +} + +pub struct KernelSysctl { + pub proc_path: String, +} +pub struct KernelSysctlPersist; +pub struct GrubCmdline { + pub grub_path: String, + pub is_cur_partition: bool, +} + +impl Configuration for KernelSysctl { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + info!("Start setting kernel.sysctl"); + for (key, key_info) in config.contents.iter() { + let proc_path = self.get_proc_path(key); + if key_info.operation == "delete" { + warn!("Failed to delete kernel.sysctl config with key \"{}\"", key); + } else if !key_info.value.is_empty() && key_info.operation.is_empty() { + fs::write(&proc_path, format!("{}\n", &key_info.value).as_bytes()) + .with_context(|| format!("Failed to write kernel.sysctl with key: \"{}\"", key))?; + info!("Configured kernel.sysctl {}={}", key, key_info.value); + } else { + warn!( + "Failed to parse kernel.sysctl, key: \"{}\", value: \"{}\", operation: \"{}\"", + key, key_info.value, key_info.operation + ); + } + } + Ok(()) + } +} + +impl KernelSysctl { + fn new(proc_path: &str) -> Self { + Self { proc_path: String::from(proc_path) } + } + + fn get_proc_path(&self, key: &str) -> PathBuf { + let path_str = format!("{}{}", self.proc_path, key.replace('.', "/")); + Path::new(&path_str).to_path_buf() + } +} + +impl Configuration for KernelSysctlPersist { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + info!("Start setting kernel.sysctl.persist"); + let mut config_path = &values::DEFAULT_KERNEL_CONFIG_PATH.to_string(); + if !config.config_path.is_empty() { + config_path = &config.config_path; + } + debug!("kernel.sysctl.persist config_path: \"{}\"", config_path); + create_config_file(config_path).with_context(|| format!("Failed to find config path \"{}\"", config_path))?; + let configs = get_and_set_configs(&mut config.contents, config_path) + .with_context(|| format!("Failed to set persist kernel configs \"{}\"", config_path))?; + write_configs_to_file(config_path, &configs).with_context(|| "Failed to write configs to file".to_string())?; + Ok(()) + } +} + +fn create_config_file(config_path: &str) -> Result<()> { + if !is_file_exist(config_path) { + let f = fs::File::create(config_path)?; + let metadata = f.metadata()?; + let mut permissions = metadata.permissions(); + permissions.set_mode(values::DEFAULT_KERNEL_CONFIG_PERM); + debug!("Create file {} with permission 0644", config_path); + } + Ok(()) +} + +fn get_and_set_configs(expect_configs: &mut HashMap, config_path: &str) -> Result> { + let f = File::open(config_path).with_context(|| format!("Failed to open config path \"{}\"", config_path))?; + let mut configs_write = Vec::new(); + for line in io::BufReader::new(f).lines() { + let line = line?; + // if line is a comment or blank + if line.starts_with('#') || line.starts_with(';') || line.trim().is_empty() { + configs_write.push(line); + continue; + } + let config_kv: Vec<&str> = line.splitn(2, '=').map(|s| s.trim()).collect(); + // if config_kv is not a key-value pair + if config_kv.len() != 2 { + bail!("could not parse sysctl config {}", line); + } + let new_key_info = expect_configs.get(config_kv[0]); + let new_config = match new_key_info { + Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config_kv, new_key_info), + Some(new_key_info) => handle_update_key(&config_kv, new_key_info), + None => config_kv.join("="), + }; + configs_write.push(new_config); + expect_configs.remove(config_kv[0]); + } + let new_config = handle_add_key(expect_configs, false); + configs_write.extend(new_config); + Ok(configs_write) +} + +fn write_configs_to_file(config_path: &str, configs: &Vec) -> Result<()> { + info!("Write configuration to file \"{}\"", config_path); + let f = File::create(config_path)?; + let mut w = BufWriter::new(f); + for line in configs { + if line.is_empty() { + continue; + } + writeln!(w, "{}", line.as_str())?; + } + w.flush().with_context(|| format!("Failed to flush file {}", config_path))?; + w.get_mut().sync_all().with_context(|| "Failed to sync".to_string())?; + debug!("Write configuration to file \"{}\" success", config_path); + Ok(()) +} + +fn handle_delete_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String { + let key = config_kv[0]; + if config_kv.len() == 1 && new_config_info.value.is_empty() { + info!("Delete configuration key: \"{}\"", key); + return String::from(""); + } else if config_kv.len() == 1 && !new_config_info.value.is_empty() { + warn!("Failed to delete key \"{}\" with inconsistent values \"nil\" and \"{}\"", key, new_config_info.value); + return key.to_string(); + } + let old_value = config_kv[1]; + if old_value != new_config_info.value { + warn!( + "Failed to delete key \"{}\" with inconsistent values \"{}\" and \"{}\"", + key, old_value, new_config_info.value + ); + return config_kv.join("="); + } + info!("Delete configuration {}={}", key, old_value); + String::new() +} + +fn handle_update_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String { + let key = config_kv[0]; + if !new_config_info.operation.is_empty() { + warn!( + "Unknown operation \"{}\", updating key \"{}\" with value \"{}\" by default", + new_config_info.operation, key, new_config_info.value + ); + } + if config_kv.len() == values::ONLY_KEY && new_config_info.value.is_empty() { + return key.to_string(); + } + let new_value = new_config_info.value.trim(); + if config_kv.len() == values::ONLY_KEY && !new_config_info.value.is_empty() { + info!("Update configuration \"{}={}\"", key, new_value); + return format!("{}={}", key, new_value); + } + if new_config_info.value.is_empty() { + warn!("Failed to update key \"{}\" with \"null\" value", key); + return config_kv.join("="); + } + info!("Update configuration \"{}={}\"", key, new_value); + format!("{}={}", key, new_value) +} + +fn handle_add_key(expect_configs: &HashMap, is_only_key_valid: bool) -> Vec { + let mut configs_write = Vec::new(); + for (key, config_info) in expect_configs.iter() { + if config_info.operation == "delete" { + warn!("Failed to delete inexistent key: \"{}\"", key); + continue; + } + if key.is_empty() || key.contains('=') { + warn!("Failed to add \"null\" key or key containing \"=\", key: \"{}\"", key); + continue; + } + if !config_info.operation.is_empty() { + warn!( + "Unknown operation \"{}\", adding key \"{}\" with value \"{}\" by default", + config_info.operation, key, config_info.value + ); + } + let (k, v) = (key.trim(), config_info.value.trim()); + if v.is_empty() && is_only_key_valid { + info!("Add configuration \"{}\"", k); + configs_write.push(k.to_string()); + } else if v.is_empty() { + warn!("Failed to add key \"{}\" with \"null\" value", k); + } else { + info!("Add configuration \"{}={}\"", k, v); + configs_write.push(format!("{}={}", k, v)); + } + } + configs_write +} + +impl Configuration for GrubCmdline { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + if self.is_cur_partition { + info!("Start setting grub.cmdline.current configuration"); + } else { + info!("Start setting grub.cmdline.next configuration"); + } + if !is_file_exist(&self.grub_path) { + bail!("Failed to find grub.cfg file"); + } + let config_partition = if cfg!(test) { + self.is_cur_partition + } else { + self.get_config_partition(RealCommandExecutor {}) + .with_context(|| "Failed to get config partition".to_string())? + }; + debug!("Config_partition: {} (false means partition A, true means partition B)", config_partition); + let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition) + .with_context(|| "Failed to set grub configs".to_string())?; + write_configs_to_file(&self.grub_path, &configs) + .with_context(|| "Failed to write configs to file".to_string())?; + Ok(()) + } +} + +impl GrubCmdline { + // get_config_partition returns false if the menuentry to be configured is A, true for menuentry B + fn get_config_partition(&self, executor: T) -> Result { + let (_, next_partition) = get_partition_info(&executor)?; + let mut flag = false; + if next_partition.menuentry == "B" { + flag = true + } + Ok(self.is_cur_partition != flag) + } +} + +fn get_and_set_grubcfg( + expect_configs: &mut HashMap, + grub_path: &str, + config_partition: bool, +) -> Result> { + let f = File::open(grub_path).with_context(|| format!("Failed to open grub.cfg \"{}\"", grub_path))?; + let re_find_cur_linux = r"^\s*linux.*root=.*"; + let re = Regex::new(re_find_cur_linux)?; + let mut configs_write = Vec::new(); + let mut match_config_partition = false; + for line in io::BufReader::new(f).lines() { + let mut line = line?; + if re.is_match(&line) { + if match_config_partition == config_partition { + line = modify_boot_cfg(expect_configs, &line)?; + } + match_config_partition = true; + } + configs_write.push(line); + } + Ok(configs_write) +} + +fn modify_boot_cfg(expect_configs: &mut HashMap, line: &String) -> Result { + trace!("Match partition that need to be configured, entering modify_boot_cfg, linux line: {}", line); + let mut new_configs = vec![" ".to_string()]; + let olg_configs: Vec<&str> = line.split(' ').collect(); + for old_config in olg_configs { + if old_config.is_empty() { + continue; + } + // At most 2 substrings can be returned to satisfy the case like root=UUID=xxxx + let config = old_config.splitn(2, '=').collect::>(); + if config.len() != values::ONLY_KEY && config.len() != values::KV_PAIR { + bail!("Failed to parse grub.cfg linux line {}", old_config); + } + let new_key_info = expect_configs.get(config[0]); + let new_config = match new_key_info { + Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config, new_key_info), + Some(new_key_info) => handle_update_key(&config, new_key_info), + None => config.join("="), + }; + if !new_config.is_empty() { + new_configs.push(new_config); + } + expect_configs.remove(config[0]); + } + let new_config = handle_add_key(expect_configs, true); + new_configs.extend(new_config); + Ok(new_configs.join(" ")) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use mockall::{mock, predicate::*}; + use tempfile::{NamedTempFile, TempDir}; + + use super::*; + use crate::sys_mgmt::{GRUB_CMDLINE_CURRENT, GRUB_CMDLINE_NEXT, KERNEL_SYSCTL, KERNEL_SYSCTL_PERSIST}; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_config_partition() { + init(); + let mut grub_cmdline = GrubCmdline { grub_path: String::from(""), is_cur_partition: true }; + let mut executor = MockCommandExec::new(); + + // the output shows that current root menuentry is A + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + + let result = grub_cmdline.get_config_partition(executor).unwrap(); + // it should return false because the current root menuentry is A and we want to configure current partition + assert_eq!(result, false); + + let mut executor = MockCommandExec::new(); + + // the output shows that current root menuentry is A + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + grub_cmdline.is_cur_partition = false; + let result = grub_cmdline.get_config_partition(executor).unwrap(); + // it should return true because the current root menuentry is A and we want to configure next partition + assert_eq!(result, true); + } + + #[test] + fn test_kernel_sysctl() { + init(); + let tmp_dir = TempDir::new().unwrap(); + assert_eq!(tmp_dir.path().exists(), true); + let kernel_sysctl = KernelSysctl::new(tmp_dir.path().to_str().unwrap()); + + let config_detail = HashMap::from([ + ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }), + ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }), + ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ]); + + let mut config = + Sysconfig { model: KERNEL_SYSCTL.to_string(), config_path: String::from(""), contents: config_detail }; + kernel_sysctl.set_config(&mut config).unwrap(); + + let result = fs::read_to_string(format!("{}{}", tmp_dir.path().to_str().unwrap(), "a")).unwrap(); + assert_eq!(result, "1\n"); + } + + #[test] + fn test_kernel_sysctl_persist() { + init(); + let comment = r"# This file is managed by KubeOS for unit testing."; + // create a tmp file with comment + let mut tmp_file = tempfile::NamedTempFile::new().unwrap(); + writeln!(tmp_file, "{}", comment).unwrap(); + writeln!(tmp_file, "a=0").unwrap(); + writeln!(tmp_file, "d=4").unwrap(); + writeln!(tmp_file, "e=5").unwrap(); + writeln!(tmp_file, "g=7").unwrap(); + let kernel_sysctl_persist = KernelSysctlPersist {}; + let config_detail = HashMap::from([ + ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }), + ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }), + ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ("f".to_string(), KeyInfo { value: "".to_string(), operation: "add".to_string() }), + ("g".to_string(), KeyInfo { value: "7".to_string(), operation: "delete".to_string() }), + ("".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }), + ("s=x".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }), + ]); + let mut config = Sysconfig { + model: KERNEL_SYSCTL_PERSIST.to_string(), + config_path: String::from(tmp_file.path().to_str().unwrap()), + contents: config_detail, + }; + kernel_sysctl_persist.set_config(&mut config).unwrap(); + let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); + let expected_res = format!("{}\n{}\n{}\n{}\n{}\n", comment, "a=1", "d=4", "e=5", "c=3"); + assert_eq!(result, expected_res); + let mut config = Sysconfig { + model: KERNEL_SYSCTL_PERSIST.to_string(), + config_path: String::from("/tmp/kubeos-test-kernel-sysctl-persist.txt"), + contents: HashMap::new(), + }; + kernel_sysctl_persist.set_config(&mut config).unwrap(); + assert!(is_file_exist(&config.config_path)); + delete_file_or_dir(&config.config_path).unwrap(); + } + + #[test] + fn write_configs_to_file_tests() { + init(); + let tmp_file = NamedTempFile::new().unwrap(); + let configs = vec!["a=1".to_string(), "b=2".to_string()]; + write_configs_to_file(tmp_file.path().to_str().unwrap(), &configs).unwrap(); + assert_eq!(fs::read(tmp_file.path()).unwrap(), b"a=1\nb=2\n"); + } + + #[test] + fn test_grub_cmdline() { + init(); + let mut tmp_file = NamedTempFile::new().unwrap(); + let mut grub_cmdline = + GrubCmdline { grub_path: tmp_file.path().to_str().unwrap().to_string(), is_cur_partition: true }; + let grub_cfg = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +} + +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=UUID=2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +}"; + writeln!(tmp_file, "{}", grub_cfg).unwrap(); + let config_second_part = HashMap::from([ + ("debug".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("quiet".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }), + ("nomodeset".to_string(), KeyInfo { value: "".to_string(), operation: "update".to_string() }), + ("oops".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("".to_string(), KeyInfo { value: "test".to_string(), operation: "".to_string() }), + ("selinux".to_string(), KeyInfo { value: "1".to_string(), operation: "delete".to_string() }), + ("acpi".to_string(), KeyInfo { value: "off".to_string(), operation: "delete".to_string() }), + ("ro".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ]); + let mut config = Sysconfig { + model: GRUB_CMDLINE_CURRENT.to_string(), + config_path: String::new(), + contents: config_second_part, + }; + grub_cmdline.set_config(&mut config).unwrap(); + grub_cmdline.is_cur_partition = false; + let config_first_part = HashMap::from([ + ("pci".to_string(), KeyInfo { value: "nomis".to_string(), operation: "".to_string() }), + ("quiet".to_string(), KeyInfo { value: "11".to_string(), operation: "delete".to_string() }), + ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "update".to_string() }), + ]); + config.contents = config_first_part; + config.model = GRUB_CMDLINE_NEXT.to_string(); + grub_cmdline.set_config(&mut config).unwrap(); + let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); + let expected_res = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 pci=nomis + initrd /boot/initramfs.img +} +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=UUID=2 ro=1 rootfstype=ext4 nomodeset oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 debug + initrd /boot/initramfs.img +} +"; + assert_eq!(result, expected_res); + + // test grub.cfg not exist + grub_cmdline.grub_path = "/tmp/grub-KubeOS-test.cfg".to_string(); + let res = grub_cmdline.set_config(&mut config); + assert!(res.is_err()); + } + + #[test] + fn test_create_config_file() { + init(); + let tmp_file = "/tmp/kubeos-test-create-config-file.txt"; + create_config_file(&tmp_file).unwrap(); + assert!(is_file_exist(&tmp_file)); + fs::remove_file(tmp_file).unwrap(); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs new file mode 100644 index 00000000..80caf291 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -0,0 +1,301 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs, os::unix::fs::PermissionsExt, path::Path}; + +use anyhow::{anyhow, Context, Result}; +use log::{debug, info}; + +use crate::{ + api::{ImageHandler, UpgradeRequest}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES}, + utils::*, +}; + +pub struct CtrImageHandler { + pub paths: PreparePath, + pub executor: T, +} + +const DEFAULT_NAMESPACE: &str = "k8s.io"; + +impl ImageHandler for CtrImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?; + self.get_image(req)?; + self.get_rootfs_archive(req, IMAGE_PERMISSION)?; + + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); + img_manager.create_os_image(IMAGE_PERMISSION) + } +} + +impl Default for CtrImageHandler { + fn default() -> Self { + Self { paths: PreparePath::default(), executor: RealCommandExecutor {} } + } +} + +impl CtrImageHandler { + #[cfg(test)] + pub fn new(paths: PreparePath, executor: T) -> Self { + Self { paths, executor } + } + + fn get_image(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + is_valid_image_name(image_name)?; + let cli: String = + if is_command_available("crictl", &self.executor) { "crictl".to_string() } else { "ctr".to_string() }; + remove_image_if_exist(&cli, image_name, &self.executor)?; + info!("Start pulling image {}", image_name); + pull_image(&cli, image_name, &self.executor)?; + info!("Start checking image digest"); + check_oci_image_digest(&cli, image_name, &req.check_sum, &self.executor)?; + Ok(()) + } + + fn get_rootfs_archive(&self, req: &UpgradeRequest, permission: u32) -> Result<()> { + let image_name = &req.container_image; + let mount_path = &self + .paths + .mount_path + .to_str() + .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?; + info!("Start getting rootfs {}", image_name); + self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?; + self.executor + .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?; + // copy os.tar from mount_path to its partent dir + self.copy_file(self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission)?; + self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?; + Ok(()) + } + + fn check_and_unmount(&self, mount_path: &str) -> Result<()> { + let ctr_snapshot_cmd = + format!("ctr -n={} snapshots ls | grep {} | awk '{{print $1}}'", DEFAULT_NAMESPACE, mount_path); + let exist_snapshot = self.executor.run_command_with_output("bash", &["-c", &ctr_snapshot_cmd])?; + if !exist_snapshot.is_empty() { + self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "unmount", mount_path])?; + self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "snapshots", "remove", mount_path])?; + } + Ok(()) + } + + fn copy_file, Q: AsRef>(&self, src: P, dst: Q, permission: u32) -> Result<()> { + let copied_bytes = fs::copy(src.as_ref(), dst.as_ref())?; + debug!("Copy {} to {}, total bytes: {}", src.as_ref().display(), dst.as_ref().display(), copied_bytes); + fs::set_permissions(dst, fs::Permissions::from_mode(permission))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{io::Write, path::PathBuf}; + + use mockall::mock; + use tempfile::NamedTempFile; + + use super::*; + use crate::api::CertsInfo; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + // mock is_command_available + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "/bin/sh" && args.contains(&"command -v crictl")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock remove_image_if_exist + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.contains(&"inspecti")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.contains(&"rmi")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock pull_image + mock_executor + .expect_run_command() + .withf(|cmd, args| { + cmd == "crictl" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest") + }) + .times(1) + .returning(|_, _| Ok(())); + // mock get_oci_image_digest + let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]"; + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "crictl" && args.contains(&"inspecti") && args.contains(&"{{.status.repoDigests}}") + }) + .times(1) + .returning(|_, _| Ok(command_output2.to_string())); + let ctr = CtrImageHandler::new(PreparePath::default(), mock_executor); + let result = ctr.get_image(&req); + assert!(result.is_ok()); + } + + #[test] + fn test_get_rootfs_archive() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + // mock check_and_unmount + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure + .times(1) + .returning(|_, _| Ok("".to_string())); + + // mock ctr mount rw + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[4] == "--rw") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + // create temp file for copy + let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file."); + writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file."); + + // Get the path of the temporary file and the path where it should be copied. + let src_dir = tmp_file.path().parent().unwrap(); + let src_file_name = tmp_file.path().file_name().unwrap().to_str().unwrap().to_string(); + let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file."); + let dst_path = dst_file.path().to_path_buf(); + + let paths = PreparePath { + persist_path: "/tmp".into(), + update_path: PathBuf::new(), + image_path: PathBuf::new(), + mount_path: src_dir.to_path_buf(), + rootfs_file: src_file_name.clone(), + tar_path: dst_path.clone(), + }; + + // mock check_and_unmount + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure + .times(1) + .returning(|_, _| Ok("".to_string())); + + let ctr = CtrImageHandler::new(paths, mock_executor); + let result = ctr.get_rootfs_archive(&req, IMAGE_PERMISSION); + assert!(result.is_ok()); + } + + #[test] + fn test_copy_file() { + // Setup: Create a temporary file and write some data to it. + let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file."); + writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file."); + + // Get the path of the temporary file and the path where it should be copied. + let src_path = tmp_file.path().to_str().unwrap().to_string(); + let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file."); + let dst_path = dst_file.path().to_str().unwrap().to_string(); + + let ctr = CtrImageHandler::default(); + let result = ctr.copy_file(&src_path, &dst_path, IMAGE_PERMISSION); + + assert!(result.is_ok()); + + let expected_content = "Hello, world!\n"; + let actual_content = fs::read_to_string(&dst_path).expect("Failed to read destination file."); + assert_eq!(expected_content, actual_content); + + // Assert the file permission + let metadata = fs::metadata(&dst_path).expect("Failed to read destination file."); + let expected_permission = 0o100600; + assert_eq!(metadata.permissions().mode(), expected_permission); + } + + #[test] + fn test_check_and_unmount() { + let mut mock_executor = MockCommandExec::new(); + + // When `run_command_with_output` is called with "bash" and the specific args, it will return Ok("snapshot_exists"). + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") + .times(1) + .returning(|_, _| Ok("snapshot_exists".to_string())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"images")) + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"snapshots")) + .times(1) + .returning(|_, _| Ok(())); + + let result = CtrImageHandler::new(PreparePath::default(), mock_executor).check_and_unmount("test_mount_path"); + + assert!(result.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs new file mode 100644 index 00000000..6d836dc4 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs @@ -0,0 +1,406 @@ +use std::{ + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; + +use anyhow::{bail, Context, Result}; +use log::{debug, info, trace}; +use reqwest::{blocking::Client, Certificate}; +use sha2::{Digest, Sha256}; + +use crate::{ + api::{CertsInfo, ImageHandler, UpgradeRequest}, + sys_mgmt::{CERTS_PATH, IMAGE_PERMISSION, PERSIST_DIR}, + utils::*, +}; + +const BUFFER: u64 = 1024 * 1024 * 10; + +pub struct DiskImageHandler { + pub paths: PreparePath, + pub executor: T, + pub certs_path: String, +} + +impl ImageHandler for DiskImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + self.download(req)?; + self.checksum_match(self.paths.image_path.to_str().unwrap_or_default(), &req.check_sum)?; + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); + Ok(img_manager) + } +} + +impl Default for DiskImageHandler { + fn default() -> Self { + Self { paths: PreparePath::default(), executor: RealCommandExecutor {}, certs_path: CERTS_PATH.to_string() } + } +} + +impl DiskImageHandler { + #[cfg(test)] + pub fn new(paths: PreparePath, executor: T, certs_path: String) -> Self { + Self { paths, executor, certs_path } + } + + fn download(&self, req: &UpgradeRequest) -> Result<()> { + let mut resp = self.send_download_request(req)?; + if resp.status() != reqwest::StatusCode::OK { + bail!("Failed to download image from {}, status: {}", req.image_url, resp.status()); + } + debug!("Received response body size: {:?}", resp.content_length().unwrap_or_default()); + let need_bytes = resp.content_length().unwrap_or_default() + BUFFER; + + check_disk_size( + i64::try_from(need_bytes).with_context(|| "Failed to transform content length from u64 to i64")?, + self.paths.image_path.parent().unwrap_or_else(|| Path::new(PERSIST_DIR)), + )?; + + let mut out = fs::File::create(&self.paths.image_path)?; + trace!("Start to save upgrade image to path {}", &self.paths.image_path.display()); + out.set_permissions(fs::Permissions::from_mode(IMAGE_PERMISSION))?; + let bytes = resp.copy_to(&mut out)?; + info!( + "Download image successfully, upgrade image path: {}, write bytes: {}", + &self.paths.image_path.display(), + bytes + ); + Ok(()) + } + + fn checksum_match(&self, file_path: &str, check_sum: &str) -> Result<()> { + info!("Start checking image checksum"); + let check_sum = check_sum.to_ascii_lowercase(); + let file = fs::read(file_path)?; + let mut hasher = Sha256::new(); + hasher.update(file); + let hash = hasher.finalize(); + // sha256sum -b /persist/update.img + let cal_sum = format!("{:X}", hash).to_ascii_lowercase(); + if cal_sum != check_sum { + delete_file_or_dir(file_path)?; + bail!("Checksum {} mismatch to {}", cal_sum, check_sum); + } + debug!("Checksum match"); + Ok(()) + } + + fn send_download_request(&self, req: &UpgradeRequest) -> Result { + let client: Client; + + if !req.image_url.starts_with("https://") { + // http request + if !req.flag_safe { + bail!("The upgrade image url is not safe"); + } + info!("Discover http request to: {}", &req.image_url); + client = Client::new(); + } else if req.mtls { + // https mtls request + client = self.load_ca_client_certs(&req.certs).with_context(|| "Failed to load client certificates")?; + info!("Discover https mtls request to: {}", &req.image_url); + } else { + // https request + client = self.load_ca_certs(&req.certs.ca_cert).with_context(|| "Failed to load CA certificates")?; + info!("Discover https request to: {}", &req.image_url); + } + + client.get(&req.image_url).send().with_context(|| format!("Failed to fetch from URL: {}", &req.image_url)) + } + + fn load_ca_certs(&self, ca_cert: &str) -> Result { + trace!("Start to load CA certificates"); + self.cert_exist(ca_cert)?; + let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(ca_cert))?)?; + let client = Client::builder().add_root_certificate(ca).build()?; + Ok(client) + } + + fn load_ca_client_certs(&self, certs: &CertsInfo) -> Result { + trace!("Start to load CA and client certificates"); + self.cert_exist(&certs.ca_cert)?; + let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(&certs.ca_cert))?)?; + + self.cert_exist(&certs.client_cert)?; + self.cert_exist(&certs.client_key)?; + let client_cert = std::fs::read(self.get_certs_path(&certs.client_cert))?; + let client_key = std::fs::read(self.get_certs_path(&certs.client_key))?; + let mut client_identity = Vec::new(); + client_identity.extend_from_slice(&client_cert); + client_identity.extend_from_slice(&client_key); + let client_id = reqwest::Identity::from_pem(&client_identity)?; + + let client = Client::builder().use_rustls_tls().add_root_certificate(ca).identity(client_id).build()?; + Ok(client) + } + + fn cert_exist(&self, cert_file: &str) -> Result<()> { + if cert_file.is_empty() { + bail!("Please provide the certificate"); + } + if !self.get_certs_path(cert_file).exists() { + bail!("Certificate does not exist: {}", cert_file); + } + Ok(()) + } + + fn get_certs_path(&self, cert: &str) -> PathBuf { + let cert_path = format!("{}{}", self.certs_path, cert); + PathBuf::from(cert_path) + } +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use mockall::mock; + use mockito; + use tempfile::NamedTempFile; + + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + #[test] + fn test_get_certs_path() { + init(); + let handler = DiskImageHandler::::default(); + let certs_path = handler.get_certs_path("ca.pem"); + assert_eq!(certs_path.to_str().unwrap(), "/etc/KubeOS/certs/ca.pem"); + } + + #[test] + fn test_cert_exist() { + init(); + // generate tmp file + let tmp_file = NamedTempFile::new().unwrap(); + let handler = + DiskImageHandler::::new(PreparePath::default(), RealCommandExecutor {}, String::new()); + let res = handler.cert_exist(tmp_file.path().to_str().unwrap()); + assert!(res.is_ok()); + + assert!(handler.cert_exist("aaa.pem").is_err()); + assert!(handler.cert_exist("").is_err()) + } + + #[test] + fn test_send_download_request() { + init(); + // This is a tmp cert only for KubeOS unit testing. + let tmp_cert = "-----BEGIN CERTIFICATE-----\n\ + MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\ + MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\ + MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\ + bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\ + Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\ + HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\ + c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\ + vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\ + -----END CERTIFICATE-----\n"; + + // This is a tmp private key only for KubeOS unit testing. + let tmp_key = "-----BEGIN PRIVATE KEY-----\n\ + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\ + Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\ + nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\ + -----END PRIVATE KEY-----\n"; + + // Create a temporary file to hold the certificate + let mut cert_file = NamedTempFile::new().unwrap(); + cert_file.write_all(tmp_cert.as_bytes()).unwrap(); + println!("cert_file: {:?}", cert_file.path().to_str().unwrap()); + + // Create a temporary file to hold the private key + let mut key_file = NamedTempFile::new().unwrap(); + key_file.write_all(tmp_key.as_bytes()).unwrap(); + // http + let handler = DiskImageHandler::::default(); + let mut req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "http://localhost:8080/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + req.flag_safe = false; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + + // https + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "/tmp".to_string(); + let tmp_cert_filename = cert_file.path().file_name().unwrap().to_str().unwrap(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "https://localhost:8081/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { + ca_cert: tmp_cert_filename.to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + + // mtls + let tmp_key = NamedTempFile::new().unwrap(); + let tmp_key_filename = tmp_key.path().file_name().unwrap().to_str().unwrap(); + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "/tmp".to_string(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "https://localhost:8082/aaa.txt".to_string(), + flag_safe: true, + mtls: true, + certs: CertsInfo { + ca_cert: tmp_cert_filename.to_string(), + client_cert: tmp_cert_filename.to_string(), + client_key: tmp_key_filename.to_string(), + }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + } + + #[test] + fn test_checksum_match() { + init(); + let mut tmp_file = NamedTempFile::new().unwrap(); + tmp_file.write(b"This is a test txt file for KubeOS test.\n").unwrap(); + let mut handler = DiskImageHandler::::default(); + handler.paths.image_path = tmp_file.path().to_path_buf(); + let mut req = UpgradeRequest { + version: "v2".into(), + check_sum: "98Ea7aff44631D183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "http://localhost:8080/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + assert_eq!(handler.paths.image_path.exists(), true); + handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum).unwrap(); + + req.check_sum = "1234567Abc".into(); + let res = handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum); + assert!(res.is_err()); + } + + #[test] + fn test_load_certs() { + init(); + // This is a tmp cert only for KubeOS unit testing. + let tmp_cert = "-----BEGIN CERTIFICATE-----\n\ + MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\ + MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\ + MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\ + bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\ + Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\ + HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\ + c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\ + vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\ + -----END CERTIFICATE-----\n"; + + // This is a tmp private key only for KubeOS unit testing. + let tmp_key = "-----BEGIN PRIVATE KEY-----\n\ + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\ + Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\ + nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\ + -----END PRIVATE KEY-----\n"; + + // Create a temporary file to hold the certificate + let mut cert_file = NamedTempFile::new().unwrap(); + cert_file.write_all(tmp_cert.as_bytes()).unwrap(); + + // Create a temporary file to hold the private key + let mut key_file = NamedTempFile::new().unwrap(); + key_file.write_all(tmp_key.as_bytes()).unwrap(); + + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "".to_string(); + let certs = CertsInfo { + ca_cert: cert_file.path().to_str().unwrap().to_string(), + client_cert: cert_file.path().to_str().unwrap().to_string(), + client_key: key_file.path().to_str().unwrap().to_string(), + }; + + let res = handler.load_ca_client_certs(&certs); + assert!(res.is_ok()); + + let res = handler.load_ca_certs(&certs.ca_cert); + assert!(res.is_ok()); + } + + #[test] + fn test_download_image() { + init(); + let tmp_file = NamedTempFile::new().unwrap(); + + let mock_executor = MockCommandExec::new(); + let mut handler = DiskImageHandler::new(PreparePath::default(), mock_executor, String::new()); + handler.executor.expect_clone().times(1).returning(|| MockCommandExec::new()); + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + handler.executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + handler.paths.image_path = tmp_file.path().to_path_buf(); + assert_eq!(true, handler.paths.image_path.exists()); + + let url = mockito::server_url(); + let upgrade_request = UpgradeRequest { + version: "v2".into(), + check_sum: "98ea7aff44631d183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: format!("{}/test.txt", url), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let _m = mockito::mock("GET", "/test.txt") + .with_status(200) + .with_body("This is a test txt file for KubeOS test.\n") + .create(); + handler.download_image(&upgrade_request).unwrap(); + assert_eq!(true, handler.paths.image_path.exists()); + assert_eq!( + fs::read(handler.paths.image_path.to_str().unwrap()).unwrap(), + "This is a test txt file for KubeOS test.\n".as_bytes() + ); + + let _m = mockito::mock("GET", "/test.txt").with_status(404).with_body("Not found").create(); + let res = handler.download_image(&upgrade_request); + assert!(res.is_err()) + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs new file mode 100644 index 00000000..4d97552c --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -0,0 +1,236 @@ +use anyhow::{Context, Result}; +use log::{debug, info, trace}; + +use crate::{ + api::{ImageHandler, UpgradeRequest}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES}, + utils::*, +}; + +pub struct DockerImageHandler { + pub paths: PreparePath, + pub container_name: String, + pub executor: T, +} + +impl ImageHandler for DockerImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?; + self.get_image(req)?; + self.get_rootfs_archive(req)?; + + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); + img_manager.create_os_image(IMAGE_PERMISSION) + } +} + +impl Default for DockerImageHandler { + fn default() -> Self { + Self { paths: PreparePath::default(), container_name: "kubeos-temp".into(), executor: RealCommandExecutor {} } + } +} + +impl DockerImageHandler { + #[cfg(test)] + pub fn new(paths: PreparePath, container_name: String, executor: T) -> Self { + Self { paths, container_name, executor } + } + + fn get_image(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + is_valid_image_name(image_name)?; + let cli = "docker"; + remove_image_if_exist(cli, image_name, &self.executor)?; + info!("Start pulling image {}", image_name); + pull_image(cli, image_name, &self.executor)?; + info!("Start checking image digest"); + check_oci_image_digest(cli, image_name, &req.check_sum, &self.executor)?; + Ok(()) + } + + fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + info!("Start getting rootfs {}", image_name); + self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?; + debug!("Create container {}", self.container_name); + let container_id = + self.executor.run_command_with_output("docker", &["create", "--name", &self.container_name, image_name])?; + debug!("Copy rootfs from container {} to {}", container_id, self.paths.update_path.display()); + self.executor.run_command( + "docker", + &[ + "cp", + format!("{}:/{}", container_id, self.paths.rootfs_file).as_str(), + self.paths.update_path.to_str().unwrap(), + ], + )?; + self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?; + Ok(()) + } + + fn check_and_rm_container(&self) -> Result<()> { + trace!("Check and remove container {}", self.container_name); + let docker_ps_cmd = format!("docker ps -a -f=name={} | awk 'NR==2' | awk '{{print $1}}'", self.container_name); + let exist_id = self.executor.run_command_with_output("bash", &["-c", &docker_ps_cmd])?; + if !exist_id.is_empty() { + info!("Remove container {} {} for cleaning environment", self.container_name, exist_id); + self.executor.run_command("docker", &["rm", exist_id.as_str()])?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use mockall::mock; + + use super::*; + use crate::api::CertsInfo; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_check_and_rm_container() { + init(); + let mut mock_executor = MockCommandExec::new(); + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" + && args.len() == 2 + && args.contains(&"docker ps -a -f=name=test | awk 'NR==2' | awk '{print $1}'") + }) + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111")) + .times(1) + .returning(|_, _| Ok(())); + + let result = + DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor).check_and_rm_container(); + assert!(result.is_ok()); + + assert_eq!(DockerImageHandler::default().container_name, "kubeos-temp"); + } + + #[test] + fn test_get_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "docker".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + // mock remove_image_if_exist + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rmi")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock pull_image + mock_executor + .expect_run_command() + .withf(|cmd, args| { + cmd == "docker" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest") + }) + .times(1) + .returning(|_, _| Ok(())); + // mock get_oci_image_digest + let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]"; + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect") && args.contains(&"{{.RepoDigests}}")) + .times(1) + .returning(|_, _| Ok(command_output2.to_string())); + + let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor); + let result = docker.get_image(&req); + assert!(result.is_ok()); + } + + #[test] + fn test_get_rootfs_archive() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "docker".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + // mock check_and_rm_container + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'") + }) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::new())); + // mock get_rootfs_archive + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "docker" && args.contains(&"create")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"cp") && args.contains(&"1111:/os.tar")) + .times(1) + .returning(|_, _| Ok(())); + // mock check_and_rm_container + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'") + }) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111")) + .times(1) + .returning(|_, _| Ok(())); + + let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor); + let result = docker.get_rootfs_archive(&req); + assert!(result.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/mod.rs b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs new file mode 100644 index 00000000..0e06f297 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs @@ -0,0 +1,23 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod config; +mod containerd_image; +mod disk_image; +mod docker_image; +mod values; + +pub use config::*; +pub use containerd_image::*; +pub use disk_image::*; +pub use docker_image::*; +pub use values::*; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/values.rs b/KubeOS-Rust/manager/src/sys_mgmt/values.rs new file mode 100644 index 00000000..b107efc3 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/values.rs @@ -0,0 +1,36 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub const KERNEL_SYSCTL: &str = "kernel.sysctl"; +pub const KERNEL_SYSCTL_PERSIST: &str = "kernel.sysctl.persist"; +pub const GRUB_CMDLINE_CURRENT: &str = "grub.cmdline.current"; +pub const GRUB_CMDLINE_NEXT: &str = "grub.cmdline.next"; + +pub const DEFAULT_PROC_PATH: &str = "/proc/sys/"; +pub const DEFAULT_KERNEL_CONFIG_PATH: &str = "/etc/sysctl.conf"; +pub const DEFAULT_GRUB_CFG_PATH: &str = "/boot/efi/EFI/openEuler/grub.cfg"; +pub const DEFAULT_GRUBENV_PATH: &str = "/boot/efi/EFI/openEuler/grubenv"; + +pub const PERSIST_DIR: &str = "/persist"; +pub const ROOTFS_ARCHIVE: &str = "os.tar"; +pub const UPDATE_DIR: &str = "KubeOS-Update"; +pub const MOUNT_DIR: &str = "kubeos-update"; +pub const OS_IMAGE_NAME: &str = "update.img"; +pub const CERTS_PATH: &str = "/etc/KubeOS/certs/"; + +pub const DEFAULT_KERNEL_CONFIG_PERM: u32 = 0o644; +pub const DEFAULT_GRUB_CFG_PERM: u32 = 0o751; +pub const IMAGE_PERMISSION: u32 = 0o600; + +pub const ONLY_KEY: usize = 1; +pub const KV_PAIR: usize = 2; +pub const NEED_BYTES: i64 = 3 * 1024 * 1024 * 1024; diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs new file mode 100644 index 00000000..9baf99e3 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -0,0 +1,310 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs, + os::{linux::fs::MetadataExt, unix::fs::DirBuilderExt}, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, bail, Context, Result}; +use log::{debug, info, trace}; +use nix::{mount, mount::MntFlags}; + +use super::executor::CommandExecutor; +use crate::sys_mgmt::{MOUNT_DIR, OS_IMAGE_NAME, PERSIST_DIR, ROOTFS_ARCHIVE, UPDATE_DIR}; + +/// * persist_path: /persist +/// +/// * update_path: /persist/KubeOS-Update +/// +/// * mount_path: /persist/KubeOS-Update/kubeos-update +/// +/// * tar_path: /persist/KubeOS-Update/os.tar +/// +/// * image_path: /persist/update.img +/// +/// * rootfs_file: os.tar +#[derive(Clone)] +pub struct PreparePath { + pub persist_path: PathBuf, + pub update_path: PathBuf, + pub mount_path: PathBuf, + pub tar_path: PathBuf, + pub image_path: PathBuf, + pub rootfs_file: String, +} + +impl Default for PreparePath { + fn default() -> Self { + let persist_dir = Path::new(PERSIST_DIR); + let update_pathbuf = persist_dir.join(UPDATE_DIR); + Self { + persist_path: persist_dir.to_path_buf(), + update_path: update_pathbuf.clone(), + mount_path: update_pathbuf.join(MOUNT_DIR), + tar_path: update_pathbuf.join(ROOTFS_ARCHIVE), + image_path: persist_dir.join(OS_IMAGE_NAME), + rootfs_file: ROOTFS_ARCHIVE.to_string(), + } + } +} + +pub fn is_file_exist>(path: P) -> bool { + path.as_ref().exists() +} + +pub fn perpare_env(prepare_path: &PreparePath, need_bytes: i64, permission: u32) -> Result<()> { + info!("Prepare environment to upgrade"); + check_disk_size(need_bytes, &prepare_path.persist_path)?; + clean_env(&prepare_path.update_path, &prepare_path.mount_path, &prepare_path.image_path)?; + fs::DirBuilder::new().recursive(true).mode(permission).create(&prepare_path.mount_path)?; + Ok(()) +} + +pub fn check_disk_size>(need_bytes: i64, path: P) -> Result<()> { + trace!("Check if there is enough disk space to upgrade"); + let fs_stat = nix::sys::statfs::statfs(path.as_ref())?; + let available_blocks = i64::try_from(fs_stat.blocks_available())?; + let available_space = available_blocks * fs_stat.block_size(); + if available_space < need_bytes { + bail!("Space is not enough for downloading"); + } + Ok(()) +} + +/// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img +pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> +where + P: AsRef + std::fmt::Debug, +{ + if is_mounted(&mount_path)? { + debug!("Umount \"{}\"", mount_path.as_ref().display()); + if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) { + bail!("Failed to umount {} in clean_env: {}", mount_path.as_ref().display(), errno); + } + } + // losetup -D? + delete_file_or_dir(&update_path).with_context(|| format!("Failed to delete {:?}", update_path))?; + delete_file_or_dir(&image_path).with_context(|| format!("Failed to delete {:?}", image_path))?; + Ok(()) +} + +pub fn delete_file_or_dir>(path: P) -> Result<()> { + if is_file_exist(&path) { + if fs::metadata(&path)?.is_file() { + info!("Delete file \"{}\"", path.as_ref().display()); + fs::remove_file(&path)?; + } else { + info!("Delete directory \"{}\"", path.as_ref().display()); + fs::remove_dir_all(&path)?; + } + } + Ok(()) +} + +pub fn is_command_available(command: &str, command_executor: &T) -> bool { + match command_executor.run_command("/bin/sh", &["-c", format!("command -v {}", command).as_str()]) { + Ok(_) => { + debug!("command {} is available", command); + true + }, + Err(_) => { + debug!("command {} is not available", command); + false + }, + } +} + +pub fn is_mounted>(mount_path: P) -> Result { + if !is_file_exist(&mount_path) { + return Ok(false); + } + // Get device ID of mountPath + let mount_meta = fs::symlink_metadata(&mount_path)?; + let dev = mount_meta.st_dev(); + + // Get device ID of mountPath's parent directory + let parent = mount_path + .as_ref() + .parent() + .ok_or_else(|| anyhow!("Failed to get parent directory of {}", mount_path.as_ref().display()))?; + let parent_meta = fs::symlink_metadata(parent)?; + let dev_parent = parent_meta.st_dev(); + Ok(dev != dev_parent) +} + +pub fn switch_boot_menuentry( + command_executor: &T, + grub_env_path: &str, + next_menuentry: &str, +) -> Result<()> { + if get_boot_mode() == "uefi" { + command_executor.run_command( + "grub2-editenv", + &[grub_env_path, "set", format!("saved_entry={}", next_menuentry).as_str()], + )?; + } else { + command_executor.run_command("grub2-set-default", &[next_menuentry])?; + } + Ok(()) +} + +pub fn get_boot_mode() -> String { + if is_file_exist("/sys/firmware/efi") { "uefi".into() } else { "bios".into() } +} + +#[cfg(test)] +mod tests { + use mockall::{mock, predicate::*}; + use tempfile::{NamedTempFile, TempDir}; + + use super::*; + use crate::utils::RealCommandExecutor; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_file_exist() { + init(); + let path = "/tmp/test_is_file_exist"; + assert_eq!(is_file_exist(path), false); + + let file = NamedTempFile::new().unwrap(); + assert_eq!(is_file_exist(file.path().to_str().unwrap()), true); + + let tmp_dir = TempDir::new().unwrap(); + assert_eq!(is_file_exist(tmp_dir.path().to_str().unwrap()), true); + } + + #[test] + fn test_prepare_env() { + init(); + let paths = PreparePath { + persist_path: PathBuf::from("/tmp"), + update_path: PathBuf::from("/tmp/test_prepare_env"), + mount_path: PathBuf::from("/tmp/test_prepare_env/kubeos-update"), + tar_path: PathBuf::from("/tmp/test_prepare_env/os.tar"), + image_path: PathBuf::from("/tmp/test_prepare_env/update.img"), + rootfs_file: "os.tar".to_string(), + }; + perpare_env(&paths, 1 * 1024 * 1024 * 1024, 0o700).unwrap(); + } + + #[test] + fn test_check_disk_size() { + init(); + let path = "/home"; + let gb: i64 = 1 * 1024 * 1024 * 1024; + let need_gb = 1 * gb; + let result = check_disk_size(need_gb, path); + assert!(result.is_ok()); + let need_gb = 10000 * gb; + let result = check_disk_size(need_gb, path); + assert!(result.is_err()); + } + + #[test] + fn test_clean_env() { + init(); + let update_path = "/tmp/test_clean_env"; + let mount_path = "/tmp/test_clean_env/kubeos-update"; + let image_path = "/tmp/test_clean_env/update.img"; + clean_env(&update_path.to_string(), &mount_path.to_string(), &image_path.to_string()).unwrap(); + } + + #[test] + fn test_delete_file_or_dir() { + init(); + let path = "/tmp/test_delete_file"; + fs::File::create(path).unwrap(); + assert_eq!(Path::new(path).exists(), true); + delete_file_or_dir(&path.to_string()).unwrap(); + assert_eq!(Path::new(path).exists(), false); + + let path = "/tmp/test_dir"; + fs::create_dir(path).unwrap(); + assert_eq!(Path::new(path).exists(), true); + delete_file_or_dir(&path.to_string()).unwrap(); + assert_eq!(Path::new(path).exists(), false); + + let path = "/tmp/nonexist"; + delete_file_or_dir(path).unwrap(); + + let path = PathBuf::new(); + delete_file_or_dir(path).unwrap(); + } + + #[test] + fn test_switch_boot_menuentry() { + init(); + let grubenv_path = "/boot/efi/EFI/openEuler/grubenv"; + let next_menuentry = "B"; + let mut mock = MockCommandExec::new(); + if get_boot_mode() == "uefi" { + mock.expect_run_command() + .withf(move |name, args| { + name == "grub2-editenv" + && args[0] == grubenv_path + && args[2] == format!("saved_entry={}", next_menuentry).as_str() + }) + .times(1) // Expect it to be called once + .returning(move |_, _| Ok(())); + } else { + mock.expect_run_command() + .withf(move |name, args| name == "grub2-set-default" && args[0] == next_menuentry) + .times(1) // Expect it to be called once + .returning(move |_, _| Ok(())); + } + + switch_boot_menuentry(&mock, grubenv_path, next_menuentry).unwrap() + } + + #[test] + fn test_get_boot_mode() { + init(); + let boot_mode = get_boot_mode(); + let executor = RealCommandExecutor {}; + let res = executor.run_command("ls", &["/sys/firmware/efi"]); + if res.is_ok() { + assert!(boot_mode == "uefi"); + } else { + assert!(boot_mode == "bios"); + } + } + + #[test] + fn test_is_command_available() { + init(); + let executor = RealCommandExecutor {}; + assert_eq!(is_command_available("ls", &executor), true); + assert_eq!(is_command_available("aaaabb", &executor), false); + } +} diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs new file mode 100644 index 00000000..d84c6853 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/container_image.rs @@ -0,0 +1,341 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::{bail, Result}; +use log::{debug, info, trace}; +use regex::Regex; + +use super::executor::CommandExecutor; + +pub fn is_valid_image_name(image: &str) -> Result<()> { + let pattern = r"^((?:[\w.-]+)(?::\d+)?/)*(?:[\w.-]+)((?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$"; + let reg_ex = Regex::new(pattern)?; + if !reg_ex.is_match(image) { + bail!("Invalid image name: {}", image); + } + debug!("Image name {} is valid", image); + Ok(()) +} + +pub fn check_oci_image_digest( + container_runtime: &str, + image_name: &str, + check_sum: &str, + command_executor: &T, +) -> Result<()> { + let image_digests = get_oci_image_digest(container_runtime, image_name, command_executor)?; + if image_digests.to_lowercase() != check_sum.to_lowercase() { + bail!("Image digest mismatch, expect {}, got {}", check_sum, image_digests); + } + Ok(()) +} + +pub fn get_oci_image_digest( + container_runtime: &str, + image_name: &str, + executor: &T, +) -> Result { + let cmd_output: String; + match container_runtime { + "crictl" => { + cmd_output = executor.run_command_with_output( + "crictl", + &["inspecti", "--output", "go-template", "--template", "{{.status.repoDigests}}", image_name], + )?; + }, + "docker" => { + cmd_output = + executor.run_command_with_output("docker", &["inspect", "--format", "{{.RepoDigests}}", image_name])?; + }, + "ctr" => { + cmd_output = executor + .run_command_with_output("ctr", &["-n", "k8s.io", "images", "ls", &format!("name=={}", image_name)])?; + // Split by whitespaces, we get vec like [REF TYPE DIGEST SIZE PLATFORMS LABELS x x x x x x] + // get the 8th element, and split by ':' to get the digest + let fields: Vec<&str> = cmd_output.split_whitespace().collect(); + if let Some(digest) = fields.get(8).and_then(|field| field.split(':').nth(1)) { + trace!("get_oci_image_digest: {}", digest); + return Ok(digest.to_string()); + } else { + bail!("Failed to get digest from ctr command output: {}", cmd_output); + } + }, + _ => { + bail!("Container runtime {} cannot be recognized", container_runtime); + }, + } + + // Parse the cmd_output to extract the digest + let parts: Vec<&str> = cmd_output.split('@').collect(); + if let Some(last_part) = parts.last() { + if last_part.starts_with("sha256") { + let parsed_parts: Vec<&str> = last_part.trim_matches(|c| c == ']').split(':').collect(); + // After spliiing by ':', we should get vec like [sha256, digests] + if parsed_parts.len() == 2 { + debug!("get_oci_image_digest: {}", parsed_parts[1]); + return Ok(parsed_parts[1].to_string()); // 1 is the index of digests + } + } + } + + bail!("Failed to get digest from command output: {}", cmd_output) +} + +pub fn pull_image(runtime: &str, image_name: &str, executor: &T) -> Result<()> { + debug!("Pull image {}", image_name); + match runtime { + "crictl" => { + executor.run_command("crictl", &["pull", image_name])?; + }, + "ctr" => { + executor.run_command( + "ctr", + &[&"-n", "k8s.io", "images", "pull", "--hosts-dir", "/etc/containerd/certs.d", image_name], + )?; + }, + "docker" => { + executor.run_command("docker", &["pull", image_name])?; + }, + _ => { + bail!("Container runtime {} cannot be recognized", runtime); + }, + } + Ok(()) +} + +pub fn remove_image_if_exist(runtime: &str, image_name: &str, executor: &T) -> Result<()> { + match runtime { + "crictl" => { + if executor.run_command("crictl", &["inspecti", image_name]).is_ok() { + executor.run_command("crictl", &["rmi", image_name])?; + info!("Remove existing upgrade image: {}", image_name); + } + }, + "ctr" => { + let output = executor.run_command_with_output( + "ctr", + &[&"-n", "k8s.io", "images", "check", &format!("name=={}", image_name)], + )?; + if !output.is_empty() { + executor.run_command("ctr", &[&"-n", "k8s.io", "images", "rm", image_name, "--sync"])?; + info!("Remove existing upgrade image: {}", image_name); + } + }, + "docker" => { + if executor.run_command("docker", &["inspect", image_name]).is_ok() { + executor.run_command("docker", &["rmi", image_name])?; + info!("Remove existing upgrade image: {}", image_name); + } + }, + _ => { + bail!("Container runtime {} cannot be recognized", runtime); + }, + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use mockall::{mock, predicate::*}; + + use super::*; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_valid_image_name() { + init(); + let correct_images = vec![ + "alpine", + "alpine:latest", + "localhost/latest", + "library/alpine", + "localhost:1234/test", + "test:1234/blaboon", + "alpine:3.7", + "docker.example.edu/gmr/alpine:3.7", + "docker.example.com:5000/gmr/alpine@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "docker.example.co.uk/gmr/alpine/test2:latest", + "registry.dobby.org/dobby/dobby-servers/arthound:2019-08-08", + "owasp/zap:3.8.0", + "registry.dobby.co/dobby/dobby-servers/github-run:2021-10-04", + "docker.elastic.co/kibana/kibana:7.6.2", + "registry.dobby.org/dobby/dobby-servers/lerphound:latest", + "registry.dobby.org/dobby/dobby-servers/marbletown-poc:2021-03-29", + "marbles/marbles:v0.38.1", + "registry.dobby.org/dobby/dobby-servers/loophole@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "sonatype/nexon:3.30.0", + "prom/node-exporter:v1.1.1", + "sosedoff/pgweb@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "sosedoff/pgweb:latest", + "registry.dobby.org/dobby/dobby-servers/arpeggio:2021-06-01", + "registry.dobby.org/dobby/antique-penguin:release-production", + "dalprodictus/halcon:6.7.5", + "antigua/antigua:v31", + "weblate/weblate:4.7.2-1", + "redis:4.0.01-alpine", + "registry.dobby.com/dobby/dobby-servers/github-run:latest", + "192.168.122.123:5000/kubeos-x86_64:2023-01", + ]; + let wrong_images = vec![ + "alpine;v1.0", + "alpine:latest@sha256:11111111111111111111111111111111", + "alpine|v1.0", + "alpine&v1.0", + "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8", + "192.168.122.123:5000/kubeos-x86_64:2023-01@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + "192.168.122.123:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + "myimage$%^&", + ":myimage", + "/myimage", + "myimage/", + "myimage:", + "myimage@@latest", + "myimage::tag", + "registry.com//myimage:tag", + " myimage", + "myimage ", + "registry.com/:tag", + "myimage:", + "", + ":tag", + "IP:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + ]; + for image in correct_images { + assert!(is_valid_image_name(image).is_ok()); + } + for image in wrong_images { + assert!(is_valid_image_name(image).is_err()); + } + } + + #[test] + fn test_get_oci_image_digest() { + init(); + let mut mock = MockCommandExec::new(); + let container_runtime = "ctr"; + let image_name = "docker.io/nginx:latest"; + let command_output1 = + "REF TYPE DIGEST SIZE PLATFORMS LABELS\ndocker.io/nginx:latest text/html sha256:1111 132.5 KIB - -\n"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + let out1 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); + let expect_output = "1111"; + assert_eq!(out1, expect_output); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok("invalid output".to_string())); + let out2 = get_oci_image_digest(container_runtime, image_name, &mock); + assert!(out2.is_err()); + + let container_runtime = "crictl"; + let command_output2 = "[docker.io/nginx@sha256:1111]"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string())); + let out3 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); + assert_eq!(out3, expect_output); + + let out4 = get_oci_image_digest("invalid", image_name, &mock); + assert!(out4.is_err()); + + let container_runtime = "crictl"; + let command_output3 = "[docker.io/nginx:sha256:1111]"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string())); + let out5 = get_oci_image_digest(container_runtime, image_name, &mock); + assert!(out5.is_err()); + } + + #[test] + fn test_check_oci_image_digest_match() { + init(); + let mut mock = MockCommandExec::new(); + let image_name = "docker.io/nginx:latest"; + let container_runtime = "crictl"; + let command_output = "[docker.io/nginx@sha256:1a2b]"; + let check_sum = "1A2B"; + mock.expect_run_command_with_output().times(2).returning(|_, _| Ok(command_output.to_string())); + let result = check_oci_image_digest(container_runtime, image_name, check_sum, &mock); + assert!(result.is_ok()); + let result = check_oci_image_digest(container_runtime, image_name, "1111", &mock); + assert!(result.is_err()); + } + + #[test] + fn test_pull_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.len() == 2 && args[0] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[3] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.len() == 2 && args[0] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + let image_name = "docker.io/nginx:latest"; + let result = pull_image("crictl", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("ctr", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("docker", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("aaa", image_name, &mock_executor); + assert!(result.is_err()); + } + + #[test] + fn test_remove_image_if_exist() { + init(); + let mut mock_executor = MockCommandExec::new(); + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"check")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("something"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"rm")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + let image_name = "docker.io/nginx:latest"; + let res = remove_image_if_exist("ctr", image_name, &mock_executor); + assert!(res.is_ok()); + + let res = remove_image_if_exist("invalid", image_name, &mock_executor); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/manager/src/utils/executor.rs b/KubeOS-Rust/manager/src/utils/executor.rs new file mode 100644 index 00000000..c87bf2ad --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/executor.rs @@ -0,0 +1,89 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::process::Command; + +use anyhow::{bail, Result}; +use log::{debug, trace}; + +pub trait CommandExecutor: Clone { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; +} + +#[derive(Clone)] +pub struct RealCommandExecutor {} + +impl CommandExecutor for RealCommandExecutor { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()> { + trace!("run_command: {} {:?}", name, args); + let output = Command::new(name).args(args).output()?; + if !output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); + let error_message = String::from_utf8_lossy(&output.stderr); + bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message); + } + debug!("run_command: {} {:?} done", name, args); + Ok(()) + } + + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result { + trace!("run_command_with_output: {} {:?}", name, args); + let output = Command::new(name).args(args).output()?; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + if !output.status.success() { + let error_message = String::from_utf8_lossy(&output.stderr); + bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message); + } + debug!("run_command_with_output: {} {:?} done", name, args); + Ok(stdout.trim_end_matches('\n').to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_run_command_with_output() { + init(); + let executor: RealCommandExecutor = RealCommandExecutor {}; + + // test run_command_with_output + let output = executor.run_command_with_output("echo", &["hello", "world"]).unwrap(); + assert_eq!(output, "hello world"); + let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "cat").as_str()]).unwrap(); + assert_eq!(out, "/usr/bin/cat"); + let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "apple").as_str()]); + assert!(out.is_err()); + } + + #[test] + fn test_run_command() { + init(); + let executor: RealCommandExecutor = RealCommandExecutor {}; + // test run_command + let out = executor.run_command("sh", &["-c", format!("command -v {}", "apple").as_str()]); + assert!(out.is_err()); + + let out = executor.run_command("sh", &["-c", format!("command -v {}", "cat").as_str()]); + assert!(out.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs new file mode 100644 index 00000000..90806cf8 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/image_manager.rs @@ -0,0 +1,206 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{self, Permissions}, + os::unix::fs::PermissionsExt, + path::PathBuf, +}; + +use anyhow::{Context, Result}; +use log::{debug, info}; + +use super::{ + clean_env, + common::{delete_file_or_dir, PreparePath}, + executor::CommandExecutor, + partition::PartitionInfo, +}; + +pub struct UpgradeImageManager { + pub paths: PreparePath, + pub next_partition: PartitionInfo, + pub executor: T, +} + +impl UpgradeImageManager { + pub fn new(paths: PreparePath, next_partition: PartitionInfo, executor: T) -> Self { + Self { paths, next_partition, executor } + } + + fn image_path_str(&self) -> Result<&str> { + self.paths.image_path.to_str().context("Failed to convert image path to string") + } + + fn mount_path_str(&self) -> Result<&str> { + self.paths.mount_path.to_str().context("Failed to convert mount path to string") + } + + fn tar_path_str(&self) -> Result<&str> { + self.paths.tar_path.to_str().context("Failed to convert tar path to string") + } + + pub fn create_image_file(&self, permission: u32) -> Result<()> { + let image_str = self.image_path_str()?; + + debug!("Create image {}", image_str); + self.executor.run_command("dd", &["if=/dev/zero", &format!("of={}", image_str), "bs=2M", "count=1024"])?; + fs::set_permissions(&self.paths.image_path, Permissions::from_mode(permission))?; + Ok(()) + } + + pub fn format_image(&self) -> Result<()> { + let image_str = self.image_path_str()?; + debug!("Format image {}", image_str); + self.executor.run_command( + format!("mkfs.{}", self.next_partition.fs_type).as_str(), + &["-L", format!("ROOT-{}", self.next_partition.menuentry).as_str(), image_str], + )?; + Ok(()) + } + + pub fn mount_image(&self) -> Result<()> { + let image_str = self.image_path_str()?; + let mount_str = self.mount_path_str()?; + debug!("Mount {} to {}", image_str, mount_str); + self.executor.run_command("mount", &["-o", "loop", image_str, mount_str])?; + Ok(()) + } + + pub fn extract_tar_to_image(&self) -> Result<()> { + let tar_str = self.tar_path_str()?; + let mount_str = self.mount_path_str()?; + debug!("Extract {} to mounted path {}", tar_str, mount_str); + self.executor.run_command("tar", &["-xvf", tar_str, "-C", mount_str])?; + Ok(()) + } + + pub fn create_os_image(self, permission: u32) -> Result { + self.create_image_file(permission)?; + self.format_image()?; + self.mount_image()?; + self.extract_tar_to_image()?; + // Pass empty image_path to clean_env but avoid deleting the upgrade image + clean_env(&self.paths.update_path, &self.paths.mount_path, &PathBuf::new())?; + Ok(self) + } + + pub fn install(&self) -> Result<()> { + let image_str = self.image_path_str()?; + let device = self.next_partition.device.as_str(); + self.executor + .run_command("dd", &[format!("if={}", image_str).as_str(), format!("of={}", device).as_str(), "bs=8M"])?; + debug!("Install image {} to {} done", image_str, device); + info!( + "Device {} is overwritten and unable to rollback to the previous version anymore if the eviction of node fails", + device + ); + delete_file_or_dir(image_str)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{fs, io::Write, path::Path}; + + use mockall::{mock, predicate::*}; + use tempfile::NamedTempFile; + + use super::*; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_update_image_manager() { + init(); + // create a dir in tmp dir + let tmp_dir = "/tmp/test_update_image_manager"; + let img_path = format!("{}/test_image", tmp_dir); + let mut temp_file = NamedTempFile::new().unwrap(); + write!(temp_file, "test content").unwrap(); // Writing s + fs::create_dir(tmp_dir).unwrap(); + let clone_img_path = img_path.clone(); + + let mut mock = MockCommandExec::new(); + //mock create_image_file + mock.expect_run_command() + .withf(|name, args| name == "dd" && args[0] == "if=/dev/zero") + .times(1) // Expect it to be called once + .returning(move |_, _| { + // simulate 'dd' by copying the contents of the temporary file + std::fs::copy(temp_file.path(), &clone_img_path).unwrap(); + Ok(()) + }); + + //mock format_image + mock.expect_run_command() + .withf(|name, args| name == "mkfs.ext4" && args[1] == "ROOT-B") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock mount_image + mock.expect_run_command() + .withf(|name, _| name == "mount") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock extract_tar_to_image + mock.expect_run_command() + .withf(|name, args| name == "tar" && args[0] == "-xvf") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock install->dd + mock.expect_run_command() + .withf(|name, _| name == "dd") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + let img_manager = UpgradeImageManager::new( + PreparePath { + persist_path: "/tmp".into(), + update_path: tmp_dir.into(), + image_path: img_path.into(), + mount_path: "/tmp/update/mount".into(), + tar_path: "/tmp/update/image.tar".into(), + rootfs_file: "image.tar".into(), + }, + PartitionInfo { device: "/dev/sda3".into(), fs_type: "ext4".into(), menuentry: "B".into() }, + mock, + ); + + let img_manager = img_manager.create_os_image(0o755).unwrap(); + let result = img_manager.install(); + assert!(result.is_ok()); + + assert_eq!(Path::new(&tmp_dir).exists(), false); + } +} diff --git a/KubeOS-Rust/manager/src/utils/mod.rs b/KubeOS-Rust/manager/src/utils/mod.rs new file mode 100644 index 00000000..caf406e3 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/mod.rs @@ -0,0 +1,23 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod common; +mod container_image; +mod executor; +mod image_manager; +mod partition; + +pub use common::*; +pub use container_image::*; +pub use executor::*; +pub use image_manager::*; +pub use partition::*; diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs new file mode 100644 index 00000000..799b4b35 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/partition.rs @@ -0,0 +1,117 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::{bail, Result}; +use log::{debug, trace}; + +use super::executor::CommandExecutor; + +#[derive(PartialEq, Debug, Default)] +pub struct PartitionInfo { + pub device: String, + pub menuentry: String, + pub fs_type: String, +} + +/// get_partition_info returns the current partition info and the next partition info. +pub fn get_partition_info(executor: &T) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> { + let lsblk = executor.run_command_with_output("lsblk", &["-lno", "NAME,MOUNTPOINTS,FSTYPE"])?; + // After split whitespace, the root directory line should have 3 elements, which are "sda2 / ext4". + let mut cur_partition = PartitionInfo::default(); + let mut next_partition = PartitionInfo::default(); + let splitted_len = 3; + trace!("get_partition_info lsblk command output:\n{}", lsblk); + for line in lsblk.lines() { + let res: Vec<&str> = line.split_whitespace().collect(); + if res.len() == splitted_len && res[1] == "/" { + debug!("root directory line: device={}, fs_type={}", res[0], res[2]); + cur_partition.device = format!("/dev/{}", res[0]).to_string(); + cur_partition.fs_type = res[2].to_string(); + next_partition.fs_type = res[2].to_string(); + if res[0].contains('2') { + // root directory is mounted on sda2, so sda3 is the next partition + cur_partition.menuentry = String::from("A"); + next_partition.menuentry = String::from("B"); + next_partition.device = format!("/dev/{}", res[0].replace('2', "3")).to_string(); + } else if res[0].contains('3') { + // root directory is mounted on sda3, so sda2 is the next partition + cur_partition.menuentry = String::from("B"); + next_partition.menuentry = String::from("A"); + next_partition.device = format!("/dev/{}", res[0].replace('3', "2")).to_string(); + } + } + } + if cur_partition.menuentry.is_empty() { + bail!("Failed to get partition info, lsblk output: {}", lsblk); + } + Ok((cur_partition, next_partition)) +} + +#[cfg(test)] +mod tests { + use mockall::{mock, predicate::*}; + + use super::*; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_partition_info() { + init(); + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + let mut mock = MockCommandExec::new(); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + let res = get_partition_info(&mock).unwrap(); + let expect_res = ( + PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() }, + PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() }, + ); + assert_eq!(res, expect_res); + + let command_output2 = "sda\nsda1 /boot/efi vfat\nsda2 ext4\nsda3 / ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string())); + let res = get_partition_info(&mock).unwrap(); + let expect_res = ( + PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() }, + PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() }, + ); + assert_eq!(res, expect_res); + + let command_output3 = ""; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string())); + let res = get_partition_info(&mock); + assert!(res.is_err()); + + let command_output4 = "sda4 / ext4"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output4.to_string())); + let res = get_partition_info(&mock); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/operator/Cargo.toml b/KubeOS-Rust/operator/Cargo.toml new file mode 100644 index 00000000..91cca265 --- /dev/null +++ b/KubeOS-Rust/operator/Cargo.toml @@ -0,0 +1,47 @@ +[package] +description = "KubeOS os-operator" +edition = "2021" +license = "MulanPSL-2.0" +name = "operator" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +name = "operator" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.44" +async-trait = "0.1" +cli = { version = "1.0.6", path = "../cli" } +env_logger = "0.9.0" +futures = "0.3.17" +h2 = "=0.3.16" +k8s-openapi = { version = "0.13.1", features = ["v1_22"] } +kube = { version = "0.66.0", features = ["derive", "runtime"] } +log = "=0.4.15" +manager = { version = "1.0.6", path = "../manager" } +regex = "=1.7.3" +reqwest = { version = "=0.12.2", default-features = false, features = [ + "json", +] } +schemars = "=0.8.10" +serde = { version = "1.0.130", features = ["derive"] } +serde_json = "1.0.68" +socket2 = "=0.4.9" +thiserror = "1.0.29" +thread_local = "=1.1.4" +tokio = { version = "=1.28.0", default-features = false, features = [ + "macros", + "rt-multi-thread", +] } +tokio-retry = "0.3" + +[dev-dependencies] +assert-json-diff = "2.0.2" +http = "0.2.9" +hyper = "0.14.25" +tower-test = "0.4.0" +mockall = { version = "=0.11.3" } +regex = "1" diff --git a/KubeOS-Rust/operator/src/controller/apiclient.rs b/KubeOS-Rust/operator/src/controller/apiclient.rs new file mode 100644 index 00000000..3642b475 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/apiclient.rs @@ -0,0 +1,110 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + +use anyhow::Result; +use apiclient_error::Error; +use async_trait::async_trait; +use kube::{ + api::{Api, Patch, PatchParams}, + Client, +}; +use serde::{Deserialize, Serialize}; +use super::{ + crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}, + values::{NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}, +}; + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceSpecPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + spec: OSInstanceSpec, +} + +impl Default for OSInstanceSpecPatch { + fn default() -> Self { + OSInstanceSpecPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceStatusPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + status: Option, +} + +impl Default for OSInstanceStatusPatch { + fn default() -> Self { + OSInstanceStatusPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }), + } + } +} + +#[derive(Clone)] +pub struct ControllerClient { + pub client: Client, +} + +impl ControllerClient { + pub fn new(client: Client) -> Self { + ControllerClient { client } + } +} + +#[async_trait] +pub trait ApplyApi: Clone + Sized + Send + Sync { + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error>; +} + +#[async_trait] +impl ApplyApi for ControllerClient { + + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() }; + osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?; + Ok(()) + } + +} +pub mod apiclient_error { + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} diff --git a/KubeOS-Rust/operator/src/controller/apiserver_mock.rs b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs new file mode 100644 index 00000000..21abd405 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs @@ -0,0 +1,995 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{borrow::BorrowMut, cell::{RefCell, RefMut}, clone, collections::BTreeMap, default}; +use regex::Regex; +use anyhow::Result; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use http::{status, Request, Response}; +use hyper::{body::to_bytes, Body}; +use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod}; +use kube::{ + api::ObjectMeta, + core::{ErrorResponse, ListMeta, ObjectList}, + Client as KubeClient, Resource, ResourceExt, +}; +use log::debug; +use mockall::mock; +use serde_json::json; + +use self::mock_error::Error; +use super::{ + crd::{Configs, OSInstanceStatus}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_CONFIG}, +}; +use crate::controller::{ + apiclient::{ApplyApi, ControllerClient}, + crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS}, + values::{LABEL_MASTER, LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, + OperatorController, +}; + +type ApiServerHandle = tower_test::mock::Handle, Response>; +pub struct ApiServerVerifier(ApiServerHandle); + + +#[derive(Clone, Debug, Default)] +pub struct K8sResources{ + pub node_list: Vec, + pub osi_list: Vec, +} + +pub enum Testcases { + Rollback(K8sResources), + ConfigNormal(K8sResources), + SkipNoOsiNode(K8sResources), + ExchangeCurrentAndNext(K8sResources), + GetConfigOSInstances(String), + CheckUpgrading(String), + GetIdleOSInstances(String), + +} + +pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) { + tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("timeout on mock apiserver") + .expect("scenario succeeded") +} + +impl ApiServerVerifier { + pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::Rollback(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + }, + Testcases::ConfigNormal(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_config_osi_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_idle_osi_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_patch_spec_config(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_spec_config(k8s_resc.clone()) + .await + }, + Testcases::SkipNoOsiNode(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + }, + Testcases::ExchangeCurrentAndNext(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + }, + _ => { + Err(Error::ArgumentError) + } + } + .expect("Case completed without errors"); + }) + } + + pub fn test_function(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::GetConfigOSInstances(error) => { + self.handler_config_osi_list_get_error(error) + .await + }, + Testcases::CheckUpgrading(error) => { + self.handler_upgrading_node_list_get_error(error) + .await + }, + Testcases::GetIdleOSInstances(error) => { + self.handler_idle_osi_list_get_error(error) + .await + }, + _ => { + Err(Error::ArgumentError) + } + } + .expect("Case completed without errors"); + }) + } + + // 获取所有的 worker 节点，对应于reconcile的第一个 get_nodes 函数 + async fn handler_worker_node_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=%21node-role.kubernetes.io%2Fcontrol-plane&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中所有的 worker 节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if !node.labels().contains_key(LABEL_MASTER){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_worker_node_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 upgrading 的节点 + async fn handler_upgrading_node_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中标签为正在升级的节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if node.labels().contains_key(LABEL_UPGRADING){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_upgrading_node_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取所有的非 upgrading 的 worker 节点 + async fn handler_worker_and_no_upgrade_noding_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + + let remove_limit = |input: &str| -> String { + let re = Regex::new(r"limit=\d+").unwrap(); + re.replace_all(input, "").to_string() + }; + + assert_eq!( + remove_limit(request.uri().to_string().as_str()), + "/api/v1/nodes?&labelSelector=%21upgrade.openeuler.org%2Fupgrading%2C%21node-role.kubernetes.io%2Fcontrol-plane&"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中所有的非 upgrading 的 worker 节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if !node.labels().contains_key(LABEL_UPGRADING) && !node.labels().contains_key(LABEL_MASTER){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_worker_and_no_upgrade_noding_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_osinstance_get_by_node_name(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_get_by_node_name: req_node_name: {:?}", req_node_name); + + let response = serde_json::to_vec(&osinstance).unwrap(); + dbg!("handler_osinstance_get_by_node_name"); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_upgrade(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_nodestatus_upgrade: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_upgrade"); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_exchange(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_nodestatus_exchange: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + + let sysconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + + let upgradeconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + + assert_eq!(spec.sysconfigs.clone(), sysconfigs); + assert_eq!(spec.upgradeconfigs.clone(), upgradeconfigs); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_exchange"); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + // 通过节点名称获取对应节点 + async fn handler_replace_node_by_name(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PUT); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get node and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut node = Node{..Default::default()}; + let mut boolean_get_node = false; + for node_iter in k8s_resc.node_list.clone() { + if node_iter.name() == req_node_name { + boolean_get_node = true; + node = node_iter.clone(); + break; + } + } + assert!(boolean_get_node); + assert_eq!(request.extensions().get(), Some(&"replace")); + + println!("handler_replace_node_by_name: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let metadata_json = body_json.get("metadata").expect("metadata object").clone(); + let metadata: ObjectMeta = serde_json::from_value(metadata_json).expect("valid metadata"); + assert!(metadata.labels.unwrap().contains_key(LABEL_UPGRADING)); + + // 修改 node 并传出 + node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string()); + + dbg!("handler_replace_node_by_name"); + let response = serde_json::to_vec(&node).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 config 的节点上的 osi + async fn handler_config_osi_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出 + let mut osis = vec![]; + for osi in k8s_resc.osi_list.clone() { + if osi.spec.nodestatus == NODE_STATUS_CONFIG{ + osis.push(osi.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: osis, + }; + + dbg!("handler_config_osi_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 config 的节点上的 osi + async fn handler_idle_osi_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出 + let mut osis = vec![]; + for osi in k8s_resc.osi_list.clone() { + if osi.spec.nodestatus == NODE_STATUS_IDLE{ + osis.push(osi.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: osis, + }; + + dbg!("handler_idle_osi_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_osinstance_patch_spec_config(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_osi_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_osi_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_spec_config: req_osi_name: {:?}", req_osi_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_CONFIG.to_string()); + + let sysconfig = Some( + Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl")), + configpath: Some(String::from("")), + contents: + Some(vec![ + Content { + key: Some(String::from("key1")), + value: Some(String::from("a")), + operation: Some(String::from("")), + }, + Content { + key: Some(String::from("key2")), + value: Some(String::from("b")), + operation: Some(String::from("")), + }, + ]), + }]), + } + ); + assert_eq!( + spec.sysconfigs.clone(), + sysconfig + ); + + dbg!("handler_osinstance_patch_spec_config"); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance.spec.sysconfigs = sysconfig; + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_config_osi_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_config_osi_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "NotFound".to_string(), + code: 404, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_upgrading_node_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_upgrading_node_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "Invalid".to_string(), + code: 400, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(400).body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_idle_osi_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_idle_osi_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "NotFound".to_string(), + code: 404, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + + Ok(self) + } + +} + +pub mod mock_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + + #[error("Parameters other than expected were entered")] + ArgumentError, + } +} + + +impl OperatorController { + pub fn test() -> (OperatorController, ApiServerVerifier) { + let (mock_service, handle) = tower_test::mock::pair::, Response>(); + let mock_k8s_client = KubeClient::new(mock_service, "default"); + let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); + let operator_controller: OperatorController = + OperatorController::new(mock_k8s_client, mock_api_client); + (operator_controller, ApiServerVerifier(handle)) + } +} + +impl OSInstance { + pub fn set_osi_default(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }, + status: Some(OSInstanceStatus { + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }), + } + } +} + +impl OS { + pub fn set_os_default() -> Self { + let mut os = OS::new("test", OSSpec::default()); + os.meta_mut().namespace = Some("default".into()); + os + } + + pub fn set_os_rollback_osversion_v1_upgradecon_v1() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string(); + os + } + + pub fn set_os_syscon_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = OPERATION_TYPE_CONFIG.to_string(); + os.spec.sysconfigs = Some( + Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("key1")), + value: Some(String::from("a")), + operation: Some(String::from("")), + }, + Content { + key: Some(String::from("key2")), + value: Some(String::from("b")), + operation: Some(String::from("")), + }, + ]), + }]), + } + ); + os + } + + pub fn set_os_skip_osversion_v2_upgradecon_v1() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os + } + + pub fn set_os_exchange_current_and_next() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + let sysconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + os.spec.sysconfigs = sysconfigs.clone(); + os.spec.upgradeconfigs = sysconfigs.clone(); + + os + } + +} + +impl K8sResources { + pub fn set_rollback_nodes_v2_and_osi_v1() -> Self { + // 创建 node1 和 node2 + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v2".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let node2 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node2".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v2".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default"); + let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default"); + + let node_list = Vec::from([node1, node2]); + let osi_list = Vec::from([osi1, osi2]); + + K8sResources{ + node_list, + osi_list + } + } + + pub fn set_nodes_v1_and_osi_v1() -> Self { + // 创建 node1 和 node2 + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let node2 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node2".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default"); + let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default"); + + let node_list = Vec::from([node1, node2]); + let osi_list = Vec::from([osi1, osi2]); + + K8sResources{ + node_list, + osi_list + } + } + + pub fn set_skip_nodes_and_osi() -> Self { + // 创建 node1 并且不设置 osi + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let node_list = Vec::from([node1]); + let osi_list = Vec::new(); + + K8sResources{ + node_list, + osi_list + } + } + +} + +impl Default for OSSpec { + fn default() -> Self { + OSSpec { + osversion: String::from("KubeOS v1"), + maxunavailable: 3, + checksum: String::from("test"), + imagetype: String::from("containerd"), + containerimage: String::from("test"), + opstype: String::from("upgrade"), + evictpodforce: true, + imageurl: String::from(""), + flagsafe: true, + mtls: false, + cacert: Some(String::from("")), + clientcert: Some(String::from("")), + clientkey: Some(String::from("")), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + } + } +} diff --git a/KubeOS-Rust/operator/src/controller/controller.rs b/KubeOS-Rust/operator/src/controller/controller.rs new file mode 100644 index 00000000..2beb8945 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/controller.rs @@ -0,0 +1,696 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +* KubeOS is licensed under the Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +* PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + + +use anyhow::Result; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::{Api, ListParams, ObjectList, PostParams}, + core::ErrorResponse, + runtime::controller::{Context, ReconcilerAction}, + Client, ResourceExt, +}; +use log::{debug, error}; +use reconciler_error::Error; + +use crate::controller::values::NODE_STATUS_UPGRADE; + +use super::{ + apiclient::ApplyApi, + crd::{Configs, OSInstance, OS}, + values::{ + LABEL_MASTER, LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, + NO_REQUEUE, OPERATION_TYPE_CONFIG, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, + REQUEUE_ERROR, REQUEUE_NORMAL, SYS_CONFIG_NAME, UPGRADE_CONFIG_NAME + }, +}; + +#[derive(Clone)] +pub struct OperatorController { + k8s_client: Client, + controller_client: T, +} + +impl OperatorController { + pub fn new(k8s_client: Client, controller_client: T) -> Self { + OperatorController { + k8s_client, + controller_client, + } + } + + // async fn reconcile( + // &self, + // os: OS, + // ctx: Context>, + // ) -> Result { + // // k8s_client 变量不是 Option 类型，而是直接的 Client 类型，Rust 会确保不为空，不用检查 + + // // Kube-rs库中的Context类型已经处理了上下文的管理，也不需要初始化空的上下文，proxy组件的rust版本也没初始化 + + // // 调用外部的Reconcile函数 + // reconcile(os, ctx).await + // } + + // 获取 worker 节点数 + async fn get_and_update_os(&self, _namespace: &str) -> Result { + + // 创建一个筛选标签的 String 数组，只找 worker 节点 + let reqs = vec![ + format!("!{}", LABEL_MASTER), + ]; + + // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量 + let nodes_items = self.get_nodes( 0, reqs).await?; + + Ok(nodes_items.items.len() as i64) + } + + // 获取 worker 节点，传入 reqs 为 String 数组，表示多个筛选条件 + async fn get_nodes(&self, limit: i64, reqs: Vec) -> Result, Error> { + + let nodes_api: Api = Api::all(self.k8s_client.clone()); + + // 将多个标签筛选器组合成一个字符串，用逗号分隔 + let label_selector = reqs.join(","); + + // 设置 ListParams + let list_params = ListParams::default() + .labels(&label_selector) + .limit(limit as u32); + + let nodes = match nodes_api.list(&list_params).await { + Ok(nodes) => nodes, + Err(e) => { + log::error!("{:?} unable to list nodes with requirements", e); + return Err(Error::KubeClient { source: e }); + }, + }; + + Ok(nodes) + } + + // 获取可以进行升级操作的最大节点数量 + async fn check_upgrading(&self, _namespace: &str, max_unavailable: i64) -> Result { + + // 设置筛选标签，选择标签为正在升级的节点 + let reqs = vec![ + LABEL_UPGRADING.to_string(), + ]; + + // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量 + let nodes_items = self.get_nodes( 0, reqs).await?; + + Ok(max_unavailable - nodes_items.items.len() as i64) + } + + // 为指定数量的节点进行升级操作 + async fn assign_upgrade(&self, os: &OS, limit: i64, namespace: &str) -> Result { + + // 创建筛选标签，只找 worker 节点，并且标签 LabelUpgrading 不存在 + let reqs = vec![ + format!("!{}", LABEL_UPGRADING), + format!("!{}", LABEL_MASTER), + ]; + + // 获取符合标签要求的节点 + let mut nodes_items = self.get_nodes( limit + 1, reqs).await?; + + // 对选定的节点进行升级，返回升级成功的数量和可能的错误 + let count = self.upgrade_nodes(os, &mut nodes_items, limit, namespace).await?; + + Ok(count >= limit) + } + + async fn upgrade_nodes(&self, os: &OS, nodes: &mut ObjectList, limit: i64, namespace: &str) -> Result { + + let mut count = 0; + + for node in nodes.iter_mut() { + // 如果已经达到升级限制，则退出循环 + if count >= limit { + break + } + + let os_version_node = node.status.clone().unwrap().node_info.unwrap().os_image; + + debug!("node name: {}, os_version_node: {}, os_version: {}", node.name(), os_version_node, os.spec.osversion); + + // 检查 os 对象中的操作系统版本是否与节点的操作系统版本不同 + if os_version_node != os.spec.osversion { + + // 尝试获取该节点上的 os 实例 + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + match osi_api.get(&node.name().clone()).await { + Ok(mut osi) => { + // info!("osinstance is exist {:?}", osi.name()); + + debug!("osinstance is exist: \n {:?} \n", osi); + + match self.update_node_and_osins(os, node, &mut osi).await { + Ok(_) => { + count += 1; + }, + Err(_) => { + continue; + }, + } + }, + Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { + debug!("failed to get osInstance {}", &node.name().clone()); + + return Err(Error::KubeClient { + source: kube::Error::Api(ErrorResponse { + reason, + status: "".to_string(), + message: "".to_string(), + code: 0 + })}); + }, + Err(_) => continue, + } + } + + } + + Ok(count) + } + + // 升级节点以及节点上的 OSinstance + async fn update_node_and_osins(&self, os: &OS, node: &mut Node, osinstance: &mut OSInstance, ) -> Result<(), Error> { + debug!("start update_node_and_OSins"); + + // 检查os实例中的升级配置版本与os对象中的升级配置版本是否匹配。osi 字段未初始化时直接进行拷贝 + let mut copy_sign = true; + if let Some(upgradeconfigs) = osinstance.spec.upgradeconfigs.clone() { + if let Some(version) = upgradeconfigs.version { + if version == os.spec.upgradeconfigs.clone().unwrap().version.unwrap() { + copy_sign = false; + } + } + } + if copy_sign { + self.deep_copy_spec_configs(os, osinstance, UPGRADE_CONFIG_NAME.to_string()).await?; + assert!(osinstance.spec.upgradeconfigs.is_some()); + } + + copy_sign = true; + // 检查os实例中的系统配置版本与os对象中的系统配置版本是否匹配。osi 字段未初始化时直接进行拷贝 + if let Some(sysconfigs) = osinstance.spec.sysconfigs.clone() { + if let Some(version) = sysconfigs.version { + if version == os.spec.sysconfigs.clone().unwrap().version.unwrap() { + copy_sign = false; + } + } + } + if copy_sign { + self.deep_copy_spec_configs(os, osinstance, SYS_CONFIG_NAME.to_string()).await?; + assert!(osinstance.spec.sysconfigs.is_some()); + if let Some(sysconfigs) = osinstance.spec.sysconfigs.as_mut() { + if let Some(configs) = &mut sysconfigs.configs { + for config in configs { + if config.model.clone().unwrap() == "grub.cmdline.current" { + config.model = Some("grub.cmdline.next".to_string()); + } + else if config.model.clone().unwrap() == "grub.cmdline.next" { + config.model = Some("grub.cmdline.current".to_string()); + } + } + } + } + } + + // 更新os实例中的状态为升级完成状态 + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + + // 把对 osinstance 的更改从内存更新到 k8s 集群 + let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; + + node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string()); + + // 把对 node 的更改从内存更新到 k8s 集群 + let node_api: Api = Api::all(self.k8s_client.clone()); + node_api.replace(&node.name(), &PostParams::default(), &node).await?; + + Ok(()) + } + + // 深拷贝 + async fn deep_copy_spec_configs(&self, os: &OS, os_instance: &mut OSInstance, config_type: String) -> Result<(), Error> { + + match config_type.as_str() { + UPGRADE_CONFIG_NAME =>{ + + if let Ok(data) = serde_json::to_vec(&os.spec.upgradeconfigs){ + + if let Ok(upgradeconfigs) = serde_json::from_slice(&data) { + os_instance.spec.upgradeconfigs = Some(upgradeconfigs); + }else { + debug!("{} Deserialization failure", config_type); + return Err(Error::Operation { value: "Deserialization".to_string()}); + } + } + else { + debug!("{} Serialization failure", config_type); + return Err(Error::Operation { value: "Serialization".to_string()}); + } + + }, + SYS_CONFIG_NAME => { + + if let Ok(data) = serde_json::to_vec(&os.spec.sysconfigs){ + + if let Ok(sysconfigs) = serde_json::from_slice(&data) { + os_instance.spec.sysconfigs = Some(sysconfigs); + }else { + debug!("{} Deserialization failure", config_type); + return Err(Error::Operation { value: "Deserialization".to_string()}); + } + } + else { + debug!("{} Serialization failure", config_type); + return Err(Error::Operation { value: "Serialization".to_string()}); + } + + }, + _ => { + debug!("configType {} cannot be recognized", config_type); + return Err(Error::Operation { value: config_type.clone() }); + }, + } + + Ok(()) + } + + // 获取可以进行配置操作的节点数量，返回的是 instance 的列表 + async fn check_config(&self, namespace: &str, max_unavailable: i64) -> Result { + let osinstances = self.get_config_osinstances(namespace).await?; + + Ok(max_unavailable - osinstances.len() as i64) + } + + // 获取所在节点状态为配置的 osinstance列表 + async fn get_config_osinstances(&self, namespace: &str) -> Result, Error> { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + // 获取所有 OSInstance 资源 + let all_osinstances = osi_api.list(&ListParams::default()).await?; + + // 在客户端进行过滤，节点状态为 NODE_STATUS_CONFIG + let osinstances: Vec = all_osinstances + .items + .into_iter() + .filter(|osi| osi.spec.nodestatus == NODE_STATUS_CONFIG) + .collect(); + + debug!("config_osi count = {:?}", osinstances.len()); + + Ok(osinstances) + } + + // 为指定数量的节点进行配置操作 + async fn assign_config(&self, _os: &OS, sysconfigs: Configs, config_version: String, limit: i64, namespace: &str) -> Result { + + debug!("start assign_config"); + + let mut osinstances = self.get_idle_os_instances(namespace, limit + 1).await?; + + let mut count = 0; + // 遍历 osi 列表 + for osi in osinstances.iter_mut() { + if count > limit { + break; + } + + let mut config_sign = true; + if let Some(sysconfigs) = osi.spec.sysconfigs.clone() { + if let Some(version) = sysconfigs.version { + debug!("node name: {:?}, config_version_node: {:?}, config_version: {:?}", osi.name(), version, config_version); + if version == config_version { + config_sign = false; + } + } + } + + // 如果版本不同或 osi 未初始化，则将新的配置信息更新到实例中，并将节点状态标记为“配置完成”。 + if config_sign { + count += 1; + osi.spec.sysconfigs = Some(sysconfigs.clone()); + osi.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + + // 把对 osinstance 的更改从内存更新到 k8s 集群 + let namespace = osi.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + self.controller_client.update_osinstance_spec(&osi.name(), &namespace, &osi.spec).await?; + } + } + + Ok(count >= limit) + } + + // 获取所在节点状态为空闲的 osinstance列表 + async fn get_idle_os_instances(&self, namespace: &str, limit: i64) -> Result, Error> { + + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + // 获取所有 OSInstance 资源 + let all_osinstances: ObjectList = osi_api.list(&ListParams::default().limit(limit as u32)).await?; + + // 在客户端进行过滤，节点状态为 NODE_STATUS_IDLE + let osinstances: Vec = all_osinstances + .items + .into_iter() + .filter(|osi| osi.spec.nodestatus == NODE_STATUS_IDLE) + .collect(); + + Ok(osinstances) + } + + +} + +// 调用的函数的具体逻辑需要进一步完成 +pub async fn reconcile( + os: OS, + ctx: Context>, +) -> Result { + + // 初始化 operator_controller 和 os，从环境变量获取NODE_NAME + debug!("start reconcile"); + let operator_controller = ctx.get_ref(); + let os_cr: &OS = &os; + + // 从 os_cr 中获取命名空间，如果命名空间不存在则返回错误 + let namespace: String = os_cr + .namespace() + .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; + + debug!("namespace : {:?}", namespace); + + // 获取 worker 节点数 + let node_num = match operator_controller.get_and_update_os(&namespace).await { + Ok(node_num) => node_num, + Err(Error::KubeClient { source: kube::Error::Api(ErrorResponse { reason, .. })}) if &reason == "NotFound" => { + return Ok(NO_REQUEUE); + }, + Err(_) => return Ok(REQUEUE_ERROR), + }; + + debug!("node_num : {:?}", node_num); + + let opstype = os_cr.spec.opstype.clone(); + let ops = opstype.as_str(); + + debug!("opstype: {}", ops); + + match ops { + // 如果是升级或者回滚 + OPERATION_TYPE_UPGRADE | OPERATION_TYPE_ROLLBACK =>{ + debug!("start upgrade OR rollback"); + + // 获取可以进行升级操作的最大节点数量 + let limit = operator_controller.check_upgrading(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?; + + debug!("limit: {}", limit); + + // 为指定数量的节点进行升级操作 + let need_requeue = operator_controller.assign_upgrade(os_cr, limit, &namespace).await?; + if need_requeue { + return Ok(REQUEUE_NORMAL); + } + }, + // 配置操作 + OPERATION_TYPE_CONFIG =>{ + debug!("start config"); + + // 检查待配置的节点数量 + let limit = operator_controller.check_config(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?; + + debug!("limit: {}", limit); + + // 指派配置任务给节点 + let sys_configs = os_cr.spec.sysconfigs.clone().unwrap(); + let version = os_cr.spec.sysconfigs.clone().unwrap().version.unwrap(); + let need_requeue = operator_controller.assign_config(os_cr, sys_configs, version, limit, &namespace).await?; + + if need_requeue { + return Ok(REQUEUE_NORMAL); + } + }, + _ =>{ + log::error!("operation {} cannot be recognized", ops); + } + } + return Ok(REQUEUE_NORMAL); +} + +pub fn error_policy( + error: &Error, + _ctx: Context>, +) -> ReconcilerAction { + error!("Reconciliation error: {}", error.to_string()); + REQUEUE_ERROR +} + +pub mod reconciler_error { + use thiserror::Error; + + use crate::controller::{apiclient::apiclient_error}; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeClient { + #[from] + source: kube::Error, + }, + + #[error("Create/Patch OSInstance reported error: {source}")] + ApplyApi { + #[from] + source: apiclient_error::Error, + }, + + #[error("Cannot get environment NODE_NAME, error: {source}")] + Env { + #[from] + source: std::env::VarError, + }, + + #[error("{}.metadata.{} is not exist", resource, value)] + MissingObjectKey { resource: String, value: String }, + + // #[error("Cannot get {}, {} is None", value, value)] + // MissingSubResource { value: String }, + + #[error("operation {} cannot be recognized", value)] + Operation { value: String }, + + // #[error("Expect OS Version is not same with Node OS Version, please upgrade first")] + // UpgradeBeforeConfig, + + // #[error("Error when drain node, error reported: {}", value)] + // DrainNode { value: String }, + } +} + +#[cfg(test)] +mod test { + use std::{borrow::Borrow, cell::RefCell, env}; + + use serde::de; + + use super::{error_policy, reconcile, reconciler_error::Error, Context, OSInstance, OperatorController, OS}; + use crate::controller::{ + apiserver_mock::{timeout_after_5s, K8sResources, Testcases}, + ControllerClient, + }; + + #[tokio::test] + async fn test_rollback() { + env::set_var("RUST_LOG", "info"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_rollback_osversion_v1_upgradecon_v1(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::Rollback(K8sResources::set_rollback_nodes_v2_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_config_normal() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::ConfigNormal(K8sResources::set_nodes_v1_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_skip_no_osi_node() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_skip_osversion_v2_upgradecon_v1(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::SkipNoOsiNode(K8sResources::set_skip_nodes_and_osi())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_exchange_current_and_next() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_exchange_current_and_next(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::ExchangeCurrentAndNext(K8sResources::set_nodes_v1_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_deep_copy_spec_configs() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let deep_copy_result = test_operator_controller.clone().deep_copy_spec_configs(&OS::set_os_default(), &mut OSInstance::set_osi_default("", ""), "test".to_string()).await; + + assert!(deep_copy_result.is_err()); + + if let Err(err) = deep_copy_result { + assert_eq!("operation test cannot be recognized".to_string(), err.borrow().to_string()); + } + } + + #[tokio::test] + async fn test_get_config_osinstances() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + let expected_error = "list error".to_string(); + + fakeserver.test_function(Testcases::GetConfigOSInstances(expected_error.clone())); + + // 执行测试 + let result = test_operator_controller.get_config_osinstances("default").await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!(expected_error, error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + + #[tokio::test] + async fn test_check_upgrading() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + fakeserver.test_function(Testcases::CheckUpgrading("label error".to_string())); + + // 执行测试 + let result = test_operator_controller.check_upgrading("default", 2).await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!("label error", error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + + + #[tokio::test] + async fn test_get_idle_osinstances() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + let expected_error = "list error".to_string(); + + fakeserver.test_function(Testcases::GetIdleOSInstances(expected_error.clone())); + + // 执行测试 + let result = test_operator_controller.get_idle_os_instances("default", 3).await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!(expected_error, error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + +} diff --git a/KubeOS-Rust/operator/src/controller/crd.rs b/KubeOS-Rust/operator/src/controller/crd.rs new file mode 100644 index 00000000..1fa63035 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/crd.rs @@ -0,0 +1,79 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)] +pub struct OSSpec { + pub osversion: String, + pub maxunavailable: i64, + pub checksum: String, + pub imagetype: String, + pub containerimage: String, + pub opstype: String, + pub evictpodforce: bool, + pub imageurl: String, + #[serde(rename = "flagSafe")] + pub flagsafe: bool, + pub mtls: bool, + pub cacert: Option, + pub clientcert: Option, + pub clientkey: Option, + #[serde(rename = "sysconfigs")] + pub sysconfigs: Option, + #[serde(rename = "upgradeconfigs")] + pub upgradeconfigs: Option, +} + +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube( + group = "upgrade.openeuler.org", + version = "v1alpha1", + kind = "OSInstance", + plural = "osinstances", + singular = "osinstance", + status = "OSInstanceStatus", + namespaced +)] +pub struct OSInstanceSpec { + pub nodestatus: String, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct OSInstanceStatus { + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Configs { + pub version: Option, + pub configs: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Config { + pub model: Option, + pub configpath: Option, + pub contents: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Content { + pub key: Option, + pub value: Option, + pub operation: Option, +} diff --git a/KubeOS-Rust/operator/src/controller/mod.rs b/KubeOS-Rust/operator/src/controller/mod.rs new file mode 100644 index 00000000..468ebfdb --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/mod.rs @@ -0,0 +1,24 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + + mod apiclient; + #[cfg(test)] + mod apiserver_mock; + mod controller; + mod crd; + mod values; + +pub use apiclient::ControllerClient; +pub use controller::{error_policy, reconcile, OperatorController}; +pub use crd::OS; + \ No newline at end of file diff --git a/KubeOS-Rust/operator/src/controller/values.rs b/KubeOS-Rust/operator/src/controller/values.rs new file mode 100644 index 00000000..267270a9 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/values.rs @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::runtime::controller::ReconcilerAction; +use tokio::time::Duration; + +#[cfg(test)] +pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; + +pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; + +pub const LABEL_MASTER: &str = "node-role.kubernetes.io/control-plane"; + +pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; +pub const OSINSTANCE_KIND: &str = "OSInstance"; +// pub const OSI_STATUS_NAME: &str = "nodestatus"; + +pub const UPGRADE_CONFIG_NAME: &str = "UpgradeConfig"; +pub const SYS_CONFIG_NAME: &str = "SysConfig"; + +pub const NODE_STATUS_IDLE: &str = "idle"; +pub const NODE_STATUS_UPGRADE: &str = "upgrade"; +pub const NODE_STATUS_CONFIG: &str = "config"; + +pub const OPERATION_TYPE_UPGRADE: &str = "upgrade"; +pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; +pub const OPERATION_TYPE_CONFIG: &str = "config"; + + +pub const NO_REQUEUE: ReconcilerAction = ReconcilerAction { requeue_after: None }; + +pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; + +pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; diff --git a/KubeOS-Rust/operator/src/main.rs b/KubeOS-Rust/operator/src/main.rs new file mode 100644 index 00000000..51c91618 --- /dev/null +++ b/KubeOS-Rust/operator/src/main.rs @@ -0,0 +1,74 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + use anyhow::Result; + use env_logger::{Builder, Env, Target}; + use futures::StreamExt; + use kube::{ + api::{Api, ListParams}, + client::Client, + runtime::controller::{Context, Controller}, + }; + use log::{error, info}; + // use std::sync::Arc; + use tokio::signal; + + mod controller; + use controller::{ + error_policy, reconcile, ControllerClient, OperatorController, OS, + }; + + const OPERATOR_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); + + + #[tokio::main] + async fn main() -> Result<()> { + // 初始化日志记录器，默认日志级别为info，输出到标准输出 + Builder::from_env(Env::default().default_filter_or("debug")).target(Target::Stdout).init(); + + // 创建一个默认的Kubernetes客户端 + let client = Client::try_default().await?; + + // 创建一个用于操作OS资源的API对象，自定义的CR + let os: Api = Api::all(client.clone()); + + // 创建一个控制器客户端 + let controller_client = ControllerClient::new(client.clone()); + // let controller_client = Arc::new(ControllerClient::new(client.clone())); + + // 创建一个OSReconciler实例 + let os_reconciler = OperatorController::new(client.clone(), controller_client.clone()); + + // 记录操作符版本和启动信息 + info!( + "os-operator version is {}, starting operator manager", + OPERATOR_VERSION.unwrap_or("Not Found") + ); + + // 创建一个新的控制器并运行，处理reconcile和error_policy，并记录错误信息 + Controller::new(os, ListParams::default()) + .run(reconcile, error_policy, Context::new(os_reconciler)) + .for_each(|res| async move { + match res { + Ok(_) => {} + Err(e) => error!("reconcile failed: {}", e.to_string()), + } + }) + .await; + + // 等待终止信号 + signal::ctrl_c().await?; + info!("os-operator terminated"); + + Ok(()) + } + \ No newline at end of file diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml new file mode 100644 index 00000000..d804ac77 --- /dev/null +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -0,0 +1,49 @@ +[package] +description = "KubeOS os-proxy" +edition = "2021" +license = "MulanPSL-2.0" +name = "proxy" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "drain" +path = "src/drain.rs" + +[[bin]] +name = "proxy" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.44" +async-trait = "0.1" +cli = { version = "1.0.6", path = "../cli" } +env_logger = "0.9.0" +futures = "0.3.17" +h2 = "=0.3.16" +k8s-openapi = { version = "0.13.1", features = ["v1_22"] } +kube = { version = "0.66.0", features = ["derive", "runtime"] } +log = "=0.4.15" +manager = { version = "1.0.6", path = "../manager" } +regex = "=1.7.3" +reqwest = { version = "=0.12.2", default-features = false, features = [ + "json", +] } +schemars = "=0.8.10" +serde = { version = "1.0.130", features = ["derive"] } +serde_json = "1.0.68" +socket2 = "=0.4.9" +thiserror = "1.0.29" +thread_local = "=1.1.4" +tokio = { version = "=1.28.0", default-features = false, features = [ + "macros", + "rt-multi-thread", +] } +tokio-retry = "0.3" + +[dev-dependencies] +assert-json-diff = "2.0.2" +http = "0.2.9" +hyper = "0.14.25" +tower-test = "0.4.0" +mockall = { version = "=0.11.3" } diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs new file mode 100644 index 00000000..b833f276 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs @@ -0,0 +1,153 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{collections::HashMap, path::Path}; + +use agent_error::Error; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use manager::api::{CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest}; + +pub struct UpgradeInfo { + pub version: String, + pub image_type: String, + pub check_sum: String, + pub container_image: String, + pub imageurl: String, + pub flagsafe: bool, + pub mtls: bool, + pub cacert: String, + pub clientcert: String, + pub clientkey: String, +} + +pub struct ConfigInfo { + pub configs: Vec, +} + +pub struct Sysconfig { + pub model: String, + pub config_path: String, + pub contents: HashMap, +} + +pub struct KeyInfo { + pub value: String, + pub operation: String, +} + +pub trait AgentMethod { + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error>; + fn upgrade_method(&self) -> Result<(), Error>; + fn rollback_method(&self) -> Result<(), Error>; + fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error>; +} +pub trait AgentCall { + fn call_agent(&self, client: &Client, method: T) -> Result<(), Error>; +} + +pub struct AgentClient { + pub agent_client: Client, + pub agent_call_client: T, +} + +impl AgentClient { + pub fn new>(socket_path: P, agent_call_client: T) -> Self { + AgentClient { agent_client: Client::new(socket_path), agent_call_client } + } +} + +#[derive(Default)] +pub struct AgentCallClient {} +impl AgentCall for AgentCallClient { + fn call_agent(&self, client: &Client, method: T) -> Result<(), Error> { + match method.call(client) { + Ok(_resp) => Ok(()), + Err(e) => Err(Error::AgentError { source: e }), + } + } +} + +impl AgentMethod for AgentClient { + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error> { + let upgrade_request = UpgradeRequest { + version: upgrade_info.version, + image_type: upgrade_info.image_type, + check_sum: upgrade_info.check_sum, + container_image: upgrade_info.container_image, + image_url: upgrade_info.imageurl, + flag_safe: upgrade_info.flagsafe, + mtls: upgrade_info.mtls, + certs: CertsInfo { + ca_cert: upgrade_info.cacert, + client_cert: upgrade_info.clientcert, + client_key: upgrade_info.clientkey, + }, + }; + match self.agent_call_client.call_agent(&self.agent_client, PrepareUpgradeMethod::new(upgrade_request)) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn upgrade_method(&self) -> Result<(), Error> { + match self.agent_call_client.call_agent(&self.agent_client, UpgradeMethod::default()) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn rollback_method(&self) -> Result<(), Error> { + match self.agent_call_client.call_agent(&self.agent_client, RollbackMethod::default()) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error> { + let mut agent_configs: Vec = Vec::new(); + for config in config_info.configs { + let mut contents_tmp: HashMap = HashMap::new(); + for (key, key_info) in config.contents.iter() { + contents_tmp.insert( + key.to_string(), + AgentKeyInfo { value: key_info.value.clone(), operation: key_info.operation.clone() }, + ); + } + agent_configs.push(AgentSysconfig { + model: config.model, + config_path: config.config_path, + contents: contents_tmp, + }) + } + let config_request = ConfigureRequest { configs: agent_configs }; + match self.agent_call_client.call_agent(&self.agent_client, ConfigureMethod::new(config_request)) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } +} + +pub mod agent_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("{source}")] + AgentError { source: anyhow::Error }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/apiclient.rs b/KubeOS-Rust/proxy/src/controller/apiclient.rs new file mode 100644 index 00000000..3afd5a51 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/apiclient.rs @@ -0,0 +1,147 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::BTreeMap; + +use anyhow::Result; +use apiclient_error::Error; +use async_trait::async_trait; +use kube::{ + api::{Api, ObjectMeta, Patch, PatchParams, PostParams}, + Client, +}; +use serde::{Deserialize, Serialize}; + +use super::{ + crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}, + values::{LABEL_OSINSTANCE, NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}, +}; + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceSpecPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + spec: OSInstanceSpec, +} + +impl Default for OSInstanceSpecPatch { + fn default() -> Self { + OSInstanceSpecPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceStatusPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + status: Option, +} + +impl Default for OSInstanceStatusPatch { + fn default() -> Self { + OSInstanceStatusPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }), + } + } +} + +#[derive(Clone)] +pub struct ControllerClient { + pub client: Client, +} + +impl ControllerClient { + pub fn new(client: Client) -> Self { + ControllerClient { client } + } +} + +#[async_trait] +pub trait ApplyApi: Clone + Sized + Send + Sync { + async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error>; + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error>; + async fn update_osinstance_status( + &self, + node_name: &str, + namespace: &str, + status: &Option, + ) -> Result<(), Error>; +} + +#[async_trait] +impl ApplyApi for ControllerClient { + async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error> { + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + let osinstance = OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + status: None, + }; + let osi_api = Api::namespaced(self.client.clone(), namespace); + osi_api.create(&PostParams::default(), &osinstance).await?; + Ok(()) + } + + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() }; + osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?; + Ok(()) + } + + async fn update_osinstance_status( + &self, + node_name: &str, + namespace: &str, + status: &Option, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_status_patch = OSInstanceStatusPatch { status: status.clone(), ..Default::default() }; + osi_api.patch_status(node_name, &PatchParams::default(), &Patch::Merge(&osi_status_patch)).await?; + Ok(()) + } +} +pub mod apiclient_error { + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs new file mode 100644 index 00000000..2b182ca8 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs @@ -0,0 +1,681 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::BTreeMap; + +use anyhow::Result; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use http::{Request, Response}; +use hyper::{body::to_bytes, Body}; +use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod}; +use kube::{ + api::ObjectMeta, + core::{ListMeta, ObjectList}, + Client as KubeClient, Resource, ResourceExt, +}; +use mockall::mock; + +use self::mock_error::Error; +use super::{ + agentclient::*, + crd::{Configs, OSInstanceStatus}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK}, +}; +use crate::controller::{ + apiclient::{ApplyApi, ControllerClient}, + crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS}, + values::{LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, + ProxyController, +}; + +type ApiServerHandle = tower_test::mock::Handle, Response>; +pub struct ApiServerVerifier(ApiServerHandle); + +pub enum Testcases { + OSInstanceNotExist(OSInstance), + UpgradeNormal(OSInstance), + UpgradeUpgradeconfigsVersionMismatch(OSInstance), + UpgradeOSInstaceNodestatusConfig(OSInstance), + UpgradeOSInstaceNodestatusIdle(OSInstance), + ConfigNormal(OSInstance), + ConfigVersionMismatchReassign(OSInstance), + ConfigVersionMismatchUpdate(OSInstance), + Rollback(OSInstance), +} + +pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) { + tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("timeout on mock apiserver") + .expect("scenario succeeded") +} + +impl ApiServerVerifier { + pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::OSInstanceNotExist(osi) => { + self.handler_osinstance_get_not_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_creation(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi) + .await + }, + Testcases::UpgradeNormal(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_upgradeconfig_v2(osi.clone()) + .await + .unwrap() + .handler_node_cordon(osi.clone()) + .await + .unwrap() + .handler_node_pod_list_get(osi) + .await + }, + Testcases::UpgradeUpgradeconfigsVersionMismatch(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_node_update_delete_label(osi.clone()) + .await + .unwrap() + .handler_node_uncordon(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::UpgradeOSInstaceNodestatusConfig(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + }, + Testcases::UpgradeOSInstaceNodestatusIdle(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_node_update_delete_label(osi.clone()) + .await + .unwrap() + .handler_node_uncordon(osi) + .await + }, + Testcases::ConfigNormal(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_sysconfig_v2(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::ConfigVersionMismatchReassign(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::ConfigVersionMismatchUpdate(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_spec_sysconfig_v2(osi) + .await + }, + Testcases::Rollback(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_upgradeconfig_v2(osi.clone()) + .await + .unwrap() + .handler_node_cordon(osi.clone()) + .await + .unwrap() + .handler_node_pod_list_get(osi) + .await + }, + } + .expect("Case completed without errors"); + }) + } + + async fn handler_osinstance_get_not_exist(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name()) + ); + let response_json = serde_json::json!( + { "status": "Failure", "message": "osinstances.upgrade.openeuler.org \"openeuler\" not found", "reason": "NotFound", "code": 404 } + ); + dbg!("handler_osinstance_get_not_exist"); + let response = serde_json::to_vec(&response_json).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + Ok(self) + } + async fn handler_osinstance_get_exist(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name()) + ); + dbg!("handler_osinstance_get_exist"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + async fn handler_osinstance_creation(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::POST); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?") + ); + dbg!("handler_osinstance_creation"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_idle(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_IDLE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_idle"); + osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_upgradeconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?", + osinstance.name() + ) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let status_json = body_json.get("status").expect("status object").clone(); + let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!( + status.upgradeconfigs.expect("upgradeconfigs is not None").clone(), + osinstance.spec.clone().upgradeconfigs.expect("upgradeconfig is not None") + ); + + osinstance.status.as_mut().unwrap().upgradeconfigs = osinstance.spec.upgradeconfigs.clone(); + + dbg!("handler_osinstance_patch_upgradeconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?", + osinstance.name() + ) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance"); + let status_json = body_json.get("status").expect("status object").clone(); + let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!( + status.sysconfigs.expect("sysconfigs is not None").clone(), + osinstance.spec.clone().sysconfigs.expect("sysconfig is not None") + ); + + osinstance.status.as_mut().unwrap().sysconfigs = osinstance.spec.sysconfigs.clone(); + + dbg!("handler_osinstance_patch_sysconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_spec_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + + assert_eq!( + spec.sysconfigs.expect("upgradeconfigs is not None").clone().version.clone().unwrap(), + String::from("v2") + ); + + osinstance.spec.sysconfigs.as_mut().unwrap().version = Some(String::from("v2")); + + dbg!("handler_osinstance_patch_spec_sysconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_get(mut self, osinstance: OSInstance) -> Result { + // return node with name = openeuler, osimage = KubeOS v1，no upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name())); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + assert_eq!(node.name(), String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1")); + dbg!("handler_node_get"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_get_with_label(mut self, osinstance: OSInstance) -> Result { + // return node with name = openeuler, osimage = KubeOS v1，has upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name())); + let mut node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + let node_labels = node.labels_mut(); + node_labels.insert(LABEL_UPGRADING.to_string(), "".to_string()); + assert_eq!(node.name(), String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1")); + assert!(node.labels().contains_key(LABEL_UPGRADING)); + dbg!("handler_node_get_with_label"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_update_delete_label(mut self, osinstance: OSInstance) -> Result { + // return node with name = openeuler, osimage = KubeOS v1，no upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PUT); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + // check request body has upgrade label + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + dbg!("handler_node_update_delete_label"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_cordon(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + assert_eq!(request.extensions().get(), Some(&"cordon")); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + dbg!("handler_node_cordon"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_uncordon(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + assert_eq!(request.extensions().get(), Some(&"cordon")); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(false), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + dbg!("handler_node_uncordon"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_pod_list_get(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!("/api/v1/pods?&fieldSelector=spec.nodeName%3D{}", osinstance.name()) + ); + assert_eq!(request.extensions().get(), Some(&"list")); + let pods_list = ObjectList:: { metadata: ListMeta::default(), items: vec![] }; + dbg!("handler_node_pod_list_get"); + let response = serde_json::to_vec(&pods_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } +} + +pub mod mock_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} + +mock! { + pub AgentCallClient{} + impl AgentCall for AgentCallClient{ + fn call_agent(&self, client:&Client, method: T) -> Result<(), agent_error::Error> { + Ok(()) + } + } + +} +impl ProxyController { + pub fn test() -> (ProxyController, ApiServerVerifier) { + let (mock_service, handle) = tower_test::mock::pair::, Response>(); + let mock_k8s_client = KubeClient::new(mock_service, "default"); + let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); + let mut mock_agent_call_client = MockAgentCallClient::new(); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + let mock_agent_client = AgentClient::new("test", mock_agent_call_client); + let proxy_controller: ProxyController = + ProxyController::new(mock_k8s_client, mock_api_client, mock_agent_client); + (proxy_controller, ApiServerVerifier(handle)) + } +} + +impl OSInstance { + pub fn set_osi_default(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }, + status: Some(OSInstanceStatus { + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }), + } + } + + pub fn set_osi_nodestatus_upgrade(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + osinstance + } + + pub fn set_osi_nodestatus_config(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = config, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance + } + + pub fn set_osi_upgradecon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.upgradeconfigs.as_mut().unwrap().version = Some(String::from("v2")); + osinstance + } + + pub fn set_osi_nodestatus_upgrade_upgradecon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + osinstance.spec.upgradeconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + osinstance + } + + pub fn set_osi_nodestatus_config_syscon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance.spec.sysconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + osinstance + } +} + +impl OS { + pub fn set_os_default() -> Self { + let mut os = OS::new("test", OSSpec::default()); + os.meta_mut().namespace = Some("default".into()); + os + } + + pub fn set_os_osversion_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.opstype = String::from("config"); + os + } + + pub fn set_os_osversion_v2_upgradecon_v2() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.upgradeconfigs = Some(Configs { version: Some(String::from("v2")), configs: None }); + os + } + + pub fn set_os_syscon_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = String::from("config"); + os.spec.sysconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + os + } + + pub fn set_os_rollback_osversion_v2_upgradecon_v2() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string(); + os.spec.upgradeconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + os + } +} + +impl Default for OSSpec { + fn default() -> Self { + OSSpec { + osversion: String::from("KubeOS v1"), + maxunavailable: 2, + checksum: String::from("test"), + imagetype: String::from("containerd"), + containerimage: String::from("test"), + opstype: String::from("upgrade"), + evictpodforce: true, + imageurl: String::from(""), + flagsafe: false, + mtls: false, + cacert: Some(String::from("")), + clientcert: Some(String::from("")), + clientkey: Some(String::from("")), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + } + } +} diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs new file mode 100644 index 00000000..80a85d1c --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -0,0 +1,556 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{collections::HashMap, env}; + +use anyhow::Result; +use drain::drain_os; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::{Api, PostParams}, + core::ErrorResponse, + runtime::controller::{Context, ReconcilerAction}, + Client, ResourceExt, +}; +use log::{debug, error, info}; +use reconciler_error::Error; + +use super::{ + agentclient::{AgentCall, AgentClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, + apiclient::ApplyApi, + crd::{Configs, Content, OSInstance, OS}, + utils::{check_version, get_config_version, ConfigOperation, ConfigType}, + values::{ + LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, + REQUEUE_ERROR, REQUEUE_NORMAL, + }, +}; + +pub async fn reconcile( + os: OS, + ctx: Context>, +) -> Result { + debug!("start reconcile"); + let proxy_controller = ctx.get_ref(); + let os_cr = &os; + let node_name = env::var("NODE_NAME")?; + let namespace: String = os_cr + .namespace() + .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; + proxy_controller.check_osi_exisit(&namespace, &node_name).await?; + let controller_res = proxy_controller.get_resources(&namespace, &node_name).await?; + let node = controller_res.node; + let mut osinstance = controller_res.osinstance; + let node_os_image = &node + .status + .as_ref() + .ok_or(Error::MissingSubResource { value: String::from("node.status") })? + .node_info + .as_ref() + .ok_or(Error::MissingSubResource { value: String::from("node.status.node_info") })? + .os_image; + debug!("os expected osversion is {},actual osversion is {}", os_cr.spec.osversion, node_os_image); + if check_version(&os_cr.spec.osversion, node_os_image) { + match ConfigType::SysConfig.check_config_version(&os, &osinstance) { + ConfigOperation::Reassign => { + debug!("start reassign"); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.sysconfigs.as_ref()), + ConfigType::SysConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + }, + ConfigOperation::UpdateConfig => { + debug!("start update config"); + osinstance.spec.sysconfigs = os_cr.spec.sysconfigs.clone(); + proxy_controller + .controller_client + .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec) + .await?; + return Ok(REQUEUE_ERROR); + }, + _ => {}, + } + proxy_controller.set_config(&mut osinstance, ConfigType::SysConfig).await?; + proxy_controller + .refresh_node(node, osinstance, &get_config_version(os_cr.spec.sysconfigs.as_ref()), ConfigType::SysConfig) + .await?; + } else { + if os_cr.spec.opstype == NODE_STATUS_CONFIG { + return Err(Error::UpgradeBeforeConfig); + } + if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) { + debug!("start reassign"); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + if node.labels().contains_key(LABEL_UPGRADING) { + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + info!( + "node has upgrade label ,but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin" + ); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + proxy_controller.set_config(&mut osinstance, ConfigType::UpgradeConfig).await?; + proxy_controller.upgrade_node(os_cr, &node).await?; + } + } + Ok(REQUEUE_NORMAL) +} + +pub fn error_policy( + error: &Error, + _ctx: Context>, +) -> ReconcilerAction { + error!("Reconciliation error:{}", error.to_string()); + REQUEUE_ERROR +} + +struct ControllerResources { + osinstance: OSInstance, + node: Node, +} +pub struct ProxyController { + k8s_client: Client, + controller_client: T, + agent_client: AgentClient, +} + +impl ProxyController { + pub fn new(k8s_client: Client, controller_client: T, agent_client: AgentClient) -> Self { + ProxyController { k8s_client, controller_client, agent_client } + } +} + +impl ProxyController { + async fn check_osi_exisit(&self, namespace: &str, node_name: &str) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + match osi_api.get(node_name).await { + Ok(osi) => { + debug!("osinstance is exist {:?}", osi.name()); + Ok(()) + }, + Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { + info!("Create OSInstance {}", node_name); + self.controller_client.create_osinstance(node_name, namespace).await?; + Ok(()) + }, + Err(err) => Err(Error::KubeClient { source: err }), + } + } + + async fn get_resources(&self, namespace: &str, node_name: &str) -> Result { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + let osinstance_cr = osi_api.get(node_name).await?; + let node_api: Api = Api::all(self.k8s_client.clone()); + let node_cr = node_api.get(node_name).await?; + Ok(ControllerResources { osinstance: osinstance_cr, node: node_cr }) + } + + async fn refresh_node( + &self, + mut node: Node, + osinstance: OSInstance, + os_config_version: &str, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start refresh_node"); + let node_api: Api = Api::all(self.k8s_client.clone()); + let labels = node.labels_mut(); + if labels.contains_key(LABEL_UPGRADING) { + labels.remove(LABEL_UPGRADING); + node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; + } + if let Some(node_spec) = &node.spec { + if let Some(node_unschedulable) = node_spec.unschedulable { + if node_unschedulable { + node_api.uncordon(&node.name()).await?; + info!("Uncordon successfully node{}", node.name()); + } + } + } + self.update_node_status(osinstance, os_config_version, config_type).await?; + Ok(()) + } + + async fn update_node_status( + &self, + mut osinstance: OSInstance, + os_config_version: &str, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start update_node_status"); + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + return Ok(()); + } + let upgradeconfig_spec_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + let sysconfig_spec_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + let sysconfig_status_version: String; + if let Some(osinstance_status) = osinstance.status.as_ref() { + sysconfig_status_version = get_config_version(osinstance_status.sysconfigs.as_ref()); + } else { + sysconfig_status_version = get_config_version(None); + } + if sysconfig_spec_version == sysconfig_status_version + || (config_type == ConfigType::SysConfig && os_config_version != sysconfig_spec_version) + || (config_type == ConfigType::UpgradeConfig && os_config_version != upgradeconfig_spec_version) + { + let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); + self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; + } + Ok(()) + } + + async fn update_osi_status(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> { + debug!("start update_osi_status"); + config_type.set_osi_status_config(osinstance); + debug!("osinstance status is update to {:?}", osinstance.status); + let namespace = &osinstance + .namespace() + .ok_or(Error::MissingObjectKey { resource: "osinstance".to_string(), value: "namespace".to_string() })?; + self.controller_client.update_osinstance_status(&osinstance.name(), namespace, &osinstance.status).await?; + Ok(()) + } + + async fn set_config(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> { + debug!("start set_config"); + let config_info = config_type.check_config_start(osinstance); + if config_info.need_config { + match config_info.configs.and_then(convert_to_agent_config) { + Some(agent_configs) => { + match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }) { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + }, + None => { + info!("config is none, No content can be configured."); + }, + }; + self.update_osi_status(osinstance, config_type).await?; + } + Ok(()) + } + + async fn upgrade_node(&self, os_cr: &OS, node: &Node) -> Result<(), Error> { + debug!("start upgrade node"); + match os_cr.spec.opstype.as_str() { + OPERATION_TYPE_UPGRADE => { + let upgrade_info = UpgradeInfo { + version: os_cr.spec.osversion.clone(), + image_type: os_cr.spec.imagetype.clone(), + check_sum: os_cr.spec.checksum.clone(), + container_image: os_cr.spec.containerimage.clone(), + flagsafe: os_cr.spec.flagsafe, + imageurl: os_cr.spec.imageurl.clone(), + mtls: os_cr.spec.mtls, + cacert: os_cr.spec.cacert.clone().unwrap_or_default(), + clientcert: os_cr.spec.clientcert.clone().unwrap_or_default(), + clientkey: os_cr.spec.clientkey.clone().unwrap_or_default(), + }; + + match self.agent_client.prepare_upgrade_method(upgrade_info) { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; + match self.agent_client.upgrade_method() { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + }, + OPERATION_TYPE_ROLLBACK => { + self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; + + match self.agent_client.rollback_method() { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + }, + _ => { + return Err(Error::Operation { value: os_cr.spec.opstype.clone() }); + }, + } + Ok(()) + } + + async fn evict_node(&self, node_name: &str, evict_pod_force: bool) -> Result<(), Error> { + debug!("start evict_node"); + let node_api = Api::all(self.k8s_client.clone()); + node_api.cordon(node_name).await?; + info!("Cordon node Successfully{}, start drain nodes", node_name); + match self.drain_node(node_name, evict_pod_force).await { + Ok(()) => {}, + Err(e) => { + node_api.uncordon(node_name).await?; + info!("Drain node {} error, uncordon node successfully", node_name); + return Err(e); + }, + } + Ok(()) + } + + async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> { + use drain::error::DrainError::*; + match drain_os(&self.k8s_client.clone(), node_name, force).await { + Err(DeletePodsError { errors, .. }) => Err(Error::DrainNode { value: errors.join("; ") }), + _ => Ok(()), + } + } +} + +fn convert_to_agent_config(configs: Configs) -> Option> { + let mut agent_configs: Vec = Vec::new(); + if let Some(config_list) = configs.configs { + for config in config_list.into_iter() { + match config.contents.and_then(convert_to_config_hashmap) { + Some(contents_tmp) => { + let config_tmp = Sysconfig { + model: config.model.unwrap_or_default(), + config_path: config.configpath.unwrap_or_default(), + contents: contents_tmp, + }; + agent_configs.push(config_tmp) + }, + None => { + info!( + "model {} which has configpath {} do not has any contents no need to configure", + config.model.unwrap_or_default(), + config.configpath.unwrap_or_default() + ); + continue; + }, + }; + } + if agent_configs.is_empty() { + info!("no contents in all models, no need to configure"); + return None; + } + return Some(agent_configs); + } + None +} + +fn convert_to_config_hashmap(contents: Vec) -> Option> { + let mut contents_tmp: HashMap = HashMap::new(); + for content in contents.into_iter() { + let key_info = + KeyInfo { value: content.value.unwrap_or_default(), operation: content.operation.unwrap_or_default() }; + contents_tmp.insert(content.key.unwrap_or_default(), key_info); + } + Some(contents_tmp) +} + +pub mod reconciler_error { + use thiserror::Error; + + use crate::controller::{agentclient::agent_error, apiclient::apiclient_error}; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeClient { + #[from] + source: kube::Error, + }, + + #[error("Create/Patch OSInstance reported error: {source}")] + ApplyApi { + #[from] + source: apiclient_error::Error, + }, + + #[error("Cannot get environment NODE_NAME, error: {source}")] + Env { + #[from] + source: std::env::VarError, + }, + + #[error("{}.metadata.{} is not exist", resource, value)] + MissingObjectKey { resource: String, value: String }, + + #[error("Cannot get {}, {} is None", value, value)] + MissingSubResource { value: String }, + + #[error("operation {} cannot be recognized", value)] + Operation { value: String }, + + #[error("Expect OS Version is not same with Node OS Version, please upgrade first")] + UpgradeBeforeConfig, + + #[error("os-agent reported error:{source}")] + Agent { source: agent_error::Error }, + + #[error("Error when drain node, error reported: {}", value)] + DrainNode { value: String }, + } +} + +#[cfg(test)] +mod test { + use std::env; + + use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS}; + use crate::controller::{ + apiserver_mock::{timeout_after_5s, MockAgentCallClient, Testcases}, + ControllerClient, + }; + + #[tokio::test] + async fn test_create_osinstance_with_no_upgrade_or_configuration() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_default(); + let context = Context::new(test_proxy_controller); + let mocksrv = + fakeserver.run(Testcases::OSInstanceNotExist(OSInstance::set_osi_default("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + #[tokio::test] + async fn test_upgrade_normal() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeNormal(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_diff_osversion_opstype_config() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeOSInstaceNodestatusConfig( + OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"), + )); + let res = reconcile(os, context.clone()).await; + timeout_after_5s(mocksrv).await; + assert!(res.is_err(), "upgrade fails due to opstype=config"); + let err = res.unwrap_err(); + assert!(err.to_string().contains("Expect OS Version is not same with Node OS Version, please upgrade first")); + error_policy(&err, context); + } + + #[tokio::test] + async fn test_upgradeconfigs_version_mismatch() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeUpgradeconfigsVersionMismatch( + OSInstance::set_osi_nodestatus_upgrade("openeuler", "default"), + )); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_upgrade_nodestatus_idle() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::UpgradeOSInstaceNodestatusIdle(OSInstance::set_osi_upgradecon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_config_normal() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::ConfigNormal(OSInstance::set_osi_nodestatus_config_syscon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_sysconfig_version_mismatch_reassign() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchReassign(OSInstance::set_osi_nodestatus_config( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_sysconfig_version_mismatch_update() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchUpdate(OSInstance::set_osi_nodestatus_upgrade( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_rollback() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_rollback_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::Rollback(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } +} diff --git a/KubeOS-Rust/proxy/src/controller/crd.rs b/KubeOS-Rust/proxy/src/controller/crd.rs new file mode 100644 index 00000000..41f333e8 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/crd.rs @@ -0,0 +1,77 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)] +pub struct OSSpec { + pub osversion: String, + pub maxunavailable: i64, + pub checksum: String, + pub imagetype: String, + pub containerimage: String, + pub opstype: String, + pub evictpodforce: bool, + pub imageurl: String, + #[serde(rename = "flagSafe")] + pub flagsafe: bool, + pub mtls: bool, + pub cacert: Option, + pub clientcert: Option, + pub clientkey: Option, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube( + group = "upgrade.openeuler.org", + version = "v1alpha1", + kind = "OSInstance", + plural = "osinstances", + singular = "osinstance", + status = "OSInstanceStatus", + namespaced +)] +pub struct OSInstanceSpec { + pub nodestatus: String, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct OSInstanceStatus { + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Configs { + pub version: Option, + pub configs: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Config { + pub model: Option, + pub configpath: Option, + pub contents: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Content { + pub key: Option, + pub value: Option, + pub operation: Option, +} diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs new file mode 100644 index 00000000..b8a4e6e5 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -0,0 +1,26 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod agentclient; +mod apiclient; +#[cfg(test)] +mod apiserver_mock; +mod controller; +mod crd; +mod utils; +mod values; + +pub use agentclient::{AgentCallClient, AgentClient}; +pub use apiclient::ControllerClient; +pub use controller::{error_policy, reconcile, ProxyController}; +pub use crd::OS; +pub use values::SOCK_PATH; diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs new file mode 100644 index 00000000..148ca24d --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/utils.rs @@ -0,0 +1,154 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use log::{debug, info}; + +use super::{ + crd::{Configs, OSInstance, OSInstanceStatus, OS}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_IDLE, NODE_STATUS_UPGRADE}, +}; + +#[derive(PartialEq, Clone, Copy)] +pub enum ConfigType { + UpgradeConfig, + SysConfig, +} + +pub enum ConfigOperation { + DoNothing, + Reassign, + UpdateConfig, +} + +pub struct ConfigInfo { + pub need_config: bool, + pub configs: Option, +} + +impl ConfigType { + pub fn check_config_version(&self, os: &OS, osinstance: &OSInstance) -> ConfigOperation { + debug!("start check_config_version"); + let node_status = &osinstance.spec.nodestatus; + if node_status == NODE_STATUS_IDLE { + debug!("node status is idle"); + return ConfigOperation::DoNothing; + }; + match self { + ConfigType::UpgradeConfig => { + let os_config_version = get_config_version(os.spec.upgradeconfigs.as_ref()); + let osi_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + debug!( + "os upgradeconfig version is{},osinstance spec upragdeconfig version is{}", + os_config_version, osi_config_version + ); + if !check_version(&os_config_version, &osi_config_version) { + info!( + "os.spec.upgradeconfig.version is not equal to oninstance.spec.upragdeconfig.version, operation: reassgin upgrade to get newest upgradeconfigs" + ); + return ConfigOperation::Reassign; + } + }, + ConfigType::SysConfig => { + let os_config_version = get_config_version(os.spec.sysconfigs.as_ref()); + let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + debug!( + "os sysconfig version is{},osinstance spec sysconfig version is{}", + os_config_version, osi_config_version + ); + if !check_version(&os_config_version, &osi_config_version) { + if node_status == NODE_STATUS_CONFIG { + info!( + "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: reassgin config to get newest sysconfigs" + ); + return ConfigOperation::Reassign; + } + if node_status == NODE_STATUS_UPGRADE { + info!( + "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: update osinstance.spec.sysconfig and reconcile" + ); + return ConfigOperation::UpdateConfig; + } + } + }, + }; + ConfigOperation::DoNothing + } + pub fn check_config_start(&self, osinstance: &OSInstance) -> ConfigInfo { + debug!("start check_config_start"); + let spec_config_version: String; + let status_config_version: String; + let configs: Option; + match self { + ConfigType::UpgradeConfig => { + spec_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + if let Some(osinstance_status) = osinstance.status.as_ref() { + status_config_version = get_config_version(osinstance_status.upgradeconfigs.as_ref()); + } else { + status_config_version = get_config_version(None); + } + configs = osinstance.spec.upgradeconfigs.clone(); + }, + ConfigType::SysConfig => { + spec_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + if let Some(osinstance_status) = osinstance.status.as_ref() { + status_config_version = get_config_version(osinstance_status.sysconfigs.as_ref()); + } else { + status_config_version = get_config_version(None); + } + configs = osinstance.spec.sysconfigs.clone(); + }, + } + debug!( + "osinstance soec config version is {},status config version is {}", + spec_config_version, status_config_version + ); + if spec_config_version != status_config_version && osinstance.spec.nodestatus != NODE_STATUS_IDLE { + return ConfigInfo { need_config: true, configs }; + } + ConfigInfo { need_config: false, configs: None } + } + pub fn set_osi_status_config(&self, osinstance: &mut OSInstance) { + match self { + ConfigType::UpgradeConfig => { + if let Some(osi_status) = &mut osinstance.status { + osi_status.upgradeconfigs = osinstance.spec.upgradeconfigs.clone(); + } else { + osinstance.status = Some(OSInstanceStatus { + upgradeconfigs: osinstance.spec.upgradeconfigs.clone(), + sysconfigs: None, + }) + } + }, + ConfigType::SysConfig => { + if let Some(osi_status) = &mut osinstance.status { + osi_status.sysconfigs = osinstance.spec.sysconfigs.clone(); + } else { + osinstance.status = + Some(OSInstanceStatus { upgradeconfigs: None, sysconfigs: osinstance.spec.sysconfigs.clone() }) + } + }, + } + } +} + +pub fn check_version(version_a: &str, version_b: &str) -> bool { + version_a.eq(version_b) +} + +pub fn get_config_version(configs: Option<&Configs>) -> String { + if let Some(configs) = configs { + if let Some(version) = configs.version.as_ref() { + return version.to_string(); + } + }; + String::from("") +} diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs new file mode 100644 index 00000000..dec905a9 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/values.rs @@ -0,0 +1,33 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::runtime::controller::ReconcilerAction; +use tokio::time::Duration; + +pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; +pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; + +pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; +pub const OSINSTANCE_KIND: &str = "OSInstance"; + +pub const NODE_STATUS_IDLE: &str = "idle"; +pub const NODE_STATUS_UPGRADE: &str = "upgrade"; +pub const NODE_STATUS_CONFIG: &str = "config"; + +pub const OPERATION_TYPE_UPGRADE: &str = "upgrade"; +pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; + +pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; + +pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; + +pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; diff --git a/KubeOS-Rust/proxy/src/drain.rs b/KubeOS-Rust/proxy/src/drain.rs new file mode 100644 index 00000000..64417df3 --- /dev/null +++ b/KubeOS-Rust/proxy/src/drain.rs @@ -0,0 +1,511 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use futures::{stream, StreamExt}; +use k8s_openapi::api::core::v1::{Pod, PodSpec, PodStatus}; +use kube::{ + api::{EvictParams, ListParams}, + core::ObjectList, + Api, Client, ResourceExt, +}; +use log::{debug, error, info}; +use reqwest::StatusCode; +use tokio::time::{sleep, Duration, Instant}; +use tokio_retry::{ + strategy::{jitter, ExponentialBackoff}, + RetryIf, +}; + +use self::error::{ + DrainError::{DeletePodsError, GetPodListsError, WaitDeletionError}, + EvictionError::{EvictionErrorNoRetry, EvictionErrorRetry}, +}; + +pub const MAX_EVICT_POD_NUM: usize = 5; +pub const EVERY_EVICTION_RETRY: Duration = Duration::from_secs(5); +pub const EVERY_DELETION_CHECK: Duration = Duration::from_secs(5); +pub const TIMEOUT: Duration = Duration::from_secs(u64::MAX); +pub const RETRY_BASE_DELAY: Duration = Duration::from_millis(100); +pub const RETRY_MAX_DELAY: Duration = Duration::from_secs(20); +pub const MAX_RETRIES_TIMES: usize = 10; + +pub async fn drain_os(client: &Client, node_name: &str, force: bool) -> Result<(), error::DrainError> { + let pods_list = get_pods_deleted(client, node_name, force).await?; + + stream::iter(pods_list) + .for_each_concurrent(MAX_EVICT_POD_NUM, move |pod| { + let k8s_client = client.clone(); + async move { + if evict_pod(&k8s_client, &pod, force).await.is_ok() { + wait_for_deletion(&k8s_client, &pod).await.ok(); + } + } + }) + .await; + + Ok(()) +} + +async fn get_pods_deleted( + client: &Client, + node_name: &str, + force: bool, +) -> Result, error::DrainError> { + let lp = ListParams { field_selector: Some(format!("spec.nodeName={}", node_name)), ..Default::default() }; + let pods_api: Api = Api::all(client.clone()); + let pods: ObjectList = match pods_api.list(&lp).await { + Ok(pods @ ObjectList { .. }) => pods, + Err(err) => { + return Err(GetPodListsError { source: err, node_name: node_name.to_string() }); + }, + }; + let mut filterd_pods_list: Vec = Vec::new(); + let mut filterd_err: Vec = Vec::new(); + let pod_filter = CombinedFilter::new(force); + for pod in pods.into_iter() { + let filter_result = pod_filter.filter(&pod); + if filter_result.status == PodDeleteStatus::Error { + filterd_err.push(filter_result.desc); + continue; + } + if filter_result.result { + filterd_pods_list.push(pod); + } + } + if !filterd_err.is_empty() { + return Err(DeletePodsError { errors: filterd_err }); + } + Ok(filterd_pods_list.into_iter()) +} + +async fn evict_pod(k8s_client: &kube::Client, pod: &Pod, force: bool) -> Result<(), error::EvictionError> { + let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); + + let error_handling_strategy = + if force { ErrorHandleStrategy::RetryStrategy } else { ErrorHandleStrategy::TolerateStrategy }; + + RetryIf::spawn( + error_handling_strategy.retry_strategy(), + || async { + loop { + let eviction_result = pod_api.evict(&pod.name_any(), &EvictParams::default()).await; + + match eviction_result { + Ok(_) => { + pod.name(); + debug!("Successfully evicted Pod '{}'", pod.name_any()); + break; + } + Err(kube::Error::Api(e)) => { + let status_code = StatusCode::from_u16(e.code); + match status_code { + Ok(StatusCode::FORBIDDEN) => { + return Err(EvictionErrorNoRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(StatusCode::NOT_FOUND) => { + return Err(EvictionErrorNoRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(StatusCode::INTERNAL_SERVER_ERROR) => { + error!( + "Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to misconfigured PodDisruptionBudgets.", + pod.name_any(), + e, + EVERY_EVICTION_RETRY.as_secs_f64() + ); + sleep(EVERY_EVICTION_RETRY).await; + continue; + } + Ok(StatusCode::TOO_MANY_REQUESTS) => { + error!("Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to PodDisruptionBugets.", + pod.name_any(), + e, + EVERY_EVICTION_RETRY.as_secs_f64() + ); + sleep(EVERY_EVICTION_RETRY).await; + continue; + } + Ok(_) => { + error!( + "Evict pod {} reported error: '{}'.", + pod.name_any(), + e + ); + return Err(EvictionErrorRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Err(_) => { + error!( + "Evict pod {} reported error: '{}'.Received invalid response code from Kubernetes API", + pod.name_any(), + e + ); + return Err(EvictionErrorRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + } + } + Err(e) => { + error!("Evict pod {} reported error: '{}' and will retry", pod.name_any(),e); + return Err(EvictionErrorRetry { + source: e, + pod_name: pod.name_any(), + }); + } + } + } + Ok(()) + }, + error_handling_strategy + ).await +} + +async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), error::DrainError> { + let start_time = Instant::now(); + + let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); + let response_error_not_found: u16 = 404; + loop { + match pod_api.get(&pod.name_any()).await { + Ok(p) if p.uid() != pod.uid() => { + let name = (&p).name_any(); + info!("Pod {} deleted.", name); + break; + }, + Ok(_) => { + info!("Pod '{}' is not yet deleted. Waiting {}s.", pod.name_any(), EVERY_DELETION_CHECK.as_secs_f64()); + }, + Err(kube::Error::Api(e)) if e.code == response_error_not_found => { + info!("Pod {} is deleted.", pod.name_any()); + break; + }, + Err(e) => { + error!( + "Get pod {} reported error: '{}', whether pod is deleted cannot be determined, waiting {}s.", + pod.name_any(), + e, + EVERY_DELETION_CHECK.as_secs_f64() + ); + }, + } + if start_time.elapsed() > TIMEOUT { + return Err(WaitDeletionError { pod_name: pod.name_any(), max_wait: TIMEOUT }); + } else { + sleep(EVERY_DELETION_CHECK).await; + } + } + Ok(()) +} + +fn get_pod_api_with_namespace(client: &kube::Client, pod: &Pod) -> Api { + match pod.metadata.namespace.as_ref() { + Some(namespace) => Api::namespaced(client.clone(), namespace), + None => Api::default_namespaced(client.clone()), + } +} + +trait NameAny { + fn name_any(&self) -> String; +} + +impl NameAny for &Pod { + fn name_any(&self) -> String { + self.metadata.name.clone().or_else(|| self.metadata.generate_name.clone()).unwrap_or_default() + } +} +trait PodFilter { + fn filter(&self, pod: &Pod) -> Box; +} + +struct FinishedOrFailedFilter {} +impl PodFilter for FinishedOrFailedFilter { + fn filter(&self, pod: &Pod) -> Box { + return match pod.status.as_ref() { + Some(PodStatus { phase: Some(phase), .. }) if phase == "Failed" || phase == "Succeeded" => { + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + }, + _ => FilterResult::create_filter_result(false, "", PodDeleteStatus::Okay), + }; + } +} +struct DaemonFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for DaemonFilter { + fn filter(&self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + return match pod.metadata.owner_references.as_ref() { + Some(owner_references) + if owner_references + .iter() + .any(|reference| reference.controller.unwrap_or(false) && reference.kind == "DaemonSet") => + { + if self.force { + let description = format!("Ignore Pod '{}': Pod is member of a DaemonSet", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Warning }) + } else { + let description = format!("Cannot drain Pod '{}': Pod is member of a DaemonSet", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) + } + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} +impl DaemonFilter { + fn new(force: bool) -> DaemonFilter { + DaemonFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } + } +} + +struct MirrorFilter {} +impl PodFilter for MirrorFilter { + fn filter(&self, pod: &Pod) -> Box { + return match pod.metadata.annotations.as_ref() { + Some(annotations) if annotations.contains_key("kubernetes.io/config.mirror") => { + let description = format!("Ignore Pod '{}': Pod is a static Mirror Pod", pod.name_any()); + FilterResult::create_filter_result(false, &description.to_string(), PodDeleteStatus::Warning) + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} + +struct LocalStorageFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for LocalStorageFilter { + fn filter(&self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + return match pod.spec.as_ref() { + Some(PodSpec { volumes: Some(volumes), .. }) if volumes.iter().any(|volume| volume.empty_dir.is_some()) => { + if self.force { + let description = format!("Force draining Pod '{}': Pod has local storage", pod.name_any()); + Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning }) + } else { + let description = format!("Cannot drain Pod '{}': Pod has local Storage", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) + } + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} +impl LocalStorageFilter { + fn new(force: bool) -> LocalStorageFilter { + LocalStorageFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } + } +} +struct UnreplicatedFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for UnreplicatedFilter { + fn filter(&self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + let is_replicated = pod.metadata.owner_references.is_some(); + + if is_replicated { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + if !is_replicated && self.force { + let description = format!("Force drain Pod '{}': Pod is unreplicated", pod.name_any()); + Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning }) + } else { + let description = format!("Cannot drain Pod '{}': Pod is unreplicated", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) + } + } +} +impl UnreplicatedFilter { + fn new(force: bool) -> UnreplicatedFilter { + UnreplicatedFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } + } +} + +struct DeletedFilter { + delete_wait_timeout: Duration, +} +impl PodFilter for DeletedFilter { + fn filter(&self, pod: &Pod) -> Box { + let now = Instant::now().elapsed(); + return match pod.metadata.deletion_timestamp.as_ref() { + Some(time) + if time.0.timestamp() != 0 + && now - Duration::from_secs(time.0.timestamp() as u64) >= self.delete_wait_timeout => + { + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} + +struct CombinedFilter { + deleted_filter: DeletedFilter, + daemon_filter: DaemonFilter, + mirror_filter: MirrorFilter, + local_storage_filter: LocalStorageFilter, + unreplicated_filter: UnreplicatedFilter, +} +impl PodFilter for CombinedFilter { + fn filter(&self, pod: &Pod) -> Box { + let mut filter_res = self.deleted_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.daemon_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.mirror_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.local_storage_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.unreplicated_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + } +} +impl CombinedFilter { + fn new(force: bool) -> CombinedFilter { + CombinedFilter { + deleted_filter: DeletedFilter { delete_wait_timeout: TIMEOUT }, + daemon_filter: DaemonFilter::new(force), + mirror_filter: MirrorFilter {}, + local_storage_filter: LocalStorageFilter::new(force), + unreplicated_filter: UnreplicatedFilter::new(force), + } + } +} + +#[derive(PartialEq, Clone, Copy)] +enum PodDeleteStatus { + Okay, + Warning, + Error, +} +struct FilterResult { + result: bool, + desc: String, + status: PodDeleteStatus, +} +impl FilterResult { + fn create_filter_result(result: bool, desc: &str, status: PodDeleteStatus) -> Box { + Box::new(FilterResult { result, desc: desc.to_string(), status }) + } +} + +enum ErrorHandleStrategy { + RetryStrategy, + TolerateStrategy, +} + +impl ErrorHandleStrategy { + fn retry_strategy(&self) -> impl Iterator { + let backoff = + ExponentialBackoff::from_millis(RETRY_BASE_DELAY.as_millis() as u64).max_delay(RETRY_MAX_DELAY).map(jitter); + + match self { + Self::TolerateStrategy => backoff.take(0), + + Self::RetryStrategy => backoff.take(MAX_RETRIES_TIMES), + } + } +} + +impl tokio_retry::Condition for ErrorHandleStrategy { + fn should_retry(&mut self, error: &error::EvictionError) -> bool { + match self { + Self::TolerateStrategy => false, + Self::RetryStrategy => matches!(error, error::EvictionError::EvictionErrorRetry { .. }), + } + } +} + +pub mod error { + use thiserror::Error; + use tokio::time::Duration; + + #[derive(Debug, Error)] + pub enum DrainError { + #[error("Get node {} pods list error reported: {}", node_name, source)] + GetPodListsError { source: kube::Error, node_name: String }, + + #[error("Pod '{}' was not deleted in the time allocated ({:.2}s).",pod_name,max_wait.as_secs_f64())] + WaitDeletionError { pod_name: String, max_wait: Duration }, + #[error("")] + DeletePodsError { errors: Vec }, + } + + #[derive(Debug, Error)] + pub enum EvictionError { + #[error("Evict Pod {} error: '{}'", pod_name, source)] + EvictionErrorRetry { source: kube::Error, pod_name: String }, + + #[error("Evict Pod {} error: '{}'", pod_name, source)] + EvictionErrorNoRetry { source: kube::Error, pod_name: String }, + } +} diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs new file mode 100644 index 00000000..5c122ba2 --- /dev/null +++ b/KubeOS-Rust/proxy/src/main.rs @@ -0,0 +1,49 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::Result; +use env_logger::{Builder, Env, Target}; +use futures::StreamExt; +use kube::{ + api::{Api, ListParams}, + client::Client, + runtime::controller::{Context, Controller}, +}; +use log::{error, info}; +mod controller; +use controller::{ + error_policy, reconcile, AgentCallClient, AgentClient, ControllerClient, ProxyController, OS, SOCK_PATH, +}; + +const PROXY_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); +#[tokio::main] +async fn main() -> Result<()> { + Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init(); + let client = Client::try_default().await?; + let os: Api = Api::all(client.clone()); + let controller_client = ControllerClient::new(client.clone()); + let agent_call_client = AgentCallClient::default(); + let agent_client = AgentClient::new(SOCK_PATH, agent_call_client); + let proxy_controller = ProxyController::new(client, controller_client, agent_client); + info!("os-proxy version is {}, start renconcile", PROXY_VERSION.unwrap_or("Not Found")); + Controller::new(os, ListParams::default()) + .run(reconcile, error_policy, Context::new(proxy_controller)) + .for_each(|res| async move { + match res { + Ok(_o) => {}, + Err(e) => error!("reconcile failed: {}", e.to_string()), + } + }) + .await; + info!("os-proxy terminated"); + Ok(()) +} diff --git a/KubeOS-Rust/proxy/tests/common/mod.rs b/KubeOS-Rust/proxy/tests/common/mod.rs new file mode 100644 index 00000000..82577597 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/common/mod.rs @@ -0,0 +1,63 @@ +use std::process::{Command, Stdio}; + +use anyhow::Result; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::ResourceExt, + client::Client, + config::{Config, KubeConfigOptions, Kubeconfig}, + Api, +}; +use manager::utils::{CommandExecutor, RealCommandExecutor}; + +pub const CLUSTER: &str = "kubeos-test"; + +pub fn run_command(cmd: &str, args: &[&str]) -> Result<()> { + let output = Command::new(cmd).args(args).stdout(Stdio::inherit()).stderr(Stdio::inherit()).output()?; + if !output.status.success() { + println!("failed to run command: {} {}\n", cmd, args.join(" ")); + } + Ok(()) +} + +pub async fn setup() -> Result { + // set PATH variable + let path = std::env::var("PATH").unwrap(); + let new_path = format!("{}:{}", path, "../../bin"); + std::env::set_var("PATH", new_path); + + // create cluster + let executor = RealCommandExecutor {}; + println!("Creating cluster"); + run_command("bash", &["./tests/setup/setup_test_env.sh"]).expect("failed to create cluster"); + + // connect to the cluster + let kind_config = executor.run_command_with_output("kind", &["get", "kubeconfig", "-n", CLUSTER]).unwrap(); + let kubeconfig = Kubeconfig::from_yaml(kind_config.as_str()).expect("failed to parse kubeconfig"); + let options = KubeConfigOptions::default(); + let config = Config::from_custom_kubeconfig(kubeconfig, &&options).await.expect("failed to create config"); + let client = Client::try_from(config).expect("failed to create client"); + // list all nodes + let nodes: Api = Api::all(client.clone()); + let node_list = nodes.list(&Default::default()).await.expect("failed to list nodes"); + for n in node_list { + println!("Found Node: {}", n.name()); + } + // check node status + let node = nodes.get("kubeos-test-worker").await.unwrap(); + let status = node.status.unwrap(); + let conditions = status.conditions.unwrap(); + for c in conditions { + if c.type_ == "Ready" { + assert_eq!(c.status, "True"); + } + } + println!("Cluster ready"); + Ok(client) +} + +pub fn clean_env() { + let executor = RealCommandExecutor {}; + println!("Cleaning cluster"); + executor.run_command("kind", &["delete", "clusters", CLUSTER]).expect("failed to clean cluster"); +} diff --git a/KubeOS-Rust/proxy/tests/drain_test.rs b/KubeOS-Rust/proxy/tests/drain_test.rs new file mode 100644 index 00000000..2f4f1501 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/drain_test.rs @@ -0,0 +1,41 @@ +mod common; + +use common::*; +use drain::drain_os; +use k8s_openapi::api::core::v1::{Node, Pod}; +use kube::Api; + +#[tokio::test] +#[ignore = "integration test"] +async fn test_drain() { + let client = setup().await.unwrap(); + // drain node + let nodes: Api = Api::all(client.clone()); + let node_name = "kubeos-test-worker"; + println!("cordon node"); + nodes.cordon(node_name).await.unwrap(); + println!("drain node"); + drain_os(&client, node_name, true).await.unwrap(); + + // assert unschedulable + println!("check node unschedulable"); + let node = nodes.get(node_name).await.unwrap(); + if let Some(spec) = node.spec { + assert_eq!(spec.unschedulable, Some(true)); + } else { + panic!("node spec is none"); + } + // list all pods on kubeos-test-worker node and all pods should belong to daemonset + println!("list all pods on kubeos-test-worker node"); + let pods: Api = Api::all(client.clone()); + let pod_list = pods.list(&Default::default()).await.unwrap(); + // check the pod is from daemonset + for p in pod_list { + if p.spec.unwrap().node_name.unwrap() == node_name { + assert_eq!(p.metadata.owner_references.unwrap()[0].kind, "DaemonSet"); + } + } + nodes.uncordon(node_name).await.unwrap(); + + clean_env() +} diff --git a/KubeOS-Rust/proxy/tests/setup/kind-config.yaml b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml new file mode 100644 index 00000000..0fe29e73 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker \ No newline at end of file diff --git a/KubeOS-Rust/proxy/tests/setup/resources.yaml b/KubeOS-Rust/proxy/tests/setup/resources.yaml new file mode 100644 index 00000000..0e449d5d --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/resources.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: example-daemonset +spec: + selector: + matchLabels: + name: example-daemonset + template: + metadata: + labels: + name: example-daemonset + spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-with-local-storage +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] + volumeMounts: + - mountPath: "/data" + name: local-volume + volumes: + - name: local-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Pod +metadata: + name: standalone-pod +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-deployment +spec: + replicas: 2 + selector: + matchLabels: + app: example + template: + metadata: + labels: + app: example + spec: + containers: + - name: nginx + image: nginx:alpine + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: DoesNotExist + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: example-pdb +spec: + minAvailable: 1 + selector: + matchLabels: + app: example +--- +apiVersion: v1 +kind: Pod +metadata: + name: resource-intensive-pod +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] + resources: + requests: + memory: "256Mi" + cpu: "500m" + limits: + memory: "512Mi" + cpu: "1000m" + diff --git a/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh new file mode 100644 index 00000000..d24d8e01 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# this bash script executes in proxy directory + +set -Eeuxo pipefail + +# Define variables +KIND_VERSION="v0.19.0" +KUBECTL_VERSION="v1.24.15" +KIND_CLUSTER_NAME="kubeos-test" +DOCKER_IMAGES=("busybox:stable" "nginx:alpine" "kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab") +NODE_IMAGE="kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab" +RESOURCE="./tests/setup/resources.yaml" +KIND_CONFIG="./tests/setup/kind-config.yaml" +BIN_PATH="../../bin/" +ARCH=$(uname -m) + +# Install kind and kubectl +install_bins() { + # if bin dir not exist then create + if [ ! -d "${BIN_PATH}" ]; then + mkdir -p "${BIN_PATH}" + fi + if [ ! -f "${BIN_PATH}"kind ]; then + echo "Installing Kind..." + # For AMD64 / x86_64 + if [ "$ARCH" = x86_64 ]; then + # add proxy if you are behind proxy + curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-amd64 + fi + # For ARM64 + if [ "$ARCH" = aarch64 ]; then + curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-arm64 + fi + chmod +x "${BIN_PATH}"kind + fi + if [ ! -f "${BIN_PATH}"kubectl ]; then + echo "Installing kubectl..." + if [ "$ARCH" = x86_64 ]; then + curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + fi + if [ "$ARCH" = aarch64 ]; then + curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/arm64/kubectl" + fi + chmod +x "${BIN_PATH}"kubectl + fi + export PATH=$PATH:"${BIN_PATH}" +} + +# Create Kind Cluster +create_cluster() { + echo "Creating Kind cluster..." + for image in "${DOCKER_IMAGES[@]}"; do + docker pull "$image" + done + kind create cluster --name "${KIND_CLUSTER_NAME}" --config "${KIND_CONFIG}" --image "${NODE_IMAGE}" +} + +# Load Docker image into Kind cluster +load_docker_image() { + echo "Loading Docker image into Kind cluster..." + DOCKER_IMAGE=$(printf "%s " "${DOCKER_IMAGES[@]:0:2}") + kind load docker-image ${DOCKER_IMAGE} --name "${KIND_CLUSTER_NAME}" +} + +# Apply Kubernetes resource files +apply_k8s_resources() { + echo "Applying Kubernetes resources from ${RESOURCE}..." + kubectl apply -f "${RESOURCE}" + echo "Waiting for nodes getting ready..." + sleep 40s +} + +main() { + export no_proxy=localhost,127.0.0.1 + install_bins + create_cluster + load_docker_image + apply_k8s_resources +} + +main diff --git a/KubeOS-Rust/rustfmt.toml b/KubeOS-Rust/rustfmt.toml new file mode 100644 index 00000000..3c565cf5 --- /dev/null +++ b/KubeOS-Rust/rustfmt.toml @@ -0,0 +1,11 @@ +# cargo +nightly fmt +version = "Two" +use_small_heuristics = "MAX" +match_block_trailing_comma = true +newline_style = "Unix" +merge_derives = false +max_width = 120 +group_imports = "StdExternalCrate" +imports_granularity = "Crate" +reorder_imports = true +unstable_features = true \ No newline at end of file diff --git a/Makefile b/Makefile index 634a3bcf..44571ab6 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,9 @@ GO_BUILD_CGO = CGO_ENABLED=1 \ CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \ ${GO_BUILD} -buildmode=pie -trimpath -tags "seccomp selinux static_build cgo netgo osusergo" -all: proxy operator agent hostshell +RUSTFLAGS := RUSTFLAGS="-C relocation_model=pic -D warnings -W unsafe_code -W rust_2021_incompatible_closure_captures -C link-arg=-s" + +all: proxy operator agent hostshell rust-kubeos # Build binary proxy: @@ -69,6 +71,15 @@ hostshell: ${GO_BUILD_CGO} ${LD_FLAGS} -o bin/hostshell cmd/admin-container/main.go strip bin/hostshell +rust-kubeos: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust + +rust-proxy: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package proxy + +rust-agent: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package os-agent + # Install CRDs into a cluster install: manifests kubectl apply -f confg/crd diff --git a/VERSION b/VERSION index ee90284c..af0b7ddb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.4 +1.0.6 diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index 98ef1f56..27ff3273 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -18,7 +18,7 @@ spec: versions: - name: v1alpha1 additionalPrinterColumns: - - name: OSVersion + - name: OS VERSION jsonPath: .spec.osversion type: string description: The version of OS diff --git a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml index a7bad3f0..df9119b4 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml @@ -22,19 +22,19 @@ spec: type: string jsonPath: .spec.nodestatus description: The status of node - - name: SYSCONFIG STATUS + - name: SYSCONFIG-VERSION-CURRENT type: string jsonPath: .status.sysconfigs.version description: The current status of sysconfig - - name: SYSCONFIG SPEC + - name: SYSCONFIG-VERSION-DESIRED type: string jsonPath: .spec.sysconfigs.version description: The expected version of sysconfig - - name: UPGRADECONFIG STATUS + - name: UPGRADECONFIG-VERSION-CURRENT type: string jsonPath: .status.upgradeconfigs.version description: The current version of upgradeconfig - - name: UPGRADECONFIG SPEC + - name: UPGRADECONFIG-VERSION-DESIRED type: string jsonPath: .spec.upgradeconfigs.version description: The expected version of upgradeconfig diff --git a/docs/quick-start.md b/docs/quick-start.md index 13bb08d9..ee6f3c31 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -1,18 +1,22 @@ # 快速使用指导 -## 编译及部署 +[TOC] -### 编译指导 +## 编译指导 * 编译环境：openEuler Linux x86/AArch64 + * 进行编译需要以下包： * golang(大于等于1.15版本) * make * git + * rust(大于等于1.57版本) + * cargo(大于等于1.57版本) + * openssl-devel ``` shell - sudo yum install golang make git - ``` + sudo yum install golang make git rust cargo openssl-devel + ``` * 使用git获取本项目的源码 @@ -27,77 +31,101 @@ ```shell cd KubeOS - sudo make - ``` - - * proxy及operator容器镜像构建 - * proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕 - * 请用户自行编写Dockerfile来构建镜像，请注意 - * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性 - * 需将operator和proxy拷贝到baseimage上 - * 请确保proxy属主和属组为root，文件权限为500 - * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500 - * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应 - * 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库 - * Dockerfile参考如下, Dockerfile也可以使用多阶段构建: - - ``` dockerfile - FROM your_baseimage - COPY ./bin/proxy /proxy - ENTRYPOINT ["/proxy"] - FROM your_baseimage - COPY --chown=6552:6552 ./bin/operator /operator - ENTRYPOINT ["/operator"] - ``` + sudo make + # 编译生成的二进制在bin目录下，查看二进制 + tree bin + bin + ├── operator + ├── os-agent + ├── proxy + ├── rust + │   ├── ... + │   └── release + │   ├── ... + │   ├── os-agent + │   └── proxy + ``` - ```shell - # 指定proxy的镜像仓库，镜像名及版本 - export IMG_PROXY=your_imageRepository/proxy_imageName:version - # 指定proxy的Dockerfile地址 - export DOCKERFILE_PROXY=your_dockerfile_proxy - # 指定operator的镜像仓库，镜像名及版本 - export IMG_OPERATOR=your_imageRepository/operator_imageName:version - # 指定operator的Dockerfile路径 - export DOCKERFILE_OPERATOR=your_dockerfile_operator - - # 镜像构建 - docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} . - docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} . - # 推送镜像到镜像仓库 - docker push ${IMG_OPERATOR} - docker push ${IMG_PROXY} - ``` + * ```bin/proxy```、```bin/os-agent```为go语言编写的proxy和os-agent，```bin/rust/release/proxy```、```bin/rust/release/os-agent```为rust语言编写的proxy和os-agent，二者功能一致。 + +## 镜像构建指导 + +### proxy及operator镜像构建指导 + +* proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕 + +* 请用户自行编写Dockerfile来构建镜像，请注意 + * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性 + * 需将operator和proxy拷贝到baseimage上 + * 请确保proxy属主和属组为root，文件权限为500 + * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500 + * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应 + +* 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库 + +* Dockerfile参考如下, Dockerfile也可以使用多阶段构建: + + ``` dockerfile + FROM your_baseimage + COPY ./bin/proxy /proxy + ENTRYPOINT ["/proxy"] + FROM your_baseimage + COPY --chown=6552:6552 ./bin/operator /operator + ENTRYPOINT ["/operator"] + ``` -* OS虚拟机镜像制作 - * 制作注意事项 - * 请确保已安装qemu-img，bc，parted，tar，yum，docker, dosfstools - * 容器OS镜像制作需要使用root权限 - * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 - * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式 - * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间 - * 容器镜像制作时不支持用户自定义配置挂载文件 - * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。 - * 请确保os-agent属主和属组为root，建议os-agent文件权限为500 - * 容器OS虚拟机镜像制作 - 进入scripts目录，执行脚本 - - ```shell - cd scripts - bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' - ``` - - * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 - * 示例命令使用的密码为```openEuler12#$``` - * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： - * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 - * qcow2 格式的系统镜像 system.qcow2。 - * 可用于升级的根文件系统分区镜像 update.img 。 - * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数 - * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) - * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 - * 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 - -### 部署指导 + ```shell + # 指定proxy的镜像仓库，镜像名及版本 + export IMG_PROXY=your_imageRepository/proxy_imageName:version + # 指定proxy的Dockerfile地址 + export DOCKERFILE_PROXY=your_dockerfile_proxy + # 指定operator的镜像仓库，镜像名及版本 + export IMG_OPERATOR=your_imageRepository/operator_imageName:version + # 指定operator的Dockerfile路径 + export DOCKERFILE_OPERATOR=your_dockerfile_operator + + # 镜像构建 + docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} . + docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} . + # 推送镜像到镜像仓库 + docker push ${IMG_OPERATOR} + docker push ${IMG_PROXY} + ``` + +### KubeOS虚拟机镜像制作指导 + +* 制作注意事项 + * 请确保已安装qemu-img，bc，parted，tar，yum，docker + * 容器OS镜像制作需要使用root权限 + * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 + * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式 + * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间 + * 容器镜像制作时不支持用户自定义配置挂载文件 + * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。 + * 请确保os-agent属主和属组为root，建议os-agent文件权限为500 + +* 容器OS虚拟机镜像制作 + 进入scripts目录，执行脚本 + + ```shell + cd scripts + bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' + ``` + + * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 + * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： + * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 + * qcow2 格式的系统镜像 system.qcow2。 + * 可用于升级的根文件系统分区镜像 update.img 。 + * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数 + * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) + * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 + +* 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 + +## 部署指导 + +### os-operator和os-proxy部署指导 * 环境要求 * openEuler Linux x86/AArch64系统 @@ -143,18 +171,35 @@ kubectl get pods -A ``` -### 使用指导 +## 使用指导 #### 注意事项 -* 容器OS升级为所有软件包原子升级，默认不在容器OS内提供单包升级能力。 -* 容器OS升级为双区升级的方式，不支持更多分区数量。 -* 单节点的升级过程的日志可在节点的/var/log/message文件查看。 -* 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。 -* 使用docker镜像升级和mtls双向认证仅支持 openEuler 22.09 及之后的版本 -* 不支持跨大版本升级 - -#### 参数说明 +* 公共注意事项 + * 仅支持虚拟机x86和arm64 UEFI场景。 + * 当前不支持集群节点OS多版本管理，即集群中OS的CR只能为一个。 + * 使用kubectl apply通过YAML创建或更新OS的CR时，不建议并发apply，当并发请求过多时，kube-apiserver会无法处理请求导致失败。 + * 如用户配置了容器镜像仓的证书或密钥，请用户保证证书或密钥文件的权限最小。 +* 升级注意事项 + * 升级为所有软件包原子升级，默认不提供单包升级能力。 + * 升级为双区升级的方式，不支持更多分区数量。 + * 当前暂不支持跨大版本升级。 + * 单节点的升级过程的日志可在节点的 /var/log/messages 文件查看。 + * 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。 + * 节点上containerd如需配置ctr使用的私有镜像，请将配置文件host.toml按照ctr指导放在/etc/containerd/certs.d目录下。 + +* 配置注意事项 + * 用户自行指定配置内容，用户需保证配置内容安全可靠 ，尤其是持久化配置（kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next），KubeOS不对参数有效性进行检验。 + * opstype=config时，若osversion与当前集群节点的OS版本不一致，配置不会进行。 + * 当前仅支持kernel参数临时配置（kernel.sysctl）、持久化配置（kernel.sysctl.persist）和grub cmdline配置（grub.cmdline.current和grub.cmdline.next）。 + * 持久化配置会写入persist持久化分区，升级重启后配置保留；kernel参数临时配置重启后不保留。 + * 配置grub.cmdline.current或grub.cmdline.next时，如为单个参数（非key=value格式参数），请指定key为该参数，value为空。 + * 进行配置删除（operation=delete）时，key=value形式的配置需保证key、value和实际配置一致。 + * 配置不支持回退，如需回退，请修改配置版本和配置内容，重新下发配置。 + * 配置出现错误，节点状态陷入config时，请将配置版本恢复成上一版本并重新下发配置，从而使节点恢复至idel状态。 但是请注意：出现错误前已经配置完成的参数无法恢复。 + * 在配置grub.cmdline.current或grub.cmdline.next时，若需要将已存在的“key=value”格式的参数更新为只有key无value格式，比如将“rd.info=0”更新成rd.info，需要先删除“key=value”，然后在下一次配置时，添加key。不支持直接更新或者更新删除动作在同一次完成。 + +#### OS CR参数说明 在集群中创建类别为OS的定制对象，设置相应字段。类别OS来自于安装和部署章节创建的CRD对象，字段及说明如下： @@ -164,22 +209,21 @@ | 参数 |参数类型 | 参数说明 | 使用说明 | 是否必选 | | -------------- | ------ | ------------------------------------------------------------ | ----- | ---------------- | - | imagetype | string | 使用的升级镜像的类型 | 需为 docker ，containerd ，或者是 disk，其他值无效，且该参数仅在升级场景有效。
**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。|是 | - | opstype | string | 进行的操作，升级,回退或者配置 | 需为 upgrade ，config 或者 rollback ，其他值无效 |是 | - | osversion | string | 用于升级或回退的镜像的OS版本 | 需为 KubeOS version , 例如: KubeOS 1.0.0|是 | - | maxunavailable | int | 同时进行升级或回退的节点数 | maxunavailable值设置为大于实际集群的节点数时也可正常部署，升级或回退时会按照集群内实际节点数进行|是 | - | containerimage | string | 用于升级的容器镜像 | 需要为容器镜像格式：[REPOSITORY/NAME[:TAG@DIGEST]](https://docs.docker.com/engine/reference/commandline/tag/#extended-description)，仅在使用容器镜像升级场景下有效|是 | - | imageurl | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如： 仅在使用磁盘镜像升级场景下有效|是 | + | imagetype | string | 升级镜像的类型 | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。
**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是 | + | opstype | string | 操作类型：升级,回退或者配置 | 仅支持upgrade ，config 或者 rollback |是 | + | osversion | string | 升级/回退的目标版本 | osversion需与节点的目标os版本对应（节点上/etc/os-release中PRETTY_NAME字段或k8s检查到的节点os版本） 例如：KubeOS 1.0.0。 |是 | + | maxunavailable | int | 每批同时进行升级/回退/配置的节点数。 | maxunavailable值大于实际节点数时，取实际节点数进行升级/回退/配置。 |是 | + | containerimage | string | 用于升级的容器镜像 | 仅在imagetype是容器类型时生效，仅支持以下3种格式的容器镜像地址： repository/name repository/name@sha256:xxxx repository/name:tag |是 | + | imageurl | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如： ，仅在使用磁盘镜像升级场景下有效 |是 | | checksum | string | 用于升级的磁盘镜像校验的checksum(SHA-256)值或者是用于升级的容器镜像的digests值 | 仅在升级场景下有效 |是 | | flagSafe | bool | 当imageurl的地址使用http协议表示是否是安全的 | 需为 true 或者 false ，仅在imageurl使用http协议时有效 |是 | | mtls | bool | 用于表示与imageurl连接是否采用https双向认证 | 需为 true 或者 false ，仅在imageurl使用https协议时有效|是 | | cacert | string | https或者https双向认证时使用的根证书文件 | 仅在imageurl使用https协议时有效| imageurl使用https协议时必选 | | clientcert | string | https双向认证时使用的客户端证书文件 | 仅在使用https双向认证时有效|mtls为true时必选 | | clientkey | string | https双向认证时使用的客户端公钥 | 仅在使用https双向认证时有效|mtls为true时必选 | - | evictpodforce | bool | 用于表示升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | - | nodeselector | string | 需要进行升级/配置/回滚操作的节点label | 用于只对具有某些特定label的节点而不是集群所有worker节点进行运维的场景，需要进行运维操作的节点需要包含key为upgrade.openeuler.org/node-selector的label，nodeselector为该label的value值，此参数不配置时，或者配置为""时默认对所有节点进行操作| 可选 | - | sysconfigs | / | 需要进行配置的参数值 | 在配置或者升级或者回退机器时有效，在升级或者回退操作之后即机器重启之后起效，详细字段说明请见```配置（Settings）指导```| 可选 | - | upgradeconfigs | / | 需要升级前进行的配置的参数值 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | + | evictpodforce | bool | 升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | + | sysconfigs | / | 配置设置 | 1. “opstype=config”时只进行配置。 2.“opstype=upgrade/rollback”时，代表升级/回退后配置，即在升级/回退重启后进行配置。```配置（Settings）指导``` | “opstype=config”时必选 | + | upgradeconfigs | / | 升级前配置设置 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | #### 升级指导 @@ -343,12 +387,13 @@ kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' ``` -* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的 imageurl, osversion, checksum, maxunavailable, flagSafe 或者containerimage字段进行相应修改。 +* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改 #### 配置（Settings）指导 * Settings参数说明: - 以进行配置时的示例yaml为例对配置的参数进行说明，示例yaml如下： + + 基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致： ```yaml apiVersion: upgrade.openeuler.org/v1alpha1 @@ -362,12 +407,9 @@ maxunavailable: edit.node.config.number containerimage: "" evictpodforce: false - imageurl: "" checksum: "" - flagSafe: false - mtls: false sysconfigs: - version: 1.0.0 + version: edit.sysconfigs.version configs: - model: kernel.sysctl contents: @@ -380,54 +422,82 @@ configpath: persist file path contents: - key: kernel param key3 - value: kernel param value3 + value: kernel param value3 + - model: grub.cmdline.current + contents: + - key: boot param key1 + - key: boot param key2 + value: boot param value2 + - key: boot param key3 + value: boot param value3 + operation: delete + - model: grub.cmdline.next + contents: + - key: boot param key4 + - key: boot param key5 + value: boot param value5 + - key: boot param key6 + value: boot param value6 + operation: delete ``` - * 配置的参数说明如下： - * version: 配置的版本,通过版本差异触发配置，请修改配置后更新 version - * configs: 具体配置内容 - * model: 进行的配置的类型，支持的配置类型请看[Settings 列表](#setting-列表) - * configpath: 如为持久化配置，配置文件路径 - * contents: 配置参数的 key / value 和对参数的操作。 - * key / value: 请看[Settings 列表](#setting-列表)对支持的配置的 key / value的说明。 - * operation: 若不指定operation，则默认为添加或更新。若指定为delete，代表删除目前OS中已配置的参数。 - **注意：** 当operation为delete时，yaml中的key/value必须和OS上想删除参数的key/value**一致**，否则删除失败。 - * upgradeconfigs与sysconfig参数相同，upgradeconfig为升级前进行的配置，仅在升级/回滚场景起效，在升级/回滚操作执行前进行配置，只进行配置或者需要升级/回滚重启后执行配置，使用sysconfigs + 配置的参数说明如下： + + | 参数 | 参数类型 | 参数说明 | 使用说明 | 配置中是否必选 | + | ---------- | -------- | --------------------------- | ------------------------------------------------------------ | ----------------------- | + | version | string | 配置的版本 | 通过version是否相等来判断配置是否触发，version为空（为""或者没有值）时同样进行判断，所以不配置sysconfigs/upgradeconfigs时，继存的version值会被清空并触发配置。 | 是 | + | configs | / | 具体配置内容 | 包含具体配置项列表。 | 是 | + | model | string | 配置的类型 | 支持的配置类型请看附录下的```Settings列表``` | 是 | + | configpath | string | 配置文件路径 | 仅在kernel.sysctl.persist配置类型中生效，请看附录下的```Settings列表```对配置文件路径的说明。 | 否 | + | contents | / | 具体key/value的值及操作类型 | 包含具体配置参数列表。 | 是 | + | key | string | 参数名称 | key不能为空，不能包含"="，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中每种配置类型对key的说明。 | 是 | + | value | string | 参数值 | key=value形式的参数中，value不能为空，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中对每种配置类型对value的说明。 | key=value形式的参数必选 | + | operation | string | 对参数进行的操作 | 仅对kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next类型的参数生效。默认为添加或更新。仅支持配置为delete，代表删除已存在的参数（key=value需完全一致才能删除）。 | 否 | + + + + * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置 + * 使用说明 + * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如上，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml + * 查看配置之前的节点的配置的版本和节点状态（NODESTATUS状态为idle） ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` * 执行命令，在集群中部署cr实例后，节点会根据配置的参数信息进行配置，再次查看节点状态(NODESTATUS变成config) ```shell kubectl apply -f upgrade_v1alpha1_os.yaml - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` * 再次查看节点的配置的版本确认节点是否配置完成(NODESTATUS恢复为idle) ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` -* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 +* 如果后续需要再次配置，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 #### 回退指导 * 回退场景 - * 虚拟机无法正常启动时，需要退回到上一可以启动的版本时进行回退操作，仅支持手动回退容器 OS 。 - * 虚拟机能够正常启动并且进入系统，需要将当前版本退回到老版本时进行回退操作，支持工具回退（类似升级方式）和手动回退，建议使用工具回退。 - * 配置出现错误，节点状态陷入config时，可以回退至上一个配置版本以恢复节点至idle状态。 - **注意**：在配置新版本时，出现错误前已经配置的参数无法回退。 + * 虚拟机无法正常启动时，可在grub启动项页面手动切换启动项，使系统回退至上一版本（即手动回退）。 + * 虚拟机能够正常启动并且进入系统时，支持工具回退和手动回退，建议使用工具回退。 + * 工具回退有两种方式： + 1. rollback模式直接回退至上一版本。 + 2. upgrade模式重新升级至上一版本 * 手动回退指导 - * 手动重启虚拟机，选择第二启动项进行回退，手动回退仅支持回退到本次升级之前的版本。 + + * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。 * 工具回退指导 * 回退至任意版本 - * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 + * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 + * YAML修改完成后执行更新命令，在集群中更新定制对象后，节点会根据配置的字段信息进行回退 ```shell @@ -499,23 +569,21 @@ * 查看节点容器 OS 版本(回退OS版本)或节点config版本&节点状态为idle(回退config版本)，确认回退是否成功。 ```shell - kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' - - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` -#### Admin容器 +## Admin容器镜像制作、部署和使用 KubeOS提供一个分离的包含sshd服务和hostshell工具的Admin容器，来帮助管理员在必要情况下登录KubeOS，其中的sshd服务由[sysmaster](https://gitee.com/openeuler/sysmaster)/systemd拉起。Admin容器部署后用户可通过ssh连接到节点的Admin容器，进入Admin容器后执行hostshell命令获取host的root shell。 -##### 部署方法 +### admin容器镜像制作 -以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/everything/aarch64/Packages/)到scripts/admin-container目录下。 +以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-SP1-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/update/aarch64/Packages/)到scripts/admin-container目录下。 -**修改**admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/docker_img)下载。 +修改admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts-sp1可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/docker_img)下载。 ```Dockerfile -FROM openeuler-22.03-lts +FROM openeuler-22.03-lts-sp1 RUN yum -y install openssh-clients util-linux @@ -546,7 +614,9 @@ bash -x kbimg.sh create admin-image -f admin-container/Dockerfile -d your_imageR docker push your_imageRepository/admin_imageName:version ``` -在master节点上部署Admin容器，需要提供ssh公钥来免密登录，**修改**并应用如下示例yaml文件: +### admin容器部署 + +在master节点上部署Admin容器，需要提供ssh公钥来免密登录，修改并应用如下示例yaml文件: ```yaml apiVersion: v1 @@ -615,6 +685,8 @@ spec: control-plane: admin-container-sysmaster ``` +### admin容器使用 + ssh到Admin容器，然后执行hostshell命令进入host root shell, 如： ```shell @@ -622,7 +694,7 @@ ssh -p your-exposed-port root@your.worker.node.ip hostshell ``` -##### hostshell +#### hostshell说明 为了保证KubeOS的轻便性，许多工具或命令没有安装在KubeOS内。因此，用户可以在制作Admin容器时，将期望使用的二进制文件放在容器内的如/usr/bin目录下。hostshell工具在执行时会将容器下的/usr/bin, /usr/sbin, /usr/local/bin, /usr/local/sbin路径添加到host root shell的环境变量。 @@ -637,8 +709,7 @@ hostshell #### kernel Settings -* kenerl.sysctl: 临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）, 示例如下: - +* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: ```yaml configs: - model: kernel.sysctl @@ -649,8 +720,7 @@ hostshell value: 0 operation: delete ``` - -* kernel.sysctl.persist: 设置持久化内核参数，key/value表示内核参数的key/value，key与value均不能为空且key不能包含“=”， configpath为配置文件路径，支持新建（需保证父目录存在），如不指定configpath默认修改/etc/sysctl.conf，示例如下： +* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: ```yaml configs: - model: kernel.sysctl.persist @@ -668,23 +738,38 @@ hostshell * grub.cmdline: 设置grub.cfg文件中的内核引导参数，该行参数在grub.cfg文件中类似如下示例： ```shell - linux /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 - ``` + linux /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + ``` * KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置： - * grub.cmdline.current：对当前分区的启动项参数进行配置。 - * grub.cmdline.next：对下一分区的启动项参数进行配置。 + + - grub.cmdline.current：对当前分区的启动项参数进行配置。 + - grub.cmdline.next：对下一分区的启动项参数进行配置。 + * 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。 + * grub.cmdline.current/next支持“key=value”（value不能为空），也支持单key。若value中有“=”，例如“root=UUID=some-uuid”，key应设置为第一个“=”前的所有字符，value为第一个“=”后的所有字符。 配置方法示例如下： + ```yaml configs: - - model: grub.cmdline.current - contents: - - key: selinux - value: 0 - - key: root - value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 - - key: panic - value: 3 - operation: delete + - model: grub.cmdline.current + contents: + - key: selinux + value: "0" + - key: root + value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 + - key: panic + value: "3" + operation: delete + - key: crash_kexec_post_notifiers + - model: grub.cmdline.next + contents: + - key: selinux + value: "0" + - key: root + value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 + - key: panic + value: "3" + operation: delete + - key: crash_kexec_post_notifiers ``` diff --git a/scripts/bootloader.sh b/scripts/bootloader.sh index 75096a38..df4be329 100644 --- a/scripts/bootloader.sh +++ b/scripts/bootloader.sh @@ -19,7 +19,7 @@ function install_grub2_x86 () cp -r /usr/lib/grub/x86_64-efi boot/efi/EFI/openEuler eval "grub2-mkimage -d /usr/lib/grub/x86_64-efi -O x86_64-efi --output=/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" - mkdir -p /boot/EFI/BOOT/ + mkdir -p /boot/efi/EFI/BOOT/ cp -f /boot/efi/EFI/openEuler/grubx64.efi /boot/efi/EFI/BOOT/BOOTX64.EFI fi } @@ -29,7 +29,7 @@ function install_grub2_efi () cp -r /usr/lib/grub/arm64-efi /boot/efi/EFI/openEuler/ eval "grub2-mkimage -d /usr/lib/grub/arm64-efi -O arm64-efi --output=/boot/efi/EFI/openEuler/grubaa64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" - mkdir -p /boot/EFI/BOOT/ + mkdir -p /boot/efi/EFI/BOOT/ cp -f /boot/efi/EFI/openEuler/grubaa64.efi /boot/efi/EFI/BOOT/BOOTAA64.EFI } -- Gitee From 6e00be8b1716780d07a20fbfc611363d1c3eb99d Mon Sep 17 00:00:00 2001 From: YouMeiYouMaoTai <15335885760@163.com> Date: Wed, 25 Sep 2024 16:36:49 +0800 Subject: [PATCH 2/2] Merge branch 'master' into dev; Feat:The rust version of the os-operator --- .gitignore | 7 + KubeOS-Rust/Cargo.lock | 2882 +++++++++++++++++ KubeOS-Rust/Cargo.toml | 12 + KubeOS-Rust/agent/Cargo.toml | 20 + KubeOS-Rust/agent/src/function.rs | 56 + KubeOS-Rust/agent/src/main.rs | 66 + KubeOS-Rust/agent/src/rpc/agent.rs | 30 + KubeOS-Rust/agent/src/rpc/agent_impl.rs | 217 ++ KubeOS-Rust/agent/src/rpc/mod.rs | 19 + KubeOS-Rust/cli/Cargo.toml | 15 + KubeOS-Rust/cli/src/client.rs | 56 + KubeOS-Rust/cli/src/lib.rs | 14 + KubeOS-Rust/cli/src/method/callable_method.rs | 54 + KubeOS-Rust/cli/src/method/configure.rs | 72 + KubeOS-Rust/cli/src/method/mod.rs | 18 + KubeOS-Rust/cli/src/method/prepare_upgrade.rs | 78 + KubeOS-Rust/cli/src/method/request.rs | 88 + KubeOS-Rust/cli/src/method/rollback.rs | 42 + KubeOS-Rust/cli/src/method/upgrade.rs | 42 + KubeOS-Rust/manager/Cargo.toml | 25 + KubeOS-Rust/manager/src/api/agent_status.rs | 21 + KubeOS-Rust/manager/src/api/mod.rs | 17 + KubeOS-Rust/manager/src/api/types.rs | 140 + KubeOS-Rust/manager/src/lib.rs | 15 + KubeOS-Rust/manager/src/sys_mgmt/config.rs | 558 ++++ .../manager/src/sys_mgmt/containerd_image.rs | 301 ++ .../manager/src/sys_mgmt/disk_image.rs | 406 +++ .../manager/src/sys_mgmt/docker_image.rs | 236 ++ KubeOS-Rust/manager/src/sys_mgmt/mod.rs | 23 + KubeOS-Rust/manager/src/sys_mgmt/values.rs | 36 + KubeOS-Rust/manager/src/utils/common.rs | 310 ++ .../manager/src/utils/container_image.rs | 341 ++ KubeOS-Rust/manager/src/utils/executor.rs | 89 + .../manager/src/utils/image_manager.rs | 206 ++ KubeOS-Rust/manager/src/utils/mod.rs | 23 + KubeOS-Rust/manager/src/utils/partition.rs | 117 + KubeOS-Rust/operator/Cargo.toml | 47 + .../operator/src/controller/apiclient.rs | 110 + .../operator/src/controller/apiserver_mock.rs | 995 ++++++ .../operator/src/controller/controller.rs | 696 ++++ KubeOS-Rust/operator/src/controller/crd.rs | 79 + KubeOS-Rust/operator/src/controller/mod.rs | 24 + KubeOS-Rust/operator/src/controller/values.rs | 43 + KubeOS-Rust/operator/src/main.rs | 74 + KubeOS-Rust/proxy/Cargo.toml | 49 + .../proxy/src/controller/agentclient.rs | 153 + KubeOS-Rust/proxy/src/controller/apiclient.rs | 147 + .../proxy/src/controller/apiserver_mock.rs | 681 ++++ .../proxy/src/controller/controller.rs | 556 ++++ KubeOS-Rust/proxy/src/controller/crd.rs | 77 + KubeOS-Rust/proxy/src/controller/mod.rs | 26 + KubeOS-Rust/proxy/src/controller/utils.rs | 154 + KubeOS-Rust/proxy/src/controller/values.rs | 33 + KubeOS-Rust/proxy/src/drain.rs | 511 +++ KubeOS-Rust/proxy/src/main.rs | 49 + KubeOS-Rust/proxy/tests/common/mod.rs | 63 + KubeOS-Rust/proxy/tests/drain_test.rs | 41 + .../proxy/tests/setup/kind-config.yaml | 5 + KubeOS-Rust/proxy/tests/setup/resources.yaml | 102 + .../proxy/tests/setup/setup_test_env.sh | 81 + KubeOS-Rust/rustfmt.toml | 11 + Makefile | 16 +- VERSION | 2 +- .../config/crd/upgrade.openeuler.org_os.yaml | 2 +- .../upgrade.openeuler.org_osinstances.yaml | 8 +- docs/quick-start.md | 377 ++- scripts/bootloader.sh | 4 +- 67 files changed, 11713 insertions(+), 155 deletions(-) create mode 100644 KubeOS-Rust/Cargo.lock create mode 100644 KubeOS-Rust/Cargo.toml create mode 100644 KubeOS-Rust/agent/Cargo.toml create mode 100644 KubeOS-Rust/agent/src/function.rs create mode 100644 KubeOS-Rust/agent/src/main.rs create mode 100644 KubeOS-Rust/agent/src/rpc/agent.rs create mode 100644 KubeOS-Rust/agent/src/rpc/agent_impl.rs create mode 100644 KubeOS-Rust/agent/src/rpc/mod.rs create mode 100644 KubeOS-Rust/cli/Cargo.toml create mode 100644 KubeOS-Rust/cli/src/client.rs create mode 100644 KubeOS-Rust/cli/src/lib.rs create mode 100644 KubeOS-Rust/cli/src/method/callable_method.rs create mode 100644 KubeOS-Rust/cli/src/method/configure.rs create mode 100644 KubeOS-Rust/cli/src/method/mod.rs create mode 100644 KubeOS-Rust/cli/src/method/prepare_upgrade.rs create mode 100644 KubeOS-Rust/cli/src/method/request.rs create mode 100644 KubeOS-Rust/cli/src/method/rollback.rs create mode 100644 KubeOS-Rust/cli/src/method/upgrade.rs create mode 100644 KubeOS-Rust/manager/Cargo.toml create mode 100644 KubeOS-Rust/manager/src/api/agent_status.rs create mode 100644 KubeOS-Rust/manager/src/api/mod.rs create mode 100644 KubeOS-Rust/manager/src/api/types.rs create mode 100644 KubeOS-Rust/manager/src/lib.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/config.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/mod.rs create mode 100644 KubeOS-Rust/manager/src/sys_mgmt/values.rs create mode 100644 KubeOS-Rust/manager/src/utils/common.rs create mode 100644 KubeOS-Rust/manager/src/utils/container_image.rs create mode 100644 KubeOS-Rust/manager/src/utils/executor.rs create mode 100644 KubeOS-Rust/manager/src/utils/image_manager.rs create mode 100644 KubeOS-Rust/manager/src/utils/mod.rs create mode 100644 KubeOS-Rust/manager/src/utils/partition.rs create mode 100644 KubeOS-Rust/operator/Cargo.toml create mode 100644 KubeOS-Rust/operator/src/controller/apiclient.rs create mode 100644 KubeOS-Rust/operator/src/controller/apiserver_mock.rs create mode 100644 KubeOS-Rust/operator/src/controller/controller.rs create mode 100644 KubeOS-Rust/operator/src/controller/crd.rs create mode 100644 KubeOS-Rust/operator/src/controller/mod.rs create mode 100644 KubeOS-Rust/operator/src/controller/values.rs create mode 100644 KubeOS-Rust/operator/src/main.rs create mode 100644 KubeOS-Rust/proxy/Cargo.toml create mode 100644 KubeOS-Rust/proxy/src/controller/agentclient.rs create mode 100644 KubeOS-Rust/proxy/src/controller/apiclient.rs create mode 100644 KubeOS-Rust/proxy/src/controller/apiserver_mock.rs create mode 100644 KubeOS-Rust/proxy/src/controller/controller.rs create mode 100644 KubeOS-Rust/proxy/src/controller/crd.rs create mode 100644 KubeOS-Rust/proxy/src/controller/mod.rs create mode 100644 KubeOS-Rust/proxy/src/controller/utils.rs create mode 100644 KubeOS-Rust/proxy/src/controller/values.rs create mode 100644 KubeOS-Rust/proxy/src/drain.rs create mode 100644 KubeOS-Rust/proxy/src/main.rs create mode 100644 KubeOS-Rust/proxy/tests/common/mod.rs create mode 100644 KubeOS-Rust/proxy/tests/drain_test.rs create mode 100644 KubeOS-Rust/proxy/tests/setup/kind-config.yaml create mode 100644 KubeOS-Rust/proxy/tests/setup/resources.yaml create mode 100644 KubeOS-Rust/proxy/tests/setup/setup_test_env.sh create mode 100644 KubeOS-Rust/rustfmt.toml diff --git a/.gitignore b/.gitignore index 5e56e040..4d173c5e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,8 @@ +# vscode settings +.vscode + +# rust dependencies +target/ + +# KubeOS bin /bin diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock new file mode 100644 index 00000000..ad175d9c --- /dev/null +++ b/KubeOS-Rust/Cargo.lock @@ -0,0 +1,2882 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom 0.2.10", + "instant", + "rand 0.8.5", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.48.5", +] + +[[package]] +name = "cli" +version = "1.0.6" +dependencies = [ + "anyhow", + "jsonrpc", + "log", + "manager", + "serde", + "serde_json", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if", + "num_cpus", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "dyn-clone" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "globset" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "h2" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.9", + "indexmap 1.9.3", + "slab", + "tokio", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http 0.2.9", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.9", + "http-body 0.4.5", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.9", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.2.0", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.25", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.25", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.2.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.2.0", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.3", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "js-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-patch" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce" +dependencies = [ + "serde", + "serde_json", + "treediff", +] + +[[package]] +name = "jsonpath_lib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f" +dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "jsonrpc" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd8d6b3f301ba426b30feca834a2a18d48d5b54e5065496b5c1b05537bee3639" +dependencies = [ + "base64 0.13.1", + "serde", + "serde_json", +] + +[[package]] +name = "jsonrpc-core" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +dependencies = [ + "futures", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "jsonrpc-derive" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "jsonrpc-ipc-server" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382bb0206323ca7cda3dcd7e245cea86d37d02457a02a975e3378fb149a48845" +dependencies = [ + "futures", + "jsonrpc-core", + "jsonrpc-server-utils", + "log", + "parity-tokio-ipc", + "parking_lot", + "tower-service", +] + +[[package]] +name = "jsonrpc-server-utils" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4" +dependencies = [ + "bytes", + "futures", + "globset", + "jsonrpc-core", + "lazy_static", + "log", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", + "unicase", +] + +[[package]] +name = "k8s-openapi" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c" +dependencies = [ + "base64 0.13.1", + "bytes", + "chrono", + "http 0.2.9", + "percent-encoding", + "serde", + "serde-value", + "serde_json", + "url", +] + +[[package]] +name = "kube" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4b96944d327b752df4f62f3a31d8694892af06fb585747c0b5e664927823d1a" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-derive", + "kube-runtime", +] + +[[package]] +name = "kube-client" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232db1af3d3680f9289cf0b4db51b2b9fee22550fc65d25869e39b23e0aaa696" +dependencies = [ + "base64 0.13.1", + "bytes", + "chrono", + "dirs-next", + "either", + "futures", + "http 0.2.9", + "http-body 0.4.5", + "hyper 0.14.25", + "hyper-timeout", + "hyper-tls 0.5.0", + "jsonpath_lib", + "k8s-openapi", + "kube-core", + "openssl", + "pem", + "pin-project", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tokio", + "tokio-native-tls", + "tokio-util 0.6.10", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de491f8c9ee97117e0b47a629753e939c2392d5d0a40f6928e582a5fba328098" +dependencies = [ + "chrono", + "form_urlencoded", + "http 0.2.9", + "json-patch", + "k8s-openapi", + "once_cell", + "schemars", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "kube-derive" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbb86bb3607245a67c8ad3a52aff41108f36b0d1e9e3e82ffb5760bfd84b965" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 1.0.109", +] + +[[package]] +name = "kube-runtime" +version = "0.66.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710729592eb30219b4e84898e91dc991fe09ccafe2c17fec4e45c3426c61abe0" +dependencies = [ + "backoff", + "dashmap", + "derivative", + "futures", + "json-patch", + "k8s-openapi", + "kube-client", + "pin-project", + "serde", + "serde_json", + "smallvec", + "thiserror", + "tokio", + "tokio-util 0.6.10", + "tracing", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" + +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.0", + "libc", + "redox_syscall 0.4.1", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4dcd960cc540667f619483fc99102f88d6118b87730e24e8fbe8054b7445e4" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "manager" +version = "1.0.6" +dependencies = [ + "anyhow", + "env_logger", + "lazy_static", + "log", + "mockall", + "mockito", + "nix", + "predicates", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "tempfile", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", +] + +[[package]] +name = "mockall" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4a1c770583dac7ab5e2f6c139153b783a53a1bbee9729613f193e59828326" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "mockito" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80f9fece9bd97ab74339fe19f4bcaf52b76dcc18e5364c7977c1838f76b38de9" +dependencies = [ + "assert-json-diff", + "httparse", + "lazy_static", + "log", + "rand 0.8.5", + "regex", + "serde_json", + "serde_urlencoded", + "similar", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", + "pin-utils", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.3", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" + +[[package]] +name = "openssl" +version = "0.10.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "operator" +version = "1.0.6" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-trait", + "cli", + "env_logger", + "futures", + "h2 0.3.16", + "http 0.2.9", + "hyper 0.14.25", + "k8s-openapi", + "kube", + "log", + "manager", + "mockall", + "regex", + "reqwest", + "schemars", + "serde", + "serde_json", + "socket2 0.4.9", + "thiserror", + "thread_local", + "tokio", + "tokio-retry", + "tower-test", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os-agent" +version = "1.0.6" +dependencies = [ + "anyhow", + "env_logger", + "jsonrpc-core", + "jsonrpc-derive", + "jsonrpc-ipc-server", + "lazy_static", + "log", + "manager", + "nix", + "serde", + "serde_json", +] + +[[package]] +name = "parity-tokio-ipc" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9981e32fb75e004cc148f5fb70342f393830e0a4aa62e3cc93b50976218d42b6" +dependencies = [ + "futures", + "libc", + "log", + "rand 0.7.3", + "tokio", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "predicates" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc3d91237f5de3bcd9d927e24d03b495adb6135097b001cea7403e2d573d00a9" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da1c2388b1513e1b605fcec39a95e0a9e8ef088f71443ef37099fa9ae6673fcb" + +[[package]] +name = "predicates-tree" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d86de6de25020a36c6d3643a86d9a6a9f552107c0559c60ea03551b5e16c032" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proxy" +version = "1.0.6" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-trait", + "cli", + "env_logger", + "futures", + "h2 0.3.16", + "http 0.2.9", + "hyper 0.14.25", + "k8s-openapi", + "kube", + "log", + "manager", + "mockall", + "regex", + "reqwest", + "schemars", + "serde", + "serde_json", + "socket2 0.4.9", + "thiserror", + "thread_local", + "tokio", + "tokio-retry", + "tower-test", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.10", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom 0.2.10", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "reqwest" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338" +dependencies = [ + "base64 0.21.5", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.3", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.2.0", + "hyper-rustls", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.10", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustls" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.5", +] + +[[package]] +name = "rustls-pki-types" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868e20fada228fefaf6b652e00cc73623d54f8171e7352c18bb281571f2d92da" + +[[package]] +name = "rustls-webpki" +version = "0.102.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "schemars" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1847b767a3d62d95cbf3d8a9f0e421cf57a0d8aa4f411d4b16525afb0284d4ed" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4d7e1b012cb3d9129567661a63755ea4b8a7386d339dc945ae187e403c6743" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" +dependencies = [ + "serde", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c4437699b6d34972de58652c68b98cb5b53a4199ab126db8e20ec8ded29a721" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_json" +version = "1.0.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +dependencies = [ + "indexmap 1.9.3", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap 1.9.3", + "ryu", + "serde", + "yaml-rust", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "similar" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +dependencies = [ + "autocfg", + "cfg-if", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "termcolor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.4.9", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-test" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89b3cbabd3ae862100094ae433e1def582cf86451b4e9bf83aa7ac1d8a7d719" +dependencies = [ + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "slab", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tokio-util 0.7.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8" +dependencies = [ + "base64 0.13.1", + "bitflags 1.3.2", + "bytes", + "futures-core", + "futures-util", + "http 0.2.9", + "http-body 0.4.5", + "http-range-header", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tower-test" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4546773ffeab9e4ea02b8872faa49bb616a80a7da66afc2f32688943f97efa7" +dependencies = [ + "futures-util", + "pin-project", + "tokio", + "tokio-test", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tracing" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "treediff" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff" +dependencies = [ + "serde_json", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" + +[[package]] +name = "web-sys" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml new file mode 100644 index 00000000..01e05b11 --- /dev/null +++ b/KubeOS-Rust/Cargo.toml @@ -0,0 +1,12 @@ +[workspace] +members = ["agent", "cli", "manager", "operator", "proxy"] +resolver = "2" + +[profile.release] +debug = false +debug-assertions = false +lto = true +opt-level = 's' +overflow-checks = false +panic = "unwind" +rpath = false diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml new file mode 100644 index 00000000..83e1b7c0 --- /dev/null +++ b/KubeOS-Rust/agent/Cargo.toml @@ -0,0 +1,20 @@ +[package] +description = "KubeOS os-agent" +edition = "2021" +license = "MulanPSL-2.0" +name = "os-agent" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +anyhow = { version = "1.0" } +env_logger = { version = "0.9" } +jsonrpc-core = { version = "18.0" } +jsonrpc-derive = { version = "18.0" } +jsonrpc-ipc-server = { version = "18.0" } +lazy_static = { version = "1.4" } +log = { version = "= 0.4.15" } +manager = { package = "manager", path = "../manager" } +nix = { version = "0.26.2" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } diff --git a/KubeOS-Rust/agent/src/function.rs b/KubeOS-Rust/agent/src/function.rs new file mode 100644 index 00000000..9789d95c --- /dev/null +++ b/KubeOS-Rust/agent/src/function.rs @@ -0,0 +1,56 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub use jsonrpc_core::Result as RpcResult; +use jsonrpc_core::{Error, ErrorCode}; +pub use jsonrpc_derive::rpc; +use log::error; + +const RPC_OP_ERROR: i64 = -1; + +pub struct RpcFunction; + +impl RpcFunction { + pub fn call(f: F) -> RpcResult + where + F: FnOnce() -> anyhow::Result, + { + (f)().map_err(|e| { + let error_message = format!("{:#}", e); + error!("{}", error_message.replace('\n', " ").replace('\r', "")); + Error { code: ErrorCode::ServerError(RPC_OP_ERROR), message: format!("{:?}", e), data: None } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rpcfunction_call() { + // Define a mock function that returns a result + fn mock_ok_function() -> anyhow::Result { + Ok(42) + } + let result: RpcResult = RpcFunction::call(mock_ok_function); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), 42); + + fn mock_err_function() -> anyhow::Result { + Err(anyhow::anyhow!("error")) + } + let result: RpcResult = RpcFunction::call(mock_err_function); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code, ErrorCode::ServerError(RPC_OP_ERROR)); + } +} diff --git a/KubeOS-Rust/agent/src/main.rs b/KubeOS-Rust/agent/src/main.rs new file mode 100644 index 00000000..cd95ef07 --- /dev/null +++ b/KubeOS-Rust/agent/src/main.rs @@ -0,0 +1,66 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{self, DirBuilder, Permissions}, + os::unix::fs::{DirBuilderExt, PermissionsExt}, + path::Path, +}; + +use env_logger::{Builder, Env, Target}; +use jsonrpc_core::{IoHandler, IoHandlerExtension}; +use jsonrpc_ipc_server::ServerBuilder; + +mod function; +mod rpc; + +use log::info; +use rpc::{Agent, AgentImpl}; + +const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; +const CARGO_PKG_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); + +fn start_and_run(sock_path: &str) { + let socket_path = Path::new(sock_path); + + // Create directory for socket if it doesn't exist + if let Some(dir_path) = socket_path.parent() { + if !dir_path.exists() { + DirBuilder::new().mode(0o750).create(dir_path).expect("Couldn't create directory for socket"); + } + } + + // Add RPC methods to IoHandler + let mut io = IoHandler::new(); + AgentImpl::default().to_delegate().augment(&mut io); + + // Build and start server + let builder = ServerBuilder::new(io); + let server = builder.start(sock_path).expect("Couldn't open socket"); + + let gid = nix::unistd::getgid(); + nix::unistd::chown(socket_path, Some(nix::unistd::ROOT), Some(gid)).expect("Couldn't set socket group"); + + // Set socket permissions to 0640 + let socket_permissions = Permissions::from_mode(0o640); + fs::set_permissions(socket_path, socket_permissions).expect("Couldn't set socket permissions"); + + info!("os-agent started, waiting for requests..."); + server.wait(); +} + +fn main() { + Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init(); + + info!("os-agent version is: {}", CARGO_PKG_VERSION.unwrap_or("NOT FOUND")); + start_and_run(SOCK_PATH); +} diff --git a/KubeOS-Rust/agent/src/rpc/agent.rs b/KubeOS-Rust/agent/src/rpc/agent.rs new file mode 100644 index 00000000..2496bfb4 --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/agent.rs @@ -0,0 +1,30 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use manager::api::{ConfigureRequest, Response, UpgradeRequest}; + +use super::function::{rpc, RpcResult}; + +#[rpc(server)] +pub trait Agent { + #[rpc(name = "prepare_upgrade")] + fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult; + + #[rpc(name = "upgrade")] + fn upgrade(&self) -> RpcResult; + + #[rpc(name = "configure")] + fn configure(&self, req: ConfigureRequest) -> RpcResult; + + #[rpc(name = "rollback")] + fn rollback(&self) -> RpcResult; +} diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs new file mode 100644 index 00000000..ab826413 --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -0,0 +1,217 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{sync::Mutex, thread, time::Duration}; + +use anyhow::{bail, Result}; +use log::{debug, info}; +use manager::{ + api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, + sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH}, + utils::{get_partition_info, switch_boot_menuentry, RealCommandExecutor}, +}; +use nix::{sys::reboot::RebootMode, unistd::sync}; + +use super::{ + agent::Agent, + function::{RpcFunction, RpcResult}, +}; + +pub struct AgentImpl { + mutex: Mutex<()>, + disable_reboot: bool, +} + +impl Agent for AgentImpl { + fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult { + RpcFunction::call(|| self.prepare_upgrade_impl(req)) + } + + fn upgrade(&self) -> RpcResult { + RpcFunction::call(|| self.upgrade_impl()) + } + + fn configure(&self, req: ConfigureRequest) -> RpcResult { + RpcFunction::call(|| self.configure_impl(req)) + } + + fn rollback(&self) -> RpcResult { + RpcFunction::call(|| self.rollback_impl()) + } +} + +impl Default for AgentImpl { + fn default() -> Self { + Self { mutex: Mutex::new(()), disable_reboot: false } + } +} + +impl AgentImpl { + fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + debug!("Received an 'prepare upgrade' request: {:?}", req); + info!("Start preparing for upgrading to version: {}", req.version); + + let handler: Box> = match req.image_type.as_str() { + "containerd" => Box::new(ImageType::Containerd(CtrImageHandler::default())), + "docker" => Box::new(ImageType::Docker(DockerImageHandler::default())), + "disk" => Box::new(ImageType::Disk(DiskImageHandler::default())), + _ => bail!("Invalid image type \"{}\"", req.image_type), + }; + + let image_manager = handler.download_image(&req)?; + info!("Ready to install image: {:?}", image_manager.paths.image_path.display()); + image_manager.install()?; + + Ok(Response { status: AgentStatus::UpgradeReady }) + } + + fn upgrade_impl(&self) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + info!("Start to upgrade"); + let command_executor = RealCommandExecutor {}; + let (_, next_partition_info) = get_partition_info(&command_executor)?; + + // based on boot mode use different command to switch boot partition + let device = next_partition_info.device.as_str(); + let menuentry = next_partition_info.menuentry.as_str(); + switch_boot_menuentry(&command_executor, DEFAULT_GRUBENV_PATH, menuentry)?; + info!("Switch to boot partition: {}, device: {}", menuentry, device); + self.reboot()?; + Ok(Response { status: AgentStatus::Upgraded }) + } + + fn configure_impl(&self, mut req: ConfigureRequest) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + debug!("Received a 'configure' request: {:?}", req); + info!("Start to configure"); + let config_map = &*CONFIG_TEMPLATE; + for config in req.configs.iter_mut() { + let config_type = &config.model; + if let Some(configuration) = config_map.get(config_type) { + debug!("Found configuration type: \"{}\"", config_type); + configuration.set_config(config)?; + } else { + bail!("Unknown configuration type: \"{}\"", config_type); + } + } + Ok(Response { status: AgentStatus::Configured }) + } + + fn rollback_impl(&self) -> Result { + let lock = self.mutex.try_lock(); + if lock.is_err() { + bail!("os-agent is processing another request"); + } + info!("Start to rollback"); + let command_executor = RealCommandExecutor {}; + let (_, next_partition_info) = get_partition_info(&command_executor)?; + switch_boot_menuentry( + &command_executor, + manager::sys_mgmt::DEFAULT_GRUBENV_PATH, + &next_partition_info.menuentry, + )?; + info!("Switch to boot partition: {}, device: {}", next_partition_info.menuentry, next_partition_info.device); + self.reboot()?; + Ok(Response { status: AgentStatus::Rollbacked }) + } + + fn reboot(&self) -> Result<()> { + info!("Wait to reboot"); + thread::sleep(Duration::from_secs(1)); + sync(); + if self.disable_reboot { + return Ok(()); + } + nix::sys::reboot::reboot(RebootMode::RB_AUTOBOOT)?; + Ok(()) + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + + use manager::api::{CertsInfo, Sysconfig}; + + use super::*; + + #[test] + fn test_reboot() { + let mut agent = AgentImpl::default(); + agent.disable_reboot = true; + let res = agent.reboot(); + assert!(res.is_ok()); + } + + #[test] + fn test_configure() { + let agent = AgentImpl::default(); + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "kernel.sysctl".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req).unwrap(); + assert_eq!(res, Response { status: AgentStatus::Configured }); + + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "invalid".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req); + assert!(res.is_err()); + + // test lock + let _lock = agent.mutex.lock().unwrap(); + let req = ConfigureRequest { + configs: vec![Sysconfig { + model: "kernel.sysctl".to_string(), + config_path: "".to_string(), + contents: HashMap::new(), + }], + }; + let res = agent.configure(req); + assert!(res.is_err()); + } + + #[test] + fn test_prepare_upgrade() { + let agent = AgentImpl::default(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "xxx".into(), + image_type: "xxx".into(), + container_image: "xxx".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let res = agent.prepare_upgrade(req); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/agent/src/rpc/mod.rs b/KubeOS-Rust/agent/src/rpc/mod.rs new file mode 100644 index 00000000..976356be --- /dev/null +++ b/KubeOS-Rust/agent/src/rpc/mod.rs @@ -0,0 +1,19 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use super::function; + +mod agent; +mod agent_impl; + +pub use agent::*; +pub use agent_impl::*; diff --git a/KubeOS-Rust/cli/Cargo.toml b/KubeOS-Rust/cli/Cargo.toml new file mode 100644 index 00000000..78d5fd51 --- /dev/null +++ b/KubeOS-Rust/cli/Cargo.toml @@ -0,0 +1,15 @@ +[package] +description = "KubeOS os-agent client" +edition = "2021" +license = "MulanPSL-2.0" +name = "cli" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dependencies] +anyhow = { version = "1.0" } +jsonrpc = { version = "0.13", features = ["simple_uds"] } +kubeos-manager = { package = "manager", path = "../manager" } +log = { version = "0.4" } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs new file mode 100644 index 00000000..37518bdc --- /dev/null +++ b/KubeOS-Rust/cli/src/client.rs @@ -0,0 +1,56 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::path::Path; + +use jsonrpc::{ + simple_uds::UdsTransport, Client as JsonRPCClient, Request as JsonRPCRequest, Response as JsonRPCResponse, +}; +use serde_json::value::RawValue; + +pub struct Client { + json_rpc_client: JsonRPCClient, +} + +pub struct Request<'a>(JsonRPCRequest<'a>); + +impl<'a> Request<'a> {} + +impl Client { + pub fn new>(socket_path: P) -> Self { + Client { json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)) } + } + + pub fn build_request<'a>(&self, command: &'a str, params: &'a [Box]) -> Request<'a> { + let json_rpc_request = self.json_rpc_client.build_request(command, params); + let request = Request(json_rpc_request); + request + } + + pub fn send_request(&self, request: Request) -> Result { + self.json_rpc_client.send_request(request.0) + } +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_client() { + let socket_path = "/tmp/KubeOS-test.sock"; + let cli = Client::new(socket_path); + let command = "example_command"; + let params = vec![]; + let request = cli.send_request(cli.build_request(command, ¶ms)); + assert!(request.is_err()); + } +} diff --git a/KubeOS-Rust/cli/src/lib.rs b/KubeOS-Rust/cli/src/lib.rs new file mode 100644 index 00000000..cd66d72f --- /dev/null +++ b/KubeOS-Rust/cli/src/lib.rs @@ -0,0 +1,14 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod client; +pub mod method; diff --git a/KubeOS-Rust/cli/src/method/callable_method.rs b/KubeOS-Rust/cli/src/method/callable_method.rs new file mode 100644 index 00000000..a174b5b8 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/callable_method.rs @@ -0,0 +1,54 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde_json::value::RawValue; + +use super::request::{parse_error, request}; +use crate::client::Client; + +pub trait RpcMethod { + type Response: serde::de::DeserializeOwned; + fn command_name(&self) -> &'static str; + fn command_params(&self) -> Vec>; + fn call(&self, client: &Client) -> Result { + let response = request(client, self.command_name(), self.command_params())?; + response.result().map_err(parse_error) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::client; + + #[derive(Default)] + struct DummyMethod; + + impl RpcMethod for DummyMethod { + type Response = String; + + fn command_name(&self) -> &'static str { + "dummy_command" + } + + fn command_params(&self) -> Vec> { + vec![] + } + } + + #[test] + fn test_call() { + let client = client::Client::new("/tmp/KubeOS-test.sock"); + let result = DummyMethod::default().call(&client); + assert!(result.is_err()); + } +} diff --git a/KubeOS-Rust/cli/src/method/configure.rs b/KubeOS-Rust/cli/src/method/configure.rs new file mode 100644 index 00000000..cca752d0 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/configure.rs @@ -0,0 +1,72 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::{to_raw_value, RawValue}; + +use crate::method::callable_method::RpcMethod; + +pub struct ConfigureMethod { + req: api::ConfigureRequest, +} + +impl ConfigureMethod { + pub fn new(req: api::ConfigureRequest) -> Self { + ConfigureMethod { req } + } + + pub fn set_configure_request(&mut self, req: api::ConfigureRequest) -> &Self { + self.req = req; + self + } +} + +impl RpcMethod for ConfigureMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "configure" + } + fn command_params(&self) -> Vec> { + vec![to_raw_value(&self.req).unwrap()] + } +} +#[cfg(test)] +mod tests { + use kubeos_manager::api::{ConfigureRequest, Sysconfig}; + + use super::*; + + #[test] + fn test_configure_method() { + let req = ConfigureRequest { configs: vec![] }; + let mut method = ConfigureMethod::new(req); + + // Test set_configure_request method + let new_req = ConfigureRequest { + configs: vec![Sysconfig { + model: "model".to_string(), + config_path: "config_path".to_string(), + contents: Default::default(), + }], + }; + method.set_configure_request(new_req); + + // Test command_name method + assert_eq!(method.command_name(), "configure"); + + // Test command_params method + let expected_params = + "RawValue({\"configs\":[{\"model\":\"model\",\"config_path\":\"config_path\",\"contents\":{}}]})"; + let actual_params = format!("{:?}", method.command_params()[0]); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/mod.rs b/KubeOS-Rust/cli/src/method/mod.rs new file mode 100644 index 00000000..e1f38bcd --- /dev/null +++ b/KubeOS-Rust/cli/src/method/mod.rs @@ -0,0 +1,18 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod callable_method; +pub mod configure; +pub mod prepare_upgrade; +pub mod request; +pub mod rollback; +pub mod upgrade; diff --git a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs new file mode 100644 index 00000000..f2034f6b --- /dev/null +++ b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs @@ -0,0 +1,78 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::{to_raw_value, RawValue}; + +use crate::method::callable_method::RpcMethod; + +pub struct PrepareUpgradeMethod { + req: api::UpgradeRequest, +} + +impl PrepareUpgradeMethod { + pub fn new(req: api::UpgradeRequest) -> Self { + PrepareUpgradeMethod { req } + } + + pub fn set_prepare_upgrade_request(&mut self, req: api::UpgradeRequest) -> &Self { + self.req = req; + self + } +} + +impl RpcMethod for PrepareUpgradeMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "prepare_upgrade" + } + fn command_params(&self) -> Vec> { + vec![to_raw_value(&self.req).unwrap()] + } +} +#[cfg(test)] +mod tests { + use kubeos_manager::api::{CertsInfo, UpgradeRequest}; + + use super::*; + + #[test] + fn test_prepare_upgrade_method() { + let req = UpgradeRequest { + version: "v1".into(), + check_sum: "".into(), + image_type: "".into(), + container_image: "".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let mut method = PrepareUpgradeMethod::new(req); + let new_req = UpgradeRequest { + version: "v2".into(), + check_sum: "xxx".into(), + image_type: "xxx".into(), + container_image: "xxx".into(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + method.set_prepare_upgrade_request(new_req); + assert_eq!(method.command_name(), "prepare_upgrade"); + + let expected_params = "RawValue({\"version\":\"v2\",\"check_sum\":\"xxx\",\"image_type\":\"xxx\",\"container_image\":\"xxx\",\"image_url\":\"\",\"flag_safe\":false,\"mtls\":false,\"certs\":{\"ca_cert\":\"\",\"client_cert\":\"\",\"client_key\":\"\"}})"; + let actual_params = format!("{:?}", method.command_params()[0]); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs new file mode 100644 index 00000000..581aa639 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/request.rs @@ -0,0 +1,88 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::anyhow; +use jsonrpc::{Error, Response}; +use log::debug; +use serde_json::value::RawValue; + +use crate::client::Client; + +pub fn request(client: &Client, command: &str, params: Vec>) -> Result { + let request = client.build_request(command, ¶ms); + let response = client.send_request(request).map_err(parse_error); + debug!("{:#?}", response); + response +} + +pub fn parse_error(error: Error) -> anyhow::Error { + match error { + Error::Transport(e) => { + anyhow!( + "Cannot connect to KubeOS os-agent unix socket, {}", + e.source().map(|e| e.to_string()).unwrap_or_else(|| "Connection timeout".to_string()) + ) + }, + Error::Json(e) => { + debug!("Json parse error: {:?}", e); + anyhow!("Failed to parse response") + }, + Error::Rpc(ref e) => { + if e.message == "Method not found" { + anyhow!("Method is unimplemented") + } else { + anyhow!("{}", e.message) + } + }, + _ => { + debug!("{:?}", error); + anyhow!("Response is invalid") + }, + } +} + +#[cfg(test)] +mod tests { + use jsonrpc::error::RpcError; + use serde::de::Error as DeError; + + use super::*; + + #[test] + fn test_parse_error() { + // Test Error::Transport + let transport_error = + Error::Transport(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "Connection timeout"))); + let result = parse_error(transport_error); + assert_eq!(result.to_string(), "Cannot connect to KubeOS os-agent unix socket, Connection timeout"); + + // Test Error::Json + let json_error = Error::Json(serde_json::Error::custom("Failed to parse response")); + let result = parse_error(json_error); + assert_eq!(result.to_string(), "Failed to parse response"); + + // Test Error::Rpc with "Method not found" message + let rpc_error = Error::Rpc(RpcError { code: -32601, message: "Method not found".to_string(), data: None }); + let result = parse_error(rpc_error); + assert_eq!(result.to_string(), "Method is unimplemented"); + + // Test Error::Rpc with other message + let rpc_error = Error::Rpc(RpcError { code: -32603, message: "Internal server error".to_string(), data: None }); + let result = parse_error(rpc_error); + assert_eq!(result.to_string(), "Internal server error"); + + // Test other Error variant + let other_error = Error::VersionMismatch; + let result = parse_error(other_error); + assert_eq!(result.to_string(), "Response is invalid"); + } +} diff --git a/KubeOS-Rust/cli/src/method/rollback.rs b/KubeOS-Rust/cli/src/method/rollback.rs new file mode 100644 index 00000000..7945f4b1 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/rollback.rs @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::RawValue; + +use crate::method::callable_method::RpcMethod; + +#[derive(Default)] +pub struct RollbackMethod {} + +impl RpcMethod for RollbackMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "rollback" + } + fn command_params(&self) -> Vec> { + vec![] + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_rollback_method() { + let method = RollbackMethod::default(); + assert_eq!(method.command_name(), "rollback"); + let expected_params = "[]"; + let actual_params = format!("{:?}", method.command_params()); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/cli/src/method/upgrade.rs b/KubeOS-Rust/cli/src/method/upgrade.rs new file mode 100644 index 00000000..f2f94cd5 --- /dev/null +++ b/KubeOS-Rust/cli/src/method/upgrade.rs @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kubeos_manager::api; +use serde_json::value::RawValue; + +use crate::method::callable_method::RpcMethod; + +#[derive(Default)] +pub struct UpgradeMethod {} + +impl RpcMethod for UpgradeMethod { + type Response = api::Response; + fn command_name(&self) -> &'static str { + "upgrade" + } + fn command_params(&self) -> Vec> { + vec![] + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_upgrade_method() { + let method = UpgradeMethod::default(); + assert_eq!(method.command_name(), "upgrade"); + let expected_params = "[]"; + let actual_params = format!("{:?}", method.command_params()); + assert_eq!(actual_params, expected_params); + } +} diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml new file mode 100644 index 00000000..f60a7c08 --- /dev/null +++ b/KubeOS-Rust/manager/Cargo.toml @@ -0,0 +1,25 @@ +[package] +description = "KubeOS os-agent manager" +edition = "2021" +license = "MulanPSL-2.0" +name = "manager" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[dev-dependencies] +mockall = { version = "=0.11.3" } +mockito = { version = "0.31.1", default-features = false } +predicates = { version = "=2.0.1" } +tempfile = { version = "3.6.0" } + +[dependencies] +anyhow = { version = "1.0" } +env_logger = { version = "0.9" } +lazy_static = { version = "1.4" } +log = { version = "0.4" } +nix = { version = "0.26.2" } +regex = { version = "1.7.3" } +reqwest = { version = "=0.12.2", features = ["blocking", "rustls-tls"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0" } +sha2 = { version = "0.10.8" } diff --git a/KubeOS-Rust/manager/src/api/agent_status.rs b/KubeOS-Rust/manager/src/api/agent_status.rs new file mode 100644 index 00000000..bb16e6bc --- /dev/null +++ b/KubeOS-Rust/manager/src/api/agent_status.rs @@ -0,0 +1,21 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)] +pub enum AgentStatus { + UpgradeReady, + Upgraded, + Rollbacked, + Configured, +} diff --git a/KubeOS-Rust/manager/src/api/mod.rs b/KubeOS-Rust/manager/src/api/mod.rs new file mode 100644 index 00000000..01c9df1a --- /dev/null +++ b/KubeOS-Rust/manager/src/api/mod.rs @@ -0,0 +1,17 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod agent_status; +mod types; + +pub use agent_status::*; +pub use types::*; diff --git a/KubeOS-Rust/manager/src/api/types.rs b/KubeOS-Rust/manager/src/api/types.rs new file mode 100644 index 00000000..98aeaa33 --- /dev/null +++ b/KubeOS-Rust/manager/src/api/types.rs @@ -0,0 +1,140 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::agent_status::*; +use crate::{ + sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler}, + utils::{CommandExecutor, UpgradeImageManager}, +}; + +#[derive(Deserialize, Serialize, Debug)] +pub struct UpgradeRequest { + pub version: String, + pub check_sum: String, + pub image_type: String, + pub container_image: String, + pub image_url: String, + pub flag_safe: bool, + pub mtls: bool, + pub certs: CertsInfo, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct CertsInfo { + pub ca_cert: String, + pub client_cert: String, + pub client_key: String, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct KeyInfo { + pub value: String, + pub operation: String, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct Sysconfig { + pub model: String, + pub config_path: String, + pub contents: HashMap, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct ConfigureRequest { + pub configs: Vec, +} + +#[derive(Deserialize, Serialize, Debug, PartialEq)] +pub struct Response { + pub status: AgentStatus, +} + +pub enum ImageType { + Containerd(CtrImageHandler), + Docker(DockerImageHandler), + Disk(DiskImageHandler), +} + +impl ImageType { + pub fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result> { + match self { + ImageType::Containerd(handler) => handler.download_image(req), + ImageType::Docker(handler) => handler.download_image(req), + ImageType::Disk(handler) => handler.download_image(req), + } + } +} +pub trait ImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result>; +} + +#[cfg(test)] +mod tests { + use anyhow::Result; + use mockall::mock; + + use super::*; + use crate::utils::PreparePath; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + #[test] + fn test_download_image() { + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: "kubeos-temp".to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + let mut mock_executor1 = MockCommandExec::new(); + mock_executor1.expect_run_command().returning(|_, _| Ok(())); + mock_executor1.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let c_handler = CtrImageHandler::new(PreparePath::default(), mock_executor1); + let image_type = ImageType::Containerd(c_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + + let mut mock_executor2 = MockCommandExec::new(); + mock_executor2.expect_run_command().returning(|_, _| Ok(())); + mock_executor2.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let docker_handler = DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor2); + let image_type = ImageType::Docker(docker_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + + let mut mock_executor3 = MockCommandExec::new(); + mock_executor3.expect_run_command().returning(|_, _| Ok(())); + mock_executor3.expect_run_command_with_output().returning(|_, _| Ok(String::new())); + let disk_handler = DiskImageHandler::new(PreparePath::default(), mock_executor3, "test".into()); + let image_type = ImageType::Disk(disk_handler); + let result = image_type.download_image(&req); + assert!(result.is_err()); + } +} diff --git a/KubeOS-Rust/manager/src/lib.rs b/KubeOS-Rust/manager/src/lib.rs new file mode 100644 index 00000000..b45cab99 --- /dev/null +++ b/KubeOS-Rust/manager/src/lib.rs @@ -0,0 +1,15 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub mod api; +pub mod sys_mgmt; +pub mod utils; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs new file mode 100644 index 00000000..138df9da --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs @@ -0,0 +1,558 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + collections::HashMap, + fs::{self, File}, + io::{self, BufRead, BufWriter, Write}, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + string::String, +}; + +use anyhow::{bail, Context, Result}; +use lazy_static::lazy_static; +use log::{debug, info, trace, warn}; +use regex::Regex; + +use crate::{api::*, sys_mgmt::values, utils::*}; + +lazy_static! { + pub static ref CONFIG_TEMPLATE: HashMap> = { + let mut config_map = HashMap::new(); + config_map.insert( + values::KERNEL_SYSCTL.to_string(), + Box::new(KernelSysctl::new(values::DEFAULT_PROC_PATH)) as Box, + ); + config_map.insert( + values::KERNEL_SYSCTL_PERSIST.to_string(), + Box::new(KernelSysctlPersist) as Box, + ); + config_map.insert( + values::GRUB_CMDLINE_CURRENT.to_string(), + Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: true }) + as Box, + ); + config_map.insert( + values::GRUB_CMDLINE_NEXT.to_string(), + Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: false }) + as Box, + ); + config_map + }; +} + +pub trait Configuration { + fn set_config(&self, config: &mut Sysconfig) -> Result<()>; +} + +pub struct KernelSysctl { + pub proc_path: String, +} +pub struct KernelSysctlPersist; +pub struct GrubCmdline { + pub grub_path: String, + pub is_cur_partition: bool, +} + +impl Configuration for KernelSysctl { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + info!("Start setting kernel.sysctl"); + for (key, key_info) in config.contents.iter() { + let proc_path = self.get_proc_path(key); + if key_info.operation == "delete" { + warn!("Failed to delete kernel.sysctl config with key \"{}\"", key); + } else if !key_info.value.is_empty() && key_info.operation.is_empty() { + fs::write(&proc_path, format!("{}\n", &key_info.value).as_bytes()) + .with_context(|| format!("Failed to write kernel.sysctl with key: \"{}\"", key))?; + info!("Configured kernel.sysctl {}={}", key, key_info.value); + } else { + warn!( + "Failed to parse kernel.sysctl, key: \"{}\", value: \"{}\", operation: \"{}\"", + key, key_info.value, key_info.operation + ); + } + } + Ok(()) + } +} + +impl KernelSysctl { + fn new(proc_path: &str) -> Self { + Self { proc_path: String::from(proc_path) } + } + + fn get_proc_path(&self, key: &str) -> PathBuf { + let path_str = format!("{}{}", self.proc_path, key.replace('.', "/")); + Path::new(&path_str).to_path_buf() + } +} + +impl Configuration for KernelSysctlPersist { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + info!("Start setting kernel.sysctl.persist"); + let mut config_path = &values::DEFAULT_KERNEL_CONFIG_PATH.to_string(); + if !config.config_path.is_empty() { + config_path = &config.config_path; + } + debug!("kernel.sysctl.persist config_path: \"{}\"", config_path); + create_config_file(config_path).with_context(|| format!("Failed to find config path \"{}\"", config_path))?; + let configs = get_and_set_configs(&mut config.contents, config_path) + .with_context(|| format!("Failed to set persist kernel configs \"{}\"", config_path))?; + write_configs_to_file(config_path, &configs).with_context(|| "Failed to write configs to file".to_string())?; + Ok(()) + } +} + +fn create_config_file(config_path: &str) -> Result<()> { + if !is_file_exist(config_path) { + let f = fs::File::create(config_path)?; + let metadata = f.metadata()?; + let mut permissions = metadata.permissions(); + permissions.set_mode(values::DEFAULT_KERNEL_CONFIG_PERM); + debug!("Create file {} with permission 0644", config_path); + } + Ok(()) +} + +fn get_and_set_configs(expect_configs: &mut HashMap, config_path: &str) -> Result> { + let f = File::open(config_path).with_context(|| format!("Failed to open config path \"{}\"", config_path))?; + let mut configs_write = Vec::new(); + for line in io::BufReader::new(f).lines() { + let line = line?; + // if line is a comment or blank + if line.starts_with('#') || line.starts_with(';') || line.trim().is_empty() { + configs_write.push(line); + continue; + } + let config_kv: Vec<&str> = line.splitn(2, '=').map(|s| s.trim()).collect(); + // if config_kv is not a key-value pair + if config_kv.len() != 2 { + bail!("could not parse sysctl config {}", line); + } + let new_key_info = expect_configs.get(config_kv[0]); + let new_config = match new_key_info { + Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config_kv, new_key_info), + Some(new_key_info) => handle_update_key(&config_kv, new_key_info), + None => config_kv.join("="), + }; + configs_write.push(new_config); + expect_configs.remove(config_kv[0]); + } + let new_config = handle_add_key(expect_configs, false); + configs_write.extend(new_config); + Ok(configs_write) +} + +fn write_configs_to_file(config_path: &str, configs: &Vec) -> Result<()> { + info!("Write configuration to file \"{}\"", config_path); + let f = File::create(config_path)?; + let mut w = BufWriter::new(f); + for line in configs { + if line.is_empty() { + continue; + } + writeln!(w, "{}", line.as_str())?; + } + w.flush().with_context(|| format!("Failed to flush file {}", config_path))?; + w.get_mut().sync_all().with_context(|| "Failed to sync".to_string())?; + debug!("Write configuration to file \"{}\" success", config_path); + Ok(()) +} + +fn handle_delete_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String { + let key = config_kv[0]; + if config_kv.len() == 1 && new_config_info.value.is_empty() { + info!("Delete configuration key: \"{}\"", key); + return String::from(""); + } else if config_kv.len() == 1 && !new_config_info.value.is_empty() { + warn!("Failed to delete key \"{}\" with inconsistent values \"nil\" and \"{}\"", key, new_config_info.value); + return key.to_string(); + } + let old_value = config_kv[1]; + if old_value != new_config_info.value { + warn!( + "Failed to delete key \"{}\" with inconsistent values \"{}\" and \"{}\"", + key, old_value, new_config_info.value + ); + return config_kv.join("="); + } + info!("Delete configuration {}={}", key, old_value); + String::new() +} + +fn handle_update_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String { + let key = config_kv[0]; + if !new_config_info.operation.is_empty() { + warn!( + "Unknown operation \"{}\", updating key \"{}\" with value \"{}\" by default", + new_config_info.operation, key, new_config_info.value + ); + } + if config_kv.len() == values::ONLY_KEY && new_config_info.value.is_empty() { + return key.to_string(); + } + let new_value = new_config_info.value.trim(); + if config_kv.len() == values::ONLY_KEY && !new_config_info.value.is_empty() { + info!("Update configuration \"{}={}\"", key, new_value); + return format!("{}={}", key, new_value); + } + if new_config_info.value.is_empty() { + warn!("Failed to update key \"{}\" with \"null\" value", key); + return config_kv.join("="); + } + info!("Update configuration \"{}={}\"", key, new_value); + format!("{}={}", key, new_value) +} + +fn handle_add_key(expect_configs: &HashMap, is_only_key_valid: bool) -> Vec { + let mut configs_write = Vec::new(); + for (key, config_info) in expect_configs.iter() { + if config_info.operation == "delete" { + warn!("Failed to delete inexistent key: \"{}\"", key); + continue; + } + if key.is_empty() || key.contains('=') { + warn!("Failed to add \"null\" key or key containing \"=\", key: \"{}\"", key); + continue; + } + if !config_info.operation.is_empty() { + warn!( + "Unknown operation \"{}\", adding key \"{}\" with value \"{}\" by default", + config_info.operation, key, config_info.value + ); + } + let (k, v) = (key.trim(), config_info.value.trim()); + if v.is_empty() && is_only_key_valid { + info!("Add configuration \"{}\"", k); + configs_write.push(k.to_string()); + } else if v.is_empty() { + warn!("Failed to add key \"{}\" with \"null\" value", k); + } else { + info!("Add configuration \"{}={}\"", k, v); + configs_write.push(format!("{}={}", k, v)); + } + } + configs_write +} + +impl Configuration for GrubCmdline { + fn set_config(&self, config: &mut Sysconfig) -> Result<()> { + if self.is_cur_partition { + info!("Start setting grub.cmdline.current configuration"); + } else { + info!("Start setting grub.cmdline.next configuration"); + } + if !is_file_exist(&self.grub_path) { + bail!("Failed to find grub.cfg file"); + } + let config_partition = if cfg!(test) { + self.is_cur_partition + } else { + self.get_config_partition(RealCommandExecutor {}) + .with_context(|| "Failed to get config partition".to_string())? + }; + debug!("Config_partition: {} (false means partition A, true means partition B)", config_partition); + let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition) + .with_context(|| "Failed to set grub configs".to_string())?; + write_configs_to_file(&self.grub_path, &configs) + .with_context(|| "Failed to write configs to file".to_string())?; + Ok(()) + } +} + +impl GrubCmdline { + // get_config_partition returns false if the menuentry to be configured is A, true for menuentry B + fn get_config_partition(&self, executor: T) -> Result { + let (_, next_partition) = get_partition_info(&executor)?; + let mut flag = false; + if next_partition.menuentry == "B" { + flag = true + } + Ok(self.is_cur_partition != flag) + } +} + +fn get_and_set_grubcfg( + expect_configs: &mut HashMap, + grub_path: &str, + config_partition: bool, +) -> Result> { + let f = File::open(grub_path).with_context(|| format!("Failed to open grub.cfg \"{}\"", grub_path))?; + let re_find_cur_linux = r"^\s*linux.*root=.*"; + let re = Regex::new(re_find_cur_linux)?; + let mut configs_write = Vec::new(); + let mut match_config_partition = false; + for line in io::BufReader::new(f).lines() { + let mut line = line?; + if re.is_match(&line) { + if match_config_partition == config_partition { + line = modify_boot_cfg(expect_configs, &line)?; + } + match_config_partition = true; + } + configs_write.push(line); + } + Ok(configs_write) +} + +fn modify_boot_cfg(expect_configs: &mut HashMap, line: &String) -> Result { + trace!("Match partition that need to be configured, entering modify_boot_cfg, linux line: {}", line); + let mut new_configs = vec![" ".to_string()]; + let olg_configs: Vec<&str> = line.split(' ').collect(); + for old_config in olg_configs { + if old_config.is_empty() { + continue; + } + // At most 2 substrings can be returned to satisfy the case like root=UUID=xxxx + let config = old_config.splitn(2, '=').collect::>(); + if config.len() != values::ONLY_KEY && config.len() != values::KV_PAIR { + bail!("Failed to parse grub.cfg linux line {}", old_config); + } + let new_key_info = expect_configs.get(config[0]); + let new_config = match new_key_info { + Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config, new_key_info), + Some(new_key_info) => handle_update_key(&config, new_key_info), + None => config.join("="), + }; + if !new_config.is_empty() { + new_configs.push(new_config); + } + expect_configs.remove(config[0]); + } + let new_config = handle_add_key(expect_configs, true); + new_configs.extend(new_config); + Ok(new_configs.join(" ")) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use mockall::{mock, predicate::*}; + use tempfile::{NamedTempFile, TempDir}; + + use super::*; + use crate::sys_mgmt::{GRUB_CMDLINE_CURRENT, GRUB_CMDLINE_NEXT, KERNEL_SYSCTL, KERNEL_SYSCTL_PERSIST}; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_config_partition() { + init(); + let mut grub_cmdline = GrubCmdline { grub_path: String::from(""), is_cur_partition: true }; + let mut executor = MockCommandExec::new(); + + // the output shows that current root menuentry is A + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + + let result = grub_cmdline.get_config_partition(executor).unwrap(); + // it should return false because the current root menuentry is A and we want to configure current partition + assert_eq!(result, false); + + let mut executor = MockCommandExec::new(); + + // the output shows that current root menuentry is A + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + grub_cmdline.is_cur_partition = false; + let result = grub_cmdline.get_config_partition(executor).unwrap(); + // it should return true because the current root menuentry is A and we want to configure next partition + assert_eq!(result, true); + } + + #[test] + fn test_kernel_sysctl() { + init(); + let tmp_dir = TempDir::new().unwrap(); + assert_eq!(tmp_dir.path().exists(), true); + let kernel_sysctl = KernelSysctl::new(tmp_dir.path().to_str().unwrap()); + + let config_detail = HashMap::from([ + ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }), + ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }), + ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ]); + + let mut config = + Sysconfig { model: KERNEL_SYSCTL.to_string(), config_path: String::from(""), contents: config_detail }; + kernel_sysctl.set_config(&mut config).unwrap(); + + let result = fs::read_to_string(format!("{}{}", tmp_dir.path().to_str().unwrap(), "a")).unwrap(); + assert_eq!(result, "1\n"); + } + + #[test] + fn test_kernel_sysctl_persist() { + init(); + let comment = r"# This file is managed by KubeOS for unit testing."; + // create a tmp file with comment + let mut tmp_file = tempfile::NamedTempFile::new().unwrap(); + writeln!(tmp_file, "{}", comment).unwrap(); + writeln!(tmp_file, "a=0").unwrap(); + writeln!(tmp_file, "d=4").unwrap(); + writeln!(tmp_file, "e=5").unwrap(); + writeln!(tmp_file, "g=7").unwrap(); + let kernel_sysctl_persist = KernelSysctlPersist {}; + let config_detail = HashMap::from([ + ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }), + ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }), + ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ("f".to_string(), KeyInfo { value: "".to_string(), operation: "add".to_string() }), + ("g".to_string(), KeyInfo { value: "7".to_string(), operation: "delete".to_string() }), + ("".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }), + ("s=x".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }), + ]); + let mut config = Sysconfig { + model: KERNEL_SYSCTL_PERSIST.to_string(), + config_path: String::from(tmp_file.path().to_str().unwrap()), + contents: config_detail, + }; + kernel_sysctl_persist.set_config(&mut config).unwrap(); + let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); + let expected_res = format!("{}\n{}\n{}\n{}\n{}\n", comment, "a=1", "d=4", "e=5", "c=3"); + assert_eq!(result, expected_res); + let mut config = Sysconfig { + model: KERNEL_SYSCTL_PERSIST.to_string(), + config_path: String::from("/tmp/kubeos-test-kernel-sysctl-persist.txt"), + contents: HashMap::new(), + }; + kernel_sysctl_persist.set_config(&mut config).unwrap(); + assert!(is_file_exist(&config.config_path)); + delete_file_or_dir(&config.config_path).unwrap(); + } + + #[test] + fn write_configs_to_file_tests() { + init(); + let tmp_file = NamedTempFile::new().unwrap(); + let configs = vec!["a=1".to_string(), "b=2".to_string()]; + write_configs_to_file(tmp_file.path().to_str().unwrap(), &configs).unwrap(); + assert_eq!(fs::read(tmp_file.path()).unwrap(), b"a=1\nb=2\n"); + } + + #[test] + fn test_grub_cmdline() { + init(); + let mut tmp_file = NamedTempFile::new().unwrap(); + let mut grub_cmdline = + GrubCmdline { grub_path: tmp_file.path().to_str().unwrap().to_string(), is_cur_partition: true }; + let grub_cfg = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +} + +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=UUID=2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + initrd /boot/initramfs.img +}"; + writeln!(tmp_file, "{}", grub_cfg).unwrap(); + let config_second_part = HashMap::from([ + ("debug".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("quiet".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }), + ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }), + ("nomodeset".to_string(), KeyInfo { value: "".to_string(), operation: "update".to_string() }), + ("oops".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }), + ("".to_string(), KeyInfo { value: "test".to_string(), operation: "".to_string() }), + ("selinux".to_string(), KeyInfo { value: "1".to_string(), operation: "delete".to_string() }), + ("acpi".to_string(), KeyInfo { value: "off".to_string(), operation: "delete".to_string() }), + ("ro".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }), + ]); + let mut config = Sysconfig { + model: GRUB_CMDLINE_CURRENT.to_string(), + config_path: String::new(), + contents: config_second_part, + }; + grub_cmdline.set_config(&mut config).unwrap(); + grub_cmdline.is_cur_partition = false; + let config_first_part = HashMap::from([ + ("pci".to_string(), KeyInfo { value: "nomis".to_string(), operation: "".to_string() }), + ("quiet".to_string(), KeyInfo { value: "11".to_string(), operation: "delete".to_string() }), + ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "update".to_string() }), + ]); + config.contents = config_first_part; + config.model = GRUB_CMDLINE_NEXT.to_string(); + grub_cmdline.set_config(&mut config).unwrap(); + let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap(); + let expected_res = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt2' + linux /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 pci=nomis + initrd /boot/initramfs.img +} +menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' { + load_video + set gfxpayload=keep + insmod gzio + insmod part_gpt + insmod ext2 + set root='hd0,gpt3' + linux /boot/vmlinuz root=UUID=2 ro=1 rootfstype=ext4 nomodeset oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 debug + initrd /boot/initramfs.img +} +"; + assert_eq!(result, expected_res); + + // test grub.cfg not exist + grub_cmdline.grub_path = "/tmp/grub-KubeOS-test.cfg".to_string(); + let res = grub_cmdline.set_config(&mut config); + assert!(res.is_err()); + } + + #[test] + fn test_create_config_file() { + init(); + let tmp_file = "/tmp/kubeos-test-create-config-file.txt"; + create_config_file(&tmp_file).unwrap(); + assert!(is_file_exist(&tmp_file)); + fs::remove_file(tmp_file).unwrap(); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs new file mode 100644 index 00000000..80caf291 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -0,0 +1,301 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs, os::unix::fs::PermissionsExt, path::Path}; + +use anyhow::{anyhow, Context, Result}; +use log::{debug, info}; + +use crate::{ + api::{ImageHandler, UpgradeRequest}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES}, + utils::*, +}; + +pub struct CtrImageHandler { + pub paths: PreparePath, + pub executor: T, +} + +const DEFAULT_NAMESPACE: &str = "k8s.io"; + +impl ImageHandler for CtrImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?; + self.get_image(req)?; + self.get_rootfs_archive(req, IMAGE_PERMISSION)?; + + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); + img_manager.create_os_image(IMAGE_PERMISSION) + } +} + +impl Default for CtrImageHandler { + fn default() -> Self { + Self { paths: PreparePath::default(), executor: RealCommandExecutor {} } + } +} + +impl CtrImageHandler { + #[cfg(test)] + pub fn new(paths: PreparePath, executor: T) -> Self { + Self { paths, executor } + } + + fn get_image(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + is_valid_image_name(image_name)?; + let cli: String = + if is_command_available("crictl", &self.executor) { "crictl".to_string() } else { "ctr".to_string() }; + remove_image_if_exist(&cli, image_name, &self.executor)?; + info!("Start pulling image {}", image_name); + pull_image(&cli, image_name, &self.executor)?; + info!("Start checking image digest"); + check_oci_image_digest(&cli, image_name, &req.check_sum, &self.executor)?; + Ok(()) + } + + fn get_rootfs_archive(&self, req: &UpgradeRequest, permission: u32) -> Result<()> { + let image_name = &req.container_image; + let mount_path = &self + .paths + .mount_path + .to_str() + .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?; + info!("Start getting rootfs {}", image_name); + self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?; + self.executor + .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?; + // copy os.tar from mount_path to its partent dir + self.copy_file(self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission)?; + self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?; + Ok(()) + } + + fn check_and_unmount(&self, mount_path: &str) -> Result<()> { + let ctr_snapshot_cmd = + format!("ctr -n={} snapshots ls | grep {} | awk '{{print $1}}'", DEFAULT_NAMESPACE, mount_path); + let exist_snapshot = self.executor.run_command_with_output("bash", &["-c", &ctr_snapshot_cmd])?; + if !exist_snapshot.is_empty() { + self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "unmount", mount_path])?; + self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "snapshots", "remove", mount_path])?; + } + Ok(()) + } + + fn copy_file, Q: AsRef>(&self, src: P, dst: Q, permission: u32) -> Result<()> { + let copied_bytes = fs::copy(src.as_ref(), dst.as_ref())?; + debug!("Copy {} to {}, total bytes: {}", src.as_ref().display(), dst.as_ref().display(), copied_bytes); + fs::set_permissions(dst, fs::Permissions::from_mode(permission))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{io::Write, path::PathBuf}; + + use mockall::mock; + use tempfile::NamedTempFile; + + use super::*; + use crate::api::CertsInfo; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + // mock is_command_available + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "/bin/sh" && args.contains(&"command -v crictl")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock remove_image_if_exist + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.contains(&"inspecti")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.contains(&"rmi")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock pull_image + mock_executor + .expect_run_command() + .withf(|cmd, args| { + cmd == "crictl" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest") + }) + .times(1) + .returning(|_, _| Ok(())); + // mock get_oci_image_digest + let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]"; + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "crictl" && args.contains(&"inspecti") && args.contains(&"{{.status.repoDigests}}") + }) + .times(1) + .returning(|_, _| Ok(command_output2.to_string())); + let ctr = CtrImageHandler::new(PreparePath::default(), mock_executor); + let result = ctr.get_image(&req); + assert!(result.is_ok()); + } + + #[test] + fn test_get_rootfs_archive() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "containerd".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + // mock check_and_unmount + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure + .times(1) + .returning(|_, _| Ok("".to_string())); + + // mock ctr mount rw + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[4] == "--rw") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + // create temp file for copy + let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file."); + writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file."); + + // Get the path of the temporary file and the path where it should be copied. + let src_dir = tmp_file.path().parent().unwrap(); + let src_file_name = tmp_file.path().file_name().unwrap().to_str().unwrap().to_string(); + let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file."); + let dst_path = dst_file.path().to_path_buf(); + + let paths = PreparePath { + persist_path: "/tmp".into(), + update_path: PathBuf::new(), + image_path: PathBuf::new(), + mount_path: src_dir.to_path_buf(), + rootfs_file: src_file_name.clone(), + tar_path: dst_path.clone(), + }; + + // mock check_and_unmount + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure + .times(1) + .returning(|_, _| Ok("".to_string())); + + let ctr = CtrImageHandler::new(paths, mock_executor); + let result = ctr.get_rootfs_archive(&req, IMAGE_PERMISSION); + assert!(result.is_ok()); + } + + #[test] + fn test_copy_file() { + // Setup: Create a temporary file and write some data to it. + let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file."); + writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file."); + + // Get the path of the temporary file and the path where it should be copied. + let src_path = tmp_file.path().to_str().unwrap().to_string(); + let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file."); + let dst_path = dst_file.path().to_str().unwrap().to_string(); + + let ctr = CtrImageHandler::default(); + let result = ctr.copy_file(&src_path, &dst_path, IMAGE_PERMISSION); + + assert!(result.is_ok()); + + let expected_content = "Hello, world!\n"; + let actual_content = fs::read_to_string(&dst_path).expect("Failed to read destination file."); + assert_eq!(expected_content, actual_content); + + // Assert the file permission + let metadata = fs::metadata(&dst_path).expect("Failed to read destination file."); + let expected_permission = 0o100600; + assert_eq!(metadata.permissions().mode(), expected_permission); + } + + #[test] + fn test_check_and_unmount() { + let mut mock_executor = MockCommandExec::new(); + + // When `run_command_with_output` is called with "bash" and the specific args, it will return Ok("snapshot_exists"). + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") + .times(1) + .returning(|_, _| Ok("snapshot_exists".to_string())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"images")) + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"snapshots")) + .times(1) + .returning(|_, _| Ok(())); + + let result = CtrImageHandler::new(PreparePath::default(), mock_executor).check_and_unmount("test_mount_path"); + + assert!(result.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs new file mode 100644 index 00000000..6d836dc4 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs @@ -0,0 +1,406 @@ +use std::{ + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; + +use anyhow::{bail, Context, Result}; +use log::{debug, info, trace}; +use reqwest::{blocking::Client, Certificate}; +use sha2::{Digest, Sha256}; + +use crate::{ + api::{CertsInfo, ImageHandler, UpgradeRequest}, + sys_mgmt::{CERTS_PATH, IMAGE_PERMISSION, PERSIST_DIR}, + utils::*, +}; + +const BUFFER: u64 = 1024 * 1024 * 10; + +pub struct DiskImageHandler { + pub paths: PreparePath, + pub executor: T, + pub certs_path: String, +} + +impl ImageHandler for DiskImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + self.download(req)?; + self.checksum_match(self.paths.image_path.to_str().unwrap_or_default(), &req.check_sum)?; + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); + Ok(img_manager) + } +} + +impl Default for DiskImageHandler { + fn default() -> Self { + Self { paths: PreparePath::default(), executor: RealCommandExecutor {}, certs_path: CERTS_PATH.to_string() } + } +} + +impl DiskImageHandler { + #[cfg(test)] + pub fn new(paths: PreparePath, executor: T, certs_path: String) -> Self { + Self { paths, executor, certs_path } + } + + fn download(&self, req: &UpgradeRequest) -> Result<()> { + let mut resp = self.send_download_request(req)?; + if resp.status() != reqwest::StatusCode::OK { + bail!("Failed to download image from {}, status: {}", req.image_url, resp.status()); + } + debug!("Received response body size: {:?}", resp.content_length().unwrap_or_default()); + let need_bytes = resp.content_length().unwrap_or_default() + BUFFER; + + check_disk_size( + i64::try_from(need_bytes).with_context(|| "Failed to transform content length from u64 to i64")?, + self.paths.image_path.parent().unwrap_or_else(|| Path::new(PERSIST_DIR)), + )?; + + let mut out = fs::File::create(&self.paths.image_path)?; + trace!("Start to save upgrade image to path {}", &self.paths.image_path.display()); + out.set_permissions(fs::Permissions::from_mode(IMAGE_PERMISSION))?; + let bytes = resp.copy_to(&mut out)?; + info!( + "Download image successfully, upgrade image path: {}, write bytes: {}", + &self.paths.image_path.display(), + bytes + ); + Ok(()) + } + + fn checksum_match(&self, file_path: &str, check_sum: &str) -> Result<()> { + info!("Start checking image checksum"); + let check_sum = check_sum.to_ascii_lowercase(); + let file = fs::read(file_path)?; + let mut hasher = Sha256::new(); + hasher.update(file); + let hash = hasher.finalize(); + // sha256sum -b /persist/update.img + let cal_sum = format!("{:X}", hash).to_ascii_lowercase(); + if cal_sum != check_sum { + delete_file_or_dir(file_path)?; + bail!("Checksum {} mismatch to {}", cal_sum, check_sum); + } + debug!("Checksum match"); + Ok(()) + } + + fn send_download_request(&self, req: &UpgradeRequest) -> Result { + let client: Client; + + if !req.image_url.starts_with("https://") { + // http request + if !req.flag_safe { + bail!("The upgrade image url is not safe"); + } + info!("Discover http request to: {}", &req.image_url); + client = Client::new(); + } else if req.mtls { + // https mtls request + client = self.load_ca_client_certs(&req.certs).with_context(|| "Failed to load client certificates")?; + info!("Discover https mtls request to: {}", &req.image_url); + } else { + // https request + client = self.load_ca_certs(&req.certs.ca_cert).with_context(|| "Failed to load CA certificates")?; + info!("Discover https request to: {}", &req.image_url); + } + + client.get(&req.image_url).send().with_context(|| format!("Failed to fetch from URL: {}", &req.image_url)) + } + + fn load_ca_certs(&self, ca_cert: &str) -> Result { + trace!("Start to load CA certificates"); + self.cert_exist(ca_cert)?; + let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(ca_cert))?)?; + let client = Client::builder().add_root_certificate(ca).build()?; + Ok(client) + } + + fn load_ca_client_certs(&self, certs: &CertsInfo) -> Result { + trace!("Start to load CA and client certificates"); + self.cert_exist(&certs.ca_cert)?; + let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(&certs.ca_cert))?)?; + + self.cert_exist(&certs.client_cert)?; + self.cert_exist(&certs.client_key)?; + let client_cert = std::fs::read(self.get_certs_path(&certs.client_cert))?; + let client_key = std::fs::read(self.get_certs_path(&certs.client_key))?; + let mut client_identity = Vec::new(); + client_identity.extend_from_slice(&client_cert); + client_identity.extend_from_slice(&client_key); + let client_id = reqwest::Identity::from_pem(&client_identity)?; + + let client = Client::builder().use_rustls_tls().add_root_certificate(ca).identity(client_id).build()?; + Ok(client) + } + + fn cert_exist(&self, cert_file: &str) -> Result<()> { + if cert_file.is_empty() { + bail!("Please provide the certificate"); + } + if !self.get_certs_path(cert_file).exists() { + bail!("Certificate does not exist: {}", cert_file); + } + Ok(()) + } + + fn get_certs_path(&self, cert: &str) -> PathBuf { + let cert_path = format!("{}{}", self.certs_path, cert); + PathBuf::from(cert_path) + } +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use mockall::mock; + use mockito; + use tempfile::NamedTempFile; + + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + #[test] + fn test_get_certs_path() { + init(); + let handler = DiskImageHandler::::default(); + let certs_path = handler.get_certs_path("ca.pem"); + assert_eq!(certs_path.to_str().unwrap(), "/etc/KubeOS/certs/ca.pem"); + } + + #[test] + fn test_cert_exist() { + init(); + // generate tmp file + let tmp_file = NamedTempFile::new().unwrap(); + let handler = + DiskImageHandler::::new(PreparePath::default(), RealCommandExecutor {}, String::new()); + let res = handler.cert_exist(tmp_file.path().to_str().unwrap()); + assert!(res.is_ok()); + + assert!(handler.cert_exist("aaa.pem").is_err()); + assert!(handler.cert_exist("").is_err()) + } + + #[test] + fn test_send_download_request() { + init(); + // This is a tmp cert only for KubeOS unit testing. + let tmp_cert = "-----BEGIN CERTIFICATE-----\n\ + MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\ + MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\ + MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\ + bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\ + Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\ + HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\ + c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\ + vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\ + -----END CERTIFICATE-----\n"; + + // This is a tmp private key only for KubeOS unit testing. + let tmp_key = "-----BEGIN PRIVATE KEY-----\n\ + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\ + Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\ + nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\ + -----END PRIVATE KEY-----\n"; + + // Create a temporary file to hold the certificate + let mut cert_file = NamedTempFile::new().unwrap(); + cert_file.write_all(tmp_cert.as_bytes()).unwrap(); + println!("cert_file: {:?}", cert_file.path().to_str().unwrap()); + + // Create a temporary file to hold the private key + let mut key_file = NamedTempFile::new().unwrap(); + key_file.write_all(tmp_key.as_bytes()).unwrap(); + // http + let handler = DiskImageHandler::::default(); + let mut req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "http://localhost:8080/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + req.flag_safe = false; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + + // https + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "/tmp".to_string(); + let tmp_cert_filename = cert_file.path().file_name().unwrap().to_str().unwrap(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "https://localhost:8081/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { + ca_cert: tmp_cert_filename.to_string(), + client_cert: "".to_string(), + client_key: "".to_string(), + }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + + // mtls + let tmp_key = NamedTempFile::new().unwrap(); + let tmp_key_filename = tmp_key.path().file_name().unwrap().to_str().unwrap(); + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "/tmp".to_string(); + let req = UpgradeRequest { + version: "v2".into(), + check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "https://localhost:8082/aaa.txt".to_string(), + flag_safe: true, + mtls: true, + certs: CertsInfo { + ca_cert: tmp_cert_filename.to_string(), + client_cert: tmp_cert_filename.to_string(), + client_key: tmp_key_filename.to_string(), + }, + }; + let res = handler.send_download_request(&req); + assert!(res.is_err()); + } + + #[test] + fn test_checksum_match() { + init(); + let mut tmp_file = NamedTempFile::new().unwrap(); + tmp_file.write(b"This is a test txt file for KubeOS test.\n").unwrap(); + let mut handler = DiskImageHandler::::default(); + handler.paths.image_path = tmp_file.path().to_path_buf(); + let mut req = UpgradeRequest { + version: "v2".into(), + check_sum: "98Ea7aff44631D183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: "http://localhost:8080/aaa.txt".to_string(), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + assert_eq!(handler.paths.image_path.exists(), true); + handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum).unwrap(); + + req.check_sum = "1234567Abc".into(); + let res = handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum); + assert!(res.is_err()); + } + + #[test] + fn test_load_certs() { + init(); + // This is a tmp cert only for KubeOS unit testing. + let tmp_cert = "-----BEGIN CERTIFICATE-----\n\ + MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\ + MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\ + MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\ + bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\ + Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\ + HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\ + c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\ + vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\ + -----END CERTIFICATE-----\n"; + + // This is a tmp private key only for KubeOS unit testing. + let tmp_key = "-----BEGIN PRIVATE KEY-----\n\ + MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\ + Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\ + nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\ + -----END PRIVATE KEY-----\n"; + + // Create a temporary file to hold the certificate + let mut cert_file = NamedTempFile::new().unwrap(); + cert_file.write_all(tmp_cert.as_bytes()).unwrap(); + + // Create a temporary file to hold the private key + let mut key_file = NamedTempFile::new().unwrap(); + key_file.write_all(tmp_key.as_bytes()).unwrap(); + + let mut handler = DiskImageHandler::::default(); + handler.certs_path = "".to_string(); + let certs = CertsInfo { + ca_cert: cert_file.path().to_str().unwrap().to_string(), + client_cert: cert_file.path().to_str().unwrap().to_string(), + client_key: key_file.path().to_str().unwrap().to_string(), + }; + + let res = handler.load_ca_client_certs(&certs); + assert!(res.is_ok()); + + let res = handler.load_ca_certs(&certs.ca_cert); + assert!(res.is_ok()); + } + + #[test] + fn test_download_image() { + init(); + let tmp_file = NamedTempFile::new().unwrap(); + + let mock_executor = MockCommandExec::new(); + let mut handler = DiskImageHandler::new(PreparePath::default(), mock_executor, String::new()); + handler.executor.expect_clone().times(1).returning(|| MockCommandExec::new()); + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + handler.executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + handler.paths.image_path = tmp_file.path().to_path_buf(); + assert_eq!(true, handler.paths.image_path.exists()); + + let url = mockito::server_url(); + let upgrade_request = UpgradeRequest { + version: "v2".into(), + check_sum: "98ea7aff44631d183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(), + image_type: "disk".into(), + container_image: "".into(), + image_url: format!("{}/test.txt", url), + flag_safe: true, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + let _m = mockito::mock("GET", "/test.txt") + .with_status(200) + .with_body("This is a test txt file for KubeOS test.\n") + .create(); + handler.download_image(&upgrade_request).unwrap(); + assert_eq!(true, handler.paths.image_path.exists()); + assert_eq!( + fs::read(handler.paths.image_path.to_str().unwrap()).unwrap(), + "This is a test txt file for KubeOS test.\n".as_bytes() + ); + + let _m = mockito::mock("GET", "/test.txt").with_status(404).with_body("Not found").create(); + let res = handler.download_image(&upgrade_request); + assert!(res.is_err()) + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs new file mode 100644 index 00000000..4d97552c --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs @@ -0,0 +1,236 @@ +use anyhow::{Context, Result}; +use log::{debug, info, trace}; + +use crate::{ + api::{ImageHandler, UpgradeRequest}, + sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES}, + utils::*, +}; + +pub struct DockerImageHandler { + pub paths: PreparePath, + pub container_name: String, + pub executor: T, +} + +impl ImageHandler for DockerImageHandler { + fn download_image(&self, req: &UpgradeRequest) -> Result> { + perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?; + self.get_image(req)?; + self.get_rootfs_archive(req)?; + + let (_, next_partition_info) = get_partition_info(&self.executor)?; + let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone()); + img_manager.create_os_image(IMAGE_PERMISSION) + } +} + +impl Default for DockerImageHandler { + fn default() -> Self { + Self { paths: PreparePath::default(), container_name: "kubeos-temp".into(), executor: RealCommandExecutor {} } + } +} + +impl DockerImageHandler { + #[cfg(test)] + pub fn new(paths: PreparePath, container_name: String, executor: T) -> Self { + Self { paths, container_name, executor } + } + + fn get_image(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + is_valid_image_name(image_name)?; + let cli = "docker"; + remove_image_if_exist(cli, image_name, &self.executor)?; + info!("Start pulling image {}", image_name); + pull_image(cli, image_name, &self.executor)?; + info!("Start checking image digest"); + check_oci_image_digest(cli, image_name, &req.check_sum, &self.executor)?; + Ok(()) + } + + fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> { + let image_name = &req.container_image; + info!("Start getting rootfs {}", image_name); + self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?; + debug!("Create container {}", self.container_name); + let container_id = + self.executor.run_command_with_output("docker", &["create", "--name", &self.container_name, image_name])?; + debug!("Copy rootfs from container {} to {}", container_id, self.paths.update_path.display()); + self.executor.run_command( + "docker", + &[ + "cp", + format!("{}:/{}", container_id, self.paths.rootfs_file).as_str(), + self.paths.update_path.to_str().unwrap(), + ], + )?; + self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?; + Ok(()) + } + + fn check_and_rm_container(&self) -> Result<()> { + trace!("Check and remove container {}", self.container_name); + let docker_ps_cmd = format!("docker ps -a -f=name={} | awk 'NR==2' | awk '{{print $1}}'", self.container_name); + let exist_id = self.executor.run_command_with_output("bash", &["-c", &docker_ps_cmd])?; + if !exist_id.is_empty() { + info!("Remove container {} {} for cleaning environment", self.container_name, exist_id); + self.executor.run_command("docker", &["rm", exist_id.as_str()])?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use mockall::mock; + + use super::*; + use crate::api::CertsInfo; + + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_check_and_rm_container() { + init(); + let mut mock_executor = MockCommandExec::new(); + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" + && args.len() == 2 + && args.contains(&"docker ps -a -f=name=test | awk 'NR==2' | awk '{print $1}'") + }) + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111")) + .times(1) + .returning(|_, _| Ok(())); + + let result = + DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor).check_and_rm_container(); + assert!(result.is_ok()); + + assert_eq!(DockerImageHandler::default().container_name, "kubeos-temp"); + } + + #[test] + fn test_get_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "docker".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + + // mock remove_image_if_exist + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rmi")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + // mock pull_image + mock_executor + .expect_run_command() + .withf(|cmd, args| { + cmd == "docker" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest") + }) + .times(1) + .returning(|_, _| Ok(())); + // mock get_oci_image_digest + let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]"; + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect") && args.contains(&"{{.RepoDigests}}")) + .times(1) + .returning(|_, _| Ok(command_output2.to_string())); + + let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor); + let result = docker.get_image(&req); + assert!(result.is_ok()); + } + + #[test] + fn test_get_rootfs_archive() { + init(); + let mut mock_executor = MockCommandExec::new(); + let image_name = "docker.io/library/busybox:latest"; + let req = UpgradeRequest { + version: "KubeOS v2".to_string(), + image_type: "docker".to_string(), + container_image: image_name.to_string(), + check_sum: "22222".to_string(), + image_url: "".to_string(), + flag_safe: false, + mtls: false, + certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() }, + }; + // mock check_and_rm_container + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'") + }) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::new())); + // mock get_rootfs_archive + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "docker" && args.contains(&"create")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"cp") && args.contains(&"1111:/os.tar")) + .times(1) + .returning(|_, _| Ok(())); + // mock check_and_rm_container + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| { + cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'") + }) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("1111"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111")) + .times(1) + .returning(|_, _| Ok(())); + + let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor); + let result = docker.get_rootfs_archive(&req); + assert!(result.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/sys_mgmt/mod.rs b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs new file mode 100644 index 00000000..0e06f297 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs @@ -0,0 +1,23 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod config; +mod containerd_image; +mod disk_image; +mod docker_image; +mod values; + +pub use config::*; +pub use containerd_image::*; +pub use disk_image::*; +pub use docker_image::*; +pub use values::*; diff --git a/KubeOS-Rust/manager/src/sys_mgmt/values.rs b/KubeOS-Rust/manager/src/sys_mgmt/values.rs new file mode 100644 index 00000000..b107efc3 --- /dev/null +++ b/KubeOS-Rust/manager/src/sys_mgmt/values.rs @@ -0,0 +1,36 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +pub const KERNEL_SYSCTL: &str = "kernel.sysctl"; +pub const KERNEL_SYSCTL_PERSIST: &str = "kernel.sysctl.persist"; +pub const GRUB_CMDLINE_CURRENT: &str = "grub.cmdline.current"; +pub const GRUB_CMDLINE_NEXT: &str = "grub.cmdline.next"; + +pub const DEFAULT_PROC_PATH: &str = "/proc/sys/"; +pub const DEFAULT_KERNEL_CONFIG_PATH: &str = "/etc/sysctl.conf"; +pub const DEFAULT_GRUB_CFG_PATH: &str = "/boot/efi/EFI/openEuler/grub.cfg"; +pub const DEFAULT_GRUBENV_PATH: &str = "/boot/efi/EFI/openEuler/grubenv"; + +pub const PERSIST_DIR: &str = "/persist"; +pub const ROOTFS_ARCHIVE: &str = "os.tar"; +pub const UPDATE_DIR: &str = "KubeOS-Update"; +pub const MOUNT_DIR: &str = "kubeos-update"; +pub const OS_IMAGE_NAME: &str = "update.img"; +pub const CERTS_PATH: &str = "/etc/KubeOS/certs/"; + +pub const DEFAULT_KERNEL_CONFIG_PERM: u32 = 0o644; +pub const DEFAULT_GRUB_CFG_PERM: u32 = 0o751; +pub const IMAGE_PERMISSION: u32 = 0o600; + +pub const ONLY_KEY: usize = 1; +pub const KV_PAIR: usize = 2; +pub const NEED_BYTES: i64 = 3 * 1024 * 1024 * 1024; diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs new file mode 100644 index 00000000..9baf99e3 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -0,0 +1,310 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs, + os::{linux::fs::MetadataExt, unix::fs::DirBuilderExt}, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, bail, Context, Result}; +use log::{debug, info, trace}; +use nix::{mount, mount::MntFlags}; + +use super::executor::CommandExecutor; +use crate::sys_mgmt::{MOUNT_DIR, OS_IMAGE_NAME, PERSIST_DIR, ROOTFS_ARCHIVE, UPDATE_DIR}; + +/// * persist_path: /persist +/// +/// * update_path: /persist/KubeOS-Update +/// +/// * mount_path: /persist/KubeOS-Update/kubeos-update +/// +/// * tar_path: /persist/KubeOS-Update/os.tar +/// +/// * image_path: /persist/update.img +/// +/// * rootfs_file: os.tar +#[derive(Clone)] +pub struct PreparePath { + pub persist_path: PathBuf, + pub update_path: PathBuf, + pub mount_path: PathBuf, + pub tar_path: PathBuf, + pub image_path: PathBuf, + pub rootfs_file: String, +} + +impl Default for PreparePath { + fn default() -> Self { + let persist_dir = Path::new(PERSIST_DIR); + let update_pathbuf = persist_dir.join(UPDATE_DIR); + Self { + persist_path: persist_dir.to_path_buf(), + update_path: update_pathbuf.clone(), + mount_path: update_pathbuf.join(MOUNT_DIR), + tar_path: update_pathbuf.join(ROOTFS_ARCHIVE), + image_path: persist_dir.join(OS_IMAGE_NAME), + rootfs_file: ROOTFS_ARCHIVE.to_string(), + } + } +} + +pub fn is_file_exist>(path: P) -> bool { + path.as_ref().exists() +} + +pub fn perpare_env(prepare_path: &PreparePath, need_bytes: i64, permission: u32) -> Result<()> { + info!("Prepare environment to upgrade"); + check_disk_size(need_bytes, &prepare_path.persist_path)?; + clean_env(&prepare_path.update_path, &prepare_path.mount_path, &prepare_path.image_path)?; + fs::DirBuilder::new().recursive(true).mode(permission).create(&prepare_path.mount_path)?; + Ok(()) +} + +pub fn check_disk_size>(need_bytes: i64, path: P) -> Result<()> { + trace!("Check if there is enough disk space to upgrade"); + let fs_stat = nix::sys::statfs::statfs(path.as_ref())?; + let available_blocks = i64::try_from(fs_stat.blocks_available())?; + let available_space = available_blocks * fs_stat.block_size(); + if available_space < need_bytes { + bail!("Space is not enough for downloading"); + } + Ok(()) +} + +/// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img +pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> +where + P: AsRef + std::fmt::Debug, +{ + if is_mounted(&mount_path)? { + debug!("Umount \"{}\"", mount_path.as_ref().display()); + if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) { + bail!("Failed to umount {} in clean_env: {}", mount_path.as_ref().display(), errno); + } + } + // losetup -D? + delete_file_or_dir(&update_path).with_context(|| format!("Failed to delete {:?}", update_path))?; + delete_file_or_dir(&image_path).with_context(|| format!("Failed to delete {:?}", image_path))?; + Ok(()) +} + +pub fn delete_file_or_dir>(path: P) -> Result<()> { + if is_file_exist(&path) { + if fs::metadata(&path)?.is_file() { + info!("Delete file \"{}\"", path.as_ref().display()); + fs::remove_file(&path)?; + } else { + info!("Delete directory \"{}\"", path.as_ref().display()); + fs::remove_dir_all(&path)?; + } + } + Ok(()) +} + +pub fn is_command_available(command: &str, command_executor: &T) -> bool { + match command_executor.run_command("/bin/sh", &["-c", format!("command -v {}", command).as_str()]) { + Ok(_) => { + debug!("command {} is available", command); + true + }, + Err(_) => { + debug!("command {} is not available", command); + false + }, + } +} + +pub fn is_mounted>(mount_path: P) -> Result { + if !is_file_exist(&mount_path) { + return Ok(false); + } + // Get device ID of mountPath + let mount_meta = fs::symlink_metadata(&mount_path)?; + let dev = mount_meta.st_dev(); + + // Get device ID of mountPath's parent directory + let parent = mount_path + .as_ref() + .parent() + .ok_or_else(|| anyhow!("Failed to get parent directory of {}", mount_path.as_ref().display()))?; + let parent_meta = fs::symlink_metadata(parent)?; + let dev_parent = parent_meta.st_dev(); + Ok(dev != dev_parent) +} + +pub fn switch_boot_menuentry( + command_executor: &T, + grub_env_path: &str, + next_menuentry: &str, +) -> Result<()> { + if get_boot_mode() == "uefi" { + command_executor.run_command( + "grub2-editenv", + &[grub_env_path, "set", format!("saved_entry={}", next_menuentry).as_str()], + )?; + } else { + command_executor.run_command("grub2-set-default", &[next_menuentry])?; + } + Ok(()) +} + +pub fn get_boot_mode() -> String { + if is_file_exist("/sys/firmware/efi") { "uefi".into() } else { "bios".into() } +} + +#[cfg(test)] +mod tests { + use mockall::{mock, predicate::*}; + use tempfile::{NamedTempFile, TempDir}; + + use super::*; + use crate::utils::RealCommandExecutor; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_file_exist() { + init(); + let path = "/tmp/test_is_file_exist"; + assert_eq!(is_file_exist(path), false); + + let file = NamedTempFile::new().unwrap(); + assert_eq!(is_file_exist(file.path().to_str().unwrap()), true); + + let tmp_dir = TempDir::new().unwrap(); + assert_eq!(is_file_exist(tmp_dir.path().to_str().unwrap()), true); + } + + #[test] + fn test_prepare_env() { + init(); + let paths = PreparePath { + persist_path: PathBuf::from("/tmp"), + update_path: PathBuf::from("/tmp/test_prepare_env"), + mount_path: PathBuf::from("/tmp/test_prepare_env/kubeos-update"), + tar_path: PathBuf::from("/tmp/test_prepare_env/os.tar"), + image_path: PathBuf::from("/tmp/test_prepare_env/update.img"), + rootfs_file: "os.tar".to_string(), + }; + perpare_env(&paths, 1 * 1024 * 1024 * 1024, 0o700).unwrap(); + } + + #[test] + fn test_check_disk_size() { + init(); + let path = "/home"; + let gb: i64 = 1 * 1024 * 1024 * 1024; + let need_gb = 1 * gb; + let result = check_disk_size(need_gb, path); + assert!(result.is_ok()); + let need_gb = 10000 * gb; + let result = check_disk_size(need_gb, path); + assert!(result.is_err()); + } + + #[test] + fn test_clean_env() { + init(); + let update_path = "/tmp/test_clean_env"; + let mount_path = "/tmp/test_clean_env/kubeos-update"; + let image_path = "/tmp/test_clean_env/update.img"; + clean_env(&update_path.to_string(), &mount_path.to_string(), &image_path.to_string()).unwrap(); + } + + #[test] + fn test_delete_file_or_dir() { + init(); + let path = "/tmp/test_delete_file"; + fs::File::create(path).unwrap(); + assert_eq!(Path::new(path).exists(), true); + delete_file_or_dir(&path.to_string()).unwrap(); + assert_eq!(Path::new(path).exists(), false); + + let path = "/tmp/test_dir"; + fs::create_dir(path).unwrap(); + assert_eq!(Path::new(path).exists(), true); + delete_file_or_dir(&path.to_string()).unwrap(); + assert_eq!(Path::new(path).exists(), false); + + let path = "/tmp/nonexist"; + delete_file_or_dir(path).unwrap(); + + let path = PathBuf::new(); + delete_file_or_dir(path).unwrap(); + } + + #[test] + fn test_switch_boot_menuentry() { + init(); + let grubenv_path = "/boot/efi/EFI/openEuler/grubenv"; + let next_menuentry = "B"; + let mut mock = MockCommandExec::new(); + if get_boot_mode() == "uefi" { + mock.expect_run_command() + .withf(move |name, args| { + name == "grub2-editenv" + && args[0] == grubenv_path + && args[2] == format!("saved_entry={}", next_menuentry).as_str() + }) + .times(1) // Expect it to be called once + .returning(move |_, _| Ok(())); + } else { + mock.expect_run_command() + .withf(move |name, args| name == "grub2-set-default" && args[0] == next_menuentry) + .times(1) // Expect it to be called once + .returning(move |_, _| Ok(())); + } + + switch_boot_menuentry(&mock, grubenv_path, next_menuentry).unwrap() + } + + #[test] + fn test_get_boot_mode() { + init(); + let boot_mode = get_boot_mode(); + let executor = RealCommandExecutor {}; + let res = executor.run_command("ls", &["/sys/firmware/efi"]); + if res.is_ok() { + assert!(boot_mode == "uefi"); + } else { + assert!(boot_mode == "bios"); + } + } + + #[test] + fn test_is_command_available() { + init(); + let executor = RealCommandExecutor {}; + assert_eq!(is_command_available("ls", &executor), true); + assert_eq!(is_command_available("aaaabb", &executor), false); + } +} diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs new file mode 100644 index 00000000..d84c6853 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/container_image.rs @@ -0,0 +1,341 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::{bail, Result}; +use log::{debug, info, trace}; +use regex::Regex; + +use super::executor::CommandExecutor; + +pub fn is_valid_image_name(image: &str) -> Result<()> { + let pattern = r"^((?:[\w.-]+)(?::\d+)?/)*(?:[\w.-]+)((?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$"; + let reg_ex = Regex::new(pattern)?; + if !reg_ex.is_match(image) { + bail!("Invalid image name: {}", image); + } + debug!("Image name {} is valid", image); + Ok(()) +} + +pub fn check_oci_image_digest( + container_runtime: &str, + image_name: &str, + check_sum: &str, + command_executor: &T, +) -> Result<()> { + let image_digests = get_oci_image_digest(container_runtime, image_name, command_executor)?; + if image_digests.to_lowercase() != check_sum.to_lowercase() { + bail!("Image digest mismatch, expect {}, got {}", check_sum, image_digests); + } + Ok(()) +} + +pub fn get_oci_image_digest( + container_runtime: &str, + image_name: &str, + executor: &T, +) -> Result { + let cmd_output: String; + match container_runtime { + "crictl" => { + cmd_output = executor.run_command_with_output( + "crictl", + &["inspecti", "--output", "go-template", "--template", "{{.status.repoDigests}}", image_name], + )?; + }, + "docker" => { + cmd_output = + executor.run_command_with_output("docker", &["inspect", "--format", "{{.RepoDigests}}", image_name])?; + }, + "ctr" => { + cmd_output = executor + .run_command_with_output("ctr", &["-n", "k8s.io", "images", "ls", &format!("name=={}", image_name)])?; + // Split by whitespaces, we get vec like [REF TYPE DIGEST SIZE PLATFORMS LABELS x x x x x x] + // get the 8th element, and split by ':' to get the digest + let fields: Vec<&str> = cmd_output.split_whitespace().collect(); + if let Some(digest) = fields.get(8).and_then(|field| field.split(':').nth(1)) { + trace!("get_oci_image_digest: {}", digest); + return Ok(digest.to_string()); + } else { + bail!("Failed to get digest from ctr command output: {}", cmd_output); + } + }, + _ => { + bail!("Container runtime {} cannot be recognized", container_runtime); + }, + } + + // Parse the cmd_output to extract the digest + let parts: Vec<&str> = cmd_output.split('@').collect(); + if let Some(last_part) = parts.last() { + if last_part.starts_with("sha256") { + let parsed_parts: Vec<&str> = last_part.trim_matches(|c| c == ']').split(':').collect(); + // After spliiing by ':', we should get vec like [sha256, digests] + if parsed_parts.len() == 2 { + debug!("get_oci_image_digest: {}", parsed_parts[1]); + return Ok(parsed_parts[1].to_string()); // 1 is the index of digests + } + } + } + + bail!("Failed to get digest from command output: {}", cmd_output) +} + +pub fn pull_image(runtime: &str, image_name: &str, executor: &T) -> Result<()> { + debug!("Pull image {}", image_name); + match runtime { + "crictl" => { + executor.run_command("crictl", &["pull", image_name])?; + }, + "ctr" => { + executor.run_command( + "ctr", + &[&"-n", "k8s.io", "images", "pull", "--hosts-dir", "/etc/containerd/certs.d", image_name], + )?; + }, + "docker" => { + executor.run_command("docker", &["pull", image_name])?; + }, + _ => { + bail!("Container runtime {} cannot be recognized", runtime); + }, + } + Ok(()) +} + +pub fn remove_image_if_exist(runtime: &str, image_name: &str, executor: &T) -> Result<()> { + match runtime { + "crictl" => { + if executor.run_command("crictl", &["inspecti", image_name]).is_ok() { + executor.run_command("crictl", &["rmi", image_name])?; + info!("Remove existing upgrade image: {}", image_name); + } + }, + "ctr" => { + let output = executor.run_command_with_output( + "ctr", + &[&"-n", "k8s.io", "images", "check", &format!("name=={}", image_name)], + )?; + if !output.is_empty() { + executor.run_command("ctr", &[&"-n", "k8s.io", "images", "rm", image_name, "--sync"])?; + info!("Remove existing upgrade image: {}", image_name); + } + }, + "docker" => { + if executor.run_command("docker", &["inspect", image_name]).is_ok() { + executor.run_command("docker", &["rmi", image_name])?; + info!("Remove existing upgrade image: {}", image_name); + } + }, + _ => { + bail!("Container runtime {} cannot be recognized", runtime); + }, + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use mockall::{mock, predicate::*}; + + use super::*; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_valid_image_name() { + init(); + let correct_images = vec![ + "alpine", + "alpine:latest", + "localhost/latest", + "library/alpine", + "localhost:1234/test", + "test:1234/blaboon", + "alpine:3.7", + "docker.example.edu/gmr/alpine:3.7", + "docker.example.com:5000/gmr/alpine@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "docker.example.co.uk/gmr/alpine/test2:latest", + "registry.dobby.org/dobby/dobby-servers/arthound:2019-08-08", + "owasp/zap:3.8.0", + "registry.dobby.co/dobby/dobby-servers/github-run:2021-10-04", + "docker.elastic.co/kibana/kibana:7.6.2", + "registry.dobby.org/dobby/dobby-servers/lerphound:latest", + "registry.dobby.org/dobby/dobby-servers/marbletown-poc:2021-03-29", + "marbles/marbles:v0.38.1", + "registry.dobby.org/dobby/dobby-servers/loophole@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "sonatype/nexon:3.30.0", + "prom/node-exporter:v1.1.1", + "sosedoff/pgweb@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8", + "sosedoff/pgweb:latest", + "registry.dobby.org/dobby/dobby-servers/arpeggio:2021-06-01", + "registry.dobby.org/dobby/antique-penguin:release-production", + "dalprodictus/halcon:6.7.5", + "antigua/antigua:v31", + "weblate/weblate:4.7.2-1", + "redis:4.0.01-alpine", + "registry.dobby.com/dobby/dobby-servers/github-run:latest", + "192.168.122.123:5000/kubeos-x86_64:2023-01", + ]; + let wrong_images = vec![ + "alpine;v1.0", + "alpine:latest@sha256:11111111111111111111111111111111", + "alpine|v1.0", + "alpine&v1.0", + "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8", + "192.168.122.123:5000/kubeos-x86_64:2023-01@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + "192.168.122.123:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + "myimage$%^&", + ":myimage", + "/myimage", + "myimage/", + "myimage:", + "myimage@@latest", + "myimage::tag", + "registry.com//myimage:tag", + " myimage", + "myimage ", + "registry.com/:tag", + "myimage:", + "", + ":tag", + "IP:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a", + ]; + for image in correct_images { + assert!(is_valid_image_name(image).is_ok()); + } + for image in wrong_images { + assert!(is_valid_image_name(image).is_err()); + } + } + + #[test] + fn test_get_oci_image_digest() { + init(); + let mut mock = MockCommandExec::new(); + let container_runtime = "ctr"; + let image_name = "docker.io/nginx:latest"; + let command_output1 = + "REF TYPE DIGEST SIZE PLATFORMS LABELS\ndocker.io/nginx:latest text/html sha256:1111 132.5 KIB - -\n"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + let out1 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); + let expect_output = "1111"; + assert_eq!(out1, expect_output); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok("invalid output".to_string())); + let out2 = get_oci_image_digest(container_runtime, image_name, &mock); + assert!(out2.is_err()); + + let container_runtime = "crictl"; + let command_output2 = "[docker.io/nginx@sha256:1111]"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string())); + let out3 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap(); + assert_eq!(out3, expect_output); + + let out4 = get_oci_image_digest("invalid", image_name, &mock); + assert!(out4.is_err()); + + let container_runtime = "crictl"; + let command_output3 = "[docker.io/nginx:sha256:1111]"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string())); + let out5 = get_oci_image_digest(container_runtime, image_name, &mock); + assert!(out5.is_err()); + } + + #[test] + fn test_check_oci_image_digest_match() { + init(); + let mut mock = MockCommandExec::new(); + let image_name = "docker.io/nginx:latest"; + let container_runtime = "crictl"; + let command_output = "[docker.io/nginx@sha256:1a2b]"; + let check_sum = "1A2B"; + mock.expect_run_command_with_output().times(2).returning(|_, _| Ok(command_output.to_string())); + let result = check_oci_image_digest(container_runtime, image_name, check_sum, &mock); + assert!(result.is_ok()); + let result = check_oci_image_digest(container_runtime, image_name, "1111", &mock); + assert!(result.is_err()); + } + + #[test] + fn test_pull_image() { + init(); + let mut mock_executor = MockCommandExec::new(); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "crictl" && args.len() == 2 && args[0] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[3] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "docker" && args.len() == 2 && args[0] == "pull") // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + + let image_name = "docker.io/nginx:latest"; + let result = pull_image("crictl", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("ctr", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("docker", image_name, &mock_executor); + assert!(result.is_ok()); + let result = pull_image("aaa", image_name, &mock_executor); + assert!(result.is_err()); + } + + #[test] + fn test_remove_image_if_exist() { + init(); + let mut mock_executor = MockCommandExec::new(); + mock_executor + .expect_run_command_with_output() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"check")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(String::from("something"))); + mock_executor + .expect_run_command() + .withf(|cmd, args| cmd == "ctr" && args.contains(&"rm")) // simplified with a closure + .times(1) + .returning(|_, _| Ok(())); + let image_name = "docker.io/nginx:latest"; + let res = remove_image_if_exist("ctr", image_name, &mock_executor); + assert!(res.is_ok()); + + let res = remove_image_if_exist("invalid", image_name, &mock_executor); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/manager/src/utils/executor.rs b/KubeOS-Rust/manager/src/utils/executor.rs new file mode 100644 index 00000000..c87bf2ad --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/executor.rs @@ -0,0 +1,89 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::process::Command; + +use anyhow::{bail, Result}; +use log::{debug, trace}; + +pub trait CommandExecutor: Clone { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; +} + +#[derive(Clone)] +pub struct RealCommandExecutor {} + +impl CommandExecutor for RealCommandExecutor { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()> { + trace!("run_command: {} {:?}", name, args); + let output = Command::new(name).args(args).output()?; + if !output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); + let error_message = String::from_utf8_lossy(&output.stderr); + bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message); + } + debug!("run_command: {} {:?} done", name, args); + Ok(()) + } + + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result { + trace!("run_command_with_output: {} {:?}", name, args); + let output = Command::new(name).args(args).output()?; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + if !output.status.success() { + let error_message = String::from_utf8_lossy(&output.stderr); + bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message); + } + debug!("run_command_with_output: {} {:?} done", name, args); + Ok(stdout.trim_end_matches('\n').to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_run_command_with_output() { + init(); + let executor: RealCommandExecutor = RealCommandExecutor {}; + + // test run_command_with_output + let output = executor.run_command_with_output("echo", &["hello", "world"]).unwrap(); + assert_eq!(output, "hello world"); + let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "cat").as_str()]).unwrap(); + assert_eq!(out, "/usr/bin/cat"); + let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "apple").as_str()]); + assert!(out.is_err()); + } + + #[test] + fn test_run_command() { + init(); + let executor: RealCommandExecutor = RealCommandExecutor {}; + // test run_command + let out = executor.run_command("sh", &["-c", format!("command -v {}", "apple").as_str()]); + assert!(out.is_err()); + + let out = executor.run_command("sh", &["-c", format!("command -v {}", "cat").as_str()]); + assert!(out.is_ok()); + } +} diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs new file mode 100644 index 00000000..90806cf8 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/image_manager.rs @@ -0,0 +1,206 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{ + fs::{self, Permissions}, + os::unix::fs::PermissionsExt, + path::PathBuf, +}; + +use anyhow::{Context, Result}; +use log::{debug, info}; + +use super::{ + clean_env, + common::{delete_file_or_dir, PreparePath}, + executor::CommandExecutor, + partition::PartitionInfo, +}; + +pub struct UpgradeImageManager { + pub paths: PreparePath, + pub next_partition: PartitionInfo, + pub executor: T, +} + +impl UpgradeImageManager { + pub fn new(paths: PreparePath, next_partition: PartitionInfo, executor: T) -> Self { + Self { paths, next_partition, executor } + } + + fn image_path_str(&self) -> Result<&str> { + self.paths.image_path.to_str().context("Failed to convert image path to string") + } + + fn mount_path_str(&self) -> Result<&str> { + self.paths.mount_path.to_str().context("Failed to convert mount path to string") + } + + fn tar_path_str(&self) -> Result<&str> { + self.paths.tar_path.to_str().context("Failed to convert tar path to string") + } + + pub fn create_image_file(&self, permission: u32) -> Result<()> { + let image_str = self.image_path_str()?; + + debug!("Create image {}", image_str); + self.executor.run_command("dd", &["if=/dev/zero", &format!("of={}", image_str), "bs=2M", "count=1024"])?; + fs::set_permissions(&self.paths.image_path, Permissions::from_mode(permission))?; + Ok(()) + } + + pub fn format_image(&self) -> Result<()> { + let image_str = self.image_path_str()?; + debug!("Format image {}", image_str); + self.executor.run_command( + format!("mkfs.{}", self.next_partition.fs_type).as_str(), + &["-L", format!("ROOT-{}", self.next_partition.menuentry).as_str(), image_str], + )?; + Ok(()) + } + + pub fn mount_image(&self) -> Result<()> { + let image_str = self.image_path_str()?; + let mount_str = self.mount_path_str()?; + debug!("Mount {} to {}", image_str, mount_str); + self.executor.run_command("mount", &["-o", "loop", image_str, mount_str])?; + Ok(()) + } + + pub fn extract_tar_to_image(&self) -> Result<()> { + let tar_str = self.tar_path_str()?; + let mount_str = self.mount_path_str()?; + debug!("Extract {} to mounted path {}", tar_str, mount_str); + self.executor.run_command("tar", &["-xvf", tar_str, "-C", mount_str])?; + Ok(()) + } + + pub fn create_os_image(self, permission: u32) -> Result { + self.create_image_file(permission)?; + self.format_image()?; + self.mount_image()?; + self.extract_tar_to_image()?; + // Pass empty image_path to clean_env but avoid deleting the upgrade image + clean_env(&self.paths.update_path, &self.paths.mount_path, &PathBuf::new())?; + Ok(self) + } + + pub fn install(&self) -> Result<()> { + let image_str = self.image_path_str()?; + let device = self.next_partition.device.as_str(); + self.executor + .run_command("dd", &[format!("if={}", image_str).as_str(), format!("of={}", device).as_str(), "bs=8M"])?; + debug!("Install image {} to {} done", image_str, device); + info!( + "Device {} is overwritten and unable to rollback to the previous version anymore if the eviction of node fails", + device + ); + delete_file_or_dir(image_str)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{fs, io::Write, path::Path}; + + use mockall::{mock, predicate::*}; + use tempfile::NamedTempFile; + + use super::*; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_update_image_manager() { + init(); + // create a dir in tmp dir + let tmp_dir = "/tmp/test_update_image_manager"; + let img_path = format!("{}/test_image", tmp_dir); + let mut temp_file = NamedTempFile::new().unwrap(); + write!(temp_file, "test content").unwrap(); // Writing s + fs::create_dir(tmp_dir).unwrap(); + let clone_img_path = img_path.clone(); + + let mut mock = MockCommandExec::new(); + //mock create_image_file + mock.expect_run_command() + .withf(|name, args| name == "dd" && args[0] == "if=/dev/zero") + .times(1) // Expect it to be called once + .returning(move |_, _| { + // simulate 'dd' by copying the contents of the temporary file + std::fs::copy(temp_file.path(), &clone_img_path).unwrap(); + Ok(()) + }); + + //mock format_image + mock.expect_run_command() + .withf(|name, args| name == "mkfs.ext4" && args[1] == "ROOT-B") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock mount_image + mock.expect_run_command() + .withf(|name, _| name == "mount") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock extract_tar_to_image + mock.expect_run_command() + .withf(|name, args| name == "tar" && args[0] == "-xvf") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + //mock install->dd + mock.expect_run_command() + .withf(|name, _| name == "dd") + .times(1) // Expect it to be called once + .returning(|_, _| Ok(())); + + let img_manager = UpgradeImageManager::new( + PreparePath { + persist_path: "/tmp".into(), + update_path: tmp_dir.into(), + image_path: img_path.into(), + mount_path: "/tmp/update/mount".into(), + tar_path: "/tmp/update/image.tar".into(), + rootfs_file: "image.tar".into(), + }, + PartitionInfo { device: "/dev/sda3".into(), fs_type: "ext4".into(), menuentry: "B".into() }, + mock, + ); + + let img_manager = img_manager.create_os_image(0o755).unwrap(); + let result = img_manager.install(); + assert!(result.is_ok()); + + assert_eq!(Path::new(&tmp_dir).exists(), false); + } +} diff --git a/KubeOS-Rust/manager/src/utils/mod.rs b/KubeOS-Rust/manager/src/utils/mod.rs new file mode 100644 index 00000000..caf406e3 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/mod.rs @@ -0,0 +1,23 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod common; +mod container_image; +mod executor; +mod image_manager; +mod partition; + +pub use common::*; +pub use container_image::*; +pub use executor::*; +pub use image_manager::*; +pub use partition::*; diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs new file mode 100644 index 00000000..799b4b35 --- /dev/null +++ b/KubeOS-Rust/manager/src/utils/partition.rs @@ -0,0 +1,117 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::{bail, Result}; +use log::{debug, trace}; + +use super::executor::CommandExecutor; + +#[derive(PartialEq, Debug, Default)] +pub struct PartitionInfo { + pub device: String, + pub menuentry: String, + pub fs_type: String, +} + +/// get_partition_info returns the current partition info and the next partition info. +pub fn get_partition_info(executor: &T) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> { + let lsblk = executor.run_command_with_output("lsblk", &["-lno", "NAME,MOUNTPOINTS,FSTYPE"])?; + // After split whitespace, the root directory line should have 3 elements, which are "sda2 / ext4". + let mut cur_partition = PartitionInfo::default(); + let mut next_partition = PartitionInfo::default(); + let splitted_len = 3; + trace!("get_partition_info lsblk command output:\n{}", lsblk); + for line in lsblk.lines() { + let res: Vec<&str> = line.split_whitespace().collect(); + if res.len() == splitted_len && res[1] == "/" { + debug!("root directory line: device={}, fs_type={}", res[0], res[2]); + cur_partition.device = format!("/dev/{}", res[0]).to_string(); + cur_partition.fs_type = res[2].to_string(); + next_partition.fs_type = res[2].to_string(); + if res[0].contains('2') { + // root directory is mounted on sda2, so sda3 is the next partition + cur_partition.menuentry = String::from("A"); + next_partition.menuentry = String::from("B"); + next_partition.device = format!("/dev/{}", res[0].replace('2', "3")).to_string(); + } else if res[0].contains('3') { + // root directory is mounted on sda3, so sda2 is the next partition + cur_partition.menuentry = String::from("B"); + next_partition.menuentry = String::from("A"); + next_partition.device = format!("/dev/{}", res[0].replace('3', "2")).to_string(); + } + } + } + if cur_partition.menuentry.is_empty() { + bail!("Failed to get partition info, lsblk output: {}", lsblk); + } + Ok((cur_partition, next_partition)) +} + +#[cfg(test)] +mod tests { + use mockall::{mock, predicate::*}; + + use super::*; + + // Mock the CommandExecutor trait + mock! { + pub CommandExec{} + impl CommandExecutor for CommandExec { + fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>; + fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result; + } + impl Clone for CommandExec { + fn clone(&self) -> Self; + } + } + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_get_partition_info() { + init(); + let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3 ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + let mut mock = MockCommandExec::new(); + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string())); + let res = get_partition_info(&mock).unwrap(); + let expect_res = ( + PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() }, + PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() }, + ); + assert_eq!(res, expect_res); + + let command_output2 = "sda\nsda1 /boot/efi vfat\nsda2 ext4\nsda3 / ext4\nsda4 /persist ext4\nsr0 iso9660\n"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string())); + let res = get_partition_info(&mock).unwrap(); + let expect_res = ( + PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() }, + PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() }, + ); + assert_eq!(res, expect_res); + + let command_output3 = ""; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string())); + let res = get_partition_info(&mock); + assert!(res.is_err()); + + let command_output4 = "sda4 / ext4"; + mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output4.to_string())); + let res = get_partition_info(&mock); + assert!(res.is_err()); + } +} diff --git a/KubeOS-Rust/operator/Cargo.toml b/KubeOS-Rust/operator/Cargo.toml new file mode 100644 index 00000000..91cca265 --- /dev/null +++ b/KubeOS-Rust/operator/Cargo.toml @@ -0,0 +1,47 @@ +[package] +description = "KubeOS os-operator" +edition = "2021" +license = "MulanPSL-2.0" +name = "operator" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +name = "operator" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.44" +async-trait = "0.1" +cli = { version = "1.0.6", path = "../cli" } +env_logger = "0.9.0" +futures = "0.3.17" +h2 = "=0.3.16" +k8s-openapi = { version = "0.13.1", features = ["v1_22"] } +kube = { version = "0.66.0", features = ["derive", "runtime"] } +log = "=0.4.15" +manager = { version = "1.0.6", path = "../manager" } +regex = "=1.7.3" +reqwest = { version = "=0.12.2", default-features = false, features = [ + "json", +] } +schemars = "=0.8.10" +serde = { version = "1.0.130", features = ["derive"] } +serde_json = "1.0.68" +socket2 = "=0.4.9" +thiserror = "1.0.29" +thread_local = "=1.1.4" +tokio = { version = "=1.28.0", default-features = false, features = [ + "macros", + "rt-multi-thread", +] } +tokio-retry = "0.3" + +[dev-dependencies] +assert-json-diff = "2.0.2" +http = "0.2.9" +hyper = "0.14.25" +tower-test = "0.4.0" +mockall = { version = "=0.11.3" } +regex = "1" diff --git a/KubeOS-Rust/operator/src/controller/apiclient.rs b/KubeOS-Rust/operator/src/controller/apiclient.rs new file mode 100644 index 00000000..3642b475 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/apiclient.rs @@ -0,0 +1,110 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + +use anyhow::Result; +use apiclient_error::Error; +use async_trait::async_trait; +use kube::{ + api::{Api, Patch, PatchParams}, + Client, +}; +use serde::{Deserialize, Serialize}; +use super::{ + crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}, + values::{NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}, +}; + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceSpecPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + spec: OSInstanceSpec, +} + +impl Default for OSInstanceSpecPatch { + fn default() -> Self { + OSInstanceSpecPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceStatusPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + status: Option, +} + +impl Default for OSInstanceStatusPatch { + fn default() -> Self { + OSInstanceStatusPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }), + } + } +} + +#[derive(Clone)] +pub struct ControllerClient { + pub client: Client, +} + +impl ControllerClient { + pub fn new(client: Client) -> Self { + ControllerClient { client } + } +} + +#[async_trait] +pub trait ApplyApi: Clone + Sized + Send + Sync { + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error>; +} + +#[async_trait] +impl ApplyApi for ControllerClient { + + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() }; + osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?; + Ok(()) + } + +} +pub mod apiclient_error { + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} diff --git a/KubeOS-Rust/operator/src/controller/apiserver_mock.rs b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs new file mode 100644 index 00000000..21abd405 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs @@ -0,0 +1,995 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{borrow::BorrowMut, cell::{RefCell, RefMut}, clone, collections::BTreeMap, default}; +use regex::Regex; +use anyhow::Result; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use http::{status, Request, Response}; +use hyper::{body::to_bytes, Body}; +use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod}; +use kube::{ + api::ObjectMeta, + core::{ErrorResponse, ListMeta, ObjectList}, + Client as KubeClient, Resource, ResourceExt, +}; +use log::debug; +use mockall::mock; +use serde_json::json; + +use self::mock_error::Error; +use super::{ + crd::{Configs, OSInstanceStatus}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_CONFIG}, +}; +use crate::controller::{ + apiclient::{ApplyApi, ControllerClient}, + crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS}, + values::{LABEL_MASTER, LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, + OperatorController, +}; + +type ApiServerHandle = tower_test::mock::Handle, Response>; +pub struct ApiServerVerifier(ApiServerHandle); + + +#[derive(Clone, Debug, Default)] +pub struct K8sResources{ + pub node_list: Vec, + pub osi_list: Vec, +} + +pub enum Testcases { + Rollback(K8sResources), + ConfigNormal(K8sResources), + SkipNoOsiNode(K8sResources), + ExchangeCurrentAndNext(K8sResources), + GetConfigOSInstances(String), + CheckUpgrading(String), + GetIdleOSInstances(String), + +} + +pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) { + tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("timeout on mock apiserver") + .expect("scenario succeeded") +} + +impl ApiServerVerifier { + pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::Rollback(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + }, + Testcases::ConfigNormal(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_config_osi_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_idle_osi_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_patch_spec_config(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_spec_config(k8s_resc.clone()) + .await + }, + Testcases::SkipNoOsiNode(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + }, + Testcases::ExchangeCurrentAndNext(k8s_resc) => { + self.handler_worker_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_upgrading_node_list_get(k8s_resc.clone()) + .await + .unwrap() + .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone()) + .await + .unwrap() + // 为两个节点上的 osi 升级，重复两次 + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_get_by_node_name(k8s_resc.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone()) + .await + .unwrap() + .handler_replace_node_by_name(k8s_resc.clone()) + .await + }, + _ => { + Err(Error::ArgumentError) + } + } + .expect("Case completed without errors"); + }) + } + + pub fn test_function(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::GetConfigOSInstances(error) => { + self.handler_config_osi_list_get_error(error) + .await + }, + Testcases::CheckUpgrading(error) => { + self.handler_upgrading_node_list_get_error(error) + .await + }, + Testcases::GetIdleOSInstances(error) => { + self.handler_idle_osi_list_get_error(error) + .await + }, + _ => { + Err(Error::ArgumentError) + } + } + .expect("Case completed without errors"); + }) + } + + // 获取所有的 worker 节点，对应于reconcile的第一个 get_nodes 函数 + async fn handler_worker_node_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=%21node-role.kubernetes.io%2Fcontrol-plane&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中所有的 worker 节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if !node.labels().contains_key(LABEL_MASTER){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_worker_node_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 upgrading 的节点 + async fn handler_upgrading_node_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中标签为正在升级的节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if node.labels().contains_key(LABEL_UPGRADING){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_upgrading_node_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取所有的非 upgrading 的 worker 节点 + async fn handler_worker_and_no_upgrade_noding_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + + let remove_limit = |input: &str| -> String { + let re = Regex::new(r"limit=\d+").unwrap(); + re.replace_all(input, "").to_string() + }; + + assert_eq!( + remove_limit(request.uri().to_string().as_str()), + "/api/v1/nodes?&labelSelector=%21upgrade.openeuler.org%2Fupgrading%2C%21node-role.kubernetes.io%2Fcontrol-plane&"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中所有的非 upgrading 的 worker 节点传出 + let mut nodes = vec![]; + for node in k8s_resc.node_list.clone() { + if !node.labels().contains_key(LABEL_UPGRADING) && !node.labels().contains_key(LABEL_MASTER){ + nodes.push(node.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: nodes, + }; + + dbg!("handler_worker_and_no_upgrade_noding_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_osinstance_get_by_node_name(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_get_by_node_name: req_node_name: {:?}", req_node_name); + + let response = serde_json::to_vec(&osinstance).unwrap(); + dbg!("handler_osinstance_get_by_node_name"); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_upgrade(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_nodestatus_upgrade: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_upgrade"); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_exchange(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_node_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_nodestatus_exchange: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + + let sysconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + + let upgradeconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + + assert_eq!(spec.sysconfigs.clone(), sysconfigs); + assert_eq!(spec.upgradeconfigs.clone(), upgradeconfigs); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_exchange"); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + // 通过节点名称获取对应节点 + async fn handler_replace_node_by_name(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PUT); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get node and send back + let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut node = Node{..Default::default()}; + let mut boolean_get_node = false; + for node_iter in k8s_resc.node_list.clone() { + if node_iter.name() == req_node_name { + boolean_get_node = true; + node = node_iter.clone(); + break; + } + } + assert!(boolean_get_node); + assert_eq!(request.extensions().get(), Some(&"replace")); + + println!("handler_replace_node_by_name: req_node_name: {:?}", req_node_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let metadata_json = body_json.get("metadata").expect("metadata object").clone(); + let metadata: ObjectMeta = serde_json::from_value(metadata_json).expect("valid metadata"); + assert!(metadata.labels.unwrap().contains_key(LABEL_UPGRADING)); + + // 修改 node 并传出 + node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string()); + + dbg!("handler_replace_node_by_name"); + let response = serde_json::to_vec(&node).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 config 的节点上的 osi + async fn handler_config_osi_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出 + let mut osis = vec![]; + for osi in k8s_resc.osi_list.clone() { + if osi.spec.nodestatus == NODE_STATUS_CONFIG{ + osis.push(osi.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: osis, + }; + + dbg!("handler_config_osi_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + // 获取环境中所有的标签为 config 的节点上的 osi + async fn handler_idle_osi_list_get(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3"); + assert_eq!(request.extensions().get(), Some(&"list")); + + // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出 + let mut osis = vec![]; + for osi in k8s_resc.osi_list.clone() { + if osi.spec.nodestatus == NODE_STATUS_IDLE{ + osis.push(osi.clone()); + } + } + + let node_list: ObjectList = ObjectList { + metadata: ListMeta { + ..Default::default() + }, + items: osis, + }; + + dbg!("handler_idle_osi_list_get"); + + let response = serde_json::to_vec(&node_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_osinstance_patch_spec_config(mut self, k8s_resc: K8sResources) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + + // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back + let req_osi_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap(); + let mut osinstance = OSInstance::set_osi_default("", ""); + let mut boolean_get_osi = false; + for osi in k8s_resc.osi_list.clone() { + if osi.name() == req_osi_name { + boolean_get_osi = true; + osinstance = osi.clone(); + break; + } + } + assert!(boolean_get_osi); + + println!("handler_osinstance_patch_spec_config: req_osi_name: {:?}", req_osi_name); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_CONFIG.to_string()); + + let sysconfig = Some( + Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl")), + configpath: Some(String::from("")), + contents: + Some(vec![ + Content { + key: Some(String::from("key1")), + value: Some(String::from("a")), + operation: Some(String::from("")), + }, + Content { + key: Some(String::from("key2")), + value: Some(String::from("b")), + operation: Some(String::from("")), + }, + ]), + }]), + } + ); + assert_eq!( + spec.sysconfigs.clone(), + sysconfig + ); + + dbg!("handler_osinstance_patch_spec_config"); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance.spec.sysconfigs = sysconfig; + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_config_osi_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_config_osi_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "NotFound".to_string(), + code: 404, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_upgrading_node_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_upgrading_node_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "Invalid".to_string(), + code: 400, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(400).body(Body::from(response)).unwrap()); + + Ok(self) + } + + async fn handler_idle_osi_list_get_error(mut self, error: String) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3"); + assert_eq!(request.extensions().get(), Some(&"list")); + + dbg!("handler_idle_osi_list_get_error"); + + // 仅序列化 ErrorResponse 部分 + let error_response = ErrorResponse { + status: "Failure".to_string(), + message: error, + reason: "NotFound".to_string(), + code: 404, + }; + + // 序列化为 JSON + let response_body = json!({ + "status": error_response.status, + "message": error_response.message, + "reason": error_response.reason, + "code": error_response.code, + }); + + // 构建 HTTP 响应 + let response = serde_json::to_vec(&response_body).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + + Ok(self) + } + +} + +pub mod mock_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + + #[error("Parameters other than expected were entered")] + ArgumentError, + } +} + + +impl OperatorController { + pub fn test() -> (OperatorController, ApiServerVerifier) { + let (mock_service, handle) = tower_test::mock::pair::, Response>(); + let mock_k8s_client = KubeClient::new(mock_service, "default"); + let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); + let operator_controller: OperatorController = + OperatorController::new(mock_k8s_client, mock_api_client); + (operator_controller, ApiServerVerifier(handle)) + } +} + +impl OSInstance { + pub fn set_osi_default(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }, + status: Some(OSInstanceStatus { + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }), + } + } +} + +impl OS { + pub fn set_os_default() -> Self { + let mut os = OS::new("test", OSSpec::default()); + os.meta_mut().namespace = Some("default".into()); + os + } + + pub fn set_os_rollback_osversion_v1_upgradecon_v1() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string(); + os + } + + pub fn set_os_syscon_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = OPERATION_TYPE_CONFIG.to_string(); + os.spec.sysconfigs = Some( + Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("key1")), + value: Some(String::from("a")), + operation: Some(String::from("")), + }, + Content { + key: Some(String::from("key2")), + value: Some(String::from("b")), + operation: Some(String::from("")), + }, + ]), + }]), + } + ); + os + } + + pub fn set_os_skip_osversion_v2_upgradecon_v1() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os + } + + pub fn set_os_exchange_current_and_next() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + let sysconfigs = Some( + Configs{ + version: Some(String::from("v2")), + configs: Some(vec![ + Config { + model: Some(String::from("grub.cmdline.current")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("a")), + value: Some(String::from("1")), + operation: Some(String::from("")), + } + ]), + }, + Config { + model: Some(String::from("grub.cmdline.next")), + configpath: Some(String::from("")), + contents: Some(vec![ + Content { + key: Some(String::from("b")), + value: Some(String::from("2")), + operation: Some(String::from("")), + } + ]), + }, + ]), + } + ); + os.spec.sysconfigs = sysconfigs.clone(); + os.spec.upgradeconfigs = sysconfigs.clone(); + + os + } + +} + +impl K8sResources { + pub fn set_rollback_nodes_v2_and_osi_v1() -> Self { + // 创建 node1 和 node2 + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v2".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let node2 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node2".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v2".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default"); + let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default"); + + let node_list = Vec::from([node1, node2]); + let osi_list = Vec::from([osi1, osi2]); + + K8sResources{ + node_list, + osi_list + } + } + + pub fn set_nodes_v1_and_osi_v1() -> Self { + // 创建 node1 和 node2 + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let node2 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node2".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default"); + let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default"); + + let node_list = Vec::from([node1, node2]); + let osi_list = Vec::from([osi1, osi2]); + + K8sResources{ + node_list, + osi_list + } + } + + pub fn set_skip_nodes_and_osi() -> Self { + // 创建 node1 并且不设置 osi + let node1 = Node { + metadata: ObjectMeta { + name: Some("openeuler-node1".into()), + labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])), + ..Default::default() + }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { + os_image: "KubeOS v1".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let node_list = Vec::from([node1]); + let osi_list = Vec::new(); + + K8sResources{ + node_list, + osi_list + } + } + +} + +impl Default for OSSpec { + fn default() -> Self { + OSSpec { + osversion: String::from("KubeOS v1"), + maxunavailable: 3, + checksum: String::from("test"), + imagetype: String::from("containerd"), + containerimage: String::from("test"), + opstype: String::from("upgrade"), + evictpodforce: true, + imageurl: String::from(""), + flagsafe: true, + mtls: false, + cacert: Some(String::from("")), + clientcert: Some(String::from("")), + clientkey: Some(String::from("")), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + } + } +} diff --git a/KubeOS-Rust/operator/src/controller/controller.rs b/KubeOS-Rust/operator/src/controller/controller.rs new file mode 100644 index 00000000..2beb8945 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/controller.rs @@ -0,0 +1,696 @@ +/* +* Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. +* KubeOS is licensed under the Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +* PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + + +use anyhow::Result; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::{Api, ListParams, ObjectList, PostParams}, + core::ErrorResponse, + runtime::controller::{Context, ReconcilerAction}, + Client, ResourceExt, +}; +use log::{debug, error}; +use reconciler_error::Error; + +use crate::controller::values::NODE_STATUS_UPGRADE; + +use super::{ + apiclient::ApplyApi, + crd::{Configs, OSInstance, OS}, + values::{ + LABEL_MASTER, LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, + NO_REQUEUE, OPERATION_TYPE_CONFIG, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, + REQUEUE_ERROR, REQUEUE_NORMAL, SYS_CONFIG_NAME, UPGRADE_CONFIG_NAME + }, +}; + +#[derive(Clone)] +pub struct OperatorController { + k8s_client: Client, + controller_client: T, +} + +impl OperatorController { + pub fn new(k8s_client: Client, controller_client: T) -> Self { + OperatorController { + k8s_client, + controller_client, + } + } + + // async fn reconcile( + // &self, + // os: OS, + // ctx: Context>, + // ) -> Result { + // // k8s_client 变量不是 Option 类型，而是直接的 Client 类型，Rust 会确保不为空，不用检查 + + // // Kube-rs库中的Context类型已经处理了上下文的管理，也不需要初始化空的上下文，proxy组件的rust版本也没初始化 + + // // 调用外部的Reconcile函数 + // reconcile(os, ctx).await + // } + + // 获取 worker 节点数 + async fn get_and_update_os(&self, _namespace: &str) -> Result { + + // 创建一个筛选标签的 String 数组，只找 worker 节点 + let reqs = vec![ + format!("!{}", LABEL_MASTER), + ]; + + // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量 + let nodes_items = self.get_nodes( 0, reqs).await?; + + Ok(nodes_items.items.len() as i64) + } + + // 获取 worker 节点，传入 reqs 为 String 数组，表示多个筛选条件 + async fn get_nodes(&self, limit: i64, reqs: Vec) -> Result, Error> { + + let nodes_api: Api = Api::all(self.k8s_client.clone()); + + // 将多个标签筛选器组合成一个字符串，用逗号分隔 + let label_selector = reqs.join(","); + + // 设置 ListParams + let list_params = ListParams::default() + .labels(&label_selector) + .limit(limit as u32); + + let nodes = match nodes_api.list(&list_params).await { + Ok(nodes) => nodes, + Err(e) => { + log::error!("{:?} unable to list nodes with requirements", e); + return Err(Error::KubeClient { source: e }); + }, + }; + + Ok(nodes) + } + + // 获取可以进行升级操作的最大节点数量 + async fn check_upgrading(&self, _namespace: &str, max_unavailable: i64) -> Result { + + // 设置筛选标签，选择标签为正在升级的节点 + let reqs = vec![ + LABEL_UPGRADING.to_string(), + ]; + + // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量 + let nodes_items = self.get_nodes( 0, reqs).await?; + + Ok(max_unavailable - nodes_items.items.len() as i64) + } + + // 为指定数量的节点进行升级操作 + async fn assign_upgrade(&self, os: &OS, limit: i64, namespace: &str) -> Result { + + // 创建筛选标签，只找 worker 节点，并且标签 LabelUpgrading 不存在 + let reqs = vec![ + format!("!{}", LABEL_UPGRADING), + format!("!{}", LABEL_MASTER), + ]; + + // 获取符合标签要求的节点 + let mut nodes_items = self.get_nodes( limit + 1, reqs).await?; + + // 对选定的节点进行升级，返回升级成功的数量和可能的错误 + let count = self.upgrade_nodes(os, &mut nodes_items, limit, namespace).await?; + + Ok(count >= limit) + } + + async fn upgrade_nodes(&self, os: &OS, nodes: &mut ObjectList, limit: i64, namespace: &str) -> Result { + + let mut count = 0; + + for node in nodes.iter_mut() { + // 如果已经达到升级限制，则退出循环 + if count >= limit { + break + } + + let os_version_node = node.status.clone().unwrap().node_info.unwrap().os_image; + + debug!("node name: {}, os_version_node: {}, os_version: {}", node.name(), os_version_node, os.spec.osversion); + + // 检查 os 对象中的操作系统版本是否与节点的操作系统版本不同 + if os_version_node != os.spec.osversion { + + // 尝试获取该节点上的 os 实例 + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + match osi_api.get(&node.name().clone()).await { + Ok(mut osi) => { + // info!("osinstance is exist {:?}", osi.name()); + + debug!("osinstance is exist: \n {:?} \n", osi); + + match self.update_node_and_osins(os, node, &mut osi).await { + Ok(_) => { + count += 1; + }, + Err(_) => { + continue; + }, + } + }, + Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { + debug!("failed to get osInstance {}", &node.name().clone()); + + return Err(Error::KubeClient { + source: kube::Error::Api(ErrorResponse { + reason, + status: "".to_string(), + message: "".to_string(), + code: 0 + })}); + }, + Err(_) => continue, + } + } + + } + + Ok(count) + } + + // 升级节点以及节点上的 OSinstance + async fn update_node_and_osins(&self, os: &OS, node: &mut Node, osinstance: &mut OSInstance, ) -> Result<(), Error> { + debug!("start update_node_and_OSins"); + + // 检查os实例中的升级配置版本与os对象中的升级配置版本是否匹配。osi 字段未初始化时直接进行拷贝 + let mut copy_sign = true; + if let Some(upgradeconfigs) = osinstance.spec.upgradeconfigs.clone() { + if let Some(version) = upgradeconfigs.version { + if version == os.spec.upgradeconfigs.clone().unwrap().version.unwrap() { + copy_sign = false; + } + } + } + if copy_sign { + self.deep_copy_spec_configs(os, osinstance, UPGRADE_CONFIG_NAME.to_string()).await?; + assert!(osinstance.spec.upgradeconfigs.is_some()); + } + + copy_sign = true; + // 检查os实例中的系统配置版本与os对象中的系统配置版本是否匹配。osi 字段未初始化时直接进行拷贝 + if let Some(sysconfigs) = osinstance.spec.sysconfigs.clone() { + if let Some(version) = sysconfigs.version { + if version == os.spec.sysconfigs.clone().unwrap().version.unwrap() { + copy_sign = false; + } + } + } + if copy_sign { + self.deep_copy_spec_configs(os, osinstance, SYS_CONFIG_NAME.to_string()).await?; + assert!(osinstance.spec.sysconfigs.is_some()); + if let Some(sysconfigs) = osinstance.spec.sysconfigs.as_mut() { + if let Some(configs) = &mut sysconfigs.configs { + for config in configs { + if config.model.clone().unwrap() == "grub.cmdline.current" { + config.model = Some("grub.cmdline.next".to_string()); + } + else if config.model.clone().unwrap() == "grub.cmdline.next" { + config.model = Some("grub.cmdline.current".to_string()); + } + } + } + } + } + + // 更新os实例中的状态为升级完成状态 + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + + // 把对 osinstance 的更改从内存更新到 k8s 集群 + let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; + + node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string()); + + // 把对 node 的更改从内存更新到 k8s 集群 + let node_api: Api = Api::all(self.k8s_client.clone()); + node_api.replace(&node.name(), &PostParams::default(), &node).await?; + + Ok(()) + } + + // 深拷贝 + async fn deep_copy_spec_configs(&self, os: &OS, os_instance: &mut OSInstance, config_type: String) -> Result<(), Error> { + + match config_type.as_str() { + UPGRADE_CONFIG_NAME =>{ + + if let Ok(data) = serde_json::to_vec(&os.spec.upgradeconfigs){ + + if let Ok(upgradeconfigs) = serde_json::from_slice(&data) { + os_instance.spec.upgradeconfigs = Some(upgradeconfigs); + }else { + debug!("{} Deserialization failure", config_type); + return Err(Error::Operation { value: "Deserialization".to_string()}); + } + } + else { + debug!("{} Serialization failure", config_type); + return Err(Error::Operation { value: "Serialization".to_string()}); + } + + }, + SYS_CONFIG_NAME => { + + if let Ok(data) = serde_json::to_vec(&os.spec.sysconfigs){ + + if let Ok(sysconfigs) = serde_json::from_slice(&data) { + os_instance.spec.sysconfigs = Some(sysconfigs); + }else { + debug!("{} Deserialization failure", config_type); + return Err(Error::Operation { value: "Deserialization".to_string()}); + } + } + else { + debug!("{} Serialization failure", config_type); + return Err(Error::Operation { value: "Serialization".to_string()}); + } + + }, + _ => { + debug!("configType {} cannot be recognized", config_type); + return Err(Error::Operation { value: config_type.clone() }); + }, + } + + Ok(()) + } + + // 获取可以进行配置操作的节点数量，返回的是 instance 的列表 + async fn check_config(&self, namespace: &str, max_unavailable: i64) -> Result { + let osinstances = self.get_config_osinstances(namespace).await?; + + Ok(max_unavailable - osinstances.len() as i64) + } + + // 获取所在节点状态为配置的 osinstance列表 + async fn get_config_osinstances(&self, namespace: &str) -> Result, Error> { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + // 获取所有 OSInstance 资源 + let all_osinstances = osi_api.list(&ListParams::default()).await?; + + // 在客户端进行过滤，节点状态为 NODE_STATUS_CONFIG + let osinstances: Vec = all_osinstances + .items + .into_iter() + .filter(|osi| osi.spec.nodestatus == NODE_STATUS_CONFIG) + .collect(); + + debug!("config_osi count = {:?}", osinstances.len()); + + Ok(osinstances) + } + + // 为指定数量的节点进行配置操作 + async fn assign_config(&self, _os: &OS, sysconfigs: Configs, config_version: String, limit: i64, namespace: &str) -> Result { + + debug!("start assign_config"); + + let mut osinstances = self.get_idle_os_instances(namespace, limit + 1).await?; + + let mut count = 0; + // 遍历 osi 列表 + for osi in osinstances.iter_mut() { + if count > limit { + break; + } + + let mut config_sign = true; + if let Some(sysconfigs) = osi.spec.sysconfigs.clone() { + if let Some(version) = sysconfigs.version { + debug!("node name: {:?}, config_version_node: {:?}, config_version: {:?}", osi.name(), version, config_version); + if version == config_version { + config_sign = false; + } + } + } + + // 如果版本不同或 osi 未初始化，则将新的配置信息更新到实例中，并将节点状态标记为“配置完成”。 + if config_sign { + count += 1; + osi.spec.sysconfigs = Some(sysconfigs.clone()); + osi.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + + // 把对 osinstance 的更改从内存更新到 k8s 集群 + let namespace = osi.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + self.controller_client.update_osinstance_spec(&osi.name(), &namespace, &osi.spec).await?; + } + } + + Ok(count >= limit) + } + + // 获取所在节点状态为空闲的 osinstance列表 + async fn get_idle_os_instances(&self, namespace: &str, limit: i64) -> Result, Error> { + + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + + // 获取所有 OSInstance 资源 + let all_osinstances: ObjectList = osi_api.list(&ListParams::default().limit(limit as u32)).await?; + + // 在客户端进行过滤，节点状态为 NODE_STATUS_IDLE + let osinstances: Vec = all_osinstances + .items + .into_iter() + .filter(|osi| osi.spec.nodestatus == NODE_STATUS_IDLE) + .collect(); + + Ok(osinstances) + } + + +} + +// 调用的函数的具体逻辑需要进一步完成 +pub async fn reconcile( + os: OS, + ctx: Context>, +) -> Result { + + // 初始化 operator_controller 和 os，从环境变量获取NODE_NAME + debug!("start reconcile"); + let operator_controller = ctx.get_ref(); + let os_cr: &OS = &os; + + // 从 os_cr 中获取命名空间，如果命名空间不存在则返回错误 + let namespace: String = os_cr + .namespace() + .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; + + debug!("namespace : {:?}", namespace); + + // 获取 worker 节点数 + let node_num = match operator_controller.get_and_update_os(&namespace).await { + Ok(node_num) => node_num, + Err(Error::KubeClient { source: kube::Error::Api(ErrorResponse { reason, .. })}) if &reason == "NotFound" => { + return Ok(NO_REQUEUE); + }, + Err(_) => return Ok(REQUEUE_ERROR), + }; + + debug!("node_num : {:?}", node_num); + + let opstype = os_cr.spec.opstype.clone(); + let ops = opstype.as_str(); + + debug!("opstype: {}", ops); + + match ops { + // 如果是升级或者回滚 + OPERATION_TYPE_UPGRADE | OPERATION_TYPE_ROLLBACK =>{ + debug!("start upgrade OR rollback"); + + // 获取可以进行升级操作的最大节点数量 + let limit = operator_controller.check_upgrading(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?; + + debug!("limit: {}", limit); + + // 为指定数量的节点进行升级操作 + let need_requeue = operator_controller.assign_upgrade(os_cr, limit, &namespace).await?; + if need_requeue { + return Ok(REQUEUE_NORMAL); + } + }, + // 配置操作 + OPERATION_TYPE_CONFIG =>{ + debug!("start config"); + + // 检查待配置的节点数量 + let limit = operator_controller.check_config(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?; + + debug!("limit: {}", limit); + + // 指派配置任务给节点 + let sys_configs = os_cr.spec.sysconfigs.clone().unwrap(); + let version = os_cr.spec.sysconfigs.clone().unwrap().version.unwrap(); + let need_requeue = operator_controller.assign_config(os_cr, sys_configs, version, limit, &namespace).await?; + + if need_requeue { + return Ok(REQUEUE_NORMAL); + } + }, + _ =>{ + log::error!("operation {} cannot be recognized", ops); + } + } + return Ok(REQUEUE_NORMAL); +} + +pub fn error_policy( + error: &Error, + _ctx: Context>, +) -> ReconcilerAction { + error!("Reconciliation error: {}", error.to_string()); + REQUEUE_ERROR +} + +pub mod reconciler_error { + use thiserror::Error; + + use crate::controller::{apiclient::apiclient_error}; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeClient { + #[from] + source: kube::Error, + }, + + #[error("Create/Patch OSInstance reported error: {source}")] + ApplyApi { + #[from] + source: apiclient_error::Error, + }, + + #[error("Cannot get environment NODE_NAME, error: {source}")] + Env { + #[from] + source: std::env::VarError, + }, + + #[error("{}.metadata.{} is not exist", resource, value)] + MissingObjectKey { resource: String, value: String }, + + // #[error("Cannot get {}, {} is None", value, value)] + // MissingSubResource { value: String }, + + #[error("operation {} cannot be recognized", value)] + Operation { value: String }, + + // #[error("Expect OS Version is not same with Node OS Version, please upgrade first")] + // UpgradeBeforeConfig, + + // #[error("Error when drain node, error reported: {}", value)] + // DrainNode { value: String }, + } +} + +#[cfg(test)] +mod test { + use std::{borrow::Borrow, cell::RefCell, env}; + + use serde::de; + + use super::{error_policy, reconcile, reconciler_error::Error, Context, OSInstance, OperatorController, OS}; + use crate::controller::{ + apiserver_mock::{timeout_after_5s, K8sResources, Testcases}, + ControllerClient, + }; + + #[tokio::test] + async fn test_rollback() { + env::set_var("RUST_LOG", "info"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_rollback_osversion_v1_upgradecon_v1(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::Rollback(K8sResources::set_rollback_nodes_v2_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_config_normal() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::ConfigNormal(K8sResources::set_nodes_v1_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_skip_no_osi_node() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_skip_osversion_v2_upgradecon_v1(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::SkipNoOsiNode(K8sResources::set_skip_nodes_and_osi())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_exchange_current_and_next() { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let os = OS::set_os_exchange_current_and_next(); + let context = Context::new(test_operator_controller); + let mocksrv = fakeserver + .run(Testcases::ExchangeCurrentAndNext(K8sResources::set_nodes_v1_and_osi_v1())); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_deep_copy_spec_configs() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + let deep_copy_result = test_operator_controller.clone().deep_copy_spec_configs(&OS::set_os_default(), &mut OSInstance::set_osi_default("", ""), "test".to_string()).await; + + assert!(deep_copy_result.is_err()); + + if let Err(err) = deep_copy_result { + assert_eq!("operation test cannot be recognized".to_string(), err.borrow().to_string()); + } + } + + #[tokio::test] + async fn test_get_config_osinstances() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + let expected_error = "list error".to_string(); + + fakeserver.test_function(Testcases::GetConfigOSInstances(expected_error.clone())); + + // 执行测试 + let result = test_operator_controller.get_config_osinstances("default").await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!(expected_error, error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + + #[tokio::test] + async fn test_check_upgrading() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + fakeserver.test_function(Testcases::CheckUpgrading("label error".to_string())); + + // 执行测试 + let result = test_operator_controller.check_upgrading("default", 2).await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!("label error", error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + + + #[tokio::test] + async fn test_get_idle_osinstances() { + env_logger::init(); + + let (test_operator_controller, fakeserver) = OperatorController::::test(); + + let expected_error = "list error".to_string(); + + fakeserver.test_function(Testcases::GetIdleOSInstances(expected_error.clone())); + + // 执行测试 + let result = test_operator_controller.get_idle_os_instances("default", 3).await; + + // 验证返回值 + assert!(result.is_err()); + if let Err(err) = result { + match err { + Error::KubeClient { source } => { + match source { + kube::Error::Api(error_response) => { + assert_eq!(expected_error, error_response.message); + }, + _ => { + assert!(false); + } + } + } + _ => { + assert!(false); + } + } + } + } + +} diff --git a/KubeOS-Rust/operator/src/controller/crd.rs b/KubeOS-Rust/operator/src/controller/crd.rs new file mode 100644 index 00000000..1fa63035 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/crd.rs @@ -0,0 +1,79 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)] +pub struct OSSpec { + pub osversion: String, + pub maxunavailable: i64, + pub checksum: String, + pub imagetype: String, + pub containerimage: String, + pub opstype: String, + pub evictpodforce: bool, + pub imageurl: String, + #[serde(rename = "flagSafe")] + pub flagsafe: bool, + pub mtls: bool, + pub cacert: Option, + pub clientcert: Option, + pub clientkey: Option, + #[serde(rename = "sysconfigs")] + pub sysconfigs: Option, + #[serde(rename = "upgradeconfigs")] + pub upgradeconfigs: Option, +} + +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube( + group = "upgrade.openeuler.org", + version = "v1alpha1", + kind = "OSInstance", + plural = "osinstances", + singular = "osinstance", + status = "OSInstanceStatus", + namespaced +)] +pub struct OSInstanceSpec { + pub nodestatus: String, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct OSInstanceStatus { + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Configs { + pub version: Option, + pub configs: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Config { + pub model: Option, + pub configpath: Option, + pub contents: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Content { + pub key: Option, + pub value: Option, + pub operation: Option, +} diff --git a/KubeOS-Rust/operator/src/controller/mod.rs b/KubeOS-Rust/operator/src/controller/mod.rs new file mode 100644 index 00000000..468ebfdb --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/mod.rs @@ -0,0 +1,24 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + + mod apiclient; + #[cfg(test)] + mod apiserver_mock; + mod controller; + mod crd; + mod values; + +pub use apiclient::ControllerClient; +pub use controller::{error_policy, reconcile, OperatorController}; +pub use crd::OS; + \ No newline at end of file diff --git a/KubeOS-Rust/operator/src/controller/values.rs b/KubeOS-Rust/operator/src/controller/values.rs new file mode 100644 index 00000000..267270a9 --- /dev/null +++ b/KubeOS-Rust/operator/src/controller/values.rs @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::runtime::controller::ReconcilerAction; +use tokio::time::Duration; + +#[cfg(test)] +pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; + +pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; + +pub const LABEL_MASTER: &str = "node-role.kubernetes.io/control-plane"; + +pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; +pub const OSINSTANCE_KIND: &str = "OSInstance"; +// pub const OSI_STATUS_NAME: &str = "nodestatus"; + +pub const UPGRADE_CONFIG_NAME: &str = "UpgradeConfig"; +pub const SYS_CONFIG_NAME: &str = "SysConfig"; + +pub const NODE_STATUS_IDLE: &str = "idle"; +pub const NODE_STATUS_UPGRADE: &str = "upgrade"; +pub const NODE_STATUS_CONFIG: &str = "config"; + +pub const OPERATION_TYPE_UPGRADE: &str = "upgrade"; +pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; +pub const OPERATION_TYPE_CONFIG: &str = "config"; + + +pub const NO_REQUEUE: ReconcilerAction = ReconcilerAction { requeue_after: None }; + +pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; + +pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; diff --git a/KubeOS-Rust/operator/src/main.rs b/KubeOS-Rust/operator/src/main.rs new file mode 100644 index 00000000..51c91618 --- /dev/null +++ b/KubeOS-Rust/operator/src/main.rs @@ -0,0 +1,74 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + use anyhow::Result; + use env_logger::{Builder, Env, Target}; + use futures::StreamExt; + use kube::{ + api::{Api, ListParams}, + client::Client, + runtime::controller::{Context, Controller}, + }; + use log::{error, info}; + // use std::sync::Arc; + use tokio::signal; + + mod controller; + use controller::{ + error_policy, reconcile, ControllerClient, OperatorController, OS, + }; + + const OPERATOR_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); + + + #[tokio::main] + async fn main() -> Result<()> { + // 初始化日志记录器，默认日志级别为info，输出到标准输出 + Builder::from_env(Env::default().default_filter_or("debug")).target(Target::Stdout).init(); + + // 创建一个默认的Kubernetes客户端 + let client = Client::try_default().await?; + + // 创建一个用于操作OS资源的API对象，自定义的CR + let os: Api = Api::all(client.clone()); + + // 创建一个控制器客户端 + let controller_client = ControllerClient::new(client.clone()); + // let controller_client = Arc::new(ControllerClient::new(client.clone())); + + // 创建一个OSReconciler实例 + let os_reconciler = OperatorController::new(client.clone(), controller_client.clone()); + + // 记录操作符版本和启动信息 + info!( + "os-operator version is {}, starting operator manager", + OPERATOR_VERSION.unwrap_or("Not Found") + ); + + // 创建一个新的控制器并运行，处理reconcile和error_policy，并记录错误信息 + Controller::new(os, ListParams::default()) + .run(reconcile, error_policy, Context::new(os_reconciler)) + .for_each(|res| async move { + match res { + Ok(_) => {} + Err(e) => error!("reconcile failed: {}", e.to_string()), + } + }) + .await; + + // 等待终止信号 + signal::ctrl_c().await?; + info!("os-operator terminated"); + + Ok(()) + } + \ No newline at end of file diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml new file mode 100644 index 00000000..d804ac77 --- /dev/null +++ b/KubeOS-Rust/proxy/Cargo.toml @@ -0,0 +1,49 @@ +[package] +description = "KubeOS os-proxy" +edition = "2021" +license = "MulanPSL-2.0" +name = "proxy" +version = "1.0.6" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "drain" +path = "src/drain.rs" + +[[bin]] +name = "proxy" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.44" +async-trait = "0.1" +cli = { version = "1.0.6", path = "../cli" } +env_logger = "0.9.0" +futures = "0.3.17" +h2 = "=0.3.16" +k8s-openapi = { version = "0.13.1", features = ["v1_22"] } +kube = { version = "0.66.0", features = ["derive", "runtime"] } +log = "=0.4.15" +manager = { version = "1.0.6", path = "../manager" } +regex = "=1.7.3" +reqwest = { version = "=0.12.2", default-features = false, features = [ + "json", +] } +schemars = "=0.8.10" +serde = { version = "1.0.130", features = ["derive"] } +serde_json = "1.0.68" +socket2 = "=0.4.9" +thiserror = "1.0.29" +thread_local = "=1.1.4" +tokio = { version = "=1.28.0", default-features = false, features = [ + "macros", + "rt-multi-thread", +] } +tokio-retry = "0.3" + +[dev-dependencies] +assert-json-diff = "2.0.2" +http = "0.2.9" +hyper = "0.14.25" +tower-test = "0.4.0" +mockall = { version = "=0.11.3" } diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs new file mode 100644 index 00000000..b833f276 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs @@ -0,0 +1,153 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{collections::HashMap, path::Path}; + +use agent_error::Error; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use manager::api::{CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest}; + +pub struct UpgradeInfo { + pub version: String, + pub image_type: String, + pub check_sum: String, + pub container_image: String, + pub imageurl: String, + pub flagsafe: bool, + pub mtls: bool, + pub cacert: String, + pub clientcert: String, + pub clientkey: String, +} + +pub struct ConfigInfo { + pub configs: Vec, +} + +pub struct Sysconfig { + pub model: String, + pub config_path: String, + pub contents: HashMap, +} + +pub struct KeyInfo { + pub value: String, + pub operation: String, +} + +pub trait AgentMethod { + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error>; + fn upgrade_method(&self) -> Result<(), Error>; + fn rollback_method(&self) -> Result<(), Error>; + fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error>; +} +pub trait AgentCall { + fn call_agent(&self, client: &Client, method: T) -> Result<(), Error>; +} + +pub struct AgentClient { + pub agent_client: Client, + pub agent_call_client: T, +} + +impl AgentClient { + pub fn new>(socket_path: P, agent_call_client: T) -> Self { + AgentClient { agent_client: Client::new(socket_path), agent_call_client } + } +} + +#[derive(Default)] +pub struct AgentCallClient {} +impl AgentCall for AgentCallClient { + fn call_agent(&self, client: &Client, method: T) -> Result<(), Error> { + match method.call(client) { + Ok(_resp) => Ok(()), + Err(e) => Err(Error::AgentError { source: e }), + } + } +} + +impl AgentMethod for AgentClient { + fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error> { + let upgrade_request = UpgradeRequest { + version: upgrade_info.version, + image_type: upgrade_info.image_type, + check_sum: upgrade_info.check_sum, + container_image: upgrade_info.container_image, + image_url: upgrade_info.imageurl, + flag_safe: upgrade_info.flagsafe, + mtls: upgrade_info.mtls, + certs: CertsInfo { + ca_cert: upgrade_info.cacert, + client_cert: upgrade_info.clientcert, + client_key: upgrade_info.clientkey, + }, + }; + match self.agent_call_client.call_agent(&self.agent_client, PrepareUpgradeMethod::new(upgrade_request)) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn upgrade_method(&self) -> Result<(), Error> { + match self.agent_call_client.call_agent(&self.agent_client, UpgradeMethod::default()) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn rollback_method(&self) -> Result<(), Error> { + match self.agent_call_client.call_agent(&self.agent_client, RollbackMethod::default()) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } + + fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error> { + let mut agent_configs: Vec = Vec::new(); + for config in config_info.configs { + let mut contents_tmp: HashMap = HashMap::new(); + for (key, key_info) in config.contents.iter() { + contents_tmp.insert( + key.to_string(), + AgentKeyInfo { value: key_info.value.clone(), operation: key_info.operation.clone() }, + ); + } + agent_configs.push(AgentSysconfig { + model: config.model, + config_path: config.config_path, + contents: contents_tmp, + }) + } + let config_request = ConfigureRequest { configs: agent_configs }; + match self.agent_call_client.call_agent(&self.agent_client, ConfigureMethod::new(config_request)) { + Ok(_resp) => Ok(()), + Err(e) => Err(e), + } + } +} + +pub mod agent_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("{source}")] + AgentError { source: anyhow::Error }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/apiclient.rs b/KubeOS-Rust/proxy/src/controller/apiclient.rs new file mode 100644 index 00000000..3afd5a51 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/apiclient.rs @@ -0,0 +1,147 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::BTreeMap; + +use anyhow::Result; +use apiclient_error::Error; +use async_trait::async_trait; +use kube::{ + api::{Api, ObjectMeta, Patch, PatchParams, PostParams}, + Client, +}; +use serde::{Deserialize, Serialize}; + +use super::{ + crd::{OSInstance, OSInstanceSpec, OSInstanceStatus}, + values::{LABEL_OSINSTANCE, NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND}, +}; + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceSpecPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + spec: OSInstanceSpec, +} + +impl Default for OSInstanceSpecPatch { + fn default() -> Self { + OSInstanceSpecPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct OSInstanceStatusPatch { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + status: Option, +} + +impl Default for OSInstanceStatusPatch { + fn default() -> Self { + OSInstanceStatusPatch { + api_version: OSINSTANCE_API_VERSION.to_string(), + kind: OSINSTANCE_KIND.to_string(), + status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }), + } + } +} + +#[derive(Clone)] +pub struct ControllerClient { + pub client: Client, +} + +impl ControllerClient { + pub fn new(client: Client) -> Self { + ControllerClient { client } + } +} + +#[async_trait] +pub trait ApplyApi: Clone + Sized + Send + Sync { + async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error>; + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error>; + async fn update_osinstance_status( + &self, + node_name: &str, + namespace: &str, + status: &Option, + ) -> Result<(), Error>; +} + +#[async_trait] +impl ApplyApi for ControllerClient { + async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error> { + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + let osinstance = OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None }, + status: None, + }; + let osi_api = Api::namespaced(self.client.clone(), namespace); + osi_api.create(&PostParams::default(), &osinstance).await?; + Ok(()) + } + + async fn update_osinstance_spec( + &self, + node_name: &str, + namespace: &str, + spec: &OSInstanceSpec, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() }; + osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?; + Ok(()) + } + + async fn update_osinstance_status( + &self, + node_name: &str, + namespace: &str, + status: &Option, + ) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.client.clone(), namespace); + let osi_status_patch = OSInstanceStatusPatch { status: status.clone(), ..Default::default() }; + osi_api.patch_status(node_name, &PatchParams::default(), &Patch::Merge(&osi_status_patch)).await?; + Ok(()) + } +} +pub mod apiclient_error { + use thiserror::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs new file mode 100644 index 00000000..2b182ca8 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs @@ -0,0 +1,681 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::collections::BTreeMap; + +use anyhow::Result; +use cli::{ + client::Client, + method::{ + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, + rollback::RollbackMethod, upgrade::UpgradeMethod, + }, +}; +use http::{Request, Response}; +use hyper::{body::to_bytes, Body}; +use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod}; +use kube::{ + api::ObjectMeta, + core::{ListMeta, ObjectList}, + Client as KubeClient, Resource, ResourceExt, +}; +use mockall::mock; + +use self::mock_error::Error; +use super::{ + agentclient::*, + crd::{Configs, OSInstanceStatus}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK}, +}; +use crate::controller::{ + apiclient::{ApplyApi, ControllerClient}, + crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS}, + values::{LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE}, + ProxyController, +}; + +type ApiServerHandle = tower_test::mock::Handle, Response>; +pub struct ApiServerVerifier(ApiServerHandle); + +pub enum Testcases { + OSInstanceNotExist(OSInstance), + UpgradeNormal(OSInstance), + UpgradeUpgradeconfigsVersionMismatch(OSInstance), + UpgradeOSInstaceNodestatusConfig(OSInstance), + UpgradeOSInstaceNodestatusIdle(OSInstance), + ConfigNormal(OSInstance), + ConfigVersionMismatchReassign(OSInstance), + ConfigVersionMismatchUpdate(OSInstance), + Rollback(OSInstance), +} + +pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) { + tokio::time::timeout(std::time::Duration::from_secs(5), handle) + .await + .expect("timeout on mock apiserver") + .expect("scenario succeeded") +} + +impl ApiServerVerifier { + pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + match cases { + Testcases::OSInstanceNotExist(osi) => { + self.handler_osinstance_get_not_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_creation(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi) + .await + }, + Testcases::UpgradeNormal(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_upgradeconfig_v2(osi.clone()) + .await + .unwrap() + .handler_node_cordon(osi.clone()) + .await + .unwrap() + .handler_node_pod_list_get(osi) + .await + }, + Testcases::UpgradeUpgradeconfigsVersionMismatch(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_node_update_delete_label(osi.clone()) + .await + .unwrap() + .handler_node_uncordon(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::UpgradeOSInstaceNodestatusConfig(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + }, + Testcases::UpgradeOSInstaceNodestatusIdle(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_node_update_delete_label(osi.clone()) + .await + .unwrap() + .handler_node_uncordon(osi) + .await + }, + Testcases::ConfigNormal(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_sysconfig_v2(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::ConfigVersionMismatchReassign(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_nodestatus_idle(osi) + .await + }, + Testcases::ConfigVersionMismatchUpdate(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_spec_sysconfig_v2(osi) + .await + }, + Testcases::Rollback(osi) => { + self.handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_osinstance_get_exist(osi.clone()) + .await + .unwrap() + .handler_node_get_with_label(osi.clone()) + .await + .unwrap() + .handler_osinstance_patch_upgradeconfig_v2(osi.clone()) + .await + .unwrap() + .handler_node_cordon(osi.clone()) + .await + .unwrap() + .handler_node_pod_list_get(osi) + .await + }, + } + .expect("Case completed without errors"); + }) + } + + async fn handler_osinstance_get_not_exist(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name()) + ); + let response_json = serde_json::json!( + { "status": "Failure", "message": "osinstances.upgrade.openeuler.org \"openeuler\" not found", "reason": "NotFound", "code": 404 } + ); + dbg!("handler_osinstance_get_not_exist"); + let response = serde_json::to_vec(&response_json).unwrap(); + send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap()); + Ok(self) + } + async fn handler_osinstance_get_exist(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name()) + ); + dbg!("handler_osinstance_get_exist"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + async fn handler_osinstance_creation(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::POST); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?") + ); + dbg!("handler_osinstance_creation"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_nodestatus_idle(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + assert_eq!(spec.nodestatus.clone(), NODE_STATUS_IDLE.to_string()); + + dbg!("handler_osinstance_patch_nodestatus_idle"); + osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_upgradeconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?", + osinstance.name() + ) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime"); + let status_json = body_json.get("status").expect("status object").clone(); + let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!( + status.upgradeconfigs.expect("upgradeconfigs is not None").clone(), + osinstance.spec.clone().upgradeconfigs.expect("upgradeconfig is not None") + ); + + osinstance.status.as_mut().unwrap().upgradeconfigs = osinstance.spec.upgradeconfigs.clone(); + + dbg!("handler_osinstance_patch_upgradeconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!( + "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?", + osinstance.name() + ) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance"); + let status_json = body_json.get("status").expect("status object").clone(); + let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status"); + + assert_eq!( + status.sysconfigs.expect("sysconfigs is not None").clone(), + osinstance.spec.clone().sysconfigs.expect("sysconfig is not None") + ); + + osinstance.status.as_mut().unwrap().sysconfigs = osinstance.spec.sysconfigs.clone(); + + dbg!("handler_osinstance_patch_sysconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_osinstance_patch_spec_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!( + request.uri().to_string(), + format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name()) + ); + + let req_body = to_bytes(request.into_body()).await.unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance"); + let spec_json = body_json.get("spec").expect("spec object").clone(); + let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec"); + + assert_eq!( + spec.sysconfigs.expect("upgradeconfigs is not None").clone().version.clone().unwrap(), + String::from("v2") + ); + + osinstance.spec.sysconfigs.as_mut().unwrap().version = Some(String::from("v2")); + + dbg!("handler_osinstance_patch_spec_sysconfig_v2"); + let response = serde_json::to_vec(&osinstance).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_get(mut self, osinstance: OSInstance) -> Result { + // return node with name = openeuler, osimage = KubeOS v1，no upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name())); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + assert_eq!(node.name(), String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1")); + dbg!("handler_node_get"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_get_with_label(mut self, osinstance: OSInstance) -> Result { + // return node with name = openeuler, osimage = KubeOS v1，has upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name())); + let mut node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: None, + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + let node_labels = node.labels_mut(); + node_labels.insert(LABEL_UPGRADING.to_string(), "".to_string()); + assert_eq!(node.name(), String::from("openeuler")); + assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1")); + assert!(node.labels().contains_key(LABEL_UPGRADING)); + dbg!("handler_node_get_with_label"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_update_delete_label(mut self, osinstance: OSInstance) -> Result { + // return node with name = openeuler, osimage = KubeOS v1，no upgrade label + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PUT); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + // check request body has upgrade label + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + dbg!("handler_node_update_delete_label"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_cordon(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + assert_eq!(request.extensions().get(), Some(&"cordon")); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + dbg!("handler_node_cordon"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_uncordon(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::PATCH); + assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name())); + assert_eq!(request.extensions().get(), Some(&"cordon")); + let node = Node { + metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() }, + spec: Some(NodeSpec { unschedulable: Some(false), ..Default::default() }), + status: Some(NodeStatus { + node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }), + ..Default::default() + }), + }; + dbg!("handler_node_uncordon"); + let response = serde_json::to_vec(&node.clone()).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } + + async fn handler_node_pod_list_get(mut self, osinstance: OSInstance) -> Result { + let (request, send) = self.0.next_request().await.expect("service not called"); + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().to_string(), + format!("/api/v1/pods?&fieldSelector=spec.nodeName%3D{}", osinstance.name()) + ); + assert_eq!(request.extensions().get(), Some(&"list")); + let pods_list = ObjectList:: { metadata: ListMeta::default(), items: vec![] }; + dbg!("handler_node_pod_list_get"); + let response = serde_json::to_vec(&pods_list).unwrap(); + send.send_response(Response::builder().body(Body::from(response)).unwrap()); + Ok(self) + } +} + +pub mod mock_error { + use thiserror::Error; + + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeError { + #[from] + source: kube::Error, + }, + } +} + +mock! { + pub AgentCallClient{} + impl AgentCall for AgentCallClient{ + fn call_agent(&self, client:&Client, method: T) -> Result<(), agent_error::Error> { + Ok(()) + } + } + +} +impl ProxyController { + pub fn test() -> (ProxyController, ApiServerVerifier) { + let (mock_service, handle) = tower_test::mock::pair::, Response>(); + let mock_k8s_client = KubeClient::new(mock_service, "default"); + let mock_api_client = ControllerClient::new(mock_k8s_client.clone()); + let mut mock_agent_call_client = MockAgentCallClient::new(); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + mock_agent_call_client.expect_call_agent::().returning(|_x, _y| Ok(())); + let mock_agent_client = AgentClient::new("test", mock_agent_call_client); + let proxy_controller: ProxyController = + ProxyController::new(mock_k8s_client, mock_api_client, mock_agent_client); + (proxy_controller, ApiServerVerifier(handle)) + } +} + +impl OSInstance { + pub fn set_osi_default(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut labels = BTreeMap::new(); + labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string()); + OSInstance { + metadata: ObjectMeta { + name: Some(node_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..ObjectMeta::default() + }, + spec: OSInstanceSpec { + nodestatus: NODE_STATUS_IDLE.to_string(), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }, + status: Some(OSInstanceStatus { + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + }), + } + } + + pub fn set_osi_nodestatus_upgrade(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + osinstance + } + + pub fn set_osi_nodestatus_config(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = config, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance + } + + pub fn set_osi_upgradecon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.upgradeconfigs.as_mut().unwrap().version = Some(String::from("v2")); + osinstance + } + + pub fn set_osi_nodestatus_upgrade_upgradecon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string(); + osinstance.spec.upgradeconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + osinstance + } + + pub fn set_osi_nodestatus_config_syscon_v2(node_name: &str, namespace: &str) -> Self { + // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1 + let mut osinstance = OSInstance::set_osi_default(node_name, namespace); + osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string(); + osinstance.spec.sysconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + osinstance + } +} + +impl OS { + pub fn set_os_default() -> Self { + let mut os = OS::new("test", OSSpec::default()); + os.meta_mut().namespace = Some("default".into()); + os + } + + pub fn set_os_osversion_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.opstype = String::from("config"); + os + } + + pub fn set_os_osversion_v2_upgradecon_v2() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.upgradeconfigs = Some(Configs { version: Some(String::from("v2")), configs: None }); + os + } + + pub fn set_os_syscon_v2_opstype_config() -> Self { + let mut os = OS::set_os_default(); + os.spec.opstype = String::from("config"); + os.spec.sysconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + os + } + + pub fn set_os_rollback_osversion_v2_upgradecon_v2() -> Self { + let mut os = OS::set_os_default(); + os.spec.osversion = String::from("KubeOS v2"); + os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string(); + os.spec.upgradeconfigs = Some(Configs { + version: Some(String::from("v2")), + configs: Some(vec![Config { + model: Some(String::from("kernel.sysctl.persist")), + configpath: Some(String::from("/persist/persist.conf")), + contents: Some(vec![Content { + key: Some(String::from("kernel.test")), + value: Some(String::from("test")), + operation: Some(String::from("delete")), + }]), + }]), + }); + os + } +} + +impl Default for OSSpec { + fn default() -> Self { + OSSpec { + osversion: String::from("KubeOS v1"), + maxunavailable: 2, + checksum: String::from("test"), + imagetype: String::from("containerd"), + containerimage: String::from("test"), + opstype: String::from("upgrade"), + evictpodforce: true, + imageurl: String::from(""), + flagsafe: false, + mtls: false, + cacert: Some(String::from("")), + clientcert: Some(String::from("")), + clientkey: Some(String::from("")), + sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }), + } + } +} diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs new file mode 100644 index 00000000..80a85d1c --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -0,0 +1,556 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{collections::HashMap, env}; + +use anyhow::Result; +use drain::drain_os; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::{Api, PostParams}, + core::ErrorResponse, + runtime::controller::{Context, ReconcilerAction}, + Client, ResourceExt, +}; +use log::{debug, error, info}; +use reconciler_error::Error; + +use super::{ + agentclient::{AgentCall, AgentClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo}, + apiclient::ApplyApi, + crd::{Configs, Content, OSInstance, OS}, + utils::{check_version, get_config_version, ConfigOperation, ConfigType}, + values::{ + LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, + REQUEUE_ERROR, REQUEUE_NORMAL, + }, +}; + +pub async fn reconcile( + os: OS, + ctx: Context>, +) -> Result { + debug!("start reconcile"); + let proxy_controller = ctx.get_ref(); + let os_cr = &os; + let node_name = env::var("NODE_NAME")?; + let namespace: String = os_cr + .namespace() + .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?; + proxy_controller.check_osi_exisit(&namespace, &node_name).await?; + let controller_res = proxy_controller.get_resources(&namespace, &node_name).await?; + let node = controller_res.node; + let mut osinstance = controller_res.osinstance; + let node_os_image = &node + .status + .as_ref() + .ok_or(Error::MissingSubResource { value: String::from("node.status") })? + .node_info + .as_ref() + .ok_or(Error::MissingSubResource { value: String::from("node.status.node_info") })? + .os_image; + debug!("os expected osversion is {},actual osversion is {}", os_cr.spec.osversion, node_os_image); + if check_version(&os_cr.spec.osversion, node_os_image) { + match ConfigType::SysConfig.check_config_version(&os, &osinstance) { + ConfigOperation::Reassign => { + debug!("start reassign"); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.sysconfigs.as_ref()), + ConfigType::SysConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + }, + ConfigOperation::UpdateConfig => { + debug!("start update config"); + osinstance.spec.sysconfigs = os_cr.spec.sysconfigs.clone(); + proxy_controller + .controller_client + .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec) + .await?; + return Ok(REQUEUE_ERROR); + }, + _ => {}, + } + proxy_controller.set_config(&mut osinstance, ConfigType::SysConfig).await?; + proxy_controller + .refresh_node(node, osinstance, &get_config_version(os_cr.spec.sysconfigs.as_ref()), ConfigType::SysConfig) + .await?; + } else { + if os_cr.spec.opstype == NODE_STATUS_CONFIG { + return Err(Error::UpgradeBeforeConfig); + } + if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) { + debug!("start reassign"); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + if node.labels().contains_key(LABEL_UPGRADING) { + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + info!( + "node has upgrade label ,but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin" + ); + proxy_controller + .refresh_node( + node, + osinstance, + &get_config_version(os_cr.spec.upgradeconfigs.as_ref()), + ConfigType::UpgradeConfig, + ) + .await?; + return Ok(REQUEUE_NORMAL); + } + proxy_controller.set_config(&mut osinstance, ConfigType::UpgradeConfig).await?; + proxy_controller.upgrade_node(os_cr, &node).await?; + } + } + Ok(REQUEUE_NORMAL) +} + +pub fn error_policy( + error: &Error, + _ctx: Context>, +) -> ReconcilerAction { + error!("Reconciliation error:{}", error.to_string()); + REQUEUE_ERROR +} + +struct ControllerResources { + osinstance: OSInstance, + node: Node, +} +pub struct ProxyController { + k8s_client: Client, + controller_client: T, + agent_client: AgentClient, +} + +impl ProxyController { + pub fn new(k8s_client: Client, controller_client: T, agent_client: AgentClient) -> Self { + ProxyController { k8s_client, controller_client, agent_client } + } +} + +impl ProxyController { + async fn check_osi_exisit(&self, namespace: &str, node_name: &str) -> Result<(), Error> { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + match osi_api.get(node_name).await { + Ok(osi) => { + debug!("osinstance is exist {:?}", osi.name()); + Ok(()) + }, + Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => { + info!("Create OSInstance {}", node_name); + self.controller_client.create_osinstance(node_name, namespace).await?; + Ok(()) + }, + Err(err) => Err(Error::KubeClient { source: err }), + } + } + + async fn get_resources(&self, namespace: &str, node_name: &str) -> Result { + let osi_api: Api = Api::namespaced(self.k8s_client.clone(), namespace); + let osinstance_cr = osi_api.get(node_name).await?; + let node_api: Api = Api::all(self.k8s_client.clone()); + let node_cr = node_api.get(node_name).await?; + Ok(ControllerResources { osinstance: osinstance_cr, node: node_cr }) + } + + async fn refresh_node( + &self, + mut node: Node, + osinstance: OSInstance, + os_config_version: &str, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start refresh_node"); + let node_api: Api = Api::all(self.k8s_client.clone()); + let labels = node.labels_mut(); + if labels.contains_key(LABEL_UPGRADING) { + labels.remove(LABEL_UPGRADING); + node = node_api.replace(&node.name(), &PostParams::default(), &node).await?; + } + if let Some(node_spec) = &node.spec { + if let Some(node_unschedulable) = node_spec.unschedulable { + if node_unschedulable { + node_api.uncordon(&node.name()).await?; + info!("Uncordon successfully node{}", node.name()); + } + } + } + self.update_node_status(osinstance, os_config_version, config_type).await?; + Ok(()) + } + + async fn update_node_status( + &self, + mut osinstance: OSInstance, + os_config_version: &str, + config_type: ConfigType, + ) -> Result<(), Error> { + debug!("start update_node_status"); + if osinstance.spec.nodestatus == NODE_STATUS_IDLE { + return Ok(()); + } + let upgradeconfig_spec_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + let sysconfig_spec_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + let sysconfig_status_version: String; + if let Some(osinstance_status) = osinstance.status.as_ref() { + sysconfig_status_version = get_config_version(osinstance_status.sysconfigs.as_ref()); + } else { + sysconfig_status_version = get_config_version(None); + } + if sysconfig_spec_version == sysconfig_status_version + || (config_type == ConfigType::SysConfig && os_config_version != sysconfig_spec_version) + || (config_type == ConfigType::UpgradeConfig && os_config_version != upgradeconfig_spec_version) + { + let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey { + resource: String::from("osinstance"), + value: String::from("namespace"), + })?; + osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string(); + self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?; + } + Ok(()) + } + + async fn update_osi_status(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> { + debug!("start update_osi_status"); + config_type.set_osi_status_config(osinstance); + debug!("osinstance status is update to {:?}", osinstance.status); + let namespace = &osinstance + .namespace() + .ok_or(Error::MissingObjectKey { resource: "osinstance".to_string(), value: "namespace".to_string() })?; + self.controller_client.update_osinstance_status(&osinstance.name(), namespace, &osinstance.status).await?; + Ok(()) + } + + async fn set_config(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> { + debug!("start set_config"); + let config_info = config_type.check_config_start(osinstance); + if config_info.need_config { + match config_info.configs.and_then(convert_to_agent_config) { + Some(agent_configs) => { + match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }) { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + }, + None => { + info!("config is none, No content can be configured."); + }, + }; + self.update_osi_status(osinstance, config_type).await?; + } + Ok(()) + } + + async fn upgrade_node(&self, os_cr: &OS, node: &Node) -> Result<(), Error> { + debug!("start upgrade node"); + match os_cr.spec.opstype.as_str() { + OPERATION_TYPE_UPGRADE => { + let upgrade_info = UpgradeInfo { + version: os_cr.spec.osversion.clone(), + image_type: os_cr.spec.imagetype.clone(), + check_sum: os_cr.spec.checksum.clone(), + container_image: os_cr.spec.containerimage.clone(), + flagsafe: os_cr.spec.flagsafe, + imageurl: os_cr.spec.imageurl.clone(), + mtls: os_cr.spec.mtls, + cacert: os_cr.spec.cacert.clone().unwrap_or_default(), + clientcert: os_cr.spec.clientcert.clone().unwrap_or_default(), + clientkey: os_cr.spec.clientkey.clone().unwrap_or_default(), + }; + + match self.agent_client.prepare_upgrade_method(upgrade_info) { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; + match self.agent_client.upgrade_method() { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + }, + OPERATION_TYPE_ROLLBACK => { + self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?; + + match self.agent_client.rollback_method() { + Ok(_resp) => {}, + Err(e) => { + return Err(Error::Agent { source: e }); + }, + } + }, + _ => { + return Err(Error::Operation { value: os_cr.spec.opstype.clone() }); + }, + } + Ok(()) + } + + async fn evict_node(&self, node_name: &str, evict_pod_force: bool) -> Result<(), Error> { + debug!("start evict_node"); + let node_api = Api::all(self.k8s_client.clone()); + node_api.cordon(node_name).await?; + info!("Cordon node Successfully{}, start drain nodes", node_name); + match self.drain_node(node_name, evict_pod_force).await { + Ok(()) => {}, + Err(e) => { + node_api.uncordon(node_name).await?; + info!("Drain node {} error, uncordon node successfully", node_name); + return Err(e); + }, + } + Ok(()) + } + + async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> { + use drain::error::DrainError::*; + match drain_os(&self.k8s_client.clone(), node_name, force).await { + Err(DeletePodsError { errors, .. }) => Err(Error::DrainNode { value: errors.join("; ") }), + _ => Ok(()), + } + } +} + +fn convert_to_agent_config(configs: Configs) -> Option> { + let mut agent_configs: Vec = Vec::new(); + if let Some(config_list) = configs.configs { + for config in config_list.into_iter() { + match config.contents.and_then(convert_to_config_hashmap) { + Some(contents_tmp) => { + let config_tmp = Sysconfig { + model: config.model.unwrap_or_default(), + config_path: config.configpath.unwrap_or_default(), + contents: contents_tmp, + }; + agent_configs.push(config_tmp) + }, + None => { + info!( + "model {} which has configpath {} do not has any contents no need to configure", + config.model.unwrap_or_default(), + config.configpath.unwrap_or_default() + ); + continue; + }, + }; + } + if agent_configs.is_empty() { + info!("no contents in all models, no need to configure"); + return None; + } + return Some(agent_configs); + } + None +} + +fn convert_to_config_hashmap(contents: Vec) -> Option> { + let mut contents_tmp: HashMap = HashMap::new(); + for content in contents.into_iter() { + let key_info = + KeyInfo { value: content.value.unwrap_or_default(), operation: content.operation.unwrap_or_default() }; + contents_tmp.insert(content.key.unwrap_or_default(), key_info); + } + Some(contents_tmp) +} + +pub mod reconciler_error { + use thiserror::Error; + + use crate::controller::{agentclient::agent_error, apiclient::apiclient_error}; + #[derive(Error, Debug)] + pub enum Error { + #[error("Kubernetes reported error: {source}")] + KubeClient { + #[from] + source: kube::Error, + }, + + #[error("Create/Patch OSInstance reported error: {source}")] + ApplyApi { + #[from] + source: apiclient_error::Error, + }, + + #[error("Cannot get environment NODE_NAME, error: {source}")] + Env { + #[from] + source: std::env::VarError, + }, + + #[error("{}.metadata.{} is not exist", resource, value)] + MissingObjectKey { resource: String, value: String }, + + #[error("Cannot get {}, {} is None", value, value)] + MissingSubResource { value: String }, + + #[error("operation {} cannot be recognized", value)] + Operation { value: String }, + + #[error("Expect OS Version is not same with Node OS Version, please upgrade first")] + UpgradeBeforeConfig, + + #[error("os-agent reported error:{source}")] + Agent { source: agent_error::Error }, + + #[error("Error when drain node, error reported: {}", value)] + DrainNode { value: String }, + } +} + +#[cfg(test)] +mod test { + use std::env; + + use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS}; + use crate::controller::{ + apiserver_mock::{timeout_after_5s, MockAgentCallClient, Testcases}, + ControllerClient, + }; + + #[tokio::test] + async fn test_create_osinstance_with_no_upgrade_or_configuration() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_default(); + let context = Context::new(test_proxy_controller); + let mocksrv = + fakeserver.run(Testcases::OSInstanceNotExist(OSInstance::set_osi_default("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + #[tokio::test] + async fn test_upgrade_normal() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeNormal(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_diff_osversion_opstype_config() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeOSInstaceNodestatusConfig( + OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"), + )); + let res = reconcile(os, context.clone()).await; + timeout_after_5s(mocksrv).await; + assert!(res.is_err(), "upgrade fails due to opstype=config"); + let err = res.unwrap_err(); + assert!(err.to_string().contains("Expect OS Version is not same with Node OS Version, please upgrade first")); + error_policy(&err, context); + } + + #[tokio::test] + async fn test_upgradeconfigs_version_mismatch() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::UpgradeUpgradeconfigsVersionMismatch( + OSInstance::set_osi_nodestatus_upgrade("openeuler", "default"), + )); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_upgrade_nodestatus_idle() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::UpgradeOSInstaceNodestatusIdle(OSInstance::set_osi_upgradecon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_config_normal() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::ConfigNormal(OSInstance::set_osi_nodestatus_config_syscon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_sysconfig_version_mismatch_reassign() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchReassign(OSInstance::set_osi_nodestatus_config( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_sysconfig_version_mismatch_update() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_syscon_v2_opstype_config(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchUpdate(OSInstance::set_osi_nodestatus_upgrade( + "openeuler", + "default", + ))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } + + #[tokio::test] + async fn test_rollback() { + let (test_proxy_controller, fakeserver) = ProxyController::::test(); + env::set_var("NODE_NAME", "openeuler"); + let os = OS::set_os_rollback_osversion_v2_upgradecon_v2(); + let context = Context::new(test_proxy_controller); + let mocksrv = fakeserver + .run(Testcases::Rollback(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"))); + reconcile(os, context.clone()).await.expect("reconciler"); + timeout_after_5s(mocksrv).await; + } +} diff --git a/KubeOS-Rust/proxy/src/controller/crd.rs b/KubeOS-Rust/proxy/src/controller/crd.rs new file mode 100644 index 00000000..41f333e8 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/crd.rs @@ -0,0 +1,77 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::CustomResource; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)] +pub struct OSSpec { + pub osversion: String, + pub maxunavailable: i64, + pub checksum: String, + pub imagetype: String, + pub containerimage: String, + pub opstype: String, + pub evictpodforce: bool, + pub imageurl: String, + #[serde(rename = "flagSafe")] + pub flagsafe: bool, + pub mtls: bool, + pub cacert: Option, + pub clientcert: Option, + pub clientkey: Option, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[kube( + group = "upgrade.openeuler.org", + version = "v1alpha1", + kind = "OSInstance", + plural = "osinstances", + singular = "osinstance", + status = "OSInstanceStatus", + namespaced +)] +pub struct OSInstanceSpec { + pub nodestatus: String, + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct OSInstanceStatus { + pub sysconfigs: Option, + pub upgradeconfigs: Option, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Configs { + pub version: Option, + pub configs: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Config { + pub model: Option, + pub configpath: Option, + pub contents: Option>, +} + +#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)] +pub struct Content { + pub key: Option, + pub value: Option, + pub operation: Option, +} diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs new file mode 100644 index 00000000..b8a4e6e5 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/mod.rs @@ -0,0 +1,26 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +mod agentclient; +mod apiclient; +#[cfg(test)] +mod apiserver_mock; +mod controller; +mod crd; +mod utils; +mod values; + +pub use agentclient::{AgentCallClient, AgentClient}; +pub use apiclient::ControllerClient; +pub use controller::{error_policy, reconcile, ProxyController}; +pub use crd::OS; +pub use values::SOCK_PATH; diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs new file mode 100644 index 00000000..148ca24d --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/utils.rs @@ -0,0 +1,154 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use log::{debug, info}; + +use super::{ + crd::{Configs, OSInstance, OSInstanceStatus, OS}, + values::{NODE_STATUS_CONFIG, NODE_STATUS_IDLE, NODE_STATUS_UPGRADE}, +}; + +#[derive(PartialEq, Clone, Copy)] +pub enum ConfigType { + UpgradeConfig, + SysConfig, +} + +pub enum ConfigOperation { + DoNothing, + Reassign, + UpdateConfig, +} + +pub struct ConfigInfo { + pub need_config: bool, + pub configs: Option, +} + +impl ConfigType { + pub fn check_config_version(&self, os: &OS, osinstance: &OSInstance) -> ConfigOperation { + debug!("start check_config_version"); + let node_status = &osinstance.spec.nodestatus; + if node_status == NODE_STATUS_IDLE { + debug!("node status is idle"); + return ConfigOperation::DoNothing; + }; + match self { + ConfigType::UpgradeConfig => { + let os_config_version = get_config_version(os.spec.upgradeconfigs.as_ref()); + let osi_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + debug!( + "os upgradeconfig version is{},osinstance spec upragdeconfig version is{}", + os_config_version, osi_config_version + ); + if !check_version(&os_config_version, &osi_config_version) { + info!( + "os.spec.upgradeconfig.version is not equal to oninstance.spec.upragdeconfig.version, operation: reassgin upgrade to get newest upgradeconfigs" + ); + return ConfigOperation::Reassign; + } + }, + ConfigType::SysConfig => { + let os_config_version = get_config_version(os.spec.sysconfigs.as_ref()); + let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + debug!( + "os sysconfig version is{},osinstance spec sysconfig version is{}", + os_config_version, osi_config_version + ); + if !check_version(&os_config_version, &osi_config_version) { + if node_status == NODE_STATUS_CONFIG { + info!( + "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: reassgin config to get newest sysconfigs" + ); + return ConfigOperation::Reassign; + } + if node_status == NODE_STATUS_UPGRADE { + info!( + "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: update osinstance.spec.sysconfig and reconcile" + ); + return ConfigOperation::UpdateConfig; + } + } + }, + }; + ConfigOperation::DoNothing + } + pub fn check_config_start(&self, osinstance: &OSInstance) -> ConfigInfo { + debug!("start check_config_start"); + let spec_config_version: String; + let status_config_version: String; + let configs: Option; + match self { + ConfigType::UpgradeConfig => { + spec_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref()); + if let Some(osinstance_status) = osinstance.status.as_ref() { + status_config_version = get_config_version(osinstance_status.upgradeconfigs.as_ref()); + } else { + status_config_version = get_config_version(None); + } + configs = osinstance.spec.upgradeconfigs.clone(); + }, + ConfigType::SysConfig => { + spec_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref()); + if let Some(osinstance_status) = osinstance.status.as_ref() { + status_config_version = get_config_version(osinstance_status.sysconfigs.as_ref()); + } else { + status_config_version = get_config_version(None); + } + configs = osinstance.spec.sysconfigs.clone(); + }, + } + debug!( + "osinstance soec config version is {},status config version is {}", + spec_config_version, status_config_version + ); + if spec_config_version != status_config_version && osinstance.spec.nodestatus != NODE_STATUS_IDLE { + return ConfigInfo { need_config: true, configs }; + } + ConfigInfo { need_config: false, configs: None } + } + pub fn set_osi_status_config(&self, osinstance: &mut OSInstance) { + match self { + ConfigType::UpgradeConfig => { + if let Some(osi_status) = &mut osinstance.status { + osi_status.upgradeconfigs = osinstance.spec.upgradeconfigs.clone(); + } else { + osinstance.status = Some(OSInstanceStatus { + upgradeconfigs: osinstance.spec.upgradeconfigs.clone(), + sysconfigs: None, + }) + } + }, + ConfigType::SysConfig => { + if let Some(osi_status) = &mut osinstance.status { + osi_status.sysconfigs = osinstance.spec.sysconfigs.clone(); + } else { + osinstance.status = + Some(OSInstanceStatus { upgradeconfigs: None, sysconfigs: osinstance.spec.sysconfigs.clone() }) + } + }, + } + } +} + +pub fn check_version(version_a: &str, version_b: &str) -> bool { + version_a.eq(version_b) +} + +pub fn get_config_version(configs: Option<&Configs>) -> String { + if let Some(configs) = configs { + if let Some(version) = configs.version.as_ref() { + return version.to_string(); + } + }; + String::from("") +} diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs new file mode 100644 index 00000000..dec905a9 --- /dev/null +++ b/KubeOS-Rust/proxy/src/controller/values.rs @@ -0,0 +1,33 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use kube::runtime::controller::ReconcilerAction; +use tokio::time::Duration; + +pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node"; +pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading"; + +pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1"; +pub const OSINSTANCE_KIND: &str = "OSInstance"; + +pub const NODE_STATUS_IDLE: &str = "idle"; +pub const NODE_STATUS_UPGRADE: &str = "upgrade"; +pub const NODE_STATUS_CONFIG: &str = "config"; + +pub const OPERATION_TYPE_UPGRADE: &str = "upgrade"; +pub const OPERATION_TYPE_ROLLBACK: &str = "rollback"; + +pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock"; + +pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) }; + +pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) }; diff --git a/KubeOS-Rust/proxy/src/drain.rs b/KubeOS-Rust/proxy/src/drain.rs new file mode 100644 index 00000000..64417df3 --- /dev/null +++ b/KubeOS-Rust/proxy/src/drain.rs @@ -0,0 +1,511 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use futures::{stream, StreamExt}; +use k8s_openapi::api::core::v1::{Pod, PodSpec, PodStatus}; +use kube::{ + api::{EvictParams, ListParams}, + core::ObjectList, + Api, Client, ResourceExt, +}; +use log::{debug, error, info}; +use reqwest::StatusCode; +use tokio::time::{sleep, Duration, Instant}; +use tokio_retry::{ + strategy::{jitter, ExponentialBackoff}, + RetryIf, +}; + +use self::error::{ + DrainError::{DeletePodsError, GetPodListsError, WaitDeletionError}, + EvictionError::{EvictionErrorNoRetry, EvictionErrorRetry}, +}; + +pub const MAX_EVICT_POD_NUM: usize = 5; +pub const EVERY_EVICTION_RETRY: Duration = Duration::from_secs(5); +pub const EVERY_DELETION_CHECK: Duration = Duration::from_secs(5); +pub const TIMEOUT: Duration = Duration::from_secs(u64::MAX); +pub const RETRY_BASE_DELAY: Duration = Duration::from_millis(100); +pub const RETRY_MAX_DELAY: Duration = Duration::from_secs(20); +pub const MAX_RETRIES_TIMES: usize = 10; + +pub async fn drain_os(client: &Client, node_name: &str, force: bool) -> Result<(), error::DrainError> { + let pods_list = get_pods_deleted(client, node_name, force).await?; + + stream::iter(pods_list) + .for_each_concurrent(MAX_EVICT_POD_NUM, move |pod| { + let k8s_client = client.clone(); + async move { + if evict_pod(&k8s_client, &pod, force).await.is_ok() { + wait_for_deletion(&k8s_client, &pod).await.ok(); + } + } + }) + .await; + + Ok(()) +} + +async fn get_pods_deleted( + client: &Client, + node_name: &str, + force: bool, +) -> Result, error::DrainError> { + let lp = ListParams { field_selector: Some(format!("spec.nodeName={}", node_name)), ..Default::default() }; + let pods_api: Api = Api::all(client.clone()); + let pods: ObjectList = match pods_api.list(&lp).await { + Ok(pods @ ObjectList { .. }) => pods, + Err(err) => { + return Err(GetPodListsError { source: err, node_name: node_name.to_string() }); + }, + }; + let mut filterd_pods_list: Vec = Vec::new(); + let mut filterd_err: Vec = Vec::new(); + let pod_filter = CombinedFilter::new(force); + for pod in pods.into_iter() { + let filter_result = pod_filter.filter(&pod); + if filter_result.status == PodDeleteStatus::Error { + filterd_err.push(filter_result.desc); + continue; + } + if filter_result.result { + filterd_pods_list.push(pod); + } + } + if !filterd_err.is_empty() { + return Err(DeletePodsError { errors: filterd_err }); + } + Ok(filterd_pods_list.into_iter()) +} + +async fn evict_pod(k8s_client: &kube::Client, pod: &Pod, force: bool) -> Result<(), error::EvictionError> { + let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); + + let error_handling_strategy = + if force { ErrorHandleStrategy::RetryStrategy } else { ErrorHandleStrategy::TolerateStrategy }; + + RetryIf::spawn( + error_handling_strategy.retry_strategy(), + || async { + loop { + let eviction_result = pod_api.evict(&pod.name_any(), &EvictParams::default()).await; + + match eviction_result { + Ok(_) => { + pod.name(); + debug!("Successfully evicted Pod '{}'", pod.name_any()); + break; + } + Err(kube::Error::Api(e)) => { + let status_code = StatusCode::from_u16(e.code); + match status_code { + Ok(StatusCode::FORBIDDEN) => { + return Err(EvictionErrorNoRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(StatusCode::NOT_FOUND) => { + return Err(EvictionErrorNoRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Ok(StatusCode::INTERNAL_SERVER_ERROR) => { + error!( + "Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to misconfigured PodDisruptionBudgets.", + pod.name_any(), + e, + EVERY_EVICTION_RETRY.as_secs_f64() + ); + sleep(EVERY_EVICTION_RETRY).await; + continue; + } + Ok(StatusCode::TOO_MANY_REQUESTS) => { + error!("Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to PodDisruptionBugets.", + pod.name_any(), + e, + EVERY_EVICTION_RETRY.as_secs_f64() + ); + sleep(EVERY_EVICTION_RETRY).await; + continue; + } + Ok(_) => { + error!( + "Evict pod {} reported error: '{}'.", + pod.name_any(), + e + ); + return Err(EvictionErrorRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + Err(_) => { + error!( + "Evict pod {} reported error: '{}'.Received invalid response code from Kubernetes API", + pod.name_any(), + e + ); + return Err(EvictionErrorRetry { + source: kube::Error::Api(e.clone()), + pod_name: pod.name_any(), + }); + } + } + } + Err(e) => { + error!("Evict pod {} reported error: '{}' and will retry", pod.name_any(),e); + return Err(EvictionErrorRetry { + source: e, + pod_name: pod.name_any(), + }); + } + } + } + Ok(()) + }, + error_handling_strategy + ).await +} + +async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), error::DrainError> { + let start_time = Instant::now(); + + let pod_api: Api = get_pod_api_with_namespace(k8s_client, pod); + let response_error_not_found: u16 = 404; + loop { + match pod_api.get(&pod.name_any()).await { + Ok(p) if p.uid() != pod.uid() => { + let name = (&p).name_any(); + info!("Pod {} deleted.", name); + break; + }, + Ok(_) => { + info!("Pod '{}' is not yet deleted. Waiting {}s.", pod.name_any(), EVERY_DELETION_CHECK.as_secs_f64()); + }, + Err(kube::Error::Api(e)) if e.code == response_error_not_found => { + info!("Pod {} is deleted.", pod.name_any()); + break; + }, + Err(e) => { + error!( + "Get pod {} reported error: '{}', whether pod is deleted cannot be determined, waiting {}s.", + pod.name_any(), + e, + EVERY_DELETION_CHECK.as_secs_f64() + ); + }, + } + if start_time.elapsed() > TIMEOUT { + return Err(WaitDeletionError { pod_name: pod.name_any(), max_wait: TIMEOUT }); + } else { + sleep(EVERY_DELETION_CHECK).await; + } + } + Ok(()) +} + +fn get_pod_api_with_namespace(client: &kube::Client, pod: &Pod) -> Api { + match pod.metadata.namespace.as_ref() { + Some(namespace) => Api::namespaced(client.clone(), namespace), + None => Api::default_namespaced(client.clone()), + } +} + +trait NameAny { + fn name_any(&self) -> String; +} + +impl NameAny for &Pod { + fn name_any(&self) -> String { + self.metadata.name.clone().or_else(|| self.metadata.generate_name.clone()).unwrap_or_default() + } +} +trait PodFilter { + fn filter(&self, pod: &Pod) -> Box; +} + +struct FinishedOrFailedFilter {} +impl PodFilter for FinishedOrFailedFilter { + fn filter(&self, pod: &Pod) -> Box { + return match pod.status.as_ref() { + Some(PodStatus { phase: Some(phase), .. }) if phase == "Failed" || phase == "Succeeded" => { + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + }, + _ => FilterResult::create_filter_result(false, "", PodDeleteStatus::Okay), + }; + } +} +struct DaemonFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for DaemonFilter { + fn filter(&self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + return match pod.metadata.owner_references.as_ref() { + Some(owner_references) + if owner_references + .iter() + .any(|reference| reference.controller.unwrap_or(false) && reference.kind == "DaemonSet") => + { + if self.force { + let description = format!("Ignore Pod '{}': Pod is member of a DaemonSet", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Warning }) + } else { + let description = format!("Cannot drain Pod '{}': Pod is member of a DaemonSet", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) + } + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} +impl DaemonFilter { + fn new(force: bool) -> DaemonFilter { + DaemonFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } + } +} + +struct MirrorFilter {} +impl PodFilter for MirrorFilter { + fn filter(&self, pod: &Pod) -> Box { + return match pod.metadata.annotations.as_ref() { + Some(annotations) if annotations.contains_key("kubernetes.io/config.mirror") => { + let description = format!("Ignore Pod '{}': Pod is a static Mirror Pod", pod.name_any()); + FilterResult::create_filter_result(false, &description.to_string(), PodDeleteStatus::Warning) + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} + +struct LocalStorageFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for LocalStorageFilter { + fn filter(&self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + return match pod.spec.as_ref() { + Some(PodSpec { volumes: Some(volumes), .. }) if volumes.iter().any(|volume| volume.empty_dir.is_some()) => { + if self.force { + let description = format!("Force draining Pod '{}': Pod has local storage", pod.name_any()); + Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning }) + } else { + let description = format!("Cannot drain Pod '{}': Pod has local Storage", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) + } + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} +impl LocalStorageFilter { + fn new(force: bool) -> LocalStorageFilter { + LocalStorageFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } + } +} +struct UnreplicatedFilter { + finished_or_failed_filter: FinishedOrFailedFilter, + force: bool, +} +impl PodFilter for UnreplicatedFilter { + fn filter(&self, pod: &Pod) -> Box { + if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + let is_replicated = pod.metadata.owner_references.is_some(); + + if is_replicated { + return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay); + } + + if !is_replicated && self.force { + let description = format!("Force drain Pod '{}': Pod is unreplicated", pod.name_any()); + Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning }) + } else { + let description = format!("Cannot drain Pod '{}': Pod is unreplicated", pod.name_any()); + Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error }) + } + } +} +impl UnreplicatedFilter { + fn new(force: bool) -> UnreplicatedFilter { + UnreplicatedFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force } + } +} + +struct DeletedFilter { + delete_wait_timeout: Duration, +} +impl PodFilter for DeletedFilter { + fn filter(&self, pod: &Pod) -> Box { + let now = Instant::now().elapsed(); + return match pod.metadata.deletion_timestamp.as_ref() { + Some(time) + if time.0.timestamp() != 0 + && now - Duration::from_secs(time.0.timestamp() as u64) >= self.delete_wait_timeout => + { + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + }, + _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay), + }; + } +} + +struct CombinedFilter { + deleted_filter: DeletedFilter, + daemon_filter: DaemonFilter, + mirror_filter: MirrorFilter, + local_storage_filter: LocalStorageFilter, + unreplicated_filter: UnreplicatedFilter, +} +impl PodFilter for CombinedFilter { + fn filter(&self, pod: &Pod) -> Box { + let mut filter_res = self.deleted_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.daemon_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.mirror_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.local_storage_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + filter_res = self.unreplicated_filter.filter(pod); + if !filter_res.result { + info!("{}", filter_res.desc); + return Box::new(FilterResult { + result: filter_res.result, + desc: filter_res.desc.clone(), + status: filter_res.status, + }); + } + + FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay) + } +} +impl CombinedFilter { + fn new(force: bool) -> CombinedFilter { + CombinedFilter { + deleted_filter: DeletedFilter { delete_wait_timeout: TIMEOUT }, + daemon_filter: DaemonFilter::new(force), + mirror_filter: MirrorFilter {}, + local_storage_filter: LocalStorageFilter::new(force), + unreplicated_filter: UnreplicatedFilter::new(force), + } + } +} + +#[derive(PartialEq, Clone, Copy)] +enum PodDeleteStatus { + Okay, + Warning, + Error, +} +struct FilterResult { + result: bool, + desc: String, + status: PodDeleteStatus, +} +impl FilterResult { + fn create_filter_result(result: bool, desc: &str, status: PodDeleteStatus) -> Box { + Box::new(FilterResult { result, desc: desc.to_string(), status }) + } +} + +enum ErrorHandleStrategy { + RetryStrategy, + TolerateStrategy, +} + +impl ErrorHandleStrategy { + fn retry_strategy(&self) -> impl Iterator { + let backoff = + ExponentialBackoff::from_millis(RETRY_BASE_DELAY.as_millis() as u64).max_delay(RETRY_MAX_DELAY).map(jitter); + + match self { + Self::TolerateStrategy => backoff.take(0), + + Self::RetryStrategy => backoff.take(MAX_RETRIES_TIMES), + } + } +} + +impl tokio_retry::Condition for ErrorHandleStrategy { + fn should_retry(&mut self, error: &error::EvictionError) -> bool { + match self { + Self::TolerateStrategy => false, + Self::RetryStrategy => matches!(error, error::EvictionError::EvictionErrorRetry { .. }), + } + } +} + +pub mod error { + use thiserror::Error; + use tokio::time::Duration; + + #[derive(Debug, Error)] + pub enum DrainError { + #[error("Get node {} pods list error reported: {}", node_name, source)] + GetPodListsError { source: kube::Error, node_name: String }, + + #[error("Pod '{}' was not deleted in the time allocated ({:.2}s).",pod_name,max_wait.as_secs_f64())] + WaitDeletionError { pod_name: String, max_wait: Duration }, + #[error("")] + DeletePodsError { errors: Vec }, + } + + #[derive(Debug, Error)] + pub enum EvictionError { + #[error("Evict Pod {} error: '{}'", pod_name, source)] + EvictionErrorRetry { source: kube::Error, pod_name: String }, + + #[error("Evict Pod {} error: '{}'", pod_name, source)] + EvictionErrorNoRetry { source: kube::Error, pod_name: String }, + } +} diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs new file mode 100644 index 00000000..5c122ba2 --- /dev/null +++ b/KubeOS-Rust/proxy/src/main.rs @@ -0,0 +1,49 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use anyhow::Result; +use env_logger::{Builder, Env, Target}; +use futures::StreamExt; +use kube::{ + api::{Api, ListParams}, + client::Client, + runtime::controller::{Context, Controller}, +}; +use log::{error, info}; +mod controller; +use controller::{ + error_policy, reconcile, AgentCallClient, AgentClient, ControllerClient, ProxyController, OS, SOCK_PATH, +}; + +const PROXY_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); +#[tokio::main] +async fn main() -> Result<()> { + Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init(); + let client = Client::try_default().await?; + let os: Api = Api::all(client.clone()); + let controller_client = ControllerClient::new(client.clone()); + let agent_call_client = AgentCallClient::default(); + let agent_client = AgentClient::new(SOCK_PATH, agent_call_client); + let proxy_controller = ProxyController::new(client, controller_client, agent_client); + info!("os-proxy version is {}, start renconcile", PROXY_VERSION.unwrap_or("Not Found")); + Controller::new(os, ListParams::default()) + .run(reconcile, error_policy, Context::new(proxy_controller)) + .for_each(|res| async move { + match res { + Ok(_o) => {}, + Err(e) => error!("reconcile failed: {}", e.to_string()), + } + }) + .await; + info!("os-proxy terminated"); + Ok(()) +} diff --git a/KubeOS-Rust/proxy/tests/common/mod.rs b/KubeOS-Rust/proxy/tests/common/mod.rs new file mode 100644 index 00000000..82577597 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/common/mod.rs @@ -0,0 +1,63 @@ +use std::process::{Command, Stdio}; + +use anyhow::Result; +use k8s_openapi::api::core::v1::Node; +use kube::{ + api::ResourceExt, + client::Client, + config::{Config, KubeConfigOptions, Kubeconfig}, + Api, +}; +use manager::utils::{CommandExecutor, RealCommandExecutor}; + +pub const CLUSTER: &str = "kubeos-test"; + +pub fn run_command(cmd: &str, args: &[&str]) -> Result<()> { + let output = Command::new(cmd).args(args).stdout(Stdio::inherit()).stderr(Stdio::inherit()).output()?; + if !output.status.success() { + println!("failed to run command: {} {}\n", cmd, args.join(" ")); + } + Ok(()) +} + +pub async fn setup() -> Result { + // set PATH variable + let path = std::env::var("PATH").unwrap(); + let new_path = format!("{}:{}", path, "../../bin"); + std::env::set_var("PATH", new_path); + + // create cluster + let executor = RealCommandExecutor {}; + println!("Creating cluster"); + run_command("bash", &["./tests/setup/setup_test_env.sh"]).expect("failed to create cluster"); + + // connect to the cluster + let kind_config = executor.run_command_with_output("kind", &["get", "kubeconfig", "-n", CLUSTER]).unwrap(); + let kubeconfig = Kubeconfig::from_yaml(kind_config.as_str()).expect("failed to parse kubeconfig"); + let options = KubeConfigOptions::default(); + let config = Config::from_custom_kubeconfig(kubeconfig, &&options).await.expect("failed to create config"); + let client = Client::try_from(config).expect("failed to create client"); + // list all nodes + let nodes: Api = Api::all(client.clone()); + let node_list = nodes.list(&Default::default()).await.expect("failed to list nodes"); + for n in node_list { + println!("Found Node: {}", n.name()); + } + // check node status + let node = nodes.get("kubeos-test-worker").await.unwrap(); + let status = node.status.unwrap(); + let conditions = status.conditions.unwrap(); + for c in conditions { + if c.type_ == "Ready" { + assert_eq!(c.status, "True"); + } + } + println!("Cluster ready"); + Ok(client) +} + +pub fn clean_env() { + let executor = RealCommandExecutor {}; + println!("Cleaning cluster"); + executor.run_command("kind", &["delete", "clusters", CLUSTER]).expect("failed to clean cluster"); +} diff --git a/KubeOS-Rust/proxy/tests/drain_test.rs b/KubeOS-Rust/proxy/tests/drain_test.rs new file mode 100644 index 00000000..2f4f1501 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/drain_test.rs @@ -0,0 +1,41 @@ +mod common; + +use common::*; +use drain::drain_os; +use k8s_openapi::api::core::v1::{Node, Pod}; +use kube::Api; + +#[tokio::test] +#[ignore = "integration test"] +async fn test_drain() { + let client = setup().await.unwrap(); + // drain node + let nodes: Api = Api::all(client.clone()); + let node_name = "kubeos-test-worker"; + println!("cordon node"); + nodes.cordon(node_name).await.unwrap(); + println!("drain node"); + drain_os(&client, node_name, true).await.unwrap(); + + // assert unschedulable + println!("check node unschedulable"); + let node = nodes.get(node_name).await.unwrap(); + if let Some(spec) = node.spec { + assert_eq!(spec.unschedulable, Some(true)); + } else { + panic!("node spec is none"); + } + // list all pods on kubeos-test-worker node and all pods should belong to daemonset + println!("list all pods on kubeos-test-worker node"); + let pods: Api = Api::all(client.clone()); + let pod_list = pods.list(&Default::default()).await.unwrap(); + // check the pod is from daemonset + for p in pod_list { + if p.spec.unwrap().node_name.unwrap() == node_name { + assert_eq!(p.metadata.owner_references.unwrap()[0].kind, "DaemonSet"); + } + } + nodes.uncordon(node_name).await.unwrap(); + + clean_env() +} diff --git a/KubeOS-Rust/proxy/tests/setup/kind-config.yaml b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml new file mode 100644 index 00000000..0fe29e73 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker \ No newline at end of file diff --git a/KubeOS-Rust/proxy/tests/setup/resources.yaml b/KubeOS-Rust/proxy/tests/setup/resources.yaml new file mode 100644 index 00000000..0e449d5d --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/resources.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: example-daemonset +spec: + selector: + matchLabels: + name: example-daemonset + template: + metadata: + labels: + name: example-daemonset + spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-with-local-storage +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] + volumeMounts: + - mountPath: "/data" + name: local-volume + volumes: + - name: local-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Pod +metadata: + name: standalone-pod +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-deployment +spec: + replicas: 2 + selector: + matchLabels: + app: example + template: + metadata: + labels: + app: example + spec: + containers: + - name: nginx + image: nginx:alpine + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: DoesNotExist + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: example-pdb +spec: + minAvailable: 1 + selector: + matchLabels: + app: example +--- +apiVersion: v1 +kind: Pod +metadata: + name: resource-intensive-pod +spec: + containers: + - name: busybox + image: busybox:stable + command: ["/bin/sh", "-c", "sleep 3600"] + resources: + requests: + memory: "256Mi" + cpu: "500m" + limits: + memory: "512Mi" + cpu: "1000m" + diff --git a/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh new file mode 100644 index 00000000..d24d8e01 --- /dev/null +++ b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# this bash script executes in proxy directory + +set -Eeuxo pipefail + +# Define variables +KIND_VERSION="v0.19.0" +KUBECTL_VERSION="v1.24.15" +KIND_CLUSTER_NAME="kubeos-test" +DOCKER_IMAGES=("busybox:stable" "nginx:alpine" "kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab") +NODE_IMAGE="kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab" +RESOURCE="./tests/setup/resources.yaml" +KIND_CONFIG="./tests/setup/kind-config.yaml" +BIN_PATH="../../bin/" +ARCH=$(uname -m) + +# Install kind and kubectl +install_bins() { + # if bin dir not exist then create + if [ ! -d "${BIN_PATH}" ]; then + mkdir -p "${BIN_PATH}" + fi + if [ ! -f "${BIN_PATH}"kind ]; then + echo "Installing Kind..." + # For AMD64 / x86_64 + if [ "$ARCH" = x86_64 ]; then + # add proxy if you are behind proxy + curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-amd64 + fi + # For ARM64 + if [ "$ARCH" = aarch64 ]; then + curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-arm64 + fi + chmod +x "${BIN_PATH}"kind + fi + if [ ! -f "${BIN_PATH}"kubectl ]; then + echo "Installing kubectl..." + if [ "$ARCH" = x86_64 ]; then + curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + fi + if [ "$ARCH" = aarch64 ]; then + curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/arm64/kubectl" + fi + chmod +x "${BIN_PATH}"kubectl + fi + export PATH=$PATH:"${BIN_PATH}" +} + +# Create Kind Cluster +create_cluster() { + echo "Creating Kind cluster..." + for image in "${DOCKER_IMAGES[@]}"; do + docker pull "$image" + done + kind create cluster --name "${KIND_CLUSTER_NAME}" --config "${KIND_CONFIG}" --image "${NODE_IMAGE}" +} + +# Load Docker image into Kind cluster +load_docker_image() { + echo "Loading Docker image into Kind cluster..." + DOCKER_IMAGE=$(printf "%s " "${DOCKER_IMAGES[@]:0:2}") + kind load docker-image ${DOCKER_IMAGE} --name "${KIND_CLUSTER_NAME}" +} + +# Apply Kubernetes resource files +apply_k8s_resources() { + echo "Applying Kubernetes resources from ${RESOURCE}..." + kubectl apply -f "${RESOURCE}" + echo "Waiting for nodes getting ready..." + sleep 40s +} + +main() { + export no_proxy=localhost,127.0.0.1 + install_bins + create_cluster + load_docker_image + apply_k8s_resources +} + +main diff --git a/KubeOS-Rust/rustfmt.toml b/KubeOS-Rust/rustfmt.toml new file mode 100644 index 00000000..3c565cf5 --- /dev/null +++ b/KubeOS-Rust/rustfmt.toml @@ -0,0 +1,11 @@ +# cargo +nightly fmt +version = "Two" +use_small_heuristics = "MAX" +match_block_trailing_comma = true +newline_style = "Unix" +merge_derives = false +max_width = 120 +group_imports = "StdExternalCrate" +imports_granularity = "Crate" +reorder_imports = true +unstable_features = true \ No newline at end of file diff --git a/Makefile b/Makefile index 634a3bcf..760eb544 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,9 @@ GO_BUILD_CGO = CGO_ENABLED=1 \ CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \ ${GO_BUILD} -buildmode=pie -trimpath -tags "seccomp selinux static_build cgo netgo osusergo" -all: proxy operator agent hostshell +RUSTFLAGS := RUSTFLAGS="-C relocation_model=pic -D warnings -W unsafe_code -W rust_2021_incompatible_closure_captures -C link-arg=-s" + +all: proxy operator agent hostshell rust-kubeos # Build binary proxy: @@ -69,6 +71,18 @@ hostshell: ${GO_BUILD_CGO} ${LD_FLAGS} -o bin/hostshell cmd/admin-container/main.go strip bin/hostshell +rust-kubeos: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust + +rust-proxy: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package proxy + +rust-agent: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package os-agent + +rust-operator: + cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package operator + # Install CRDs into a cluster install: manifests kubectl apply -f confg/crd diff --git a/VERSION b/VERSION index ee90284c..af0b7ddb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.4 +1.0.6 diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index 98ef1f56..27ff3273 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -18,7 +18,7 @@ spec: versions: - name: v1alpha1 additionalPrinterColumns: - - name: OSVersion + - name: OS VERSION jsonPath: .spec.osversion type: string description: The version of OS diff --git a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml index a7bad3f0..df9119b4 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml @@ -22,19 +22,19 @@ spec: type: string jsonPath: .spec.nodestatus description: The status of node - - name: SYSCONFIG STATUS + - name: SYSCONFIG-VERSION-CURRENT type: string jsonPath: .status.sysconfigs.version description: The current status of sysconfig - - name: SYSCONFIG SPEC + - name: SYSCONFIG-VERSION-DESIRED type: string jsonPath: .spec.sysconfigs.version description: The expected version of sysconfig - - name: UPGRADECONFIG STATUS + - name: UPGRADECONFIG-VERSION-CURRENT type: string jsonPath: .status.upgradeconfigs.version description: The current version of upgradeconfig - - name: UPGRADECONFIG SPEC + - name: UPGRADECONFIG-VERSION-DESIRED type: string jsonPath: .spec.upgradeconfigs.version description: The expected version of upgradeconfig diff --git a/docs/quick-start.md b/docs/quick-start.md index 13bb08d9..ee6f3c31 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -1,18 +1,22 @@ # 快速使用指导 -## 编译及部署 +[TOC] -### 编译指导 +## 编译指导 * 编译环境：openEuler Linux x86/AArch64 + * 进行编译需要以下包： * golang(大于等于1.15版本) * make * git + * rust(大于等于1.57版本) + * cargo(大于等于1.57版本) + * openssl-devel ``` shell - sudo yum install golang make git - ``` + sudo yum install golang make git rust cargo openssl-devel + ``` * 使用git获取本项目的源码 @@ -27,77 +31,101 @@ ```shell cd KubeOS - sudo make - ``` - - * proxy及operator容器镜像构建 - * proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕 - * 请用户自行编写Dockerfile来构建镜像，请注意 - * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性 - * 需将operator和proxy拷贝到baseimage上 - * 请确保proxy属主和属组为root，文件权限为500 - * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500 - * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应 - * 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库 - * Dockerfile参考如下, Dockerfile也可以使用多阶段构建: - - ``` dockerfile - FROM your_baseimage - COPY ./bin/proxy /proxy - ENTRYPOINT ["/proxy"] - FROM your_baseimage - COPY --chown=6552:6552 ./bin/operator /operator - ENTRYPOINT ["/operator"] - ``` + sudo make + # 编译生成的二进制在bin目录下，查看二进制 + tree bin + bin + ├── operator + ├── os-agent + ├── proxy + ├── rust + │   ├── ... + │   └── release + │   ├── ... + │   ├── os-agent + │   └── proxy + ``` - ```shell - # 指定proxy的镜像仓库，镜像名及版本 - export IMG_PROXY=your_imageRepository/proxy_imageName:version - # 指定proxy的Dockerfile地址 - export DOCKERFILE_PROXY=your_dockerfile_proxy - # 指定operator的镜像仓库，镜像名及版本 - export IMG_OPERATOR=your_imageRepository/operator_imageName:version - # 指定operator的Dockerfile路径 - export DOCKERFILE_OPERATOR=your_dockerfile_operator - - # 镜像构建 - docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} . - docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} . - # 推送镜像到镜像仓库 - docker push ${IMG_OPERATOR} - docker push ${IMG_PROXY} - ``` + * ```bin/proxy```、```bin/os-agent```为go语言编写的proxy和os-agent，```bin/rust/release/proxy```、```bin/rust/release/os-agent```为rust语言编写的proxy和os-agent，二者功能一致。 + +## 镜像构建指导 + +### proxy及operator镜像构建指导 + +* proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕 + +* 请用户自行编写Dockerfile来构建镜像，请注意 + * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性 + * 需将operator和proxy拷贝到baseimage上 + * 请确保proxy属主和属组为root，文件权限为500 + * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500 + * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应 + +* 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库 + +* Dockerfile参考如下, Dockerfile也可以使用多阶段构建: + + ``` dockerfile + FROM your_baseimage + COPY ./bin/proxy /proxy + ENTRYPOINT ["/proxy"] + FROM your_baseimage + COPY --chown=6552:6552 ./bin/operator /operator + ENTRYPOINT ["/operator"] + ``` -* OS虚拟机镜像制作 - * 制作注意事项 - * 请确保已安装qemu-img，bc，parted，tar，yum，docker, dosfstools - * 容器OS镜像制作需要使用root权限 - * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 - * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式 - * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间 - * 容器镜像制作时不支持用户自定义配置挂载文件 - * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。 - * 请确保os-agent属主和属组为root，建议os-agent文件权限为500 - * 容器OS虚拟机镜像制作 - 进入scripts目录，执行脚本 - - ```shell - cd scripts - bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' - ``` - - * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 - * 示例命令使用的密码为```openEuler12#$``` - * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： - * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 - * qcow2 格式的系统镜像 system.qcow2。 - * 可用于升级的根文件系统分区镜像 update.img 。 - * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数 - * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) - * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 - * 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 - -### 部署指导 + ```shell + # 指定proxy的镜像仓库，镜像名及版本 + export IMG_PROXY=your_imageRepository/proxy_imageName:version + # 指定proxy的Dockerfile地址 + export DOCKERFILE_PROXY=your_dockerfile_proxy + # 指定operator的镜像仓库，镜像名及版本 + export IMG_OPERATOR=your_imageRepository/operator_imageName:version + # 指定operator的Dockerfile路径 + export DOCKERFILE_OPERATOR=your_dockerfile_operator + + # 镜像构建 + docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} . + docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} . + # 推送镜像到镜像仓库 + docker push ${IMG_OPERATOR} + docker push ${IMG_PROXY} + ``` + +### KubeOS虚拟机镜像制作指导 + +* 制作注意事项 + * 请确保已安装qemu-img，bc，parted，tar，yum，docker + * 容器OS镜像制作需要使用root权限 + * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库 + * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式 + * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间 + * 容器镜像制作时不支持用户自定义配置挂载文件 + * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。 + * 请确保os-agent属主和属组为root，建议os-agent文件权限为500 + +* 容器OS虚拟机镜像制作 + 进入scripts目录，执行脚本 + + ```shell + cd scripts + bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' + ``` + + * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。 + * 容器 OS 镜像制作完成后，会在 scripts 目录下生成： + * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。 + * qcow2 格式的系统镜像 system.qcow2。 + * 可用于升级的根文件系统分区镜像 update.img 。 + * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数 + * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool) + * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。 + +* 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。 + +## 部署指导 + +### os-operator和os-proxy部署指导 * 环境要求 * openEuler Linux x86/AArch64系统 @@ -143,18 +171,35 @@ kubectl get pods -A ``` -### 使用指导 +## 使用指导 #### 注意事项 -* 容器OS升级为所有软件包原子升级，默认不在容器OS内提供单包升级能力。 -* 容器OS升级为双区升级的方式，不支持更多分区数量。 -* 单节点的升级过程的日志可在节点的/var/log/message文件查看。 -* 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。 -* 使用docker镜像升级和mtls双向认证仅支持 openEuler 22.09 及之后的版本 -* 不支持跨大版本升级 - -#### 参数说明 +* 公共注意事项 + * 仅支持虚拟机x86和arm64 UEFI场景。 + * 当前不支持集群节点OS多版本管理，即集群中OS的CR只能为一个。 + * 使用kubectl apply通过YAML创建或更新OS的CR时，不建议并发apply，当并发请求过多时，kube-apiserver会无法处理请求导致失败。 + * 如用户配置了容器镜像仓的证书或密钥，请用户保证证书或密钥文件的权限最小。 +* 升级注意事项 + * 升级为所有软件包原子升级，默认不提供单包升级能力。 + * 升级为双区升级的方式，不支持更多分区数量。 + * 当前暂不支持跨大版本升级。 + * 单节点的升级过程的日志可在节点的 /var/log/messages 文件查看。 + * 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。 + * 节点上containerd如需配置ctr使用的私有镜像，请将配置文件host.toml按照ctr指导放在/etc/containerd/certs.d目录下。 + +* 配置注意事项 + * 用户自行指定配置内容，用户需保证配置内容安全可靠 ，尤其是持久化配置（kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next），KubeOS不对参数有效性进行检验。 + * opstype=config时，若osversion与当前集群节点的OS版本不一致，配置不会进行。 + * 当前仅支持kernel参数临时配置（kernel.sysctl）、持久化配置（kernel.sysctl.persist）和grub cmdline配置（grub.cmdline.current和grub.cmdline.next）。 + * 持久化配置会写入persist持久化分区，升级重启后配置保留；kernel参数临时配置重启后不保留。 + * 配置grub.cmdline.current或grub.cmdline.next时，如为单个参数（非key=value格式参数），请指定key为该参数，value为空。 + * 进行配置删除（operation=delete）时，key=value形式的配置需保证key、value和实际配置一致。 + * 配置不支持回退，如需回退，请修改配置版本和配置内容，重新下发配置。 + * 配置出现错误，节点状态陷入config时，请将配置版本恢复成上一版本并重新下发配置，从而使节点恢复至idel状态。 但是请注意：出现错误前已经配置完成的参数无法恢复。 + * 在配置grub.cmdline.current或grub.cmdline.next时，若需要将已存在的“key=value”格式的参数更新为只有key无value格式，比如将“rd.info=0”更新成rd.info，需要先删除“key=value”，然后在下一次配置时，添加key。不支持直接更新或者更新删除动作在同一次完成。 + +#### OS CR参数说明 在集群中创建类别为OS的定制对象，设置相应字段。类别OS来自于安装和部署章节创建的CRD对象，字段及说明如下： @@ -164,22 +209,21 @@ | 参数 |参数类型 | 参数说明 | 使用说明 | 是否必选 | | -------------- | ------ | ------------------------------------------------------------ | ----- | ---------------- | - | imagetype | string | 使用的升级镜像的类型 | 需为 docker ，containerd ，或者是 disk，其他值无效，且该参数仅在升级场景有效。
**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。|是 | - | opstype | string | 进行的操作，升级,回退或者配置 | 需为 upgrade ，config 或者 rollback ，其他值无效 |是 | - | osversion | string | 用于升级或回退的镜像的OS版本 | 需为 KubeOS version , 例如: KubeOS 1.0.0|是 | - | maxunavailable | int | 同时进行升级或回退的节点数 | maxunavailable值设置为大于实际集群的节点数时也可正常部署，升级或回退时会按照集群内实际节点数进行|是 | - | containerimage | string | 用于升级的容器镜像 | 需要为容器镜像格式：[REPOSITORY/NAME[:TAG@DIGEST]](https://docs.docker.com/engine/reference/commandline/tag/#extended-description)，仅在使用容器镜像升级场景下有效|是 | - | imageurl | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如： 仅在使用磁盘镜像升级场景下有效|是 | + | imagetype | string | 升级镜像的类型 | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。
**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是 | + | opstype | string | 操作类型：升级,回退或者配置 | 仅支持upgrade ，config 或者 rollback |是 | + | osversion | string | 升级/回退的目标版本 | osversion需与节点的目标os版本对应（节点上/etc/os-release中PRETTY_NAME字段或k8s检查到的节点os版本） 例如：KubeOS 1.0.0。 |是 | + | maxunavailable | int | 每批同时进行升级/回退/配置的节点数。 | maxunavailable值大于实际节点数时，取实际节点数进行升级/回退/配置。 |是 | + | containerimage | string | 用于升级的容器镜像 | 仅在imagetype是容器类型时生效，仅支持以下3种格式的容器镜像地址： repository/name repository/name@sha256:xxxx repository/name:tag |是 | + | imageurl | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如： ，仅在使用磁盘镜像升级场景下有效 |是 | | checksum | string | 用于升级的磁盘镜像校验的checksum(SHA-256)值或者是用于升级的容器镜像的digests值 | 仅在升级场景下有效 |是 | | flagSafe | bool | 当imageurl的地址使用http协议表示是否是安全的 | 需为 true 或者 false ，仅在imageurl使用http协议时有效 |是 | | mtls | bool | 用于表示与imageurl连接是否采用https双向认证 | 需为 true 或者 false ，仅在imageurl使用https协议时有效|是 | | cacert | string | https或者https双向认证时使用的根证书文件 | 仅在imageurl使用https协议时有效| imageurl使用https协议时必选 | | clientcert | string | https双向认证时使用的客户端证书文件 | 仅在使用https双向认证时有效|mtls为true时必选 | | clientkey | string | https双向认证时使用的客户端公钥 | 仅在使用https双向认证时有效|mtls为true时必选 | - | evictpodforce | bool | 用于表示升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | - | nodeselector | string | 需要进行升级/配置/回滚操作的节点label | 用于只对具有某些特定label的节点而不是集群所有worker节点进行运维的场景，需要进行运维操作的节点需要包含key为upgrade.openeuler.org/node-selector的label，nodeselector为该label的value值，此参数不配置时，或者配置为""时默认对所有节点进行操作| 可选 | - | sysconfigs | / | 需要进行配置的参数值 | 在配置或者升级或者回退机器时有效，在升级或者回退操作之后即机器重启之后起效，详细字段说明请见```配置（Settings）指导```| 可选 | - | upgradeconfigs | / | 需要升级前进行的配置的参数值 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | + | evictpodforce | bool | 升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | + | sysconfigs | / | 配置设置 | 1. “opstype=config”时只进行配置。 2.“opstype=upgrade/rollback”时，代表升级/回退后配置，即在升级/回退重启后进行配置。```配置（Settings）指导``` | “opstype=config”时必选 | + | upgradeconfigs | / | 升级前配置设置 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | #### 升级指导 @@ -343,12 +387,13 @@ kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' ``` -* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的 imageurl, osversion, checksum, maxunavailable, flagSafe 或者containerimage字段进行相应修改。 +* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改 #### 配置（Settings）指导 * Settings参数说明: - 以进行配置时的示例yaml为例对配置的参数进行说明，示例yaml如下： + + 基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致： ```yaml apiVersion: upgrade.openeuler.org/v1alpha1 @@ -362,12 +407,9 @@ maxunavailable: edit.node.config.number containerimage: "" evictpodforce: false - imageurl: "" checksum: "" - flagSafe: false - mtls: false sysconfigs: - version: 1.0.0 + version: edit.sysconfigs.version configs: - model: kernel.sysctl contents: @@ -380,54 +422,82 @@ configpath: persist file path contents: - key: kernel param key3 - value: kernel param value3 + value: kernel param value3 + - model: grub.cmdline.current + contents: + - key: boot param key1 + - key: boot param key2 + value: boot param value2 + - key: boot param key3 + value: boot param value3 + operation: delete + - model: grub.cmdline.next + contents: + - key: boot param key4 + - key: boot param key5 + value: boot param value5 + - key: boot param key6 + value: boot param value6 + operation: delete ``` - * 配置的参数说明如下： - * version: 配置的版本,通过版本差异触发配置，请修改配置后更新 version - * configs: 具体配置内容 - * model: 进行的配置的类型，支持的配置类型请看[Settings 列表](#setting-列表) - * configpath: 如为持久化配置，配置文件路径 - * contents: 配置参数的 key / value 和对参数的操作。 - * key / value: 请看[Settings 列表](#setting-列表)对支持的配置的 key / value的说明。 - * operation: 若不指定operation，则默认为添加或更新。若指定为delete，代表删除目前OS中已配置的参数。 - **注意：** 当operation为delete时，yaml中的key/value必须和OS上想删除参数的key/value**一致**，否则删除失败。 - * upgradeconfigs与sysconfig参数相同，upgradeconfig为升级前进行的配置，仅在升级/回滚场景起效，在升级/回滚操作执行前进行配置，只进行配置或者需要升级/回滚重启后执行配置，使用sysconfigs + 配置的参数说明如下： + + | 参数 | 参数类型 | 参数说明 | 使用说明 | 配置中是否必选 | + | ---------- | -------- | --------------------------- | ------------------------------------------------------------ | ----------------------- | + | version | string | 配置的版本 | 通过version是否相等来判断配置是否触发，version为空（为""或者没有值）时同样进行判断，所以不配置sysconfigs/upgradeconfigs时，继存的version值会被清空并触发配置。 | 是 | + | configs | / | 具体配置内容 | 包含具体配置项列表。 | 是 | + | model | string | 配置的类型 | 支持的配置类型请看附录下的```Settings列表``` | 是 | + | configpath | string | 配置文件路径 | 仅在kernel.sysctl.persist配置类型中生效，请看附录下的```Settings列表```对配置文件路径的说明。 | 否 | + | contents | / | 具体key/value的值及操作类型 | 包含具体配置参数列表。 | 是 | + | key | string | 参数名称 | key不能为空，不能包含"="，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中每种配置类型对key的说明。 | 是 | + | value | string | 参数值 | key=value形式的参数中，value不能为空，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中对每种配置类型对value的说明。 | key=value形式的参数必选 | + | operation | string | 对参数进行的操作 | 仅对kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next类型的参数生效。默认为添加或更新。仅支持配置为delete，代表删除已存在的参数（key=value需完全一致才能删除）。 | 否 | + + + + * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置 + * 使用说明 + * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如上，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml + * 查看配置之前的节点的配置的版本和节点状态（NODESTATUS状态为idle） ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` * 执行命令，在集群中部署cr实例后，节点会根据配置的参数信息进行配置，再次查看节点状态(NODESTATUS变成config) ```shell kubectl apply -f upgrade_v1alpha1_os.yaml - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` * 再次查看节点的配置的版本确认节点是否配置完成(NODESTATUS恢复为idle) ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` -* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 +* 如果后续需要再次配置，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 #### 回退指导 * 回退场景 - * 虚拟机无法正常启动时，需要退回到上一可以启动的版本时进行回退操作，仅支持手动回退容器 OS 。 - * 虚拟机能够正常启动并且进入系统，需要将当前版本退回到老版本时进行回退操作，支持工具回退（类似升级方式）和手动回退，建议使用工具回退。 - * 配置出现错误，节点状态陷入config时，可以回退至上一个配置版本以恢复节点至idle状态。 - **注意**：在配置新版本时，出现错误前已经配置的参数无法回退。 + * 虚拟机无法正常启动时，可在grub启动项页面手动切换启动项，使系统回退至上一版本（即手动回退）。 + * 虚拟机能够正常启动并且进入系统时，支持工具回退和手动回退，建议使用工具回退。 + * 工具回退有两种方式： + 1. rollback模式直接回退至上一版本。 + 2. upgrade模式重新升级至上一版本 * 手动回退指导 - * 手动重启虚拟机，选择第二启动项进行回退，手动回退仅支持回退到本次升级之前的版本。 + + * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。 * 工具回退指导 * 回退至任意版本 - * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 + * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。 + * YAML修改完成后执行更新命令，在集群中更新定制对象后，节点会根据配置的字段信息进行回退 ```shell @@ -499,23 +569,21 @@ * 查看节点容器 OS 版本(回退OS版本)或节点config版本&节点状态为idle(回退config版本)，确认回退是否成功。 ```shell - kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage' - - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version' ``` -#### Admin容器 +## Admin容器镜像制作、部署和使用 KubeOS提供一个分离的包含sshd服务和hostshell工具的Admin容器，来帮助管理员在必要情况下登录KubeOS，其中的sshd服务由[sysmaster](https://gitee.com/openeuler/sysmaster)/systemd拉起。Admin容器部署后用户可通过ssh连接到节点的Admin容器，进入Admin容器后执行hostshell命令获取host的root shell。 -##### 部署方法 +### admin容器镜像制作 -以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/everything/aarch64/Packages/)到scripts/admin-container目录下。 +以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-SP1-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/update/aarch64/Packages/)到scripts/admin-container目录下。 -**修改**admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/docker_img)下载。 +修改admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts-sp1可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/docker_img)下载。 ```Dockerfile -FROM openeuler-22.03-lts +FROM openeuler-22.03-lts-sp1 RUN yum -y install openssh-clients util-linux @@ -546,7 +614,9 @@ bash -x kbimg.sh create admin-image -f admin-container/Dockerfile -d your_imageR docker push your_imageRepository/admin_imageName:version ``` -在master节点上部署Admin容器，需要提供ssh公钥来免密登录，**修改**并应用如下示例yaml文件: +### admin容器部署 + +在master节点上部署Admin容器，需要提供ssh公钥来免密登录，修改并应用如下示例yaml文件: ```yaml apiVersion: v1 @@ -615,6 +685,8 @@ spec: control-plane: admin-container-sysmaster ``` +### admin容器使用 + ssh到Admin容器，然后执行hostshell命令进入host root shell, 如： ```shell @@ -622,7 +694,7 @@ ssh -p your-exposed-port root@your.worker.node.ip hostshell ``` -##### hostshell +#### hostshell说明 为了保证KubeOS的轻便性，许多工具或命令没有安装在KubeOS内。因此，用户可以在制作Admin容器时，将期望使用的二进制文件放在容器内的如/usr/bin目录下。hostshell工具在执行时会将容器下的/usr/bin, /usr/sbin, /usr/local/bin, /usr/local/sbin路径添加到host root shell的环境变量。 @@ -637,8 +709,7 @@ hostshell #### kernel Settings -* kenerl.sysctl: 临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）, 示例如下: - +* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: ```yaml configs: - model: kernel.sysctl @@ -649,8 +720,7 @@ hostshell value: 0 operation: delete ``` - -* kernel.sysctl.persist: 设置持久化内核参数，key/value表示内核参数的key/value，key与value均不能为空且key不能包含“=”， configpath为配置文件路径，支持新建（需保证父目录存在），如不指定configpath默认修改/etc/sysctl.conf，示例如下： +* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下: ```yaml configs: - model: kernel.sysctl.persist @@ -668,23 +738,38 @@ hostshell * grub.cmdline: 设置grub.cfg文件中的内核引导参数，该行参数在grub.cfg文件中类似如下示例： ```shell - linux /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 - ``` + linux /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 + ``` * KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置： - * grub.cmdline.current：对当前分区的启动项参数进行配置。 - * grub.cmdline.next：对下一分区的启动项参数进行配置。 + + - grub.cmdline.current：对当前分区的启动项参数进行配置。 + - grub.cmdline.next：对下一分区的启动项参数进行配置。 + * 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。 + * grub.cmdline.current/next支持“key=value”（value不能为空），也支持单key。若value中有“=”，例如“root=UUID=some-uuid”，key应设置为第一个“=”前的所有字符，value为第一个“=”后的所有字符。 配置方法示例如下： + ```yaml configs: - - model: grub.cmdline.current - contents: - - key: selinux - value: 0 - - key: root - value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 - - key: panic - value: 3 - operation: delete + - model: grub.cmdline.current + contents: + - key: selinux + value: "0" + - key: root + value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 + - key: panic + value: "3" + operation: delete + - key: crash_kexec_post_notifiers + - model: grub.cmdline.next + contents: + - key: selinux + value: "0" + - key: root + value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94 + - key: panic + value: "3" + operation: delete + - key: crash_kexec_post_notifiers ``` diff --git a/scripts/bootloader.sh b/scripts/bootloader.sh index 75096a38..df4be329 100644 --- a/scripts/bootloader.sh +++ b/scripts/bootloader.sh @@ -19,7 +19,7 @@ function install_grub2_x86 () cp -r /usr/lib/grub/x86_64-efi boot/efi/EFI/openEuler eval "grub2-mkimage -d /usr/lib/grub/x86_64-efi -O x86_64-efi --output=/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" - mkdir -p /boot/EFI/BOOT/ + mkdir -p /boot/efi/EFI/BOOT/ cp -f /boot/efi/EFI/openEuler/grubx64.efi /boot/efi/EFI/BOOT/BOOTX64.EFI fi } @@ -29,7 +29,7 @@ function install_grub2_efi () cp -r /usr/lib/grub/arm64-efi /boot/efi/EFI/openEuler/ eval "grub2-mkimage -d /usr/lib/grub/arm64-efi -O arm64-efi --output=/boot/efi/EFI/openEuler/grubaa64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux" - mkdir -p /boot/EFI/BOOT/ + mkdir -p /boot/efi/EFI/BOOT/ cp -f /boot/efi/EFI/openEuler/grubaa64.efi /boot/efi/EFI/BOOT/BOOTAA64.EFI } -- Gitee