From fb74ff58f86438c5dbd0f26806ad4b6b6f8e9ff6 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 1 Aug 2024 04:32:57 +0000 Subject: [PATCH] feat:add kbimg-rs framework Currently, KubeOS vm/pxe/upgrade images are built by kbimg bash scripts which is hard for users to customize their own OS. Hence, a Rust binary kbimg is developed for dynamically generate scripts for build OS from users' TOML configuration. Signed-off-by: Yuhang Wei --- .taplo.toml | 6 + KubeOS-Rust/Cargo.lock | 148 ++++++++++++++++++++++- KubeOS-Rust/Cargo.toml | 2 +- KubeOS-Rust/kbimg/Cargo.toml | 15 +++ KubeOS-Rust/kbimg/kbimg.toml | 43 +++++++ KubeOS-Rust/kbimg/src/admin_container.rs | 12 ++ KubeOS-Rust/kbimg/src/commands.rs | 122 +++++++++++++++++++ KubeOS-Rust/kbimg/src/docker_img.rs | 12 ++ KubeOS-Rust/kbimg/src/main.rs | 117 ++++++++++++++++++ KubeOS-Rust/kbimg/src/repo.rs | 12 ++ KubeOS-Rust/kbimg/src/utils.rs | 62 ++++++++++ KubeOS-Rust/kbimg/src/values.rs | 2 + 12 files changed, 551 insertions(+), 2 deletions(-) create mode 100644 .taplo.toml create mode 100644 KubeOS-Rust/kbimg/Cargo.toml create mode 100644 KubeOS-Rust/kbimg/kbimg.toml create mode 100644 KubeOS-Rust/kbimg/src/admin_container.rs create mode 100644 KubeOS-Rust/kbimg/src/commands.rs create mode 100644 KubeOS-Rust/kbimg/src/docker_img.rs create mode 100644 KubeOS-Rust/kbimg/src/main.rs create mode 100644 KubeOS-Rust/kbimg/src/repo.rs create mode 100644 KubeOS-Rust/kbimg/src/utils.rs create mode 100644 KubeOS-Rust/kbimg/src/values.rs diff --git a/.taplo.toml b/.taplo.toml new file mode 100644 index 00000000..2af124ff --- /dev/null +++ b/.taplo.toml @@ -0,0 +1,6 @@ +[[rule]] + +[rule.formatting] +indent_string = " " +reorder_arrays = true +reorder_keys = true diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock index 93e3d07d..4ffc2bc2 100644 --- a/KubeOS-Rust/Cargo.lock +++ b/KubeOS-Rust/Cargo.lock @@ -187,6 +187,45 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "clap" +version = "3.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive", + "clap_lex", + "indexmap 1.9.3", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "cli" version = "1.0.6" @@ -632,6 +671,12 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1067,6 +1112,19 @@ dependencies = [ "url", ] +[[package]] +name = "kbimg" +version = "1.0.5" +dependencies = [ + "anyhow", + "clap", + "env_logger", + "log", + "regex", + "serde", + "toml 0.7.6", +] + [[package]] name = "kube" version = "0.66.0" @@ -1453,6 +1511,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "parity-tokio-ipc" version = "0.9.0" @@ -1587,7 +1651,31 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" dependencies = [ - "toml", + "toml 0.5.11", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", ] [[package]] @@ -2013,6 +2101,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2194,6 +2291,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + [[package]] name = "thiserror" version = "1.0.50" @@ -2371,6 +2474,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +dependencies = [ + "indexmap 2.2.6", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -2825,6 +2962,15 @@ version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml index 68ee670a..2886023f 100644 --- a/KubeOS-Rust/Cargo.toml +++ b/KubeOS-Rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["agent", "cli", "manager", "proxy"] +members = ["agent", "cli", "kbimg", "manager", "proxy"] resolver = "2" [profile.release] diff --git a/KubeOS-Rust/kbimg/Cargo.toml b/KubeOS-Rust/kbimg/Cargo.toml new file mode 100644 index 00000000..db206ea5 --- /dev/null +++ b/KubeOS-Rust/kbimg/Cargo.toml @@ -0,0 +1,15 @@ +[package] +description = "KubeOS kbimg" +edition = "2021" +license = "MulanPSL-2.0" +name = "kbimg" +version = "1.0.5" + +[dependencies] +anyhow = { version = "1.0" } +clap = { version = "=3.2.23", features = ["derive"] } +env_logger = { version = "0.9" } +log = { version = "= 0.4.15" } +regex = { version = "1.7.3" } +serde = { version = "1.0", features = ["derive"] } +toml = { version = "=0.7.6" } diff --git a/KubeOS-Rust/kbimg/kbimg.toml b/KubeOS-Rust/kbimg/kbimg.toml new file mode 100644 index 00000000..f57c6578 --- /dev/null +++ b/KubeOS-Rust/kbimg/kbimg.toml @@ -0,0 +1,43 @@ +[from_repo] +agent_path = "./agent" +image_type = "vm-repo" +legacy_bios = true +repo_path = "./openEuler.repo" +root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0" +version = "v1" + +[from_dockerimg] +image_name = "" +image_type = "vm-docker" + +[admin_container] +dockerfile = "" +image_name = "" + +[[users]] +groups = ["admin"] +name = "foo" +passwd = "foo" +sudo = "ALL=(ALL) ALL" + +[[users]] +groups = ["example"] +name = "bar" +passwd = "bar" + +[[copy_files]] +dst = "/persist/dst-file1" +src = "./src-file1" + +[[copy_files]] +dst = "/persist/dst-file2" +src = "./src-file2" + +[grub] +passwd = "foo" + +[systemd_service] +name = ["containerd", "kubelet"] + +[chroot_script] +path = "./chroot.sh" diff --git a/KubeOS-Rust/kbimg/src/admin_container.rs b/KubeOS-Rust/kbimg/src/admin_container.rs new file mode 100644 index 00000000..e611c72c --- /dev/null +++ b/KubeOS-Rust/kbimg/src/admin_container.rs @@ -0,0 +1,12 @@ +use std::path::PathBuf; + +use crate::{commands::AdminContainerInfo, Config, CreateImage}; + +impl CreateImage for AdminContainerInfo { + fn perpare(&self) -> anyhow::Result<()> { + todo!() + } + fn generate_scripts(&self, config: Config) -> anyhow::Result { + todo!() + } +} diff --git a/KubeOS-Rust/kbimg/src/commands.rs b/KubeOS-Rust/kbimg/src/commands.rs new file mode 100644 index 00000000..a68899bc --- /dev/null +++ b/KubeOS-Rust/kbimg/src/commands.rs @@ -0,0 +1,122 @@ +use std::path::PathBuf; + +use clap::{Args, Parser, Subcommand}; +use serde::Deserialize; + +#[derive(Parser)] +#[clap(name = "kbimg")] +#[clap(author, version, about)] +#[clap(long_about = "A tool for creating KubeOS images.")] +pub struct Cli { + /// Path to the detailed configuration toml file + #[clap(short, long, value_parser)] + pub config: Option, + /// Enable debug mode, keep the scripts after execution + #[clap(short, long, action)] + pub debug: bool, + #[clap(subcommand)] + pub commands: Option, +} + +#[derive(Subcommand, Debug, Deserialize)] +pub enum Commands { + /// Create a new container image for upgrading KubeOS + #[clap(name = "upgrade-image")] + UpgradeImage(RepoInfo), + /// Create a new KubeOS vm image from repo + #[clap(name = "vm-repo")] + VMRepo(RepoInfo), + /// Create a new KubeOS vm image from docker image + #[clap(name = "vm-docker")] + VMDocker(DockerInfo), + /// Create a new KubeOS pxe image from repo + #[clap(name = "pxe-repo")] + PxeRepo(RepoInfo), + /// Create a new KubeOS pxe image from docker image + #[clap(name = "pxe-docker")] + PxeDocker(DockerInfo), + /// Create a KubeOS admin-container image + #[clap(name = "admin-container")] + AdminContainer(AdminContainerInfo), +} + +#[derive(Args, Debug, Deserialize, Clone)] +pub struct RepoInfo { + /// Required: KubeOS version + #[clap(short, long, value_parser)] + pub version: String, + /// Required: Repo path for installing packages + #[clap(short = 'p', long, value_parser)] + pub repo_path: PathBuf, + /// Required: Path to the agent binary + #[clap(short = 'b', long, value_parser)] + pub agent_path: PathBuf, + /// Required: Encrypted password for root user + #[clap(short = 'e', long, value_parser)] + pub root_passwd: String, + /// Optional: boot mode, default is uefi, enable this flag for legacy bios + #[clap(short, long, value_parser)] + pub legacy_bios: bool, + #[clap(skip)] + pub image_type: String, +} + +#[derive(Args, Debug, Deserialize, Clone)] +pub struct DockerInfo { + /// Required: Name of the container image + #[clap(short, long, value_parser)] + pub image_name: String, + #[clap(skip)] + pub image_type: String, +} + +#[derive(Args, Debug, Deserialize, Clone)] +pub struct AdminContainerInfo { + /// Required: Name of the container image + #[clap(short, long, value_parser)] + pub image_name: String, + /// Required: Path to the Dockerfile + #[clap(short, long, value_parser)] + pub dockerfile: PathBuf, +} + +#[derive(Debug, Deserialize, Default, Clone)] +pub struct Config { + pub from_repo: Option, + pub from_dockerimg: Option, + pub admin_container: Option, + pub users: Option>, + pub copy_files: Option>, + pub grub: Option, + pub systemd_service: Option, + pub chroot_script: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct User { + pub name: String, + pub passwd: String, + pub groups: Option>, + pub sudo: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct CopyFile { + pub src: String, + pub dst: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Grub { + pub passwd: Option, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct SystemdService { + pub name: Vec, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct ChrootScript { + pub path: String, +} diff --git a/KubeOS-Rust/kbimg/src/docker_img.rs b/KubeOS-Rust/kbimg/src/docker_img.rs new file mode 100644 index 00000000..d65cf023 --- /dev/null +++ b/KubeOS-Rust/kbimg/src/docker_img.rs @@ -0,0 +1,12 @@ +use std::path::PathBuf; + +use crate::{commands::DockerInfo, Config, CreateImage}; + +impl CreateImage for DockerInfo { + fn perpare(&self) -> anyhow::Result<()> { + todo!() + } + fn generate_scripts(&self, config: Config) -> anyhow::Result { + todo!() + } +} diff --git a/KubeOS-Rust/kbimg/src/main.rs b/KubeOS-Rust/kbimg/src/main.rs new file mode 100644 index 00000000..e0f8e66b --- /dev/null +++ b/KubeOS-Rust/kbimg/src/main.rs @@ -0,0 +1,117 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +use std::{fs, path::PathBuf, process::exit}; + +use anyhow::Result; +use clap::Parser; +use env_logger::{Builder, Env, Target}; +use log::{debug, error, info}; + +mod admin_container; +mod commands; +mod docker_img; +mod repo; +mod utils; +mod values; + +use utils::execute_scripts; + +use crate::commands::{Cli, Commands, Config}; + +trait CreateImage { + /// validate cmd args, check disk size and other prepare work + fn perpare(&self) -> Result<()>; + /// generate scripts for creating image. If debug is enabled, keep the scripts, otherwise execute them + fn generate_scripts(&self, config: Config) -> Result; +} + +fn process(info: Box, config: Config) -> Result<()> { + info.perpare()?; + let path = info.generate_scripts(config)?; + execute_scripts(path)?; + Ok(()) +} + +fn main() { + let cli = Cli::parse(); + if cli.debug { + Builder::from_env(Env::default().default_filter_or("debug")).target(Target::Stdout).init(); + } else { + Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init(); + } + match cli.config { + Some(config) => { + info!("Loading config file"); + debug!("Config file path: {:?}", config); + let content = fs::read_to_string(config).unwrap(); + let data: Config = toml::from_str(&content).unwrap(); + debug!("Config: {:?}", data); + let info = if let Some(info) = data.from_repo.clone() { + Some(Box::new(info) as Box) + } else if let Some(info) = data.from_dockerimg.clone() { + Some(Box::new(info) as Box) + } else if let Some(info) = data.admin_container.clone() { + Some(Box::new(info) as Box) + } else { + None + }; + if let Some(i) = info { + match process(i, data) { + Ok(_) => { + info!("Image created successfully"); + }, + Err(e) => { + error!("Failed to create image: {:?}", e); + }, + } + } + exit(0); + }, + None => {}, + } + let info = match cli.commands { + Some(Commands::UpgradeImage(mut info)) => { + info.image_type = "upgrade".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::VMRepo(mut info)) => { + info.image_type = "vm-repo".to_string(); + debug!("VMRepo: {:?}", info); + Some(Box::new(info) as Box) + }, + Some(Commands::VMDocker(mut info)) => { + info.image_type = "vm-docker".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::PxeRepo(mut info)) => { + info.image_type = "pxe-repo".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::PxeDocker(mut info)) => { + info.image_type = "pxe-docker".to_string(); + Some(Box::new(info) as Box) + }, + Some(Commands::AdminContainer(info)) => Some(Box::new(info) as Box), + None => None, + }; + if let Some(i) = info { + match process(i, Config::default()) { + Ok(_) => { + info!("Image created successfully"); + }, + Err(e) => { + error!("Failed to create image: {:?}", e); + }, + } + } +} diff --git a/KubeOS-Rust/kbimg/src/repo.rs b/KubeOS-Rust/kbimg/src/repo.rs new file mode 100644 index 00000000..cdc8ac1b --- /dev/null +++ b/KubeOS-Rust/kbimg/src/repo.rs @@ -0,0 +1,12 @@ +use std::path::PathBuf; + +use crate::{commands::RepoInfo, Config, CreateImage}; + +impl CreateImage for RepoInfo { + fn perpare(&self) -> anyhow::Result<()> { + todo!() + } + fn generate_scripts(&self, config: Config) -> anyhow::Result { + todo!() + } +} diff --git a/KubeOS-Rust/kbimg/src/utils.rs b/KubeOS-Rust/kbimg/src/utils.rs new file mode 100644 index 00000000..2983a41f --- /dev/null +++ b/KubeOS-Rust/kbimg/src/utils.rs @@ -0,0 +1,62 @@ +use std::{fs::File, io::Write, path::PathBuf, process::Command}; + +use anyhow::bail; +use log::{debug, error, info, trace}; + +pub(crate) fn execute_scripts(script: PathBuf) -> anyhow::Result<()> { + if !script.exists() { + bail!("Script does not exist: {:?}", script); + } + let status = Command::new("bash").arg(&script).status()?; + if !status.success() { + bail!("Failed to execute script: {}\n", script.display()); + } + Ok(()) +} + +pub(crate) fn write_vector_to_file(scripts: Vec, file_name: &str) -> anyhow::Result<()> { + debug!("Writing scripts to file: {:?}", file_name); + let mut file = File::create(file_name)?; + // set permissions based on regulation + for line in scripts { + write!(file, "{}", line)?; + } + Ok(()) +} + +/// Check if the input parameter is valid +pub(crate) fn is_valid_param + std::fmt::Debug>(param: S) -> bool { + let special_chars = vec!["|", ";", "&", "&&", "||", ">", ">>", "<", ",", "#", "!", "$"]; + !param.as_ref().chars().any(|c| special_chars.contains(&c.to_string().as_str())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init() { + let _ = env_logger::builder() + .target(env_logger::Target::Stdout) + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[test] + fn test_is_valid_param() { + init(); + assert_eq!(is_valid_param("test"), true); + assert_eq!(is_valid_param("test|test"), false); + assert_eq!(is_valid_param("test;test"), false); + assert_eq!(is_valid_param("test&test"), false); + assert_eq!(is_valid_param("test&&test"), false); + assert_eq!(is_valid_param("test||test"), false); + assert_eq!(is_valid_param("test>test"), false); + assert_eq!(is_valid_param("test>>test"), false); + assert_eq!(is_valid_param("test