From f6733a835afaeab8daaae85b4c2b50926a89b636 Mon Sep 17 00:00:00 2001 From: Yuting Nie Date: Sat, 15 Jul 2023 12:12:32 +0800 Subject: [PATCH] Complete rk8s 0.2.0 development Signed-off-by: Yuting Nie --- rk8s/Cargo.toml | 15 + rk8s/LICENSE-APACHE | 201 ++++++++++ rk8s/LICENSE-MIT | 21 ++ rk8s/LICENSE-THIRD-PARTY | 0 rk8s/README.md | 66 ++++ rk8s/src/config.rs | 195 ++++++++++ rk8s/src/deploy/docker.rs | 170 +++++++++ rk8s/src/deploy/etcd.rs | 409 +++++++++++++++++++++ rk8s/src/deploy/kube_apiserver.rs | 376 +++++++++++++++++++ rk8s/src/deploy/kube_controller_manager.rs | 231 ++++++++++++ rk8s/src/deploy/kube_proxy.rs | 274 ++++++++++++++ rk8s/src/deploy/kube_scheduler.rs | 220 +++++++++++ rk8s/src/deploy/kubectl.rs | 151 ++++++++ rk8s/src/deploy/kubelet.rs | 240 ++++++++++++ rk8s/src/deploy/mod.rs | 10 + rk8s/src/deploy/pre_check.rs | 101 +++++ rk8s/src/deploy/prepare_kube.rs | 113 ++++++ rk8s/src/init/etcd.rs | 360 ++++++++++++++++++ rk8s/src/init/kube_apiserver.rs | 365 ++++++++++++++++++ rk8s/src/init/kube_controller_manager.rs | 218 +++++++++++ rk8s/src/init/kube_ctl.rs | 154 ++++++++ rk8s/src/init/kube_let.rs | 229 ++++++++++++ rk8s/src/init/kube_proxy.rs | 254 +++++++++++++ rk8s/src/init/kube_scheduler.rs | 205 +++++++++++ rk8s/src/init/mod.rs | 8 + rk8s/src/init/pre_check.rs | 87 +++++ rk8s/src/install/cfssl.rs | 59 +++ rk8s/src/install/docker.rs | 139 +++++++ rk8s/src/install/etcd.rs | 56 +++ rk8s/src/install/kubernetes.rs | 89 +++++ rk8s/src/install/mod.rs | 4 + rk8s/src/join/etcd.rs | 90 +++++ rk8s/src/join/mod.rs | 1 + rk8s/src/main.rs | 14 + rk8s/src/rk8s.rs | 172 +++++++++ 35 files changed, 5297 insertions(+) create mode 100644 rk8s/Cargo.toml create mode 100644 rk8s/LICENSE-APACHE create mode 100644 rk8s/LICENSE-MIT create mode 100644 rk8s/LICENSE-THIRD-PARTY create mode 100644 rk8s/README.md create mode 100644 rk8s/src/config.rs create mode 100644 rk8s/src/deploy/docker.rs create mode 100644 rk8s/src/deploy/etcd.rs create mode 100644 rk8s/src/deploy/kube_apiserver.rs create mode 100644 rk8s/src/deploy/kube_controller_manager.rs create mode 100644 rk8s/src/deploy/kube_proxy.rs create mode 100644 rk8s/src/deploy/kube_scheduler.rs create mode 100644 rk8s/src/deploy/kubectl.rs create mode 100644 rk8s/src/deploy/kubelet.rs create mode 100644 rk8s/src/deploy/mod.rs create mode 100644 rk8s/src/deploy/pre_check.rs create mode 100644 rk8s/src/deploy/prepare_kube.rs create mode 100644 rk8s/src/init/etcd.rs create mode 100644 rk8s/src/init/kube_apiserver.rs create mode 100644 rk8s/src/init/kube_controller_manager.rs create mode 100644 rk8s/src/init/kube_ctl.rs create mode 100644 rk8s/src/init/kube_let.rs create mode 100644 rk8s/src/init/kube_proxy.rs create mode 100644 rk8s/src/init/kube_scheduler.rs create mode 100644 rk8s/src/init/mod.rs create mode 100644 rk8s/src/init/pre_check.rs create mode 100644 rk8s/src/install/cfssl.rs create mode 100644 rk8s/src/install/docker.rs create mode 100644 rk8s/src/install/etcd.rs create mode 100644 rk8s/src/install/kubernetes.rs create mode 100644 rk8s/src/install/mod.rs create mode 100644 rk8s/src/join/etcd.rs create mode 100644 rk8s/src/join/mod.rs create mode 100644 rk8s/src/main.rs create mode 100644 rk8s/src/rk8s.rs diff --git a/rk8s/Cargo.toml b/rk8s/Cargo.toml new file mode 100644 index 00000000..8e54bf92 --- /dev/null +++ b/rk8s/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rk8s" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = { version = "4.3.0", features = ["derive"] } +regex = "1.8.3" +serde = { version = "1.0.163", features = ["derive"] } +serde_json = "1.0.96" +serde_yaml = "0.9.21" +tracing = "0.1.37" +tracing-subscriber = "0.3.17" diff --git a/rk8s/LICENSE-APACHE b/rk8s/LICENSE-APACHE new file mode 100644 index 00000000..b651cf1c --- /dev/null +++ b/rk8s/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/LICENSE-2.0 + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2022 Open Rust Initiative + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/rk8s/LICENSE-MIT b/rk8s/LICENSE-MIT new file mode 100644 index 00000000..91127361 --- /dev/null +++ b/rk8s/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Open Rust Initiative + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/rk8s/LICENSE-THIRD-PARTY b/rk8s/LICENSE-THIRD-PARTY new file mode 100644 index 00000000..e69de29b diff --git a/rk8s/README.md b/rk8s/README.md new file mode 100644 index 00000000..69b8e79c --- /dev/null +++ b/rk8s/README.md @@ -0,0 +1,66 @@ +# Rk8s - A Kubernetes Distribution with Rust + +For now, we are woking on a deploy tool helps you set up a working kubernetes cluster in just a few steps. + +## Usage + +Prerequisite: + +[CentOS Stream 9](https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/iso/): the currently tested working distro, since the rust container runtime [youki](https://github.com/containers/youki) has a limit on linux kernel version. + +gcc: compiler which rustc depends on. + +[cargo](https://rustup.rs/) command available: which means you need to have rust installed. + +cfssl: If you do not have one installed, you can run `rk8s install cfssl` after the configuration is generated. + +First, clone & build the crate: + +```bash +git clone https://github.com/open-rust-initiative/rk8s.git +cd rk8s +cargo build +cp target/debug/rk8s /usr/bin +``` + +Then you should now have a working `rk8s` ready for deploying. + +> The `rk8s` command needs root privilege. + +#### Deploying outside the cluster: + +1. `rk8s generate config` will generate a folder named `rk8s` under `/root` directory. + +2. Change the content in `/root/rk8s/cfg/config.yaml`, specify the machines' IP addresses and their according roles (master or worker) in `instance_hosts` filed, if your deploying machine (the machine running `rk8s`) will be outside of cluster, then `instance_ip` and `instance_name` fields are irrelevant. + +3. `ssh-keygen` to generate a key for ssh connection across machines, and `ssh-copy-id -i root@` notifies machines to be deployed. + +4. `rk8s install cfssl` to install cfssl-related tools for later use. + +5. Run `rk8s deploy`. + +Then you should have a working cluster, ssh to the master node and run `kubectl get nodes`, you should see the master node is ready. + +#### Deploying inside the cluster: + +1. Run `rk8s generate config` on every machine to be deployed. + +2. Run `rk8s precheck` to turn off *selinux*, *firewall* and so forth. + +3. Change the content in `/root/rk8s/cfg/config.yaml`, specify the machines' IP addresses and their according roles (master or worker) in `instance_hosts` filed, `instance_ip` and `instance_name` fields need to be set correctly according to the `IP` and `hostname` of the current machine. + +4. Run `rk8s install etcd` on each instance. + +5. For masters run: + + ```bash + $ rk8s install kubernetes + $ rk8s install docker + ``` + +6. Run `rk8s init` on master node. + +7. Run `rk8s join` on worker node. + +After `rk8s` finishes its job, the cluster is now set, the deploying machine is inside the cluster. + diff --git a/rk8s/src/config.rs b/rk8s/src/config.rs new file mode 100644 index 00000000..a3e36043 --- /dev/null +++ b/rk8s/src/config.rs @@ -0,0 +1,195 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{Read, Write}; + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +pub struct Config { + // The machine running this program. + pub instance_name: String, + // The ip address of running server. + pub instance_ip: String, + pub instance_hosts: HashMap, + + // Fields needed by `install cfssl` command. + pub cfssl_url: String, + pub cfssljson_url: String, + pub cfsslcertinfo_url: String, + // Fields needed by `install etcd` command. + pub etcd_url: String, + // Fields needed by `install docker` command. + pub docker_url: String, + // Fields needed by `install kubernetes` command. + pub kubernetes_url: String, + + // Fields needed by `etcd` phase. + pub etcd_ca_CN: String, + pub etcd_CN: String, + pub etcd_key_algo: String, + pub etcd_key_size: i64, + pub etcd_expiry: String, + pub etcd_usages: Vec, + pub etcd_names_C: String, + pub etcd_names_L: String, + pub etcd_names_ST: String, + + // Fields needed by `kube_apiserver` phase. + pub kube_apiserver_CN: String, + pub kube_apiserver_key_algo: String, + pub kube_apiserver_key_size: i64, + pub kube_apiserver_expiry: String, + pub kube_apiserver_usages: Vec, + pub kube_apiserver_names_C: String, + pub kube_apiserver_names_L: String, + pub kube_apiserver_names_ST: String, + pub kube_apiserver_names_O: String, + pub kube_apiserver_names_OU: String, + + // Fields needed by `kube_controller_manager` phase. + pub kube_controller_manager_CN: String, + pub kube_controller_manager_key_algo: String, + pub kube_controller_manager_key_size: i64, + pub kube_controller_manager_names_C: String, + pub kube_controller_manager_names_L: String, + pub kube_controller_manager_names_ST: String, + pub kube_controller_manager_names_O: String, + pub kube_controller_manager_names_OU: String, + + // Fields needed by `kube_scheduler` phase. + pub kube_scheduler_CN: String, + pub kube_scheduler_key_algo: String, + pub kube_scheduler_key_size: i64, + pub kube_scheduler_names_C: String, + pub kube_scheduler_names_L: String, + pub kube_scheduler_names_ST: String, + pub kube_scheduler_names_O: String, + pub kube_scheduler_names_OU: String, + + // Fields needed by `kube_ctl` phase. + pub kube_ctl_CN: String, + pub kube_ctl_key_algo: String, + pub kube_ctl_key_size: i64, + pub kube_ctl_names_C: String, + pub kube_ctl_names_L: String, + pub kube_ctl_names_ST: String, + pub kube_ctl_names_O: String, + pub kube_ctl_names_OU: String, + + // Fields needed by `kube_proxy` phase. + pub kube_proxy_CN: String, + pub kube_proxy_key_algo: String, + pub kube_proxy_key_size: i64, + pub kube_proxy_names_C: String, + pub kube_proxy_names_L: String, + pub kube_proxy_names_ST: String, + pub kube_proxy_names_O: String, + pub kube_proxy_names_OU: String, +} + +impl Config { + pub fn init() -> Config { + tracing::info!("Reading config file..."); + let mut file = File::open("cfg/config.yaml").expect("File `config.yaml` does not exist!"); + let mut content = vec![]; + file.read_to_end(&mut content) + .expect("Error happened when trying to read content of `config.yaml`"); + let config = serde_yaml::from_slice(&content) + .expect("Something went wrong while parsing config.yaml"); + tracing::info!("Config read"); + config + } +} + +pub fn generate_config_template() { + let config = Config { + instance_name: "master01".to_owned(), + instance_ip: "192.168.157.130".to_owned(), + instance_hosts: { + let mut map = HashMap::new(); + map.insert( + "192.168.157.130".to_owned(), + "master01".to_owned(), + ); + map + }, + + cfssl_url: "https://pkg.cfssl.org/R1.2/cfssl_linux-amd64".to_owned(), + cfssljson_url: "https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64".to_owned(), + cfsslcertinfo_url: "https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64".to_owned(), + etcd_url: "https://github.com/etcd-io/etcd/releases/download/v3.4.9/etcd-v3.4.9-linux-amd64.tar.gz".to_owned(), + docker_url: "https://download.docker.com/linux/static/stable/x86_64/docker-20.10.9.tgz".to_owned(), + kubernetes_url: "https://dl.k8s.io/v1.20.15/kubernetes-server-linux-amd64.tar.gz".to_owned(), + + etcd_ca_CN: "etcd CA".to_owned(), + etcd_CN: "etcd".to_owned(), + etcd_key_algo: "rsa".to_owned(), + etcd_key_size: 2048, + etcd_expiry: "87600h".to_owned(), + etcd_usages: vec![ + "signing".to_owned(), + "key encipherment".to_owned(), + "server auth".to_owned(), + "client auth".to_owned(), + ], + etcd_names_C: "CN".to_owned(), + etcd_names_L: "Beijing".to_owned(), + etcd_names_ST: "Beijing".to_owned(), + + kube_apiserver_CN: "kubernetes".to_owned(), + kube_apiserver_key_algo: "rsa".to_owned(), + kube_apiserver_key_size: 2048, + kube_apiserver_expiry: "87600h".to_owned(), + kube_apiserver_usages: vec![ + "signing".to_owned(), + "key encipherment".to_owned(), + "server auth".to_owned(), + "client auth".to_owned(), + ], + kube_apiserver_names_C: "CN".to_owned(), + kube_apiserver_names_L: "Beijing".to_owned(), + kube_apiserver_names_ST: "Beijing".to_owned(), + kube_apiserver_names_O: "k8s".to_owned(), + kube_apiserver_names_OU: "System".to_owned(), + + kube_controller_manager_CN: "system:kube-controller-manager".to_owned(), + kube_controller_manager_key_algo: "rsa".to_owned(), + kube_controller_manager_key_size: 2048, + kube_controller_manager_names_C: "CN".to_owned(), + kube_controller_manager_names_L: "Beijing".to_owned(), + kube_controller_manager_names_ST: "Beijing".to_owned(), + kube_controller_manager_names_O: "system:masters".to_owned(), + kube_controller_manager_names_OU: "System".to_owned(), + + kube_scheduler_CN: "system:kube-scheduler".to_owned(), + kube_scheduler_key_algo: "rsa".to_owned(), + kube_scheduler_key_size: 2048, + kube_scheduler_names_C: "CN".to_owned(), + kube_scheduler_names_L: "Beijing".to_owned(), + kube_scheduler_names_ST: "Beijing".to_owned(), + kube_scheduler_names_O: "system:masters".to_owned(), + kube_scheduler_names_OU: "System".to_owned(), + + kube_ctl_CN: "admin".to_owned(), + kube_ctl_key_algo: "rsa".to_owned(), + kube_ctl_key_size: 2048, + kube_ctl_names_C: "CN".to_owned(), + kube_ctl_names_L: "Beijing".to_owned(), + kube_ctl_names_ST: "Beijing".to_owned(), + kube_ctl_names_O: "system:masters".to_owned(), + kube_ctl_names_OU: "System".to_owned(), + + kube_proxy_CN: "system:kube-proxy".to_owned(), + kube_proxy_key_algo: "rsa".to_owned(), + kube_proxy_key_size: 2048, + kube_proxy_names_C: "CN".to_owned(), + kube_proxy_names_L: "Beijing".to_owned(), + kube_proxy_names_ST: "Beijing".to_owned(), + kube_proxy_names_O: "k8s".to_owned(), + kube_proxy_names_OU: "System".to_owned(), + }; + + let yaml = serde_yaml::to_string(&config).unwrap(); + let mut file = File::create("cfg/config.yaml").unwrap(); + file.write_all(yaml.as_bytes()).unwrap(); +} diff --git a/rk8s/src/deploy/docker.rs b/rk8s/src/deploy/docker.rs new file mode 100644 index 00000000..2d579287 --- /dev/null +++ b/rk8s/src/deploy/docker.rs @@ -0,0 +1,170 @@ +use std::env; +use std::fs; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::config::Config; + +struct DockerCfg; + +impl DockerCfg { + fn generate() { + // Send to /etc/docker/daemon.json + let mut docker_cfg = File::create("to_send/daemon.json") + .expect("Error happened when trying to create docker config file"); + // Set youki (pre-built) as the default runtime. + let content = r#"{ + "default-runtime": "youki", + "runtimes": { + "youki": { + "path": "/usr/bin/youki" + } + }, + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver": "json-file", + "log-opts": { + "max-size": "100m" + }, + "storage-driver": "overlay2" +} +"#; + docker_cfg + .write_all(content.as_bytes()) + .expect("Error happened when trying to write docker unit file"); + } +} + +struct DockerUnit; + +impl DockerUnit { + fn generate() { + let mut docker_unit = File::create("to_send/docker.service") + .expect("Error happened when trying to create docker unit file"); + let content = r#"[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +After=network-online.target firewalld.service +Wants=network-online.target + +[Service] +Type=notify +ExecStart=/usr/bin/dockerd +ExecReload=/bin/kill -s HUP $MAINPID +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TimeoutStartSec=0 +Delegate=yes +KillMode=process +Restart=on-failure +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target +"#; + docker_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write docker unit file"); + } +} + +pub fn start(config: &Config) { + // Deploy docker to all hosts according to their name. + // Docker does not distinguish masters or workers. + tracing::info!("Preparing mutual .json, .service and docker binaries..."); + tracing::info!("Change working directory into `docker`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/docker"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `etcd`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + // Prepare directory to be sent. + let path = PathBuf::from("to_send/"); + check_dir_exist_or_create(path); + + tracing::info!("untaring docker binaries..."); + Command::new("tar") + .arg("-zxf") + .arg("/rk8s/preparation/docker-20.10.9.tgz") + .status() + .expect("Error happened when trying to untar `docker` executable"); + + tracing::info!("Generating docker.service file to to_send/"); + DockerUnit::generate(); + tracing::info!("docker.service generated"); + + tracing::info!("Generating daemon.json to to_send/"); + DockerCfg::generate(); + tracing::info!("daemon.json generated"); + + for (ip, name) in &config.instance_hosts { + tracing::info!("Found instance {} on {},", name, ip); + + Command::new("scp") + .arg("docker/containerd") + .arg("docker/containerd-shim") + .arg("docker/containerd-shim-runc-v2") + .arg("docker/ctr") + .arg("docker/docker") + .arg("docker/dockerd") + .arg("docker/docker-init") + .arg("docker/docker-proxy") + .arg("docker/runc") + // Send youki along with the process + .arg("/rk8s/preparation/youki") + .arg(format!("root@{}:/usr/bin", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("mkdir /etc/docker") + .status() + .expect("Error happened when trying to create config directory on other nodes"); + + Command::new("scp") + .arg("to_send/daemon.json") + .arg(format!("root@{}:/etc/docker", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + + Command::new("scp") + .arg("to_send/docker.service") + .arg(format!("root@{}:/usr/lib/systemd/system/", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + + tracing::info!("Docker installed on {}, starting...", name); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl start docker") + .status() + .expect("Error happened when trying to start docker"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl enable docker") + .status() + .expect("Error happened when trying to enable docker"); + tracing::info!("Docker started on {}", ip); + } + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `etcd`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} + +fn check_dir_exist_or_create(path: PathBuf) { + if !path.is_dir() { + fs::create_dir_all(path).expect("Error happened when trying to create path"); + } +} diff --git a/rk8s/src/deploy/etcd.rs b/rk8s/src/deploy/etcd.rs new file mode 100644 index 00000000..fc98c950 --- /dev/null +++ b/rk8s/src/deploy/etcd.rs @@ -0,0 +1,409 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::thread; + +#[derive(Serialize, Deserialize, Debug)] +struct CAConfig { + signing: Signing, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Signing { + default: SignDefault, + profiles: SignProfiles, +} + +#[derive(Serialize, Deserialize, Debug)] +struct SignDefault { + expiry: String, +} + +#[derive(Serialize, Deserialize, Debug)] +struct SignProfiles { + www: WWWProfile, +} + +#[derive(Serialize, Deserialize, Debug)] +struct WWWProfile { + expiry: String, + usages: Vec, +} + +impl CAConfig { + fn from(config: &Config) -> CAConfig { + CAConfig { + signing: Signing { + default: SignDefault { + expiry: config.etcd_expiry.to_owned(), + }, + profiles: SignProfiles { + www: WWWProfile { + expiry: config.etcd_expiry.to_owned(), + usages: config.etcd_usages.to_owned(), + }, + }, + }, + } + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct CACsr { + CN: String, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, +} + +impl CACsr { + fn from(config: &Config) -> CACsr { + CACsr { + CN: config.etcd_ca_CN.to_owned(), + key: Key { + algo: config.etcd_key_algo.to_owned(), + size: config.etcd_key_size.to_owned(), + }, + names: vec![Name { + C: config.etcd_names_C.to_owned(), + L: config.etcd_names_L.to_owned(), + ST: config.etcd_names_ST.to_owned(), + }], + } + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct ServerCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +impl ServerCsr { + fn from(config: &Config) -> ServerCsr { + ServerCsr { + CN: config.etcd_CN.to_owned(), + hosts: { + let mut hosts = Vec::new(); + for ip in config.instance_hosts.keys() { + hosts.push(ip.to_owned()); + } + hosts + }, + key: Key { + algo: config.etcd_key_algo.to_owned(), + size: config.etcd_key_size.to_owned(), + }, + names: vec![Name { + C: config.etcd_names_C.to_owned(), + L: config.etcd_names_L.to_owned(), + ST: config.etcd_names_ST.to_owned(), + }], + } + } +} + +struct ETCDCfg; + +impl ETCDCfg { + fn generate(current_ip: &String, current_name: &String, config: &Config) { + let mut etcd_conf = File::create(format!("to_send/{}/etcd.conf", current_ip)) + .expect("Error happened when trying to create etcd configuration file"); + + writeln!(&mut etcd_conf, "#[Member]") + .expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf, "ETCD_NAME=\"etcd_{}\"", current_name) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_DATA_DIR=\"/var/lib/etcd/default.etcd\"" + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_LISTEN_PEER_URLS=\"https://{}:2380\"", + current_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_LISTEN_CLIENT_URLS=\"https://{}:2379\"", + current_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf).expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf, "#[Clustering]") + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_INITIAL_ADVERTISE_PEER_URLS=\"https://{}:2380\"", + current_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_ADVERTISE_CLIENT_URLS=\"https://{}:2379\"", + current_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + let mut buffer = String::new(); + for (ip, name) in &config.instance_hosts { + buffer.push_str(format!("etcd_{}=https://{}:2380,", name, ip).as_str()); + } + buffer.pop(); + writeln!(&mut etcd_conf, "ETCD_INITIAL_CLUSTER=\"{}\"", buffer) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_INITIAL_CLUSTER_TOKEN=\"etcd-cluster\"" + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf, "ETCD_INITIAL_CLUSTER_STATE=\"new\"") + .expect("Error happened when trying to write `etcd.conf`"); + } +} + +struct ETCDUnit; + +impl ETCDUnit { + fn generate() { + let mut etcd_unit = File::create("to_send/etcd.service") + .expect("Error happened when trying to create etcd unit file"); + let content = r#"[Unit] +Description=Etcd Server +After=network.target +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +EnvironmentFile=/opt/etcd/cfg/etcd.conf +ExecStart=/opt/etcd/bin/etcd \ +--cert-file=/opt/etcd/ssl/server.pem \ +--key-file=/opt/etcd/ssl/server-key.pem \ +--peer-cert-file=/opt/etcd/ssl/server.pem \ +--peer-key-file=/opt/etcd/ssl/server-key.pem \ +--trusted-ca-file=/opt/etcd/ssl/ca.pem \ +--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \ +--logger=zap +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +"#; + etcd_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write etcd unit file"); + } +} + +pub fn start(config: &Config) { + // Deploy etcd to all hosts according to their name. + // Etcd does not distinguish masters or workers. + tracing::info!("Preparing mutual .pem, .service and etcd binaries..."); + tracing::info!("Change working directory into `etcd`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/etcd"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `etcd`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `ca-config.json`..."); + let ca_config = CAConfig::from(config); + let content = serde_json::to_string_pretty(&ca_config) + .expect("Error happened when trying to serialize `ca-config.json`"); + let mut ca_config_file = File::create("ca-config.json") + .expect("Error happened when trying to create `ca-config.json`"); + ca_config_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `ca-config.json`"); + tracing::info!("`ca-config.json` generated"); + + tracing::info!("Start generating `ca-config.json`..."); + let ca_csr = CACsr::from(config); + let content = serde_json::to_string_pretty(&ca_csr) + .expect("Error happened when trying to serialize `ca-csr.json`"); + let mut ca_csr_file = + File::create("ca-csr.json").expect("Error happened when trying to create `ca-csr.json`"); + ca_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `ca-csr.json`"); + tracing::info!("`ca-csr.json` generated"); + + tracing::info!("Generating self-signed CA certificate..."); + let cfssl_ca = Command::new("cfssl") + .arg("gencert") + .arg("-initca") + .arg("ca-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("ca") + .arg("-") + .stdin(Stdio::from(cfssl_ca.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed CA certificate generated"); + + tracing::info!("Start generating `server-csr.json`..."); + let server_csr = ServerCsr::from(config); + let content = serde_json::to_string_pretty(&server_csr) + .expect("Error happened when trying to serialize `server-csr.json`"); + let mut server_csr_file = File::create("server-csr.json") + .expect("Error happened when trying to create `server-csr.json`"); + server_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `server-csr.json`"); + tracing::info!("`server-csr.json` generated"); + + tracing::info!("Generating self-signed etcd https certificate..."); + let cfssl_etcd = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=www") + .arg("server-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("server") + .stdin(Stdio::from(cfssl_etcd.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed CA certificate generated"); + + // Prepare directory to be sent. + let cfg_path = PathBuf::from("to_send/etcd/cfg"); + check_dir_exist_or_create(cfg_path); + let bin_path = PathBuf::from("to_send/etcd/bin"); + check_dir_exist_or_create(bin_path); + let ssl_path = PathBuf::from("to_send/etcd/ssl"); + check_dir_exist_or_create(ssl_path); + for ip in config.instance_hosts.keys() { + let path = PathBuf::from("to_send"); + let path = path.join(ip); + check_dir_exist_or_create(path); + } + + tracing::info!("Untaring prepared etcd binary file"); + Command::new("tar") + .arg("-zxf") + .arg("/rk8s/preparation/etcd-v3.4.9-linux-amd64.tar.gz") + .status() + .expect("Error happened when trying to untar `etcd` executable"); + + tracing::info!("Copying binaries to to_send/etcd/bin..."); + Command::new("cp") + .arg("etcd-v3.4.9-linux-amd64/etcd") + .arg("etcd-v3.4.9-linux-amd64/etcdctl") + .arg("to_send/etcd/bin") + .status() + .expect("Error happened when trying to copy binaries to `to_send/etcd/bin`"); + tracing::info!("Binaries prepared"); + + tracing::info!("Copying certificates to to_send/etcd/ssl..."); + Command::new("cp") + .arg("ca.pem") + .arg("ca-key.pem") + .arg("server-key.pem") + .arg("server.pem") + .arg("to_send/etcd/ssl") + .status() + .expect("Error happened when trying to copy certificates to `to_send/etcd/ssl`"); + tracing::info!("Certificates copied"); + + tracing::info!("Generating `etcd.service` to to_send/..."); + ETCDUnit::generate(); + tracing::info!("`etcd.service` prepared"); + + // Sending files to nodes. + let mut handles = Vec::new(); + for (ip, name) in &config.instance_hosts { + tracing::info!("Found instance {} on {},", name, ip); + ETCDCfg::generate(ip, name, config); + + Command::new("scp") + .arg("-r") + .arg("to_send/etcd") + .arg(format!("root@{}:/opt/", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + Command::new("scp") + .arg("to_send/etcd.service") + .arg(format!("root@{}:/usr/lib/systemd/system/", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + Command::new("scp") + .arg(format!("to_send/{}/etcd.conf", ip)) + .arg(format!("root@{}:/opt/etcd/cfg", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + + let concurrent_ip = ip.clone(); + let concurrent_name = name.clone(); + let handle = thread::spawn(move || { + // Starting etcd. + tracing::info!("Etcd installed on {}, starting...", concurrent_name); + Command::new("ssh") + .arg(format!("root@{}", concurrent_ip)) + .arg("systemctl daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("ssh") + .arg(format!("root@{}", concurrent_ip)) + .arg("systemctl start etcd") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("ssh") + .arg(format!("root@{}", concurrent_ip)) + .arg("systemctl enable etcd") + .status() + .expect("Error happened when trying to reload systemd daemons"); + tracing::info!("Etcd started on {}, start...", concurrent_ip); + }); + handles.push(handle); + } + + // Wait for jobs to complete. + for handle in handles { + handle.join().unwrap(); + } + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `etcd`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} + +fn check_dir_exist_or_create(path: PathBuf) { + if !path.is_dir() { + fs::create_dir_all(path).expect("Error happened when trying to create path"); + } +} diff --git a/rk8s/src/deploy/kube_apiserver.rs b/rk8s/src/deploy/kube_apiserver.rs new file mode 100644 index 00000000..b985c297 --- /dev/null +++ b/rk8s/src/deploy/kube_apiserver.rs @@ -0,0 +1,376 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +#[derive(Serialize, Deserialize, Debug)] +struct CAConfig { + signing: Signing, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Signing { + default: SignDefault, + profiles: SignProfiles, +} + +#[derive(Serialize, Deserialize, Debug)] +struct SignDefault { + expiry: String, +} + +#[derive(Serialize, Deserialize, Debug)] +struct SignProfiles { + kubernetes: KubernetesProfile, +} + +#[derive(Serialize, Deserialize, Debug)] +struct KubernetesProfile { + expiry: String, + usages: Vec, +} + +impl CAConfig { + fn from(config: &Config) -> CAConfig { + CAConfig { + signing: Signing { + default: SignDefault { + expiry: config.kube_apiserver_expiry.to_owned(), + }, + profiles: SignProfiles { + kubernetes: KubernetesProfile { + expiry: config.kube_apiserver_expiry.to_owned(), + usages: config.kube_apiserver_usages.to_owned(), + }, + }, + }, + } + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct CACsr { + CN: String, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl CACsr { + fn from(config: &Config) -> CACsr { + CACsr { + CN: config.kube_apiserver_CN.to_owned(), + key: Key { + algo: config.kube_apiserver_key_algo.to_owned(), + size: config.kube_apiserver_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_apiserver_names_C.to_owned(), + L: config.kube_apiserver_names_L.to_owned(), + ST: config.kube_apiserver_names_ST.to_owned(), + O: config.kube_apiserver_names_O.to_owned(), + OU: config.kube_apiserver_names_OU.to_owned(), + }], + } + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct ServerCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +impl ServerCsr { + fn from(config: &Config) -> ServerCsr { + ServerCsr { + CN: config.kube_apiserver_CN.to_owned(), + hosts: { + let mut hosts = vec![ + "10.0.0.1".to_string(), + "127.0.0.1".to_string(), + "kubernetes".to_string(), + "kubernetes.default".to_string(), + "kubernetes.default.svc".to_string(), + "kubernetes.default.svc.cluster".to_string(), + "kubernetes.default.svc.cluster.local".to_string(), + ]; + for ip in config.instance_hosts.keys() { + hosts.push(ip.to_owned()); + } + hosts + }, + key: Key { + algo: config.kube_apiserver_key_algo.to_owned(), + size: config.kube_apiserver_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_apiserver_names_C.to_owned(), + L: config.kube_apiserver_names_L.to_owned(), + ST: config.kube_apiserver_names_ST.to_owned(), + O: config.kube_apiserver_names_O.to_owned(), + OU: config.kube_apiserver_names_OU.to_owned(), + }], + } + } +} + +struct KubeApiserverCfg; + +impl KubeApiserverCfg { + fn generate(current_ip: &String, config: &Config) { + let mut apiserver_conf = File::create(format!( + "to_send/{}/apiserver/kube-apiserver.conf", + current_ip + )) + .expect("Error happened when trying to create kube-apiserver configuration file"); + + writeln!( + &mut apiserver_conf, + r#"KUBE_APISERVER_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \"# + ) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + let mut buffer = String::new(); + for ip in config.instance_hosts.keys() { + buffer.push_str(format!("https://{}:2379,", ip).as_str()); + } + buffer.pop(); + writeln!(&mut apiserver_conf, "--etcd-servers={}", buffer) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + writeln!(&mut apiserver_conf, "--bind-address={}", current_ip) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + writeln!(&mut apiserver_conf, "--secure-port=6443") + .expect("Error happened when trying to write `kube-apiserver.conf`"); + writeln!(&mut apiserver_conf, "--advertise-address={}", current_ip) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + writeln!( + &mut apiserver_conf, +r#"--allow-privileged=true \ +--service-cluster-ip-range=10.0.0.0/24 \ +--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \ +--authorization-mode=RBAC,Node \ +--enable-bootstrap-token-auth=true \ +--token-auth-file=/opt/kubernetes/cfg/token.csv \ +--service-node-port-range=30000-32767 \ +--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \ +--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \ +--tls-cert-file=/opt/kubernetes/ssl/server.pem \ +--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \ +--client-ca-file=/opt/kubernetes/ssl/ca.pem \ +--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \ +--service-account-issuer=api \ +--service-account-signing-key-file=/opt/kubernetes/ssl/server-key.pem \ +--etcd-cafile=/opt/etcd/ssl/ca.pem \ +--etcd-certfile=/opt/etcd/ssl/server.pem \ +--etcd-keyfile=/opt/etcd/ssl/server-key.pem \ +--requestheader-client-ca-file=/opt/kubernetes/ssl/ca.pem \ +--proxy-client-cert-file=/opt/kubernetes/ssl/server.pem \ +--proxy-client-key-file=/opt/kubernetes/ssl/server-key.pem \ +--requestheader-allowed-names=kubernetes \ +--requestheader-extra-headers-prefix=X-Remote-Extra- \ +--requestheader-group-headers=X-Remote-Group \ +--requestheader-username-headers=X-Remote-User \ +--enable-aggregator-routing=true \ +--audit-log-maxage=30 \ +--audit-log-maxbackup=3 \ +--audit-log-maxsize=100 \ +--audit-log-path=/opt/kubernetes/logs/k8s-audit.log""#, + ) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + } +} + +struct KubeApiserverUnit; + +impl KubeApiserverUnit { + fn generate() { + let mut kube_apiserver_unit = File::create("to_send/kube-apiserver.service") + .expect("Error happened when trying to create kube-apiserver unit file"); + let content = r#"[Unit] +Description=Kubernetes API Server +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kube-apiserver.conf +ExecStart=/opt/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS +Restart=on-failure + +[Install] +WantedBy=multi-user.target +"#; + kube_apiserver_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kube-apiserver unit file"); + } +} + +pub fn start(config: &Config) { + tracing::info!("kube_apiserver phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `ca-config.json`..."); + let ca_config = CAConfig::from(config); + let content = serde_json::to_string_pretty(&ca_config) + .expect("Error happened when trying to serialize `ca-config.json`"); + let mut ca_config_file = File::create("ca-config.json") + .expect("Error happened when trying to create `ca-config.json`"); + ca_config_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `ca-config.json`"); + tracing::info!("`ca-config.json` generated"); + + tracing::info!("Start generating `ca-csr.json`..."); + let ca_csr = CACsr::from(config); + let content = serde_json::to_string_pretty(&ca_csr) + .expect("Error happened when trying to serialize `ca-csr.json`"); + let mut ca_csr_file = + File::create("ca-csr.json").expect("Error happened when trying to create `ca-csr.json`"); + ca_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `ca-csr.json`"); + tracing::info!("`ca-csr.json` generated"); + + tracing::info!("Generating self-signed CA certificate..."); + let cfssl_ca = Command::new("cfssl") + .arg("gencert") + .arg("-initca") + .arg("ca-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("ca") + .arg("-") + .stdin(Stdio::from(cfssl_ca.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed CA certificate generated"); + + tracing::info!("Start generating `server-csr.json`..."); + let server_csr = ServerCsr::from(config); + let content = serde_json::to_string_pretty(&server_csr) + .expect("Error happened when trying to serialize `server-csr.json`"); + let mut server_csr_file = File::create("server-csr.json") + .expect("Error happened when trying to create `server-csr.json`"); + server_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `server-csr.json`"); + tracing::info!("`server-csr.json` generated"); + + tracing::info!("Generating self-signed kube_apiserver https certificate..."); + let cfssl_kube = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("server-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("server") + .stdin(Stdio::from(cfssl_kube.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed CA certificate generated"); + + tracing::info!("Generating `token.csv` to to_send/..."); + let mut token = + File::create("to_send/token.csv").expect("Error happened when trying to write token file"); + token.write_all(b"4136692876ad4b01bb9dd0988480ebba,kubelet-bootstrap,10001,\"system:node-bootstrapper\"").expect("Error happened when trying to write `token.csv`"); + tracing::info!("`token.csv` generated"); + + tracing::info!("Generating `kube-apiserver.service` to to_send/..."); + KubeApiserverUnit::generate(); + tracing::info!("`kube-apiserver.service` generated"); + + for (ip, name) in &config.instance_hosts { + // Apiserver config, ssl are only needed by masters. + if name.contains("master") { + Command::new("scp") + .arg("ca.pem") + .arg("ca-key.pem") + .arg("server-key.pem") + .arg("server.pem") + .arg(format!("root@{}:/opt/kubernetes/ssl", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Certificates sent to master on {}", ip); + + tracing::info!("Generating `kube-apiserver.conf` to /opt/kubernetes/cfg..."); + KubeApiserverCfg::generate(ip, config); + tracing::info!("`kube-apiserver.conf` generated"); + + Command::new("scp") + .arg("to_send/token.csv") + .arg(format!("to_send/{}/apiserver/kube-apiserver.conf", ip)) + .arg(format!("root@{}:/opt/kubernetes/cfg", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Configurations sent to master on {}", ip); + + Command::new("scp") + .arg("to_send/kube-apiserver.service") + .arg(format!("root@{}:/usr/lib/systemd/system/", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Systemd service sent to master on {}", ip); + + // Starting apiserver... + tracing::info!("kube-apiserver installed on {}, starting...", name); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl start kube-apiserver") + .status() + .expect("Error happened when trying to start apiserver"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl enable kube-apiserver") + .status() + .expect("Error happened when trying to enable apiserver"); + tracing::info!("kube-apiserver started on {}", ip); + } + } + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/deploy/kube_controller_manager.rs b/rk8s/src/deploy/kube_controller_manager.rs new file mode 100644 index 00000000..8ec8b701 --- /dev/null +++ b/rk8s/src/deploy/kube_controller_manager.rs @@ -0,0 +1,231 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +struct KubeControllerManagerCfg; + +impl KubeControllerManagerCfg { + fn generate() { + let mut controller_conf = File::create("to_send/kube-controller-manager.conf").expect( + "Error happened when trying to create kube-controller-manager configuration file", + ); + + writeln!( + &mut controller_conf, + r#"KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \ +--leader-elect=true \ +--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig \ +--bind-address=127.0.0.1 \ +--allocate-node-cidrs=true \ +--cluster-cidr=10.244.0.0/16 \ +--service-cluster-ip-range=10.0.0.0/24 \ +--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \ +--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \ +--root-ca-file=/opt/kubernetes/ssl/ca.pem \ +--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \ +--cluster-signing-duration=87600h0m0s" +"# + ) + .expect("Error happened when trying to write `kube-controller-manager.conf`"); + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct KubeControllerManagerCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl KubeControllerManagerCsr { + fn from(config: &Config) -> KubeControllerManagerCsr { + KubeControllerManagerCsr { + CN: config.kube_controller_manager_CN.to_owned(), + hosts: vec![], + key: Key { + algo: config.kube_controller_manager_key_algo.to_owned(), + size: config.kube_controller_manager_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_controller_manager_names_C.to_owned(), + L: config.kube_controller_manager_names_L.to_owned(), + ST: config.kube_controller_manager_names_ST.to_owned(), + O: config.kube_controller_manager_names_O.to_owned(), + OU: config.kube_controller_manager_names_OU.to_owned(), + }], + } + } +} + +struct KubeControllerManagerUnit; + +impl KubeControllerManagerUnit { + fn generate() { + let mut kube_controller_unit = File::create("to_send/kube-controller-manager.service") + .expect("Error happened when trying to create kube-controller-manager unit file"); + let content = r#"[Unit] +Description=Kubernetes Controller Manager +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kube-controller-manager.conf +ExecStart=/opt/kubernetes/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS +Restart=on-failure + +[Install] +WantedBy=multi-user.target +"#; + kube_controller_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kube-controller manager unit file"); + } +} + +pub fn start(config: &Config) { + tracing::info!("kube_controller_manager phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `kube-controller-manager-csr.json`..."); + let kube_controller_manager_csr = KubeControllerManagerCsr::from(config); + let content = serde_json::to_string_pretty(&kube_controller_manager_csr) + .expect("Error happened when trying to serialize `kube-controller-manager-csr.json`"); + let mut kube_controller_manager_csr_file = File::create("kube-controller-manager-csr.json") + .expect("Error happened when trying to create `kube-controller-manager-csr.json`"); + kube_controller_manager_csr_file + .write_all(content.as_bytes()) + .expect( + "Error happened when trying to write content to `kube-controller-manager-csr.json`", + ); + tracing::info!("`kube-controller-manager-csr.json` generated"); + + tracing::info!("Generating self-signed kube_controller_manager https certificate..."); + let cfssl_kube_controller = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("kube-controller-manager-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("kube-controller-manager") + .stdin(Stdio::from(cfssl_kube_controller.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed kube_controller_manager CA certificate generated"); + + tracing::info!("Generating `kube-controller-manager.conf` to to_send/..."); + KubeControllerManagerCfg::generate(); + tracing::info!("`kube-controller-manager.conf` generated"); + + tracing::info!("Generating `kube-controller-manager.service` to to_send/"); + KubeControllerManagerUnit::generate(); + tracing::info!("`kube-controller-manager.service` generated"); + + for (ip, name) in &config.instance_hosts { + if name.contains("master") { + Command::new("scp") + .arg("kube-controller-manager.pem") + .arg("kube-controller-manager-key.pem") + .arg(format!("root@{}:/opt/kubernetes/ssl", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Certificates sent to master on {}", ip); + + Command::new("scp") + .arg("to_send/kube-controller-manager.conf") + .arg(format!("root@{}:/opt/kubernetes/cfg", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Configurations sent to master on {}", ip); + + Command::new("scp") + .arg("to_send/kube-controller-manager.service") + .arg(format!("root@{}:/usr/lib/systemd/system/", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Systemd service sent to master on {}", ip); + + // Generate kubeconfig on remote master. + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg(format!("kubectl config set-cluster kubernetes --certificate-authority=/opt/kubernetes/ssl/ca.pem --embed-certs=true \ + --server=https://{}:6443 --kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig", ip)) + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config set-credentials kube-controller-manager --client-certificate=/opt/kubernetes/ssl/kube-controller-manager.pem \ + --client-key=/opt/kubernetes/ssl/kube-controller-manager-key.pem --embed-certs=true --kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config set-context default --cluster=kubernetes --user=kube-controller-manager \ + --kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config use-context default --kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + + // Starting controller manager... + tracing::info!("kube-controller-manager installed on {}, starting...", name); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl start kube-controller-manager") + .status() + .expect("Error happened when trying to start controller-manager"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl enable kube-controller-manager") + .status() + .expect("Error happened when trying to enable controller-manager"); + tracing::info!("kube-controller-manager started on {}", ip); + } + } + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/deploy/kube_proxy.rs b/rk8s/src/deploy/kube_proxy.rs new file mode 100644 index 00000000..ba8de654 --- /dev/null +++ b/rk8s/src/deploy/kube_proxy.rs @@ -0,0 +1,274 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +struct KubeProxyCfg; + +impl KubeProxyCfg { + fn generate() { + let mut kube_proxy_conf = File::create("to_send/kube-proxy.conf") + .expect("Error happened when trying to create kube-proxy configuration file"); + + writeln!( + &mut kube_proxy_conf, + r#"KUBE_PROXY_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \ +--config=/opt/kubernetes/cfg/kube-proxy-config.yml" +"# + ) + .expect("Error happened when trying to write `kube-proxy.conf`"); + } +} + +struct KubeProxyConfig; + +impl KubeProxyConfig { + fn generate(current_ip: &String, current_name: &String) { + let mut kube_proxy_config = File::create(format!( + "to_send/{}/kube_proxy/kube-proxy-config.yml", + current_ip + )) + .expect("Error happened when trying to create kube-proxy configuration file"); + + writeln!( + &mut kube_proxy_config, + r#"kind: KubeProxyConfiguration +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +bindAddress: 0.0.0.0 +metricsBindAddress: 0.0.0.0:10249 +clientConnection: + kubeconfig: /opt/kubernetes/cfg/kube-proxy.kubeconfig"# + ) + .expect("Error happened when trying to write `kube-proxy-config.yml`"); + writeln!( + &mut kube_proxy_config, + "hostnameOverride: {} \\", + current_name + ) + .expect("Error happened when trying to write `kube-proxy-config.yml`"); + writeln!( + &mut kube_proxy_config, + r#"clusterCIDR: 10.244.0.0/16 +"#, + ) + .expect("Error happened when trying to write `kube-proxy-config.yml`"); + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct KubeProxyCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl KubeProxyCsr { + fn from(config: &Config) -> KubeProxyCsr { + KubeProxyCsr { + CN: config.kube_proxy_CN.to_owned(), + hosts: vec![], + key: Key { + algo: config.kube_proxy_key_algo.to_owned(), + size: config.kube_proxy_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_proxy_names_C.to_owned(), + L: config.kube_proxy_names_L.to_owned(), + ST: config.kube_proxy_names_ST.to_owned(), + O: config.kube_proxy_names_O.to_owned(), + OU: config.kube_proxy_names_OU.to_owned(), + }], + } + } +} + +struct KubeProxyUnit; + +impl KubeProxyUnit { + fn generate() { + let mut proxy_unit = File::create("to_send/kube-proxy.service") + .expect("Error happened when trying to create kube-proxy unit file"); + let content = r#"[Unit] +Description=Kubernetes Proxy +After=network.target + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kube-proxy.conf +ExecStart=/opt/kubernetes/bin/kube-proxy $KUBE_PROXY_OPTS +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +"#; + proxy_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kube-proxy unit file"); + } +} + +pub fn start(config: &Config) { + tracing::info!("kube_proxy phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `kube-proxy-csr.json`..."); + let kube_proxy_csr = KubeProxyCsr::from(config); + let content = serde_json::to_string_pretty(&kube_proxy_csr) + .expect("Error happened when trying to serialize `kube-proxy-csr.json`"); + let mut kube_proxy_csr_file = File::create("kube-proxy-csr.json") + .expect("Error happened when trying to create `kube-proxy-csr.json`"); + kube_proxy_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `kube-proxy-csr.json`"); + tracing::info!("`kube-proxy-csr.json` generated"); + + tracing::info!("Generating self-signed kube_proxy https certificate..."); + let cfssl_kube_proxy = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("kube-proxy-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("kube-proxy") + .stdin(Stdio::from(cfssl_kube_proxy.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed kube_proxy CA certificate generated"); + + tracing::info!("Generating `kube-proxy.conf` to to_send/..."); + KubeProxyCfg::generate(); + tracing::info!("`kube-proxy.conf` generated"); + + tracing::info!("Generating `kube-proxy.service` to to_send/..."); + KubeProxyUnit::generate(); + tracing::info!("`kube-proxy.service` generated"); + + for (ip, name) in &config.instance_hosts { + if name.contains("master") { + tracing::info!("Generating `kube-proxy-config.yml`..."); + KubeProxyConfig::generate(ip, name); + tracing::info!("`kube-proxy-config.yml` generated"); + + Command::new("scp") + .arg("kube-proxy.pem") + .arg("kube-proxy-key.pem") + .arg(format!("root@{}:/opt/kubernetes/ssl", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Certificates sent to master on {}", ip); + + Command::new("scp") + .arg("to_send/kube-proxy.conf") + .arg(format!("to_send/{}/kube_proxy/kube-proxy-config.yml", ip)) + .arg(format!("root@{}:/opt/kubernetes/cfg", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Configurations sent to master on {}", ip); + + Command::new("scp") + .arg("to_send/kube-proxy.service") + .arg(format!("root@{}:/usr/lib/systemd/system/", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Systemd service sent to master on {}", ip); + + // Generate kubeconfig on remote master. + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg(format!("kubectl config set-cluster kubernetes --certificate-authority=/opt/kubernetes/ssl/ca.pem --embed-certs=true \ + --server=https://{}:6443 --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig", ip)) + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config set-credentials kube-proxy --client-certificate=/opt/kubernetes/ssl/kube-proxy.pem \ + --client-key=/opt/kubernetes/ssl/kube-proxy-key.pem --embed-certs=true --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg( + "kubectl config set-context default --cluster=kubernetes --user=kube-proxy \ + --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig", + ) + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config use-context default --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + + // Starting kubelet... + tracing::info!("kubelet installed on {}, starting...", name); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl start kube-proxy") + .status() + .expect("Error happened when trying to start kube-proxy"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl enable kube-proxy") + .status() + .expect("Error happened when trying to enable kube-proxy"); + tracing::info!("kube-proxy started on {}", ip); + + Command::new("scp") + .arg("/rk8s/preparation/calico.yaml") + .arg(format!("root@{}:/root", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Systemd service sent to master on {}", ip); + tracing::info!("Deploying Calico..."); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl apply -f /root/calico.yaml") + .status() + .expect("Error happened when trying to enable kubelet"); + } + } + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/deploy/kube_scheduler.rs b/rk8s/src/deploy/kube_scheduler.rs new file mode 100644 index 00000000..0bef7766 --- /dev/null +++ b/rk8s/src/deploy/kube_scheduler.rs @@ -0,0 +1,220 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +struct KubeSchedulerCfg; + +impl KubeSchedulerCfg { + fn generate() { + let mut scheduler_conf = File::create("to_send/kube-scheduler.conf") + .expect("Error happened when trying to create kube-scheduler configuration file"); + + writeln!( + &mut scheduler_conf, + r#"KUBE_SCHEDULER_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \ +--leader-elect \ +--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig \ +--bind-address=127.0.0.1" +"# + ) + .expect("Error happened when trying to write `kube-scheduler.conf`"); + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct KubeSchedulerCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl KubeSchedulerCsr { + fn from(config: &Config) -> KubeSchedulerCsr { + KubeSchedulerCsr { + CN: config.kube_scheduler_CN.to_owned(), + hosts: vec![], + key: Key { + algo: config.kube_scheduler_key_algo.to_owned(), + size: config.kube_scheduler_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_scheduler_names_C.to_owned(), + L: config.kube_scheduler_names_L.to_owned(), + ST: config.kube_scheduler_names_ST.to_owned(), + O: config.kube_scheduler_names_O.to_owned(), + OU: config.kube_scheduler_names_OU.to_owned(), + }], + } + } +} + +struct KubeSchedulerUnit; + +impl KubeSchedulerUnit { + fn generate() { + let mut scheduler_unit = File::create("to_send/kube-scheduler.service") + .expect("Error happened when trying to create kube-scheduler unit file"); + let content = r#"[Unit] +Description=Kubernetes Scheduler +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kube-scheduler.conf +ExecStart=/opt/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS +Restart=on-failure + +[Install] +WantedBy=multi-user.target +"#; + scheduler_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kube-scheduler unit file"); + } +} + +pub fn start(config: &Config) { + tracing::info!("kube_apiserver phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `kube-scheduler-csr.json`..."); + let kube_scheduler_csr = KubeSchedulerCsr::from(config); + let content = serde_json::to_string_pretty(&kube_scheduler_csr) + .expect("Error happened when trying to serialize `kube-scheduler-csr.json`"); + let mut kube_scheduler_csr_file = File::create("kube-scheduler-csr.json") + .expect("Error happened when trying to create `kube-scheduler-csr.json`"); + kube_scheduler_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `kube-scheduler-csr.json`"); + tracing::info!("`kube-scheduler-csr.json` generated"); + + tracing::info!("Generating self-signed kube_scheduler certificate..."); + let cfssl_kube_scheduler = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("kube-scheduler-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("kube-scheduler") + .stdin(Stdio::from(cfssl_kube_scheduler.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed kube_scheduler CA certificate generated"); + + tracing::info!("Generating `kube-scheduler.conf` to to_send/..."); + KubeSchedulerCfg::generate(); + tracing::info!("`kube-scheduler.conf` generated"); + + tracing::info!("Generating `kube-scheduler.service` to /usr/lib/systemd/system/"); + KubeSchedulerUnit::generate(); + tracing::info!("`kube-scheduler.service` generated"); + + for (ip, name) in &config.instance_hosts { + if name.contains("master") { + Command::new("scp") + .arg("kube-scheduler.pem") + .arg("kube-scheduler-key.pem") + .arg(format!("root@{}:/opt/kubernetes/ssl", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Certificates sent to master on {}", ip); + + Command::new("scp") + .arg("to_send/kube-scheduler.conf") + .arg(format!("root@{}:/opt/kubernetes/cfg", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Configurations sent to master on {}", ip); + + Command::new("scp") + .arg("to_send/kube-scheduler.service") + .arg(format!("root@{}:/usr/lib/systemd/system/", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Systemd service sent to master on {}", ip); + + // Generate kubeconfig on remote master. + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg(format!("kubectl config set-cluster kubernetes --certificate-authority=/opt/kubernetes/ssl/ca.pem --embed-certs=true \ + --server=https://{}:6443 --kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig", ip)) + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config set-credentials kube-scheduler --client-certificate=/opt/kubernetes/ssl/kube-scheduler.pem \ + --client-key=/opt/kubernetes/ssl/kube-scheduler-key.pem --embed-certs=true --kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config set-context default --cluster=kubernetes --user=kube-scheduler \ + --kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config use-context default --kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + + // Starting scheduler... + tracing::info!("kube-scheduler installed on {}, starting...", name); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl start kube-scheduler") + .status() + .expect("Error happened when trying to start scheduler"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl enable kube-scheduler") + .status() + .expect("Error happened when trying to enable scheduler"); + tracing::info!("kube-scheduler started on {}", ip); + } + } + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/deploy/kubectl.rs b/rk8s/src/deploy/kubectl.rs new file mode 100644 index 00000000..b2921de3 --- /dev/null +++ b/rk8s/src/deploy/kubectl.rs @@ -0,0 +1,151 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct KubectlCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl KubectlCsr { + fn from(config: &Config) -> KubectlCsr { + KubectlCsr { + CN: config.kube_ctl_CN.to_owned(), + hosts: vec![], + key: Key { + algo: config.kube_ctl_key_algo.to_owned(), + size: config.kube_ctl_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_ctl_names_C.to_owned(), + L: config.kube_ctl_names_L.to_owned(), + ST: config.kube_ctl_names_ST.to_owned(), + O: config.kube_ctl_names_O.to_owned(), + OU: config.kube_ctl_names_OU.to_owned(), + }], + } + } +} + +pub fn start(config: &Config) { + tracing::info!("kube_apiserver phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `admin-csr.json`..."); + let admin_csr = KubectlCsr::from(config); + let content = serde_json::to_string_pretty(&admin_csr) + .expect("Error happened when trying to serialize `admin-csr.json`"); + let mut admin_csr_file = File::create("admin-csr.json") + .expect("Error happened when trying to create `admin-csr.json`"); + admin_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `admin-csr.json`"); + tracing::info!("`admin-csr.json` generated"); + + tracing::info!("Generating self-signed kubectl https certificate..."); + let cfssl_kubectl = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("admin-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("admin") + .stdin(Stdio::from(cfssl_kubectl.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed kubectl CA certificate generated"); + + for (ip, name) in &config.instance_hosts { + if name.contains("master") { + Command::new("scp") + .arg("admin.pem") + .arg("admin-key.pem") + .arg(format!("root@{}:/opt/kubernetes/ssl", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Certificates sent to master on {}", ip); + + // Create .kube directory under /root + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("mkdir /root/.kube") + .status() + .expect("Error happened when trying to create directory"); + + // Generate kubeconfig on remote master. + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg(format!("kubectl config set-cluster kubernetes --certificate-authority=/opt/kubernetes/ssl/ca.pem --embed-certs=true \ + --server=https://{}:6443 --kubeconfig=/root/.kube/config", ip)) + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config set-credentials cluster-admin --client-certificate=/opt/kubernetes/ssl/admin.pem \ + --client-key=/opt/kubernetes/ssl/admin-key.pem --embed-certs=true --kubeconfig=/root/.kube/config") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg( + "kubectl config set-context default --cluster=kubernetes --user=cluster-admin \ + --kubeconfig=/root/.kube/config", + ) + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config use-context default --kubeconfig=/root/.kube/config") + .status() + .expect("Error happened when trying to execute kubectl"); + + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap") + .status() + .expect("Error happened when trying to execute kubectl"); + + tracing::info!("kubectl is now ready on {}", ip); + } + } + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/deploy/kubelet.rs b/rk8s/src/deploy/kubelet.rs new file mode 100644 index 00000000..8344123e --- /dev/null +++ b/rk8s/src/deploy/kubelet.rs @@ -0,0 +1,240 @@ +use crate::config::Config; +use regex::Regex; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::Command; +use std::{thread, time}; + +struct KubeletCfg; + +impl KubeletCfg { + fn generate(current_ip: &String, current_name: &String) { + let mut kubelet_conf = File::create(format!("to_send/{}/kubelet/kubelet.conf", current_ip)) + .expect("Error happened when trying to create kubelet configuration file"); + + writeln!( + &mut kubelet_conf, + r#"KUBELET_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \"#, + ) + .expect("Error happened when trying to write `kubelet.conf`"); + writeln!(&mut kubelet_conf, "--hostname-override={} \\", current_name) + .expect("Error happened when trying to write `kubelet.conf`"); + writeln!( + &mut kubelet_conf, + r#"--network-plugin=cni \ +--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \ +--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \ +--config=/opt/kubernetes/cfg/kubelet-config.yml \ +--cert-dir=/opt/kubernetes/ssl \ +--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0" +"# + ) + .expect("Error happened when trying to write `kubelet.conf`"); + } +} + +struct KubeletConfig; + +impl KubeletConfig { + fn generate() { + let mut kubelet_config = File::create("to_send/kubelet-config.yml") + .expect("Error happened when trying to create kubelet configuration file"); + + writeln!( + &mut kubelet_config, + r#"kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +address: 0.0.0.0 +port: 10250 +readOnlyPort: 10255 +cgroupDriver: systemd +clusterDNS: +- 10.0.0.2 +clusterDomain: cluster.local +failSwapOn: false +authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 2m0s + enabled: true + x509: + clientCAFile: /opt/kubernetes/ssl/ca.pem +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 5m0s + cacheUnauthorizedTTL: 30s +evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% +maxOpenFiles: 1000000 +maxPods: 110 +"#, + ) + .expect("Error happened when trying to write `kubelet-config.yml`"); + } +} + +struct KubeletUnit; + +impl KubeletUnit { + fn generate() { + let mut kubelet_unit = File::create("to_send/kubelet.service") + .expect("Error happened when trying to create kubelet unit file"); + let content = r#"[Unit] +Description=Kubernetes Kubelet +After=docker.service + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kubelet.conf +ExecStart=/opt/kubernetes/bin/kubelet $KUBELET_OPTS +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +"#; + kubelet_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kubelet unit file"); + } +} + +pub fn start(config: &Config) { + // kube-scheduler + tracing::info!("kube_apiserver phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Generating `kubelet-config.yml` to to_send/..."); + KubeletConfig::generate(); + tracing::info!("`kubelet-config.yml` generated"); + + tracing::info!("Generating `kubelet.service` to to_send/..."); + KubeletUnit::generate(); + tracing::info!("`kubelet.service` generated"); + + let mut master_ip = ""; + for (ip, name) in &config.instance_hosts { + tracing::info!("Found instance {} on {},", name, ip); + if name.contains("master") { + master_ip = ip; + tracing::info!("Generating `kubelet.conf`..."); + KubeletCfg::generate(ip, name); + tracing::info!("`kubelet.conf` generated"); + + Command::new("scp") + .arg(format!("to_send/{}/kubelet/kubelet.conf", ip)) + .arg("to_send/kubelet-config.yml") + .arg(format!("root@{}:/opt/kubernetes/cfg", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Configurations sent to master on {}", ip); + + Command::new("scp") + .arg("to_send/kubelet.service") + .arg(format!("root@{}:/usr/lib/systemd/system/", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + tracing::info!("Systemd service sent to master on {}", ip); + + // Generate kubeconfig on remote master. + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg(format!("kubectl config set-cluster kubernetes --certificate-authority=/opt/kubernetes/ssl/ca.pem --embed-certs=true \ + --server=https://{}:6443 --kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig", ip)) + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config set-credentials kubelet-bootstrap --token=4136692876ad4b01bb9dd0988480ebba \ + --kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap \ + --kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("kubectl config use-context default --kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + + // Starting kubelet... + tracing::info!("kubelet installed on {}, starting...", name); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl start kubelet") + .status() + .expect("Error happened when trying to start kubelet"); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl enable kubelet") + .status() + .expect("Error happened when trying to enable kubelet"); + tracing::info!("kubelet started on {}", ip); + } + } + + loop { + thread::sleep(time::Duration::from_secs(1)); + let output = Command::new("ssh") + .arg(format!("root@{}", master_ip)) + .arg("kubectl get csr") + .output() + .unwrap() + .stdout; + if !output.is_empty() { + break; + } + tracing::info!("Waiting other etcd nodes to join cluster"); + } + + // kubectl approve master node csr. + let output = Command::new("ssh") + .arg(format!("root@{}", master_ip)) + .arg("kubectl get csr") + .output() + .unwrap() + .stdout; + let output = String::from_utf8(output).unwrap(); + tracing::info!("Output of kubectl is: {}", output); + let csr = Regex::new(r"node-csr-\S*").unwrap(); + let mut res = ""; + for word in output.split_whitespace() { + if csr.is_match(word) { + res = word; + break; + } + } + tracing::info!("Retrieved csr is: {}", res); + Command::new("ssh") + .arg(format!("root@{}", master_ip)) + .arg(format!("kubectl certificate approve {}", res)) + .status() + .expect("Error happened when trying to approve csr from node"); + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/deploy/mod.rs b/rk8s/src/deploy/mod.rs new file mode 100644 index 00000000..e7c37c47 --- /dev/null +++ b/rk8s/src/deploy/mod.rs @@ -0,0 +1,10 @@ +pub mod docker; +pub mod etcd; +pub mod kube_apiserver; +pub mod kube_controller_manager; +pub mod kube_proxy; +pub mod kube_scheduler; +pub mod kubectl; +pub mod kubelet; +pub mod pre_check; +pub mod prepare_kube; diff --git a/rk8s/src/deploy/pre_check.rs b/rk8s/src/deploy/pre_check.rs new file mode 100644 index 00000000..ea36c637 --- /dev/null +++ b/rk8s/src/deploy/pre_check.rs @@ -0,0 +1,101 @@ +use crate::config::Config; +use std::fs::{self, File}; +use std::io::Write; +use std::process::Command; + +pub fn start(config: &Config) { + tracing::info!("Pre check started"); + fs::create_dir("pre_check").unwrap(); + let mut k8s_conf = + File::create("k8s.conf").expect("Error happened when trying to create `k8s.conf` file"); + k8s_conf + .write_all( + b"net.bridge.bridge-nf-call-ip6tables = 1\nnet.bridge.bridge-nf-call-iptables = 1\n", + ) + .expect("Error happened when trying to write to `/etc/sysctl.d/k8s.conf`"); + let mut k8s_module = File::create("pre_check/k8s.conf") + .expect("Error happened when trying to create `k8s.conf` file"); + k8s_module + .write_all(b"br_netfilter\n") + .expect("Error happened when trying to write to `/etc/sysctl.d/k8s.conf`"); + + for (ip, name) in &config.instance_hosts { + tracing::info!("Found {} on {}, start pre-setting", name, ip); + // Stop `firewalld` daemon. + tracing::info!("Stopping `firewalld` daemon..."); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl stop firewalld") + .status() + .expect("Error happened when trying to stop firewalld daemon"); + tracing::info!("`firewalld` daemon stopped"); + + // Disable `firewalld` daemon. + tracing::info!("Disabling `firewalld` daemon..."); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("systemctl disable firewalld") + .status() + .expect("Error happened when trying to disable firewalld daemon"); + tracing::info!("`firewalld` daemon disabled"); + + // Turn off selinux. + tracing::info!("Disabling selinux..."); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("sed -i s/enforcing/disabled/ /etc/selinux/config") + .status() + .expect("Error happened when trying to disable swap partition"); + + // Turn off swap. + tracing::info!("Disabling swap partition..."); + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("sed -ri \"s/.*swap.*/#&/\" /etc/fstab") + .status() + .expect("Error happened when trying to disable swap partition"); + tracing::info!("swap partition disabled"); + + // Set Master `hostname`. + tracing::info!("Setting master's `hostname` to master01..."); + // Acquire ip and name of this instance. + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg(format!("hostnamectl set-hostname {}", name)) + .status() + .expect("Error happened when trying to set hostname of master"); + tracing::info!("`hostname` set to master01"); + + // Set `/etc/hosts` file. + tracing::info!("Setting `/etc/hosts` according to configuration..."); + let mut buffer = String::new(); + for (ip, name) in &config.instance_hosts { + buffer.push_str(format!("{} {}\n", ip, name).as_str()); + } + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg(format!("echo \"{}\" >> /etc/hosts", buffer)) + .status() + .expect("Error happened when trying to write hosts"); + tracing::info!("`/etc/hosts` set"); + + tracing::info!("Setting `/etc/sysctl.d/k8s.conf` according to configuration..."); + Command::new("scp") + .arg("k8s.conf") + .arg(format!("root@{}:/etc/sysctl.d", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + Command::new("scp") + .arg("pre_check/k8s.conf") + .arg(format!("root@{}:/etc/modules-load.d", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + + Command::new("ssh") + .arg(format!("root@{}", ip)) + .arg("sysctl --system") + .status() + .expect("Error happened when activating `k8s.conf`"); + tracing::info!("`/etc/sysctl.d/k8s.conf` set"); + } +} diff --git a/rk8s/src/deploy/prepare_kube.rs b/rk8s/src/deploy/prepare_kube.rs new file mode 100644 index 00000000..9e94d291 --- /dev/null +++ b/rk8s/src/deploy/prepare_kube.rs @@ -0,0 +1,113 @@ +use crate::config::Config; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +pub fn start(config: &Config) { + tracing::info!("Start preparing kubernetes binaries..."); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + // Prepare directory to be sent. + let cfg_path = PathBuf::from("to_send/kubernetes/cfg"); + check_dir_exist_or_create(cfg_path); + let bin_path = PathBuf::from("to_send/kubernetes/bin"); + check_dir_exist_or_create(bin_path); + let ssl_path = PathBuf::from("to_send/kubernetes/ssl"); + check_dir_exist_or_create(ssl_path); + let logs_path = PathBuf::from("to_send/kubernetes/logs"); + check_dir_exist_or_create(logs_path); + + tracing::info!("Untaring prepared kubernetes binary file"); + Command::new("tar") + .arg("-zxf") + .arg("/rk8s/preparation/kubernetes-server-linux-amd64.tar.gz") + .status() + .expect("Error happened when trying to untar `kubernetes` executable"); + + tracing::info!("Copying binaries to to_send/"); + Command::new("cp") + // Needed by master. + .arg("kubernetes/server/bin/kube-apiserver") + .arg("kubernetes/server/bin/kube-controller-manager") + .arg("kubernetes/server/bin/kube-scheduler") + .arg("kubernetes/server/bin/kubectl") + // Needed by worker. + .arg("kubernetes/server/bin/kubelet") + .arg("kubernetes/server/bin/kube-proxy") + .arg("to_send/") + .status() + .expect("Error happened when trying to copy binaries to `to_send/etcd/bin`"); + tracing::info!("Binaries prepared"); + + for (ip, name) in &config.instance_hosts { + tracing::info!( + "Found node: {} on {}, sending kubernetes skeleton and worker binaries...", + name, + ip + ); + Command::new("scp") + .arg("-r") + .arg("to_send/kubernetes") + .arg(format!("root@{}:/opt/", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + Command::new("scp") + .arg("to_send/kubelet") + .arg("to_send/kube-proxy") + .arg(format!("root@{}:/opt/kubernetes/bin", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + + let path = PathBuf::from("to_send").join(ip); + if name.contains("master") { + // Only master need apiserver. + let apiserver_path = path.join("apiserver"); + check_dir_exist_or_create(apiserver_path); + + // Only master need controller manager. + let controller_path = path.join("controller_manager"); + check_dir_exist_or_create(controller_path); + + // Only master need scheduler. + let scheduler_path = path.join("scheduler"); + check_dir_exist_or_create(scheduler_path); + + tracing::info!("Found master: {} on {}, sending kubernetes apiserver, controller-manager, scheduler, kubectl...", name, ip); + Command::new("scp") + .arg("to_send/kube-apiserver") + .arg("to_send/kube-controller-manager") + .arg("to_send/kube-scheduler") + .arg(format!("root@{}:/opt/kubernetes/bin", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + Command::new("scp") + .arg("to_send/kubectl") + .arg(format!("root@{}:/usr/bin", ip)) + .status() + .expect("Error happened when trying to send files to other nodes"); + } + + let kubelet_path = path.join("kubelet"); + check_dir_exist_or_create(kubelet_path); + + let kube_proxy_path = path.join("kube_proxy"); + check_dir_exist_or_create(kube_proxy_path); + } + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} + +fn check_dir_exist_or_create(path: PathBuf) { + if !path.is_dir() { + fs::create_dir_all(path).expect("Error happened when trying to create path"); + } +} diff --git a/rk8s/src/init/etcd.rs b/rk8s/src/init/etcd.rs new file mode 100644 index 00000000..61e595cb --- /dev/null +++ b/rk8s/src/init/etcd.rs @@ -0,0 +1,360 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +#[derive(Serialize, Deserialize, Debug)] +struct CAConfig { + signing: Signing, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Signing { + default: SignDefault, + profiles: SignProfiles, +} + +#[derive(Serialize, Deserialize, Debug)] +struct SignDefault { + expiry: String, +} + +#[derive(Serialize, Deserialize, Debug)] +struct SignProfiles { + www: WWWProfile, +} + +#[derive(Serialize, Deserialize, Debug)] +struct WWWProfile { + expiry: String, + usages: Vec, +} + +impl CAConfig { + fn from(config: &Config) -> CAConfig { + CAConfig { + signing: Signing { + default: SignDefault { + expiry: config.etcd_expiry.to_owned(), + }, + profiles: SignProfiles { + www: WWWProfile { + expiry: config.etcd_expiry.to_owned(), + usages: config.etcd_usages.to_owned(), + }, + }, + }, + } + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct CACsr { + CN: String, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, +} + +impl CACsr { + fn from(config: &Config) -> CACsr { + CACsr { + CN: config.etcd_ca_CN.to_owned(), + key: Key { + algo: config.etcd_key_algo.to_owned(), + size: config.etcd_key_size.to_owned(), + }, + names: vec![Name { + C: config.etcd_names_C.to_owned(), + L: config.etcd_names_L.to_owned(), + ST: config.etcd_names_ST.to_owned(), + }], + } + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct ServerCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +impl ServerCsr { + fn from(config: &Config) -> ServerCsr { + ServerCsr { + CN: config.etcd_CN.to_owned(), + hosts: { + let mut hosts = Vec::new(); + for ip in config.instance_hosts.keys() { + hosts.push(ip.to_owned()); + } + hosts + }, + key: Key { + algo: config.etcd_key_algo.to_owned(), + size: config.etcd_key_size.to_owned(), + }, + names: vec![Name { + C: config.etcd_names_C.to_owned(), + L: config.etcd_names_L.to_owned(), + ST: config.etcd_names_ST.to_owned(), + }], + } + } +} + +struct ETCDCfg; + +impl ETCDCfg { + fn generate(config: &Config) { + let mut etcd_conf = File::create("/opt/etcd/cfg/etcd.conf") + .expect("Error happened when trying to create etcd configuration file"); + + writeln!(&mut etcd_conf, "#[Member]") + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_NAME=\"etcd_{}\"", + config.instance_name + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_DATA_DIR=\"/var/lib/etcd/default.etcd\"" + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_LISTEN_PEER_URLS=\"https://{}:2380\"", + config.instance_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_LISTEN_CLIENT_URLS=\"https://{}:2379\"", + config.instance_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf).expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf, "#[Clustering]") + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_INITIAL_ADVERTISE_PEER_URLS=\"https://{}:2380\"", + config.instance_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_ADVERTISE_CLIENT_URLS=\"https://{}:2379\"", + config.instance_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + let mut buffer = String::new(); + for (ip, name) in &config.instance_hosts { + buffer.push_str(format!("etcd_{}=https://{}:2380,", name, ip).as_str()); + } + buffer.pop(); + writeln!(&mut etcd_conf, "ETCD_INITIAL_CLUSTER=\"{}\"", buffer) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_INITIAL_CLUSTER_TOKEN=\"etcd-cluster\"" + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf, "ETCD_INITIAL_CLUSTER_STATE=\"new\"") + .expect("Error happened when trying to write `etcd.conf`"); + } +} + +struct ETCDUnit; + +impl ETCDUnit { + fn generate() { + let mut etcd_unit = File::create("/usr/lib/systemd/system/etcd.service") + .expect("Error happened when trying to create etcd unit file"); + let content = r#"[Unit] +Description=Etcd Server +After=network.target +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +EnvironmentFile=/opt/etcd/cfg/etcd.conf +ExecStart=/opt/etcd/bin/etcd \ +--cert-file=/opt/etcd/ssl/server.pem \ +--key-file=/opt/etcd/ssl/server-key.pem \ +--peer-cert-file=/opt/etcd/ssl/server.pem \ +--peer-key-file=/opt/etcd/ssl/server-key.pem \ +--trusted-ca-file=/opt/etcd/ssl/ca.pem \ +--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \ +--logger=zap +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +"#; + etcd_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write etcd unit file"); + } +} + +pub fn start(config: &Config) { + tracing::info!("etcd phase started"); + tracing::info!("Change working directory into `etcd`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/etcd"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `etcd`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `ca-config.json`..."); + let ca_config = CAConfig::from(config); + let content = serde_json::to_string_pretty(&ca_config) + .expect("Error happened when trying to serialize `ca-config.json`"); + let mut ca_config_file = File::create("ca-config.json") + .expect("Error happened when trying to create `ca-config.json`"); + ca_config_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `ca-config.json`"); + tracing::info!("`ca-config.json` generated"); + + tracing::info!("Start generating `ca-config.json`..."); + let ca_csr = CACsr::from(config); + let content = serde_json::to_string_pretty(&ca_csr) + .expect("Error happened when trying to serialize `ca-csr.json`"); + let mut ca_csr_file = + File::create("ca-csr.json").expect("Error happened when trying to create `ca-csr.json`"); + ca_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `ca-csr.json`"); + tracing::info!("`ca-csr.json` generated"); + + tracing::info!("Generating self-signed CA certificate..."); + let cfssl_ca = Command::new("cfssl") + .arg("gencert") + .arg("-initca") + .arg("ca-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("ca") + .arg("-") + .stdin(Stdio::from(cfssl_ca.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed CA certificate generated"); + + tracing::info!("Start generating `server-csr.json`..."); + let server_csr = ServerCsr::from(config); + let content = serde_json::to_string_pretty(&server_csr) + .expect("Error happened when trying to serialize `server-csr.json`"); + let mut server_csr_file = File::create("server-csr.json") + .expect("Error happened when trying to create `server-csr.json`"); + server_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `server-csr.json`"); + tracing::info!("`server-csr.json` generated"); + + tracing::info!("Generating self-signed etcd https certificate..."); + let cfssl_etcd = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=www") + .arg("server-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("server") + .stdin(Stdio::from(cfssl_etcd.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed CA certificate generated"); + + tracing::info!("Copying certificates to /opt/etcd/ssl..."); + Command::new("cp") + .arg("ca.pem") + .arg("ca-key.pem") + .arg("server-key.pem") + .arg("server.pem") + .arg("/opt/etcd/ssl") + .status() + .expect("Error happened when trying to copy certificates to `/opt/etcd/ssl`"); + tracing::info!("ertificates copied"); + + tracing::info!("Generating `etcd.conf` to /opt/etcd/cfg..."); + ETCDCfg::generate(config); + tracing::info!("`etcd.conf` generated"); + + tracing::info!("Generating `etcd.service` to /usr/lib/systemd/system/..."); + ETCDUnit::generate(); + tracing::info!("`etcd.service` generated"); + + tracing::info!("Sending etcd to worker nodes..."); + for ip in config.instance_hosts.keys() { + if *ip != config.instance_ip { + Command::new("scp") + .arg("-r") + .arg("/opt/etcd") + .arg(format!("root@{}:/opt/", ip)) + .status() + .expect("Error happened when trying to send files to other worker nodes"); + Command::new("scp") + .arg("/usr/lib/systemd/system/etcd.service") + .arg(format!("root@{}:/usr/lib/systemd/system/", ip)) + .status() + .expect("Error happened when trying to send files to other worker nodes"); + } + } + tracing::info!("Files sent to other worker nodes"); + + Command::new("systemctl") + .arg("daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("systemctl") + .arg("enable") + .arg("etcd") + .status() + .expect("Error happened when trying to enable `etcd.service`"); + Command::new("systemctl") + .arg("start") + .arg("etcd") + .status() + .expect("Error happened when trying to start `etcd.service`"); + tracing::info!("Master is now etcd set"); + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `etcd`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/init/kube_apiserver.rs b/rk8s/src/init/kube_apiserver.rs new file mode 100644 index 00000000..e629486a --- /dev/null +++ b/rk8s/src/init/kube_apiserver.rs @@ -0,0 +1,365 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +#[derive(Serialize, Deserialize, Debug)] +struct CAConfig { + signing: Signing, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Signing { + default: SignDefault, + profiles: SignProfiles, +} + +#[derive(Serialize, Deserialize, Debug)] +struct SignDefault { + expiry: String, +} + +#[derive(Serialize, Deserialize, Debug)] +struct SignProfiles { + kubernetes: KubernetesProfile, +} + +#[derive(Serialize, Deserialize, Debug)] +struct KubernetesProfile { + expiry: String, + usages: Vec, +} + +impl CAConfig { + fn from(config: &Config) -> CAConfig { + CAConfig { + signing: Signing { + default: SignDefault { + expiry: config.kube_apiserver_expiry.to_owned(), + }, + profiles: SignProfiles { + kubernetes: KubernetesProfile { + expiry: config.kube_apiserver_expiry.to_owned(), + usages: config.kube_apiserver_usages.to_owned(), + }, + }, + }, + } + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct CACsr { + CN: String, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl CACsr { + fn from(config: &Config) -> CACsr { + CACsr { + CN: config.kube_apiserver_CN.to_owned(), + key: Key { + algo: config.kube_apiserver_key_algo.to_owned(), + size: config.kube_apiserver_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_apiserver_names_C.to_owned(), + L: config.kube_apiserver_names_L.to_owned(), + ST: config.kube_apiserver_names_ST.to_owned(), + O: config.kube_apiserver_names_O.to_owned(), + OU: config.kube_apiserver_names_OU.to_owned(), + }], + } + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct ServerCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +impl ServerCsr { + fn from(config: &Config) -> ServerCsr { + ServerCsr { + CN: config.kube_apiserver_CN.to_owned(), + hosts: { + let mut hosts = vec![ + "10.0.0.1".to_string(), + "127.0.0.1".to_string(), + "kubernetes".to_string(), + "kubernetes.default".to_string(), + "kubernetes.default.svc".to_string(), + "kubernetes.default.svc.cluster".to_string(), + "kubernetes.default.svc.cluster.local".to_string(), + ]; + for ip in config.instance_hosts.keys() { + hosts.push(ip.to_owned()); + } + hosts + }, + key: Key { + algo: config.kube_apiserver_key_algo.to_owned(), + size: config.kube_apiserver_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_apiserver_names_C.to_owned(), + L: config.kube_apiserver_names_L.to_owned(), + ST: config.kube_apiserver_names_ST.to_owned(), + O: config.kube_apiserver_names_O.to_owned(), + OU: config.kube_apiserver_names_OU.to_owned(), + }], + } + } +} + +struct KubeApiserverCfg; + +impl KubeApiserverCfg { + fn generate(config: &Config) { + let mut apiserver_conf = File::create("/opt/kubernetes/cfg/kube-apiserver.conf") + .expect("Error happened when trying to create kube-apiserver configuration file"); + + writeln!( + &mut apiserver_conf, + r#"KUBE_APISERVER_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \"# + ) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + let mut buffer = String::new(); + for ip in config.instance_hosts.keys() { + buffer.push_str(format!("https://{}:2379,", ip).as_str()); + } + buffer.pop(); + writeln!(&mut apiserver_conf, "--etcd-servers={}", buffer) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + writeln!(&mut apiserver_conf, "--bind-address={}", config.instance_ip) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + writeln!(&mut apiserver_conf, "--secure-port=6443") + .expect("Error happened when trying to write `kube-apiserver.conf`"); + writeln!( + &mut apiserver_conf, + "--advertise-address={}", + config.instance_ip + ) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + writeln!( + &mut apiserver_conf, +r#"--allow-privileged=true \ +--service-cluster-ip-range=10.0.0.0/24 \ +--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \ +--authorization-mode=RBAC,Node \ +--enable-bootstrap-token-auth=true \ +--token-auth-file=/opt/kubernetes/cfg/token.csv \ +--service-node-port-range=30000-32767 \ +--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \ +--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \ +--tls-cert-file=/opt/kubernetes/ssl/server.pem \ +--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \ +--client-ca-file=/opt/kubernetes/ssl/ca.pem \ +--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \ +--service-account-issuer=api \ +--service-account-signing-key-file=/opt/kubernetes/ssl/server-key.pem \ +--etcd-cafile=/opt/etcd/ssl/ca.pem \ +--etcd-certfile=/opt/etcd/ssl/server.pem \ +--etcd-keyfile=/opt/etcd/ssl/server-key.pem \ +--requestheader-client-ca-file=/opt/kubernetes/ssl/ca.pem \ +--proxy-client-cert-file=/opt/kubernetes/ssl/server.pem \ +--proxy-client-key-file=/opt/kubernetes/ssl/server-key.pem \ +--requestheader-allowed-names=kubernetes \ +--requestheader-extra-headers-prefix=X-Remote-Extra- \ +--requestheader-group-headers=X-Remote-Group \ +--requestheader-username-headers=X-Remote-User \ +--enable-aggregator-routing=true \ +--audit-log-maxage=30 \ +--audit-log-maxbackup=3 \ +--audit-log-maxsize=100 \ +--audit-log-path=/opt/kubernetes/logs/k8s-audit.log""#, + ) + .expect("Error happened when trying to write `kube-apiserver.conf`"); + } +} + +struct KubeApiserverUnit; + +impl KubeApiserverUnit { + fn generate() { + let mut kube_apiserver_unit = + File::create("/usr/lib/systemd/system/kube-apiserver.service") + .expect("Error happened when trying to create kube-apiserver unit file"); + let content = r#"[Unit] +Description=Kubernetes API Server +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kube-apiserver.conf +ExecStart=/opt/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS +Restart=on-failure + +[Install] +WantedBy=multi-user.target +"#; + kube_apiserver_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kube-apiserver unit file"); + } +} + +pub fn start(config: &Config) { + tracing::info!("kube_apiserver phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `ca-config.json`..."); + let ca_config = CAConfig::from(config); + let content = serde_json::to_string_pretty(&ca_config) + .expect("Error happened when trying to serialize `ca-config.json`"); + let mut ca_config_file = File::create("ca-config.json") + .expect("Error happened when trying to create `ca-config.json`"); + ca_config_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `ca-config.json`"); + tracing::info!("`ca-config.json` generated"); + + tracing::info!("Start generating `ca-csr.json`..."); + let ca_csr = CACsr::from(config); + let content = serde_json::to_string_pretty(&ca_csr) + .expect("Error happened when trying to serialize `ca-csr.json`"); + let mut ca_csr_file = + File::create("ca-csr.json").expect("Error happened when trying to create `ca-csr.json`"); + ca_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `ca-csr.json`"); + tracing::info!("`ca-csr.json` generated"); + + tracing::info!("Generating self-signed CA certificate..."); + let cfssl_ca = Command::new("cfssl") + .arg("gencert") + .arg("-initca") + .arg("ca-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("ca") + .arg("-") + .stdin(Stdio::from(cfssl_ca.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed CA certificate generated"); + + tracing::info!("Start generating `server-csr.json`..."); + let server_csr = ServerCsr::from(config); + let content = serde_json::to_string_pretty(&server_csr) + .expect("Error happened when trying to serialize `server-csr.json`"); + let mut server_csr_file = File::create("server-csr.json") + .expect("Error happened when trying to create `server-csr.json`"); + server_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `server-csr.json`"); + tracing::info!("`server-csr.json` generated"); + + tracing::info!("Generating self-signed kube_apiserver https certificate..."); + let cfssl_kube = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("server-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("server") + .stdin(Stdio::from(cfssl_kube.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed CA certificate generated"); + + tracing::info!("Copying certificates to /opt/kubernetes/ssl..."); + Command::new("cp") + .arg("ca.pem") + .arg("ca-key.pem") + .arg("server-key.pem") + .arg("server.pem") + .arg("/opt/kubernetes/ssl") + .status() + .expect("Error happened when trying to copy certificates to `/opt/kubernetes/ssl`"); + tracing::info!("ertificates copied"); + + tracing::info!("Generating `kube-apiserver.conf` to /opt/kubernetes/cfg..."); + KubeApiserverCfg::generate(config); + tracing::info!("`kube-apiserver.conf` generated"); + + tracing::info!("Generating `token.csv` to /opt/kubernetes/cfg/token.csv..."); + let mut token = File::create("/opt/kubernetes/cfg/token.csv") + .expect("Error happened when trying to write token file"); + token.write_all(b"4136692876ad4b01bb9dd0988480ebba,kubelet-bootstrap,10001,\"system:node-bootstrapper\"").expect("Error happened when trying to write `token.csv`"); + tracing::info!("`token.csv` generated"); + + tracing::info!("Generating `kube-apiserver.service` to /usr/lib/systemd/system/"); + KubeApiserverUnit::generate(); + tracing::info!("`kube-apiserver.service` generated"); + + // tracing::info!("Sending etcd to worker nodes..."); + // for (ip, _) in &config.instance_hosts { + // if *ip != config.instance_ip { + // Command::new("scp").arg("-r").arg("/opt/etcd").arg(format!("root@{}:/opt/", ip)).status().expect("Error happened when trying to send files to other worker nodes"); + // Command::new("scp").arg("/usr/lib/systemd/system/etcd.service").arg(format!("root@{}:/usr/lib/systemd/system/", ip)).status().expect("Error happened when trying to send files to other worker nodes"); + // } + // } + // tracing::info!("Files sent to other worker nodes"); + + Command::new("systemctl") + .arg("daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("systemctl") + .arg("enable") + .arg("kube-apiserver") + .status() + .expect("Error happened when trying to enable `kube-apiserver.service`"); + Command::new("systemctl") + .arg("start") + .arg("kube-apiserver") + .status() + .expect("Error happened when trying to start `kube-apiserver.service`"); + tracing::info!("Master's apiserver is now set"); + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/init/kube_controller_manager.rs b/rk8s/src/init/kube_controller_manager.rs new file mode 100644 index 00000000..a03d725a --- /dev/null +++ b/rk8s/src/init/kube_controller_manager.rs @@ -0,0 +1,218 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +struct KubeControllerManagerCfg; + +impl KubeControllerManagerCfg { + fn generate() { + let mut controller_conf = File::create("/opt/kubernetes/cfg/kube-controller-manager.conf") + .expect( + "Error happened when trying to create kube-controller-manager configuration file", + ); + + writeln!( + &mut controller_conf, + r#"KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \ +--leader-elect=true \ +--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig \ +--bind-address=127.0.0.1 \ +--allocate-node-cidrs=true \ +--cluster-cidr=10.244.0.0/16 \ +--service-cluster-ip-range=10.0.0.0/24 \ +--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \ +--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \ +--root-ca-file=/opt/kubernetes/ssl/ca.pem \ +--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \ +--cluster-signing-duration=87600h0m0s" +"# + ) + .expect("Error happened when trying to write `kube-controller-manager.conf`"); + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct KubeControllerManagerCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl KubeControllerManagerCsr { + fn from(config: &Config) -> KubeControllerManagerCsr { + KubeControllerManagerCsr { + CN: config.kube_controller_manager_CN.to_owned(), + hosts: vec![], + key: Key { + algo: config.kube_controller_manager_key_algo.to_owned(), + size: config.kube_controller_manager_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_controller_manager_names_C.to_owned(), + L: config.kube_controller_manager_names_L.to_owned(), + ST: config.kube_controller_manager_names_ST.to_owned(), + O: config.kube_controller_manager_names_O.to_owned(), + OU: config.kube_controller_manager_names_OU.to_owned(), + }], + } + } +} + +struct KubeControllerManagerUnit; + +impl KubeControllerManagerUnit { + fn generate() { + let mut kube_controller_unit = + File::create("/usr/lib/systemd/system/kube-controller-manager.service") + .expect("Error happened when trying to create kube-controller-manager unit file"); + let content = r#"[Unit] +Description=Kubernetes Controller Manager +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kube-controller-manager.conf +ExecStart=/opt/kubernetes/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS +Restart=on-failure + +[Install] +WantedBy=multi-user.target +"#; + kube_controller_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kube-controller manager unit file"); + } +} + +pub fn start(config: &Config) { + // kube-controller-manager + tracing::info!("kube_apiserver phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Generating `kube-controller-manager.conf` to /opt/kubernetes/cfg..."); + KubeControllerManagerCfg::generate(); + tracing::info!("`kube-controller-manager.conf` generated"); + + tracing::info!("Start generating `kube-controller-manager-csr.json`..."); + let kube_controller_manager_csr = KubeControllerManagerCsr::from(config); + let content = serde_json::to_string_pretty(&kube_controller_manager_csr) + .expect("Error happened when trying to serialize `kube-controller-manager-csr.json`"); + let mut kube_controller_manager_csr_file = File::create("kube-controller-manager-csr.json") + .expect("Error happened when trying to create `kube-controller-manager-csr.json`"); + kube_controller_manager_csr_file + .write_all(content.as_bytes()) + .expect( + "Error happened when trying to write content to `kube-controller-manager-csr.json`", + ); + tracing::info!("`kube-controller-manager-csr.json` generated"); + + tracing::info!("Generating self-signed kube_controller_manager https certificate..."); + let cfssl_kube_controller = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("kube-controller-manager-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("kube-controller-manager") + .stdin(Stdio::from(cfssl_kube_controller.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed kube_controller_manager CA certificate generated"); + + tracing::info!("Generating `kubeconfig` using `kubectl`"); + Command::new("kubectl") + .arg("config") + .arg("set-cluster") + .arg("kubernetes") + .arg("--certificate-authority=/opt/kubernetes/ssl/ca.pem") + .arg("--embed-certs=true") + .arg(format!("--server=https://{}:6443", config.instance_ip)) + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-credentials") + .arg("kube-controller-manager") + .arg("--client-certificate=./kube-controller-manager.pem") + .arg("--client-key=./kube-controller-manager-key.pem") + .arg("--embed-certs=true") + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-context") + .arg("default") + .arg("--cluster=kubernetes") + .arg("--user=kube-controller-manager") + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("use-context") + .arg("default") + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + + tracing::info!("Generating `kube-controller-manager.service` to /usr/lib/systemd/system/"); + KubeControllerManagerUnit::generate(); + tracing::info!("`kube-controller-manager.service` generated"); + + Command::new("systemctl") + .arg("daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("systemctl") + .arg("enable") + .arg("kube-controller-manager") + .status() + .expect("Error happened when trying to enable `kube-controller-manager.service`"); + Command::new("systemctl") + .arg("start") + .arg("kube-controller-manager") + .status() + .expect("Error happened when trying to start `kube-controller-manager.service`"); + tracing::info!("Master's controller manager is now set"); + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/init/kube_ctl.rs b/rk8s/src/init/kube_ctl.rs new file mode 100644 index 00000000..0cbe4c0e --- /dev/null +++ b/rk8s/src/init/kube_ctl.rs @@ -0,0 +1,154 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct KubeSchedulerCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl KubeSchedulerCsr { + fn from(config: &Config) -> KubeSchedulerCsr { + KubeSchedulerCsr { + CN: config.kube_ctl_CN.to_owned(), + hosts: vec![], + key: Key { + algo: config.kube_ctl_key_algo.to_owned(), + size: config.kube_ctl_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_ctl_names_C.to_owned(), + L: config.kube_ctl_names_L.to_owned(), + ST: config.kube_ctl_names_ST.to_owned(), + O: config.kube_ctl_names_O.to_owned(), + OU: config.kube_ctl_names_OU.to_owned(), + }], + } + } +} + +pub fn start(config: &Config) { + // kube-ctl + tracing::info!("kube_apiserver phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `admin-csr.json`..."); + let admin_csr = KubeSchedulerCsr::from(config); + let content = serde_json::to_string_pretty(&admin_csr) + .expect("Error happened when trying to serialize `admin-csr.json`"); + let mut admin_csr_file = File::create("admin-csr.json") + .expect("Error happened when trying to create `admin-csr.json`"); + admin_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `admin-csr.json`"); + tracing::info!("`admin-csr.json` generated"); + + tracing::info!("Generating self-signed kubectl https certificate..."); + let cfssl_kube_scheduler = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("admin-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("admin") + .stdin(Stdio::from(cfssl_kube_scheduler.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed kubectl CA certificate generated"); + + // Check /root/.kube directory exist or not + let dot_kube = Path::new("/root/.kube"); + if !dot_kube.is_dir() { + fs::create_dir_all("/root/.kube") + .expect("Error happened when trying to create `.kube` directory"); + } + + tracing::info!("Generating `kubeconfig` using `kubectl`"); + Command::new("kubectl") + .arg("config") + .arg("set-cluster") + .arg("kubernetes") + .arg("--certificate-authority=/opt/kubernetes/ssl/ca.pem") + .arg("--embed-certs=true") + .arg(format!("--server=https://{}:6443", config.instance_ip)) + .arg("--kubeconfig=/root/.kube/config") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-credentials") + .arg("cluster-admin") + .arg("--client-certificate=./admin.pem") + .arg("--client-key=./admin-key.pem") + .arg("--embed-certs=true") + .arg("--kubeconfig=/root/.kube/config") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-context") + .arg("default") + .arg("--cluster=kubernetes") + .arg("--user=cluster-admin") + .arg("--kubeconfig=/root/.kube/config") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("use-context") + .arg("default") + .arg("--kubeconfig=/root/.kube/config") + .status() + .expect("Error happened when trying to execute kubectl"); + + Command::new("kubectl") + .arg("create") + .arg("clusterrolebinding") + .arg("kubelet-bootstrap") + .arg("--clusterrole=system:node-bootstrapper") + .arg("--user=kubelet-bootstrap") + .status() + .expect("Error happened when trying to execute kubectl"); + tracing::info!("Master's scheduler is now set"); + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/init/kube_let.rs b/rk8s/src/init/kube_let.rs new file mode 100644 index 00000000..d414d236 --- /dev/null +++ b/rk8s/src/init/kube_let.rs @@ -0,0 +1,229 @@ +use crate::config::Config; +use regex::Regex; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::Command; + +struct KubeletCfg; + +impl KubeletCfg { + fn generate(config: &Config) { + let mut kubelet_conf = File::create("/opt/kubernetes/cfg/kubelet.conf") + .expect("Error happened when trying to create kubelet configuration file"); + + writeln!( + &mut kubelet_conf, + r#"KUBELET_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \"#, + ) + .expect("Error happened when trying to write `kubelet.conf`"); + writeln!( + &mut kubelet_conf, + "--hostname-override={} \\", + config.instance_name + ) + .expect("Error happened when trying to write `kubelet.conf`"); + writeln!( + &mut kubelet_conf, + r#"--network-plugin=cni \ +--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \ +--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \ +--config=/opt/kubernetes/cfg/kubelet-config.yml \ +--cert-dir=/opt/kubernetes/ssl \ +--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0" +"# + ) + .expect("Error happened when trying to write `kubelet.conf`"); + } +} + +struct KubeletConfig; + +impl KubeletConfig { + fn generate() { + let mut kubelet_config = File::create("/opt/kubernetes/cfg/kubelet-config.yml") + .expect("Error happened when trying to create kubelet configuration file"); + + writeln!( + &mut kubelet_config, + r#"kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +address: 0.0.0.0 +port: 10250 +readOnlyPort: 10255 +cgroupDriver: systemd +clusterDNS: +- 10.0.0.2 +clusterDomain: cluster.local +failSwapOn: false +authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 2m0s + enabled: true + x509: + clientCAFile: /opt/kubernetes/ssl/ca.pem +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 5m0s + cacheUnauthorizedTTL: 30s +evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% +maxOpenFiles: 1000000 +maxPods: 110 +"#, + ) + .expect("Error happened when trying to write `kubelet-config.yml`"); + } +} + +struct KubeletUnit; + +impl KubeletUnit { + fn generate() { + let mut kubelet_unit = File::create("/usr/lib/systemd/system/kubelet.service") + .expect("Error happened when trying to create kubelet unit file"); + let content = r#"[Unit] +Description=Kubernetes Kubelet +After=docker.service + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kubelet.conf +ExecStart=/opt/kubernetes/bin/kubelet $KUBELET_OPTS +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +"#; + kubelet_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kubelet unit file"); + } +} + +pub fn start(config: &Config) { + // kube-scheduler + tracing::info!("kube_apiserver phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Generating `kubelet.conf` to /opt/kubernetes/cfg..."); + KubeletCfg::generate(config); + tracing::info!("`kubelet.conf` generated"); + + tracing::info!("Generating `kubelet-config.yml` to /opt/kubernetes/cfg..."); + KubeletConfig::generate(); + tracing::info!("`kubelet-config.yml` generated"); + + tracing::info!("Generating `kubeconfig` using `kubectl`"); + Command::new("kubectl") + .arg("config") + .arg("set-cluster") + .arg("kubernetes") + .arg("--certificate-authority=/opt/kubernetes/ssl/ca.pem") + .arg("--embed-certs=true") + .arg(format!("--server=https://{}:6443", config.instance_ip)) + .arg("--kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-credentials") + .arg("kubelet-bootstrap") + .arg("--token=4136692876ad4b01bb9dd0988480ebba") + .arg("--kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-context") + .arg("default") + .arg("--cluster=kubernetes") + .arg("--user=kubelet-bootstrap") + .arg("--kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("use-context") + .arg("default") + .arg("--kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + + tracing::info!("Generating `kubelet.service` to /usr/lib/systemd/system/"); + KubeletUnit::generate(); + tracing::info!("`kubelet.service` generated"); + + Command::new("systemctl") + .arg("daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("systemctl") + .arg("enable") + .arg("kubelet") + .status() + .expect("Error happened when trying to enable `kubelet.service`"); + Command::new("systemctl") + .arg("start") + .arg("kubelet") + .status() + .expect("Error happened when trying to start `kubelet.service`"); + tracing::info!("Master's kubelet is now set"); + + loop { + let output = Command::new("kubectl") + .arg("get") + .arg("csr") + .output() + .unwrap() + .stdout; + if !output.is_empty() { + break; + } + tracing::info!("Waiting other etcd nodes to join cluster"); + } + + // kubectl approve master node csr. + let output = Command::new("kubectl") + .arg("get") + .arg("csr") + .output() + .unwrap() + .stdout; + let output = String::from_utf8(output).unwrap(); + tracing::info!("Output of kubectl is: {}", output); + let csr = Regex::new(r"node-csr-\S*").unwrap(); + let mut res = ""; + for word in output.split_whitespace() { + if csr.is_match(word) { + res = word; + break; + } + } + tracing::info!("Retrieved csr is: {}", res); + Command::new("kubectl") + .arg("certificate") + .arg("approve") + .arg(res) + .status() + .expect("Error happened when trying to approve csr from node"); + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/init/kube_proxy.rs b/rk8s/src/init/kube_proxy.rs new file mode 100644 index 00000000..9ee13b79 --- /dev/null +++ b/rk8s/src/init/kube_proxy.rs @@ -0,0 +1,254 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +struct KubeProxyCfg; + +impl KubeProxyCfg { + fn generate() { + let mut kube_proxy_conf = File::create("/opt/kubernetes/cfg/kube-proxy.conf") + .expect("Error happened when trying to create kube-proxy configuration file"); + + writeln!( + &mut kube_proxy_conf, + r#"KUBE_PROXY_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \ +--config=/opt/kubernetes/cfg/kube-proxy-config.yml" +"# + ) + .expect("Error happened when trying to write `kube-proxy.conf`"); + } +} + +struct KubeProxyConfig; + +impl KubeProxyConfig { + fn generate(config: &Config) { + let mut kube_proxy_config = File::create("/opt/kubernetes/cfg/kube-proxy-config.yml") + .expect("Error happened when trying to create kube-proxy configuration file"); + + writeln!( + &mut kube_proxy_config, + r#"kind: KubeProxyConfiguration +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +bindAddress: 0.0.0.0 +metricsBindAddress: 0.0.0.0:10249 +clientConnection: + kubeconfig: /opt/kubernetes/cfg/kube-proxy.kubeconfig"# + ) + .expect("Error happened when trying to write `kube-proxy-config.yml`"); + writeln!( + &mut kube_proxy_config, + "hostnameOverride: {} \\", + config.instance_name + ) + .expect("Error happened when trying to write `kube-proxy-config.yml`"); + writeln!( + &mut kube_proxy_config, + r#"clusterCIDR: 10.244.0.0/16 +"#, + ) + .expect("Error happened when trying to write `kube-proxy-config.yml`"); + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct KubeProxyCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl KubeProxyCsr { + fn from(config: &Config) -> KubeProxyCsr { + KubeProxyCsr { + CN: config.kube_proxy_CN.to_owned(), + hosts: vec![], + key: Key { + algo: config.kube_proxy_key_algo.to_owned(), + size: config.kube_proxy_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_proxy_names_C.to_owned(), + L: config.kube_proxy_names_L.to_owned(), + ST: config.kube_proxy_names_ST.to_owned(), + O: config.kube_proxy_names_O.to_owned(), + OU: config.kube_proxy_names_OU.to_owned(), + }], + } + } +} + +struct KubeProxyUnit; + +impl KubeProxyUnit { + fn generate() { + let mut proxy_unit = File::create("/usr/lib/systemd/system/kube-proxy.service") + .expect("Error happened when trying to create kube-proxy unit file"); + let content = r#"[Unit] +Description=Kubernetes Proxy +After=network.target + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kube-proxy.conf +ExecStart=/opt/kubernetes/bin/kube-proxy $KUBE_PROXY_OPTS +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +"#; + proxy_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kube-proxy unit file"); + } +} + +pub fn start(config: &Config) { + // kube-proxy + tracing::info!("kube_proxy phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Start generating `kube-proxy-csr.json`..."); + let kube_proxy_csr = KubeProxyCsr::from(config); + let content = serde_json::to_string_pretty(&kube_proxy_csr) + .expect("Error happened when trying to serialize `kube-proxy-csr.json`"); + let mut kube_proxy_csr_file = File::create("kube-proxy-csr.json") + .expect("Error happened when trying to create `kube-proxy-csr.json`"); + kube_proxy_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `kube-proxy-csr.json`"); + tracing::info!("`kube-proxy-csr.json` generated"); + + tracing::info!("Generating self-signed kube_proxy https certificate..."); + let cfssl_kube_proxy = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("kube-proxy-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("kube-proxy") + .stdin(Stdio::from(cfssl_kube_proxy.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed kube_proxy CA certificate generated"); + + tracing::info!("Generating `kube-proxy.conf` to /opt/kubernetes/cfg..."); + KubeProxyCfg::generate(); + tracing::info!("`kube-proxy.conf` generated"); + + tracing::info!("Generating `kube-proxy-config.yml` to /opt/kubernetes/cfg..."); + KubeProxyConfig::generate(config); + tracing::info!("`kube-proxy-config.yml` generated"); + + tracing::info!("Generating `kubeconfig` using `kubectl`"); + Command::new("kubectl") + .arg("config") + .arg("set-cluster") + .arg("kubernetes") + .arg("--certificate-authority=/opt/kubernetes/ssl/ca.pem") + .arg("--embed-certs=true") + .arg(format!("--server=https://{}:6443", config.instance_ip)) + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-credentials") + .arg("kube-proxy") + .arg("--client-certificate=./kube-proxy.pem") + .arg("--client-key=./kube-proxy-key.pem") + .arg("--embed-certs=true") + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-context") + .arg("default") + .arg("--cluster=kubernetes") + .arg("--user=kube-proxy") + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("use-context") + .arg("default") + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + + tracing::info!("Generating `kube-proxy.service` to /usr/lib/systemd/system/"); + KubeProxyUnit::generate(); + tracing::info!("`kube-proxy.service` generated"); + + Command::new("systemctl") + .arg("daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("systemctl") + .arg("enable") + .arg("kube-proxy") + .status() + .expect("Error happened when trying to enable `kube-proxy.service`"); + Command::new("systemctl") + .arg("start") + .arg("kube-proxy") + .status() + .expect("Error happened when trying to start `kube-proxy.service`"); + tracing::info!("Master's proxy is now set"); + + tracing::info!("Deploying Calico..."); + Command::new("curl") + .arg("https://docs.projectcalico.org/v3.20/manifests/calico.yaml") + .arg("-o") + .arg("calico.yaml") + .status() + .expect("Error happened when trying to download Calico"); + Command::new("kubectl") + .arg("apply") + .arg("-f") + .arg("calico.yaml") + .status() + .expect("Error happened when trying to apply calico.yaml"); + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/init/kube_scheduler.rs b/rk8s/src/init/kube_scheduler.rs new file mode 100644 index 00000000..e0ecdfef --- /dev/null +++ b/rk8s/src/init/kube_scheduler.rs @@ -0,0 +1,205 @@ +use crate::config::Config; +use serde::{Deserialize, Serialize}; +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +struct KubeSchedulerCfg; + +impl KubeSchedulerCfg { + fn generate() { + let mut scheduler_conf = File::create("/opt/kubernetes/cfg/kube-scheduler.conf") + .expect("Error happened when trying to create kube-scheduler configuration file"); + + writeln!( + &mut scheduler_conf, + r#"KUBE_SCHEDULER_OPTS="--logtostderr=false \ +--v=2 \ +--log-dir=/opt/kubernetes/logs \ +--leader-elect \ +--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig \ +--bind-address=127.0.0.1" +"# + ) + .expect("Error happened when trying to write `kube-controller-manager.conf`"); + } +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct KubeSchedulerCsr { + CN: String, + hosts: Vec, + key: Key, + names: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +struct Key { + algo: String, + size: i64, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Deserialize, Debug)] +struct Name { + C: String, + L: String, + ST: String, + O: String, + OU: String, +} + +impl KubeSchedulerCsr { + fn from(config: &Config) -> KubeSchedulerCsr { + KubeSchedulerCsr { + CN: config.kube_scheduler_CN.to_owned(), + hosts: vec![], + key: Key { + algo: config.kube_scheduler_key_algo.to_owned(), + size: config.kube_scheduler_key_size.to_owned(), + }, + names: vec![Name { + C: config.kube_scheduler_names_C.to_owned(), + L: config.kube_scheduler_names_L.to_owned(), + ST: config.kube_scheduler_names_ST.to_owned(), + O: config.kube_scheduler_names_O.to_owned(), + OU: config.kube_scheduler_names_OU.to_owned(), + }], + } + } +} + +struct KubeSchedulerUnit; + +impl KubeSchedulerUnit { + fn generate() { + let mut scheduler_unit = File::create("/usr/lib/systemd/system/kube-scheduler.service") + .expect("Error happened when trying to create kube-scheduler unit file"); + let content = r#"[Unit] +Description=Kubernetes Scheduler +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +EnvironmentFile=/opt/kubernetes/cfg/kube-scheduler.conf +ExecStart=/opt/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS +Restart=on-failure + +[Install] +WantedBy=multi-user.target +"#; + scheduler_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write kube-scheduler unit file"); + } +} + +pub fn start(config: &Config) { + // kube-scheduler + tracing::info!("kube_apiserver phase started"); + tracing::info!("Change working directory into `k8s`"); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/k8s"); + env::set_current_dir(work_dir).expect("Error happened when trying to change into `k8s`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + + tracing::info!("Generating `kube-scheduler.conf` to /opt/kubernetes/cfg..."); + KubeSchedulerCfg::generate(); + tracing::info!("`kube-scheduler.conf` generated"); + + tracing::info!("Start generating `kube-scheduler-csr.json`..."); + let kube_scheduler_csr = KubeSchedulerCsr::from(config); + let content = serde_json::to_string_pretty(&kube_scheduler_csr) + .expect("Error happened when trying to serialize `kube-scheduler-csr.json`"); + let mut kube_scheduler_csr_file = File::create("kube-scheduler-csr.json") + .expect("Error happened when trying to create `kube-scheduler-csr.json`"); + kube_scheduler_csr_file + .write_all(content.as_bytes()) + .expect("Error happened when trying to write content to `kube-scheduler--csr.json`"); + tracing::info!("`kube-scheduler-csr.json` generated"); + + tracing::info!("Generating self-signed kube_controller_manager https certificate..."); + let cfssl_kube_scheduler = Command::new("cfssl") + .arg("gencert") + .arg("-ca=ca.pem") + .arg("-ca-key=ca-key.pem") + .arg("-config=ca-config.json") + .arg("-profile=kubernetes") + .arg("kube-scheduler-csr.json") + .stdout(Stdio::piped()) + .spawn() + .expect("Error happened when trying to execute `cfssl` command"); + Command::new("cfssljson") + .arg("-bare") + .arg("kube-scheduler") + .stdin(Stdio::from(cfssl_kube_scheduler.stdout.unwrap())) + .status() + .expect("Error happened when trying to execute `cfssljson`"); + tracing::info!("Self-signed kube_scheduler CA certificate generated"); + + tracing::info!("Generating `kubeconfig` using `kubectl`"); + Command::new("kubectl") + .arg("config") + .arg("set-cluster") + .arg("kubernetes") + .arg("--certificate-authority=/opt/kubernetes/ssl/ca.pem") + .arg("--embed-certs=true") + .arg(format!("--server=https://{}:6443", config.instance_ip)) + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-credentials") + .arg("kube-scheduler") + .arg("--client-certificate=./kube-scheduler.pem") + .arg("--client-key=./kube-scheduler-key.pem") + .arg("--embed-certs=true") + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("set-context") + .arg("default") + .arg("--cluster=kubernetes") + .arg("--user=kube-scheduler") + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + Command::new("kubectl") + .arg("config") + .arg("use-context") + .arg("default") + .arg("--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig") + .status() + .expect("Error happened when trying to execute kubectl"); + + tracing::info!("Generating `kube-scheduler.service` to /usr/lib/systemd/system/"); + KubeSchedulerUnit::generate(); + tracing::info!("`kube-scheduler.service` generated"); + + Command::new("systemctl") + .arg("daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("systemctl") + .arg("enable") + .arg("kube-scheduler") + .status() + .expect("Error happened when trying to enable `kube-scheduler.service`"); + Command::new("systemctl") + .arg("start") + .arg("kube-scheduler") + .status() + .expect("Error happened when trying to start `kube-scheduler.service`"); + tracing::info!("Master's scheduler is now set"); + + env::set_current_dir(prev_dir).expect("Error happened when trying to change into `/rk8s`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/init/mod.rs b/rk8s/src/init/mod.rs new file mode 100644 index 00000000..7242bf5b --- /dev/null +++ b/rk8s/src/init/mod.rs @@ -0,0 +1,8 @@ +pub mod etcd; +pub mod kube_apiserver; +pub mod kube_controller_manager; +pub mod kube_ctl; +pub mod kube_let; +pub mod kube_proxy; +pub mod kube_scheduler; +pub mod pre_check; diff --git a/rk8s/src/init/pre_check.rs b/rk8s/src/init/pre_check.rs new file mode 100644 index 00000000..963123f4 --- /dev/null +++ b/rk8s/src/init/pre_check.rs @@ -0,0 +1,87 @@ +use crate::config::Config; +use std::fs::File; +use std::io::Write; +use std::process::Command; + +pub fn start(config: &Config) { + tracing::info!("Pre check started"); + + // Stop `firewalld` daemon. + tracing::info!("Stopping `firewalld` daemon..."); + Command::new("systemctl") + .arg("stop") + .arg("firewalld") + .status() + .expect("Error happened when trying to stop firewalld daemon"); + tracing::info!("`firewalld` daemon stopped"); + + // Disable `firewalld` daemon. + tracing::info!("Disabling `firewalld` daemon..."); + Command::new("systemctl") + .arg("disable") + .arg("firewalld") + .status() + .expect("Error happened when trying to disable firewalld daemon"); + tracing::info!("`firewalld` daemon disabled"); + + // Turn off selinux. + tracing::info!("Disabling selinux..."); + Command::new("sed") + .arg("-i") + .arg("s/enforcing/disabled/") + .arg("/etc/selinux/config") + .status() + .expect("Error happened when trying to disable swap partition"); + + // Turn off swap. + tracing::info!("Disabling swap partition..."); + Command::new("sed") + .arg("-ri") + .arg("s/.*swap.*/#&/") + .arg("/etc/fstab") + .status() + .expect("Error happened when trying to disable swap partition"); + tracing::info!("swap partition disabled"); + + // Set Master `hostname`. + tracing::info!("Setting master's `hostname` to master01..."); + // Acquire ip and name of this instance. + Command::new("hostnamectl") + .arg("set-hostname") + .arg(&config.instance_name) + .status() + .expect("Error happened when trying to set hostname of master"); + tracing::info!("`hostname` set to master01"); + + // Set `/etc/hosts` file. + tracing::info!("Setting `/etc/hosts` according to configuration..."); + let mut hosts_file = File::options() + .append(true) + .open("/etc/hosts") + .expect("Error happened when trying to append cluster information to `/etc/hosts`"); + for (ip, name) in &config.instance_hosts { + writeln!(&mut hosts_file, "{} {}", ip, name) + .expect("Error happened when trying to write `/etc/hosts`"); + } + tracing::info!("`/etc/hosts` set"); + + // Set IPv4 iptables. + tracing::info!("Setting `/etc/sysctl.d/k8s.conf` according to configuration..."); + let mut k8s_conf = File::create("/etc/sysctl.d/k8s.conf") + .expect("Error happened when trying to create `k8s.conf` file"); + k8s_conf + .write_all( + b"net.bridge.bridge-nf-call-ip6tables = 1\nnet.bridge.bridge-nf-call-iptables = 1\n", + ) + .expect("Error happened when trying to write to `/etc/sysctl.d/k8s.conf`"); + Command::new("sysctl") + .arg("--system") + .status() + .expect("Error happened when activating `k8s.conf`"); + tracing::info!("`/etc/sysctl.d/k8s.conf` set"); + + // Synchronize time and date using `ntpdate`. + // TODO + tracing::info!("Synchronizing date and time..."); + tracing::info!("Time and date synchronized"); +} diff --git a/rk8s/src/install/cfssl.rs b/rk8s/src/install/cfssl.rs new file mode 100644 index 00000000..13e5ce8d --- /dev/null +++ b/rk8s/src/install/cfssl.rs @@ -0,0 +1,59 @@ +use std::env; +use std::path::Path; +use std::process::Command; + +pub fn start() { + tracing::info!("Start Installing cfssl..."); + let prev_dir = Path::new("/rk8s"); + let work_dir = Path::new("/rk8s/preparation"); + env::set_current_dir(work_dir) + .expect("Error happened when trying to change into `praparation`"); + tracing::info!("Changed to {}", env::current_dir().unwrap().display()); + Command::new("cp") + .arg("cfssl") + .arg("/usr/local/bin/cfssl") + .status() + .expect("Error happened when trying to copy `cfssl`"); + tracing::info!("cfssl copied"); + Command::new("chmod") + .arg("+x") + .arg("/usr/local/bin/cfssl") + .status() + .expect("Error happened when trying to set `cfssl` executable"); + tracing::info!("cfssl is ready"); + + tracing::info!("Start installing cfssljson..."); + Command::new("cp") + .arg("cfssljson") + .arg("/usr/local/bin/cfssljson") + .status() + .expect("Error happened when trying to copy `cfssljson`"); + tracing::info!("cfssljson copied"); + Command::new("chmod") + .arg("+x") + .arg("/usr/local/bin/cfssljson") + .status() + .expect("Error happened when trying to set `cfssljson` executable"); + tracing::info!("cfssljson is ready"); + + tracing::info!("Start installing cfsslcertinfo..."); + Command::new("cp") + .arg("cfssl-certinfo") + .arg("/usr/local/bin/cfssl-certinfo") + .status() + .expect("Error happened when trying to copy `cfssl-certinfo`"); + tracing::info!("cfssl-certinfo copied"); + Command::new("chmod") + .arg("+x") + .arg("/usr/local/bin/cfssl-certinfo") + .status() + .expect("Error happened when trying to set `cfssl-certinfo` executable"); + tracing::info!("cfssl-certinfo is ready"); + + env::set_current_dir(prev_dir) + .expect("Error happened when trying to change into `praparation`"); + tracing::info!( + "Change working directory back to {}", + env::current_dir().unwrap().display() + ); +} diff --git a/rk8s/src/install/docker.rs b/rk8s/src/install/docker.rs new file mode 100644 index 00000000..630716b4 --- /dev/null +++ b/rk8s/src/install/docker.rs @@ -0,0 +1,139 @@ +use std::fs; +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; +use std::process::Command; + +use crate::config::Config; + +struct DockerCfg; + +impl DockerCfg { + fn generate() { + let mut docker_cfg = File::create("/etc/docker/daemon.json") + .expect("Error happened when trying to create docker config file"); + let content = r#"{ + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver": "json-file", + "log-opts": { + "max-size": "100m" + }, + "storage-driver": "overlay2" +} +"#; + docker_cfg + .write_all(content.as_bytes()) + .expect("Error happened when trying to write docker unit file"); + } +} + +struct DockerUnit; + +impl DockerUnit { + fn generate() { + let mut docker_unit = File::create("/usr/lib/systemd/system/docker.service") + .expect("Error happened when trying to create docker unit file"); + let content = r#"[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +After=network-online.target firewalld.service +Wants=network-online.target + +[Service] +Type=notify +ExecStart=/usr/bin/dockerd +ExecReload=/bin/kill -s HUP $MAINPID +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +TimeoutStartSec=0 +Delegate=yes +KillMode=process +Restart=on-failure +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target +"#; + docker_unit + .write_all(content.as_bytes()) + .expect("Error happened when trying to write docker unit file"); + } +} + +pub fn start(config: &Config) { + tracing::info!("Start installing docker"); + tracing::info!("Downloading docker binary from docker_URL"); + Command::new("curl") + .arg("-L") + .arg(&config.docker_url) + .arg("-o") + .arg("docker/docker-20.10.9.tgz") + .status() + .expect("Error happened when trying to download `etcd`"); + + if PathBuf::from("/rk8s/docker/docker-20.10.9.tgz").is_file() { + tracing::info!("docker downloaded"); + + tracing::info!("untaring downloaded file"); + Command::new("tar") + .arg("-zxf") + .arg("docker/docker-20.10.9.tgz") + .arg("--directory") + .arg("docker") + .status() + .expect("Error happened when trying to untar `docker` executable"); + + Command::new("cp") + .arg("docker/docker/containerd") + .arg("docker/docker/containerd-shim") + .arg("docker/docker/containerd-shim-runc-v2") + .arg("docker/docker/ctr") + .arg("docker/docker/docker") + .arg("docker/docker/dockerd") + .arg("docker/docker/docker-init") + .arg("docker/docker/docker-proxy") + .arg("docker/docker/runc") + .arg("/usr/bin") + .status() + .expect("Error happened when trying to copy `docker` executable to `/usr/bin`"); + + tracing::info!("Generating docker.service file to /usr/lib/systemd/system/"); + DockerUnit::generate(); + tracing::info!("docker.service generated"); + + let config_dir = PathBuf::from("/etc/docker"); + check_dir_exist_or_create(config_dir); + tracing::info!("Generating daemon.json to /etc/docker/"); + DockerCfg::generate(); + tracing::info!("daemon.json generated"); + + tracing::info!("docker is ready"); + } else { + tracing::error!("docker not downloaded, please try again"); + } + + Command::new("systemctl") + .arg("daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("systemctl") + .arg("enable") + .arg("docker") + .status() + .expect("Error happened when trying to enable `docker.service`"); + Command::new("systemctl") + .arg("start") + .arg("docker") + .status() + .expect("Error happened when trying to start `docker.service`"); + + tracing::info!("docker is now installed"); +} + +fn check_dir_exist_or_create(path: PathBuf) { + if !path.is_dir() { + fs::create_dir_all(path).expect("Error happened when trying to create path"); + } +} diff --git a/rk8s/src/install/etcd.rs b/rk8s/src/install/etcd.rs new file mode 100644 index 00000000..111197f8 --- /dev/null +++ b/rk8s/src/install/etcd.rs @@ -0,0 +1,56 @@ +use crate::config::Config; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +pub fn start(config: &Config) { + tracing::info!("Start downloading etcd..."); + // Retrieve file name from URL. + let fields: Vec<&str> = config.etcd_url.as_str().split('/').collect(); + let filename = fields[fields.len() - 1]; + let fields: Vec<&str> = filename.split(".tar.").collect(); + let folder_name = fields[0]; + Command::new("curl") + .arg("-L") + .arg(&config.etcd_url) + .arg("-o") + .arg(format!("etcd/{}.tar.gz", folder_name)) + .status() + .expect("Error happened when trying to download `etcd`"); + if PathBuf::from(format!("/rk8s/etcd/{}.tar.gz", folder_name)).is_file() { + tracing::info!("etcd downloaded"); + + tracing::info!("untaring downloaded file"); + Command::new("tar") + .arg("-zxf") + .arg(format!("etcd/{}.tar.gz", folder_name)) + .arg("--directory") + .arg("etcd") + .status() + .expect("Error happened when trying to untar `etcd` executable"); + + let cfg_path = PathBuf::from("/opt/etcd/cfg"); + check_dir_exist_or_create(cfg_path); + let bin_path = PathBuf::from("/opt/etcd/bin"); + check_dir_exist_or_create(bin_path); + let ssl_path = PathBuf::from("/opt/etcd/ssl"); + check_dir_exist_or_create(ssl_path); + + Command::new("cp") + .arg(format!("etcd/{}/etcd", folder_name)) + .arg(format!("etcd/{}/etcdctl", folder_name)) + .arg("/opt/etcd/bin") + .status() + .expect("Error happened when trying to copy `etcd` executable to `/opt/etcd/bin`"); + + tracing::info!("etcd is ready"); + } else { + tracing::error!("etcd not downloaded, please try again"); + } +} + +fn check_dir_exist_or_create(path: PathBuf) { + if !path.is_dir() { + fs::create_dir_all(path).expect("Error happened when trying to create path"); + } +} diff --git a/rk8s/src/install/kubernetes.rs b/rk8s/src/install/kubernetes.rs new file mode 100644 index 00000000..a9cf0281 --- /dev/null +++ b/rk8s/src/install/kubernetes.rs @@ -0,0 +1,89 @@ +use crate::config::Config; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +pub fn start(config: &Config) { + tracing::info!("Start downloading kubernetes..."); + // Retrieve file name from URL. + let filename = "kubernetes-server-linux-amd64"; + Command::new("curl") + .arg("-L") + .arg(&config.kubernetes_url) + .arg("-o") + .arg(format!("k8s/{}.tar.gz", filename)) + .status() + .expect("Error happened when trying to download `kubernetes`"); + if PathBuf::from(format!("/rk8s/k8s/{}.tar.gz", filename)).is_file() { + tracing::info!("kubernetes downloaded"); + + tracing::info!("untaring downloaded file"); + Command::new("tar") + .arg("-zxf") + .arg(format!("k8s/{}.tar.gz", filename)) + .arg("--directory") + .arg("k8s") + .status() + .expect("Error happened when trying to untar `kubernetes` executable"); + + let cfg_path = PathBuf::from("/opt/kubernetes/cfg"); + check_dir_exist_or_create(cfg_path); + let bin_path = PathBuf::from("/opt/kubernetes/bin"); + check_dir_exist_or_create(bin_path); + let ssl_path = PathBuf::from("/opt/kubernetes/ssl"); + check_dir_exist_or_create(ssl_path); + let logs_path = PathBuf::from("/opt/kubernetes/logs"); + check_dir_exist_or_create(logs_path); + + Command::new("cp") + .arg("k8s/kubernetes/server/bin/kube-apiserver") + .arg("k8s/kubernetes/server/bin/kube-scheduler") + .arg("k8s/kubernetes/server/bin/kube-controller-manager") + .arg("k8s/kubernetes/server/bin/kube-proxy") + .arg("k8s/kubernetes/server/bin/kubelet") + .arg("/opt/kubernetes/bin") + .status() + .expect("Error happened when trying to copy `kubernetes` executable to `/opt/kubernetes/bin`"); + + // Prepare content to be sent to worker nodes. + let kubernetes_bin = PathBuf::from("to_send/kubernetes/bin"); + check_dir_exist_or_create(kubernetes_bin); + let kubernetes_cfg = PathBuf::from("to_send/kubernetes/cfg"); + check_dir_exist_or_create(kubernetes_cfg); + let kubernetes_ssl = PathBuf::from("to_send/kubernetes/ssl"); + check_dir_exist_or_create(kubernetes_ssl); + let kubernetes_logs = PathBuf::from("to_send/kubernetes/logs"); + check_dir_exist_or_create(kubernetes_logs); + Command::new("cp") + .arg("k8s/kubernetes/server/bin/kube-proxy") + .arg("k8s/kubernetes/server/bin/kubelet") + .arg("to_send/kubernetes/bin") + .status() + .expect("Error happened when trying to copy `kubernetes` executable to `/opt/kubernetes/bin`"); + + Command::new("cp") + .arg("k8s/kubernetes/server/bin/kube-apiserver") + .arg("k8s/kubernetes/server/bin/kube-scheduler") + .arg("k8s/kubernetes/server/bin/kube-controller-manager") + .arg("k8s/kubernetes/server/bin/kube-proxy") + .arg("k8s/kubernetes/server/bin/kubelet") + .arg("/opt/kubernetes/bin") + .status() + .expect("Error happened when trying to copy `kubernetes` executable to `/opt/kubernetes/bin`"); + Command::new("cp") + .arg("k8s/kubernetes/server/bin/kubectl") + .arg("/usr/bin") + .status() + .expect("Error happened when trying to copy `kubernetes` executable to `/usr/bin`"); + + // tracing::info!("kubernetes is ready"); + } else { + tracing::error!("kubernetes not downloaded, please try again"); + } +} + +fn check_dir_exist_or_create(path: PathBuf) { + if !path.is_dir() { + fs::create_dir_all(path).expect("Error happened when trying to create path"); + } +} diff --git a/rk8s/src/install/mod.rs b/rk8s/src/install/mod.rs new file mode 100644 index 00000000..70eb12b2 --- /dev/null +++ b/rk8s/src/install/mod.rs @@ -0,0 +1,4 @@ +pub mod cfssl; +pub mod docker; +pub mod etcd; +pub mod kubernetes; diff --git a/rk8s/src/join/etcd.rs b/rk8s/src/join/etcd.rs new file mode 100644 index 00000000..43986161 --- /dev/null +++ b/rk8s/src/join/etcd.rs @@ -0,0 +1,90 @@ +use crate::config::Config; +use std::fs::File; +use std::io::Write; +use std::process::Command; + +struct ETCDCfg; + +impl ETCDCfg { + fn generate(config: &Config) { + let mut etcd_conf = File::create("/opt/etcd/cfg/etcd.conf") + .expect("Error happened when trying to create etcd configuration file"); + + writeln!(&mut etcd_conf, "#[Member]") + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_NAME=\"etcd_{}\"", + config.instance_name + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_DATA_DIR=\"/var/lib/etcd/default.etcd\"" + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_LISTEN_PEER_URLS=\"https://{}:2380\"", + config.instance_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_LISTEN_CLIENT_URLS=\"https://{}:2379\"", + config.instance_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf).expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf, "#[Clustering]") + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_INITIAL_ADVERTISE_PEER_URLS=\"https://{}:2380\"", + config.instance_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_ADVERTISE_CLIENT_URLS=\"https://{}:2379\"", + config.instance_ip + ) + .expect("Error happened when trying to write `etcd.conf`"); + let mut buffer = String::new(); + for (ip, name) in &config.instance_hosts { + buffer.push_str(format!("etcd_{}=https://{}:2380,", name, ip).as_str()); + } + buffer.pop(); + writeln!(&mut etcd_conf, "ETCD_INITIAL_CLUSTER=\"{}\"", buffer) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!( + &mut etcd_conf, + "ETCD_INITIAL_CLUSTER_TOKEN=\"etcd-cluster\"" + ) + .expect("Error happened when trying to write `etcd.conf`"); + writeln!(&mut etcd_conf, "ETCD_INITIAL_CLUSTER_STATE=\"new\"") + .expect("Error happened when trying to write `etcd.conf`"); + } +} + +pub fn start(config: &Config) { + tracing::info!("Overwriting `/opt/etcd/cfg/etcd.conf` according to rk8s configuration"); + ETCDCfg::generate(config); + tracing::info!("`/opt/etcd/cfg/etcd.conf` is now set"); + + Command::new("systemctl") + .arg("daemon-reload") + .status() + .expect("Error happened when trying to reload systemd daemons"); + Command::new("systemctl") + .arg("enable") + .arg("etcd") + .status() + .expect("Error happened when trying to enable `etcd.service`"); + Command::new("systemctl") + .arg("start") + .arg("etcd") + .status() + .expect("Error happened when trying to start `etcd.service`"); + tracing::info!("etcd_{} is now etcd set", config.instance_name); +} diff --git a/rk8s/src/join/mod.rs b/rk8s/src/join/mod.rs new file mode 100644 index 00000000..29780d57 --- /dev/null +++ b/rk8s/src/join/mod.rs @@ -0,0 +1 @@ +pub mod etcd; diff --git a/rk8s/src/main.rs b/rk8s/src/main.rs new file mode 100644 index 00000000..424667c5 --- /dev/null +++ b/rk8s/src/main.rs @@ -0,0 +1,14 @@ +mod config; +mod deploy; +mod init; +mod install; +mod join; +mod rk8s; + +use rk8s::run_command; + +fn main() { + tracing_subscriber::fmt::init(); + tracing::info!("rk8s started"); + run_command(); +} diff --git a/rk8s/src/rk8s.rs b/rk8s/src/rk8s.rs new file mode 100644 index 00000000..9d6d3463 --- /dev/null +++ b/rk8s/src/rk8s.rs @@ -0,0 +1,172 @@ +use crate::config; +use crate::config::Config; +use crate::deploy; +use crate::init; +use crate::install; +use crate::join; +use clap::{Parser, Subcommand}; +use std::env; +use std::fs; +use std::path::Path; + +#[derive(Parser)] +#[command(name = "rk8s")] +#[command(author = "Ruoqing He ")] +#[command(version = "0.1.0")] +#[command( + about = "A rust implementation of Kubeadm, easily bootstrap a secure Kubernetes cluster", + long_about = " + +Introduction: + + ┌──────────────────────────────────────────────────────────┐ + │ RK8S │ + │ A kubeadm implementation written in Rust │ + │ Easily bootstrap a secure Kubernetes cluster │ + │ │ + │ Please give us feedback at: │ + │ │ + └──────────────────────────────────────────────────────────┘ + +Example usage: + + Create a two-machine cluster with one control-plane node + (which controls the cluster), and one worker node + (where your workloads, like Pods and Deployments run). + + ┌──────────────────────────────────────────────────────────┐ + │ On the first machine: │ + ├──────────────────────────────────────────────────────────┤ + │ control-plane# kubeadm init │ + └──────────────────────────────────────────────────────────┘ + + ┌──────────────────────────────────────────────────────────┐ + │ On the second machine: │ + ├──────────────────────────────────────────────────────────┤ + │ worker# kubeadm join │ + └──────────────────────────────────────────────────────────┘ + + You can then repeat the second step on as many other machines as you like. +" +)] +struct Cli { + #[arg(long)] + flag1: Option, + #[arg(long)] + flag2: Option, + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + Deploy, + Precheck, + Init, + Join, + Install { target: String }, + Generate { target: String }, +} + +pub fn run_command() { + // Change working directory + let work_root = Path::new("/rk8s"); + if !work_root.is_dir() { + fs::create_dir("/rk8s").expect("Error happened when trying to create `/rk8s` directory"); + fs::create_dir("/rk8s/cfg") + .expect("Error happened when trying to create `/rk8s/cfg` directory"); + fs::create_dir("/rk8s/etcd") + .expect("Error happened when trying to create `/rk8s/etcd` directory"); + fs::create_dir("/rk8s/docker") + .expect("Error happened when trying to create `/rk8s/docker` directory"); + fs::create_dir("/rk8s/k8s") + .expect("Error happened when trying to create `/rk8s/k8s` directory"); + fs::create_dir("/rk8s/preparation") + .expect("Error happened when trying to create `/rk8s/preparation` directory"); + } + env::set_current_dir(work_root) + .expect("Error happened when trying to change working directory"); + + let cli = Cli::parse(); + + match &cli.command { + Commands::Deploy => { + // Read configuration file. + let adm_config = Config::init(); + deploy::pre_check::start(&adm_config); + deploy::etcd::start(&adm_config); + deploy::docker::start(&adm_config); + deploy::prepare_kube::start(&adm_config); + deploy::kube_apiserver::start(&adm_config); + deploy::kube_controller_manager::start(&adm_config); + deploy::kube_scheduler::start(&adm_config); + deploy::kubectl::start(&adm_config); + deploy::kubelet::start(&adm_config); + deploy::kube_proxy::start(&adm_config); + } + Commands::Precheck => { + // Read configuration file. + let adm_config = Config::init(); + init::pre_check::start(&adm_config); + } + Commands::Init => { + // Read configuration file. + let adm_config = Config::init(); + tracing::info!("Init subcommand invoked."); + init::etcd::start(&adm_config); + init::kube_apiserver::start(&adm_config); + init::kube_controller_manager::start(&adm_config); + init::kube_scheduler::start(&adm_config); + init::kube_ctl::start(&adm_config); + init::kube_let::start(&adm_config); + init::kube_proxy::start(&adm_config); + } + Commands::Join => { + // Read configuration file. + let adm_config = Config::init(); + tracing::info!("Join subcommand invoked."); + join::etcd::start(&adm_config); + } + Commands::Install { target } => { + // Read configuration file. + let adm_config = Config::init(); + match target.as_str() { + "etcd" => { + tracing::info!("Installing etcd..."); + install::etcd::start(&adm_config); + } + "cfssl" => { + tracing::info!("Installing cfssl..."); + install::cfssl::start(); + tracing::info!("cfssl installation complete"); + } + "docker" => { + tracing::info!("Installing docker..."); + install::docker::start(&adm_config); + tracing::info!("docker installation complete"); + } + "kubernetes" => { + tracing::info!("Installing kubernetes..."); + install::kubernetes::start(&adm_config); + tracing::info!("kubernetes installation complete"); + } + _ => { + tracing::info!("Unknown target"); + } + } + } + Commands::Generate { target } => { + // Generate `config_template` do not require reading configuration. + match target.as_str() { + "config" => { + tracing::info!("Generating `config-template.yaml`..."); + config::generate_config_template(); + tracing::info!("`config_template.yaml` generated under `rk8s/cfg`"); + } + _ => { + tracing::info!("Unknown target"); + } + } + } + } +} -- Gitee