diff --git a/cmd/deploy.go b/cmd/deploy.go index 2620df40d26c9e5ce7f45b3b44045ca8822232f3..ace4146c74d6a636e84ce7f6c8fdbabe99376a70 100755 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -25,7 +25,7 @@ import ( "nestos-kubernetes-deployer/pkg/cert" "nestos-kubernetes-deployer/pkg/configmanager" "nestos-kubernetes-deployer/pkg/configmanager/asset" - "nestos-kubernetes-deployer/pkg/fileserver" + "nestos-kubernetes-deployer/pkg/httpserver" "nestos-kubernetes-deployer/pkg/ignition/machine" "nestos-kubernetes-deployer/pkg/infra" "nestos-kubernetes-deployer/pkg/kubeclient" @@ -117,8 +117,8 @@ func getClusterConfig(options *opts.OptionsList) (*asset.ClusterAsset, error) { } // startHttpService initializes the HTTP file service, adds files to the cache, and starts the service. -func startHttpService(conf *asset.ClusterAsset) (*fileserver.HttpFileService, error) { - fileService := fileserver.NewFileService(configmanager.GetBootstrapIgnPort()) +func startHttpService(conf *asset.ClusterAsset) (*httpserver.HttpFileService, error) { + fileService := httpserver.NewFileService(configmanager.GetBootstrapIgnPort()) // Ignition files are divided into three types: // control plane ignition files for initializing the cluster, diff --git a/cmd/destroy.go b/cmd/destroy.go index 3b96436b5117227d4a05a84befef71771219df52..ade2fba3b812a1e737d7c8c25e252905f3e26b11 100644 --- a/cmd/destroy.go +++ b/cmd/destroy.go @@ -20,8 +20,6 @@ import ( "nestos-kubernetes-deployer/cmd/command/opts" "nestos-kubernetes-deployer/pkg/configmanager" "nestos-kubernetes-deployer/pkg/infra" - "os" - "path/filepath" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -66,10 +64,10 @@ func runDestroyCmd(cmd *cobra.Command, args []string) error { } // delete asset files - filepath := filepath.Join(persistDir, clusterID) - if err := os.RemoveAll(filepath); err != nil { + if err := configmanager.Delete(clusterID); err != nil { logrus.Errorf("Failed to clean the asset files") return err } + return nil } diff --git a/cmd/extend.go b/cmd/extend.go index 64a1a47c3ec6238d5ca0f3fd21a34ebed9789249..81d2915eaf83e0723b0b5c101dc3041a10e7a58c 100755 --- a/cmd/extend.go +++ b/cmd/extend.go @@ -16,15 +16,25 @@ limitations under the License. package cmd import ( + "context" "fmt" + "io/ioutil" "nestos-kubernetes-deployer/cmd/command" "nestos-kubernetes-deployer/cmd/command/opts" "nestos-kubernetes-deployer/pkg/configmanager" "nestos-kubernetes-deployer/pkg/configmanager/asset" + "nestos-kubernetes-deployer/pkg/httpserver" + "nestos-kubernetes-deployer/pkg/ignition/machine" "nestos-kubernetes-deployer/pkg/infra" + "nestos-kubernetes-deployer/pkg/kubeclient" + "time" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ) func NewExtendCommand() *cobra.Command { @@ -50,7 +60,7 @@ func runExtendCmd(cmd *cobra.Command, args []string) error { num, err := cmd.Flags().GetUint("num") if err != nil { - logrus.Errorf("Failed to get assets directory: %v", err) + logrus.Errorf("Failed to get the number of extended nodes: %v", err) return err } @@ -64,9 +74,12 @@ func runExtendCmd(cmd *cobra.Command, args []string) error { logrus.Errorf("Failed to get cluster config using the cluster id: %v", err) return err } - extendArray(clusterConfig, int(num)) + newHostnames := extendArray(clusterConfig, int(num)) + + fileService := httpserver.NewFileService(configmanager.GetBootstrapIgnPort()) + defer fileService.Stop() - if err := extendCluster(clusterConfig); err != nil { + if err := extendCluster(clusterConfig, fileService); err != nil { logrus.Errorf("Failed to extend %s cluster: %v", clusterID, err) return err } @@ -74,13 +87,20 @@ func runExtendCmd(cmd *cobra.Command, args []string) error { logrus.Errorf("Failed to persist the cluster asset: %v", err) return err } - logrus.Infof("To access 'cluster-id:%s' cluster using 'kubectl', run 'export KUBECONFIG=%s'", clusterID, clusterConfig.AdminKubeConfig) + + logrus.Infof("Waiting for cluster extend nodes to be ready...") + if err := checkNodesReady(clusterConfig, newHostnames); err != nil { + return err + } + + logrus.Infof("The cluster id:%s node is extended successfully", clusterID) return nil } -func extendArray(c *asset.ClusterAsset, count int) { +func extendArray(c *asset.ClusterAsset, count int) []string { num := len(c.Worker) + var newHostnames []string for i := 0; i < count; i++ { hostname := fmt.Sprintf("k8s-worker%02d", num+i+1) c.Worker = append(c.Worker, asset.NodeAsset{ @@ -93,10 +113,24 @@ func extendArray(c *asset.ClusterAsset, count int) { }, Ignitions: c.Worker[i].Ignitions, }) + newHostnames = append(newHostnames, hostname) } + return newHostnames } -func extendCluster(conf *asset.ClusterAsset) error { +func extendCluster(conf *asset.ClusterAsset, fileService *httpserver.HttpFileService) error { + data, err := ioutil.ReadFile(conf.Worker[0].CreateIgnPath) + if err != nil { + logrus.Errorf("error reading Ignition file: %v", err) + return err + } + + fileService.AddFileToCache(machine.WorkerIgnFilename, data) + if err := fileService.Start(); err != nil { + logrus.Errorf("error starting file service: %v", err) + return err + } + // regenerate worker.tf var worker infra.Infra if err := worker.Generate(conf, "worker"); err != nil { @@ -113,3 +147,54 @@ func extendCluster(conf *asset.ClusterAsset) error { return nil } + +// waitUntilNodesReady waits until all nodes are ready within a given timeout +func waitUntilNodesReady(ctx context.Context, clientset *kubernetes.Clientset, nodeNames []string, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + for { + time.Sleep(10 * time.Second) + select { + case <-ctx.Done(): + return ctx.Err() + default: + allNodesReady := true + for _, nodeName := range nodeNames { + node, err := clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + allNodesReady = false + break + } + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status != corev1.ConditionTrue { + allNodesReady = false + break + } + } + } + if allNodesReady { + return nil + } + } + } +} + +// checkNodesReady waits for all nodes to be ready +func checkNodesReady(conf *asset.ClusterAsset, nodeNames []string) error { + clientset, err := kubeclient.CreateClient(conf.Kubernetes.AdminKubeConfig) + if err != nil { + logrus.Errorf("error creating Kubernetes client: %v", err) + return err + } + + // Wait for nodes to be ready + timeout := 30 * time.Minute + err = waitUntilNodesReady(context.Background(), clientset, nodeNames, timeout) + if err != nil { + logrus.Errorf("error waiting for nodes to be ready: %v", err) + return err + } + + return nil +} diff --git a/pkg/configmanager/asset/clusterasset.go b/pkg/configmanager/asset/clusterasset.go index 42194b78f4999e4241fab6aeb5b77c82e24dff89..c5b9d40d1e5ad67983a7a7476ae0cd5aff3adfdc 100644 --- a/pkg/configmanager/asset/clusterasset.go +++ b/pkg/configmanager/asset/clusterasset.go @@ -172,7 +172,7 @@ type Kubernetes struct { Pause_Image string Release_Image_URL string Token string - AdminKubeConfig string `json:"-" yaml:"-"` + AdminKubeConfig string CertificateKey string Network diff --git a/pkg/configmanager/globalconfig/globalconfig.go b/pkg/configmanager/globalconfig/globalconfig.go index b1f0faa7df14f9de6de8ca4ef8d7c6a628bf48e2..5fc1a128f1c8a5f1e11e9b8d2bb85ba8214b090e 100644 --- a/pkg/configmanager/globalconfig/globalconfig.go +++ b/pkg/configmanager/globalconfig/globalconfig.go @@ -21,41 +21,66 @@ import ( "nestos-kubernetes-deployer/cmd/command/opts" "nestos-kubernetes-deployer/pkg/utils" "os" + "path/filepath" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) +const GlobalConfigFile = "global_config.yaml" + func InitGlobalConfig(opts *opts.OptionsList) (*GlobalConfig, error) { globalAsset := &GlobalConfig{ - Log_Level: "default log level", + Log_Level: "default log level", + ClusterConfig_Path: "", + PersistDir: opts.RootOptDir, // default persist directory + BootstrapUrl: BootstrapUrl{ + BootstrapIgnPort: "9080", // default port + }, + } + configFile := filepath.Join(globalAsset.PersistDir, GlobalConfigFile) + if _, err := os.Stat(configFile); err == nil { + configData, err := os.ReadFile(configFile) + if err != nil { + logrus.Errorf("Failed to read config file: %s\n", err) + return nil, err + } + err = yaml.Unmarshal(configData, globalAsset) + if err != nil { + logrus.Errorf("Failed to unmarshal config data: %s\n", err) + return nil, err + } } if opts.NKD.Log_Level != "" { globalAsset.Log_Level = opts.NKD.Log_Level } - - persistDir := opts.RootOptDir - if err := os.MkdirAll(persistDir, 0644); err != nil { - return nil, err + if opts.NKD.BootstrapIgnHost != "" { + globalAsset.BootstrapIgnHost = opts.NKD.BootstrapIgnHost } - globalAsset.PersistDir = persistDir - globalAsset.BootstrapIgnHost = opts.NKD.BootstrapIgnHost - globalAsset.BootstrapIgnPort = opts.NKD.BootstrapIgnPort + if opts.NKD.BootstrapIgnPort != "" { + globalAsset.BootstrapIgnPort = opts.NKD.BootstrapIgnPort + } + if !utils.IsPortOpen(globalAsset.BootstrapIgnPort) { + return nil, fmt.Errorf("The port %s is occupied.", globalAsset.BootstrapIgnPort) + } if globalAsset.BootstrapIgnHost == "" { if ip, err := utils.GetLocalIP(); err != nil { - return nil, fmt.Errorf("failed to get local IP: %v", err) + logrus.Errorf("failed to get local IP: %v", err) + return nil, err } else { globalAsset.BootstrapIgnHost = ip } } - if globalAsset.BootstrapIgnPort == "" { - // HTTP service default port - globalAsset.BootstrapIgnPort = "9080" + if err := os.MkdirAll(globalAsset.PersistDir, 0644); err != nil { + return nil, err } - if !utils.IsPortOpen(globalAsset.BootstrapIgnPort) { - return nil, fmt.Errorf("The port %s is occupied.", globalAsset.BootstrapIgnPort) + if err := globalAsset.Persist(); err != nil { + return nil, err } return globalAsset, nil @@ -75,13 +100,30 @@ type BootstrapUrl struct { BootstrapIgnPort string `yaml:"bootstrap_ign_port"` } -// TODO: Delete deletes the global asset. -func (ga *GlobalConfig) Delete() error { +// Delete deletes the global asset. +func (ga *GlobalConfig) Delete(persistFilePath string) error { + if _, err := os.Stat(persistFilePath); os.IsNotExist(err) { + return nil + } + + if err := os.Remove(persistFilePath); err != nil { + logrus.Errorf("failed to delete global config file: %v", err) + return err + } + return nil } -// TODO: Persist persists the global asset. func (ga *GlobalConfig) Persist() error { - // TODO + globalConfigData, err := yaml.Marshal(ga) + if err != nil { + logrus.Errorf("failed to marshal global config: %v", err) + return err + } + + if err := os.WriteFile(ga.PersistDir, globalConfigData, 0644); err != nil { + logrus.Errorf("failed to write global config file: %v", err) + return err + } return nil } diff --git a/pkg/configmanager/manager.go b/pkg/configmanager/manager.go index 5da047c78e7986925f9df3b603b9204f6efc16a3..4920120e8ca7189e51c38f7006c7f526d3aa334d 100644 --- a/pkg/configmanager/manager.go +++ b/pkg/configmanager/manager.go @@ -33,6 +33,8 @@ var ClusterAsset = map[string]*asset.ClusterAsset{} // var InfraAsset = map[string]*asset.InfraAsset{} +const clusterConfigFile string = "cluster_config.yaml" + func Initial(opts *opts.OptionsList) error { // Init global asset globalConfig, err := globalconfig.InitGlobalConfig(opts) @@ -41,7 +43,7 @@ func Initial(opts *opts.OptionsList) error { } GlobalConfig = globalConfig - files, err := filepath.Glob(filepath.Join(globalConfig.PersistDir, "*", "cluster_config.yaml")) + files, err := filepath.Glob(filepath.Join(globalConfig.PersistDir, "*", clusterConfigFile)) if err != nil { return err } diff --git a/pkg/fileserver/server.go b/pkg/httpserver/server.go similarity index 99% rename from pkg/fileserver/server.go rename to pkg/httpserver/server.go index 3dce06e60900c002321be07a3d2bdbc5506f5928..094acc37e34e2ab137f589a222b9ceebb1fb27f4 100644 --- a/pkg/fileserver/server.go +++ b/pkg/httpserver/server.go @@ -15,7 +15,7 @@ limitations under the License. */ // service.go -package fileserver +package httpserver import ( "errors" diff --git a/test/server_test.go b/test/httpserver_test/httpserver_test.go similarity index 98% rename from test/server_test.go rename to test/httpserver_test/httpserver_test.go index e6bcf26b1e64d18ab0b417988346151270da6f40..98647fe0030a2847676b4b07e4dd2ea08a45ccf6 100644 --- a/test/server_test.go +++ b/test/httpserver_test/httpserver_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fileserver +package httpserver_test import ( "io/ioutil"