From c87508242599e90cc4e5efc737ab6d821e92adc1 Mon Sep 17 00:00:00 2001 From: lauk001 Date: Wed, 5 Jun 2024 15:01:13 +0800 Subject: [PATCH] Optimize cluster http and tftp running code --- cmd/deploy.go | 162 +++++++++++++++++------------------ cmd/extend.go | 77 +++++++---------- pkg/httpserver/httpserver.go | 8 ++ pkg/infra/ipxe.go | 63 -------------- pkg/infra/pxe.go | 90 ------------------- pkg/tftpserver/tftpserver.go | 8 ++ 6 files changed, 127 insertions(+), 281 deletions(-) delete mode 100644 pkg/infra/ipxe.go delete mode 100644 pkg/infra/pxe.go diff --git a/cmd/deploy.go b/cmd/deploy.go index 26b3a49..9502784 100755 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -32,6 +32,7 @@ import ( "nestos-kubernetes-deployer/pkg/infra" "nestos-kubernetes-deployer/pkg/kubeclient" "nestos-kubernetes-deployer/pkg/osmanager" + "nestos-kubernetes-deployer/pkg/tftpserver" "nestos-kubernetes-deployer/pkg/utils" "net/http" "os" @@ -78,11 +79,12 @@ func runDeployCmd(cmd *cobra.Command, args []string) error { return err } - if err := deployCluster(config); err != nil { - logrus.Errorf("Failed to deploy %s cluster: %v", clusterID, err) + if err := createCluster(config); err != nil { + logrus.Errorf("Failed to create cluster: %v", err) return err } + logrus.Info("Cluster deployment completed successfully!") logrus.Infof("To access 'cluster-id:%s' cluster using 'kubectl', run 'export KUBECONFIG=%s'", clusterID, config.AdminKubeConfig) return nil } @@ -118,55 +120,10 @@ func getClusterConfig(options *opts.OptionsList) (*asset.ClusterAsset, error) { return config, nil } -func deployCluster(conf *asset.ClusterAsset) error { - hs := httpserver.NewHTTPService(configmanager.GetBootstrapIgnPort()) - defer hs.Stop() - - if err := createCluster(conf, hs); err != nil { - logrus.Errorf("Failed to create cluster: %v", err) - return err - } +func createCluster(conf *asset.ClusterAsset) error { + httpService := httpserver.NewHTTPService(configmanager.GetBootstrapIgnPort()) + defer httpService.Stop() - kubeClient, err := kubeclient.CreateClient(conf.Kubernetes.AdminKubeConfig) - if err != nil { - logrus.Errorf("Failed to create kubernetes client %v", err) - return err - } - - if err := waitForAPIReady(kubeClient); err != nil { - logrus.Errorf("Failed while waiting for Kubernetes API to be ready: %v", err) - return err - } - // Set kubeconfig environment variable - os.Setenv("KUBECONFIG", conf.Kubernetes.AdminKubeConfig) - - // Apply network plugin - if err := applyNetworkPlugin(conf.Network.Plugin, conf.IsNestOS); err != nil { - logrus.Errorf("Failed to apply network plugin: %v", err) - return err - } - logrus.Info("Network plugin deployment completed successfully.") - - if conf.Housekeeper.DeployHousekeeper { - logrus.Info("Starting deployment of Housekeeper...") - if err := deployHousekeeper(conf.Housekeeper, conf.Kubernetes.AdminKubeConfig); err != nil { - logrus.Errorf("Failed to deploy operator: %v", err) - return err - } - logrus.Info("Housekeeper deployment completed successfully.") - } - - // Wait for pods to be ready - if err := waitForPodsReady(kubeClient); err != nil { - logrus.Errorf("Failed while waiting for pods to be in 'Ready' state: %v", err) - return err - } - - logrus.Info("Cluster deployment completed successfully!") - return nil -} - -func createCluster(conf *asset.ClusterAsset, httpService *httpserver.HTTPService) error { osMgr := osmanager.NewOSManager(conf) if err := osMgr.GenerateOSConfig(); err != nil { logrus.Errorf("Error generating OS config: %v", err) @@ -203,12 +160,7 @@ func createCluster(conf *asset.ClusterAsset, httpService *httpserver.HTTPService p := infra.InfraPlatform{} switch strings.ToLower(conf.Platform) { case "libvirt": - go func() { - if err := httpService.Start(); err != nil { - logrus.Errorf("error starting http service: %v", err) - return - } - }() + httpserver.StartHTTPService(httpService) libvirtMaster := &infra.Libvirt{ PersistDir: configmanager.GetPersistDir(), @@ -235,12 +187,7 @@ func createCluster(conf *asset.ClusterAsset, httpService *httpserver.HTTPService return err } case "openstack": - go func() { - if err := httpService.Start(); err != nil { - logrus.Errorf("error starting http service: %v", err) - return - } - }() + httpserver.StartHTTPService(httpService) openstackMaster := &infra.OpenStack{ PersistDir: configmanager.GetPersistDir(), @@ -267,36 +214,83 @@ func createCluster(conf *asset.ClusterAsset, httpService *httpserver.HTTPService } case "pxe": pxeConfig := conf.InfraPlatform.(*infraasset.PXEAsset) - pxe := &infra.PXE{ - IP: pxeConfig.IP, - HTTPServerPort: pxeConfig.HTTPServerPort, - HTTPRootDir: pxeConfig.HTTPRootDir, - TFTPServerPort: pxeConfig.TFTPServerPort, - TFTPRootDir: pxeConfig.TFTPRootDir, - HTTPService: httpService, - } - p.SetInfra(pxe) - if err := p.Deploy(); err != nil { - logrus.Errorf("Failed to deploy PXE:%v", err) - return err - } + httpService.Port = pxeConfig.HTTPServerPort + httpService.DirPath = pxeConfig.HTTPRootDir + httpserver.StartHTTPService(httpService) + + tftpService := tftpserver.NewTFTPService(pxeConfig.IP, pxeConfig.TFTPServerPort, pxeConfig.TFTPRootDir) + go func() { + select { + case <-httpService.Ch: + logrus.Info("tftp server stop") + tftpService.Stop() + return + } + }() + go func() { + if err := tftpService.Start(); err != nil { + logrus.Errorf("error starting http service: %v", err) + return + } + }() + defer tftpService.Stop() + case "ipxe": ipxeConfig := conf.InfraPlatform.(*infraasset.IPXEAsset) - ipxe := &infra.IPXE{ - Port: ipxeConfig.Port, - FilePath: ipxeConfig.FilePath, - OSInstallTreePath: ipxeConfig.OSInstallTreePath, - HTTPService: httpService, - } - p.SetInfra(ipxe) - if err := p.Deploy(); err != nil { - logrus.Errorf("Failed to deploy IPXE:%v", err) + httpService.Port = ipxeConfig.Port + httpService.DirPath = ipxeConfig.OSInstallTreePath + fileContent, err := os.ReadFile(ipxeConfig.FilePath) + if err != nil { return err } + httpService.AddFileToCache(constants.IPXECfg, fileContent) + httpserver.StartHTTPService(httpService) + default: return errors.New("unsupported platform") } + if err := clusterCreatePost(conf); err != nil { + return err + } + return nil +} + +func clusterCreatePost(conf *asset.ClusterAsset) error { + kubeClient, err := kubeclient.CreateClient(conf.Kubernetes.AdminKubeConfig) + if err != nil { + logrus.Errorf("Failed to create kubernetes client %v", err) + return err + } + + if err := waitForAPIReady(kubeClient); err != nil { + logrus.Errorf("Failed while waiting for Kubernetes API to be ready: %v", err) + return err + } + // Set kubeconfig environment variable + os.Setenv("KUBECONFIG", conf.Kubernetes.AdminKubeConfig) + + // Apply network plugin + if err := applyNetworkPlugin(conf.Network.Plugin, conf.IsNestOS); err != nil { + logrus.Errorf("Failed to apply network plugin: %v", err) + return err + } + logrus.Info("Network plugin deployment completed successfully.") + + if conf.Housekeeper.DeployHousekeeper { + logrus.Info("Starting deployment of Housekeeper...") + if err := deployHousekeeper(conf.Housekeeper, conf.Kubernetes.AdminKubeConfig); err != nil { + logrus.Errorf("Failed to deploy operator: %v", err) + return err + } + logrus.Info("Housekeeper deployment completed successfully.") + } + + // Wait for pods to be ready + if err := waitForPodsReady(kubeClient); err != nil { + logrus.Errorf("Failed while waiting for pods to be in 'Ready' state: %v", err) + return err + } return nil } diff --git a/cmd/extend.go b/cmd/extend.go index 0057c9f..c1a3f4e 100755 --- a/cmd/extend.go +++ b/cmd/extend.go @@ -30,6 +30,7 @@ import ( "nestos-kubernetes-deployer/pkg/kubeclient" "nestos-kubernetes-deployer/pkg/osmanager" "nestos-kubernetes-deployer/pkg/terraform" + "nestos-kubernetes-deployer/pkg/tftpserver" "os" "strings" "time" @@ -77,8 +78,6 @@ func runExtendCmd(cmd *cobra.Command, args []string) error { return err } - httpService := httpserver.NewHTTPService(configmanager.GetBootstrapIgnPort()) - num, err := cmd.Flags().GetUint("num") if err != nil { platform := strings.ToLower(clusterConfig.Platform) @@ -90,17 +89,11 @@ func runExtendCmd(cmd *cobra.Command, args []string) error { newHostnames := extendArray(clusterConfig, int(num)) - if err := extendCluster(clusterConfig, httpService); err != nil { + if err := extendCluster(clusterConfig, newHostnames); err != nil { logrus.Errorf("Failed to extend %s cluster: %v", clusterID, err) return err } - logrus.Infof("Waiting for cluster extend nodes to be ready...") - if err := checkNodesReady(clusterConfig, newHostnames); err != nil { - return err - } - - httpService.Stop() logrus.Infof("The cluster nodes are extended successfully") return nil @@ -125,7 +118,10 @@ func extendArray(c *asset.ClusterAsset, count int) []string { return newHostnames } -func extendCluster(conf *asset.ClusterAsset, httpService *httpserver.HTTPService) error { +func extendCluster(conf *asset.ClusterAsset, nodeNames []string) error { + httpService := httpserver.NewHTTPService(configmanager.GetBootstrapIgnPort()) + defer httpService.Stop() + data, err := os.ReadFile(conf.BootConfig.Worker.Path) if err != nil { logrus.Errorf("error reading boot config file: %v", err) @@ -152,12 +148,7 @@ func extendCluster(conf *asset.ClusterAsset, httpService *httpserver.HTTPService p := infra.InfraPlatform{} switch strings.ToLower(conf.Platform) { case "libvirt": - go func() { - if err := httpService.Start(); err != nil { - logrus.Errorf("error starting http service: %v", err) - return - } - }() + httpserver.StartHTTPService(httpService) // regenerate worker.tf var worker terraform.Infra @@ -178,12 +169,7 @@ func extendCluster(conf *asset.ClusterAsset, httpService *httpserver.HTTPService return err } case "openstack": - go func() { - if err := httpService.Start(); err != nil { - logrus.Errorf("error starting http service: %v", err) - return - } - }() + httpserver.StartHTTPService(httpService) // regenerate worker.tf var worker terraform.Infra @@ -205,36 +191,39 @@ func extendCluster(conf *asset.ClusterAsset, httpService *httpserver.HTTPService } case "pxe": pxeConfig := conf.InfraPlatform.(*infraasset.PXEAsset) - pxe := &infra.PXE{ - IP: pxeConfig.IP, - HTTPServerPort: pxeConfig.HTTPServerPort, - HTTPRootDir: pxeConfig.HTTPRootDir, - TFTPServerPort: pxeConfig.TFTPServerPort, - TFTPRootDir: pxeConfig.TFTPRootDir, - HTTPService: httpService, - } - p.SetInfra(pxe) - if err := p.Extend(); err != nil { - logrus.Errorf("Failed to extend worker nodes:%v", err) - return err - } + httpService.Port = pxeConfig.HTTPServerPort + httpService.DirPath = pxeConfig.HTTPRootDir + httpserver.StartHTTPService(httpService) + + tftpService := tftpserver.NewTFTPService(pxeConfig.IP, pxeConfig.TFTPServerPort, pxeConfig.TFTPRootDir) + go func() { + if err := tftpService.Start(); err != nil { + logrus.Errorf("error starting http service: %v", err) + return + } + }() + defer tftpService.Stop() + case "ipxe": ipxeConfig := conf.InfraPlatform.(*infraasset.IPXEAsset) - ipxe := &infra.IPXE{ - Port: ipxeConfig.Port, - FilePath: ipxeConfig.FilePath, - OSInstallTreePath: ipxeConfig.OSInstallTreePath, - HTTPService: httpService, - } - p.SetInfra(ipxe) - if err := p.Extend(); err != nil { - logrus.Errorf("Failed to extend worker nodes:%v", err) + httpService.Port = ipxeConfig.Port + httpService.DirPath = ipxeConfig.OSInstallTreePath + fileContent, err := os.ReadFile(ipxeConfig.FilePath) + if err != nil { return err } + httpService.AddFileToCache(constants.IPXECfg, fileContent) + httpserver.StartHTTPService(httpService) + default: return errors.New("unsupported platform") } + logrus.Infof("Waiting for cluster extend nodes to be ready...") + if err := checkNodesReady(conf, nodeNames); err != nil { + return err + } + return nil } diff --git a/pkg/httpserver/httpserver.go b/pkg/httpserver/httpserver.go index 5d0ad22..fb62b5b 100644 --- a/pkg/httpserver/httpserver.go +++ b/pkg/httpserver/httpserver.go @@ -186,3 +186,11 @@ func (hs *HTTPService) Stop() error { hs.server = nil return nil } + +func StartHTTPService(httpService *HTTPService) { + go func() { + if err := httpService.Start(); err != nil { + logrus.Errorf("error starting HTTP service: %v", err) + } + }() +} diff --git a/pkg/infra/ipxe.go b/pkg/infra/ipxe.go deleted file mode 100644 index 9dd35f0..0000000 --- a/pkg/infra/ipxe.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2024 KylinSoft Co., Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package infra - -import ( - "fmt" - "nestos-kubernetes-deployer/pkg/constants" - "nestos-kubernetes-deployer/pkg/httpserver" - "os" -) - -type IPXE struct { - Port string - FilePath string - OSInstallTreePath string - HTTPService *httpserver.HTTPService -} - -func (i *IPXE) deployHTTP(port string, dirPath string, filePath string) error { - i.HTTPService.Port = port - i.HTTPService.DirPath = dirPath - - fileContent, err := os.ReadFile(filePath) - if err != nil { - return err - } - - if err := i.HTTPService.AddFileToCache(constants.IPXECfg, fileContent); err != nil { - return err - } - - if err := i.HTTPService.Start(); err != nil { - return fmt.Errorf("error starting http service: %v", err) - } - - return nil -} - -func (i *IPXE) Deploy() error { - return i.deployHTTP(i.Port, i.OSInstallTreePath, i.FilePath) -} - -func (i *IPXE) Extend() error { - return i.Deploy() -} - -func (i *IPXE) Destroy() error { - return nil -} diff --git a/pkg/infra/pxe.go b/pkg/infra/pxe.go deleted file mode 100644 index 3fbd403..0000000 --- a/pkg/infra/pxe.go +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright 2024 KylinSoft Co., Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package infra - -import ( - "nestos-kubernetes-deployer/pkg/httpserver" - "nestos-kubernetes-deployer/pkg/tftpserver" - - "github.com/sirupsen/logrus" -) - -type PXE struct { - IP string - HTTPServerPort string - HTTPRootDir string - TFTPServerPort string - TFTPRootDir string - HTTPService *httpserver.HTTPService -} - -func (p *PXE) deployHTTP(port string, dirPath string) error { - p.HTTPService.Port = port - p.HTTPService.DirPath = dirPath - if err := p.HTTPService.Start(); err != nil { - return err - } - - return nil -} - -func (p *PXE) deployTFTP(ip string, port string, rootDir string) error { - tftpService := &tftpserver.TFTPService{ - IP: ip, - Port: port, - RootDir: rootDir, - } - go func() { - select { - case <-p.HTTPService.Ch: - logrus.Info("tftp server stop") - tftpService.Stop() - return - } - }() - - if err := tftpService.Start(); err != nil { - return err - } - defer tftpService.Stop() - - return nil -} - -func (p *PXE) Deploy() error { - go func() { - err := p.deployHTTP(p.HTTPServerPort, p.HTTPRootDir) - if err != nil { - logrus.Errorf("PXE deploy http server err: %v", err) - return - } - }() - - if err := p.deployTFTP(p.IP, p.TFTPServerPort, p.TFTPRootDir); err != nil { - return err - } - - return nil -} - -func (p *PXE) Extend() error { - return p.Deploy() -} - -func (p *PXE) Destroy() error { - return nil -} diff --git a/pkg/tftpserver/tftpserver.go b/pkg/tftpserver/tftpserver.go index 6037a14..15c9bde 100644 --- a/pkg/tftpserver/tftpserver.go +++ b/pkg/tftpserver/tftpserver.go @@ -32,6 +32,14 @@ type TFTPService struct { server *tftp.Server } +func NewTFTPService(ip string, port string, rootDir string) *TFTPService { + return &TFTPService{ + IP: ip, + Port: port, + RootDir: rootDir, + } +} + func (t *TFTPService) Start() error { tftpHandler := TFTPHandler{ RootDir: t.RootDir, -- Gitee