diff --git a/app/apis/nkd/const.go b/app/apis/nkd/const.go index 35274993c60dc9b2e5afc76a1b60c879e9c4edd7..29b65a0c67a035826ad85c0e2c258d74c016ec9a 100644 --- a/app/apis/nkd/const.go +++ b/app/apis/nkd/const.go @@ -22,15 +22,16 @@ var ( NkdClusterName = "example nkd cluster" // system - HostName = "master" + MasterHostName = "master" + WorkerHostName = "worker" // Hostname2 = "master01" // Hostname3 = "master01" - Username = "user1" + Username = "root" Password = "********" // repo Secret = []map[string]string{{"repousre": "********"}} - Registry = "kubeadm.k8s.io/v1beta3" + Registry = "registry.cn-hangzhou.aliyuncs.com/google_containers" // infra Platform = "openstack" @@ -50,7 +51,7 @@ var ( Openstack_WorkerNodeName = []string{"node01", "node02", "node03"} Openstack_Internal_network = "existing internal net name" Openstack_External_network = "existing external net name" - Openstack_Master_ip = []string{"*.*.*.*", "*.*.*.*", "*.*.*.*"} + Openstack_Master_ip = []string{"10.1.10.51", "10.1.10.52", "10.1.10.53"} Openstack_Worker_ip = []string{"*.*.*.*", "*.*.*.*", "*.*.*.*"} Openstack_Flavor_Name = "existing flavor name" Openstack_Glance_Name = "existing glance name" @@ -71,7 +72,7 @@ var ( BindPort int32 = 6643 // nodeRegistration - CriSocket = "/var/run/dockershim.sock" + CriSocket = "/var/run/isulad.sock" ImagePullPolicy = "IfNotPresent" Name = "node" Taints = []Taint{} @@ -85,13 +86,14 @@ var ( // etcd LocalDir = "/var/lib/etcd" - ImageRepository = "registry.k8s.io" - KubernetesVersion = "1.23.0" + ImageRepository = "registry.cn-hangzhou.aliyuncs.com/google_containers" + KubernetesVersion = "1.23.10" DnsDomain = "cluster.local" - ServiceSubnet = "10.96.0.0/12" + ServiceSubnet = "10.96.0.0/16" + PodSubnet = "10.100.0.0/16" // worker - APIServerEndpoint = "kube-apiserver:6443" + APIServerEndpoint = "10.1.10.51:6443" Token = "abcdef.0123456789abcdef" UnsafeSkipCAVerification = true WorkerDiscoverTimeout = "5m0s" @@ -103,7 +105,10 @@ var ( Master_Count = 3 Worker_Count = 3 + SSHKey = "ssh-rsa AAAAB3N... root@localhost.localdomain" // containerdaemon - Pauseversion = "pause:3.6" + PauseImageTag = "3.6" + CorednsImageTag = "v1.8.6" + ReleaseImageURl = "" ) diff --git a/app/apis/nkd/types.go b/app/apis/nkd/types.go index ed5782c6ca692d35871b056ed31e4c289863a8bf..e86522c6e62e8e29f9c73a023d707c61a66a2a40 100644 --- a/app/apis/nkd/types.go +++ b/app/apis/nkd/types.go @@ -21,12 +21,13 @@ import ( ) type System struct { - Count int - HostName string - Ips []string - Username string `yaml:"username"` - Password string `yaml:"password"` - SSHKey string + Count int + MasterHostName string + WorkerHostName string + Ips []string + Username string `yaml:"username"` + Password string `yaml:"password"` + SSHKey string } type Openstack struct { @@ -340,7 +341,9 @@ type Addon struct { } type ContainerDaemon struct { - Pause string + PauseImageTag string + CorednsImageTag string + ReleaseImageURl string } type Master struct { diff --git a/app/cmd/init.go b/app/cmd/init.go index 1dbfb7f5de9c21622e9e4423301d444ee4fab33e..3b357f0812242576958cd715988edc303fbddd85 100755 --- a/app/cmd/init.go +++ b/app/cmd/init.go @@ -58,7 +58,7 @@ func NewInitDefaultNkdConfigCommand() *cobra.Command { }, } phases.NewGenerateCertsCmd() - initRunner.AppendPhase(phases.NewGenerateCertsCmd()) + // initRunner.AppendPhase(phases.NewGenerateCertsCmd()) initRunner.AppendPhase(phases.NewGenerateIgnCmd()) initRunner.AppendPhase(phases.NewGenerateTFCmd()) cmd.PersistentFlags().StringVarP(&config, "config", "c", "", "config for init") diff --git a/app/cmd/phases/config/print.go b/app/cmd/phases/config/print.go index df4c54535eda93ce492dfd7b437d4b53fb6e41f5..4fa810c0253b7cb64bea1260cfe79d3a9e43674b 100644 --- a/app/cmd/phases/config/print.go +++ b/app/cmd/phases/config/print.go @@ -97,11 +97,13 @@ func DefaultedStaticWorkerConfiguration(internalconfig *nkd.Worker) *nkd.Worker } system1 := nkd.System{ - Count: nkd.Master_Count, - Ips: nkd.Openstack_Master_ip, - HostName: nkd.HostName, - Username: nkd.Username, - Password: nkd.Password, + Count: nkd.Master_Count, + Ips: nkd.Openstack_Master_ip, + WorkerHostName: nkd.WorkerHostName, + MasterHostName: nkd.MasterHostName, + Username: nkd.Username, + Password: nkd.Password, + SSHKey: nkd.SSHKey, } vmsize := nkd.Size{ @@ -142,7 +144,9 @@ func DefaultedStaticWorkerConfiguration(internalconfig *nkd.Worker) *nkd.Worker } containerdaemon := nkd.ContainerDaemon{ - Pause: nkd.Pauseversion, + PauseImageTag: nkd.PauseImageTag, + CorednsImageTag: nkd.CorednsImageTag, + ReleaseImageURl: nkd.ReleaseImageURl, } internalconfig.Node = nkd.WorkerNode @@ -159,11 +163,12 @@ func DefaultedStaticMasterConfiguration(internalconfig *nkd.Master) *nkd.Master cluster := nkd.Cluster{Name: nkd.NkdClusterName} system1 := nkd.System{ - Count: nkd.Master_Count, - Ips: nkd.Openstack_Master_ip, - HostName: nkd.HostName, - Username: nkd.Username, - Password: nkd.Password, + Count: nkd.Master_Count, + Ips: nkd.Openstack_Master_ip, + MasterHostName: nkd.MasterHostName, + Username: nkd.Username, + Password: nkd.Password, + SSHKey: nkd.SSHKey, } repo := nkd.Repo{ @@ -224,7 +229,7 @@ func DefaultedStaticMasterConfiguration(internalconfig *nkd.Master) *nkd.Master Etcd: nkd.Etcd{Local: &nkd.LocalEtcd{DataDir: nkd.LocalDir}}, ImageRepository: nkd.ImagePullPolicy, KubernetesVersion: nkd.KubernetesVersion, - Networking: nkd.Networking{DNSDomain: nkd.DnsDomain, ServiceSubnet: nkd.ServiceSubnet}, + Networking: nkd.Networking{DNSDomain: nkd.DnsDomain, ServiceSubnet: nkd.ServiceSubnet, PodSubnet: nkd.PodSubnet}, APIServer: apiServer, } @@ -236,7 +241,9 @@ func DefaultedStaticMasterConfiguration(internalconfig *nkd.Master) *nkd.Master } containerdaemon := nkd.ContainerDaemon{ - Pause: nkd.Pauseversion, + PauseImageTag: nkd.PauseImageTag, + CorednsImageTag: nkd.CorednsImageTag, + ReleaseImageURl: nkd.ReleaseImageURl, } internalconfig.Node = nkd.MasterNode diff --git a/app/cmd/phases/init/ign.go b/app/cmd/phases/init/ign.go index 8a7b3f09be7d010818e9927296e46564f7c22d6e..2fc9ec9e5163b05c41e6dfbc245e767614e9567e 100644 --- a/app/cmd/phases/init/ign.go +++ b/app/cmd/phases/init/ign.go @@ -21,6 +21,7 @@ import ( "io" "nestos-kubernetes-deployer/app/apis/nkd" "nestos-kubernetes-deployer/app/cmd/phases/workflow" + "net" "os" "path" "path/filepath" @@ -47,13 +48,16 @@ type commonTemplateData struct { APIServerURL string Hsip string //HostName + IP ImageRegistry string - PodSandboxImage string + PauseImageTag string KubeVersion string ServiceSubnet string PodSubnet string Token string NodeType string NodeName string + CorednsImageTag string + IpSegment string + ReleaseImageURl string } var ( @@ -80,22 +84,36 @@ func runGenerateIgnConfig(r workflow.RunData, node string) error { ) if node == "master" { nodeCount = data.MasterCfg().System.Count - hostName = data.MasterCfg().System.HostName + hostName = data.MasterCfg().System.MasterHostName + ip := net.ParseIP(data.MasterCfg().System.Ips[0]) + if ip == nil { + logrus.Warning("Invalid ip address!") + return nil + } + ipSegment := ip.To4() + ipSegment[2] = 0 + ipSegment[3] = 0 for i := 0; i < nodeCount; i++ { oneNodeName = fmt.Sprintf("%s%02d", hostName, i+1) - temp := data.MasterCfg().System.Ips[i] + oneNodeName + "\n" + temp := data.MasterCfg().System.Ips[i] + " " + oneNodeName + "\n" hsip = hsip + temp } for j := 0; j < nodeCount; j++ { - ctd := getMasterTmplData(data.MasterCfg(), j+1) + ctd := getMasterTmplData(data.MasterCfg(), j+1, ipSegment.String(), hsip) if err := generateConfig(ctd); err != nil { return err } } } else { nodeCount = data.WorkerCfg().System.Count + hostName = data.WorkerCfg().System.MasterHostName + for i := 0; i < len(data.WorkerCfg().System.Ips); i++ { + oneNodeName = fmt.Sprintf("%s%02d", hostName, i+1) + temp := data.WorkerCfg().System.Ips[i] + " " + oneNodeName + "\n" + hsip = hsip + temp + } for j := 0; j < nodeCount; j++ { - ctd := getWorkerTmplData(data.WorkerCfg(), j+1) + ctd := getWorkerTmplData(data.WorkerCfg(), j+1, hsip) if err := generateConfig(ctd); err != nil { return err } @@ -104,31 +122,37 @@ func runGenerateIgnConfig(r workflow.RunData, node string) error { return nil } -func getMasterTmplData(nkdConfig *nkd.Master, count int) *commonTemplateData { - oneNodeName := fmt.Sprintf("%s%d", nkdConfig.System.HostName, count) +func getMasterTmplData(nkdConfig *nkd.Master, count int, ip string, hsip string) *commonTemplateData { + oneNodeName := fmt.Sprintf("%s%d", nkdConfig.System.MasterHostName, count) return &commonTemplateData{ SSHKey: nkdConfig.System.SSHKey, - APIServerURL: "", + APIServerURL: nkdConfig.System.Ips[0], + Hsip: hsip, ImageRegistry: nkdConfig.Repo.Registry, - PodSandboxImage: "", + PauseImageTag: nkdConfig.ContainerDaemon.PauseImageTag, ServiceSubnet: nkdConfig.Kubeadm.Networking.ServiceSubnet, PodSubnet: nkdConfig.Kubeadm.Networking.PodSubnet, - Token: "", + Token: "abcdef.0123456789abcdef", NodeName: oneNodeName, NodeType: "master", + CorednsImageTag: nkdConfig.ContainerDaemon.CorednsImageTag, + IpSegment: ip, + ReleaseImageURl: nkdConfig.ContainerDaemon.ReleaseImageURl, } } -func getWorkerTmplData(nkdConfig *nkd.Worker, count int) *commonTemplateData { - oneNodeName := fmt.Sprintf("%s%d", nkdConfig.System.HostName, count) +func getWorkerTmplData(nkdConfig *nkd.Worker, count int, hsip string) *commonTemplateData { + oneNodeName := fmt.Sprintf("%s%d", nkdConfig.System.WorkerHostName, count) return &commonTemplateData{ SSHKey: nkdConfig.System.SSHKey, - APIServerURL: "", + APIServerURL: nkdConfig.Worker.Discovery.BootstrapToken.APIServerEndpoint, + Hsip: hsip, ImageRegistry: nkdConfig.Repo.Registry, - PodSandboxImage: "", + PauseImageTag: nkdConfig.ContainerDaemon.PauseImageTag, Token: nkdConfig.Worker.Discovery.TlsBootstrapToken, NodeName: oneNodeName, NodeType: "worker", + ReleaseImageURl: nkdConfig.ContainerDaemon.ReleaseImageURl, } } @@ -177,7 +201,11 @@ func generateConfig(ctd *commonTemplateData) error { return err } ignName := fmt.Sprintf("%s%s", ctd.NodeName, ".ign") - if err := generateFile(&config, "./", ignName); err != nil { + filepath := "./master" + if ctd.NodeType == "worker" { + filepath = "./worker" + } + if err := generateFile(&config, filepath, ignName); err != nil { logrus.Errorf("failed to generate ignition file: %v", err) return err } diff --git a/data/ignition/master/files/etc/init-config.yaml.template b/data/ignition/master/files/etc/init-config.yaml.template index 8831ddca0b30e349b6f621b2596a293a93949d3e..089228ceafc1e7b35ba2ea90322cc4650d432402 100644 --- a/data/ignition/master/files/etc/init-config.yaml.template +++ b/data/ignition/master/files/etc/init-config.yaml.template @@ -28,3 +28,5 @@ networking: dnsDomain: "cluster.local" dns: type: CoreDNS + imageRepository: {{.ImageRegistry}}/coredns + imageTag: {{.CorednsImageTag}} \ No newline at end of file diff --git a/data/ignition/master/files/etc/isulad/daemon.json.template b/data/ignition/master/files/etc/isulad/daemon.json.template index 4994ed3a137fba9bcea5eb4a30f78aec42fc5554..e5540c103dcd5885120d550c9481bdd031261827 100644 --- a/data/ignition/master/files/etc/isulad/daemon.json.template +++ b/data/ignition/master/files/etc/isulad/daemon.json.template @@ -29,7 +29,7 @@ "insecure-registries": [ "{{.ImageRegistry}}" ], - "pod-sandbox-image": "{{.PodSandboxImage}}", + "pod-sandbox-image": "{{.ImageRegistry}}/pause:{{.PauseImageTag}}", "native.umask": "secure", "network-plugin": "cni", "cni-bin-dir": "/opt/cni/bin", diff --git a/data/ignition/master/files/etc/nkd/node.pivot.sh.template b/data/ignition/master/files/etc/nkd/node.pivot.sh.template new file mode 100644 index 0000000000000000000000000000000000000000..86a09bd874bb4d99aa9993332b816ae263b301d0 --- /dev/null +++ b/data/ignition/master/files/etc/nkd/node.pivot.sh.template @@ -0,0 +1,16 @@ +#!/bin/sh +service_name="isulad" +if systemctl is-active --quiet $service_name ; then + echo "$service_name is already running" +else + echo "$service_name is not running, starting..." + systemctl start $service_name + systemctl enable $service_name + if [ $? -eq 0 ]; then + echo "$service_name starting success." + else + echo "unable to start $service_name." + exit 1 + fi +fi +rpm-ostree rebase --experimental ostree-unverified-image:docker://{{.ReleaseImageURl}} --bypass-driver diff --git a/data/ignition/master/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf b/data/ignition/master/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf index f6daaef95b1b42baaddfe1e7fcd950a2e79e661a..19475e59ac9ff8ae31d62589e6060846b2b744d6 100644 --- a/data/ignition/master/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf +++ b/data/ignition/master/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf @@ -8,4 +8,4 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env # the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= -ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS diff --git a/data/ignition/master/systemd/disable-selinux.service b/data/ignition/master/systemd/disable-selinux.service index d29557b7fd58d26df2bece1f45df51b3f4927a05..ed2657b29d202f3119f9a443c35f6a5b3f0668e7 100644 --- a/data/ignition/master/systemd/disable-selinux.service +++ b/data/ignition/master/systemd/disable-selinux.service @@ -1,6 +1,7 @@ [Unit] Description=disable selinux for kubernetes ConditionPathExists=!/var/nkd/log/disable-selinux.stamp +ConditionPathExists=/var/nkd/log/node-pivot.stamp [Service] Type=oneshot diff --git a/data/ignition/master/systemd/init-cluster.service b/data/ignition/master/systemd/init-cluster.service index a31c52b544b4aa939e630221610c0fd01ee155ef..417b9e82c8c2ca1e31ad7fa864d4f33a19f20451 100644 --- a/data/ignition/master/systemd/init-cluster.service +++ b/data/ignition/master/systemd/init-cluster.service @@ -1,9 +1,10 @@ [Unit] Description=init kubernetes cluster -Requires=set-kernel-para.service disable-selinux.service -After=set-kernel-para.service disable-selinux.service +Requires=set-kernel-para.service disable-selinux.service release-image-pivot.service +After=set-kernel-para.service disable-selinux.service release-image-pivot.service ConditionPathExists=/var/nkd/log/set-kernel-para.stamp ConditionPathExists=/var/nkd/log/disable-selinux.stamp +ConditionPathExists=/var/nkd/log/node-pivot.stamp ConditionPathExists=!/var/nkd/log/init-cluster.stamp [Service] diff --git a/data/ignition/master/systemd/install-cni-plugin.service b/data/ignition/master/systemd/install-cni-plugin.service.template similarity index 39% rename from data/ignition/master/systemd/install-cni-plugin.service rename to data/ignition/master/systemd/install-cni-plugin.service.template index d779a770da22ef1f715ac676cd0be48b743d8d06..a6126158c649fee4c804b766d18d73bb747053ce 100644 --- a/data/ignition/master/systemd/install-cni-plugin.service +++ b/data/ignition/master/systemd/install-cni-plugin.service.template @@ -2,11 +2,15 @@ Description=install cni network plugin for kubernetes Requires=init-cluster.service After=init-cluster.service +ConditionPathExists=/var/nkd/log/init-k8s-cluster.stamp [Service] Type=oneshot RemainAfterExit=yes +ExecStart=bash -c "sed -i 's#usr/libexec/#opt/libexec/#g' /etc/nkd/calico.yaml" +ExecStart=bash -c "sed -i 's/# - name: CALICO_IPV4POOL_CIDR/- name: CALICO_IPV4POOL_CIDR/g' /etc/nkd/calico.yaml" +ExecStart=bash -c "sed -i 's?# value: "{{.IpSegment}}/16"? value: "10.100.0.0/16"?g' /etc/nkd/calico.yaml" ExecStart=kubectl apply -f /etc/calico.yaml --kubeconfig=/etc/kubernetes/admin.conf [Install] -WantedBy=multi-user.target +WantedBy=multi-user.target \ No newline at end of file diff --git a/data/ignition/master/systemd/kubelet.service b/data/ignition/master/systemd/kubelet.service index 3adb4a3c282348d0182c9134e02f0e0b1bcf4a8a..a041dd6bebf63e51a4f3b524a3c775c165896237 100644 --- a/data/ignition/master/systemd/kubelet.service +++ b/data/ignition/master/systemd/kubelet.service @@ -3,9 +3,10 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/ Wants=network-online.target After=network-online.target +ConditionPathExists=/var/nkd/log/node-pivot.stamp [Service] -ExecStart=/usr/local/bin/kubelet +ExecStart=/usr/bin/kubelet Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/data/ignition/master/systemd/release-image-pivot.service b/data/ignition/master/systemd/release-image-pivot.service new file mode 100644 index 0000000000000000000000000000000000000000..aa8ba3590273caa557bca307214c7dee46b47564 --- /dev/null +++ b/data/ignition/master/systemd/release-image-pivot.service @@ -0,0 +1,15 @@ +[Unit] +Description=Pivot node to the nkd release image +Wants=network-online.target +After=network-online.target +ConditionPathExists=!/var/nkd/log/node-pivot.stamp + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=sh /etc/nkd/node-pivot.sh +ExecStart=/bin/touch /var/nkd/log/node-pivot.stamp +ExecStart=systemctl reboot + +[Install] +WantedBy=multi-user.target diff --git a/data/ignition/master/systemd/set-kernel-para.service b/data/ignition/master/systemd/set-kernel-para.service index d63023a9d7436b40bb5d24a9d5beb0a75408259f..1a9fcd776417a89572ad3097f5d372074753b90e 100644 --- a/data/ignition/master/systemd/set-kernel-para.service +++ b/data/ignition/master/systemd/set-kernel-para.service @@ -1,6 +1,7 @@ [Unit] Description=set kernel para for Kubernetes ConditionPathExists=!/var/nkd/log/set-kernel-para.stamp +ConditionPathExists=/var/nkd/log/node-pivot.stamp [Service] Type=oneshot diff --git a/data/ignition/worker/files/etc/isulad/daemon.json.template b/data/ignition/worker/files/etc/isulad/daemon.json.template index 4994ed3a137fba9bcea5eb4a30f78aec42fc5554..e5540c103dcd5885120d550c9481bdd031261827 100644 --- a/data/ignition/worker/files/etc/isulad/daemon.json.template +++ b/data/ignition/worker/files/etc/isulad/daemon.json.template @@ -29,7 +29,7 @@ "insecure-registries": [ "{{.ImageRegistry}}" ], - "pod-sandbox-image": "{{.PodSandboxImage}}", + "pod-sandbox-image": "{{.ImageRegistry}}/pause:{{.PauseImageTag}}", "native.umask": "secure", "network-plugin": "cni", "cni-bin-dir": "/opt/cni/bin", diff --git a/data/ignition/worker/files/etc/join-config.yaml.template b/data/ignition/worker/files/etc/join-config.yaml.template index 12f98f2716c6743a37a9b57dc41eac01a9913969..6ecac36ecc7e136976704dc82abf60dacf08e5f3 100644 --- a/data/ignition/worker/files/etc/join-config.yaml.template +++ b/data/ignition/worker/files/etc/join-config.yaml.template @@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1beta2 caCertPath: /etc/kubernetes/pki/ca.crt discovery: bootstrapToken: - apiServerEndpoint: {{.APIServerURL}}:6443 + apiServerEndpoint: {{.APIServerURL}} token: {{.Token}} unsafeSkipCAVerification: true timeout: 5m0s diff --git a/data/ignition/worker/files/etc/nkd/node.pivot.sh.template b/data/ignition/worker/files/etc/nkd/node.pivot.sh.template new file mode 100644 index 0000000000000000000000000000000000000000..86a09bd874bb4d99aa9993332b816ae263b301d0 --- /dev/null +++ b/data/ignition/worker/files/etc/nkd/node.pivot.sh.template @@ -0,0 +1,16 @@ +#!/bin/sh +service_name="isulad" +if systemctl is-active --quiet $service_name ; then + echo "$service_name is already running" +else + echo "$service_name is not running, starting..." + systemctl start $service_name + systemctl enable $service_name + if [ $? -eq 0 ]; then + echo "$service_name starting success." + else + echo "unable to start $service_name." + exit 1 + fi +fi +rpm-ostree rebase --experimental ostree-unverified-image:docker://{{.ReleaseImageURl}} --bypass-driver diff --git a/data/ignition/worker/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf b/data/ignition/worker/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf index f6daaef95b1b42baaddfe1e7fcd950a2e79e661a..19475e59ac9ff8ae31d62589e6060846b2b744d6 100644 --- a/data/ignition/worker/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf +++ b/data/ignition/worker/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf @@ -8,4 +8,4 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env # the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= -ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS diff --git a/data/ignition/worker/systemd/disable-selinux.service b/data/ignition/worker/systemd/disable-selinux.service index d29557b7fd58d26df2bece1f45df51b3f4927a05..ed2657b29d202f3119f9a443c35f6a5b3f0668e7 100644 --- a/data/ignition/worker/systemd/disable-selinux.service +++ b/data/ignition/worker/systemd/disable-selinux.service @@ -1,6 +1,7 @@ [Unit] Description=disable selinux for kubernetes ConditionPathExists=!/var/nkd/log/disable-selinux.stamp +ConditionPathExists=/var/nkd/log/node-pivot.stamp [Service] Type=oneshot diff --git a/data/ignition/worker/systemd/join-cluster.service b/data/ignition/worker/systemd/join-cluster.service index e4a6ad72b3b7762f18c940885f82c4d7266b8854..a8b598c25d6c69d1367f678c9415b731227fce2b 100644 --- a/data/ignition/worker/systemd/join-cluster.service +++ b/data/ignition/worker/systemd/join-cluster.service @@ -1,15 +1,16 @@ [Unit] Description=init kubernetes cluster -Requires=set-kernel-para.service disable-selinux.service -After=set-kernel-para.service disable-selinux.service +Requires=set-kernel-para.service disable-selinux.service release-image-pivot.service +After=set-kernel-para.service disable-selinux.service release-image-pivot.service ConditionPathExists=/var/nkd/log/set-kernel-para.stamp ConditionPathExists=/var/nkd/log/disable-selinux.stamp +ConditionPathExists=/var/nkd/log/node-pivot.stamp ConditionPathExists=!/var/nkd/log/join-cluster.stamp [Service] Type=oneshot RemainAfterExit=yes -ExecStart=kubeadm join {{.APIServerURL}}:6443 --token {{.Token}} --discovery-token-ca-cert-hash {{.TokenCACertHash}} --cri-socket=/var/run/isulad.sock +ExecStart=kubeadm join --config=/etc/nkd/join-config.yaml ExecStart=/bin/touch /var/nkd/log/join-cluster.stamp [Install] diff --git a/data/ignition/worker/systemd/kubelet.service b/data/ignition/worker/systemd/kubelet.service index 3adb4a3c282348d0182c9134e02f0e0b1bcf4a8a..a041dd6bebf63e51a4f3b524a3c775c165896237 100644 --- a/data/ignition/worker/systemd/kubelet.service +++ b/data/ignition/worker/systemd/kubelet.service @@ -3,9 +3,10 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/ Wants=network-online.target After=network-online.target +ConditionPathExists=/var/nkd/log/node-pivot.stamp [Service] -ExecStart=/usr/local/bin/kubelet +ExecStart=/usr/bin/kubelet Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/data/ignition/worker/systemd/release-image-pivot.service b/data/ignition/worker/systemd/release-image-pivot.service new file mode 100644 index 0000000000000000000000000000000000000000..aa8ba3590273caa557bca307214c7dee46b47564 --- /dev/null +++ b/data/ignition/worker/systemd/release-image-pivot.service @@ -0,0 +1,15 @@ +[Unit] +Description=Pivot node to the nkd release image +Wants=network-online.target +After=network-online.target +ConditionPathExists=!/var/nkd/log/node-pivot.stamp + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=sh /etc/nkd/node-pivot.sh +ExecStart=/bin/touch /var/nkd/log/node-pivot.stamp +ExecStart=systemctl reboot + +[Install] +WantedBy=multi-user.target diff --git a/data/ignition/worker/systemd/set-kernel-para.service b/data/ignition/worker/systemd/set-kernel-para.service index d63023a9d7436b40bb5d24a9d5beb0a75408259f..1a9fcd776417a89572ad3097f5d372074753b90e 100644 --- a/data/ignition/worker/systemd/set-kernel-para.service +++ b/data/ignition/worker/systemd/set-kernel-para.service @@ -1,6 +1,7 @@ [Unit] Description=set kernel para for Kubernetes ConditionPathExists=!/var/nkd/log/set-kernel-para.stamp +ConditionPathExists=/var/nkd/log/node-pivot.stamp [Service] Type=oneshot diff --git a/resource/templates/terraform/master.tf.tpl b/resource/templates/terraform/master.tf.tpl index 96169f03f77087f39de716d1452325d365d5b276..b237ca04bd1d78d5059d72d757e424679638d851 100644 --- a/resource/templates/terraform/master.tf.tpl +++ b/resource/templates/terraform/master.tf.tpl @@ -20,7 +20,7 @@ variable "instance_count" { } variable "instance_name" { - default = "{{.System.HostName}}" + default = "{{.System.MasterHostName}}" } resource "openstack_compute_flavor_v2" "flavor" { diff --git a/resource/templates/terraform/worker.tf.tpl b/resource/templates/terraform/worker.tf.tpl index 9a052e95f1c3a225d49d4e1a9a0bd4d8a6a1b244..80718e9c1d831d51baede42dedc80169c701e3da 100644 --- a/resource/templates/terraform/worker.tf.tpl +++ b/resource/templates/terraform/worker.tf.tpl @@ -20,7 +20,7 @@ variable "instance_count" { } variable "instance_name" { - default = "{{.System.HostName}}" + default = "{{.System.WorkerHostName}}" } resource "openstack_compute_flavor_v2" "flavor" {