From b0cc31da584a70b57d0247c790cd6ab7a7bed13d Mon Sep 17 00:00:00 2001 From: Haibin Lee Date: Tue, 31 Oct 2023 14:15:39 +0800 Subject: [PATCH 1/4] WIP: trying for fix problems when add CM to an existing cluster --- README.md | 14 +++- ansible.cfg | 6 +- inventories/opengauss/hosts.ini | 11 +-- roles/opengauss/defaults/main.yml | 9 +++ roles/opengauss/handlers/main.yml | 3 +- .../opengauss/tasks/deploy/cluster_check.yml | 13 +++- .../tasks/deploy/cluster_manager.yml | 68 +++++++++++++++++++ roles/opengauss/tasks/deploy/expansion.yml | 50 +++++++------- roles/opengauss/tasks/deploy/install.yml | 35 ++++++---- roles/opengauss/tasks/os/common_set.yml | 12 ++++ roles/opengauss/tasks/os/user.yml | 9 ++- roles/opengauss/tasks/post_tasks.yml | 18 ++--- roles/opengauss/tasks/runtime_facts.yml | 1 + .../opengauss/templates/cluster_master.xml.j2 | 2 +- .../templates/cluster_replicas.xml.j2 | 10 ++- 15 files changed, 191 insertions(+), 70 deletions(-) create mode 100644 roles/opengauss/tasks/deploy/cluster_manager.yml diff --git a/README.md b/README.md index 505c674..566731d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Ansible role for openGauss -自动部署 openGauss，根据分组定义，自适应部署单点、一主一从、一主多从以及级联节点等架构模式。 +自动部署 openGauss，根据分组定义，自适应部署单点、一主一备、一主多备以及级联节点等架构模式。 # 已适配的系统 @@ -15,7 +15,7 @@ # 优势特点 -1. 1 主 4 从 1 级联的架构，十分钟内部署完成（不含安装包的下载时间）。 +1. 1 主 4 备 1 级联的架构，十分钟内部署完成（不含安装包的下载时间）。 1. 自动匹配 CPU 架构以及操作系统。 1. 支持自定义 cluster_config.xml.j2 模板，优先使用 `{{ inventory_dir }}/templates/openGauss/cluster_config.xml.j2`。 1. 默认自动生成数据库管理员密码，也可自定义变量进行替换。全程自动化，无交互步骤。 @@ -59,6 +59,16 @@ * 2023-10-13: 支持集群扩容。 * 2023-10-18: 支持 openEuler 20.03 LTS SP3。 +* 2023-10-26: 大量修复和优化，可完美实现从单点部署，到逐步扩容为 1 主 4 备 4 级联的架构。 + ``` + 已测试的扩容场景如下 + + 1 主 + -> 1 主 1 备 + -> 1 主 1 备 1 级联 + -> 1 主 2 备 2 级联 + -> 1 主 4 备 4 级联 + ``` # 开发指南 diff --git a/ansible.cfg b/ansible.cfg index b38abad..ef8ea22 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -9,9 +9,9 @@ pipelining = True become = True host_key_checking = False deprecation_warnings = False -callback_whitelist = profile_tasks,log_plays -display_skipped_hosts = False -#stdout_callback = yaml +callback_whitelist = profile_tasks,timer,profile_roles +; display_skipped_hosts = False +; stdout_callback = yaml [callback_log_plays] log_folder = ./log/ \ No newline at end of file diff --git a/inventories/opengauss/hosts.ini b/inventories/opengauss/hosts.ini index 12aa6e7..1168a45 100644 --- a/inventories/opengauss/hosts.ini +++ b/inventories/opengauss/hosts.ini @@ -1,20 +1,15 @@ ; 主机，仅设置 1 个目标机。 [opengauss_master] -192.168.56.17 +192.168.56.18 ; 备机，可设置若干个或留空。不可大于 8。 [opengauss_follower] -192.168.56.11 -192.168.56.15 -192.168.56.12 192.168.56.13 +192.168.56.15 ; 级联机，可设置若干个或留空。 [opengauss_cascade] -192.168.56.19 -192.168.56.18 -192.168.56.16 -192.168.56.14 +192.168.56.11 ; 以上 3 个分组的合并组，勿动。 [opengauss:children] diff --git a/roles/opengauss/defaults/main.yml b/roles/opengauss/defaults/main.yml index b71fc4f..e9223b4 100644 --- a/roles/opengauss/defaults/main.yml +++ b/roles/opengauss/defaults/main.yml @@ -33,6 +33,15 @@ opengauss_env: length=8 ) | password_hash }} + ca_file_pass: >- + {{ + lookup( + 'password', + inventory_dir + '/credentials/opengauss_ca_file_pass', + chars=['ascii_letters', 'digits', 'punctuation'], + length=8 + ) + }} iface_mtu: 8192 clusterName: openGauss dataPortBase: 15400 diff --git a/roles/opengauss/handlers/main.yml b/roles/opengauss/handlers/main.yml index 798c84c..3005875 100644 --- a/roles/opengauss/handlers/main.yml +++ b/roles/opengauss/handlers/main.yml @@ -4,4 +4,5 @@ ansible.builtin.service: name: systemd-logind state: restarted - enabled: true \ No newline at end of file + enabled: true + daemon_reload: true diff --git a/roles/opengauss/tasks/deploy/cluster_check.yml b/roles/opengauss/tasks/deploy/cluster_check.yml index 3bdad59..406fa5d 100644 --- a/roles/opengauss/tasks/deploy/cluster_check.yml +++ b/roles/opengauss/tasks/deploy/cluster_check.yml @@ -46,5 +46,16 @@ vars: og_expansion: true delegate_to: "{{ og_master }}" - when: "groups['opengauss_expand'] is defined" run_once: true + when: "groups['opengauss_expand'] is defined" + +- name: Add CM cluster + when: + - "cluster_detail is succeeded" + - "'cmserver' not in cluster_detail.stdout" + - "og_cm_enabled" + ansible.builtin.import_tasks: + file: cluster_manager.yml + delegate_to: "{{ og_master }}" + run_once: true + become_user: "{{ og_user }}" \ No newline at end of file diff --git a/roles/opengauss/tasks/deploy/cluster_manager.yml b/roles/opengauss/tasks/deploy/cluster_manager.yml new file mode 100644 index 0000000..f640f52 --- /dev/null +++ b/roles/opengauss/tasks/deploy/cluster_manager.yml @@ -0,0 +1,68 @@ +# - name: Upload CM install package +# ansible.builtin.unarchive: +# src: "{{ playbook_dir }}/downloaded_files/{{ og_pkg_name }}" +# dest: "/tmp/" +# include: "*-cm.tar.gz" +# owner: "{{ og_user }}" +# mode: "0644" + +- name: Stop cluster + ansible.builtin.command: + gs_om -t stop + changed_when: false + +- name: Create initial cmserver list + loop: "{{ og_all_nodes }}" + loop_control: + loop_var: node + extended: true + ansible.builtin.add_host: + hostname: "{{ ansible_loop.index }}_{{ node }}" + groups: + - opengauss_cm + node_ip: "{{ node }}" + +- name: "Change ownership to {{ og_upload_path }}" + ansible.builtin.file: + path: "{{ og_upload_path }}" + owner: "{{ og_user }}" + recurse: true + become_user: root + +- name: "Remove /opt/openGauss/install/om/openGauss-5.0.0-openEuler-64bit-cm.tar.gz" + ansible.builtin.file: + path: "/opt/openGauss/install/om/openGauss-5.0.0-openEuler-64bit-cm.tar.gz" + state: absent + become_user: root + loop: "{{ og_all_nodes }}" + delegate_to: "{{ item }}" + +- name: "Create cluster_config.xml under {{ og_upload_path }}" + ansible.builtin.template: + src: "{{ item }}" + dest: "{{ og_upload_path }}/cluster_config.xml" + owner: "{{ og_user }}" + group: "{{ og_group }}" + mode: "0644" + backup: true + lstrip_blocks: true + with_first_found: + - "{{ inventory_dir }}/templates/cluster_config.xml.j2" + - "cluster_config.xml.j2" + +- name: Install CM + ansible.builtin.expect: + command: "./cm_install -X {{ og_upload_path }}/cluster_config.xml --cmpkg {{ og_upload_path }}/openGauss-5.0.0-openEuler-64bit-cm.tar.gz" + responses: + (?i)password: "{{ og_ca_pass }}" + chdir: "{{ og_home }}/install/app/tool/cm_tool" + changed_when: false + register: cm_install + until: "'CM exists' in cm_install.stdout" + retries: 10 + delay: 6 + +- name: Start cluster + ansible.builtin.command: + gs_om -t start + changed_when: false diff --git a/roles/opengauss/tasks/deploy/expansion.yml b/roles/opengauss/tasks/deploy/expansion.yml index af64d2b..aa2d419 100644 --- a/roles/opengauss/tasks/deploy/expansion.yml +++ b/roles/opengauss/tasks/deploy/expansion.yml @@ -2,15 +2,15 @@ ansible.builtin.debug: msg: "{{ groups['opengauss_expand'] }}" -# - name: Debug -# ansible.builtin.debug: -# msg: "{{ split_line }}" -# loop: "{{ cluster_detail.stdout_lines | sort }}" -# loop_control: -# loop_var: line -# when: "'cmserver' in line" -# vars: -# split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" +- name: Debug + ansible.builtin.debug: + msg: "{{ split_line }}" + loop: "{{ cluster_detail.stdout_lines | sort }}" + loop_control: + loop_var: line + when: "'cmserver' in line" + vars: + split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" - name: Create current cmserver list loop: "{{ cluster_detail.stdout_lines | sort }}" @@ -34,8 +34,7 @@ ansible.builtin.set_fact: og_expand_cm_ips: >- {%- for node in (groups['opengauss_cm'] | sort) -%} - {{ hostvars[node]['node_ip'] }} - {{ (loop.nextitem is defined) | ternary(',', '') }} + {{ hostvars[node]['node_ip'] }}{{ (loop.nextitem is defined) | ternary(',', '') }} {%- endfor -%} {%- if groups['opengauss_expand'] is defined -%} {%- for node in (groups['opengauss_expand'] | sort) -%} @@ -44,8 +43,7 @@ {%- endif -%} og_cm_names: >- {%- for node in (groups['opengauss_cm'] | sort) -%} - og{{ hostvars[node]['node_ip'] | ipaddr('int') }} - {{ (loop.nextitem is defined) | ternary(',', '') }} + og{{ hostvars[node]['node_ip'] | ipaddr('int') }}{{ (loop.nextitem is defined) | ternary(',', '') }} {%- endfor -%} {%- if groups['opengauss_expand'] is defined -%} {%- for node in (groups['opengauss_expand'] | sort) -%} @@ -54,15 +52,15 @@ {%- endif -%} when: "groups['opengauss_cm'] is defined" -# - name: Debug -# ansible.builtin.debug: -# msg: "{{ split_line }}" -# loop: "{{ cluster_detail.stdout_lines | sort }}" -# loop_control: -# loop_var: line -# when: "(og_data_path + '/dn') in line" -# vars: -# split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" +- name: Debug + ansible.builtin.debug: + msg: "{{ split_line }}" + loop: "{{ cluster_detail.stdout_lines | sort }}" + loop_control: + loop_var: line + when: "(og_data_path + '/dn') in line" + vars: + split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" - name: Create current data node list loop: "{{ cluster_detail.stdout_lines | sort }}" @@ -70,7 +68,7 @@ loop_var: line when: "(og_data_path + '/dn') in line" ansible.builtin.add_host: - hostname: "{{ node_info[3] }}_{{ node_info[1] }}" + hostname: "{{ node_info[4] }}_{{ node_info[1] }}" groups: - opengauss_dn node_ip: "{{ node_info[2] }}" @@ -94,10 +92,12 @@ ,og{{ node | ipaddr('int') }},{{ og_data_path }}/dn {%- endfor -%} og_expansion: "{{ groups['opengauss_expand'] is defined }}" - when: "groups['opengauss_dn'] is defined" + when: + - "groups['opengauss_dn'] is defined" - name: Cluster will be expand - when: og_expansion + when: + - "og_expansion" block: - name: Update /etc/hosts ansible.builtin.blockinfile: diff --git a/roles/opengauss/tasks/deploy/install.yml b/roles/opengauss/tasks/deploy/install.yml index b49ab60..1202cd5 100644 --- a/roles/opengauss/tasks/deploy/install.yml +++ b/roles/opengauss/tasks/deploy/install.yml @@ -11,7 +11,27 @@ - "{{ inventory_dir }}/templates/cluster_config.xml.j2" - "cluster_config.xml.j2" -- name: First time deploy +- name: Cluster expanding + ansible.builtin.shell: >- + . /home/{{ og_user }}/.bashrc && \ + {{ og_upload_path }}/script/gs_expansion \ + -U {{ og_user }} \ + -G {{ og_group }} \ + -X {{ og_upload_path }}/cluster_config.xml \ + -h {{ groups['opengauss_expand'] | join(',') }} + become_user: root + changed_when: false + when: "og_expansion" + +- name: Install CM when needed + ansible.builtin.import_tasks: + file: cluster_manager.yml + when: + - "groups['opengauss_cm'] is not defined" + - "og_cm_enabled" + become_user: "{{ og_user }}" + +- name: Starting deploy when: "not og_expansion" block: - name: Starting pre install @@ -32,16 +52,3 @@ changed_when: false become_user: "{{ og_user }}" become_flags: "-i" - -- name: Starting expand - ansible.builtin.shell: | - . /home/{{ og_user }}/.bashrc - - {{ og_upload_path }}/script/gs_expansion \ - -U {{ og_user }} \ - -G {{ og_group }} \ - -X {{ og_upload_path }}/cluster_config.xml \ - -h {{ groups['opengauss_expand'] | join(',') }} - become_user: root - changed_when: false - when: "og_expansion" diff --git a/roles/opengauss/tasks/os/common_set.yml b/roles/opengauss/tasks/os/common_set.yml index db7c970..4c18b4b 100644 --- a/roles/opengauss/tasks/os/common_set.yml +++ b/roles/opengauss/tasks/os/common_set.yml @@ -18,6 +18,7 @@ option: RemoveIPC value: "no" mode: "0644" + notify: Restart systemd-logind.service - name: Config /usr/lib/systemd/system/systemd-logind.service ini_file: @@ -57,6 +58,7 @@ - bison - nano - htop + - python3-pexpect update_cache: true use: "{{ custom_pkg_mgr | default(ansible_pkg_mgr) }}" @@ -123,3 +125,13 @@ ansible.builtin.command: swapoff -a changed_when: false + +- name: Config /etc/sysctl.d/50-opengauss.conf + ansible.posix.sysctl: + sysctl_file: /etc/sysctl.d/50-opengauss.conf + name: "{{ sysctl.key }}" + value: "{{ sysctl.value }}" + with_dict: "{{ combined_vars.opengauss_sysctl }}" + loop_control: + label: "{{ sysctl.key }} = {{ sysctl.value }}" + loop_var: sysctl diff --git a/roles/opengauss/tasks/os/user.yml b/roles/opengauss/tasks/os/user.yml index 21c6d7d..1d1f72c 100644 --- a/roles/opengauss/tasks/os/user.yml +++ b/roles/opengauss/tasks/os/user.yml @@ -21,10 +21,13 @@ group: "{{ og_group }}" recurse: true - - name: "Create .bashrc under /home/{{ og_user }}" - ansible.builtin.file: + - name: Config command alias + ansible.builtin.lineinfile: path: "/home/{{ og_user }}/.bashrc" - state: touch + create: true + line: "{{ item }}" owner: "{{ og_user }}" group: "{{ og_group }}" mode: "0644" + with_items: + - "alias gs_detail='gs_om -t status --detail'" diff --git a/roles/opengauss/tasks/post_tasks.yml b/roles/opengauss/tasks/post_tasks.yml index 026d3e7..5af0b3d 100644 --- a/roles/opengauss/tasks/post_tasks.yml +++ b/roles/opengauss/tasks/post_tasks.yml @@ -1,12 +1,12 @@ -- name: Config /etc/sysctl.d/50-opengauss.conf - ansible.posix.sysctl: - sysctl_file: /etc/sysctl.d/50-opengauss.conf - name: "{{ sysctl.key }}" - value: "{{ sysctl.value }}" - with_dict: "{{ combined_vars.opengauss_sysctl }}" - loop_control: - label: "{{ sysctl.key }} = {{ sysctl.value }}" - loop_var: sysctl +# - name: Config /etc/sysctl.d/50-opengauss.conf +# ansible.posix.sysctl: +# sysctl_file: /etc/sysctl.d/50-opengauss.conf +# name: "{{ sysctl.key }}" +# value: "{{ sysctl.value }}" +# with_dict: "{{ combined_vars.opengauss_sysctl }}" +# loop_control: +# label: "{{ sysctl.key }} = {{ sysctl.value }}" +# loop_var: sysctl - name: Remove authorized key ansible.posix.authorized_key: diff --git a/roles/opengauss/tasks/runtime_facts.yml b/roles/opengauss/tasks/runtime_facts.yml index c92fd42..5664479 100644 --- a/roles/opengauss/tasks/runtime_facts.yml +++ b/roles/opengauss/tasks/runtime_facts.yml @@ -3,6 +3,7 @@ og_user: "{{ combined_vars.opengauss_env.user_name }}" og_group: "{{ combined_vars.opengauss_env.user_group }}" og_user_pass: "{{ combined_vars.opengauss_env.user_pass }}" + og_ca_pass: "{{ combined_vars.opengauss_env.ca_file_pass }}" og_home: "{{ combined_vars.opengauss_home }}" og_paths: "{{ combined_vars.opengauss_paths }}" og_ver: "{{ combined_vars.opengauss_version }}" diff --git a/roles/opengauss/templates/cluster_master.xml.j2 b/roles/opengauss/templates/cluster_master.xml.j2 index b69e702..3202603 100644 --- a/roles/opengauss/templates/cluster_master.xml.j2 +++ b/roles/opengauss/templates/cluster_master.xml.j2 @@ -12,7 +12,7 @@ -{% if og_cm_enabled %} +{% if og_cm_enabled and groups['opengauss_cm'] is defined %} diff --git a/roles/opengauss/templates/cluster_replicas.xml.j2 b/roles/opengauss/templates/cluster_replicas.xml.j2 index f6dae6e..efa96b6 100644 --- a/roles/opengauss/templates/cluster_replicas.xml.j2 +++ b/roles/opengauss/templates/cluster_replicas.xml.j2 @@ -7,11 +7,15 @@ -{% if (groups['opengauss'] | count) > 2 %} +{% if og_cm_enabled and groups['opengauss_cm'] is defined %} + - + - +{% if node in groups['opengauss_cascade'] %} + + +{% endif %} {% endif %} -- Gitee From 4c4751a03099f5ead9049d2ee03a91ad37f8168e Mon Sep 17 00:00:00 2001 From: Haibin Lee Date: Fri, 3 Nov 2023 20:11:22 +0800 Subject: [PATCH 2/4] =?UTF-8?q?WIP:=20=E5=BE=97=E5=88=B0=20openGauss=20?= =?UTF-8?q?=E7=A0=94=E5=8F=91=E5=9B=A2=E9=98=9F=E7=9A=84=E6=94=AF=E6=8C=81?= =?UTF-8?q?=EF=BC=8C=E4=BA=86=E8=A7=A3=E4=BA=86=E6=89=A9=E5=AE=B9=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=E4=B8=80=E4=BA=9B=E7=89=B9=E6=AE=8A=E5=9C=BA=E6=99=AF?= =?UTF-8?q?=E6=89=80=E9=9C=80=E7=9A=84=E6=AD=A5=E9=AA=A4=EF=BC=8C=20=20=20?= =?UTF-8?q?=20=20=20=E6=AD=A3=E5=9C=A8=E5=AE=8C=E5=96=84=E8=AF=A5=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=EF=BC=8C=E6=89=8B=E5=8A=A8=E6=93=8D=E4=BD=9C=E5=B7=B2?= =?UTF-8?q?=E5=8F=AF=E4=BB=A5=E8=A7=A3=E5=86=B3=EF=BC=8C=E4=B8=8B=E5=91=A8?= =?UTF-8?q?=E9=87=8D=E6=9E=84=E9=83=A8=E5=88=86=E4=BB=A3=E7=A0=81=EF=BC=8C?= =?UTF-8?q?=E5=B8=8C=E6=9C=9B=E4=B8=8D=E4=BC=9A=E6=9C=89=E6=96=B0=E5=9D=91?= =?UTF-8?q?=20=20=20=20=20=20=E5=95=A6=EF=BD=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +- ansible-docker/Dockerfile | 18 +- inventories/opengauss/cluster_config.xml | 71 ++++++ .../opengauss/group_vars/opengauss.yml | 2 + inventories/opengauss/hosts.ini | 13 +- roles/opengauss/defaults/main.yml | 2 +- .../opengauss/tasks/deploy/cluster_check.yml | 47 ++-- .../opengauss/tasks/deploy/cluster_expand.yml | 25 ++ .../tasks/deploy/cluster_manager.yml | 223 +++++++++++++----- roles/opengauss/tasks/deploy/install.yml | 31 ++- roles/opengauss/tasks/deploy/main.yml | 4 +- .../{expansion.yml => runtime_groups.yml} | 65 +++-- roles/opengauss/tasks/deploy/upload.yml | 7 +- roles/opengauss/tasks/main.yml | 39 +-- roles/opengauss/tasks/os/common_set.yml | 2 +- roles/opengauss/tasks/pre_tasks.yml | 20 +- roles/opengauss/tasks/runtime_facts.yml | 4 +- .../opengauss/templates/cluster_master.xml.j2 | 16 +- .../templates/cluster_replicas.xml.j2 | 7 +- roles/pre_tasks/tasks/main.yml | 7 - roles/pre_tasks/tasks/os/openEuler.yml | 6 + 21 files changed, 410 insertions(+), 203 deletions(-) create mode 100644 inventories/opengauss/cluster_config.xml create mode 100644 roles/opengauss/tasks/deploy/cluster_expand.yml rename roles/opengauss/tasks/deploy/{expansion.yml => runtime_groups.yml} (75%) create mode 100644 roles/pre_tasks/tasks/os/openEuler.yml diff --git a/.gitignore b/.gitignore index 36bd50c..8a24b2c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,9 @@ downloaded_files inventories/opengauss/credentials inventories/opengauss/assembled_vars inventories/opengauss/report* +inventories/opengauss/hostvars vagrant/.vagrant vagrant/packer_cache vagrant/output* -log \ No newline at end of file +log +test.yml \ No newline at end of file diff --git a/ansible-docker/Dockerfile b/ansible-docker/Dockerfile index 8c55e8a..6c9cd7f 100644 --- a/ansible-docker/Dockerfile +++ b/ansible-docker/Dockerfile @@ -4,14 +4,22 @@ ARG TIMEZONE="Asia/Shanghai" ADD ssh_config /root/.ssh/config -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.cloud.tencent.com/g' /etc/apk/repositories \ +RUN addgroup dbgrp \ + && adduser -D -G dbgrp omm \ + && sed -i 's/dl-cdn.alpinelinux.org/mirrors.cloud.tencent.com/g' /etc/apk/repositories \ + && apk upgrade -U --no-cache \ && apk --no-cache add bzip2 nano ansible fish byobu \ sshpass tzdata rsync wget curl net-tools tar unzip \ - openssh-client py3-netaddr musl-locales py3-jmespath + openssh-client py3-netaddr musl-locales py3-jmespath \ + inetutils-telnet \ + && ln -sf /usr/share/zoneinfo/$TIMEZONE /etc/localtime \ + && sed -i 's/\/bin\/ash/\/usr\/bin\/fish/g' /etc/passwd + +USER omm RUN byobu-select-backend tmux \ && byobu-ctrl-a screen \ && byobu-enable \ - && ln -sf /usr/share/zoneinfo/$TIMEZONE /etc/localtime \ - && sed -i 's/\/bin\/ash/\/usr\/bin\/fish/g' /etc/passwd \ - && fish -c "alias pansible='ansible-playbook' && funcsave pansible" \ No newline at end of file + && fish -c "alias pansible='ansible-playbook' && funcsave pansible" + + diff --git a/inventories/opengauss/cluster_config.xml b/inventories/opengauss/cluster_config.xml new file mode 100644 index 0000000..0492b96 --- /dev/null +++ b/inventories/opengauss/cluster_config.xml @@ -0,0 +1,71 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/inventories/opengauss/group_vars/opengauss.yml b/inventories/opengauss/group_vars/opengauss.yml index d0f31c0..9193dc2 100644 --- a/inventories/opengauss/group_vars/opengauss.yml +++ b/inventories/opengauss/group_vars/opengauss.yml @@ -1,3 +1,5 @@ # ansible_python_interpreter: "python3" opengauss_version: 5.0.0 + +global_pkg_mirror: https://mirrors.aliyun.com diff --git a/inventories/opengauss/hosts.ini b/inventories/opengauss/hosts.ini index 1168a45..cafc2f9 100644 --- a/inventories/opengauss/hosts.ini +++ b/inventories/opengauss/hosts.ini @@ -2,14 +2,19 @@ [opengauss_master] 192.168.56.18 -; 备机，可设置若干个或留空。不可大于 8。 +; 备机，可设置若干个或留空。 [opengauss_follower] -192.168.56.13 +; 192.168.56.13 192.168.56.15 +; 192.168.56.19 ; 级联机，可设置若干个或留空。 [opengauss_cascade] -192.168.56.11 +; 192.168.56.11 +; 192.168.56.12 +; 192.168.56.14 +; 192.168.56.16 +192.168.56.17 ; 以上 3 个分组的合并组，勿动。 [opengauss:children] @@ -23,7 +28,7 @@ opengauss_master opengauss_follower ; -; 备节点分组 +; 备节点分组，总数不可大于 8。 [opengauss_replica_nodes:children] opengauss_cascade opengauss_follower diff --git a/roles/opengauss/defaults/main.yml b/roles/opengauss/defaults/main.yml index e9223b4..72a6e53 100644 --- a/roles/opengauss/defaults/main.yml +++ b/roles/opengauss/defaults/main.yml @@ -47,7 +47,7 @@ opengauss_env: dataPortBase: 15400 cmServerPortBase: 15300 azName: AZ1 - swap_off: true # 关闭 swap + swap_off: false # 关闭 swap syncNum: 0 # dataNode1_syncNum # locale: SQL_ASCII diff --git a/roles/opengauss/tasks/deploy/cluster_check.yml b/roles/opengauss/tasks/deploy/cluster_check.yml index 406fa5d..fc22190 100644 --- a/roles/opengauss/tasks/deploy/cluster_check.yml +++ b/roles/opengauss/tasks/deploy/cluster_check.yml @@ -1,8 +1,7 @@ - name: Check cluster status block: - name: "Query cluster detail" - ansible.builtin.command: >- - gs_om -t status --detail + ansible.builtin.command: "gs_om -t status --detail" changed_when: false register: cluster_detail become_user: "{{ og_user }}" @@ -10,18 +9,18 @@ run_once: true rescue: - - name: Start pre tasks + - name: Import pre tasks ansible.builtin.import_tasks: file: pre_tasks.yml become_user: root - - name: Start deploy + - name: Import deploy tasks ansible.builtin.import_tasks: file: deploy/main.yml delegate_to: "{{ og_master }}" run_once: true -- name: Check expansion status +- name: Tasks for expansion when: "cluster_detail is succeeded" block: - name: Create expansion list @@ -33,29 +32,39 @@ when: "item not in cluster_detail.stdout" run_once: true - - name: Start pre tasks + - name: Import pre tasks ansible.builtin.include_tasks: file: pre_tasks.yml when: - "groups['opengauss_expand'] is defined" - "inventory_hostname in groups['opengauss_expand']" - - name: Start expansion + - name: Tasks for CM cluster + when: + - "'cmserver' not in cluster_detail.stdout" + - "og_cm_enabled" + block: + - name: Import cluster manager tasks + ansible.builtin.import_tasks: + file: cluster_manager.yml + delegate_to: "{{ og_master }}" + become_user: "{{ og_user }}" + run_once: true + + - name: Refresh cluster status + ansible.builtin.command: + gs_om -t status --detail + changed_when: false + register: cluster_detail1 + become_user: "{{ og_user }}" + delegate_to: "{{ og_master }}" + run_once: true + + - name: Import cluster expand tasks ansible.builtin.import_tasks: - file: expansion.yml + file: cluster_expand.yml vars: og_expansion: true delegate_to: "{{ og_master }}" run_once: true when: "groups['opengauss_expand'] is defined" - -- name: Add CM cluster - when: - - "cluster_detail is succeeded" - - "'cmserver' not in cluster_detail.stdout" - - "og_cm_enabled" - ansible.builtin.import_tasks: - file: cluster_manager.yml - delegate_to: "{{ og_master }}" - run_once: true - become_user: "{{ og_user }}" \ No newline at end of file diff --git a/roles/opengauss/tasks/deploy/cluster_expand.yml b/roles/opengauss/tasks/deploy/cluster_expand.yml new file mode 100644 index 0000000..135b633 --- /dev/null +++ b/roles/opengauss/tasks/deploy/cluster_expand.yml @@ -0,0 +1,25 @@ +- name: Import runtime groups tasks + ansible.builtin.import_tasks: + file: runtime_groups.yml + +- name: Cluster will be expand + when: + - "og_expansion" + block: + - name: Update /etc/hosts + ansible.builtin.blockinfile: + path: /etc/hosts + marker: "# {mark} OPENGAUSS NODES" + block: | + {% for node in og_all_nodes %} + {{ node }} og{{ node | ipaddr('int') }} og-{{ node | replace('.', '-') }} + {% endfor %} + delegate_to: "{{ node }}" + loop: "{{ og_all_nodes }}" + loop_control: + loop_var: node + + - name: Import deploy tasks + ansible.builtin.import_tasks: + file: deploy/main.yml + delegate_to: "{{ og_master }}" diff --git a/roles/opengauss/tasks/deploy/cluster_manager.yml b/roles/opengauss/tasks/deploy/cluster_manager.yml index f640f52..e49f2b8 100644 --- a/roles/opengauss/tasks/deploy/cluster_manager.yml +++ b/roles/opengauss/tasks/deploy/cluster_manager.yml @@ -1,68 +1,169 @@ -# - name: Upload CM install package -# ansible.builtin.unarchive: -# src: "{{ playbook_dir }}/downloaded_files/{{ og_pkg_name }}" -# dest: "/tmp/" -# include: "*-cm.tar.gz" -# owner: "{{ og_user }}" -# mode: "0644" +- name: Import runtime groups tasks + ansible.builtin.import_tasks: + file: runtime_groups.yml -- name: Stop cluster - ansible.builtin.command: - gs_om -t stop - changed_when: false +- name: Deploy CM into existing data cluster + block: + - name: Stop cluster + ansible.builtin.command: + gs_om -t stop + changed_when: false -- name: Create initial cmserver list - loop: "{{ og_all_nodes }}" - loop_control: - loop_var: node - extended: true - ansible.builtin.add_host: - hostname: "{{ ansible_loop.index }}_{{ node }}" - groups: - - opengauss_cm - node_ip: "{{ node }}" + # 把现有的数据节点加到 'opengauss_cm' 组 + - name: Create initial cmserver list + ansible.builtin.add_host: + hostname: >- + {%- if og_cluster_config.data_port in line -%} + {{ node_info[4] }}_{{ node_info[1] }} + {%- else -%} + {{ node_info[3] }}_{{ node_info[1] }} + {%- endif -%} + groups: + - opengauss_cm + node_ip: "{{ node_info[2] }}" + node_name: "{{ node_info[1] }}" + loop: "{{ cluster_detail.stdout_lines | sort }}" + loop_control: + loop_var: line + when: "(og_data_path + '/dn') in line" + vars: + node_info: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" -- name: "Change ownership to {{ og_upload_path }}" - ansible.builtin.file: - path: "{{ og_upload_path }}" - owner: "{{ og_user }}" - recurse: true - become_user: root + # - name: Create initial cmserver list + # ansible.builtin.add_host: + # hostname: "{{ hostvars[node]['node_ip'] }}" + # groups: + # - opengauss_cm + # loop: "{{ groups['opengauss_dn'] }}" + # loop_control: + # loop_var: node + # label: "{{ hostvars[node]['node_ip'] }}" -- name: "Remove /opt/openGauss/install/om/openGauss-5.0.0-openEuler-64bit-cm.tar.gz" - ansible.builtin.file: - path: "/opt/openGauss/install/om/openGauss-5.0.0-openEuler-64bit-cm.tar.gz" - state: absent - become_user: root - loop: "{{ og_all_nodes }}" - delegate_to: "{{ item }}" + - name: "Import packages upload tasks" + ansible.builtin.import_tasks: + file: upload.yml + become_user: root -- name: "Create cluster_config.xml under {{ og_upload_path }}" - ansible.builtin.template: - src: "{{ item }}" - dest: "{{ og_upload_path }}/cluster_config.xml" - owner: "{{ og_user }}" - group: "{{ og_group }}" - mode: "0644" - backup: true - lstrip_blocks: true - with_first_found: - - "{{ inventory_dir }}/templates/cluster_config.xml.j2" - - "cluster_config.xml.j2" + - name: Get cm package name + ansible.builtin.set_fact: + cm_pkg: "{{ file }}" + loop: "{{ og_upload.files | default([]) }}" + loop_control: + loop_var: file + when: + - "'cm.tar.gz' in file" -- name: Install CM - ansible.builtin.expect: - command: "./cm_install -X {{ og_upload_path }}/cluster_config.xml --cmpkg {{ og_upload_path }}/openGauss-5.0.0-openEuler-64bit-cm.tar.gz" - responses: - (?i)password: "{{ og_ca_pass }}" - chdir: "{{ og_home }}/install/app/tool/cm_tool" - changed_when: false - register: cm_install - until: "'CM exists' in cm_install.stdout" - retries: 10 - delay: 6 + - name: "Change file permission of {{ cm_pkg }}" + ansible.builtin.file: + path: "{{ og_upload_path }}/{{ cm_pkg }}" + owner: "{{ og_user }}" + group: "{{ og_group }}" + mode: "0640" + become_user: root -- name: Start cluster - ansible.builtin.command: - gs_om -t start - changed_when: false + - name: Create some paths + ansible.builtin.file: + path: "{{ item.0 }}" + state: directory + owner: "{{ og_user }}" + group: "{{ og_group }}" + mode: "0750" + delegate_to: "{{ item.1 }}" + become_user: root + with_nested: + - ['{{ og_log_path }}/omm/cm/cm_server', '{{ og_log_path }}/omm/cm/cm_agent'] + - "{{ og_all_nodes }}" + vars: + og_log_path: "{{ combined_vars.opengauss_paths.gaussdbLogPath }}" + + - name: "Remove /opt/openGauss/install/om/{{ cm_pkg }}" + ansible.builtin.file: + path: "/opt/openGauss/install/om/{{ cm_pkg }}" + state: absent + become_user: root + loop: "{{ og_all_nodes }}" + delegate_to: "{{ item }}" + + # 这里创建一个临时的用于安装 CM 的 cluster_config.xml + - name: "Create cluster_config.xml under {{ og_upload_path }}" + ansible.builtin.template: + src: "{{ item }}" + dest: "/tmp/cluster_config.xml" + owner: "{{ og_user }}" + group: "{{ og_group }}" + mode: "0644" + backup: true + lstrip_blocks: true + with_first_found: + - "{{ inventory_dir }}/templates/cluster_config.xml.j2" + - "cluster_config.xml.j2" + vars: + og_expand_cm_ips: >- + {%- for node in groups['opengauss_cm'] -%} + {{ hostvars[node]['node_ip'] }}{{ (loop.nextitem is defined) | ternary(',', '') }} + {%- endfor -%} + og_cm_names: >- + {%- for node in groups['opengauss_cm'] -%} + {{ hostvars[node]['node_name'] }}{{ (loop.nextitem is defined) | ternary(',', '') }} + {%- endfor -%} + temp_data_nodes: >- + {{ og_data_path }}/dn + {%- for node in groups['opengauss_cm'] if hostvars[node]['node_ip'] != og_master -%} + ,{{ hostvars[node]['node_name'] }},{{ og_data_path }}/dn + {%- endfor -%} + og_replicas: >- + [' + {%- for node in groups['opengauss_cm'] if hostvars[node]['node_ip'] != og_master -%} + {{ hostvars[node]['node_ip'] }}{{ (loop.nextitem is defined) | ternary("','", "") }} + {%- endfor -%} + '] + + # 这里需要建一个假的定时任务，包含 'om_monitor'。后续 cm_install 时会检查，否则失败。 + - name: Fake an om_monitor cron job before install + ansible.builtin.cron: + name: openGauss om monitor + special_time: yearly + job: "{{ og_home }}/install/app/bin/om_monitor" + loop: "{{ groups['opengauss_cm'] }}" + loop_control: + loop_var: node + label: "{{ hostvars[node]['node_ip'] }}" + delegate_to: "{{ hostvars[node]['node_ip'] }}" + + - name: Install CM + ansible.builtin.expect: + command: "./cm_install -X /tmp/cluster_config.xml --cmpkg {{ og_upload_path }}/{{ cm_pkg }}" + responses: + (?i)password: "{{ og_ca_pass }}" + chdir: "{{ og_home }}/install/app/tool/cm_tool" + changed_when: false + register: cm_install + until: + - "cm_install.stdout is defined" + - "'CM exists' in cm_install.stdout" + retries: 3 + timeout: 300 + ignore_errors: true + + rescue: + - name: Install pexpect + ansible.builtin.package: + name: "{{ python_name }}-pexpect" + become_user: root + + - name: Repeat tasks + ansible.builtin.include_tasks: + file: cluster_manager.yml + + always: + - name: Debug + ansible.builtin.debug: + var: cm_install + + - name: Start cluster + ansible.builtin.command: + gs_om -t restart + with_items: + - "cm_ctl start" + - "gs_om -t start" + changed_when: false diff --git a/roles/opengauss/tasks/deploy/install.yml b/roles/opengauss/tasks/deploy/install.yml index 1202cd5..6d0c34f 100644 --- a/roles/opengauss/tasks/deploy/install.yml +++ b/roles/opengauss/tasks/deploy/install.yml @@ -1,15 +1,25 @@ -- name: "Create cluster_config.xml under {{ og_upload_path }}" +- name: "Create {{ inventory_dir }}/cluster_config.xml in local" ansible.builtin.template: src: "{{ item }}" - dest: "{{ og_upload_path }}/cluster_config.xml" + dest: "{{ inventory_dir }}/cluster_config.xml" owner: "{{ og_user }}" group: "{{ og_group }}" mode: "0644" - backup: true lstrip_blocks: true with_first_found: - "{{ inventory_dir }}/templates/cluster_config.xml.j2" - "cluster_config.xml.j2" + delegate_to: localhost + become: false + +- name: "Upload cluster_config.xml to {{ og_upload_path }}" + ansible.builtin.copy: + src: "{{ inventory_dir }}/cluster_config.xml" + dest: "{{ og_upload_path }}/" + owner: "{{ og_user }}" + group: "{{ og_group }}" + mode: "0644" + backup: true - name: Cluster expanding ansible.builtin.shell: >- @@ -18,19 +28,12 @@ -U {{ og_user }} \ -G {{ og_group }} \ -X {{ og_upload_path }}/cluster_config.xml \ - -h {{ groups['opengauss_expand'] | join(',') }} + -h {{ groups['opengauss_expand'] | join(',') }} \ + --time-out {{ (og_all_nodes | count) * 600 }} become_user: root changed_when: false when: "og_expansion" -- name: Install CM when needed - ansible.builtin.import_tasks: - file: cluster_manager.yml - when: - - "groups['opengauss_cm'] is not defined" - - "og_cm_enabled" - become_user: "{{ og_user }}" - - name: Starting deploy when: "not og_expansion" block: @@ -40,6 +43,7 @@ -U {{ og_user }} \ -G {{ og_group }} \ -X {{ og_upload_path }}/cluster_config.xml \ + --skip-hostname-set \ --non-interactive changed_when: false @@ -48,7 +52,8 @@ gs_install \ -X {{ og_upload_path }}/cluster_config.xml \ --gsinit-parameter="--pwpasswd={{ og_db_pass }}" \ - --gsinit-parameter="--locale={{ og_locale }}" + --gsinit-parameter="--locale={{ og_locale }}" \ + --time-out {{ (og_all_nodes | count) * 600 }} changed_when: false become_user: "{{ og_user }}" become_flags: "-i" diff --git a/roles/opengauss/tasks/deploy/main.yml b/roles/opengauss/tasks/deploy/main.yml index b7ed008..c58adbf 100644 --- a/roles/opengauss/tasks/deploy/main.yml +++ b/roles/opengauss/tasks/deploy/main.yml @@ -39,11 +39,11 @@ changed_when: false rescue: - - name: Upload packages + - name: Import upload tasks ansible.builtin.import_tasks: file: deploy/upload.yml always: - - name: In progressing + - name: Import installation tasks ansible.builtin.import_tasks: file: deploy/install.yml diff --git a/roles/opengauss/tasks/deploy/expansion.yml b/roles/opengauss/tasks/deploy/runtime_groups.yml similarity index 75% rename from roles/opengauss/tasks/deploy/expansion.yml rename to roles/opengauss/tasks/deploy/runtime_groups.yml index aa2d419..b3b0819 100644 --- a/roles/opengauss/tasks/deploy/expansion.yml +++ b/roles/opengauss/tasks/deploy/runtime_groups.yml @@ -1,6 +1,9 @@ -- name: Expansion list - ansible.builtin.debug: - msg: "{{ groups['opengauss_expand'] }}" +# - name: Expansion list +# ansible.builtin.debug: +# msg: "{{ groups['opengauss_expand'] }}" + +- ansible.builtin.debug: + var: cluster_detail.stdout_lines - name: Debug ansible.builtin.debug: @@ -13,19 +16,19 @@ split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" - name: Create current cmserver list - loop: "{{ cluster_detail.stdout_lines | sort }}" - loop_control: - loop_var: line - when: "'cmserver' in line" ansible.builtin.add_host: hostname: "{{ node_info[3] }}_{{ node_info[1] }}" groups: - opengauss_cm node_ip: "{{ node_info[2] }}" + loop: "{{ cluster_detail.stdout_lines | sort }}" + loop_control: + loop_var: line vars: node_info: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" + when: "'cmserver' in line" -- name: Current config manager servers +- name: Current cluster manager servers ansible.builtin.debug: msg: "{{ groups['opengauss_cm'] }}" when: "groups['opengauss_cm'] is defined" @@ -62,17 +65,23 @@ vars: split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" +# 这里需要判断输出了多少列，因为有 CM 的集群里，不输出数据库端口的那一列。 - name: Create current data node list - loop: "{{ cluster_detail.stdout_lines | sort }}" - loop_control: - loop_var: line - when: "(og_data_path + '/dn') in line" ansible.builtin.add_host: - hostname: "{{ node_info[4] }}_{{ node_info[1] }}" + hostname: >- + {%- if og_cluster_config.data_port in line -%} + {{ node_info[4] }}_{{ node_info[1] }}_{{ node_info[7] }} + {%- else -%} + {{ node_info[3] }}_{{ node_info[1] }}_{{ node_info[6] }} + {%- endif -%} groups: - opengauss_dn node_ip: "{{ node_info[2] }}" node_name: "{{ node_info[1] }}" + loop: "{{ cluster_detail.stdout_lines | sort }}" + loop_control: + loop_var: line + when: "(og_data_path + '/dn') in line" vars: node_info: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" @@ -88,31 +97,11 @@ {%- for node in (groups['opengauss_dn'] | sort) if (hostvars[node]['node_ip'] != og_master) -%} ,{{ hostvars[node]['node_name'] }},{{ og_data_path }}/dn {%- endfor -%} - {%- for node in (groups['opengauss_expand'] | sort) -%} - ,og{{ node | ipaddr('int') }},{{ og_data_path }}/dn - {%- endfor -%} + {%- if groups['opengauss_expand'] is defined -%} + {%- for node in (groups['opengauss_expand'] | sort) -%} + ,og{{ node | ipaddr('int') }},{{ og_data_path }}/dn + {%- endfor -%} + {%- endif -%} og_expansion: "{{ groups['opengauss_expand'] is defined }}" when: - "groups['opengauss_dn'] is defined" - -- name: Cluster will be expand - when: - - "og_expansion" - block: - - name: Update /etc/hosts - ansible.builtin.blockinfile: - path: /etc/hosts - marker: "# {mark} OPENGAUSS NODES" - block: | - {% for node in og_all_nodes %} - {{ node }} og{{ node | ipaddr('int') }} og-{{ node | replace('.', '-') }} - {% endfor %} - delegate_to: "{{ node }}" - loop: "{{ og_all_nodes }}" - loop_control: - loop_var: node - - - name: Start expand - ansible.builtin.import_tasks: - file: deploy/main.yml - delegate_to: "{{ og_master }}" diff --git a/roles/opengauss/tasks/deploy/upload.yml b/roles/opengauss/tasks/deploy/upload.yml index cc2145b..08c20bc 100644 --- a/roles/opengauss/tasks/deploy/upload.yml +++ b/roles/opengauss/tasks/deploy/upload.yml @@ -5,13 +5,18 @@ path: "{{ og_upload_path }}" state: directory mode: "0755" + owner: "{{ og_user }}" + group: "{{ og_group }}" - name: "Uploading {{ og_pkg_name }}" ansible.builtin.unarchive: src: "{{ playbook_dir }}/downloaded_files/{{ og_pkg_name }}" dest: "{{ og_upload_path }}/" list_files: true - creates: "{{ og_upload_path }}/version.cfg" + extra_opts: + - "--owner={{ og_user }}" + - "--group={{ og_group }}" + # creates: "{{ og_upload_path }}/version.cfg" register: og_upload - name: "Extracting om tar file" diff --git a/roles/opengauss/tasks/main.yml b/roles/opengauss/tasks/main.yml index e52c96d..4053f41 100644 --- a/roles/opengauss/tasks/main.yml +++ b/roles/opengauss/tasks/main.yml @@ -4,48 +4,21 @@ - name: Tasks always run tags: always block: - - name: Combine vars + - name: Import vars combine tasks ansible.builtin.import_role: name: "pre_tasks" tasks_from: "vars_combine.yml" - - name: Set runtime facts - ansible.builtin.import_tasks: runtime_facts.yml + - name: Import runtime facts tasks + ansible.builtin.import_tasks: + file: runtime_facts.yml - - name: Check cluster status + - name: Import cluster check tasks ansible.builtin.import_tasks: file: deploy/cluster_check.yml - # - name: Cluster will be expand - # when: og_expansion - # block: - # - name: Start pre tasks - # ansible.builtin.import_tasks: - # file: pre_tasks.yml - # when: "inventory_hostname in groups['opengauss_expand']" - - # - name: Update /etc/hosts - # ansible.builtin.blockinfile: - # path: /etc/hosts - # marker: "# {mark} OPENGAUSS NODES" - # block: | - # {% for node in og_all_nodes %} - # {{ node }} og{{ node | ipaddr('int') }} og-{{ node | replace('.', '-') }} - # {% endfor %} - # delegate_to: "{{ node }}" - # loop: "{{ og_all_nodes }}" - # loop_control: - # loop_var: node - # run_once: true - - # - name: Start deploy - # ansible.builtin.import_tasks: - # file: deploy/main.yml - # delegate_to: "{{ og_master }}" - # run_once: true - always: - - name: Run post tasks + - name: Import post tasks ansible.builtin.import_tasks: file: post_tasks.yml tags: always diff --git a/roles/opengauss/tasks/os/common_set.yml b/roles/opengauss/tasks/os/common_set.yml index 4c18b4b..c6aeb3f 100644 --- a/roles/opengauss/tasks/os/common_set.yml +++ b/roles/opengauss/tasks/os/common_set.yml @@ -58,7 +58,7 @@ - bison - nano - htop - - python3-pexpect + - "{{ python_name }}-pexpect" update_cache: true use: "{{ custom_pkg_mgr | default(ansible_pkg_mgr) }}" diff --git a/roles/opengauss/tasks/pre_tasks.yml b/roles/opengauss/tasks/pre_tasks.yml index 6bd44a4..b6deabd 100644 --- a/roles/opengauss/tasks/pre_tasks.yml +++ b/roles/opengauss/tasks/pre_tasks.yml @@ -1,26 +1,30 @@ -- name: "Custom sets for {{ ansible_distribution }}" - ansible.builtin.include_tasks: "{{ item }}" +- name: "Import custom tasks for {{ ansible_distribution }}" + ansible.builtin.include_tasks: + file: "{{ item }}" with_first_found: - "os/{{ ansible_distribution | replace(' ', '_') }}.yml" - "os/not_supported.yml" -- name: Common sets for all distribution - ansible.builtin.import_tasks: os/common_set.yml +- name: Import Common tasks for all distribution + ansible.builtin.import_tasks: + file: os/common_set.yml -- name: "Custom sets for {{ ansible_os_family }}" - ansible.builtin.include_tasks: "{{ item }}" +- name: "Import custom tasks for {{ ansible_os_family }}" + ansible.builtin.include_tasks: + file: "{{ item }}" with_first_found: - "os/{{ ansible_os_family }}.yml" - "os/not_supported.yml" -- name: Config user on hosts +- name: Import user config tasks ansible.builtin.include_tasks: file: os/user.yml with_items: "{{ groups['opengauss_expand'] | default(og_all_nodes) }}" loop_control: loop_var: node + run_once: true -- name: Config ssh on hosts +- name: Import ssh config tasks ansible.builtin.include_tasks: file: os/ssh.yml with_items: diff --git a/roles/opengauss/tasks/runtime_facts.yml b/roles/opengauss/tasks/runtime_facts.yml index 5664479..aed6832 100644 --- a/roles/opengauss/tasks/runtime_facts.yml +++ b/roles/opengauss/tasks/runtime_facts.yml @@ -13,8 +13,8 @@ og_cluster_config: az_name: "{{ combined_vars.opengauss_env.azName | default('AZ1') }}" az_priority: "{{ combined_vars.opengauss_env.azPriority | default('1') }}" - port_base: "{{ combined_vars.opengauss_env.cmServerPortBase }}" - port_data: "{{ combined_vars.opengauss_env.dataPortBase }}" + cm_port: "{{ combined_vars.opengauss_env.cmServerPortBase }}" + data_port: "{{ combined_vars.opengauss_env.dataPortBase }}" sync_num: "{{ combined_vars.opengauss_env.syncNum }}" - name: Set runtime facts - 2 diff --git a/roles/opengauss/templates/cluster_master.xml.j2 b/roles/opengauss/templates/cluster_master.xml.j2 index 3202603..b4987c6 100644 --- a/roles/opengauss/templates/cluster_master.xml.j2 +++ b/roles/opengauss/templates/cluster_master.xml.j2 @@ -9,14 +9,22 @@ - - + + -{% if og_cm_enabled and groups['opengauss_cm'] is defined %} +{% if og_cm_enabled %} - + diff --git a/roles/opengauss/templates/cluster_replicas.xml.j2 b/roles/opengauss/templates/cluster_replicas.xml.j2 index efa96b6..b12be8a 100644 --- a/roles/opengauss/templates/cluster_replicas.xml.j2 +++ b/roles/opengauss/templates/cluster_replicas.xml.j2 @@ -1,19 +1,20 @@ -{% for node in (groups['opengauss_replica_nodes'] | sort) %} +{% for node in og_replicas %} -{% if og_cm_enabled and groups['opengauss_cm'] is defined %} +{% if og_cm_enabled %} - + {% if node in groups['opengauss_cascade'] %} + {% endif %} {% endif %} diff --git a/roles/pre_tasks/tasks/main.yml b/roles/pre_tasks/tasks/main.yml index 9ca5d2a..8566c4d 100644 --- a/roles/pre_tasks/tasks/main.yml +++ b/roles/pre_tasks/tasks/main.yml @@ -21,13 +21,6 @@ global_bond_iface: "{{ iface_name.stdout | trim }}" global_bond_ip: "{{ ansible_host | default(inventory_hostname) }}" -# - name: "Set `inventory_dir` equal to `playbook_dir`" -# ansible.builtin.set_fact: -# inventory_dir: "{{ playbook_dir }}" -# when: -# - inventory_dir is not defined -# - playbook_dir is defined - - name: Set python vars ansible.builtin.set_fact: python_name: "{{ (ansible_python_interpreter | default(discovered_interpreter_python)) | basename }}" diff --git a/roles/pre_tasks/tasks/os/openEuler.yml b/roles/pre_tasks/tasks/os/openEuler.yml new file mode 100644 index 0000000..2ad5be8 --- /dev/null +++ b/roles/pre_tasks/tasks/os/openEuler.yml @@ -0,0 +1,6 @@ +- name: "Replace sources mirror with {{ global_pkg_mirror }}" + ansible.builtin.replace: + path: "/etc/yum.repos.d/openEuler.repo" + regexp: "http(s)://repo.openeuler.org" + replace: "{{ global_pkg_mirror }}/openeuler" + when: "global_pkg_mirror is defined" -- Gitee From 92bb6649d4e43087416cf5a98bdd8545253c1529 Mon Sep 17 00:00:00 2001 From: Haibin Lee Date: Mon, 6 Nov 2023 20:15:41 +0800 Subject: [PATCH 3/4] =?UTF-8?q?REV:=20=E5=9F=BA=E6=9C=AC=E8=A7=A3=E5=86=B3?= =?UTF-8?q?=E6=89=A9=E5=AE=B9=E9=97=AE=E9=A2=98=E3=80=82=E6=AC=A2=E8=BF=8E?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 18 +++++- ansible.cfg | 2 +- docs/04-expansion.md | 15 ++--- docs/99-get-involved.md | 6 ++ inventories/opengauss/cluster_config.xml | 39 ++++++++++-- .../opengauss/group_vars/opengauss.yml | 2 +- inventories/opengauss/hosts.ini | 4 +- .../opengauss/tasks/deploy/cluster_check.yml | 23 +++++++ .../tasks/deploy/cluster_manager.yml | 62 ++++++++++++------- roles/opengauss/tasks/deploy/install.yml | 32 ++++++---- .../opengauss/tasks/deploy/runtime_groups.yml | 40 ++++++------ roles/opengauss/tasks/os/CentOS.yml | 6 ++ roles/opengauss/tasks/os/common_set.yml | 5 +- roles/opengauss/tasks/os/openEuler.yml | 3 + roles/pre_tasks/tasks/os/openEuler.yml | 2 +- 15 files changed, 186 insertions(+), 73 deletions(-) create mode 100644 docs/99-get-involved.md diff --git a/README.md b/README.md index 566731d..e56e86b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ # Ansible role for openGauss -自动部署 openGauss，根据分组定义，自适应部署单点、一主一备、一主多备以及级联节点等架构模式。 +本项目基于 openGauss 的官方安装脚本，结合 Ansible 自动化编排流程，实现自动部署 openGauss。 + +使用者只需给目标机器编排好集群角色，即可通过一行命令，自适应部署单点、一主一备、一主多备以及级联节点等架构模式。 + +本项目适用于以下场景或用户： + +1. 同一网络环境下的快速部署。 +1. 需要多次重新部署数据库集群的测试工程师。 # 已适配的系统 @@ -13,9 +20,16 @@ * 5.1.0 * 5.0.0 +# 不足之处 + +1. 暂不支持 DCF 模式的集群部署。 +1. 暂不支持离线部署。 +1. 暂不支持多地容灾部署。 + # 优势特点 -1. 1 主 4 备 1 级联的架构，十分钟内部署完成（不含安装包的下载时间）。 +1. 以 1 主 4 备 1 级联的架构为例，十分钟内部署完成（不含 openGauss 压缩包及 Linux 系统安装包的下载时间）。 +1. 支持从 1 主单节点，逐步扩展为多节点集群。 1. 自动匹配 CPU 架构以及操作系统。 1. 支持自定义 cluster_config.xml.j2 模板，优先使用 `{{ inventory_dir }}/templates/openGauss/cluster_config.xml.j2`。 1. 默认自动生成数据库管理员密码，也可自定义变量进行替换。全程自动化，无交互步骤。 diff --git a/ansible.cfg b/ansible.cfg index ef8ea22..8930901 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -11,7 +11,7 @@ host_key_checking = False deprecation_warnings = False callback_whitelist = profile_tasks,timer,profile_roles ; display_skipped_hosts = False -; stdout_callback = yaml +stdout_callback = yaml [callback_log_plays] log_folder = ./log/ \ No newline at end of file diff --git a/docs/04-expansion.md b/docs/04-expansion.md index 40ff58c..8a390f1 100644 --- a/docs/04-expansion.md +++ b/docs/04-expansion.md @@ -10,7 +10,13 @@ 备机数量不可超过 8 台。 -假设原编排为 1 主 1 备 1 级联。 +### 请注意： + +### 如果是对 1 主 1 备进行扩容，会增加一些部署 Cluster Manager 的流程，期间需要停止已有的数据节点。请确保扩容前没有数据库读写操作。 + +### 如果是从 1 主单节点开始扩容，请先增加 1 个备节点，扩容为 1 主 1 备的架构后，再进行多节点同时扩容。 + +假设原编排为 1 主 1 备。 ``` ; 主服务器组，仅设置 1 个目标机。 @@ -20,14 +26,9 @@ ; 从服务器组，可设置若干个或留空。 [opengauss_follower] 192.168.56.12 - -; 级联服务器组，可设置若干个或留空。 -[opengauss_cascade] -192.168.56.13 - ``` -增加 2 台备机，1 台级联机，则在对应的组里增加目标服务器。 +增加 2 台备机，2 台级联机，则在对应的组里增加目标服务器。 ``` ; 主服务器组，仅设置 1 个目标机。 diff --git a/docs/99-get-involved.md b/docs/99-get-involved.md new file mode 100644 index 0000000..b1cfd0e --- /dev/null +++ b/docs/99-get-involved.md @@ -0,0 +1,6 @@ +# 贡献你的代码 + +## 前提条件 + +1. 需要你非常熟悉 [Ansible](https://docs.ansible.com/ansible/latest/collections/index.html) 的模块文档，一切编写以官方文档为准。 +1. 不论你使用哪款 IDE 编辑器，请务必安装启动与 ansible-lint 相关的插件。确保你的代码编写规范符合官方要求。 \ No newline at end of file diff --git a/inventories/opengauss/cluster_config.xml b/inventories/opengauss/cluster_config.xml index 0492b96..2acc8bb 100644 --- a/inventories/opengauss/cluster_config.xml +++ b/inventories/opengauss/cluster_config.xml @@ -2,13 +2,13 @@ - + - + @@ -25,16 +25,16 @@ - + - - + + - + @@ -51,6 +51,21 @@ + + + + + + + + + + + + + + + @@ -66,6 +81,18 @@ + + + + + + + + + + + + diff --git a/inventories/opengauss/group_vars/opengauss.yml b/inventories/opengauss/group_vars/opengauss.yml index 9193dc2..036acd3 100644 --- a/inventories/opengauss/group_vars/opengauss.yml +++ b/inventories/opengauss/group_vars/opengauss.yml @@ -2,4 +2,4 @@ opengauss_version: 5.0.0 -global_pkg_mirror: https://mirrors.aliyun.com +# global_pkg_mirror: https://mirrors.aliyun.com diff --git a/inventories/opengauss/hosts.ini b/inventories/opengauss/hosts.ini index cafc2f9..a88f118 100644 --- a/inventories/opengauss/hosts.ini +++ b/inventories/opengauss/hosts.ini @@ -6,14 +6,14 @@ [opengauss_follower] ; 192.168.56.13 192.168.56.15 -; 192.168.56.19 +192.168.56.19 ; 级联机，可设置若干个或留空。 [opengauss_cascade] ; 192.168.56.11 ; 192.168.56.12 ; 192.168.56.14 -; 192.168.56.16 +192.168.56.16 192.168.56.17 ; 以上 3 个分组的合并组，勿动。 diff --git a/roles/opengauss/tasks/deploy/cluster_check.yml b/roles/opengauss/tasks/deploy/cluster_check.yml index fc22190..35fb681 100644 --- a/roles/opengauss/tasks/deploy/cluster_check.yml +++ b/roles/opengauss/tasks/deploy/cluster_check.yml @@ -20,6 +20,29 @@ delegate_to: "{{ og_master }}" run_once: true + - name: Repeat tasks + ansible.builtin.include_tasks: + file: cluster_check.yml + +- name: Mission aborted + run_once: true + when: "cluster_detail.stdout | regex_search(keywords, multiline=True, ignorecase=True)" + vars: + keywords: >- + "(repair|down)" + block: + - name: Print cluster status + ansible.builtin.debug: + msg: | + {{ cluster_detail.stdout_lines }} + + - name: Abort tasks when conditions matched + ansible.builtin.fail: + msg: | + 集群状态有异常，请检查并手动修复后，再运行 playbook。 + Cluster status is not NORMAL, pls fix them + and run the playbook again. + - name: Tasks for expansion when: "cluster_detail is succeeded" block: diff --git a/roles/opengauss/tasks/deploy/cluster_manager.yml b/roles/opengauss/tasks/deploy/cluster_manager.yml index e49f2b8..f62edc0 100644 --- a/roles/opengauss/tasks/deploy/cluster_manager.yml +++ b/roles/opengauss/tasks/deploy/cluster_manager.yml @@ -14,9 +14,9 @@ ansible.builtin.add_host: hostname: >- {%- if og_cluster_config.data_port in line -%} - {{ node_info[4] }}_{{ node_info[1] }} + {{ node_info[4] }}_{{ node_info[1] }}_{{ node_info[7] }} {%- else -%} - {{ node_info[3] }}_{{ node_info[1] }} + {{ node_info[3] }}_{{ node_info[1] }}_{{ node_info[6] }} {%- endif -%} groups: - opengauss_cm @@ -29,16 +29,6 @@ vars: node_info: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" - # - name: Create initial cmserver list - # ansible.builtin.add_host: - # hostname: "{{ hostvars[node]['node_ip'] }}" - # groups: - # - opengauss_cm - # loop: "{{ groups['opengauss_dn'] }}" - # loop_control: - # loop_var: node - # label: "{{ hostvars[node]['node_ip'] }}" - - name: "Import packages upload tasks" ansible.builtin.import_tasks: file: upload.yml @@ -85,7 +75,7 @@ delegate_to: "{{ item }}" # 这里创建一个临时的用于安装 CM 的 cluster_config.xml - - name: "Create cluster_config.xml under {{ og_upload_path }}" + - name: "Create /tmp/cluster_config.xml" ansible.builtin.template: src: "{{ item }}" dest: "/tmp/cluster_config.xml" @@ -156,14 +146,44 @@ file: cluster_manager.yml always: - - name: Debug - ansible.builtin.debug: - var: cm_install + # - name: Debug + # ansible.builtin.debug: + # var: cm_install + + - name: Start the primary data node + ansible.builtin.command: + "gs_om -t start -h {{ hostvars[node]['node_name'] }}" + loop: "{{ groups['opengauss_dn'] }}" + loop_control: + loop_var: node + when: "'Primary' in node" + changed_when: false + + - name: Wait for primary node started + ansible.builtin.wait_for: + host: "{{ hostvars[node]['node_ip'] }}" + port: "{{ og_cluster_config.cm_port }}" + timeout: "600" + loop: "{{ groups['opengauss_dn'] }}" + loop_control: + loop_var: node + when: "'Primary' in node" + changed_when: false + + - name: Start the rest of data nodes + ansible.builtin.command: + "gs_om -t start -h {{ hostvars[node]['node_name'] }}" + loop: "{{ groups['opengauss_dn'] }}" + loop_control: + loop_var: node + when: "'Primary' not in node" + changed_when: false - - name: Start cluster + - name: Wait for the cluster started ansible.builtin.command: - gs_om -t restart - with_items: - - "cm_ctl start" - - "gs_om -t start" + "gs_om -t status" + register: gs_status + until: "'Normal' in gs_status.stdout" + retries: 30 + delay: 10 changed_when: false diff --git a/roles/opengauss/tasks/deploy/install.yml b/roles/opengauss/tasks/deploy/install.yml index 6d0c34f..7c14193 100644 --- a/roles/opengauss/tasks/deploy/install.yml +++ b/roles/opengauss/tasks/deploy/install.yml @@ -21,18 +21,29 @@ mode: "0644" backup: true -- name: Cluster expanding - ansible.builtin.shell: >- - . /home/{{ og_user }}/.bashrc && \ - {{ og_upload_path }}/script/gs_expansion \ - -U {{ og_user }} \ - -G {{ og_group }} \ - -X {{ og_upload_path }}/cluster_config.xml \ - -h {{ groups['opengauss_expand'] | join(',') }} \ - --time-out {{ (og_all_nodes | count) * 600 }} +- name: Start expansion become_user: root - changed_when: false when: "og_expansion" + block: + - name: Starting pre install + ansible.builtin.command: >- + {{ og_upload_path }}/script/gs_preinstall \ + -U {{ og_user }} \ + -G {{ og_group }} \ + -X {{ og_upload_path }}/cluster_config.xml \ + --non-interactive + changed_when: false + + - name: Cluster expanding + ansible.builtin.shell: >- + . /home/{{ og_user }}/.bashrc && \ + {{ og_upload_path }}/script/gs_expansion \ + -U {{ og_user }} \ + -G {{ og_group }} \ + -X {{ og_upload_path }}/cluster_config.xml \ + -h {{ groups['opengauss_expand'] | join(',') }} \ + --time-out {{ (og_all_nodes | count) * 600 }} + changed_when: false - name: Starting deploy when: "not og_expansion" @@ -43,7 +54,6 @@ -U {{ og_user }} \ -G {{ og_group }} \ -X {{ og_upload_path }}/cluster_config.xml \ - --skip-hostname-set \ --non-interactive changed_when: false diff --git a/roles/opengauss/tasks/deploy/runtime_groups.yml b/roles/opengauss/tasks/deploy/runtime_groups.yml index b3b0819..838b02e 100644 --- a/roles/opengauss/tasks/deploy/runtime_groups.yml +++ b/roles/opengauss/tasks/deploy/runtime_groups.yml @@ -2,18 +2,18 @@ # ansible.builtin.debug: # msg: "{{ groups['opengauss_expand'] }}" -- ansible.builtin.debug: - var: cluster_detail.stdout_lines +# - ansible.builtin.debug: +# var: cluster_detail.stdout_lines -- name: Debug - ansible.builtin.debug: - msg: "{{ split_line }}" - loop: "{{ cluster_detail.stdout_lines | sort }}" - loop_control: - loop_var: line - when: "'cmserver' in line" - vars: - split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" +# - name: Debug +# ansible.builtin.debug: +# msg: "{{ split_line }}" +# loop: "{{ cluster_detail.stdout_lines | sort }}" +# loop_control: +# loop_var: line +# when: "'cmserver' in line" +# vars: +# split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" - name: Create current cmserver list ansible.builtin.add_host: @@ -55,15 +55,15 @@ {%- endif -%} when: "groups['opengauss_cm'] is defined" -- name: Debug - ansible.builtin.debug: - msg: "{{ split_line }}" - loop: "{{ cluster_detail.stdout_lines | sort }}" - loop_control: - loop_var: line - when: "(og_data_path + '/dn') in line" - vars: - split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" +# - name: Debug +# ansible.builtin.debug: +# msg: "{{ split_line }}" +# loop: "{{ cluster_detail.stdout_lines | sort }}" +# loop_control: +# loop_var: line +# when: "(og_data_path + '/dn') in line" +# vars: +# split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" # 这里需要判断输出了多少列，因为有 CM 的集群里，不输出数据库端口的那一列。 - name: Create current data node list diff --git a/roles/opengauss/tasks/os/CentOS.yml b/roles/opengauss/tasks/os/CentOS.yml index b133805..e78a1f0 100644 --- a/roles/opengauss/tasks/os/CentOS.yml +++ b/roles/opengauss/tasks/os/CentOS.yml @@ -2,6 +2,9 @@ ansible.builtin.package: name: epel-release use: "{{ custom_pkg_mgr | default(ansible_pkg_mgr) }}" + register: pkg_inst + until: pkg_inst is succeeded + retries: 3 - name: Install dependencies ansible.builtin.package: @@ -9,3 +12,6 @@ - python36 - redhat-lsb-core use: "{{ custom_pkg_mgr | default(ansible_pkg_mgr) }}" + register: pkg_inst + until: pkg_inst is succeeded + retries: 3 diff --git a/roles/opengauss/tasks/os/common_set.yml b/roles/opengauss/tasks/os/common_set.yml index c6aeb3f..c5b13f1 100644 --- a/roles/opengauss/tasks/os/common_set.yml +++ b/roles/opengauss/tasks/os/common_set.yml @@ -59,8 +59,11 @@ - nano - htop - "{{ python_name }}-pexpect" - update_cache: true + # update_cache: true use: "{{ custom_pkg_mgr | default(ansible_pkg_mgr) }}" + register: pkg_inst + until: pkg_inst is succeeded + retries: 3 - name: Enable ntpd service ansible.builtin.service: diff --git a/roles/opengauss/tasks/os/openEuler.yml b/roles/opengauss/tasks/os/openEuler.yml index f0ecedb..4637580 100644 --- a/roles/opengauss/tasks/os/openEuler.yml +++ b/roles/opengauss/tasks/os/openEuler.yml @@ -4,6 +4,9 @@ - libnsl - readline-devel use: "{{ custom_pkg_mgr | default(ansible_pkg_mgr) }}" + register: pkg_inst + until: pkg_inst is succeeded + retries: 3 - name: Create soft link ansible.builtin.file: diff --git a/roles/pre_tasks/tasks/os/openEuler.yml b/roles/pre_tasks/tasks/os/openEuler.yml index 2ad5be8..43bbd8a 100644 --- a/roles/pre_tasks/tasks/os/openEuler.yml +++ b/roles/pre_tasks/tasks/os/openEuler.yml @@ -1,6 +1,6 @@ - name: "Replace sources mirror with {{ global_pkg_mirror }}" ansible.builtin.replace: path: "/etc/yum.repos.d/openEuler.repo" - regexp: "http(s)://repo.openeuler.org" + regexp: "http(s|)://repo.openeuler.org" replace: "{{ global_pkg_mirror }}/openeuler" when: "global_pkg_mirror is defined" -- Gitee From f46ead11d6319c257ca20686af833edf4eae09bc Mon Sep 17 00:00:00 2001 From: Haibin Lee Date: Fri, 10 Nov 2023 22:13:21 +0800 Subject: [PATCH 4/4] =?UTF-8?q?REV:=20=E7=BB=88=E4=BA=8E=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E5=8F=AF=E4=BB=A5=E7=9B=B4=E6=8E=A5=E4=BB=8E=E5=8D=95=E7=82=B9?= =?UTF-8?q?=E6=89=A9=E5=AE=B9=E5=88=B0=E5=A4=9A=E8=8A=82=E7=82=B9=E5=95=A6?= =?UTF-8?q?=EF=BC=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- README.md | 14 +- docs/03-deploy.md | 2 + docs/04-expansion.md | 4 +- docs/99-get-involved.md | 5 +- inventories/opengauss/cluster_config.xml | 68 +++---- inventories/opengauss/hosts.ini | 14 +- roles/opengauss/README.md | 2 +- roles/opengauss/defaults/main.yml | 2 +- .../opengauss/tasks/deploy/cluster_check.yml | 88 +++----- .../opengauss/tasks/deploy/cluster_expand.yml | 54 +++-- .../tasks/deploy/cluster_manager.yml | 191 +++++++++--------- roles/opengauss/tasks/deploy/install.yml | 4 +- roles/opengauss/tasks/deploy/known_hosts.yml | 26 ++- roles/opengauss/tasks/deploy/main.yml | 24 ++- .../opengauss/tasks/deploy/runtime_groups.yml | 189 +++++++++-------- roles/opengauss/tasks/main.yml | 59 +++++- roles/opengauss/tasks/os/common_set.yml | 14 -- roles/opengauss/tasks/os/user.yml | 2 +- roles/opengauss/tasks/pre_tasks.yml | 2 +- roles/opengauss/tasks/runtime_facts.yml | 22 +- .../opengauss/templates/cluster_config.xml.j2 | 26 ++- .../opengauss/templates/cluster_master.xml.j2 | 50 +++-- .../templates/cluster_replicas.xml.j2 | 35 +++- roles/opengauss/templates/report.md.j2 | 2 +- 25 files changed, 511 insertions(+), 391 deletions(-) diff --git a/.gitignore b/.gitignore index 8a24b2c..d3bc778 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ vagrant/.vagrant vagrant/packer_cache vagrant/output* log -test.yml \ No newline at end of file +test.yml +inventories/opengauss/cluster_config.xml diff --git a/README.md b/README.md index e56e86b..5d676c1 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ 本项目适用于以下场景或用户： -1. 同一网络环境下的快速部署。 +1. 同一网络环境下的快速部署以及扩容。 1. 需要多次重新部署数据库集群的测试工程师。 # 已适配的系统 @@ -28,6 +28,8 @@ # 优势特点 +1. 自带 Dockerfile，可通过 docker-compose 在本地启动一个 Ansible 容器，免除不同系统安装 Ansible 所带来的兼容性问题。 +1. 部署完成后自动生成部署报告，markdown 格式。 1. 以 1 主 4 备 1 级联的架构为例，十分钟内部署完成（不含 openGauss 压缩包及 Linux 系统安装包的下载时间）。 1. 支持从 1 主单节点，逐步扩展为多节点集群。 1. 自动匹配 CPU 架构以及操作系统。 @@ -71,8 +73,7 @@ # 更新日志 -* 2023-10-13: 支持集群扩容。 -* 2023-10-18: 支持 openEuler 20.03 LTS SP3。 +* 2023-11-10：优化流程，解决扩容难点，可直接从单节点扩容到多节点。 * 2023-10-26: 大量修复和优化，可完美实现从单点部署，到逐步扩容为 1 主 4 备 4 级联的架构。 ``` 已测试的扩容场景如下 @@ -83,6 +84,13 @@ -> 1 主 2 备 2 级联 -> 1 主 4 备 4 级联 ``` +* 2023-10-18: 支持 openEuler 20.03 LTS SP3。 +* 2023-10-13: 支持集群扩容。 + +# 待开发功能 + +1. 提供 DCF 模式的部署。 +1. 基于 gs_guc 批量配置自动修改功能。 # 开发指南 diff --git a/docs/03-deploy.md b/docs/03-deploy.md index 764b302..665c987 100644 --- a/docs/03-deploy.md +++ b/docs/03-deploy.md @@ -10,6 +10,8 @@ 关于 byobu 的使用，大家可自行搜索一些教程。在 byobu 里运行的程序或命令，不会因为 SSH 断开而终止。 + 也可以很方便地在同一个 SSH 连接里，切换不同的窗口。 + 基本键盘操作有： ``` diff --git a/docs/04-expansion.md b/docs/04-expansion.md index 8a390f1..2057ab8 100644 --- a/docs/04-expansion.md +++ b/docs/04-expansion.md @@ -12,9 +12,9 @@ ### 请注意： -### 如果是对 1 主 1 备进行扩容，会增加一些部署 Cluster Manager 的流程，期间需要停止已有的数据节点。请确保扩容前没有数据库读写操作。 +### 如果是对 1 主 1 备进行扩容，会增加一些部署 Cluster Manager 的流程，请确保扩容前没有数据库读写操作。 -### 如果是从 1 主单节点开始扩容，请先增加 1 个备节点，扩容为 1 主 1 备的架构后，再进行多节点同时扩容。 +### 如果是从 1 主单节点开始扩容，建议先增加 1 个备节点，扩容为 1 主 1 备的架构后，再进行多节点同时扩容。 假设原编排为 1 主 1 备。 diff --git a/docs/99-get-involved.md b/docs/99-get-involved.md index b1cfd0e..2f291ef 100644 --- a/docs/99-get-involved.md +++ b/docs/99-get-involved.md @@ -3,4 +3,7 @@ ## 前提条件 1. 需要你非常熟悉 [Ansible](https://docs.ansible.com/ansible/latest/collections/index.html) 的模块文档，一切编写以官方文档为准。 -1. 不论你使用哪款 IDE 编辑器，请务必安装启动与 ansible-lint 相关的插件。确保你的代码编写规范符合官方要求。 \ No newline at end of file +1. 不论你使用哪款 IDE 编辑器，请务必安装启动与 ansible-lint 相关的插件。确保你的代码编写规范符合官方要求。 + +## 规范要求 +1. 每个 task 的 name，相当于作用解释，请务必清晰表达。 \ No newline at end of file diff --git a/inventories/opengauss/cluster_config.xml b/inventories/opengauss/cluster_config.xml index 2acc8bb..ad3066f 100644 --- a/inventories/opengauss/cluster_config.xml +++ b/inventories/opengauss/cluster_config.xml @@ -2,13 +2,19 @@ - - + + @@ -25,20 +31,29 @@ - - - - + + - - - - + + + + - @@ -49,36 +64,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -91,8 +77,10 @@ + + diff --git a/inventories/opengauss/hosts.ini b/inventories/opengauss/hosts.ini index a88f118..5353de2 100644 --- a/inventories/opengauss/hosts.ini +++ b/inventories/opengauss/hosts.ini @@ -12,9 +12,9 @@ [opengauss_cascade] ; 192.168.56.11 ; 192.168.56.12 -; 192.168.56.14 +192.168.56.14 192.168.56.16 -192.168.56.17 +; 192.168.56.17 ; 以上 3 个分组的合并组，勿动。 [opengauss:children] @@ -22,16 +22,10 @@ opengauss_master opengauss_follower opengauss_cascade -; 同步复制节点分组 -[opengauss_sync_nodes:children] -opengauss_master -opengauss_follower - -; ; 备节点分组，总数不可大于 8。 -[opengauss_replica_nodes:children] -opengauss_cascade +[opengauss_replicas:children] opengauss_follower +opengauss_cascade ; 机器的 SSH 信息，请根据你的实际情况修改。 [opengauss:vars] diff --git a/roles/opengauss/README.md b/roles/opengauss/README.md index 660ff95..17ab665 100644 --- a/roles/opengauss/README.md +++ b/roles/opengauss/README.md @@ -2,7 +2,7 @@ 自动部署 openGauss，根据分组定义，自适应部署单点、一主一从、一主多从以及级联节点等架构模式。 -目前仅支持初次部署，不支持对集群架构做变更。 +目前仅支持初次部署以及扩容，暂不支持对集群角色做变更。 # 优势特点 diff --git a/roles/opengauss/defaults/main.yml b/roles/opengauss/defaults/main.yml index 72a6e53..8eb539e 100644 --- a/roles/opengauss/defaults/main.yml +++ b/roles/opengauss/defaults/main.yml @@ -95,7 +95,7 @@ opengauss_download: og_hostname: "og{{ inventory_hostname | ipaddr('int') }}" og_all_nodes: "{{ groups['opengauss'] | sort }}" og_master: "{{ groups['opengauss_master'] | first }}" -og_replicas: "{{ groups['opengauss_replica_nodes'] | sort }}" +og_replicas: "{{ groups['opengauss_replicas'] | sort }}" og_upload_path: "/opt/software/openGauss" og_cm_enabled: "{{ (groups['opengauss'] | count) > 2 }}" og_expansion: false diff --git a/roles/opengauss/tasks/deploy/cluster_check.yml b/roles/opengauss/tasks/deploy/cluster_check.yml index 35fb681..50c7f0e 100644 --- a/roles/opengauss/tasks/deploy/cluster_check.yml +++ b/roles/opengauss/tasks/deploy/cluster_check.yml @@ -3,91 +3,63 @@ - name: "Query cluster detail" ansible.builtin.command: "gs_om -t status --detail" changed_when: false - register: cluster_detail + register: gs_detail become_user: "{{ og_user }}" delegate_to: "{{ og_master }}" run_once: true + - name: Import tasks to create runtime groups + ansible.builtin.import_tasks: + file: deploy/runtime_groups.yml + run_once: true + rescue: - name: Import pre tasks ansible.builtin.import_tasks: file: pre_tasks.yml become_user: root + - name: Import runtime tasks + ansible.builtin.import_tasks: + file: deploy/runtime_groups.yml + - name: Import deploy tasks ansible.builtin.import_tasks: file: deploy/main.yml delegate_to: "{{ og_master }}" run_once: true + vars: + cm_nodes: "{{ (groups['opengauss_cm'] is defined) | ternary(groups['opengauss_cm'], '') | sort }}" + dn_nodes: "{{ (groups['opengauss_dn'] is defined) | ternary(groups['opengauss_dn'], '') | sort }}" + ep_nodes: "{{ (groups['opengauss_ep'] is defined) | ternary(groups['opengauss_ep'], '') | sort }}" - name: Repeat tasks ansible.builtin.include_tasks: - file: cluster_check.yml + file: deploy/cluster_check.yml - name: Mission aborted - run_once: true - when: "cluster_detail.stdout | regex_search(keywords, multiline=True, ignorecase=True)" + when: + - "gs_detail is succeeded" + - "gs_detail.stdout | regex_search(keywords, multiline=True, ignorecase=True)" vars: keywords: >- "(repair|down)" + run_once: true block: - - name: Print cluster status - ansible.builtin.debug: - msg: | - {{ cluster_detail.stdout_lines }} - - - name: Abort tasks when conditions matched + - name: Abort tasks when cluster is unstable ansible.builtin.fail: msg: | 集群状态有异常，请检查并手动修复后，再运行 playbook。 - Cluster status is not NORMAL, pls fix them + Cluster status is not NORMAL, pls fix it and run the playbook again. + --------------------------------------------- + {{ gs_detail.stdout_lines }} -- name: Tasks for expansion - when: "cluster_detail is succeeded" - block: - - name: Create expansion list - ansible.builtin.add_host: - hostname: "{{ item }}" - groups: - - opengauss_expand - loop: "{{ og_replicas }}" - when: "item not in cluster_detail.stdout" - run_once: true - - - name: Import pre tasks - ansible.builtin.include_tasks: - file: pre_tasks.yml - when: - - "groups['opengauss_expand'] is defined" - - "inventory_hostname in groups['opengauss_expand']" - - - name: Tasks for CM cluster - when: - - "'cmserver' not in cluster_detail.stdout" - - "og_cm_enabled" - block: - - name: Import cluster manager tasks - ansible.builtin.import_tasks: - file: cluster_manager.yml - delegate_to: "{{ og_master }}" - become_user: "{{ og_user }}" - run_once: true - - - name: Refresh cluster status - ansible.builtin.command: - gs_om -t status --detail - changed_when: false - register: cluster_detail1 - become_user: "{{ og_user }}" - delegate_to: "{{ og_master }}" - run_once: true - - - name: Import cluster expand tasks + always: + - name: Import post tasks ansible.builtin.import_tasks: - file: cluster_expand.yml - vars: - og_expansion: true - delegate_to: "{{ og_master }}" - run_once: true - when: "groups['opengauss_expand'] is defined" + file: post_tasks.yml + + - name: End play + ansible.builtin.meta: + end_play diff --git a/roles/opengauss/tasks/deploy/cluster_expand.yml b/roles/opengauss/tasks/deploy/cluster_expand.yml index 135b633..a41f0ae 100644 --- a/roles/opengauss/tasks/deploy/cluster_expand.yml +++ b/roles/opengauss/tasks/deploy/cluster_expand.yml @@ -1,25 +1,35 @@ -- name: Import runtime groups tasks +- name: Import pre tasks + ansible.builtin.include_tasks: + file: pre_tasks.yml + when: "inventory_hostname in ep_nodes" + +- name: Import deploy tasks ansible.builtin.import_tasks: - file: runtime_groups.yml + file: deploy/main.yml + delegate_to: "{{ og_master }}" + run_once: true + +# # 3 节点或以上，且未部署 CM 的情况。 +# - name: Tasks for CM cluster +# when: +# - "(cm_nodes | count) < 2" +# - "og_cm_enabled" +# run_once: true +# block: +# - name: Import cluster manager tasks +# ansible.builtin.import_tasks: +# file: cluster_manager.yml +# delegate_to: "{{ og_master }}" +# become_user: "{{ og_user }}" -- name: Cluster will be expand - when: - - "og_expansion" - block: - - name: Update /etc/hosts - ansible.builtin.blockinfile: - path: /etc/hosts - marker: "# {mark} OPENGAUSS NODES" - block: | - {% for node in og_all_nodes %} - {{ node }} og{{ node | ipaddr('int') }} og-{{ node | replace('.', '-') }} - {% endfor %} - delegate_to: "{{ node }}" - loop: "{{ og_all_nodes }}" - loop_control: - loop_var: node +# - name: Refresh cluster status +# ansible.builtin.command: +# gs_om -t status --detail +# changed_when: false +# register: gs_detail1 +# become_user: "{{ og_user }}" +# delegate_to: "{{ og_master }}" - - name: Import deploy tasks - ansible.builtin.import_tasks: - file: deploy/main.yml - delegate_to: "{{ og_master }}" +# - name: Import runtime groups tasks +# ansible.builtin.import_tasks: +# file: deploy/runtime_groups.yml diff --git a/roles/opengauss/tasks/deploy/cluster_manager.yml b/roles/opengauss/tasks/deploy/cluster_manager.yml index f62edc0..b706640 100644 --- a/roles/opengauss/tasks/deploy/cluster_manager.yml +++ b/roles/opengauss/tasks/deploy/cluster_manager.yml @@ -1,38 +1,10 @@ -- name: Import runtime groups tasks - ansible.builtin.import_tasks: - file: runtime_groups.yml - - name: Deploy CM into existing data cluster block: - - name: Stop cluster - ansible.builtin.command: - gs_om -t stop - changed_when: false - - # 把现有的数据节点加到 'opengauss_cm' 组 - - name: Create initial cmserver list - ansible.builtin.add_host: - hostname: >- - {%- if og_cluster_config.data_port in line -%} - {{ node_info[4] }}_{{ node_info[1] }}_{{ node_info[7] }} - {%- else -%} - {{ node_info[3] }}_{{ node_info[1] }}_{{ node_info[6] }} - {%- endif -%} - groups: - - opengauss_cm - node_ip: "{{ node_info[2] }}" - node_name: "{{ node_info[1] }}" - loop: "{{ cluster_detail.stdout_lines | sort }}" - loop_control: - loop_var: line - when: "(og_data_path + '/dn') in line" - vars: - node_info: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" - - - name: "Import packages upload tasks" + - name: Import upload tasks ansible.builtin.import_tasks: file: upload.yml become_user: root + # run_once: true - name: Get cm package name ansible.builtin.set_fact: @@ -42,6 +14,7 @@ loop_var: file when: - "'cm.tar.gz' in file" + # run_once: true - name: "Change file permission of {{ cm_pkg }}" ansible.builtin.file: @@ -50,6 +23,7 @@ group: "{{ og_group }}" mode: "0640" become_user: root + # run_once: true - name: Create some paths ansible.builtin.file: @@ -74,55 +48,89 @@ loop: "{{ og_all_nodes }}" delegate_to: "{{ item }}" - # 这里创建一个临时的用于安装 CM 的 cluster_config.xml - - name: "Create /tmp/cluster_config.xml" + # 这里需要建一个假的定时任务，包含 'om_monitor'。后续 cm_install 时会检查，否则失败。 + - name: Fake an om_monitor cron job before install + ansible.builtin.cron: + name: openGauss om monitor + special_time: yearly + job: "{{ og_home }}/install/app/bin/om_monitor" + loop: "{{ dn_nodes }}" + loop_control: + loop_var: node + label: "{{ hostvars[node]['node_ip'] }}" + delegate_to: "{{ hostvars[node]['node_ip'] }}" + + - name: "Create cluster_config.xml in local path '{{ inventory_dir }}'" ansible.builtin.template: src: "{{ item }}" - dest: "/tmp/cluster_config.xml" + dest: "{{ inventory_dir }}/cluster_config.xml" owner: "{{ og_user }}" group: "{{ og_group }}" mode: "0644" - backup: true lstrip_blocks: true with_first_found: - "{{ inventory_dir }}/templates/cluster_config.xml.j2" - "cluster_config.xml.j2" - vars: - og_expand_cm_ips: >- - {%- for node in groups['opengauss_cm'] -%} - {{ hostvars[node]['node_ip'] }}{{ (loop.nextitem is defined) | ternary(',', '') }} - {%- endfor -%} - og_cm_names: >- - {%- for node in groups['opengauss_cm'] -%} - {{ hostvars[node]['node_name'] }}{{ (loop.nextitem is defined) | ternary(',', '') }} - {%- endfor -%} - temp_data_nodes: >- - {{ og_data_path }}/dn - {%- for node in groups['opengauss_cm'] if hostvars[node]['node_ip'] != og_master -%} - ,{{ hostvars[node]['node_name'] }},{{ og_data_path }}/dn - {%- endfor -%} - og_replicas: >- - [' - {%- for node in groups['opengauss_cm'] if hostvars[node]['node_ip'] != og_master -%} - {{ hostvars[node]['node_ip'] }}{{ (loop.nextitem is defined) | ternary("','", "") }} - {%- endfor -%} - '] + delegate_to: localhost + become: false + # run_once: true + + - name: "Upload cluster_config.xml to {{ og_upload_path }}" + ansible.builtin.copy: + src: "{{ inventory_dir }}/cluster_config.xml" + dest: "{{ og_upload_path }}/" + owner: "{{ og_user }}" + group: "{{ og_group }}" + mode: "0644" + backup: true - # 这里需要建一个假的定时任务，包含 'om_monitor'。后续 cm_install 时会检查，否则失败。 - - name: Fake an om_monitor cron job before install - ansible.builtin.cron: - name: openGauss om monitor - special_time: yearly - job: "{{ og_home }}/install/app/bin/om_monitor" - loop: "{{ groups['opengauss_cm'] }}" + - name: Step 1 | Switchover once to avoid 'Term of primary is invalid or not maximal' error + ansible.builtin.command: + gs_ctl switchover -D {{ og_data_path }}/dn + changed_when: false + loop: "{{ dn_nodes }}" + loop_control: + loop_var: node + delegate_to: "{{ hostvars[node]['node_ip'] }}" + when: "'Standby' in node" + register: switchover_status + until: "switchover_status is succeeded" + retries: 9 + delay: 5 + + # - name: Select a standby node and the primary node + # ansible.builtin.set_fact: + # standby_nodes: >- + # {% for node in dn_nodes if 'Standby' in node %} + # {{ hostvars[node]['node_ip'] }} + # {% endfor %} + # primary_nodes: >- + # {% for node in dn_nodes if 'Primary' in node %} + # {{ hostvars[node]['node_ip'] }} + # {% endfor %} + + - name: "Step 2 | Switchover back to {{ og_master }}" + ansible.builtin.command: + gs_ctl switchover -D {{ og_data_path }}/dn + changed_when: false + loop: "{{ dn_nodes }}" loop_control: loop_var: node - label: "{{ hostvars[node]['node_ip'] }}" delegate_to: "{{ hostvars[node]['node_ip'] }}" + when: "'Primary' in node" + register: switchover_status + until: "switchover_status is succeeded" + retries: 9 + delay: 5 + + # - name: Stop all nodes + # ansible.builtin.command: + # gs_om -t stop + # changed_when: false - name: Install CM ansible.builtin.expect: - command: "./cm_install -X /tmp/cluster_config.xml --cmpkg {{ og_upload_path }}/{{ cm_pkg }}" + command: "./cm_install -X {{ og_upload_path }}/cluster_config.xml --cmpkg {{ og_upload_path }}/{{ cm_pkg }}" responses: (?i)password: "{{ og_ca_pass }}" chdir: "{{ og_home }}/install/app/tool/cm_tool" @@ -134,6 +142,7 @@ retries: 3 timeout: 300 ignore_errors: true + become_user: "{{ og_user }}" rescue: - name: Install pexpect @@ -150,34 +159,34 @@ # ansible.builtin.debug: # var: cm_install - - name: Start the primary data node - ansible.builtin.command: - "gs_om -t start -h {{ hostvars[node]['node_name'] }}" - loop: "{{ groups['opengauss_dn'] }}" - loop_control: - loop_var: node - when: "'Primary' in node" - changed_when: false - - - name: Wait for primary node started - ansible.builtin.wait_for: - host: "{{ hostvars[node]['node_ip'] }}" - port: "{{ og_cluster_config.cm_port }}" - timeout: "600" - loop: "{{ groups['opengauss_dn'] }}" - loop_control: - loop_var: node - when: "'Primary' in node" - changed_when: false - - - name: Start the rest of data nodes - ansible.builtin.command: - "gs_om -t start -h {{ hostvars[node]['node_name'] }}" - loop: "{{ groups['opengauss_dn'] }}" - loop_control: - loop_var: node - when: "'Primary' not in node" - changed_when: false + # - name: Start the primary data node + # ansible.builtin.command: + # "gs_om -t start -h {{ hostvars[node]['node_name'] }}" + # loop: "{{ dn_nodes }}" + # loop_control: + # loop_var: node + # when: "'Primary' in node" + # changed_when: false + + # - name: Wait for primary node started + # ansible.builtin.wait_for: + # host: "{{ hostvars[node]['node_ip'] }}" + # port: "{{ og_cluster_config.db_port }}" + # timeout: "600" + # loop: "{{ dn_nodes }}" + # loop_control: + # loop_var: node + # when: "'Primary' in node" + # changed_when: false + + # - name: Start the rest of data nodes + # ansible.builtin.command: + # "gs_om -t start -h {{ hostvars[node]['node_name'] }}" + # loop: "{{ dn_nodes }}" + # loop_control: + # loop_var: node + # when: "'Primary' not in node" + # changed_when: false - name: Wait for the cluster started ansible.builtin.command: diff --git a/roles/opengauss/tasks/deploy/install.yml b/roles/opengauss/tasks/deploy/install.yml index 7c14193..5bcce4e 100644 --- a/roles/opengauss/tasks/deploy/install.yml +++ b/roles/opengauss/tasks/deploy/install.yml @@ -1,4 +1,4 @@ -- name: "Create {{ inventory_dir }}/cluster_config.xml in local" +- name: "Create cluster_config.xml in local path '{{ inventory_dir }}'" ansible.builtin.template: src: "{{ item }}" dest: "{{ inventory_dir }}/cluster_config.xml" @@ -41,7 +41,7 @@ -U {{ og_user }} \ -G {{ og_group }} \ -X {{ og_upload_path }}/cluster_config.xml \ - -h {{ groups['opengauss_expand'] | join(',') }} \ + -h {{ groups['opengauss_ep'] | join(',') }} \ --time-out {{ (og_all_nodes | count) * 600 }} changed_when: false diff --git a/roles/opengauss/tasks/deploy/known_hosts.yml b/roles/opengauss/tasks/deploy/known_hosts.yml index 286b335..21e04b2 100644 --- a/roles/opengauss/tasks/deploy/known_hosts.yml +++ b/roles/opengauss/tasks/deploy/known_hosts.yml @@ -1,16 +1,30 @@ -- name: "Update .ssh/known_hosts" +- name: "Update /root/.ssh/known_hosts" ansible.builtin.blockinfile: - path: "{{ og_ssh.home }}/.ssh/known_hosts" - owner: "{{ og_ssh.user }}" - group: "{{ og_ssh.group }}" + path: "/root/.ssh/known_hosts" + owner: "root" + group: "root" + create: true + mode: "0644" + block: | + {% for key in (host_keys | sort) %} + {{ key }} + {% endfor %} + loop: "{{ og_all_nodes }}" + loop_control: + loop_var: node + delegate_to: "{{ node }}" + +- name: "Update ~/.ssh/known_hosts for user '{{ og_user }}'" + ansible.builtin.blockinfile: + path: "/home/{{ og_user }}/.ssh/known_hosts" + owner: "{{ og_user }}" + group: "{{ og_group }}" create: true mode: "0644" block: | {% for key in (host_keys | sort) %} {{ key }} {% endfor %} - vars: - host_keys: "{{ known_host_keys.results | map(attribute='stdout_lines') | flatten }}" loop: "{{ og_all_nodes }}" loop_control: loop_var: node diff --git a/roles/opengauss/tasks/deploy/main.yml b/roles/opengauss/tasks/deploy/main.yml index c58adbf..8346313 100644 --- a/roles/opengauss/tasks/deploy/main.yml +++ b/roles/opengauss/tasks/deploy/main.yml @@ -1,3 +1,16 @@ +- name: Update /etc/hosts + ansible.builtin.blockinfile: + path: /etc/hosts + marker: "# {mark} OPENGAUSS NODES" + block: | + {% for node in og_all_nodes %} + {{ node }} og{{ node | ipaddr('int') }} + {% endfor %} + delegate_to: "{{ node }}" + loop: "{{ og_all_nodes }}" + loop_control: + loop_var: node + - name: Scan hosts key ansible.builtin.command: >- ssh-keyscan -p {{ host_port }} {{ node }},og{{ node | ipaddr('int') }} @@ -12,15 +25,8 @@ - name: Config known hosts ansible.builtin.include_tasks: file: deploy/known_hosts.yml - with_items: - - user: root - group: root - home: /root - - user: "{{ og_user }}" - group: "{{ og_group }}" - home: "/home/{{ og_user }}" - loop_control: - loop_var: og_ssh + vars: + host_keys: "{{ known_host_keys.results | map(attribute='stdout_lines') | flatten }}" - name: Config authorized keys ansible.builtin.include_tasks: diff --git a/roles/opengauss/tasks/deploy/runtime_groups.yml b/roles/opengauss/tasks/deploy/runtime_groups.yml index 838b02e..d757156 100644 --- a/roles/opengauss/tasks/deploy/runtime_groups.yml +++ b/roles/opengauss/tasks/deploy/runtime_groups.yml @@ -1,107 +1,124 @@ # - name: Expansion list # ansible.builtin.debug: -# msg: "{{ groups['opengauss_expand'] }}" +# msg: "{{ groups['opengauss_ep'] }}" # - ansible.builtin.debug: -# var: cluster_detail.stdout_lines +# var: gs_detail.stdout_lines # - name: Debug # ansible.builtin.debug: # msg: "{{ split_line }}" -# loop: "{{ cluster_detail.stdout_lines | sort }}" +# loop: "{{ gs_detail.stdout_lines | sort }}" # loop_control: # loop_var: line # when: "'cmserver' in line" # vars: # split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" -- name: Create current cmserver list - ansible.builtin.add_host: - hostname: "{{ node_info[3] }}_{{ node_info[1] }}" - groups: - - opengauss_cm - node_ip: "{{ node_info[2] }}" - loop: "{{ cluster_detail.stdout_lines | sort }}" - loop_control: - loop_var: line - vars: - node_info: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" - when: "'cmserver' in line" +# 如果已部署集群 +- name: If deployed + block: + # 不在集群里的 IP 加入到扩展节点分组。 + - name: Create group 'opengauss_ep' + ansible.builtin.add_host: + hostname: "{{ node }}" + groups: + - opengauss_ep + loop: "{{ og_replicas }}" + loop_control: + loop_var: node + when: "node not in gs_detail.stdout" -- name: Current cluster manager servers - ansible.builtin.debug: - msg: "{{ groups['opengauss_cm'] }}" - when: "groups['opengauss_cm'] is defined" + # 按编号顺序保存现有 CM 服务器列表。 + - name: Create current cmserver list + ansible.builtin.add_host: + hostname: "{{ node_info[3] }}_{{ node_info[1] }}" + groups: + - opengauss_cm + node_ip: "{{ node_info[2] }}" + loop: "{{ gs_detail.stdout_lines | sort }}" + loop_control: + loop_var: line + vars: + node_info: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" + when: "'cmserver' in line" -- name: Set 'og_expand_cm_ips' facts - ansible.builtin.set_fact: - og_expand_cm_ips: >- - {%- for node in (groups['opengauss_cm'] | sort) -%} - {{ hostvars[node]['node_ip'] }}{{ (loop.nextitem is defined) | ternary(',', '') }} - {%- endfor -%} - {%- if groups['opengauss_expand'] is defined -%} - {%- for node in (groups['opengauss_expand'] | sort) -%} - ,{{ node }} - {%- endfor -%} - {%- endif -%} - og_cm_names: >- - {%- for node in (groups['opengauss_cm'] | sort) -%} - og{{ hostvars[node]['node_ip'] | ipaddr('int') }}{{ (loop.nextitem is defined) | ternary(',', '') }} - {%- endfor -%} - {%- if groups['opengauss_expand'] is defined -%} - {%- for node in (groups['opengauss_expand'] | sort) -%} - ,og{{ node | ipaddr('int') }} - {%- endfor -%} - {%- endif -%} - when: "groups['opengauss_cm'] is defined" + - name: Print out current cluster manager servers + ansible.builtin.debug: + msg: "{{ groups['opengauss_cm'] }}" + when: "groups['opengauss_cm'] is defined" -# - name: Debug -# ansible.builtin.debug: -# msg: "{{ split_line }}" -# loop: "{{ cluster_detail.stdout_lines | sort }}" + # 按编号顺序保存现有 DN 服务器列表。 + # 这里需要判断输出了多少列，因为有 CM 的集群里，会隐藏数据库端口 Port 的那一列。 + - name: Create current data nodes list + ansible.builtin.add_host: + hostname: >- + {%- if og_cluster_config.db_port in line -%} + {{ node_info[4] }}_{{ node_info[1] }}_{{ node_info[7] }} + {%- else -%} + {{ node_info[3] }}_{{ node_info[1] }}_{{ node_info[6] }} + {%- endif -%} + groups: + - opengauss_dn + node_ip: "{{ node_info[2] }}" + node_name: "{{ node_info[1] }}" + loop: "{{ gs_detail.stdout_lines | sort }}" + loop_control: + loop_var: line + label: "{{ line }}" + when: "(og_data_path + '/dn') in line" + vars: + node_info: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" + +# # 如果未部署集群 +# - name: Create init data nodes list +# ansible.builtin.add_host: +# hostname: "{{ node }}" +# groups: +# - opengauss_dn +# node_ip: "{{ node }}" +# node_name: "og{{ node | ipaddr('int') }}" +# loop: "{{ groups['opengauss'] }}" # loop_control: -# loop_var: line -# when: "(og_data_path + '/dn') in line" -# vars: -# split_line: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" +# loop_var: node +# extended: true +# when: "gs_detail.stdout_lines is falsy" + +# - name: Current data nodes +# ansible.builtin.debug: +# msg: "{{ groups['opengauss_dn'] }}" +# when: "groups['opengauss_dn'] is defined" -# 这里需要判断输出了多少列，因为有 CM 的集群里，不输出数据库端口的那一列。 -- name: Create current data node list - ansible.builtin.add_host: - hostname: >- - {%- if og_cluster_config.data_port in line -%} - {{ node_info[4] }}_{{ node_info[1] }}_{{ node_info[7] }} - {%- else -%} - {{ node_info[3] }}_{{ node_info[1] }}_{{ node_info[6] }} - {%- endif -%} - groups: - - opengauss_dn - node_ip: "{{ node_info[2] }}" - node_name: "{{ node_info[1] }}" - loop: "{{ cluster_detail.stdout_lines | sort }}" - loop_control: - loop_var: line - when: "(og_data_path + '/dn') in line" - vars: - node_info: "{{ line | regex_replace(' {1,}', '|') | split('|') }}" +# # 如果未部署 CM 且为单节点 +# - name: Create groups for CM deploy | Single node mode +# when: +# - "groups['opengauss_cm'] is not defined" +# - "(groups['opengauss_dn'] | count) == 1" +# - "og_cm_enabled" +# block: +# - name: Create init cm nodes list +# ansible.builtin.add_host: +# hostname: "{{ node }}" +# groups: +# - opengauss_cm +# node_ip: "{{ node }}" +# node_name: "og{{ node | ipaddr('int') }}" +# loop: "{{ groups['opengauss_ep'] | first }}" +# loop_control: +# loop_var: node -- name: Current data nodes - ansible.builtin.debug: - msg: "{{ groups['opengauss_dn'] }}" - when: "groups['opengauss_dn'] is defined" +# - name: Create init dn nodes list +# ansible.builtin.add_host: +# hostname: "{{ node }}" +# groups: +# - opengauss_cm +# node_ip: "{{ node }}" +# node_name: "og{{ node | ipaddr('int') }}" +# loop: "{{ groups['opengauss_dn'] | first }}" +# loop_control: +# loop_var: node -- name: Set 'og_data_nodes' facts - ansible.builtin.set_fact: - og_data_nodes: >- - {{ og_data_path }}/dn - {%- for node in (groups['opengauss_dn'] | sort) if (hostvars[node]['node_ip'] != og_master) -%} - ,{{ hostvars[node]['node_name'] }},{{ og_data_path }}/dn - {%- endfor -%} - {%- if groups['opengauss_expand'] is defined -%} - {%- for node in (groups['opengauss_expand'] | sort) -%} - ,og{{ node | ipaddr('int') }},{{ og_data_path }}/dn - {%- endfor -%} - {%- endif -%} - og_expansion: "{{ groups['opengauss_expand'] is defined }}" - when: - - "groups['opengauss_dn'] is defined" +# - name: Current cm nodes +# ansible.builtin.debug: +# msg: "{{ groups['opengauss_cm'] }}" +# when: "groups['opengauss_cm'] is defined" diff --git a/roles/opengauss/tasks/main.yml b/roles/opengauss/tasks/main.yml index 4053f41..2861b36 100644 --- a/roles/opengauss/tasks/main.yml +++ b/roles/opengauss/tasks/main.yml @@ -2,7 +2,6 @@ # tasks file for openGauss - name: Tasks always run - tags: always block: - name: Import vars combine tasks ansible.builtin.import_role: @@ -17,6 +16,64 @@ ansible.builtin.import_tasks: file: deploy/cluster_check.yml + - name: Import cluster expand tasks + ansible.builtin.import_tasks: + file: deploy/cluster_expand.yml + vars: + cm_nodes: "{{ (groups['opengauss_cm'] is defined) | ternary(groups['opengauss_cm'], '') | sort }}" + dn_nodes: "{{ (groups['opengauss_dn'] is defined) | ternary(groups['opengauss_dn'], '') | sort }}" + ep_nodes: "{{ (groups['opengauss_ep'] is defined) | ternary(groups['opengauss_ep'], '') | sort }}" + # 当现有架构不足 3 节点时，先扩容数据节点（CM），再部署管理节点（CM） + og_expansion: true + og_cm_enabled: >- + {{ + ( + (groups['opengauss_cm'] is not defined) + and + (dn_nodes | count ) < 3 + ) + | + ternary( + false, + true + ) + }} + when: "groups['opengauss_ep'] is defined" + + - name: Import cluster manager deploy + become_user: "{{ og_user }}" + delegate_to: "{{ og_master }}" + run_once: true + when: + - "groups['opengauss_cm'] is not defined" + - "og_cm_enabled" + block: + - name: Import cluster checking tasks + ansible.builtin.import_tasks: + file: deploy/cluster_check.yml + + - name: Import cluster manager deploy tasks + ansible.builtin.import_tasks: + file: deploy/cluster_manager.yml + vars: + cm_nodes: "{{ (groups['opengauss_dn'] is defined) | ternary(groups['opengauss_dn'], '') | sort }}" + dn_nodes: "{{ (groups['opengauss_dn'] is defined) | ternary(groups['opengauss_dn'], '') | sort }}" + ep_nodes: "" + og_expansion: false + og_cm_enabled: >- + {{ + ( + (groups['opengauss_cm'] is not defined) + and + (dn_nodes | count ) < 3 + ) + | + ternary( + false, + true + ) + }} + always: - name: Import post tasks ansible.builtin.import_tasks: diff --git a/roles/opengauss/tasks/os/common_set.yml b/roles/opengauss/tasks/os/common_set.yml index c5b13f1..b040330 100644 --- a/roles/opengauss/tasks/os/common_set.yml +++ b/roles/opengauss/tasks/os/common_set.yml @@ -71,20 +71,6 @@ state: started enabled: true -- name: Config /etc/hosts - ansible.builtin.blockinfile: - path: /etc/hosts - marker: "# {mark} OPENGAUSS NODES" - block: | - {% for node in og_all_nodes %} - {{ node }} og{{ node | ipaddr('int') }} og-{{ node | replace('.', '-') }} - {% endfor %} - delegate_to: "{{ node }}" - loop: "{{ og_all_nodes }}" - loop_control: - loop_var: node - run_once: true - - name: Get backIp1's iface name ansible.builtin.shell: cmd: | diff --git a/roles/opengauss/tasks/os/user.yml b/roles/opengauss/tasks/os/user.yml index 1d1f72c..12e5f6a 100644 --- a/roles/opengauss/tasks/os/user.yml +++ b/roles/opengauss/tasks/os/user.yml @@ -21,7 +21,7 @@ group: "{{ og_group }}" recurse: true - - name: Config command alias + - name: "Config command alias for user '{{ og_user }}'" ansible.builtin.lineinfile: path: "/home/{{ og_user }}/.bashrc" create: true diff --git a/roles/opengauss/tasks/pre_tasks.yml b/roles/opengauss/tasks/pre_tasks.yml index b6deabd..1d0a3fc 100644 --- a/roles/opengauss/tasks/pre_tasks.yml +++ b/roles/opengauss/tasks/pre_tasks.yml @@ -19,7 +19,7 @@ - name: Import user config tasks ansible.builtin.include_tasks: file: os/user.yml - with_items: "{{ groups['opengauss_expand'] | default(og_all_nodes) }}" + with_items: "{{ groups['opengauss_ep'] | default(og_all_nodes) }}" loop_control: loop_var: node run_once: true diff --git a/roles/opengauss/tasks/runtime_facts.yml b/roles/opengauss/tasks/runtime_facts.yml index aed6832..892de13 100644 --- a/roles/opengauss/tasks/runtime_facts.yml +++ b/roles/opengauss/tasks/runtime_facts.yml @@ -14,22 +14,20 @@ az_name: "{{ combined_vars.opengauss_env.azName | default('AZ1') }}" az_priority: "{{ combined_vars.opengauss_env.azPriority | default('1') }}" cm_port: "{{ combined_vars.opengauss_env.cmServerPortBase }}" - data_port: "{{ combined_vars.opengauss_env.dataPortBase }}" + db_port: "{{ combined_vars.opengauss_env.dataPortBase }}" sync_num: "{{ combined_vars.opengauss_env.syncNum }}" - name: Set runtime facts - 2 ansible.builtin.set_fact: - og_all_names: >- - og{{ og_master | ipaddr('int') }}{% for node in og_replicas %},og{{ node | ipaddr('int') }}{% endfor %} - og_all_ips: >- - {{ og_master }}{% for node in og_replicas %},{{ node }}{% endfor %} - og_all_dn_nodes: >- - {{ og_data_path }}/dn{% for node in og_replicas %},og{{ node | ipaddr('int') }},{{ og_data_path }}/dn{% endfor %} og_pkg_url: >- {{ combined_vars.opengauss_download - [ansible_architecture] - [(ansible_distribution | replace(' ', '_') | lower) - + '_' - + ansible_distribution_major_version] - | replace(opengauss_version, og_ver) }} + [ + ansible_architecture + ] + [ + ( + ansible_distribution | replace(' ', '_') | lower + ) + '_' + ansible_distribution_major_version + ] | replace(opengauss_version, og_ver) + }} og_pkg_name: "openGauss_{{ og_ver }}_{{ ansible_distribution }}_{{ ansible_distribution_version }}_{{ ansible_architecture }}.tar.gz" diff --git a/roles/opengauss/templates/cluster_config.xml.j2 b/roles/opengauss/templates/cluster_config.xml.j2 index 57f2180..a7e8607 100644 --- a/roles/opengauss/templates/cluster_config.xml.j2 +++ b/roles/opengauss/templates/cluster_config.xml.j2 @@ -2,20 +2,34 @@ - - + + -{{ lookup('template', 'cluster_master.xml.j2') }} -{% if (og_replicas | count) > 0 %} -{{ lookup('template', 'cluster_replicas.xml.j2') }} -{% endif %} +{{ lookup('ansible.builtin.template', 'cluster_master.xml.j2') }} + +{{ lookup('ansible.builtin.template', 'cluster_replicas.xml.j2') }} + diff --git a/roles/opengauss/templates/cluster_master.xml.j2 b/roles/opengauss/templates/cluster_master.xml.j2 index b4987c6..c6651d0 100644 --- a/roles/opengauss/templates/cluster_master.xml.j2 +++ b/roles/opengauss/templates/cluster_master.xml.j2 @@ -9,26 +9,48 @@ - - + -{% if og_cm_enabled %} + {%- if og_cm_enabled -%} - - - -{% endif %} + + + + {%- endif -%} \ No newline at end of file diff --git a/roles/opengauss/templates/cluster_replicas.xml.j2 b/roles/opengauss/templates/cluster_replicas.xml.j2 index b12be8a..cbffafa 100644 --- a/roles/opengauss/templates/cluster_replicas.xml.j2 +++ b/roles/opengauss/templates/cluster_replicas.xml.j2 @@ -1,24 +1,43 @@ +{% for node in dn_nodes if hostvars[node]['node_ip'] != og_master %} + + + + + + + + +{% if og_cm_enabled %} + + + +{% endif %} + +{% if hostvars[node]['node_ip'] in groups['opengauss_cascade'] %} + + +{% endif %} + +{% endfor %} + +{% for node in ep_nodes %} - -{% for node in og_replicas %} -{% if og_cm_enabled %} +{% if og_cm_enabled %} -{% if node in groups['opengauss_cascade'] %} +{% endif %} +{% if node in groups['opengauss_cascade'] %} -{% endif %} {% endif %} - -{% endfor %} - +{% endfor %} \ No newline at end of file diff --git a/roles/opengauss/templates/report.md.j2 b/roles/opengauss/templates/report.md.j2 index 81bbb65..2bf54f6 100644 --- a/roles/opengauss/templates/report.md.j2 +++ b/roles/opengauss/templates/report.md.j2 @@ -16,4 +16,4 @@ | Linux 用户名称 | {{ og_user }} | | Linux 用户密码 | {{ lookup('password', inventory_dir + '/credentials/opengauss_omm_pass', chars=['ascii_letters', 'digits']) }} | | Linux 用户组 | {{ og_group }} | -| 数据库 root 密码 |{{ combined_vars.opengauss_db_set.root_pass }} | \ No newline at end of file +| 数据库 root 密码 | {{ combined_vars.opengauss_db_set.root_pass }} | \ No newline at end of file -- Gitee