diff --git a/plugins/boostkit/config.yaml b/plugins/boostkit/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..575e390310ed690a2b0e2a5717d478a77e43be7b --- /dev/null +++ b/plugins/boostkit/config.yaml @@ -0,0 +1,31 @@ +all: + hosts: + host1: + ansible_host: HOST_IP # e.g. 192.168.1.101 + ansible_port: 22 + ansible_user: root + ansible_password: "" + # 按实际代理需要添加，添加时删除以下第一行和所需行的井号；如需全局使用代理请移动至 all:vars + #proxy_env: + # http_proxy: + # https_proxy: + # ftp_proxy: + # no_proxy: + # 可在此添加更多主机 + children: + AccLib: + hosts: + host1: + # 可在此包含更多主机 + vars: + install_list: + - 'ALL' + vars: + # 跳过所有基本检查项 + skip_check: false + # pip 源地址，如不涉及 pip 安装可删除此项 + pip_index_url: "https://mirrors.huaweicloud.com/repository/pypi/simple" + # 临时文件存放目录 + temp_path: /tmp + # 跳过 ssh 校验，如需禁用此功能，请删除以下配置项 + ansible_ssh_common_args: '-o StrictHostKeyChecking=no' diff --git a/plugins/boostkit/main.yaml b/plugins/boostkit/main.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c77268cf6060963ddea0703aedc216e9bf9d16a --- /dev/null +++ b/plugins/boostkit/main.yaml @@ -0,0 +1,15 @@ +name: boostkit +description: BoostKit +action: + AccLib-show: + description: show available components in AccLib + tasks: + - name: show available components in AccLib + playbook: AccLib-show.yml + scope: AccLib + AccLib-install: + description: install components in AccLib + tasks: + - name: install components in AccLib + playbook: AccLib-install.yml + scope: AccLib diff --git a/plugins/boostkit/workspace/AccLib-install.yml b/plugins/boostkit/workspace/AccLib-install.yml new file mode 100644 index 0000000000000000000000000000000000000000..274e4278c3453c87e30c39b4831491b55efecbdb --- /dev/null +++ b/plugins/boostkit/workspace/AccLib-install.yml @@ -0,0 +1,6 @@ +- hosts: all + environment: "{{ proxy_env | default({}) }}" + roles: + - check + - AccLib/prepare + - AccLib/install diff --git a/plugins/boostkit/workspace/AccLib-show.yml b/plugins/boostkit/workspace/AccLib-show.yml new file mode 100644 index 0000000000000000000000000000000000000000..ffa7bebf88108c0578fb9b2127caf4136088120c --- /dev/null +++ b/plugins/boostkit/workspace/AccLib-show.yml @@ -0,0 +1,6 @@ +- hosts: all + environment: "{{ proxy_env | default({}) }}" + roles: + - check + - AccLib/prepare + - AccLib/show diff --git a/plugins/boostkit/workspace/roles/AccLib/install/tasks/execute_install.yml b/plugins/boostkit/workspace/roles/AccLib/install/tasks/execute_install.yml new file mode 100644 index 0000000000000000000000000000000000000000..d2a4a00299752679c817b56de268687cae24118e --- /dev/null +++ b/plugins/boostkit/workspace/roles/AccLib/install/tasks/execute_install.yml @@ -0,0 +1,24 @@ +- name: Get timestamp + command: date +%Y%m%d%H%M%S + register: timestamp + changed_when: false + +- name: Execute install script for {{ item }} + shell: | + python3 install.py --component {{ item }} > {{ temp_path }}/BoostKit-AccLib/scripts/{{ timestamp.stdout }}.log 2>&1 + args: + chdir: "{{ temp_path }}/BoostKit-AccLib/scripts" + register: install_result + changed_when: false + ignore_errors: true + tags: install + +- name: Show execution result + debug: + msg: "Installation completed for {{ item }}. Log file: {{ temp_path }}/BoostKit-AccLib/scripts/{{ timestamp.stdout }}.log" + when: install_result is succeeded + +- name: Fail if installation failed + fail: + msg: "Installation failed for {{ item }}. Check log file: {{ temp_path }}/BoostKit-AccLib/scripts/{{ timestamp.stdout }}.log" + when: install_result is failed diff --git a/plugins/boostkit/workspace/roles/AccLib/install/tasks/main.yml b/plugins/boostkit/workspace/roles/AccLib/install/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..525a999aaaa33ec3c20db82dcb7ae394722f5a16 --- /dev/null +++ b/plugins/boostkit/workspace/roles/AccLib/install/tasks/main.yml @@ -0,0 +1,3 @@ +- name: "[Script] Execute install script" + include_tasks: execute_install.yml + loop: "{{ (install_list if install_list is defined and (install_list is iterable and not install_list is string) else [install_list]) if install_list is defined and install_list | length > 0 else ['ALL'] }}" diff --git a/plugins/boostkit/workspace/roles/AccLib/prepare/files/BoostKit-AccLib/scripts/install.py b/plugins/boostkit/workspace/roles/AccLib/prepare/files/BoostKit-AccLib/scripts/install.py new file mode 100644 index 0000000000000000000000000000000000000000..c6c0abbfddfdf0dd6068f884f83f958200268f3d --- /dev/null +++ b/plugins/boostkit/workspace/roles/AccLib/prepare/files/BoostKit-AccLib/scripts/install.py @@ -0,0 +1,477 @@ +import argparse +import os.path +import re +import xml.etree.ElementTree as ET +import subprocess +from contextlib import contextmanager + +quiet = True + + +def check_version(src, tar, op=">="): + src_list = src.split('.') + tar_list = tar.split('.') + + for index, str in enumerate(tar_list): + if '*' in str or index >= len(src_list): + break + + if op == ">=": + if int(src_list[index]) > int(str): + return True + elif int(src_list[index]) == int(str): + continue + else: + return False + elif op == "<": + if int(src_list[index]) < int(str): + return True + elif int(src_list[index]) == int(str): + continue + else: + return False + elif op == "=": + if int(src_list[index]) != int(str): + return False + else: + return False + return True + + +class Runcmd(object): + @staticmethod + def sendcmd(cmd, workspace=None, check=True, extra_env: dict = None): + env = os.environ.copy() + if extra_env: + for key in extra_env: + env[key] = extra_env.get(key) + + if not quiet: + print(f"cmd: {cmd}") + result = subprocess.run(cmd, cwd=workspace, shell=True, check=check, capture_output=True, encoding='utf-8', + env=env) + if not quiet: + print(f"done\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}\nreturn code: {result.returncode}") + return '\n'.join(result.stdout.split('\n')).strip("\n") + + +class Installer(object): + __root = None + __function_dict = dict() + + def __init__(self, xml_path): + self.__root = ET.parse(xml_path).getroot() + self.__ROOT_PATH = os.path.abspath(os.path.dirname(os.getcwd())) + self.__SOURCE_PATH = os.path.join(self.__ROOT_PATH, "source/") + self.__init_func_dict() + self.__platform = self.__check_platform() + + def __check_platform(self): + kunpeng_platform = {"0xd01": "Kunpeng 920", "0xd02": "Kunpeng 920 V200"} + implementer = Runcmd.sendcmd("cat /proc/cpuinfo | grep 'CPU implementer' | head -n 1 | awk '{print $NF}'") + part = Runcmd.sendcmd("cat /proc/cpuinfo | grep 'CPU part' | head -n 1 | awk '{print $NF}'") + + if implementer == "0x48": + return kunpeng_platform.get(part) + else: + return Runcmd.sendcmd("lscpu | grep 'BIOS Model name:' | awk '{print $NF}'") + + def __check_env(self): + with self.__process("检查代理"): + Runcmd.sendcmd("curl www.baidu.com") + + with self.__process("检查yum源, 并安装必要工具"): + Runcmd.sendcmd("yum makecache") + Runcmd.sendcmd("yum install -y git tar gcc gcc-c++ patch") + Runcmd.sendcmd("git config --global https.sslverify false") + Runcmd.sendcmd("git config --global http.sslverify false") + + def __chech_env_in_profile(self, key, value): + libpath = os.getenv(key) + if libpath and value in libpath: + return True + + env_list = Runcmd.sendcmd(f"cat /etc/profile | grep {key}", check=False).split("\n") + for env in env_list: + if value in env and not re.search("#.+export", env): + return True + + return False + + def __get_element_text(self, tag: str): + tags = [child.tag for child in self.__root.find(tag)] + texts = [child.text for child in self.__root.find(tag)] + info = dict() + + for index, tag in enumerate(tags): + info[tag] = texts[index] + + return info + + @contextmanager + def __process(self, topic): + try: + print(topic) + yield + print(f"{topic}成功") + except subprocess.CalledProcessError as e: + print(f"{topic}存在执行命令失败, 报错信息如下:\n{e.stderr}") + raise e + except Exception as e: + print(f"{topic}失败") + raise e + + def __install_ksl(self): + Runcmd.sendcmd("rm -rf ./KSL", workspace=f"{self.__ROOT_PATH}") + Runcmd.sendcmd(f"mkdir -p KSL/rpm", workspace=f"{self.__ROOT_PATH}") + + ksl_info = self.__get_element_text("KSL") + with self.__process("安装KSL"): + res = Runcmd.sendcmd("cat /etc/os-release") + name = re.search("NAME=\"(.+)\"", res).group(1) + assert name == "openEuler", "仅支持在openEuler上安装" + os_version = re.search("VERSION=\"(.+)\"", res).group(1).replace(' ', '').replace(')', '').replace('(', '-') + pkg_version = ksl_info.get("version") + url = ksl_info.get('url').format(f"{name}-{os_version}", pkg_version) + + Runcmd.sendcmd(f"wget {url} --no-check-certificate", workspace=f"{self.__ROOT_PATH}/KSL/rpm") + Runcmd.sendcmd("rpm -e boostkit-ksl", check=False) + Runcmd.sendcmd("rpm -ivh rpm/*.rpm", workspace=f"{self.__ROOT_PATH}/KSL") + + def __install_hyperscan_dep(self): + self.__install_ksl() + + Runcmd.sendcmd("rm -rf ./Hyperscan", workspace=f"{self.__ROOT_PATH}") + Runcmd.sendcmd(f"mkdir -p Hyperscan/resource", workspace=f"{self.__ROOT_PATH}") + + info = self.__get_element_text("Hyperscan") + + with self.__process("安装ragel"): + download = info.get("ragel") + bag = info.get("ragel").split('/')[-1] + + Runcmd.sendcmd(f"wget {download} --no-check-certificate", workspace=f"{self.__ROOT_PATH}/Hyperscan") + Runcmd.sendcmd(f"tar -zxf {bag}", workspace=f"{self.__ROOT_PATH}/Hyperscan") + + dir = Runcmd.sendcmd(f"ls | grep ragel | grep -v tar", workspace=f"{self.__ROOT_PATH}/Hyperscan") + Runcmd.sendcmd(f"./configure", workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}") + Runcmd.sendcmd(f"make -j && make install", workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}") + Runcmd.sendcmd(f"ragel -v") + + with self.__process("安装boost"): + Runcmd.sendcmd("yum install -y boost boost-devel") + + with self.__process("安装pcre"): + download = info.get("pcre") + bag = info.get("pcre").split('/')[-1] + + Runcmd.sendcmd(f"wget {download} --no-check-certificate", workspace=f"{self.__ROOT_PATH}/Hyperscan") + Runcmd.sendcmd(f"tar -zxf {bag}", workspace=f"{self.__ROOT_PATH}/Hyperscan") + + with self.__process("安装yum依赖包"): + Runcmd.sendcmd("yum install -y sqlite sqlite-devel") + stdout = Runcmd.sendcmd("pkg-config --libs sqlite3") + assert "-lsqlite3" in stdout, "安装sqlite失败" + Runcmd.sendcmd("yum install -y cmake make gcc gcc-c++") + + def __install_hyperscan(self): + self.__install_hyperscan_dep() + + info = self.__get_element_text("Hyperscan") + + with self.__process("安装Hyperscan"): + stdout = Runcmd.sendcmd("lscpu | grep Architecture") + platform = "x86" if "x86" in stdout else "arm" + download = info.get(f"code_{platform}") + branch = info.get(f"branch_{platform}") + dir = download.split('/')[-1].split('.')[0] + + res = Runcmd.sendcmd("rpm -qa | grep boostkit-ksl") + ksl_version = re.search("boostkit-ksl-(\d+.\d+.\d+)", res).group(1) + if platform == "arm" and check_version(ksl_version, "2.4.0"): + Runcmd.sendcmd(f"git clone {download} -b khsel", workspace=f"{self.__ROOT_PATH}/Hyperscan") + Runcmd.sendcmd("cp hyperscan/khsel.patch ./", workspace=f"{self.__ROOT_PATH}/Hyperscan") + Runcmd.sendcmd(f"git checkout {branch}", workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}") + Runcmd.sendcmd(f"mv ../khsel.patch ./", workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}") + Runcmd.sendcmd(f"git apply khsel.patch", workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}") + else: + Runcmd.sendcmd(f"git clone {download} -b {branch}", workspace=f"{self.__ROOT_PATH}/Hyperscan") + + boost_path = "/usr/include" + Runcmd.sendcmd(f"ln -s {boost_path}/boost include/boost", workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}") + + pcre_path = Runcmd.sendcmd("ls | grep pcre | grep -v tar", workspace=f"{self.__ROOT_PATH}/Hyperscan") + Runcmd.sendcmd(f"cp -rf ../{pcre_path} ./pcre", workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}") + + stdout = Runcmd.sendcmd("cmake --version | grep version", workspace=f"{self.__ROOT_PATH}/Hyperscan") + version = stdout.split()[-1] + if check_version(version, "2.8.0", op='<'): + Runcmd.sendcmd( + "sed -i 's/CMAKE_POLICY(SET CMP0026 OLD)/#CMAKE_POLICY(SET CMP0026 OLD)/g' CMakeLists.txt", + workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}/pcre") + + Runcmd.sendcmd("mkdir -p build", workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}") + Runcmd.sendcmd("cmake .. && make -j", workspace=f"{self.__ROOT_PATH}/Hyperscan/{dir}/build") + + def __install_compress_decompress_tools(self): + Runcmd.sendcmd("rm -rf ZIP", workspace=f"{self.__ROOT_PATH}") + Runcmd.sendcmd("mkdir ZIP", workspace=f"{self.__ROOT_PATH}") + + with self.__process("安装软算测试工具lzbench"): + Runcmd.sendcmd("git clone https://gitee.com/kunpeng_compute/lzbench.git --depth=1", + workspace=f"{self.__ROOT_PATH}/ZIP") + Runcmd.sendcmd("make -j", workspace=f"{self.__ROOT_PATH}/ZIP/lzbench") + + if "KAE" in str(self.__function_dict.keys()) or "QAT" in str(self.__function_dict.keys()): + with self.__process("安装带宽测试工具"): + Runcmd.sendcmd(f"make platform='{self.__platform}'", + workspace=f"{self.__ROOT_PATH}/resources/bandwidth") + Runcmd.sendcmd(f"cp ./bin/bandwidth {self.__ROOT_PATH}/ZIP", + workspace=f"{self.__ROOT_PATH}/resources/bandwidth") + Runcmd.sendcmd("make clean", workspace=f"{self.__ROOT_PATH}/resources/bandwidth") + + def __install_kae_dep(self): + with self.__process("安装KAE相关依赖"): + Runcmd.sendcmd( + "yum install -y make kernel-devel libtool numactl-devel openssl-devel chrpath lz4-devel zstd-devel zlib-devel") + + def __install_kae(self): + version = Runcmd.sendcmd("openssl version | awk '{print $2}'") + if check_version(version, "1.*", "="): + engine_path = "/usr/local/lib/engines-1.1/" + component = "engine" + elif check_version(version, "3.*", "="): + engine_path = "/usr/local/lib/engines-3.0/" + component = "engine3" + else: + assert False, f"unsupport openssl version: {version}" + + self.__install_kae_dep() + + info = self.__get_element_text("KAE") + Runcmd.sendcmd("rm -rf ./KAE", workspace=f"{self.__ROOT_PATH}") + Runcmd.sendcmd("mkdir KAE", workspace=f"{self.__ROOT_PATH}") + + with self.__process("下载编译KAE源码"): + download = info.get("kae_code") + branch = info.get("kae_code_tag") + dir = download.split('/')[-1].split('.')[0] + + Runcmd.sendcmd(f"git clone {download} -b {branch} --depth=1", workspace=f"{self.__ROOT_PATH}/KAE") + Runcmd.sendcmd("bash build.sh cleanup", workspace=f"{self.__ROOT_PATH}/KAE/{dir}") + Runcmd.sendcmd("bash build.sh driver", workspace=f"{self.__ROOT_PATH}/KAE/{dir}") + Runcmd.sendcmd("bash build.sh uadk", workspace=f"{self.__ROOT_PATH}/KAE/{dir}") + Runcmd.sendcmd(f"bash build.sh {component}", workspace=f"{self.__ROOT_PATH}/KAE/{dir}") + Runcmd.sendcmd("bash build.sh zlib", workspace=f"{self.__ROOT_PATH}/KAE/{dir}") + + if "Kunpeng 920 V200" == self.__platform: + Runcmd.sendcmd("bash build.sh lz4", workspace=f"{self.__ROOT_PATH}/KAE/{dir}") + Runcmd.sendcmd("bash build.sh zstd", workspace=f"{self.__ROOT_PATH}/KAE/{dir}") + + with self.__process("确认KAE模块使能"): + res = Runcmd.sendcmd("ls /sys/class/uacce", check=False) + if not res: + Runcmd.sendcmd("rmmod hisi_zip && rmmod hisi_sec2 && rmmod hisi_hpre") + Runcmd.sendcmd("inmod hisi_hpre && inmod hisi_sec2 && inmod hisi_zip") + Runcmd.sendcmd("ls /sys/class/uacce") + Runcmd.sendcmd("ls /usr/local/lib/libwd*") + + res = Runcmd.sendcmd("ls /usr/local/kaezip/lib", check=False) + assert "kaezip" in res, "没有找到kaezip相关动态库" + if "Kunpeng 920 V200" == self.__platform: + res = Runcmd.sendcmd("ls /usr/local/kaelz4/lib", check=False) + assert "kaelz4" in res, "没有找到kaelz4相关动态库" + res = Runcmd.sendcmd("ls /usr/local/kaezstd/lib", check=False) + assert "kaezstd" in res, "没有找到kaezstd相关动态库" + + res = Runcmd.sendcmd(f"ls {engine_path}", check=False) + assert "kae.so" in res, "没有找到engine相关动态库" + + if not self.__chech_env_in_profile("OPENSSL_ENGINES", engine_path): + Runcmd.sendcmd(f"echo 'export OPENSSL_ENGINES={engine_path}:$OPENSSL_ENGINES' >> /etc/profile") + + print("成功安装KAE, 已经添加必要环境变量, 执行source /etc/profile使能") + + def __install_libgcrypt_dep(self): + with self.__process("安装libgcrypt相关依赖"): + Runcmd.sendcmd("yum install -y texinfo transfig hwloc hwloc-devel autoconf automake") + + def __install_libgcrypt(self): + self.__install_libgcrypt_dep() + + gpg_info = Runcmd.sendcmd("find /usr/local/lib -name '*libgpg-error.so*'") + if "libgpg-error.so" in gpg_info: + print("已经安装了libgpg-error，不用再安装了") + return + + Runcmd.sendcmd("rm -rf libgpg", workspace=f"{self.__ROOT_PATH}") + with self.__process("安装libgcrypt"): + Runcmd.sendcmd(f"unzip libgpg-error-1.51.zip -d {self.__ROOT_PATH}/libgpg", + workspace=f"{self.__ROOT_PATH}/resources/libgcrypt") + Runcmd.sendcmd("chmod 777 autogen.sh; ./autogen.sh", + workspace=f"{self.__ROOT_PATH}/libgpg/libgpg-error-1.51") + Runcmd.sendcmd("chmod 777 configure; ./configure --enable-maintainer-mode", + workspace=f"{self.__ROOT_PATH}/libgpg/libgpg-error-1.51") + Runcmd.sendcmd("make -j; make install", workspace=f"{self.__ROOT_PATH}/libgpg/libgpg-error-1.51") + + if not self.__chech_env_in_profile("LD_LIBRARY_PATH", "/usr/local/lib"): + Runcmd.sendcmd("echo 'export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH' >> /etc/profile") + + def __install_qat_dep(self): + with self.__process("安装QAT依赖包"): + Runcmd.sendcmd("yum install -y systemd-devel pciutils libudev-devel readline-devel libxml2-devel " + "boost-devel elfutils-libelf-devel python3 libnl3-devel kernel-devel-$(uname -r) " + "gcc gcc-c++ yasm nasm zlib openssl-devel zlib-devel make") + + def __install_qat(self): + self.__install_qat_dep() + + Runcmd.sendcmd(f"rm -rf ./QAT", workspace=f"{self.__ROOT_PATH}") + Runcmd.sendcmd("mkdir -p QAT/qat_driver", workspace=f"{self.__ROOT_PATH}") + + info = self.__get_element_text("QAT") + + with self.__process("安装部署QAT驱动"): + download = info.get("driver") + bag = download.split('/')[-1] + decompresser = "unzip" if "zip" in bag else "tar -xf" + qat_hw_root = f"{self.__ROOT_PATH}/QAT/qat_driver" + + os.putenv("ICP_ROOT", qat_hw_root) + Runcmd.sendcmd(f"wget {download} --no-check-certificate", workspace=f"{self.__ROOT_PATH}/QAT/qat_driver") + Runcmd.sendcmd(f"{decompresser} {bag}", workspace=f"{self.__ROOT_PATH}/QAT/qat_driver") + Runcmd.sendcmd("./configure", workspace=f"{self.__ROOT_PATH}/QAT/qat_driver") + Runcmd.sendcmd("make && make install", workspace=f"{self.__ROOT_PATH}/QAT/qat_driver") + + Runcmd.sendcmd("service qat_service status && service qat_service restart") + + with self.__process("安装部署QAT引擎"): + download = info.get("code_engine") + branch = info.get("branch_engine") + dir = download.split('/')[-1].split('.')[0] + + Runcmd.sendcmd(f"git clone {download} -b {branch} --depth=1", workspace=f"{self.__ROOT_PATH}/QAT") + Runcmd.sendcmd(f"./autogen.sh && ./configure --with-qat_hw-dir={qat_hw_root}", + workspace=f"{self.__ROOT_PATH}/QAT/{dir}") + Runcmd.sendcmd(f"make && make install", workspace=f"{self.__ROOT_PATH}/QAT/{dir}") + + with self.__process("安装部署QATzip"): + download = info.get("code_zip") + branch = info.get("branch_zip") + dir = download.split('/')[-1].split('.')[0] + + Runcmd.sendcmd(f"git clone {download} -b {branch} --depth=1", workspace=f"{self.__ROOT_PATH}/QAT") + Runcmd.sendcmd(f"./autogen.sh && ./configure --with-ICP_ROOT={qat_hw_root}", + workspace=f"{self.__ROOT_PATH}/QAT/{dir}") + Runcmd.sendcmd("make clean && make all install", workspace=f"{self.__ROOT_PATH}/QAT/{dir}") + + res = Runcmd.sendcmd("ls /usr/local/lib") + assert "qatzip.so" in res, "没有找到qatzip动态库" + + self.__install_compress_decompress_tools() + + def __install_hct_dep(self): + with self.__process("安装HCT依赖"): + Runcmd.sendcmd( + "yum install -y numactl libuuid-devel kernel-`uname -r` kernel-devel-`uname -r` python3-unversioned-command") + + def __install_hct(self): + self.__install_hct_dep() + + info = self.__get_element_text("HCT") + Runcmd.sendcmd("rm -rf HCT", workspace=f"{self.__ROOT_PATH}") + Runcmd.sendcmd("mkdir -p HCT/build", workspace=f"{self.__ROOT_PATH}") + + with self.__process("安装HCT"): + download = info.get("devkit") + version = info.get("version") + dir = download.split('/')[-1].split('.')[0] + + Runcmd.sendcmd(f"git clone {download} -n --filter=blob:none", workspace=f"{self.__ROOT_PATH}/HCT") + Runcmd.sendcmd(f"git restore --staged hct/pkg/{version} && git restore hct/pkg/{version}", + workspace=f"{self.__ROOT_PATH}/HCT/{dir}") + Runcmd.sendcmd(f"rpm -ivh --nodeps hct-*.rpm", workspace=f"{self.__ROOT_PATH}/HCT/{dir}/hct/pkg/{version}", + check=False) + Runcmd.sendcmd(f"cp ./Makefile {self.__ROOT_PATH}/HCT/build", + workspace=f"{self.__ROOT_PATH}/HCT/{dir}/hct/pkg/{version}") + Runcmd.sendcmd("make && make install", workspace=f"{self.__ROOT_PATH}/HCT/build") + Runcmd.sendcmd("modprobe hct && /opt/hygon/hct/hct/script/hctconfig start") + + def __install_kqmalloc_dep(self): + with self.__process("安装kqmalloc编译依赖"): + Runcmd.sendcmd("yum install -y autoconf gcc gcc-c++") + + def __install_kqmalloc(self): + self.__install_kqmalloc_dep() + + Runcmd.sendcmd(f"rm -rf ./KQMalloc", workspace=f"{self.__ROOT_PATH}") + Runcmd.sendcmd(f"mkdir -p KQMalloc/resource", workspace=f"{self.__ROOT_PATH}") + + with self.__process("安装kqmalloc"): + self.__install_ksl() + res = Runcmd.sendcmd("rpm -qa | grep boostkit-ksl") + ksl_version = re.search("boostkit-ksl-(\d+.\d+.\d+)", res).group(1) + assert check_version(ksl_version, "2.4.0"), "ksl版本过低，请指定2.4.0以上版本" + + def __init_func_dict(self): + self.__function_dict["Hyperscan"] = self.__install_hyperscan + self.__function_dict["Libgcrypt"] = self.__install_libgcrypt + res = Runcmd.sendcmd("lscpu | grep 'BIOS Model name:'") + if "Kunpeng" in res: + self.__function_dict["KAE"] = self.__install_kae + self.__function_dict["KQMalloc"] = self.__install_kqmalloc + elif "INTEL" in res: + self.__function_dict["QAT"] = self.__install_qat + elif "Hygon" in res: + self.__function_dict["HCT"] = self.__install_hct + else: + self.__function_dict["SOFT_COMPRESS"] = self.__install_compress_decompress_tools + + def get_support(self): + return ' '.join(self.__function_dict.keys()) + + def install(self, components): + self.__check_env() + + if "ALL" == components: + for func in self.__function_dict.values(): + try: + func() + except Exception as e: + print(e) + if not args.ignore: + raise e + else: + for component in components.split(','): + func = self.__function_dict.get(component) + if func: + try: + func() + except Exception as e: + print(e) + if not args.ignore: + raise e + else: + raise f"没有找到安装方法, {component}" + + +if __name__ == '__main__': + installer = Installer('version.xml') + supports = installer.get_support() + + parser = argparse.ArgumentParser() + parser.add_argument("--component", type=str, + help=f"support component eg. ALL | component | 'component1 component2...', default ALL", + default='ALL') + parser.add_argument("-q", '--quiet', action='store_true', help="print quiet") + parser.add_argument('--ignore', action='store_true', + help="ignore error for each installing component while component is ALL") + parser.add_argument("-i", '--info', action='store_true', help="check supported components") + args = parser.parse_args() + quiet = args.quiet + info_t = args.info + if info_t: + print(supports) + exit(0) + + installer.install(args.component) diff --git a/plugins/boostkit/workspace/roles/AccLib/prepare/files/BoostKit-AccLib/scripts/requirements.txt b/plugins/boostkit/workspace/roles/AccLib/prepare/files/BoostKit-AccLib/scripts/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9a2a89e6375dc707b84bcbc07394fd89de4f27ea --- /dev/null +++ b/plugins/boostkit/workspace/roles/AccLib/prepare/files/BoostKit-AccLib/scripts/requirements.txt @@ -0,0 +1 @@ +argparse>=1.4.0 diff --git a/plugins/boostkit/workspace/roles/AccLib/prepare/files/BoostKit-AccLib/scripts/version.xml b/plugins/boostkit/workspace/roles/AccLib/prepare/files/BoostKit-AccLib/scripts/version.xml new file mode 100644 index 0000000000000000000000000000000000000000..7af95043333031ea421c920164c894de62d1fd6c --- /dev/null +++ b/plugins/boostkit/workspace/roles/AccLib/prepare/files/BoostKit-AccLib/scripts/version.xml @@ -0,0 +1,35 @@ + + + + 2.4.0 + https://repo.oepkgs.net/openEuler/rpm/{}/extras/aarch64/Packages/b/boostkit-ksl-{}-1.aarch64.rpm + + + 2.5.0 + https://repo.oepkgs.net/openEuler/rpm/{}/extras/aarch64/Packages/b/boostkit-hmpp-{}-1.aarch64.rpm + + + https://repo.oepkgs.net/openEuler/rpm/openEuler-24.03-LTS/contrib/oedp/files/ragel-6.10.tar.gz + https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz + https://repo.oepkgs.net/openEuler/rpm/openEuler-24.03-LTS/contrib/oedp/files/pcre-8.43.tar.gz + https://gitee.com/kunpengcompute/hyperscan.git + v5.4.2.aarch64 + https://github.com/intel/hyperscan.git + v5.4.2 + + + https://gitee.com/kunpengcompute/KAE.git + kae2 + + + https://downloadmirror.intel.com/843052/QAT20.L.1.2.30-00078.tar.gz + https://github.com/intel/QAT_Engine.git + v1.8.1 + https://github.com/intel/QATzip.git + v1.3.0 + + + https://gitee.com/anolis/hygon-devkit.git + hct_2.1.0.20241030_release + + \ No newline at end of file diff --git a/plugins/boostkit/workspace/roles/AccLib/prepare/tasks/main.yml b/plugins/boostkit/workspace/roles/AccLib/prepare/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..9597ff15b4c3f181da27b2f2c480b274c8dedd20 --- /dev/null +++ b/plugins/boostkit/workspace/roles/AccLib/prepare/tasks/main.yml @@ -0,0 +1,17 @@ +- name: "[Dependency] Install python3-libselinux" + yum: + name: python3-libselinux + state: present + tags: dependency + +- name: "[File] Ensure destination directory exists" + file: + path: "{{ temp_path }}" + state: directory + tags: file_transfer + +- name: "[File] Transfer local file to remote host" + copy: + src: "BoostKit-AccLib" + dest: "{{ temp_path }}" + tags: file_transfer diff --git a/plugins/boostkit/workspace/roles/AccLib/show/tasks/main.yml b/plugins/boostkit/workspace/roles/AccLib/show/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..5652f83f8b589e76c03138ae81fba9a28649b1fc --- /dev/null +++ b/plugins/boostkit/workspace/roles/AccLib/show/tasks/main.yml @@ -0,0 +1,13 @@ +- name: "[Script] Get info and display" + shell: | + python3 install.py -i + args: + chdir: "{{ temp_path }}/BoostKit-AccLib/scripts" + register: info_output + changed_when: false + tags: get_info + +- name: "[Script] Display execution output" + debug: + msg: "{{ info_output.stdout_lines }}" + tags: get_info diff --git a/plugins/boostkit/workspace/roles/check/tasks/main.yml b/plugins/boostkit/workspace/roles/check/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..c058d647f957f71baf20dc042e53b299b4f308e1 --- /dev/null +++ b/plugins/boostkit/workspace/roles/check/tasks/main.yml @@ -0,0 +1,84 @@ +- name: "[Check] Skip all checks if skip_check is true" + debug: + msg: "Skipping all checks" + when: > + skip_check is defined and + (skip_check == true or skip_check | string == "true") + tags: never + +- block: + - name: "[Check] Initialize check results" + set_fact: + check_results: [ ] + check_failed: "false" + + - name: "[CPU] Verify Kunpeng-920 processor" + shell: | + grep -m1 "CPU implementer" /proc/cpuinfo | awk '{print $4}' + register: cpu_implementer + changed_when: false + tags: hardware_check + + - name: "[CPU] Record CPU check result" + set_fact: + check_results: "{{ check_results + ['CPU implementer: ' + cpu_implementer.stdout] }}" + check_failed: "{{ check_failed or (cpu_implementer.stdout != '0x48') }}" + tags: hardware_check + + - name: "[Network] Test connectivity to Baidu using curl" + command: >- + curl -sSf --connect-timeout 5 + -w "HTTP %{http_code}" -o /dev/null + https://www.baidu.com + register: network_test + ignore_errors: yes + changed_when: false + tags: network_check + + - name: "[Network] Record network check result" + set_fact: + check_results: >- + {{ check_results + ['Network test: ' + + (network_test is failed | ternary('Failed', 'Success (' + network_test.stdout + ')'))] }} + check_failed: "{{ check_failed or (network_test is failed) }}" + tags: network_check + + - name: "[Python] Validate pip repository accessibility" + pip: + name: pip + extra_args: >- + -i "{{ pip_index_url | default('https://pypi.org/simple', true) }}" + --disable-pip-version-check + --timeout 30 + --retries 2 + state: present + when: pip_index_url is defined + register: pip_validation + ignore_errors: yes + changed_when: false + tags: python_check + + - name: "[Python] Record pip check result" + set_fact: + check_results: "{{ check_results + ['Pip repository: ' + (pip_validation is failed | ternary('Unavailable', 'Available'))] }}" + check_failed: "{{ check_failed or (pip_validation is failed) }}" + when: pip_index_url is defined + tags: python_check + + - name: "[Check] Display all check results" + debug: + msg: "{{ item }}" + loop: "{{ check_results }}" + + - name: "[Check] Abort if any check failed" + fail: + msg: >- + Pre-deployment checks failed: + {% for result in check_results %} + - {{ result }} + {% endfor %} + Please fix the issues and try again. + when: check_failed + when: > + skip_check is not defined or + (skip_check != true and skip_check | string != "true")