From 49246c0d62bf88a0ec0dfdf1166a24e829017092 Mon Sep 17 00:00:00 2001 From: lixinyu Date: Tue, 9 Jan 2024 14:40:17 +0800 Subject: [PATCH] nativesdk: optimize the research for nativesdk * in oebuild, when we want to build, we rely on nativesdk and toolchians, as for nativesdk, we commonly set it's path to a exact value, so when nativesdk path is changed, it will occour the err. however the nativesdk path is not fixed, so we should fix the alternative about this, so everytime we should get the nativesdk environment shell by detect no more by a default value. Signed-off-by: lixinyu --- .../app/plugins/bitbake/in_container.py | 2 +- src/oebuild/app/plugins/bitbake/in_host.py | 2 +- src/oebuild/app/plugins/deploy/com_target.py | 3 +- .../app/plugins/deploy/deploy-target.py | 3 -- src/oebuild/app/plugins/run_qemu/run_qemu.py | 2 +- src/oebuild/local_conf.py | 38 +++++++++++++--- src/oebuild/util.py | 44 ++++++++++++++++++- 7 files changed, 80 insertions(+), 14 deletions(-) diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index ab1753f..37f1032 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -279,7 +279,7 @@ class InContainer(BaseBuild): # read container default user .bashrc content content = self._get_bashrc_content(container=container) - init_sdk_command = '. /opt/buildtools/nativesdk/environment-setup-x86_64-pokysdk-linux' + init_sdk_command = f'. {oebuild_util.NATIVESDK_DIR}/{oebuild_util.get_nativesdk_environment(container=container)}' set_template = f'export TEMPLATECONF="{oebuild_util.CONTAINER_SRC}/yocto-meta-openeuler/.oebuild"' init_oe_comand = f'. {oebuild_util.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ {oebuild_util.CONTAINER_BUILD}/{build_dir_name}' diff --git a/src/oebuild/app/plugins/bitbake/in_host.py b/src/oebuild/app/plugins/bitbake/in_host.py index 1eb0e65..1f2db45 100644 --- a/src/oebuild/app/plugins/bitbake/in_host.py +++ b/src/oebuild/app/plugins/bitbake/in_host.py @@ -109,7 +109,7 @@ initialization operations''') pty.spawn("bash") def _mk_build_sh(self, nativesdk_dir, build_dir): - init_sdk_command = f'. {nativesdk_dir}/environment-setup-x86_64-pokysdk-linux' + init_sdk_command = f'. {nativesdk_dir}/{oebuild_util.get_nativesdk_environment(nativesdk_dir)}' set_template = f'export TEMPLATECONF="{self.configure.source_dir()}/yocto-meta-openeuler/.oebuild"' init_oe_command = f'. {self.configure.source_dir()}/yocto-poky/oe-init-build-env {build_dir}' ps1_command = 'PS1="\\u\\h:\\W> "' diff --git a/src/oebuild/app/plugins/deploy/com_target.py b/src/oebuild/app/plugins/deploy/com_target.py index 115f4ea..7902a86 100644 --- a/src/oebuild/app/plugins/deploy/com_target.py +++ b/src/oebuild/app/plugins/deploy/com_target.py @@ -35,6 +35,7 @@ class ComTarget: self.client:DockerProxy = None self.container_id = None self.work_dir = os.getcwd() + self.old_bashrc = None def exec(self, str_args: str, fun): @@ -174,7 +175,7 @@ class ComTarget: self._check_change_ugid(container=container) # read container default user .bashrc content content = self._get_bashrc_content(container=container) - init_sdk_command = f'. {oebuild_util.SDK_ABSOLATE_PATH}' + init_sdk_command = f'. {oebuild_util.NATIVESDK_DIR}/{oebuild_util.get_nativesdk_environment(container=container)}' build_dir_name = os.path.basename(self.work_dir) init_oe_command = f'. {oebuild_util.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ {oebuild_util.CONTAINER_BUILD}/{build_dir_name}' diff --git a/src/oebuild/app/plugins/deploy/deploy-target.py b/src/oebuild/app/plugins/deploy/deploy-target.py index dfe8e1c..83117fe 100644 --- a/src/oebuild/app/plugins/deploy/deploy-target.py +++ b/src/oebuild/app/plugins/deploy/deploy-target.py @@ -10,10 +10,7 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -import os -import sys import argparse -import re import textwrap import logging diff --git a/src/oebuild/app/plugins/run_qemu/run_qemu.py b/src/oebuild/app/plugins/run_qemu/run_qemu.py index c3422b2..141dee3 100644 --- a/src/oebuild/app/plugins/run_qemu/run_qemu.py +++ b/src/oebuild/app/plugins/run_qemu/run_qemu.py @@ -195,7 +195,7 @@ now, you can continue run `oebuild runqemu` in compile directory # read container default user .bashrc content content = self._get_bashrc_content(container=container) - init_sdk_command = '. /opt/buildtools/nativesdk/environment-setup-x86_64-pokysdk-linux' + init_sdk_command = f'. {oebuild_util.NATIVESDK_DIR}/{oebuild_util.get_nativesdk_environment(container=container)}' init_oe_command = f'. {oebuild_util.CONTAINER_SRC}/yocto-poky/oe-init-build-env {oebuild_util.CONTAINER_BUILD}' init_command = [init_sdk_command, init_oe_command] new_content = oebuild_util.init_bashrc_content(content, init_command) diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index d43ac33..0aa2238 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -12,9 +12,12 @@ See the Mulan PSL v2 for more details. import os import re +import sys from oebuild.parse_compile import ParseCompile from oebuild.parse_template import BUILD_IN_DOCKER, BUILD_IN_HOST +import oebuild.util as oebuild_util +from oebuild.m_log import logger class BaseLocalConf(ValueError): ''' @@ -30,9 +33,7 @@ class NativesdkNotValid(BaseLocalConf): ''' NATIVESDK_DIR_NAME = "OPENEULER_NATIVESDK_SYSROOT" -NATIVESDK_SYSROOT = "sysroots/x86_64-pokysdk-linux" OPENEULER_SP_DIR = "OPENEULER_SP_DIR" -NATIVESDK_ENVIRONMENT = "environment-setup-x86_64-pokysdk-linux" SSTATE_MIRRORS = "SSTATE_MIRRORS" SSTATE_DIR = "SSTATE_DIR" TMP_DIR = "TMPDIR" @@ -40,6 +41,30 @@ TMP_DIR = "TMPDIR" NATIVE_GCC_DIR = '/usr1/openeuler/native_gcc' SSTATE_CACHE = '/usr1/openeuler/sstate-cache' +def get_nativesdk_sysroot(nativesdk_dir = oebuild_util.NATIVESDK_DIR): + ''' + return environment initialization shell, if nativesdk directory is not exists + or can not find any initialization shell, raise error + ''' + sysroot_dir = os.path.join(nativesdk_dir, "sysroots") + if not os.path.isdir(nativesdk_dir): + logger.error("the %s is not exists", nativesdk_dir) + sys.exit(1) + if not os.path.isdir(sysroot_dir): + logger.error("the %s is not value", nativesdk_dir) + sys.exit(1) + # list items in nativesdk to find environment shell + list_items = os.listdir(sysroot_dir) + for item in list_items: + ret = re.match("^(x86_64-)[a-zA-Z0-9]{1,}(-linux)$", item) + if ret is not None: + abs_path = os.path.join(sysroot_dir, item) + if os.path.isdir(abs_path): + return item + logger.error("can not find any sysroot directory") + sys.exit(1) + + def match_and_add(new_str: str, content: str): ''' math line in content when the new_str not exist and added @@ -122,7 +147,8 @@ class LocalConf: self.check_nativesdk_valid(parse_compile.nativesdk_dir) if parse_compile.nativesdk_dir is None: raise ValueError("please set nativesdk dir") - nativesdk_sys_dir = os.path.join(parse_compile.nativesdk_dir, NATIVESDK_SYSROOT) + nativesdk_sysroot = get_nativesdk_sysroot(parse_compile.nativesdk_dir) + nativesdk_sys_dir = os.path.join(parse_compile.nativesdk_dir, nativesdk_sysroot) content = match_and_replace( pre=NATIVESDK_DIR_NAME, new_str=f'{NATIVESDK_DIR_NAME} = "{nativesdk_sys_dir}"', @@ -224,9 +250,11 @@ class LocalConf: if not os.path.exists(nativesdk_dir): raise NativesdkNotExist(f"nativesdk directory: {nativesdk_dir} not exist") - nativesdk_environment_dir = os.path.join(nativesdk_dir, NATIVESDK_ENVIRONMENT) + nativesdk_environment_path = os.path.join( + nativesdk_dir, + oebuild_util.get_nativesdk_environment(nativesdk_dir)) - with open(nativesdk_environment_dir, 'r', encoding='utf-8') as r_f: + with open(nativesdk_environment_path, 'r', encoding='utf-8') as r_f: for line in r_f.readlines(): line = line.strip('\n') if not line.startswith("export OECORE_NATIVE_SYSROOT="): diff --git a/src/oebuild/util.py b/src/oebuild/util.py index c4c8ed9..748860b 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -15,11 +15,15 @@ import os import time import random import getpass +import sys +import re from ruamel.yaml import YAML from docker.errors import DockerException +from docker.models.containers import Container from oebuild.docker_proxy import DockerProxy +from oebuild.m_log import logger from oebuild.version import __version__ CONFIG_YAML = 'config.yaml' @@ -31,8 +35,44 @@ CONTAINER_BUILD = '/home/openeuler/build' DEFAULT_DOCKER = "swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container:latest" CONTAINER_SRC = '/usr1/openeuler/src' CONTAINER_USER = "openeuler" -SDK_ABSOLATE_PATH = "/opt/buildtools/nativesdk/environment-setup-x86_64-pokysdk-linux" - +NATIVESDK_DIR = "/opt/buildtools/nativesdk" + +def get_nativesdk_environment(nativesdk_dir=NATIVESDK_DIR, container: Container = None): + ''' + return environment initialization shell, if nativesdk directory is not exists + or can not find any initialization shell, raise error + ''' + if container is None: + if not os.path.isdir(nativesdk_dir): + logger.error("the %s directory is not exists", nativesdk_dir) + sys.exit(1) + # list items in nativesdk to find environment shell + list_items = os.listdir(nativesdk_dir) + for item in list_items: + ret = re.match("^(environment-setup-)", item) + if ret is not None: + abs_path = os.path.join(nativesdk_dir, item) + if os.path.isfile(abs_path) and not os.path.islink(abs_path): + return item + else: + res = container.exec_run("ls -al", user=CONTAINER_USER, workdir=nativesdk_dir) + if res.exit_code != 0: + logger.error("can not find any nativesdk environment initialization shell") + sys.exit(res.exit_code) + list_items = res.output.decode("utf-8").split("\n") + for item in list_items: + item:str = item + # notice: the item is like format with "drwxr-xr-x 3 openeuler openeuler 4096 Nov 8 08:10 ." + # so we must get the last clip from split with space + item_split = item.split(" ") + if len(item_split) <= 0: + continue + ret = re.match("^(environment-setup-)", item_split[len(item_split) - 1]) + if ret is not None and item_split[0].startswith("-"): + return item_split[len(item_split) - 1] + + logger.error("can not find any nativesdk environment initialization shell") + sys.exit(1) def read_yaml(yaml_dir : pathlib.Path): ''' -- Gitee