diff --git a/src/oebuild/app/conf/plugins.yaml b/src/oebuild/app/conf/plugins.yaml index e4b6de0ba683691db2f1de8171b96404b3e89e55..aa2541f736dc7fbd8364e0ca42f504b796d1849c 100644 --- a/src/oebuild/app/conf/plugins.yaml +++ b/src/oebuild/app/conf/plugins.yaml @@ -32,3 +32,6 @@ plugins: - name: mplugin class: MPlugin path: plugins/m_plugin/m_plugin.py +- name: toolchain + class: Toolchain + path: plugins/toolchain/toolchain.py diff --git a/src/oebuild/app/plugins/bitbake/base_build.py b/src/oebuild/app/plugins/bitbake/base_build.py index b0c830db8efc1f9d8da074eaa9f63556339c94fa..39e3bcc5e7045a17e0534dc1ef9a00ef83ae6b36 100644 --- a/src/oebuild/app/plugins/bitbake/base_build.py +++ b/src/oebuild/app/plugins/bitbake/base_build.py @@ -11,10 +11,9 @@ See the Mulan PSL v2 for more details. ''' import os -import oebuild.util as oebuild_util +import oebuild.const as oebuild_const from oebuild.local_conf import LocalConf from oebuild.bblayers import BBLayers -import oebuild.const as oebuild_const class BaseBuild: @@ -22,23 +21,13 @@ class BaseBuild: The class provides the basic methods that the build class inherits ''' - def _set_tmpfile_content(self, content: str): - while True: - tmp_file = oebuild_util.generate_random_str(6) - if os.path.exists(tmp_file): - continue - with open(tmp_file, 'w', encoding="utf-8") as w_f: - w_f.write(content) - break - return tmp_file - - def replace_local_conf(self, parse_compile, local_dir, src_dir=None): + def replace_local_conf(self, compile_param, local_path, src_dir=None): ''' replace some param in local.conf, the LocalConf will be instantiated and exec update ''' - local_conf = LocalConf(local_conf_dir=local_dir) - local_conf.update(parse_compile=parse_compile, src_dir=src_dir) + local_conf = LocalConf(local_path) + local_conf.update(compile_param, src_dir) def add_bblayers(self, bblayers_dir: str, pre_dir: str, base_dir: str, layers): ''' @@ -49,29 +38,5 @@ class BaseBuild: ''' bblayers = BBLayers(bblayers_dir=bblayers_dir, base_dir=base_dir) - pre_dir = os.path.join(pre_dir, 'yocto-poky/..') + pre_dir = os.path.join(pre_dir, f'{oebuild_const.YOCTO_POKY}/..') bblayers.add_layer(pre_dir=pre_dir, layers=layers) - - def _restore_bashrc_content(self, old_content): - new_content = '' - for line in old_content.split('\n'): - line: str = line - if line.endswith(oebuild_const.BASH_END_FLAG) or line.replace(" ", '') == '': - continue - new_content = new_content + line + '\n' - return new_content - - def _add_bashrc(self, content: str, line: str): - if not content.endswith('\n'): - content = content + '\n' - content = content + line + oebuild_const.BASH_END_FLAG + '\n' - - return content - - def _init_bashrc_content(self, old_content, init_command: list): - new_content = self._restore_bashrc_content(old_content=old_content) - - for command in init_command: - new_content = new_content + command + oebuild_const.BASH_END_FLAG + '\n' - - return new_content diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index 17b367841227b893935e78a891590b3930f1943f..a65cb6398996bda822ede2830b9e798345b21f3d 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -19,7 +19,8 @@ from docker.errors import DockerException from oebuild.command import OebuildCommand from oebuild.configure import Configure -from oebuild.parse_compile import ParseCompile, CheckCompileError +from oebuild.struct import CompileParam +from oebuild.parse_param import ParseCompileParam from oebuild.parse_env import ParseEnv import oebuild.util as oebuild_util from oebuild.app.plugins.bitbake.in_container import InContainer @@ -87,23 +88,23 @@ class Bitbake(OebuildCommand): if not os.path.exists('.env'): os.mknod('.env') - try: - parse_compile = ParseCompile(self.compile_conf_dir) - except CheckCompileError as c_e: - logger.error(str(c_e)) - sys.exit(-1) + compile_param_dict = oebuild_util.read_yaml(self.compile_conf_dir) + compile_param: CompileParam = ParseCompileParam.parse_to_obj( + compile_param_dict) # if has manifest.yaml, init layer repo with it yocto_dir = os.path.join(self.configure.source_dir(), "yocto-meta-openeuler") manifest_path = os.path.join(yocto_dir, ".oebuild/manifest.yaml") - parse_compile.check_with_version(self.configure.source_dir(), - manifest_path=manifest_path) + oebuild_util.download_repo_from_manifest( + repo_list=compile_param.repos, + src_dir=self.configure.source_dir(), + manifest_path=manifest_path) parse_env = ParseEnv(env_dir='.env') - if parse_compile.build_in == oebuild_const.BUILD_IN_HOST: + if compile_param.build_in == oebuild_const.BUILD_IN_HOST: in_host = InHost(self.configure) - in_host.exec(parse_compile=parse_compile, command=command) + in_host.exec(compile_param=compile_param, command=command) sys.exit(0) try: @@ -114,7 +115,7 @@ class Bitbake(OebuildCommand): in_container = InContainer(self.configure) in_container.exec(parse_env=parse_env, - parse_compile=parse_compile, + compile_param=compile_param, command=command) def check_support_bitbake(self, ): diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index 4a0cb5fadd7335600014296edca3045796407b3e..abf26593619803d6eb375035c0946fe554871a4f 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -9,46 +9,22 @@ EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -import re import os import sys from docker.models.containers import Container, ExecResult -from oebuild.parse_env import ParseEnv, EnvContainer +from oebuild.parse_env import ParseEnv from oebuild.docker_proxy import DockerProxy from oebuild.configure import Configure -from oebuild.parse_compile import ParseCompile, DockerParam +from oebuild.struct import CompileParam, DockerParam from oebuild.m_log import logger from oebuild.app.plugins.bitbake.base_build import BaseBuild +from oebuild.bashrc import Bashrc import oebuild.util as oebuild_util import oebuild.const as oebuild_const -def split_command(generate_command): - """ - - Args: - generate_command: generate command - - Returns: - - """ - if generate_command.split(' ')[-1] == 'toolchain': - command = """ - ./cross-tools/update.sh ###!!!###\n - cp config_aarch64 .config && ct-ng build ###!!!###\n - cp config_aarch64-musl .config && ct-ng build ###!!!###\n - cp config_arm32 .config && ct-ng build ###!!!###\n - cp config_x86_64 .config && ct-ng build ###!!!###\n - cp config_riscv64 .config && ct-ng build ###!!!###\n""" - else: - command = "./cross-tools/update.sh ###!!!###\n" - for command_info in generate_command.split(' ')[2:]: - command = f"{command}cp {command_info} .config && ct-ng build ###!!!###\n" - return command - - class InContainer(BaseBuild): ''' bitbake command execute in container @@ -58,36 +34,32 @@ class InContainer(BaseBuild): self.configure = configure self.client = DockerProxy() self.container_id = None + self.bashrc = None - def exec(self, parse_env: ParseEnv, parse_compile: ParseCompile, command): + def exec(self, parse_env: ParseEnv, compile_param: CompileParam, command): ''' execute bitbake command ''' logger.info("Bitbake starting ...") - docker_param: DockerParam = None - if parse_compile.docker_param is not None: - docker_param = parse_compile.docker_param - else: - docker_param = self._trans_docker_param( - docker_image=parse_compile.docker_image, - toolchain_dir=parse_compile.toolchain_dir, - sstate_cache=parse_compile.sstate_cache) - # check docker image if exists docker_proxy = DockerProxy() + docker_param = compile_param.docker_param if not docker_proxy.is_image_exists(docker_param.image): logger.error('''The docker image does not exists, please run fellow command: `oebuild update docker`''') sys.exit(-1) - self.deal_env_container(env=parse_env, docker_param=docker_param) - self.exec_compile(parse_compile=parse_compile, command=command) + self.container_id = oebuild_util.deal_env_container( + env=parse_env, docker_param=docker_param) + self.bashrc = Bashrc() + self.bashrc.set_container(container=self.client.get_container(self.container_id)) + self.exec_compile(compile_param=compile_param, command=command) def _trans_docker_param(self, docker_image: str, toolchain_dir: str = None, - sstate_cache: str = None) -> DockerParam: + sstate_mirrors: str = None) -> DockerParam: ''' this function is to adapt the old compile.yaml ''' @@ -99,8 +71,8 @@ class InContainer(BaseBuild): os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()))) if toolchain_dir is not None: volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) - if sstate_cache is not None: - volumns.append(sstate_cache + ":" + oebuild_const.SSTATE_CACHE) + if sstate_mirrors is not None: + volumns.append(sstate_mirrors + ":" + oebuild_const.SSTATE_MIRRORS) docker_param = DockerParam( image=docker_image, @@ -110,35 +82,7 @@ class InContainer(BaseBuild): ) return docker_param - def deal_env_container(self, env: ParseEnv, docker_param: DockerParam): - ''' - This operation realizes the processing of the container, - controls how the container is processed by parsing the env - variable, if the container does not exist, or the original - environment and the current environment that needs to be set - are inconsistent, you need to create a new container, otherwise - directly enable the sleeping container - ''' - if env.container is None \ - or env.container.short_id is None \ - or not self.client.is_container_exists(env.container.short_id): - # judge which container - container: Container = self.client.create_container( - image=docker_param.image, - parameters=docker_param.parameters, - volumes=docker_param.volumns, - command=docker_param.command) - - env_container = EnvContainer(container.short_id) - env.set_env_container(env_container) - env.export_env() - - self.container_id = env.container.short_id - container: Container = self.client.get_container(self.container_id) # type: ignore - if not self.client.is_container_running(container): - self.client.start_container(container) - - def exec_compile(self, parse_compile: ParseCompile, command: str = ""): + def exec_compile(self, compile_param: CompileParam, command: str = ""): ''' execute compile task ''' @@ -159,20 +103,19 @@ class InContainer(BaseBuild): bblayers_dir=bblayers_dir, pre_dir=oebuild_const.CONTAINER_SRC, base_dir=self.configure.source_dir(), - layers=parse_compile.layers) + layers=compile_param.layers) # replace local_conf - local_dir = os.path.join(os.getcwd(), 'conf', 'local.conf') + local_path = os.path.join(os.getcwd(), 'conf', 'local.conf') self.replace_local_conf( - parse_compile=parse_compile, local_dir=local_dir) + compile_param=compile_param, + local_path=local_path) # add auto execute command for example: bitbake busybox if command is not None and command != "": - content = self._get_bashrc_content(container=container) - if "bitbake toolchain" in command: - command = split_command(command) - new_content = self._add_bashrc(content=content, line=command) - self.update_bashrc(container=container, content=new_content) + content = self.bashrc.get_bashrc_content() + new_content = self.bashrc.add_bashrc(content=content, line=command) + self.bashrc.update_bashrc(content=new_content) res: ExecResult = self.client.container_exec_command( container=container, command="bash .bashrc", @@ -190,15 +133,15 @@ class InContainer(BaseBuild): logger.info(line[0].decode().strip('\n')) sys.exit(exit_code) else: - content = self._get_bashrc_content(container=container) + content = self.bashrc.get_bashrc_content() for b_s in oebuild_const.BASH_BANNER.split('\n'): b_s = f"echo {b_s}" - content = self._add_bashrc(content=content, line=b_s) - self.update_bashrc(container=container, content=content) + content = self.bashrc.add_bashrc(content=content, line=b_s) + self.bashrc.update_bashrc(content=content) os.system( f"docker exec -it -u {oebuild_const.CONTAINER_USER} {container.short_id} bash") - self.restore_bashrc(container=container) + self.bashrc.restore_bashrc() def init_bitbake(self, container: Container): ''' @@ -206,7 +149,9 @@ class InContainer(BaseBuild): bblayers.conf and local.conf if exists in 10 seconds, otherwise raise init bitbake faild ''' - self._check_change_ugid(container=container) + self.client.check_change_ugid( + container=container, + container_user=oebuild_const.CONTAINER_USER) self._install_sudo(container=container) res = self.client.container_exec_command( @@ -221,64 +166,6 @@ class InContainer(BaseBuild): raise ValueError(res.output.decode()) # raise ValueError("bitbake init faild") - def _check_change_ugid(self, container: Container): - res = self.client.container_exec_command( - container=container, - user='root', - command=f"id {oebuild_const.CONTAINER_USER}", - params={ - "work_space": f"/home/{oebuild_const.CONTAINER_USER}", - "stream": False - }) - - if res.exit_code != 0: - raise ValueError("check docker user id faild") - - res_cont = res.output.decode() - - cuids = res_cont.split(' ') - # get uid from container in default user - pattern = re.compile(r'(?<=uid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))') - match_uid = pattern.search(cuids[0]) - if match_uid: - cuid = match_uid.group() - else: - raise ValueError(f"can not get container {oebuild_const.CONTAINER_USER} uid") - # get gid from container in default user - pattern = re.compile(r'(?<=gid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))') - match_gid = pattern.search(cuids[1]) - if match_gid: - cgid = match_gid.group() - else: - raise ValueError(f"can not get container {oebuild_const.CONTAINER_USER} gid") - - # judge host uid and gid are same with container uid and gid - # if not same and change container uid and gid equal to host's uid and gid - if os.getuid() != cuid: - self._change_container_uid(container=container, uid=os.getuid()) - if os.getgid() != cgid: - self._change_container_gid(container=container, gid=os.getgid()) - - def _change_container_uid(self, container: Container, uid: int): - self.client.container_exec_command( - container=container, - user='root', - command=f"usermod -u {uid} {oebuild_const.CONTAINER_USER}", - params={ - "work_space": f"/home/{oebuild_const.CONTAINER_USER}", - "stream": False - }) - - def _change_container_gid(self, container: Container, gid: int): - self.client.container_exec_command( - container=container, - user='root', - command=f"groupmod -g {gid} {oebuild_const.CONTAINER_USER}", - params={ - "work_space": f"/home/{oebuild_const.CONTAINER_USER}", - "stream": False - }) - def _install_sudo(self, container: Container): self._replace_yum_mirror(container=container) @@ -335,7 +222,7 @@ sed -i 's/repo.openeuler.org/mirrors.huaweicloud.com\/openeuler/g' /etc/yum.repo substitutions, and finally writing the initialization script ''' # read container default user .bashrc content - content = self._get_bashrc_content(container=container) + content = self.bashrc.get_bashrc_content() # get host proxy information and set in container host_proxy = oebuild_util.get_host_proxy(oebuild_const.PROXY_LIST) @@ -351,9 +238,9 @@ sed -i 's/repo.openeuler.org/mirrors.huaweicloud.com\/openeuler/g' /etc/yum.repo init_oe_comand = f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ {oebuild_const.CONTAINER_BUILD}/{build_dir_name}' init_command = [init_proxy_command, init_sdk_command, set_template, init_oe_comand] - new_content = self._init_bashrc_content(content, init_command) + new_content = self.bashrc.init_bashrc_content(content, init_command) - self.update_bashrc(container=container, content=new_content) + self.bashrc.update_bashrc(content=new_content) def _set_container_proxy(self, host_proxy): init_proxy_command = "" @@ -362,47 +249,3 @@ sed -i 's/repo.openeuler.org/mirrors.huaweicloud.com\/openeuler/g' /etc/yum.repo command = f'export {key}={value}; git config --global {key_git} {value};' init_proxy_command = f'{init_proxy_command} {command}' return init_proxy_command - - def update_bashrc(self, container: Container, content: str): - ''' - update user initialization script by replace file, first create - a file and writed content and copy it to container's .bashrc, finally - remove it - ''' - tmp_file = self._set_tmpfile_content(content) - self.client.copy_to_container( - container=container, - source_path=tmp_file, - to_path=f'/home/{oebuild_const.CONTAINER_USER}') - container.exec_run( - cmd=f''' -mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINER_USER}/.bashrc - ''', - user="root" - ) - os.remove(tmp_file) - - def restore_bashrc(self, container: Container): - ''' - Restoring .bashrc will strip out the command line - content added during bitbake initialization - ''' - old_content = self._get_bashrc_content(container=container) - self.update_bashrc(container=container, - content=self._restore_bashrc_content(old_content=old_content)) - - def _get_bashrc_content(self, container: Container): - res = self.client.container_exec_command( - container=container, - command=f"cat /home/{oebuild_const.CONTAINER_USER}/.bashrc", - user="root", - params={ - "work_space": None, - "stream": False - }) - - if res.exit_code != 0: - logger.error(res.output) - sys.exit(1) - - return res.output.decode() diff --git a/src/oebuild/app/plugins/bitbake/in_host.py b/src/oebuild/app/plugins/bitbake/in_host.py index 8ea2e06efa23e4f45be50227f5bcabec26548829..1a0500f87227186ba9840b1cacbbd8f2c9996113 100644 --- a/src/oebuild/app/plugins/bitbake/in_host.py +++ b/src/oebuild/app/plugins/bitbake/in_host.py @@ -12,14 +12,14 @@ See the Mulan PSL v2 for more details. import os import subprocess import pty -import shutil import sys from oebuild.local_conf import NativesdkNotExist, NativesdkNotValid from oebuild.configure import Configure -from oebuild.parse_compile import ParseCompile +from oebuild.parse_param import CompileParam from oebuild.m_log import logger from oebuild.app.plugins.bitbake.base_build import BaseBuild +from oebuild.bashrc import Bashrc import oebuild.util as oebuild_util import oebuild.const as oebuild_const @@ -32,13 +32,14 @@ class InHost(BaseBuild): def __init__(self, configure: Configure): self.configure = configure self.container_id = None + self.bashrc = Bashrc() - def exec(self, parse_compile: ParseCompile, command): + def exec(self, compile_param: CompileParam, command): ''' execute bitbake commands ''' self._init_build_sh(build_dir=os.getcwd()) - self._mk_build_sh(nativesdk_dir=parse_compile.nativesdk_dir, build_dir=os.getcwd()) + self._mk_build_sh(nativesdk_dir=compile_param.nativesdk_dir, build_dir=os.getcwd()) self.init_bitbake() # add bblayers, this action must before replace local_conf @@ -47,13 +48,13 @@ class InHost(BaseBuild): bblayers_dir=bblayers_dir, pre_dir=self.configure.source_dir(), base_dir=self.configure.source_dir(), - layers=parse_compile.layers) + layers=compile_param.layers) - local_dir = os.path.join(os.getcwd(), 'conf', 'local.conf') + local_path = os.path.join(os.getcwd(), 'conf', 'local.conf') try: self.replace_local_conf( - parse_compile=parse_compile, - local_dir=local_dir, + compile_param=compile_param, + local_path=local_path, src_dir=self.configure.source_dir()) except NativesdkNotExist as n_e: logger.error(str(n_e)) @@ -82,10 +83,11 @@ initialization operations''') build_sh_dir = os.path.join(os.getcwd(), 'build.sh') source_build_str = f"source {build_sh_dir}" - content = self._get_bashrc_content() - content = self._restore_bashrc_content(old_content=content) - new_content = self._add_bashrc(content, line=source_build_str) - self.update_bashrc(new_content) + # content = self.bashrc.get_bashrc_content() + content = self.bashrc.get_bashrc_content() + content = self.bashrc.restore_bashrc_content(old_content=content) + new_content = self.bashrc.add_bashrc(content, line=source_build_str) + self.bashrc.update_bashrc(content=new_content) pty.spawn("bash") def _bash_build(self,): @@ -150,14 +152,3 @@ initialization operations''') substitutions, and finally writing the initialization script ''' subprocess.getoutput("bash build.sh") - - def _get_bashrc_content(self,): - return subprocess.getoutput('cat $HOME/.bashrc') - - def update_bashrc(self, content: str): - ''' - update user initialization script by replace file, first create - a file and writed content and mv it to host's .bashrc - ''' - tmp_file = self._set_tmpfile_content(content) - shutil.move(tmp_file, os.path.join(os.environ['HOME'], '.bashrc')) diff --git a/src/oebuild/app/plugins/deploy/com_target.py b/src/oebuild/app/plugins/deploy/com_target.py index 0066cc0f45f4285c8bf9db28ec9d384751c2754d..24f221d2a7c5bb5f7ecfc3030d25b2b7a1d22748 100644 --- a/src/oebuild/app/plugins/deploy/com_target.py +++ b/src/oebuild/app/plugins/deploy/com_target.py @@ -13,7 +13,6 @@ See the Mulan PSL v2 for more details. import os import sys import re -import logging from docker.models.containers import Container from docker.errors import DockerException @@ -21,12 +20,10 @@ from docker.errors import DockerException from oebuild.docker_proxy import DockerProxy from oebuild.configure import Configure import oebuild.util as oebuild_util -from oebuild.parse_compile import ParseCompile, CheckCompileError +from oebuild.parse_param import ParseCompileParam from oebuild.m_log import logger import oebuild.const as oebuild_const -logger = logging.getLogger() - TARGET_DIR_NAME = "target_dev" TARGET_SCRIPT_NAME = "oebuild_dev" @@ -51,7 +48,7 @@ class ComTarget: self.client.delete_container(container=container, is_force=True) except DockerException: print(f""" -the container {self.container_id} failed to be destroyed, please run +the container {self.container_id} failed to be destroyed, please run `docker rm {self.container_id}` @@ -79,9 +76,9 @@ the container {self.container_id} failed to be destroyed, please run sys.exit(-1) if not self._check_conf_directory(): - logger.error(''' - You must work in a exist build directory, that mean you built before or initialize environment at least - ''') + logger.error('You must work in a exist build directory, ' + 'that mean you built before or initialize ' + 'environment at least') sys.exit(-1) logger.info("Initializing environment, please wait ...") @@ -96,9 +93,9 @@ the container {self.container_id} failed to be destroyed, please run line=f"export PATH=$PATH:/home/openeuler/{TARGET_DIR_NAME}") content = oebuild_util.add_bashrc( content=content, - line=f""" - mv -f /home/{oebuild_const.CONTAINER_USER}/{self.old_bashrc} /home/{oebuild_const.CONTAINER_USER}/.bashrc - """) + line=(f"mv -f /home/{oebuild_const.CONTAINER_USER}/{self.old_bashrc} " + f"/home/{oebuild_const.CONTAINER_USER}/.bashrc") + ) content = oebuild_util.add_bashrc( content=content, line=f"{TARGET_SCRIPT_NAME} {fun} {str_args}") @@ -123,11 +120,9 @@ the container {self.container_id} failed to be destroyed, please run the deploy feature should only be runed in docker build type ''' compile_path = os.path.join(self.work_dir, "compile.yaml") - try: - parse_compile = ParseCompile(compile_path) - except CheckCompileError as c_e: - logger.error(str(c_e)) - sys.exit(-1) + compile_dict = oebuild_util.read_yaml(yaml_path=compile_path) + parse_compile = ParseCompileParam.parse_to_obj(compile_param_dict=compile_dict) + if parse_compile.build_in != oebuild_const.BUILD_IN_DOCKER: return False return True @@ -194,12 +189,10 @@ the container {self.container_id} failed to be destroyed, please run old_bash = oebuild_util.generate_random_str(6) self.client.container_exec_command( container=container, - command=f""" - cp /home/{oebuild_const.CONTAINER_USER}/.bashrc /home/{oebuild_const.CONTAINER_USER}/{old_bash} - """, + command=(f"cp /home/{oebuild_const.CONTAINER_USER}/.bashrc " + f"/home/{oebuild_const.CONTAINER_USER}/{old_bash}"), user="root", - work_space=None, - stream=False) + params={'stream': False}) self.old_bashrc = old_bash def init_bash(self, container: Container): @@ -226,8 +219,7 @@ the container {self.container_id} failed to be destroyed, please run container=container, command=f"cat /home/{oebuild_const.CONTAINER_USER}/.bashrc", user="root", - work_space=None, - stream=False).output + params={'stream': False}).output return content.decode() @@ -243,9 +235,8 @@ the container {self.container_id} failed to be destroyed, please run source_path=tmp_file, to_path=f'/home/{oebuild_const.CONTAINER_USER}') container.exec_run( - cmd=f""" - mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINER_USER}/.bashrc - """, + cmd=(f"mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} " + f"/home/{oebuild_const.CONTAINER_USER}/.bashrc"), user="root" ) os.remove(tmp_file) @@ -279,12 +270,14 @@ the container {self.container_id} failed to be destroyed, please run return new_content def _check_change_ugid(self, container: Container): + params = { + 'work_space': f"/home/{oebuild_const.CONTAINER_USER}", + 'stream': False} res = self.client.container_exec_command( container=container, user='root', command=f"id {oebuild_const.CONTAINER_USER}", - work_space=f"/home/{oebuild_const.CONTAINER_USER}", - stream=False) + params=params) if res.exit_code != 0: raise ValueError("check docker user id faild") @@ -314,17 +307,23 @@ the container {self.container_id} failed to be destroyed, please run self._change_container_gid(container=container, gid=os.getgid()) def _change_container_uid(self, container: Container, uid: int): + params = { + 'work_space': f"/home/{oebuild_const.CONTAINER_USER}", + 'stream': False + } self.client.container_exec_command( container=container, user='root', command=f"usermod -u {uid} {oebuild_const.CONTAINER_USER}", - work_space=f"/home/{oebuild_const.CONTAINER_USER}", - stream=False) + params=params) def _change_container_gid(self, container: Container, gid: int): + params = { + 'work_space': f"/home/{oebuild_const.CONTAINER_USER}", + 'stream': False + } self.client.container_exec_command( container=container, user='root', command=f"groupmod -g {gid} {oebuild_const.CONTAINER_USER}", - work_space=f"/home/{oebuild_const.CONTAINER_USER}", - stream=False) + params=params) diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index 1da8b7a0b3fc7bb412147d29eb9e9f9d7bad9963..1dcaf5c34cc94d1566be380cd776500acfebd0f7 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -12,7 +12,6 @@ See the Mulan PSL v2 for more details. import argparse import re -import resource import subprocess import textwrap import os @@ -24,11 +23,13 @@ from shutil import rmtree from kconfiglib import Kconfig from menuconfig import menuconfig from prettytable import PrettyTable +from ruamel.yaml.scalarstring import LiteralScalarString from oebuild.command import OebuildCommand import oebuild.util as oebuild_util -from oebuild.configure import Configure, YoctoEnv -from oebuild.parse_template import BaseParseTemplate, ParseTemplate +from oebuild.configure import Configure +from oebuild.parse_template import BaseParseTemplate, \ + ParseTemplate, get_docker_param_dict, parse_repos_layers_local_obj from oebuild.m_log import logger, INFO_COLOR from oebuild.check_docker_tag import CheckDockerTag import oebuild.const as oebuild_const @@ -53,7 +54,7 @@ class Generate(OebuildCommand): self.configure = Configure() self.nativesdk_dir = None self.toolchain_dir = None - self.sstate_cache = None + self.sstate_mirrors = None self.tmp_dir = None self.oebuild_kconfig_path = os.path.expanduser( '~') + '/.local/oebuild_kconfig/' @@ -82,8 +83,8 @@ class Generate(OebuildCommand): ''') parser.add_argument('-s', - '--state_cache', - dest='sstate_cache', + '--state_mirrors', + dest='sstate_mirrors', help=''' this param is for SSTATE_MIRRORS ''') @@ -170,36 +171,32 @@ class Generate(OebuildCommand): This parameter marks the mode at build time, and is built in the container by docker ''') - parser.add_argument('-sdk', - '--sdk', - dest='sdk', + parser.add_argument('--nativesdk', + dest='nativesdk', action="store_true", help=''' This parameter is used to indicate whether to build an SDK ''') - parser.add_argument('-toolchain', - '--toolchain', + parser.add_argument('--toolchain', dest='toolchain', action="store_true", help=''' This parameter is used to indicate whether to build an toolchain ''') - parser.add_argument('-tn', - '--toolchain_name', + parser.add_argument('--toolchain_name', dest='toolchain_name', action='append', help=''' This parameter is used to toolchain name ''') - parser.add_argument('-at', - '--auto_build', + parser.add_argument('--auto_build', dest='auto_build', action="store_true", help=''' - This parameter is used to auto_build + This parameter is used for nativesdk and toolchain build ''') return parser @@ -233,14 +230,27 @@ class Generate(OebuildCommand): build_in = args.build_in auto_build = bool(args.auto_build) - if args.sdk: - self.build_sdk(args.build_in, args.directory, args.toolchain_dir, - args.nativesdk_dir, auto_build) + if args.nativesdk: + # this is for default directory is qemu-aarch64 + if args.directory is None or args.directory == '': + args.directory = "nativesdk" + build_dir = self._init_build_dir(args=args) + if build_dir is None: + sys.exit(0) + self.build_sdk(args.build_in, build_dir, auto_build) + self._print_nativesdk(build_dir=build_dir) sys.exit(0) if args.toolchain: + # this is for default directory is qemu-aarch64 + if args.directory is None or args.directory == '': + args.directory = "toolchain" toolchain_name_list = args.toolchain_name if args.toolchain_name else [] - self.tool_chain(toolchain_name_list, auto_build) + build_dir = self._init_build_dir(args=args) + if build_dir is None: + sys.exit(0) + self.build_toolchain(build_dir, toolchain_name_list, auto_build) + self._print_toolchain(build_dir=build_dir) sys.exit(0) if args.build_in == oebuild_const.BUILD_IN_HOST: @@ -255,8 +265,8 @@ class Generate(OebuildCommand): if args.toolchain_dir != '': self.toolchain_dir = args.toolchain_dir - if args.sstate_cache is not None: - self.sstate_cache = args.sstate_cache + if args.sstate_mirrors is not None: + self.sstate_mirrors = args.sstate_mirrors if args.tmp_dir is not None: self.tmp_dir = args.tmp_dir @@ -300,7 +310,7 @@ class Generate(OebuildCommand): if os.path.exists(os.path.join(build_dir, 'compile.yaml')): os.remove(os.path.join(build_dir, 'compile.yaml')) - docker_image = YoctoEnv().get_docker_image(yocto_dir=yocto_dir) + docker_image = oebuild_util.get_docker_image_from_yocto(yocto_dir=yocto_dir) if docker_image is None: check_docker_tag = CheckDockerTag(args.docker_tag, self.configure) oebuild_config = self.configure.parse_oebuild_config() @@ -332,19 +342,20 @@ class Generate(OebuildCommand): out_dir = pathlib.Path(os.path.join(build_dir, 'compile.yaml')) + param = parser_template.get_default_generate_compile_conf_param() + param['nativesdk'] = self.nativesdk_dir + param['toolchain_dir'] = self.toolchain_dir + param['build_in'] = build_in + param['sstate_mirrors'] = self.sstate_mirrors + param['tmp_dir'] = self.tmp_dir + param['datetime'] = args.datetime + param['is_disable_fetch'] = args.is_disable_fetch + param['docker_image'] = docker_image + param['src_dir'] = self.configure.source_dir() + param['compile_dir'] = build_dir oebuild_util.write_yaml( out_dir, - parser_template.generate_compile_conf( - nativesdk_dir=self.nativesdk_dir, - toolchain_dir=self.toolchain_dir, - build_in=build_in, - sstate_cache=self.sstate_cache, - tmp_dir=self.tmp_dir, - datetime=args.datetime, - is_disable_fetch=args.is_disable_fetch, - docker_image=docker_image, - src_dir=self.configure.source_dir(), - compile_dir=build_dir)) + parser_template.generate_compile_conf(param)) self._print_generate(build_dir=build_dir) @@ -358,6 +369,34 @@ please run follow command: cd {build_dir} oebuild bitbake +============================================= +''' + logger.info(format_dir) + + def _print_nativesdk(self, build_dir): + format_dir = f''' +generate compile.yaml successful + +please run follow command: +============================================= + +cd {build_dir} +oebuild bitbake or oebuild bitbake buildtools-extended-tarball + +============================================= +''' + logger.info(format_dir) + + def _print_toolchain(self, build_dir): + format_dir = f''' +generate toolchain.yaml successful + +please run follow command: +============================================= + +cd {build_dir} +oebuild toolchain + ============================================= ''' logger.info(format_dir) @@ -422,20 +461,26 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") if os.path.exists(build_dir): logger.warning("the build directory %s already exists", build_dir) while True: - in_res = input(""" - do you want to overwrite it? the overwrite action will replace the compile.yaml - to new and delete conf directory, enter Y for yes, N for no:""") - if in_res not in ["Y", "y", "yes", "N", "n", "no"]: + in_res = input(f""" + do you want to overwrite it({os.path.basename(build_dir)})? the overwrite + action will replace the compile.yaml to new and delete conf directory, + enter Y for yes, N for no, C for create:""") + if in_res not in ["Y", "y", "yes", "N", "n", "no", "C", "c", "create"]: print(""" wrong input""") continue if in_res in ['N', 'n', 'no']: return None + if in_res in ['C', 'c', 'create']: + in_res = input(""" + please enter now build name, we will create it under build directory:""") + build_dir = os.path.join(self.configure.build_dir(), in_res) + if os.path.exists(build_dir): + continue break if os.path.exists(os.path.join(build_dir, "conf")): rmtree(os.path.join(build_dir, "conf")) - else: - os.makedirs(build_dir) + os.makedirs(build_dir) return build_dir def list_info(self, ): @@ -669,10 +714,10 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") generate_command += ['-f', feature.lower()] if native_sdk: - generate_command = ['-sdk'] + generate_command = ['--nativesdk'] if tool_chain: - generate_command = ['-toolchain'] + generate_command = ['--toolchain'] if toolchain_list: for toolchain_info in toolchain_list: @@ -683,38 +728,53 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") return generate_command - def build_sdk(self, build_in, directory, toolchain_dir, nativesdk_dir, auto_build): + def build_sdk(self, build_in, build_dir, auto_build): """ Args: build_in: host or docker directory: build dir - toolchain_dir: toolchain dir - nativesdk_dir: nativesdk dir auto_build: auto_build - Returns: """ - args_dir = directory if directory else "nativeSDK" - subprocess.run(f'rm -rf {args_dir}', shell=True, check=False) - subprocess.run(f'oebuild generate -d {args_dir}', shell=True, check=False) - build_dir = os.path.join(self.configure.build_dir(), args_dir) - yaml_path = pathlib.Path(f"{build_dir}/compile.yaml") - if os.path.exists(yaml_path): - info = oebuild_util.read_yaml(yaml_path) - info['machine'] = 'sdk' - if build_in == 'host': - info['build_in'] = 'host' - del info['docker_param'] - info['nativesdk_dir'] = os.path.abspath(nativesdk_dir) - info['toolchain_dir'] = os.path.abspath(toolchain_dir) - oebuild_util.write_yaml(yaml_path, info) - os.chdir(build_dir) + compile_dir = os.path.join(self.configure.build_dir(), build_dir) + compile_yaml_path = f"{compile_dir}/compile.yaml" + common_yaml_path = os.path.join( + self.configure.source_yocto_dir(), '.oebuild/common.yaml') + repos, layers, local_conf = parse_repos_layers_local_obj(common_yaml_path) + info = { + 'repos': repos, + 'layers': layers, + 'local_conf': local_conf + } + if build_in == 'host': + info['build_in'] = 'host' + else: + docker_image = get_docker_image( + yocto_dir=self.configure.source_yocto_dir(), + docker_tag="latest", + configure=self.configure + ) + info['docker_param'] = get_docker_param_dict( + docker_image=docker_image, + src_dir=self.configure.source_dir(), + compile_dir=compile_dir, + toolchain_dir=None, + sstate_mirrors=None + ) + # add nativesdk conf + nativesdk_yaml_path = os.path.join( + self.configure.source_yocto_dir(), '.oebuild/nativesdk/local.conf') + with open(nativesdk_yaml_path, 'r', encoding='utf-8') as f: + local_conf += f.read()+"\n" + info['local_conf'] = LiteralScalarString(local_conf) + oebuild_util.write_yaml(compile_yaml_path, info) if auto_build: + os.chdir(compile_dir) subprocess.run('oebuild bitbake buildtools-extended-tarball', shell=True, check=False) - def tool_chain(self, toolchain_name_list, auto_build): + def build_toolchain(self, build_dir, toolchain_name_list, auto_build): """ Args: @@ -724,31 +784,69 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") Returns: """ - cross_path = os.path.join(self.configure.source_yocto_dir(), ".oebuild/cross-tools") - if not os.path.exists(cross_path): + source_cross_dir = os.path.join(self.configure.source_yocto_dir(), ".oebuild/cross-tools") + if not os.path.exists(source_cross_dir): logger.error('Build dependency not downloaded, not supported for build. Please ' 'download the latest yocto meta openeuler repository') sys.exit(-1) - toolchain_list = os.listdir(os.path.join(cross_path, 'configs')) - for config in toolchain_list: - if re.search('xml', config): - toolchain_list.remove(config) - if toolchain_name_list and not set(toolchain_name_list).issubset(toolchain_list): - logger.error(f'toolchain name not exists, toolchain list is {toolchain_list}') - sys.exit(-1) - build_dir = os.path.join(self.configure.build_dir(), 'toolchain') - if os.path.exists(build_dir): - os.chdir(os.path.dirname(build_dir)) - subprocess.run(f'rm -rf {build_dir}', shell=True, check=False) - subprocess.run('oebuild generate -d toolchain', shell=True, check=False) - subprocess.check_output(f'cp -r {cross_path} {build_dir}', shell=True) - os.chdir(build_dir) - resource.setrlimit(resource.RLIMIT_NOFILE, (1048576, 1048576)) - subprocess.check_output('./cross-tools/prepare.sh ../toolchain/', shell=True) - logger.info('build toolchain...') - toolchain_name = ' ' + ' '.join(toolchain_name_list) if toolchain_name_list else '' + # add toolchain.yaml to compile + docker_param = get_docker_param_dict( + docker_image=oebuild_const.DEFAULT_DOCKER, + src_dir=self.configure.source_dir(), + compile_dir=build_dir, + toolchain_dir=None, + sstate_mirrors=None) + config_list = [] + for toolchain_name in toolchain_name_list: + if toolchain_name.startwith("config_"): + config_list.append(toolchain_name) + continue + config_list.append("config_" + toolchain_name) + oebuild_util.write_yaml( + yaml_path=os.path.join(build_dir, 'toolchain.yaml'), + data={ + 'config_list': config_list, + 'docker_param': docker_param + } + ) if auto_build: - subprocess.run(f'oebuild bitbake toolchain{toolchain_name}', shell=True, check=False) + subprocess.run('oebuild toolchain auto', shell=True, check=False) + + +def get_docker_image(yocto_dir, docker_tag, configure: Configure): + ''' + xxx + ''' + docker_image = oebuild_util.get_docker_image_from_yocto(yocto_dir=yocto_dir) + if docker_image is None: + check_docker_tag = CheckDockerTag(docker_tag, configure) + oebuild_config = configure.parse_oebuild_config() + if check_docker_tag.get_tag() is not None: + docker_tag = str(check_docker_tag.get_tag()) + else: + # select docker image + while True: + print(''' +If the system does not recognize which container image to use, select the +following container, enter it numerically, and enter q to exit:''') + image_list = check_docker_tag.get_tags() + + for key, value in enumerate(image_list): + print( + f"{key}, {oebuild_config.docker.repo_url}:{value}") + k = input("please entry number:") + if k == "q": + sys.exit(0) + try: + index = int(k) + docker_tag = image_list[index] + break + except IndexError: + print("please entry true number") + docker_tag = docker_tag.strip() + docker_tag = docker_tag.strip('\n') + docker_image = f"{oebuild_config.docker.repo_url}:{docker_tag}" + return docker_image def basic_config(): @@ -769,7 +867,7 @@ def basic_config(): basic_str = textwrap.dedent(f""" comment " THIS IS BASIC CONFIG " config BASIC-SSTATE_CACHE--S - string "sstate_cache (this param is for SSTATE_MIRRORS)" + string "sstate_mirrors (this param is for SSTATE_MIRRORS)" default "None" config BASIC-SSTATE_DIR--S_DIR string "sstate_dir (this param is for SSTATE_DIR)" diff --git a/src/oebuild/app/plugins/manifest/manifest.py b/src/oebuild/app/plugins/manifest/manifest.py index cb488a27a4c0ab113debc782468c2fb6e04d9925..b373077a96ee6de8f887e04148d5262bfcc30867 100644 --- a/src/oebuild/app/plugins/manifest/manifest.py +++ b/src/oebuild/app/plugins/manifest/manifest.py @@ -114,7 +114,7 @@ class Manifest(OebuildCommand): sys.stdout.flush() print() manifest_list = dict(sorted(manifest_list.items(), key=lambda s: s[0])) - oebuild_util.write_yaml(yaml_dir=pathlib.Path(manifest_dir), + oebuild_util.write_yaml(yaml_path=pathlib.Path(manifest_dir), data={'manifest_list': manifest_list}) self._add_manifest_banner(manifest_dir=os.path.abspath(manifest_dir)) diff --git a/src/oebuild/app/plugins/toolchain/toolchain.py b/src/oebuild/app/plugins/toolchain/toolchain.py new file mode 100644 index 0000000000000000000000000000000000000000..613e712baec1106b46fbedd16ca5d59691319e3c --- /dev/null +++ b/src/oebuild/app/plugins/toolchain/toolchain.py @@ -0,0 +1,186 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import textwrap +import argparse +import sys +import os +import subprocess + +from docker.models.containers import ExecResult + +from oebuild.command import OebuildCommand +from oebuild.m_log import logger +import oebuild.util as oebuild_util +import oebuild.const as oebuild_const +from oebuild.parse_param import ParseToolchainParam +from oebuild.parse_env import ParseEnv +from oebuild.bashrc import Bashrc +from oebuild.docker_proxy import DockerProxy +from oebuild.configure import Configure + + +class Toolchain(OebuildCommand): + ''' + The toolchain provides the ability to build a cross-compilation chain. + ''' + + help_msg = 'generate manifest from oebuild workspace' + description = textwrap.dedent('''\ + manifest provides the manifest function of generating dependent + source repositories in the build working directory, and can restore + relevant source repositories based on the manifest file + ''') + + def __init__(self): + self._toolchain_yaml_path = os.path.join(os.getcwd(), 'toolchain.yaml') + self.bashrc = Bashrc() + self.client = DockerProxy() + self.container_id = None + + super().__init__('toolchain', self.help_msg, self.description) + + def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: + parser = self._parser(parser_adder, + usage=''' + + %(prog)s [auto | ] + +''') + + return parser + + def do_run(self, args: argparse.Namespace, unknown=None): + if '-h' in unknown or '--help' in unknown: + self.print_help_msg() + sys.exit(0) + + if not self._check_support_toolchain(): + logger.error( + "Please do it in compile workspace which contain toolchain.yaml") + sys.exit(-1) + + toolchain_dict = oebuild_util.read_yaml(self._toolchain_yaml_path) + toolchain_obj = ParseToolchainParam().parse_to_obj(toolchain_param_dict=toolchain_dict) + + if not os.path.exists('.env'): + os.mknod('.env') + parse_env = ParseEnv(env_dir='.env') + + self._prepare_build_env() + + self.container_id = oebuild_util.deal_env_container( + env=parse_env, docker_param=toolchain_obj.docker_param) + self.bashrc.set_container( + container=self.client.get_container(container_id=self.container_id)) + self.client.check_change_ugid( + container=self.client.get_container(container_id=self.container_id), + container_user=oebuild_const.CONTAINER_USER) + + if unknown is None or len(unknown) == 0: + content = self.bashrc.get_bashrc_content() + for b_s in oebuild_const.TOOLCHAIN_BASH_BANNER.split('\n'): + b_s = f"echo {b_s}" + content = self.bashrc.add_bashrc(content=content, line=b_s) + self.bashrc.update_bashrc(content=content) + # 调起容器环境 + build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) + docker_exec_list = [ + "docker", + "exec", + "-it", + "-u", + oebuild_const.CONTAINER_USER, + "-w", + build_dir, + self.container_id, + "bash" + ] + os.system(" ".join(docker_exec_list)) + elif unknown[0] == "auto": + self.auto_build(config_list=toolchain_obj.config_list) + else: + cross_tools_dir = os.path.join( + Configure().source_yocto_dir(), ".oebuild/cross-tools/configs") + config_list = os.listdir(cross_tools_dir) + config_name: str = unknown[0] + if not config_name.startswith("config_"): + config_name = f"config_{config_name}" + if config_name not in config_list: + logger.error("please enter valid toolchain name") + print("the valid toolchain list:") + for config in config_list: + if config.startswith("config_"): + print(config) + print("you can run oebuild toolchain aarch64 or oebuild toolchain config_aarch64") + self._build_toolchain(config_name=config_name) + self.bashrc.restore_bashrc() + + def auto_build(self, config_list): + ''' + is for auto build toolchains + ''' + for config in config_list: + self._build_toolchain(config_name=config) + + def _build_toolchain(self, config_name): + ''' + build toolchain with config + ''' + content = self.bashrc.get_bashrc_content() + src_config_dir = os.path.join( + oebuild_const.CONTAINER_SRC, + "yocto-meta-openeuler", + ".oebuild", + "cross-tools", + "configs") + content = self.bashrc.add_bashrc(content=content, line="./cross-tools/prepare.sh ./") + content = self.bashrc.add_bashrc( + content=content, + line=f"cp {src_config_dir}/{config_name} .config && ct-ng build") + self.bashrc.update_bashrc(content=content) + res: ExecResult = self.client.container_exec_command( + container=self.client.get_container(self.container_id), + command="bash .bashrc", + user=oebuild_const.CONTAINER_USER, + params={ + "work_space": f"/home/{oebuild_const.CONTAINER_USER}", + "demux": True + }) + exit_code = 0 + for line in res.output: + if line[1] is not None: + logger.error(line[1].decode().strip('\n')) + exit_code = 1 + else: + logger.info(line[0].decode().strip('\n')) + if exit_code != 0: + sys.exit(exit_code) + + def _check_support_toolchain(self): + return os.path.exists(self._toolchain_yaml_path) + + def _prepare_build_env(self): + # create cross-tools symblic + src_cross_dir = os.path.join( + oebuild_const.CONTAINER_SRC, + "yocto-meta-openeuler", + ".oebuild", + "cross-tools") + subprocess.run(f'ln -sf {src_cross_dir} cross-tools', shell=True, check=False) + # create config.xml symblic + cross_tools_dir = os.path.join( + Configure().source_yocto_dir(), ".oebuild/cross-tools/configs") + config_dir_list = os.listdir(cross_tools_dir) + for config_path in config_dir_list: + src_config_path = os.path.join(src_cross_dir, f"configs/{config_path}") + subprocess.run(f'ln -sf {src_config_path} {config_path}', shell=True, check=False) diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py index 9a3ef1f1927c003c30f03d274f7e7cbe10fb81e6..8891ba7648d8cf4d64ac963b6fc43f97621fb3a6 100644 --- a/src/oebuild/app/plugins/update/update.py +++ b/src/oebuild/app/plugins/update/update.py @@ -14,15 +14,13 @@ import argparse import os import textwrap import sys -import pathlib from docker.errors import DockerException import oebuild.util as oebuild_util from oebuild.command import OebuildCommand -from oebuild.parse_template import OebuildRepo -from oebuild.parse_compile import ParseCompile -from oebuild.configure import Configure, ConfigBasicRepo, YoctoEnv +from oebuild.parse_param import ParseCompileParam +from oebuild.configure import Configure, ConfigBasicRepo from oebuild.docker_proxy import DockerProxy from oebuild.ogit import OGit from oebuild.check_docker_tag import CheckDockerTag @@ -95,11 +93,6 @@ class Update(OebuildCommand): logger.error('Your current directory had not finished init') sys.exit(-1) - # if args.list is not None: - # if args.list == "docker": - # self.list_image_tag() - # return - update_yocto, update_docker, update_layer = False, False, False if args.item is None: update_yocto, update_docker, update_layer = True, True, True @@ -150,30 +143,22 @@ class Update(OebuildCommand): # or /compile.yaml where in build directory repos = None if os.path.exists(os.path.join(os.getcwd(), "compile.yaml")): - parse_compile = ParseCompile( - compile_conf_dir=os.path.join(os.getcwd(), "compile.yaml")) - repos = parse_compile.repos + compile_param_dict = oebuild_util.read_yaml(os.path.join(os.getcwd(), "compile.yaml")) + compile_param = ParseCompileParam.parse_compile_param_to_obj( + compile_param_dict=compile_param_dict) + repos = compile_param.repos else: - common_path = pathlib.Path( - os.path.join(yocto_dir, ".oebuild/common.yaml")) - repos = oebuild_util.read_yaml(yaml_dir=common_path)['repos'] + common_path = os.path.join(yocto_dir, ".oebuild/common.yaml") + repos = oebuild_util.trans_dict_key_to_list( + oebuild_util.read_yaml(yaml_path=common_path)['repos']) if repos is None: - return - for _, item in repos.items(): - if isinstance(item, OebuildRepo): - local_dir = os.path.join(self.configure.source_dir(), - item.path) - key_repo = OGit(repo_dir=local_dir, - remote_url=item.url, - branch=item.refspec) - else: - local_dir = os.path.join(self.configure.source_dir(), - item['path']) - key_repo = OGit(repo_dir=local_dir, - remote_url=item['url'], - branch=item['refspec']) - key_repo.clone_or_pull_repo() + sys.exit(0) + + oebuild_util.download_repo_from_manifest( + repo_list=repos, + src_dir=self.configure.source_dir(), + manifest_path=self.configure.yocto_manifest_dir()) def get_basic_repo(self, ): ''' @@ -216,7 +201,7 @@ class Update(OebuildCommand): docker_image = docker_config.repo_url + ":" + check_docker_tag.get_tag( ) else: - docker_image = YoctoEnv().get_docker_image( + docker_image = oebuild_util.get_docker_image_from_yocto( self.configure.source_yocto_dir()) if docker_image is None or docker_image == "": if check_docker_tag.get_tag( diff --git a/src/oebuild/bashrc.py b/src/oebuild/bashrc.py new file mode 100644 index 0000000000000000000000000000000000000000..681089f4b2e7aebba81953802f1e54f3e8462d55 --- /dev/null +++ b/src/oebuild/bashrc.py @@ -0,0 +1,139 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import os +import sys +import shutil +import subprocess + +from docker.models.containers import Container + +import oebuild.util as oebuild_util +import oebuild.const as oebuild_const +from oebuild.docker_proxy import DockerProxy +from oebuild.m_log import logger + + +class Bashrc: + ''' + is for modify ${HOME}/.bashrc + ''' + def __init__(self,) -> None: + self.container = None + self.client = None + + def set_container(self, container: Container): + ''' + xxx + ''' + self.container = container + self.client = DockerProxy() + + def update_bashrc(self, content: str): + ''' + update user initialization script by replace file, first create + a file and writed content and move it to .bashrc + ''' + tmp_file = self._set_tmpfile_content(content) + # deal host bashrc + if self.container is None: + shutil.move(tmp_file, os.path.join(os.environ['HOME'], '.bashrc')) + return + # deal container bashrc + self.client.copy_to_container( + container=self.container, + source_path=tmp_file, + to_path=f'/home/{oebuild_const.CONTAINER_USER}') + self.container.exec_run( + cmd=f''' +mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINER_USER}/.bashrc + ''', + user="root" + ) + os.remove(tmp_file) + + def restore_bashrc(self): + ''' + Restoring .bashrc will strip out the command line + content added during bitbake initialization + ''' + old_content = self.get_bashrc_content() + self.update_bashrc(content=self.restore_bashrc_content(old_content=old_content)) + + def get_bashrc_content(self,): + ''' + xxx + ''' + # return host bashrc + if self.container is None: + return subprocess.getoutput('cat $HOME/.bashrc') + # deal container bashrc + res = self.client.container_exec_command( + container=self.container, + command=f"cat /home/{oebuild_const.CONTAINER_USER}/.bashrc", + user="root", + params={ + "work_space": None, + "stream": False + }) + + if res.exit_code != 0: + logger.error(res.output) + sys.exit(1) + + return res.output.decode() + + @staticmethod + def restore_bashrc_content(old_content): + ''' + xxx + ''' + new_content = '' + for line in old_content.split('\n'): + line: str = line + if line.endswith(oebuild_const.BASH_END_FLAG) or line.replace(" ", '') == '': + continue + new_content = new_content + line + '\n' + return new_content + + @staticmethod + def add_bashrc(content: str, line: str): + ''' + xxx + ''' + if not content.endswith('\n'): + content = content + '\n' + content = content + line + oebuild_const.BASH_END_FLAG + '\n' + + return content + + @staticmethod + def init_bashrc_content(old_content, init_command: list): + ''' + xxx + ''' + new_content = Bashrc.restore_bashrc_content(old_content=old_content) + + for command in init_command: + new_content = new_content + command + oebuild_const.BASH_END_FLAG + '\n' + + return new_content + + def _set_tmpfile_content(self, content: str): + while True: + tmp_file = oebuild_util.generate_random_str(6) + if os.path.exists(tmp_file): + continue + with open(tmp_file, 'w', encoding="utf-8") as w_f: + w_f.write(content) + break + return tmp_file diff --git a/src/oebuild/bblayers.py b/src/oebuild/bblayers.py index 021f7cc0901223524ac1a9a7825bce72c90a3064..92df74a1be5f17a24dceea7dd8ba09b4ce2582e6 100644 --- a/src/oebuild/bblayers.py +++ b/src/oebuild/bblayers.py @@ -84,4 +84,5 @@ class BBLayers: layer_conf_dir = os.path.join(layer_dir, 'conf', 'layer.conf') if not os.path.exists(layer_conf_dir): + print(layer_conf_dir) raise ValueError("invalid layer") diff --git a/src/oebuild/configure.py b/src/oebuild/configure.py index 25adeb367d782a2b17d66854a73d270a82ad0255..9ee56effc5329e8c76a13de8d3cce7f8f8fbe980 100644 --- a/src/oebuild/configure.py +++ b/src/oebuild/configure.py @@ -132,6 +132,13 @@ class Configure: yocto_dir = yocto_config.path return os.path.join(Configure.source_dir(), yocto_dir) + @staticmethod + def yocto_manifest_dir(): + ''' + return .oebuild/manifest.yaml + ''' + return os.path.join(Configure.source_yocto_dir(), ".oebuild/manifest.yaml") + @staticmethod def source_poky_dir(): ''' @@ -154,6 +161,13 @@ class Configure: ''' return os.path.join(Configure.oebuild_topdir(), 'build') + @staticmethod + def source_nativesdk_dir(): + ''' + returns yocto-meta-openeuler/.oebuild/nativesdk + ''' + return os.path.join(Configure.source_yocto_dir(), '.oebuild/nativesdk') + @staticmethod def env_dir(): ''' @@ -169,7 +183,7 @@ class Configure: ''' config = oebuild_util.read_yaml( - yaml_dir=pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG)) + yaml_path=os.path.join(Configure.oebuild_dir(), oebuild_const.CONFIG)) tag_map = {} for key, value in config['docker']['tag_map'].items(): @@ -211,31 +225,8 @@ class Configure: try: oebuild_util.write_yaml( - yaml_dir=pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG), + yaml_path=pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG), data=data) return True except TypeError: return False - - -class YoctoEnv: - ''' - the YoctoEnv class is used for yocto-meta-openeuler/.oebuild/env.yaml - ''' - @staticmethod - def get_docker_image(yocto_dir): - ''' - determine yocto-meta-openeuler's supported docker image in env.yaml - ''' - if not os.path.exists(yocto_dir): - raise ValueError("the yocto direction is not exists") - - env_path = os.path.join(yocto_dir, ".oebuild/env.yaml") - if not os.path.exists(env_path): - return None - - env_parse = oebuild_util.read_yaml(pathlib.Path(env_path)) - if "docker_image" in env_parse: - return str(env_parse['docker_image']) - - return None diff --git a/src/oebuild/const.py b/src/oebuild/const.py index afd139e7e9144d5bfe6ab96b388ff7dac8458f8c..0e2d434d4e7500018eea68f68779938a79e56a67 100644 --- a/src/oebuild/const.py +++ b/src/oebuild/const.py @@ -32,13 +32,25 @@ SSTATE_DIR = "SSTATE_DIR" TMP_DIR = "TMPDIR" NATIVE_GCC_DIR = '/usr1/openeuler/native_gcc' -SSTATE_CACHE = '/usr1/openeuler/sstate-cache' +SSTATE_MIRRORS = '/usr1/openeuler/sstate-cache' # used for bitbake/in_container.py BASH_BANNER = ''' - Welcome to the openEuler Embedded build environment, - where you can run 'bitbake openeuler-image' to build - standard images + Welcome to the openEuler Embedded build environment, where you + can run [bitbake recipe] to build what you want, or you ran + run [bitbake -h] for help +''' + +# used for toolchain/toolchain.py +TOOLCHAIN_BASH_BANNER = ''' + Welcome to the openEuler Embedded build environment, where you + can create openEuler Embedded cross-chains tools by follows: + "./cross-tools/prepare.sh" + "cp config_aarch64 .config && ct-ng build" + "cp config_aarch64-musl .config && ct-ng build" + "cp config_arm32 .config && ct-ng build" + "cp config_x86_64 .config && ct-ng build" + "cp config_riscv64 .config && ct-ng build" ''' # used for configure.py diff --git a/src/oebuild/docker_proxy.py b/src/oebuild/docker_proxy.py index 175f8a5bfd195cc8e638314320b18bd6c0602adf..aa7382c9c31eeed2cda6f5c5f0d1c2c4bf9467ad 100644 --- a/src/oebuild/docker_proxy.py +++ b/src/oebuild/docker_proxy.py @@ -16,6 +16,7 @@ import tarfile import subprocess import sys from typing import List +import re import docker from docker.errors import ImageNotFound, NotFound @@ -225,3 +226,77 @@ class DockerProxy: sys.exit(res.returncode) container_id = res.stdout.strip() return self.get_container(container_id=container_id) + + def check_change_ugid(self, container: Container, container_user): + ''' + the function is to check and change container user uid and gid to same with host's, + together also alter directory pointed uid and gid + ''' + res = self.container_exec_command( + container=container, + user='root', + command=f"id {container_user}", + params={ + "work_space": f"/home/{container_user}", + "stream": False + }) + + if res.exit_code != 0: + raise ValueError("check docker user id faild") + + res_cont = res.output.decode() + + cuids = res_cont.split(' ') + # get uid from container in default user + pattern = re.compile(r'(?<=uid=)\d{1,}(?=\(' + container_user + r'\))') + match_uid = pattern.search(cuids[0]) + if match_uid: + cuid = match_uid.group() + else: + raise ValueError(f"can not get container {container_user} uid") + # get gid from container in default user + pattern = re.compile(r'(?<=gid=)\d{1,}(?=\(' + container_user + r'\))') + match_gid = pattern.search(cuids[1]) + if match_gid: + cgid = match_gid.group() + else: + raise ValueError(f"can not get container {container_user} gid") + + # judge host uid and gid are same with container uid and gid + # if not same and change container uid and gid equal to host's uid and gid + if os.getuid() != cuid: + self.change_container_uid( + container=container, + uid=os.getuid(), + container_user=container_user) + if os.getgid() != cgid: + self.change_container_gid( + container=container, + gid=os.getgid(), + container_user=container_user) + + def change_container_uid(self, container: Container, uid: int, container_user): + ''' + alter container user pointed uid + ''' + self.container_exec_command( + container=container, + user='root', + command=f"usermod -u {uid} {container_user}", + params={ + "work_space": f"/home/{container_user}", + "stream": False + }) + + def change_container_gid(self, container: Container, gid: int, container_user): + ''' + alter container pointed gid + ''' + self.container_exec_command( + container=container, + user='root', + command=f"groupmod -g {gid} {container_user}", + params={ + "work_space": f"/home/{container_user}", + "stream": False + }) diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index ab9bdc855ca29ffe8b2121ede4b14c81604b4133..f67f7240014f593c9729e331632af057bbff985d 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -14,10 +14,10 @@ import os import re import sys -from oebuild.parse_compile import ParseCompile import oebuild.util as oebuild_util from oebuild.m_log import logger import oebuild.const as oebuild_const +from oebuild.struct import CompileParam class BaseLocalConf(ValueError): @@ -93,153 +93,93 @@ def match_and_replace(pre: str, new_str: str, content: str): return content -# pylint:disable=[R0914,R0911,R0912,R0915] class LocalConf: ''' LocalConf corresponds to the local.conf configuration file, which can be modified by specifying parameters ''' - def __init__(self, local_conf_dir: str): - self.local_dir = local_conf_dir + def __init__(self, local_conf_path: str): + self.local_path = local_conf_path - def update(self, parse_compile: ParseCompile, src_dir=None): + def update(self, compile_param: CompileParam, src_dir=None): ''' update local.conf by ParseCompile ''' - local_dir = self.local_dir - if not os.path.exists(local_dir): - raise ValueError(f'{local_dir} not exists') + if not os.path.exists(self.local_path): + raise ValueError(f'{self.local_path} not exists') - if parse_compile.local_conf is None: - return - - with open(local_dir, 'r', encoding='utf-8') as r_f: + with open(self.local_path, 'r', encoding='utf-8') as r_f: content = r_f.read() - # replace machine - content = match_and_replace( - pre='MACHINE ', - new_str=f'MACHINE = "{parse_compile.machine}"', - content=content) - - if parse_compile.machine == 'sdk': - build_dir = re.search('.*(?=build)', self.local_dir) - if not build_dir: - logger.error('build_dir is not exists !') - sys.exit(-1) - change_conf = (build_dir.group() + - '/src/yocto-meta-openeuler/.oebuild/nativesdk/local.conf') - if not os.path.exists(change_conf): - logger.error('local.conf is not exists !') - sys.exit(-1) - - with open(change_conf, 'r', encoding='utf-8') as change_conf: - change_lines = change_conf.readlines() - for change_line in change_lines: - pre = change_line.replace('#', '').strip() if '#' in change_line else change_line - content = match_and_replace( - pre=pre, - new_str=change_line, - content=content) + pre_content = self._deal_other_local_param(compile_param=compile_param, src_dir=src_dir) + + compile_param.local_conf = f'{pre_content}\n{compile_param.local_conf}' + self._add_content_to_local_conf(content=content, local_conf=compile_param.local_conf) + + def _deal_other_local_param(self, compile_param: CompileParam, src_dir): + pre_content = "" + # add MACHINE + if compile_param.machine is not None: + pre_content += f'MACHINE = "{compile_param.machine}"\n' # replace toolchain - if parse_compile.toolchain_dir is not None: - if parse_compile.build_in == oebuild_const.BUILD_IN_DOCKER: + if compile_param.toolchain_dir is not None: + if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: replace_toolchain_str = f''' -{parse_compile.toolchain_type} = "{oebuild_const.NATIVE_GCC_DIR}"''' +{compile_param.toolchain_type} = "{oebuild_const.NATIVE_GCC_DIR}"''' else: replace_toolchain_str = f''' -{parse_compile.toolchain_type} = "{parse_compile.toolchain_dir}"''' - content = match_and_replace( - pre=parse_compile.toolchain_type, - new_str=replace_toolchain_str, - content=content - ) +{compile_param.toolchain_type} = "{compile_param.toolchain_dir}"''' + + pre_content += replace_toolchain_str # replace nativesdk OPENEULER_SP_DIR - if parse_compile.build_in == oebuild_const.BUILD_IN_HOST: - self.check_nativesdk_valid(parse_compile.nativesdk_dir) - if parse_compile.nativesdk_dir is None: + if compile_param.build_in == oebuild_const.BUILD_IN_HOST: + self.check_nativesdk_valid(compile_param.nativesdk_dir) + if compile_param.nativesdk_dir is None: raise ValueError("please set nativesdk dir") - nativesdk_sysroot = get_nativesdk_sysroot(parse_compile.nativesdk_dir) - nativesdk_sys_dir = os.path.join(parse_compile.nativesdk_dir, nativesdk_sysroot) - content = match_and_replace( - pre=oebuild_const.NATIVESDK_DIR_NAME, - new_str=f'{oebuild_const.NATIVESDK_DIR_NAME} = "{nativesdk_sys_dir}"', - content=content - ) - - content = match_and_replace( - pre=oebuild_const.OPENEULER_SP_DIR, - new_str=f"{oebuild_const.OPENEULER_SP_DIR} = '{src_dir}'", - content=content - ) + nativesdk_sysroot = get_nativesdk_sysroot(compile_param.nativesdk_dir) + nativesdk_sys_dir = os.path.join(compile_param.nativesdk_dir, nativesdk_sysroot) + + pre_content += f'{oebuild_const.NATIVESDK_DIR_NAME} = "{nativesdk_sys_dir}"\n' + pre_content += f'{oebuild_const.OPENEULER_SP_DIR} = "{src_dir}"\n' # replace sstate_cache - if parse_compile.sstate_cache is not None: - if os.path.islink(parse_compile.sstate_cache): - new_str = f"file://.* {parse_compile.sstate_cache}/PATH;downloadfilename=PATH" + if compile_param.sstate_mirrors is not None: + if os.path.islink(compile_param.sstate_mirrors): + new_str = f"file://.* {compile_param.sstate_mirrors}/PATH;downloadfilename=PATH" else: - if parse_compile.build_in == oebuild_const.BUILD_IN_DOCKER: - new_str = f"file://.* file://{oebuild_const.SSTATE_CACHE}/PATH" + if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: + new_str = f"file://.* file://{oebuild_const.SSTATE_MIRRORS}/PATH" else: - new_str = f"file://.* file://{parse_compile.sstate_cache}/PATH" - content = match_and_replace( - pre=oebuild_const.SSTATE_MIRRORS, - new_str=f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"', - content=content - ) + new_str = f"file://.* file://{compile_param.sstate_mirrors}/PATH" + pre_content += f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"\n' # replace sstate_dir - if parse_compile.sstate_dir is not None: - content = match_and_replace( - pre=oebuild_const.SSTATE_DIR, - new_str=f'{oebuild_const.SSTATE_DIR} = "{parse_compile.sstate_dir}"', - content=content - ) + if compile_param.sstate_dir is not None: + pre_content += f'{oebuild_const.SSTATE_DIR} = "{compile_param.sstate_dir}"\n' # replace tmpdir - if parse_compile.tmp_dir is not None: - content = match_and_replace( - pre=oebuild_const.TMP_DIR, - new_str=f'{oebuild_const.TMP_DIR} = "{parse_compile.tmp_dir}"', - content=content - ) + if compile_param.tmp_dir is not None: + pre_content += f'{oebuild_const.TMP_DIR} = "{compile_param.tmp_dir}"\n' - content = self.match_lib_param(content=content) + return pre_content + def _add_content_to_local_conf(self, content, local_conf): user_content_flag = "#===========the content is user added==================" - if user_content_flag not in content and parse_compile.local_conf != "": + if user_content_flag not in content and local_conf != "": # check if exists remark sysmbol, if exists and replace it - for line in parse_compile.local_conf.split('\n'): + for line in local_conf.split('\n'): if line.startswith("#"): r_line = line.lstrip("#").strip(" ") content = content.replace(r_line, line) content += f"\n{user_content_flag}\n" - content += parse_compile.local_conf + content += local_conf - if content is None: - return - with open(local_dir, 'w', encoding="utf-8") as r_f: + with open(self.local_path, 'w', encoding="utf-8") as r_f: r_f.write(content) - def match_lib_param(self, content: str): - ''' - add params LIBC, TCMODE-LIBC, crypt - ''' - lib_param_list = { - "LIBC": "glibc", - "TCMODE-LIBC": "glibc-external", - "crypt": "libxcrypt-external"} - - for key, value in lib_param_list.items(): - content = match_and_replace( - pre=key, - new_str=f'{key} = "{value}"', - content=content) - return content - def check_nativesdk_valid(self, nativesdk_dir): ''' Check whether the set nativesdk is valid, check whether diff --git a/src/oebuild/ogit.py b/src/oebuild/ogit.py index 1acf466a8baef479c0983e8a2cb32a1ee9fd3b66..b67447e808318fbf592772d7b42ac063074d1705 100644 --- a/src/oebuild/ogit.py +++ b/src/oebuild/ogit.py @@ -24,7 +24,7 @@ class OGit: owner git to print progress in clone action ''' - def __init__(self, repo_dir, remote_url, branch) -> None: + def __init__(self, repo_dir, remote_url, branch=None) -> None: self._repo_dir = repo_dir self._remote_url = remote_url self._branch = branch @@ -101,6 +101,7 @@ class OGit: except GitCommandError: logger.error("update faild") return False + logger.info("Fetching into %s successful\n", self._repo_dir) return True @staticmethod diff --git a/src/oebuild/parse_compile.py b/src/oebuild/parse_compile.py deleted file mode 100644 index 44f7cd5dd15ff4902eb96ec13fbf355af10b4dfc..0000000000000000000000000000000000000000 --- a/src/oebuild/parse_compile.py +++ /dev/null @@ -1,285 +0,0 @@ -''' -Copyright (c) 2023 openEuler Embedded -oebuild is licensed under Mulan PSL v2. -You can use this software according to the terms and conditions of the Mulan PSL v2. -You may obtain a copy of Mulan PSL v2 at: - http://license.coscl.org.cn/MulanPSL2 -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. -See the Mulan PSL v2 for more details. -''' - -import os -from dataclasses import dataclass -import pathlib -from typing import Optional, List -import sys - -import oebuild.util as oebuild_util -from oebuild.ogit import OGit -from oebuild.parse_template import PlatformTemplate, ParseTemplate -import oebuild.const as oebuild_const -from oebuild.m_log import logger - - -@dataclass -class DockerParam: - ''' - DockerParam defines the various parameters required for container startup - ''' - # point out the docker image - image: str - # point out the parameter for create container - parameters: str - # point out the volumns for create container - volumns: List[str] - # point out the command for create container - command: str - - -@dataclass -class BuildParam: - ''' - Carries the compilation-related parameters. - ''' - toolchain_dir: Optional[str] - - nativesdk_dir: Optional[str] - - not_use_repos: bool - - sstate_cache: Optional[str] - - sstate_dir: Optional[str] - - tmp_dir: Optional[str] - - docker_image: Optional[str] - - -@dataclass -class Compile(PlatformTemplate, BuildParam): - ''' - Compile is the parsed object of compile.yaml and is used to manipulate the build file - ''' - build_in: str - - docker_param: Optional[DockerParam] - - -class BaseParseCompileError(ValueError): - ''' - parse compile basic error - ''' - - -class CheckCompileError(BaseParseCompileError): - ''' - compile.yaml parse check faild error - ''' - - -class ParseCompile: - ''' - This class is used to parse compile.yaml and - download the relevant code repository - ''' - - def __init__(self, compile_conf_dir): - ''' - The initialization operation is used to parse the compile.yaml - file and perform a series of checks before parsing - ''' - if not os.path.exists(compile_conf_dir): - raise ValueError('compile.yaml is not exists') - - compile_conf_dir = pathlib.Path(compile_conf_dir) - data = oebuild_util.read_yaml(compile_conf_dir) - - try: - self.check_compile_conf(data=data) - except Exception as e_p: - raise e_p - - docker_param: DockerParam = None - if "docker_param" in data and data['docker_param'] is not None: - try: - dparam = data['docker_param'] - if isinstance(dparam, list): - dparam = " ".join(dparam) - docker_param = DockerParam( - image=dparam['image'], - parameters=dparam['parameters'], - volumns=dparam['volumns'], - command=dparam['command'] - ) - except KeyError: - logger.error("the compile.yaml parse failed, please check it again") - sys.exit(1) - - self.compile = Compile( - build_in=oebuild_const.BUILD_IN_DOCKER if 'build_in' not in data else data['build_in'], - machine=data['machine'], - toolchain_type=data['toolchain_type'], - toolchain_dir=None if 'toolchain_dir' not in data else data['toolchain_dir'], - nativesdk_dir=None if 'nativesdk_dir' not in data else data['nativesdk_dir'], - sstate_cache=None if 'sstate_cache' not in data else data['sstate_cache'], - sstate_dir=None if 'sstate_dir' not in data else data['sstate_dir'], - tmp_dir=None if 'tmp_dir' not in data else data['tmp_dir'], - not_use_repos=False if 'not_use_repos' not in data else data['not_use_repos'], - repos=None if "repos" not in data else ParseTemplate.parse_oebuild_repo(data['repos']), - local_conf=None if "local_conf" not in data else data['local_conf'], - layers=None if "layers" not in data else data['layers'], - docker_image=None if "docker_image" not in data else data['docker_image'], - docker_param=docker_param) - - @property - def build_in(self): - ''' - return attr of buildin - ''' - return self.compile.build_in - - @property - def machine(self): - ''' - return attr of machine - ''' - return self.compile.machine - - @property - def toolchain_type(self): - ''' - return attr of toolchain_type - ''' - return self.compile.toolchain_type - - @property - def toolchain_dir(self): - ''' - return attr of toolchain_dir - ''' - return self.compile.toolchain_dir - - @property - def nativesdk_dir(self): - ''' - return attr of nativesdk_dir - ''' - return self.compile.nativesdk_dir - - @property - def local_conf(self): - ''' - return attr of local_conf path - ''' - return self.compile.local_conf - - @property - def layers(self): - ''' - return attr of layers - ''' - return self.compile.layers - - @property - def not_use_repos(self): - ''' - return attr of not_use_repos - ''' - return self.compile.not_use_repos - - @property - def sstate_cache(self): - ''' - return attr of sstate_cache - ''' - return self.compile.sstate_cache - - @property - def sstate_dir(self): - ''' - return attr of sstate_cache - ''' - return self.compile.sstate_dir - - @property - def tmp_dir(self): - ''' - return attr of tmp_dir - ''' - return self.compile.tmp_dir - - @property - def repos(self): - ''' - return attr of repos - ''' - return self.compile.repos - - @property - def docker_image(self): - ''' - return attr of repos, it will be abandoned in new version - ''' - return self.compile.docker_image - - @property - def docker_param(self): - ''' - return attr of docker_param - ''' - return self.compile.docker_param - - def check_with_version(self, base_dir, manifest_path): - ''' - Download the repos set in compile.yaml based on the given base path - ''' - manifest = None - if os.path.exists(manifest_path): - manifest = oebuild_util.read_yaml(pathlib.Path(manifest_path))['manifest_list'] - if not self.compile.not_use_repos: - repos = self.compile.repos - if repos is None: - return - for repo_local, repo in repos.items(): - repo_dir = os.path.join(base_dir, repo.path) - if not os.path.exists(repo_dir): - repo_git = OGit(repo_dir=repo_dir, remote_url=repo.url, branch=repo.refspec) - repo_git.clone_or_pull_repo() - if manifest is None: - continue - if repo_local in manifest: - repo_git = OGit(repo_dir=repo_dir, remote_url=repo.url, branch=repo.refspec) - repo_item = manifest[repo_local] - repo_git.check_out_version(version=repo_item['version']) - - @staticmethod - def check_compile_conf(data): - ''' - Check whether the compile.yaml content is compliant - ''' - - keys = { - "machine": "the key machine is None", - "toolchain_type": "the key toolchain_type is None" - } - - for key, value in keys.items(): - if key not in data: - raise CheckCompileError(value) - - if "toolchain_dir" in data and data['toolchain_dir'] is not None: - if not os.path.exists(data['toolchain_dir']): - raise CheckCompileError(f"the toolchain_dir {data['toolchain_dir']} is not exist") - - if "repos" in data: - for _, repo in data['repos'].items(): - required_keys = { - "url": "the key url is None", - "path": "the key path is None", - "refspec": "the key refspec is None"} - for key, value in required_keys.items(): - if key not in repo: - raise CheckCompileError(value) diff --git a/src/oebuild/parse_param.py b/src/oebuild/parse_param.py new file mode 100644 index 0000000000000000000000000000000000000000..f0c960603df6d3acbe9145141313a230b0c09f3b --- /dev/null +++ b/src/oebuild/parse_param.py @@ -0,0 +1,190 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +from typing import Dict + +from oebuild.struct import RepoParam, DockerParam, CompileParam, ToolchainParam +import oebuild.util as oebuild_util +import oebuild.const as oebuild_const + + +class ParseRepoParam: + ''' + RepoParam: + remote_url: str + version: str + ''' + @staticmethod + def parse_to_obj(repo_param_dict: Dict[str, str]) -> RepoParam: + ''' + parse dict to RepoParam + ''' + return RepoParam( + remote_url=repo_param_dict['remote_url'], + version=repo_param_dict['version'] + ) + + @staticmethod + def parse_to_dict(repo_param_obj: RepoParam) -> Dict[str, str]: + ''' + parse RepoParam to dict + ''' + return { + "remote_url": repo_param_obj.remote_url, + "version": repo_param_obj.version + } + + +class ParseDockerParam: + ''' + class DockerParam: + image: str + parameters: str + volumns: list[str] + command: str + ''' + @staticmethod + def parse_to_obj(docker_param_dict: Dict[str, str | list[str]]) -> DockerParam: + ''' + parse dict to DockerParam + ''' + return DockerParam( + image=docker_param_dict['image'], + parameters=docker_param_dict['parameters'], + volumns=docker_param_dict['volumns'], + command=docker_param_dict['command'] + ) + + @staticmethod + def parse_to_dict(docker_param_obj: DockerParam) -> Dict[str, str | list[str]]: + ''' + parse dict to DockerParam + ''' + return { + 'image': docker_param_obj.image, + 'parameters': docker_param_obj.parameters, + 'volumns': docker_param_obj.volumns, + 'command': docker_param_obj.command + } + + +class ParseCompileParam: + ''' + CompileParam: + build_in: str + machine: str + toolchain_type: str + repos: Optional[list] + layers: Optional[list] + local_conf: Optional[LiteralScalarString] + docker_param: DockerParam + + sstate_mirrors: Optional[str] + sstate_dir: Optional[str] + tmp_dir: Optional[str] + + toolchain_dir: Optional[str] + nativesdk_dir: Optional[str] + + ''' + @staticmethod + def parse_to_obj(compile_param_dict) -> CompileParam: + ''' + The initialization operation is used to parse the compile.yaml + file and perform a series of checks before parsing + ''' + docker_param: DockerParam = None + if "docker_param" in compile_param_dict and compile_param_dict['docker_param'] is not None: + docker_param = ParseDockerParam.parse_to_obj( + docker_param_dict=compile_param_dict['docker_param'] + ) + + # for old version + repos = [] + if "repos" in compile_param_dict: + repos = oebuild_util.trans_dict_key_to_list(compile_param_dict['repos']) + + def get_value_from_dict(key, dictobj, default_value=None): + if key not in dictobj: + return default_value + return dictobj[key] + + return CompileParam( + build_in=get_value_from_dict('build_in', + compile_param_dict, + oebuild_const.BUILD_IN_DOCKER), + machine=get_value_from_dict('machine', compile_param_dict, None), + toolchain_type=get_value_from_dict('toolchain_type', compile_param_dict, None), + toolchain_dir=get_value_from_dict('toolchain_dir', compile_param_dict, None), + nativesdk_dir=get_value_from_dict('nativesdk_dir', compile_param_dict, None), + sstate_mirrors=get_value_from_dict('sstate_mirrors', compile_param_dict, None), + sstate_dir=get_value_from_dict('sstate_dir', compile_param_dict, None), + tmp_dir=get_value_from_dict('tmp_dir', compile_param_dict, None), + repos=None if len(repos) == 0 else repos, + local_conf=get_value_from_dict('local_conf', compile_param_dict, None), + layers=get_value_from_dict('layers', compile_param_dict, None), + docker_param=docker_param) + + @staticmethod + def parse_to_dict(compile_param: CompileParam): + ''' + xxx + ''' + compile_obj = {} + compile_obj['build_in'] = compile_param.build_in + compile_obj['machine'] = compile_param.machine + compile_obj['toolchain_type'] = compile_param.toolchain_type + if compile_param.toolchain_dir is not None: + compile_obj['toolchain_dir'] = compile_param.toolchain_dir + if compile_param.nativesdk_dir is not None: + compile_obj['nativesdk_dir'] = compile_param.nativesdk_dir + if compile_param.sstate_mirrors is not None: + compile_obj['sstate_mirrors'] = compile_param.sstate_mirrors + if compile_param.tmp_dir is not None: + compile_obj['tmp_dir'] = compile_param.tmp_dir + if compile_param.repos is not None: + compile_obj['repos'] = compile_param.repos + if compile_param.local_conf is not None: + compile_obj['local_conf'] = compile_param.local_conf + if compile_param.layers is not None: + compile_obj['layers'] = compile_param.layers + if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: + compile_obj['docker_param'] = compile_param.docker_param + + return compile_obj + + +class ParseToolchainParam: + ''' + ToolchainParam: + config_list: Optional[list] + docker_param: DockerParam + ''' + @staticmethod + def parse_to_obj(toolchain_param_dict) -> ToolchainParam: + ''' + parse dict to RepoParam + ''' + return ToolchainParam( + config_list=toolchain_param_dict['config_list'], + docker_param=ParseDockerParam.parse_to_obj(toolchain_param_dict['docker_param']) + ) + + @staticmethod + def parse_to_dict(toolchain_param_obj: ToolchainParam): + ''' + parse ToolchainParam to dict + ''' + return { + "config_list": toolchain_param_obj.config_list, + "docker_param": ParseDockerParam.parse_to_dict(toolchain_param_obj.docker_param) + } diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 5055cb06c02a4f7b9a9e4e8eda4ed6f0f01383db..376ee2683ac1a615543483d19a0eed4df8f32887 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -12,7 +12,7 @@ See the Mulan PSL v2 for more details. import logging import sys from dataclasses import dataclass -from typing import Dict, Optional +from typing import Optional import pathlib import os @@ -20,26 +20,7 @@ from ruamel.yaml.scalarstring import LiteralScalarString import oebuild.util as oebuild_util import oebuild.const as oebuild_const - - -@dataclass -class OebuildRepo: - ''' - object repo is to record template repo info, repo struct is: - repo_name: - url: str - path: str - refspec: str - object repo transfer string to struct to use it next easily - ''' - - repo_name: str - - url: str - - path: str - - refspec: str +from oebuild.struct import RepoParam @dataclass @@ -47,7 +28,7 @@ class Template: ''' basic template for paltform and feature ''' - repos: Optional[Dict[str, 'OebuildRepo']] + repos: Optional[list] layers: Optional[list] @@ -131,7 +112,10 @@ class ParseTemplate: try: data = oebuild_util.read_yaml(config_dir) - repo_dict = None if 'repos' not in data else self.parse_oebuild_repo(data['repos']) + repo_list = None + if 'repos' in data: + repo_list = oebuild_util.trans_dict_key_to_list( + self.parse_oebuild_repo(data['repos'])) layers = None if 'layers' not in data else data['layers'] local_conf = None if 'local_conf' not in data else data['local_conf'] @@ -148,7 +132,7 @@ class ParseTemplate: self.platform_template = PlatformTemplate( machine=data['machine'], toolchain_type=data['toolchain_type'], - repos=repo_dict, + repos=repo_list, local_conf=None if local_conf is None else LiteralScalarString(local_conf), layers=None if layers is None else layers) return @@ -164,13 +148,14 @@ class ParseTemplate: if 'support' in data: support_arch = data['support'].split('|') if self.platform not in support_arch: - raise FeatureNotSupport(f''' -your arch is {self.platform}, the feature is not supported, please check your application support archs -''') + raise FeatureNotSupport(f'your arch is {self.platform},' + ' the feature is not supported,' + 'please check your application ' + 'support archs') self.feature_template.append(FeatureTemplate( feature_name=LiteralScalarString(os.path.splitext(config_name)[0]), - repos=repo_dict, + repos=repo_list, support=support_arch, local_conf=None if local_conf is None else LiteralScalarString(local_conf), layers=None if layers is None else layers @@ -179,53 +164,48 @@ your arch is {self.platform}, the feature is not supported, please check your ap except Exception as e_p: raise e_p - def generate_compile_conf(self, - nativesdk_dir=None, - toolchain_dir=None, - build_in: str = oebuild_const.BUILD_IN_DOCKER, - sstate_cache=None, - tmp_dir=None, - datetime=None, - is_disable_fetch=False, - docker_image: str = None, - src_dir: str = None, - compile_dir: str = None): + def get_default_generate_compile_conf_param(self, ): ''' - first param common yaml + return default generate_compile_conf param ''' - common_yaml_dir = os.path.join(self.yocto_dir, '.oebuild', 'common.yaml') - if not os.path.exists(common_yaml_dir): - raise CommonNotFound('can not find .oebuild/common.yaml in yocto-meta-openeuler') - + return { + "nativesdk_dir": None, + "toolchain_dir": None, + "build_in": oebuild_const.BUILD_IN_DOCKER, + "sstate_mirrors": None, + "tmp_dir": None, + "datetime": None, + "is_disable_fetch": False, + "docker_image": None, + "src_dir": None, + "compile_dir": None + } + + def generate_compile_conf(self, param): + ''' + param obj: + nativesdk_dir=None, + toolchain_dir=None, + build_in: str = oebuild_const.BUILD_IN_DOCKER, + sstate_mirrors=None, + tmp_dir=None, + datetime=None, + is_disable_fetch=False, + docker_image: str = None, + src_dir: str = None, + compile_dir: str = None + ''' + # first param common yaml if self.platform_template is None: raise PlatformNotAdd('please set platform template first') - - common_yaml_dir = pathlib.Path(common_yaml_dir) - data = oebuild_util.read_yaml(common_yaml_dir) - - repos = {} - if 'repos' in data: - repos.update(data['repos']) - layers = [] - if 'layers' in data: - layers.extend(data['layers']) - local_conf = LiteralScalarString('') - if 'local_conf' in data: - local_conf += LiteralScalarString(data['local_conf']) + common_yaml_path = os.path.join(self.yocto_dir, '.oebuild', 'common.yaml') + repos, layers, local_conf = parse_repos_layers_local_obj(common_yaml_path) if self.platform_template.repos is not None: - for repo_name, oebuild_repo in self.platform_template.repos.items(): - if repo_name in repos: - continue - repos[repo_name] = { - 'url': oebuild_repo.url, - 'path': oebuild_repo.path, - 'refspec': oebuild_repo.refspec - } + repos.extend(oebuild_util.trans_dict_key_to_list(self.platform_template.repos)) if self.platform_template.layers is not None: - self.platform_template.layers.extend(layers) - layers = self.platform_template.layers + layers.extend(self.platform_template.layers) if self.platform_template.local_conf is not None: local_conf = LiteralScalarString(self.platform_template.local_conf + local_conf) @@ -233,67 +213,52 @@ your arch is {self.platform}, the feature is not supported, please check your ap for feature in self.feature_template: feature: FeatureTemplate = feature if feature.repos is not None: - for repo_name, oebuild_repo in feature.repos.items(): - if repo_name in repos: - continue - repos[repo_name] = { - 'url': oebuild_repo.url, - 'path': oebuild_repo.path, - 'refspec': oebuild_repo.refspec - } + repos.extend(oebuild_util.trans_dict_key_to_list(feature.repos)) if feature.layers is not None: layers.extend(feature.layers) if feature.local_conf is not None: local_conf = LiteralScalarString(feature.local_conf + '\n' + local_conf) - if datetime is not None: - datetime_str = LiteralScalarString(f'DATETIME = "{datetime}"') + if param['datetime'] is not None: + datetime_str = LiteralScalarString(f'DATETIME = "{param["datetime"]}"') local_conf = LiteralScalarString(local_conf + '\n' + datetime_str) - if is_disable_fetch: + if param['is_disable_fetch']: disable_fetch_str = LiteralScalarString('OPENEULER_FETCH = "disable"') local_conf = LiteralScalarString(local_conf + '\n' + disable_fetch_str) compile_conf = {} - compile_conf['build_in'] = build_in + compile_conf['build_in'] = param['build_in'] compile_conf['machine'] = self.platform_template.machine compile_conf['toolchain_type'] = self.platform_template.toolchain_type - - if nativesdk_dir is not None: - compile_conf['nativesdk_dir'] = nativesdk_dir - if toolchain_dir is not None: - compile_conf['toolchain_dir'] = toolchain_dir - if sstate_cache is not None: - compile_conf['sstate_cache'] = sstate_cache - if tmp_dir is not None: - compile_conf['tmp_dir'] = tmp_dir - + compile_conf = self._deal_non_essential_compile_conf_param(param, compile_conf) compile_conf['repos'] = repos compile_conf['local_conf'] = local_conf compile_conf['layers'] = layers - if build_in == oebuild_const.BUILD_IN_HOST: + if param['build_in'] == oebuild_const.BUILD_IN_HOST: return compile_conf - parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS - volumns = [] - volumns.append("/dev/net/tun:/dev/net/tun") - volumns.append(src_dir + ':' + oebuild_const.CONTAINER_SRC) - volumns.append(compile_dir + ':' + - os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) - if toolchain_dir is not None: - volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) - if sstate_cache is not None: - volumns.append(sstate_cache + ":" + oebuild_const.SSTATE_CACHE) - - docker_param = {} - docker_param['image'] = docker_image - docker_param['parameters'] = parameters - docker_param['volumns'] = volumns - docker_param['command'] = "bash" - compile_conf['docker_param'] = docker_param + compile_conf['docker_param'] = get_docker_param_dict( + docker_image=param['docker_image'], + src_dir=param['src_dir'], + compile_dir=param['compile_dir'], + toolchain_dir=param['toolchain_dir'], + sstate_mirrors=param['sstate_mirrors'] + ) + + return compile_conf + def _deal_non_essential_compile_conf_param(self, param, compile_conf): + if param['nativesdk_dir'] is not None: + compile_conf['nativesdk_dir'] = param['nativesdk_dir'] + if param['toolchain_dir'] is not None: + compile_conf['toolchain_dir'] = param['toolchain_dir'] + if param['sstate_mirrors'] is not None: + compile_conf['sstate_mirrors'] = param['sstate_mirrors'] + if param['tmp_dir'] is not None: + compile_conf['tmp_dir'] = param['tmp_dir'] return compile_conf @staticmethod @@ -303,10 +268,53 @@ your arch is {self.platform}, the feature is not supported, please check your ap ''' repo_cict = {} for name, repo in repos.items(): - repo_cict[name] = OebuildRepo( - repo_name=name, - url=repo['url'], - path=repo['path'], - refspec=repo['refspec']) + repo_cict[name] = RepoParam( + remote_url=repo['url'], + version=repo['refspec']) return repo_cict + + +def get_docker_param_dict(docker_image, src_dir, compile_dir, toolchain_dir, sstate_mirrors): + ''' + transfer docker param to dict + ''' + parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS + volumns = [] + volumns.append("/dev/net/tun:/dev/net/tun") + volumns.append(src_dir + ':' + oebuild_const.CONTAINER_SRC) + volumns.append(compile_dir + ":" + os.path.join( + oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) + if toolchain_dir is not None: + volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) + if sstate_mirrors is not None: + volumns.append(sstate_mirrors + ":" + oebuild_const.SSTATE_MIRRORS) + + docker_param = {} + docker_param['image'] = docker_image + docker_param['parameters'] = parameters + docker_param['volumns'] = volumns + docker_param['command'] = "bash" + + return docker_param + + +def parse_repos_layers_local_obj(common_yaml_path): + ''' + parse from yaml to repos, layers and local + ''' + if not os.path.exists(common_yaml_path): + logging.error('can not find .oebuild/common.yaml in yocto-meta-openeuler') + sys.exit(-1) + data = oebuild_util.read_yaml(common_yaml_path) + + repos = [] + if 'repos' in data: + repos.extend(oebuild_util.trans_dict_key_to_list(data['repos'])) + layers = [] + if 'layers' in data: + layers.extend(data['layers']) + local_conf = LiteralScalarString('') + if 'local_conf' in data: + local_conf += LiteralScalarString(data['local_conf']) + return repos, layers, local_conf diff --git a/src/oebuild/struct.py b/src/oebuild/struct.py new file mode 100644 index 0000000000000000000000000000000000000000..15728f8f879e90bfd13e7fcdb265321592409286 --- /dev/null +++ b/src/oebuild/struct.py @@ -0,0 +1,101 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' +from dataclasses import dataclass +from typing import Optional +from ruamel.yaml.scalarstring import LiteralScalarString + + +@dataclass +class RepoParam: + ''' + object repo is to record template repo info, repo struct is: + repo_name: + url: str + refspec: str + object repo transfer string to struct to use it next easily + ''' + remote_url: str + version: str + + +@dataclass +class OebuildEnv: + ''' + xxx + ''' + workdir: str + openeuler_layer: RepoParam + build_list: Optional[list] + + +@dataclass +class DockerParam: + ''' + DockerParam defines the various parameters required for container startup + ''' + # point out the docker image + image: str + # point out the parameter for create container + parameters: str + # point out the volumns for create container + volumns: list[str] + # point out the command for create container + command: str + + +@dataclass +class CompileLocalParam: + ''' + xxx + ''' + sstate_mirrors: Optional[str] + sstate_dir: Optional[str] + tmp_dir: Optional[str] + + +@dataclass +class CompileParamComm: + ''' + xxx + ''' + build_in: str + machine: str + toolchain_type: str + repos: Optional[list] + layers: Optional[list] + local_conf: Optional[LiteralScalarString] + docker_param: DockerParam + + +@dataclass +class CompileParamHost: + ''' + xxx + ''' + toolchain_dir: Optional[str] + nativesdk_dir: Optional[str] + + +@dataclass +class CompileParam(CompileParamComm, CompileLocalParam, CompileParamHost): + ''' + Compile is the parsed object of compile.yaml and is used to manipulate the build file + ''' + + +@dataclass +class ToolchainParam: + ''' + xxx + ''' + config_list: Optional[list] + docker_param: DockerParam diff --git a/src/oebuild/util.py b/src/oebuild/util.py index 9db1e4b4b7d853b33f8919ff8cecd432aaf85a2f..2e9bfbed2c63eb856d3eb7493a5c48501e106104 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -28,6 +28,10 @@ from oebuild.docker_proxy import DockerProxy from oebuild.m_log import logger from oebuild.version import __version__ import oebuild.const as oebuild_const +from oebuild.ogit import OGit +from oebuild.struct import RepoParam, DockerParam +from oebuild.parse_param import ParseRepoParam +from oebuild.parse_env import EnvContainer, ParseEnv def get_nativesdk_environment(nativesdk_dir=oebuild_const.NATIVESDK_DIR, @@ -74,15 +78,17 @@ def get_nativesdk_environment(nativesdk_dir=oebuild_const.NATIVESDK_DIR, sys.exit(1) -def read_yaml(yaml_dir: pathlib.Path): +def read_yaml(yaml_path): ''' read yaml file and parse it to object ''' - if not os.path.exists(yaml_dir.absolute()): - raise ValueError(f"yaml_dir can not find in :{yaml_dir.absolute()}") + if isinstance(yaml_path, str): + yaml_path = pathlib.Path(yaml_path) + if not os.path.exists(yaml_path.absolute()): + raise ValueError(f"yaml_dir can not find in :{yaml_path.absolute()}") try: - with open(yaml_dir.absolute(), 'r', encoding='utf-8') as r_f: + with open(yaml_path, 'r', encoding='utf-8') as r_f: yaml = YAML() data = yaml.load(r_f.read()) return data @@ -90,16 +96,18 @@ def read_yaml(yaml_dir: pathlib.Path): raise e_p -def write_yaml(yaml_dir: pathlib.Path, data): +def write_yaml(yaml_path, data): ''' write data to yaml file ''' - if not os.path.exists(yaml_dir.absolute()): - if not os.path.exists(os.path.dirname(yaml_dir.absolute())): - os.makedirs(os.path.dirname(yaml_dir.absolute())) - os.mknod(yaml_dir) + if isinstance(yaml_path, str): + yaml_path = pathlib.Path(yaml_path) + if not os.path.exists(yaml_path.absolute()): + if not os.path.exists(os.path.dirname(yaml_path.absolute())): + os.makedirs(os.path.dirname(yaml_path.absolute())) + os.mknod(yaml_path) - with open(yaml_dir, 'w', encoding='utf-8') as w_f: + with open(yaml_path, 'w', encoding='utf-8') as w_f: yaml = YAML() yaml.dump(data, w_f) @@ -272,6 +280,80 @@ def get_host_proxy(proxy_name): return host_proxy +def download_repo_from_manifest(repo_list, src_dir, manifest_path): + ''' + Download the repos set in compile.yaml based on the given base path + ''' + manifest = None + if os.path.exists(manifest_path): + manifest = read_yaml(pathlib.Path(manifest_path))['manifest_list'] + for repo_name in repo_list: + repo_dir = os.path.join(src_dir, repo_name) + if repo_name in manifest: + repo_obj: RepoParam = ParseRepoParam.parse_to_obj(manifest[repo_name]) + repo_git = OGit(repo_dir=repo_dir, remote_url=repo_obj.remote_url, branch=None) + repo_git.check_out_version(version=repo_obj.version) + + +def get_docker_image_from_yocto(yocto_dir): + ''' + determine yocto-meta-openeuler's supported docker image in env.yaml + ''' + if not os.path.exists(yocto_dir): + raise ValueError("the yocto direction is not exists") + + env_path = os.path.join(yocto_dir, ".oebuild/env.yaml") + if not os.path.exists(env_path): + return None + + env_parse = read_yaml(yaml_path=env_path) + if "docker_image" in env_parse: + return str(env_parse['docker_image']) + + return None + + +def deal_env_container(env: ParseEnv, docker_param: DockerParam): + ''' + This operation realizes the processing of the container, + controls how the container is processed by parsing the env + variable, if the container does not exist, or the original + environment and the current environment that needs to be set + are inconsistent, you need to create a new container, otherwise + directly enable the sleeping container + ''' + docker_proxy = DockerProxy() + if env.container is None \ + or env.container.short_id is None \ + or not docker_proxy.is_container_exists(env.container.short_id): + # judge which container + container: Container = docker_proxy.create_container( + image=docker_param.image, + parameters=docker_param.parameters, + volumes=docker_param.volumns, + command=docker_param.command) + + env_container = EnvContainer(container.short_id) + env.set_env_container(env_container) + env.export_env() + + container_id = env.container.short_id + container: Container = docker_proxy.get_container(container_id) # type: ignore + if not docker_proxy.is_container_running(container): + docker_proxy.start_container(container) + return container_id + + +def trans_dict_key_to_list(obj): + ''' + transfer dict key to list + ''' + res = [] + for key in obj: + res.append(key) + return res + + @contextmanager def suppress_print(): '''