From 6c3b3f9aeaa0016336952e8a6faba67bd60ec48e Mon Sep 17 00:00:00 2001 From: lixinyu Date: Sat, 27 Jan 2024 16:21:57 +0800 Subject: [PATCH] code: format code to follow the flake8 style guide * format code to follow the flake8 style guide and keep max-line-length to no more than 100 characters Signed-off-by: lixinyu --- src/oebuild/app/main.py | 28 ++-- src/oebuild/app/plugins/bitbake/base_build.py | 4 +- src/oebuild/app/plugins/bitbake/bitbake.py | 4 +- .../app/plugins/bitbake/in_container.py | 23 ++-- src/oebuild/app/plugins/bitbake/in_host.py | 21 +-- src/oebuild/app/plugins/clear/clear.py | 11 +- src/oebuild/app/plugins/demo/demo.py | 7 +- src/oebuild/app/plugins/deploy/com_target.py | 28 ++-- .../app/plugins/deploy/deploy_target.py | 12 +- .../app/plugins/deploy/target_dev/deploy.py | 109 ++++++++++------ src/oebuild/app/plugins/generate/generate.py | 120 +++++++++--------- src/oebuild/app/plugins/init/init.py | 27 ++-- src/oebuild/app/plugins/m_env/m_env.py | 2 + src/oebuild/app/plugins/m_plugin/m_plugin.py | 7 +- src/oebuild/app/plugins/manifest/manifest.py | 23 ++-- src/oebuild/app/plugins/run_qemu/run_qemu.py | 10 +- src/oebuild/app/plugins/update/update.py | 31 ++--- src/oebuild/bb/utils.py | 15 ++- src/oebuild/bblayers.py | 4 +- src/oebuild/check_docker_tag.py | 4 +- src/oebuild/command.py | 7 +- src/oebuild/configure.py | 20 ++- src/oebuild/docker_proxy.py | 40 +++--- src/oebuild/local_conf.py | 53 ++++---- src/oebuild/oebuild_parser.py | 6 +- src/oebuild/ogit.py | 8 +- src/oebuild/parse_compile.py | 14 +- src/oebuild/parse_env.py | 8 +- src/oebuild/parse_template.py | 41 +++--- src/oebuild/spec.py | 5 +- src/oebuild/util.py | 29 ++++- 31 files changed, 423 insertions(+), 298 deletions(-) diff --git a/src/oebuild/app/main.py b/src/oebuild/app/main.py index 88674fc..ac10b47 100644 --- a/src/oebuild/app/main.py +++ b/src/oebuild/app/main.py @@ -20,25 +20,27 @@ import colorama import oebuild.util as oebuild_util from oebuild.auto_completion import AutoCompletion from oebuild.version import __version__ -from oebuild.spec import get_spec,_ExtCommand +from oebuild.spec import get_spec, _ExtCommand from oebuild.command import OebuildCommand from oebuild.m_log import logger -from oebuild.oebuild_parser import OebuildArgumentParser,OebuildHelpAction +from oebuild.oebuild_parser import OebuildArgumentParser, OebuildHelpAction APP = "app" + class OebuildApp: ''' The execution body of the oebuild tool, and all oebuild commands are resolved and executed by this body ''' + def __init__(self): self.base_oebuild_dir = oebuild_util.get_base_oebuild() self.oebuild_parser = None self.subparsers = {} self.cmd = None try: - plugins_dir = pathlib.Path(self.base_oebuild_dir,'app/conf','plugins.yaml') + plugins_dir = pathlib.Path(self.base_oebuild_dir, 'app/conf', 'plugins.yaml') self.oebuild_plugins_path = os.path.expanduser('~') + '/.local/oebuild_plugins/' self.append_plugins_dir = pathlib.Path(self.oebuild_plugins_path, 'append_plugins.yaml') self.command_ext = self.get_command_ext(oebuild_util.read_yaml(plugins_dir)['plugins']) @@ -66,11 +68,9 @@ class OebuildApp: path=app['path']) return command_ext - def _load_extension_specs(self, ): self.command_spec = extension_commands(APP, self.command_ext) - def _setup_parsers(self): # Set up and install command-line argument parsers. @@ -109,18 +109,16 @@ class OebuildApp: return parser, subparser_gen - def _check_command(self, args): if args.help or \ - args.command is None or \ - args.command not in self.command_ext or \ - args.command == 'help': + args.command is None or \ + args.command not in self.command_ext or \ + args.command == 'help': self.help() return False return True - def run_command(self, argv): ''' Parse command line arguments and run the OebuildCommand. @@ -135,10 +133,10 @@ class OebuildApp: # Finally, run the command. self._run_extension(args.command, unknown) - def _run_extension(self, name:str, unknown): + def _run_extension(self, name: str, unknown): # Check a program invariant. spec = self.command_spec[name] - cmd:OebuildCommand = oebuild_util.get_instance(spec.factory) + cmd: OebuildCommand = oebuild_util.get_instance(spec.factory) parser = self.subparsers[name] @@ -164,6 +162,7 @@ class OebuildApp: ''' self.oebuild_parser.print_help() + def check_user(): ''' check execute user must in normal user @@ -173,7 +172,8 @@ def check_user(): return False return True -def extension_commands(pre_dir, commandlist:OrderedDict): + +def extension_commands(pre_dir, commandlist: OrderedDict): ''' Get descriptions of available extension commands. The return value is an ordered map from project paths to lists of @@ -187,6 +187,7 @@ def extension_commands(pre_dir, commandlist:OrderedDict): return specs + def main(argv=None): ''' oebuild main entrypoint @@ -199,5 +200,6 @@ def main(argv=None): app = OebuildApp() app.run(argv or sys.argv[1:]) + if __name__ == "__main__": main() diff --git a/src/oebuild/app/plugins/bitbake/base_build.py b/src/oebuild/app/plugins/bitbake/base_build.py index ccfec74..b0c830d 100644 --- a/src/oebuild/app/plugins/bitbake/base_build.py +++ b/src/oebuild/app/plugins/bitbake/base_build.py @@ -16,10 +16,12 @@ from oebuild.local_conf import LocalConf from oebuild.bblayers import BBLayers import oebuild.const as oebuild_const + class BaseBuild: ''' The class provides the basic methods that the build class inherits ''' + def _set_tmpfile_content(self, content: str): while True: tmp_file = oebuild_util.generate_random_str(6) @@ -30,7 +32,7 @@ class BaseBuild: break return tmp_file - def replace_local_conf(self, parse_compile, local_dir, src_dir = None): + def replace_local_conf(self, parse_compile, local_dir, src_dir=None): ''' replace some param in local.conf, the LocalConf will be instantiated and exec update diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index 4179271..512e485 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -26,12 +26,14 @@ from oebuild.app.plugins.bitbake.in_host import InHost from oebuild.m_log import logger import oebuild.const as oebuild_const + class Bitbake(OebuildCommand): ''' Bitbake instructions can enter the build interactive environment and then directly run bitbake-related instructions,or run bitbake command directly, for example: `oebuild bitbake busybox` ''' + def __init__(self): self.compile_conf_dir = os.path.join(os.getcwd(), 'compile.yaml') self.configure = Configure() @@ -61,7 +63,7 @@ class Bitbake(OebuildCommand): return parser - def do_run(self, args: argparse.ArgumentParser, unknown = None): + def do_run(self, args: argparse.ArgumentParser, unknown=None): ''' The BitBake execution logic is: the first step is to prepare the code that initializes diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index f750487..7bf8b31 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -13,7 +13,7 @@ import re import os import sys -from docker.models.containers import Container,ExecResult +from docker.models.containers import Container, ExecResult from oebuild.parse_env import ParseEnv, EnvContainer from oebuild.docker_proxy import DockerProxy @@ -24,6 +24,7 @@ from oebuild.app.plugins.bitbake.base_build import BaseBuild import oebuild.util as oebuild_util import oebuild.const as oebuild_const + class InContainer(BaseBuild): ''' bitbake command execute in container @@ -40,7 +41,7 @@ class InContainer(BaseBuild): ''' logger.info("Bitbake starting ...") - docker_param:DockerParam = None + docker_param: DockerParam = None if parse_compile.docker_param is not None: docker_param = parse_compile.docker_param else: @@ -56,7 +57,7 @@ class InContainer(BaseBuild): `oebuild update docker`''') return - self.deal_env_container(env=parse_env,docker_param=docker_param) + self.deal_env_container(env=parse_env, docker_param=docker_param) self.exec_compile(parse_compile=parse_compile, command=command) def _trans_docker_param(self, @@ -71,7 +72,7 @@ class InContainer(BaseBuild): volumns.append("/dev/net/tun:/dev/net/tun") volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) volumns.append(os.getcwd() + ':' + - os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()))) + os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()))) if toolchain_dir is not None: volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) if sstate_cache is not None: @@ -95,11 +96,11 @@ class InContainer(BaseBuild): directly enable the sleeping container ''' if env.container is None \ - or env.container.short_id is None \ - or not self.client.is_container_exists(env.container.short_id): + or env.container.short_id is None \ + or not self.client.is_container_exists(env.container.short_id): # judge which container - container:Container = self.client.create_container( - image = docker_param.image, + container: Container = self.client.create_container( + image=docker_param.image, parameters=docker_param.parameters, volumes=docker_param.volumns, command=docker_param.command) @@ -109,7 +110,7 @@ class InContainer(BaseBuild): env.export_env() self.container_id = env.container.short_id - container:Container = self.client.get_container(self.container_id) # type: ignore + container: Container = self.client.get_container(self.container_id) # type: ignore if not self.client.is_container_running(container): self.client.start_container(container) @@ -117,7 +118,7 @@ class InContainer(BaseBuild): ''' execute compile task ''' - container:Container = self.client.get_container(self.container_id) # type: ignore + container: Container = self.client.get_container(self.container_id) # type: ignore self.init_bash(container=container, build_dir_name=os.path.basename(os.getcwd())) @@ -146,7 +147,7 @@ class InContainer(BaseBuild): content = self._get_bashrc_content(container=container) new_content = self._add_bashrc(content=content, line=command) self.update_bashrc(container=container, content=new_content) - res:ExecResult = self.client.container_exec_command( + res: ExecResult = self.client.container_exec_command( container=container, command="bash .bashrc", user=oebuild_const.CONTAINER_USER, diff --git a/src/oebuild/app/plugins/bitbake/in_host.py b/src/oebuild/app/plugins/bitbake/in_host.py index 7c258cb..68a696d 100644 --- a/src/oebuild/app/plugins/bitbake/in_host.py +++ b/src/oebuild/app/plugins/bitbake/in_host.py @@ -23,6 +23,7 @@ from oebuild.app.plugins.bitbake.base_build import BaseBuild import oebuild.util as oebuild_util import oebuild.const as oebuild_const + class InHost(BaseBuild): ''' bitbake command execute in host @@ -67,13 +68,13 @@ initialization operations''') return if command is not None and command != "": - self._append_build_sh(str_list=[command], build_dir= os.getcwd()) + self._append_build_sh(str_list=[command], build_dir=os.getcwd()) with subprocess.Popen('bash build.sh', - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - text=True) as s_p: + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + text=True) as s_p: if s_p.returncode is not None and s_p.returncode != 0: err_msg = '' if s_p.stderr is not None: @@ -96,9 +97,9 @@ initialization operations''') for b_s in oebuild_const.BASH_BANNER.split('\n'): b_s = f"echo {b_s}{oebuild_const.BASH_END_FLAG}" banner_list.append(b_s) - self._append_build_sh(str_list=banner_list, build_dir= os.getcwd()) + self._append_build_sh(str_list=banner_list, build_dir=os.getcwd()) append_str = f"sed -i '/{oebuild_const.BASH_END_FLAG}/d' $HOME/.bashrc" - self._append_build_sh(str_list = [append_str], build_dir= os.getcwd()) + self._append_build_sh(str_list=[append_str], build_dir=os.getcwd()) build_sh_dir = os.path.join(os.getcwd(), 'build.sh') source_build_str = f"source {build_sh_dir}" @@ -120,7 +121,7 @@ initialization operations''') ps1_command = 'PS1="\\u\\h:\\W> "' self._append_build_sh( - str_list= [init_sdk_command, set_template, init_oe_command, ps1_command], + str_list=[init_sdk_command, set_template, init_oe_command, ps1_command], build_dir=build_dir ) @@ -130,7 +131,7 @@ initialization operations''') os.remove(build_sh_dir) os.mknod(build_sh_dir) - def _append_build_sh(self, str_list:list, build_dir): + def _append_build_sh(self, str_list: list, build_dir): build_sh_dir = os.path.join(build_dir, 'build.sh') if not os.path.exists(build_sh_dir): raise ValueError("build.sh not exists") diff --git a/src/oebuild/app/plugins/clear/clear.py b/src/oebuild/app/plugins/clear/clear.py index c6663f8..62a5a11 100644 --- a/src/oebuild/app/plugins/clear/clear.py +++ b/src/oebuild/app/plugins/clear/clear.py @@ -23,6 +23,7 @@ from oebuild.configure import Configure from oebuild.docker_proxy import DockerProxy from oebuild.m_log import logger + class Clear(OebuildCommand): ''' for some clear task @@ -39,7 +40,7 @@ class Clear(OebuildCommand): will be generated, such as containers,so this command can remove unimportant products, such as containers ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -55,7 +56,7 @@ class Clear(OebuildCommand): return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): # perpare parse help command if self.pre_parse_help(args, unknown): return @@ -80,8 +81,8 @@ class Clear(OebuildCommand): build_list = os.listdir(self.configure.build_dir()) for build_dir in build_list: build_dir = os.path.join(self.configure.build_dir(), build_dir) - if os.path.exists(os.path.join(build_dir,".env")): - env_list.append(os.path.join(build_dir,".env")) + if os.path.exists(os.path.join(build_dir, ".env")): + env_list.append(os.path.join(build_dir, ".env")) # traversal every env file and get container_id, and then try to stop it and rm it for env in env_list: @@ -91,7 +92,7 @@ class Clear(OebuildCommand): container = self.client.get_container(container_id=container_id) DockerProxy().stop_container(container=container) DockerProxy().delete_container(container=container) - logger.info("Delete container: %s successful",container.short_id) + logger.info("Delete container: %s successful", container.short_id) except DockerException: continue except KeyError: diff --git a/src/oebuild/app/plugins/demo/demo.py b/src/oebuild/app/plugins/demo/demo.py index df3b6f3..5548ec7 100644 --- a/src/oebuild/app/plugins/demo/demo.py +++ b/src/oebuild/app/plugins/demo/demo.py @@ -8,6 +8,7 @@ from oebuild.configure import Configure logger = logging.getLogger() + class Demo(OebuildCommand): def __init__(self): @@ -18,7 +19,7 @@ class Demo(OebuildCommand): description=textwrap.dedent('''\ this is your description message ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -31,6 +32,6 @@ class Demo(OebuildCommand): return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): args = args.parse_args(unknown) - pass \ No newline at end of file + pass diff --git a/src/oebuild/app/plugins/deploy/com_target.py b/src/oebuild/app/plugins/deploy/com_target.py index 9babf70..0066cc0 100644 --- a/src/oebuild/app/plugins/deploy/com_target.py +++ b/src/oebuild/app/plugins/deploy/com_target.py @@ -30,14 +30,16 @@ logger = logging.getLogger() TARGET_DIR_NAME = "target_dev" TARGET_SCRIPT_NAME = "oebuild_dev" + class ComTarget: ''' The class is used to deploy-target and undeploy-target, this is a main body, the deploy-target and undeploy-target is just a interface and finally go into ComTarget ''' + def __init__(self) -> None: self.configure = Configure() - self.client:DockerProxy = None + self.client: DockerProxy = None self.container_id = None self.work_dir = os.getcwd() self.old_bashrc = None @@ -84,7 +86,7 @@ the container {self.container_id} failed to be destroyed, please run logger.info("Initializing environment, please wait ...") self.deal_env_container(oebuild_const.DEFAULT_DOCKER) - container:Container = self.client.get_container(self.container_id) + container: Container = self.client.get_container(self.container_id) self._make_and_copy_lib(container=container) self.bak_bash(container=container) self.init_bash(container=container) @@ -143,7 +145,7 @@ the container {self.container_id} failed to be destroyed, please run ''' return os.path.exists(self.configure.source_poky_dir()) - def _make_and_copy_lib(self, container:Container): + def _make_and_copy_lib(self, container: Container): # everytime, we should make sure that script is updated, so we make a rm action before copy container.exec_run(f"rm -rf /home/openeuler/{TARGET_DIR_NAME}") # copy package lib to docker @@ -155,7 +157,7 @@ the container {self.container_id} failed to be destroyed, please run to_path="/home/openeuler/") container.exec_run(f"chmod 755 /home/openeuler/{TARGET_DIR_NAME}/{TARGET_SCRIPT_NAME}") - def deal_env_container(self,docker_image:str): + def deal_env_container(self, docker_image: str): ''' This operation realizes the processing of the container, controls how the container is processed by parsing the env @@ -169,18 +171,18 @@ the container {self.container_id} failed to be destroyed, please run volumns.append("/dev/net/tun:/dev/net/tun") volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) volumns.append(os.path.join(self.configure.build_dir(), cwd_name) - + ':' + - os.path.join(oebuild_const.CONTAINER_BUILD, cwd_name)) + + ':' + + os.path.join(oebuild_const.CONTAINER_BUILD, cwd_name)) parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS - container:Container = self.client.create_container( - image=docker_image, - parameters=parameters, - volumes=volumns, - command="bash") + container: Container = self.client.create_container( + image=docker_image, + parameters=parameters, + volumes=volumns, + command="bash") self.container_id = container.short_id - container:Container = self.client.get_container(self.container_id) # type: ignore + container: Container = self.client.get_container(self.container_id) # type: ignore if not self.client.is_container_running(container): self.client.start_container(container) @@ -286,7 +288,7 @@ the container {self.container_id} failed to be destroyed, please run if res.exit_code != 0: raise ValueError("check docker user id faild") - res_cont:str = res.output.decode() + res_cont: str = res.output.decode() cuids = res_cont.split(' ') # get uid from container in default user diff --git a/src/oebuild/app/plugins/deploy/deploy_target.py b/src/oebuild/app/plugins/deploy/deploy_target.py index 83117fe..91d0f4d 100644 --- a/src/oebuild/app/plugins/deploy/deploy_target.py +++ b/src/oebuild/app/plugins/deploy/deploy_target.py @@ -20,10 +20,12 @@ from oebuild.app.plugins.deploy.com_target import ComTarget logger = logging.getLogger() + class DeployTarget(OebuildCommand): ''' we use package in a ''' + def __init__(self) -> None: super().__init__( '{}', @@ -33,7 +35,7 @@ Deploys a recipe's build output (i.e. the output of the do_install task) to a li overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand. ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -44,7 +46,7 @@ oebuild deploy-target [-h] [-c] [-s] [-n] [-p] [--no-check-space] [-e SSH_EXEC] return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): if '-h' in unknown or '--help' in unknown: self.print_help_msg() return @@ -80,10 +82,12 @@ options: --no-strip Do not strip executables prior to deploy """) + class UnDeployTarget(OebuildCommand): ''' we use package in a ''' + def __init__(self) -> None: super().__init__( '{}', @@ -91,7 +95,7 @@ class UnDeployTarget(OebuildCommand): textwrap.dedent('''\ Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target. ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -102,7 +106,7 @@ oebuild undeploy-target [-h] [-c] [-s] [-a] [-n] [-e SSH_EXEC] [-P PORT] [-I KEY return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): if '-h' in unknown or '--help' in unknown: self.print_help_msg() return diff --git a/src/oebuild/app/plugins/deploy/target_dev/deploy.py b/src/oebuild/app/plugins/deploy/target_dev/deploy.py index f799cb1..233e179 100644 --- a/src/oebuild/app/plugins/deploy/target_dev/deploy.py +++ b/src/oebuild/app/plugins/deploy/target_dev/deploy.py @@ -22,6 +22,7 @@ logger = logging.getLogger('devtool') deploylist_path = '/.devtool' + def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False): """ Prepare a shell script for running on the target to @@ -79,11 +80,13 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals lines.append('do') lines.append(' checkpath=`dirname "$checkpath"`') lines.append('done') - lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') + lines.append( + r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') # First line of the file is the total space lines.append('total=`head -n1 $3`') lines.append('if [ $total -gt $freespace ] ; then') - lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') + lines.append( + ' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') lines.append(' exit 1') lines.append('fi') if not nopreserve: @@ -134,7 +137,6 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals return '\n'.join(lines) - def deploy(args, config, basepath, workspace): """Entry point for the devtool 'deploy' subcommand""" import math @@ -158,12 +160,12 @@ def deploy(args, config, basepath, workspace): rd = tinfoil.parse_recipe(args.recipename) except Exception as e: raise DevtoolError('Exception parsing recipe %s: %s' % - (args.recipename, e)) + (args.recipename, e)) recipe_outdir = rd.getVar('D') if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): raise DevtoolError('No files to deploy - have you built the %s ' - 'recipe? If so, the install step has not installed ' - 'any files.' % args.recipename) + 'recipe? If so, the install step has not installed ' + 'any files.' % args.recipename) if args.strip and not args.dry_run: # Fakeroot copy to new destination @@ -171,10 +173,11 @@ def deploy(args, config, basepath, workspace): recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped') if os.path.isdir(recipe_outdir): exec_fakeroot(rd, "rm -rf %s" % recipe_outdir, shell=True) - exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) + exec_fakeroot(rd, "cp -af %s %s" % + (os.path.join(srcdir, '.'), recipe_outdir), shell=True) os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), - rd.getVar('base_libdir'), rd) + rd.getVar('base_libdir'), rd) filelist = [] inodes = set({}) @@ -230,9 +233,9 @@ def deploy(args, config, basepath, workspace): tmpscript = '/tmp/devtool_deploy.sh' tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') shellscript = _prepare_remote_script(deploy=True, - verbose=args.show_status, - nopreserve=args.no_preserve, - nocheckspace=args.no_check_space) + verbose=args.show_status, + nopreserve=args.no_preserve, + nocheckspace=args.no_check_space) # Write out the script to a file with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: f.write(shellscript) @@ -242,18 +245,20 @@ def deploy(args, config, basepath, workspace): for fpath, fsize in filelist: f.write('%s %d\n' % (fpath, fsize)) # Copy them to the target - ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, + extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) if ret != 0: raise DevtoolError('Failed to copy script to %s - rerun with -s to ' - 'get a complete error message' % args.target) + 'get a complete error message' % args.target) finally: shutil.rmtree(tmpdir) # Now run the script - ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) + ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, + args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) if ret != 0: raise DevtoolError('Deploy failed - rerun with -s to get a complete ' - 'error message') + 'error message') logger.info('Successfully deployed %s' % recipe_outdir) @@ -267,12 +272,15 @@ def deploy(args, config, basepath, workspace): return 0 + def undeploy(args, config, basepath, workspace): """Entry point for the devtool 'undeploy' subcommand""" if args.all and args.recipename: - raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target') + raise argparse_oe.ArgumentUsageError( + 'Cannot specify -a/--all with a recipe name', 'undeploy-target') elif not args.recipename and not args.all: - raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') + raise argparse_oe.ArgumentUsageError( + 'If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') extraoptions = '' if args.no_host_check: @@ -296,20 +304,23 @@ def undeploy(args, config, basepath, workspace): tmpdir = tempfile.mkdtemp(prefix='devtool') try: tmpscript = '/tmp/devtool_undeploy.sh' - shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all) + shellscript = _prepare_remote_script( + deploy=False, dryrun=args.dry_run, undeployall=args.all) # Write out the script to a file with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: f.write(shellscript) # Copy it to the target - ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, + extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) if ret != 0: raise DevtoolError('Failed to copy script to %s - rerun with -s to ' - 'get a complete error message' % args.target) + 'get a complete error message' % args.target) finally: shutil.rmtree(tmpdir) # Now run the script - ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True) + ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, + extraoptions, args.target, tmpscript, args.recipename), shell=True) if ret != 0: raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' 'error message') @@ -327,24 +338,33 @@ def register_commands(subparsers, context): description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.', group='testbuild') parser_deploy.add_argument('recipename', help='Recipe to deploy') - parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') - parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') - parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') - parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') - parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true') - parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true') + parser_deploy.add_argument( + 'target', help='Live target machine running an ssh server: user@hostname[:destdir]') + parser_deploy.add_argument('-c', '--no-host-check', + help='Disable ssh host key checking', action='store_true') + parser_deploy.add_argument('-s', '--show-status', + help='Show progress/status output', action='store_true') + parser_deploy.add_argument( + '-n', '--dry-run', help='List files to be deployed only', action='store_true') + parser_deploy.add_argument('-p', '--no-preserve', + help='Do not preserve existing files', action='store_true') + parser_deploy.add_argument( + '--no-check-space', help='Do not check for available space before deploying', action='store_true') parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') - parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') + parser_deploy.add_argument( + '-P', '--port', help='Specify port to use for connection to the target') parser_deploy.add_argument('-I', '--key', help='Specify ssh private key for connection to the target') strip_opts = parser_deploy.add_mutually_exclusive_group(required=False) strip_opts.add_argument('-S', '--strip', - help='Strip executables prior to deploying (default: %(default)s). ' - 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.', - default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')), - action='store_true') - strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') + help='Strip executables prior to deploying (default: %(default)s). ' + 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.', + default=oe.types.boolean(context.config.get( + 'Deploy', 'strip', default='0')), + action='store_true') + strip_opts.add_argument( + '--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') parser_deploy.set_defaults(func=deploy) @@ -352,15 +372,22 @@ def register_commands(subparsers, context): help='Undeploy recipe output files in live target machine', description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.', group='testbuild') - parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') - parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname') - parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') - parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') - parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') - parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true') + parser_undeploy.add_argument( + 'recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') + parser_undeploy.add_argument( + 'target', help='Live target machine running an ssh server: user@hostname') + parser_undeploy.add_argument('-c', '--no-host-check', + help='Disable ssh host key checking', action='store_true') + parser_undeploy.add_argument( + '-s', '--show-status', help='Show progress/status output', action='store_true') + parser_undeploy.add_argument( + '-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') + parser_undeploy.add_argument( + '-n', '--dry-run', help='List files to be undeployed only', action='store_true') parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') - parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') + parser_undeploy.add_argument( + '-P', '--port', help='Specify port to use for connection to the target') parser_undeploy.add_argument('-I', '--key', - help='Specify ssh private key for connection to the target') + help='Specify ssh private key for connection to the target') parser_undeploy.set_defaults(func=undeploy) diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index fa75fa2..4271421 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -21,17 +21,19 @@ from prettytable import PrettyTable from oebuild.command import OebuildCommand import oebuild.util as oebuild_util -from oebuild.parse_compile import ParseCompile,CheckCompileError +from oebuild.parse_compile import ParseCompile, CheckCompileError from oebuild.configure import Configure, YoctoEnv from oebuild.parse_template import BaseParseTemplate, ParseTemplate from oebuild.m_log import logger, INFO_COLOR from oebuild.check_docker_tag import CheckDockerTag import oebuild.const as oebuild_const + class Generate(OebuildCommand): ''' compile.yaml is generated according to different command parameters by generate ''' + def __init__(self): self.configure = Configure() self.nativesdk_dir = None @@ -49,7 +51,7 @@ class Generate(OebuildCommand): way to specify compile.yaml, users can directly specify the file after customizing the build configuration file ''' - )) + )) def do_add_parser(self, parser_adder): parser = self._parser( @@ -59,82 +61,82 @@ class Generate(OebuildCommand): %(prog)s [-p platform] [-f features] [-t toolchain_dir] [-d build_directory] [-l list] [-b_in build_in] ''') - parser.add_argument('-l', '--list', dest='list',action = "store_true", - help=''' + parser.add_argument('-l', '--list', dest='list', action="store_true", + help=''' will list support archs and features ''' - ) + ) parser.add_argument('-p', '--platform', dest='platform', default="qemu-aarch64", - help=''' + help=''' this param is for arch, you can find it in yocto-meta-openeuler/.oebuild/platform ''' - ) + ) - parser.add_argument('-s','--state_cache', dest='sstate_cache', - help=''' + parser.add_argument('-s', '--state_cache', dest='sstate_cache', + help=''' this param is for SSTATE_MIRRORS ''' - ) + ) parser.add_argument('-s_dir', '--sstate_dir', dest='sstate_dir', - help=''' + help=''' this param is for SSTATE_DIR ''' - ) + ) parser.add_argument('-m', '--tmp_dir', dest='tmp_dir', - help=''' + help=''' this param is for tmp directory, the build result will be stored in ''' - ) + ) parser.add_argument('-f', '--features', dest='features', action='append', - help=''' + help=''' this param is feature, it's a reuse command ''' - ) + ) parser.add_argument('-c', '--compile_dir', dest='compile_dir', - help=''' + help=''' this param is for compile.yaml directory ''' - ) + ) parser.add_argument('-d', '--directory', dest='directory', - help=''' + help=''' this param is build directory, the default is same to platform ''' - ) + ) - parser.add_argument('-t', '--toolchain_dir', dest='toolchain_dir', default = '', - help=''' + parser.add_argument('-t', '--toolchain_dir', dest='toolchain_dir', default='', + help=''' this param is for external toolchain dir, if you want use your own toolchain ''' - ) + ) - parser.add_argument('-n', '--nativesdk_dir', dest='nativesdk_dir', default = '', - help=''' + parser.add_argument('-n', '--nativesdk_dir', dest='nativesdk_dir', default='', + help=''' this param is for external nativesdk dir, the param will be useful when you want to build in host ''' - ) + ) - parser.add_argument('-tag', '--docker_tag', dest='docker_tag', default = '', - help=''' + parser.add_argument('-tag', '--docker_tag', dest='docker_tag', default='', + help=''' when build in docker, the param can be point docker image ''' - ) + ) - parser.add_argument('-dt', '--datetime', dest = "datetime", - help=''' + parser.add_argument('-dt', '--datetime', dest="datetime", + help=''' this param is add DATETIME to local.conf, the value format is 20231212010101 ''') parser.add_argument('-df', '--disable_fetch', - dest = "is_disable_fetch", - action = "store_true", - help=''' + dest="is_disable_fetch", + action="store_true", + help=''' this param is set openeuler_fetch in local.conf, the default value is enable, if set -df, the OPENEULER_FETCH will set to 'disable' ''') @@ -142,13 +144,13 @@ class Generate(OebuildCommand): '--build_in', dest='build_in', choices=[oebuild_const.BUILD_IN_DOCKER, oebuild_const.BUILD_IN_HOST], - default = oebuild_const.BUILD_IN_DOCKER, help=''' + default=oebuild_const.BUILD_IN_DOCKER, help=''' This parameter marks the mode at build time, and is built in the container by docker ''') return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): # perpare parse help command if self.pre_parse_help(args, unknown): return @@ -217,8 +219,8 @@ class Generate(OebuildCommand): try: self._add_platform_template(args=args, - yocto_oebuild_dir=yocto_oebuild_dir, - parser_template=parser_template) + yocto_oebuild_dir=yocto_oebuild_dir, + parser_template=parser_template) except BaseParseTemplate as b_t: logger.error(str(b_t)) return @@ -228,8 +230,8 @@ class Generate(OebuildCommand): try: self._add_features_template(args=args, - yocto_oebuild_dir=yocto_oebuild_dir, - parser_template=parser_template) + yocto_oebuild_dir=yocto_oebuild_dir, + parser_template=parser_template) except BaseParseTemplate as b_t: logger.error(str(b_t)) self._list_feature() @@ -238,8 +240,8 @@ class Generate(OebuildCommand): logger.error(str(v_e)) return - if os.path.exists(os.path.join(build_dir,'compile.yaml')): - os.remove(os.path.join(build_dir,'compile.yaml')) + if os.path.exists(os.path.join(build_dir, 'compile.yaml')): + os.remove(os.path.join(build_dir, 'compile.yaml')) docker_image = YoctoEnv().get_docker_image(yocto_dir=yocto_dir) if docker_image is None: @@ -255,7 +257,7 @@ class Generate(OebuildCommand): following container, enter it numerically, and enter q to exit:''') image_list = check_docker_tag.get_tags() - for key,value in enumerate(image_list): + for key, value in enumerate(image_list): print(f"{key}, {oebuild_config.docker.repo_url}:{value}") k = input("please entry number:") if k == "q": @@ -270,20 +272,20 @@ class Generate(OebuildCommand): docker_tag = docker_tag.strip('\n') docker_image = f"{oebuild_config.docker.repo_url}:{docker_tag}" - out_dir = pathlib.Path(os.path.join(build_dir,'compile.yaml')) + out_dir = pathlib.Path(os.path.join(build_dir, 'compile.yaml')) oebuild_util.write_yaml(out_dir, parser_template.generate_compile_conf( - nativesdk_dir= self.nativesdk_dir, - toolchain_dir= self.toolchain_dir, + nativesdk_dir=self.nativesdk_dir, + toolchain_dir=self.toolchain_dir, build_in=build_in, - sstate_cache= self.sstate_cache, - tmp_dir = self.tmp_dir, - datetime = args.datetime, - is_disable_fetch = args.is_disable_fetch, + sstate_cache=self.sstate_cache, + tmp_dir=self.tmp_dir, + datetime=args.datetime, + is_disable_fetch=args.is_disable_fetch, docker_image=docker_image, src_dir=self.configure.source_dir(), compile_dir=build_dir - )) + )) self._print_generate(build_dir=build_dir) @@ -320,8 +322,8 @@ oebuild bitbake try: parser_template.add_template( os.path.join(yocto_oebuild_dir, - 'platform', - args.platform+'.yaml')) + 'platform', + args.platform+'.yaml')) except BaseParseTemplate as e_p: raise e_p else: @@ -335,8 +337,8 @@ wrong platform, please run `oebuild generate -l` to view support platform""") if feature + '.yaml' in os.listdir(os.path.join(yocto_oebuild_dir, 'features')): try: parser_template.add_template(os.path.join(yocto_oebuild_dir, - 'features', - feature+'.yaml')) + 'features', + feature+'.yaml')) except BaseParseTemplate as b_t: raise b_t else: @@ -368,7 +370,7 @@ wrong platform, please run `oebuild generate -l` to view support feature""") print(""" wrong input""") continue - if in_res in ['N','n', 'no']: + if in_res in ['N', 'n', 'no']: return None break if os.path.exists(os.path.join(build_dir, "conf")): @@ -404,13 +406,13 @@ wrong platform, please run `oebuild generate -l` to view support feature""") yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") list_feature = os.listdir(os.path.join(yocto_oebuild_dir, 'features')) print("the feature list is:") - table = PrettyTable(['feature name','support arch']) + table = PrettyTable(['feature name', 'support arch']) table.align = "l" for feature in list_feature: if feature.endswith('.yml'): - feature_name = feature.replace('.yml','') + feature_name = feature.replace('.yml', '') if feature.endswith('.yaml'): - feature_name = feature.replace('.yaml','') + feature_name = feature.replace('.yaml', '') feat = oebuild_util.read_yaml(pathlib.Path(os.path.join(yocto_oebuild_dir, 'features', feature))) diff --git a/src/oebuild/app/plugins/init/init.py b/src/oebuild/app/plugins/init/init.py index e342058..f0eefa6 100644 --- a/src/oebuild/app/plugins/init/init.py +++ b/src/oebuild/app/plugins/init/init.py @@ -22,6 +22,7 @@ from oebuild.configure import Configure, ConfigBasicRepo, Config from oebuild.m_log import logger import oebuild.const as oebuild_const + class Init(OebuildCommand): ''' Directory initialization directive, mainly used to initialize @@ -50,7 +51,7 @@ class Init(OebuildCommand): consideration of the build configuration of OEbuild, and can be easily called by third parties ''' - )) + )) def do_add_parser(self, parser_adder): self._parser( @@ -60,11 +61,11 @@ class Init(OebuildCommand): %(prog)s [directory] [-u yocto_remote_url] [-b branch] ''') - parser_adder.add_argument('-u','--yocto_remote_url', dest = 'yocto_remote_url', - help='''Specifies the remote of yocto-meta-openeuler''') + parser_adder.add_argument('-u', '--yocto_remote_url', dest='yocto_remote_url', + help='''Specifies the remote of yocto-meta-openeuler''') - parser_adder.add_argument('-b', '--branch', dest = 'branch', - help='''Specifies the branch of yocto-meta-openeuler''') + parser_adder.add_argument('-b', '--branch', dest='branch', + help='''Specifies the branch of yocto-meta-openeuler''') parser_adder.add_argument( 'directory', nargs='?', default=None, @@ -72,7 +73,7 @@ class Init(OebuildCommand): return parser_adder - def do_run(self, args: argparse.ArgumentParser, unknown = None): + def do_run(self, args: argparse.ArgumentParser, unknown=None): ''' detach target dicrectory if finished init, if inited, just put out err msg and exit ''' @@ -100,9 +101,9 @@ class Init(OebuildCommand): return os.chdir(args.directory) - oebuild_config:Config = self.configure.parse_oebuild_config() + oebuild_config: Config = self.configure.parse_oebuild_config() - yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] + yocto_config: ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] if args.yocto_remote_url is not None: yocto_config.remote_url = args.yocto_remote_url if args.branch is not None: @@ -111,7 +112,7 @@ class Init(OebuildCommand): self.configure.update_oebuild_config(oebuild_config) - logger.info("init %s successful",args.directory) + logger.info("init %s successful", args.directory) format_msg = f''' There is a build configuration example file under {args.directory}/.oebuild/compile.yaml.sample, if you want to block complex generate instructions, you can directly copy a configuration file, @@ -139,7 +140,7 @@ please execute the follow commands next return True @staticmethod - def create_oebuild_directory(updir : str): + def create_oebuild_directory(updir: str): ''' create oebuild config directory ''' @@ -152,7 +153,7 @@ please execute the follow commands next return "" @staticmethod - def create_src_directory(updir : str): + def create_src_directory(updir: str): ''' this is desctiption ''' @@ -165,7 +166,7 @@ please execute the follow commands next return None @staticmethod - def copy_config_file(updir : str): + def copy_config_file(updir: str): ''' copy oebuild config to some directory ''' @@ -176,7 +177,7 @@ please execute the follow commands next logger.error("mkdir config faild") @staticmethod - def copy_compile_file(updir : str): + def copy_compile_file(updir: str): ''' copy oebuild compile.yaml.sample to some directory ''' diff --git a/src/oebuild/app/plugins/m_env/m_env.py b/src/oebuild/app/plugins/m_env/m_env.py index 9552ae1..4a242e6 100644 --- a/src/oebuild/app/plugins/m_env/m_env.py +++ b/src/oebuild/app/plugins/m_env/m_env.py @@ -28,6 +28,7 @@ from oebuild.configure import Configure import oebuild.util as oebuild_util from oebuild.m_log import logger + class Menv(OebuildCommand): ''' the class is used to manager sdk environment, the sdk environment refers to openEuler @@ -35,6 +36,7 @@ class Menv(OebuildCommand): Embedded system you built. for example, the sdk with qt image can be used to develop apps runned in qt system, the sdk with ros image can be used to develop apps runned in ros system ''' + def __init__(self): self.configure = Configure() self.oebuild_env_path = os.path.expanduser('~') + '/.local/oebuild_env/' diff --git a/src/oebuild/app/plugins/m_plugin/m_plugin.py b/src/oebuild/app/plugins/m_plugin/m_plugin.py index af3403a..5c4a245 100644 --- a/src/oebuild/app/plugins/m_plugin/m_plugin.py +++ b/src/oebuild/app/plugins/m_plugin/m_plugin.py @@ -31,6 +31,7 @@ class MPlugin(OebuildCommand): manager it, the plugin must be in oebuild plugin standard, when you install the plugin you developped, you can use it through oebuild. """ + def __init__(self): self.configure = Configure() self.oebuild_plugin_commands = ['install', 'list', 'enable', 'disable', 'remove'] @@ -148,10 +149,10 @@ class MPlugin(OebuildCommand): append_command_ext = {} if args.plugin_name in self.command_ext.keys() \ - or args.plugin_name in append_command_ext.keys(): + or args.plugin_name in append_command_ext.keys(): while True: user_input = input( -f'Do you want to overwrite the existing plugin ({args.plugin_name}) in oebuild(Y/N)') + f'Do you want to overwrite the existing plugin ({args.plugin_name}) in oebuild(Y/N)') if user_input.lower() == 'y': break if user_input.lower() == 'n': @@ -293,7 +294,7 @@ f'Do you want to overwrite the existing plugin ({args.plugin_name}) in oebuild(Y def_name += "," if file_line.startswith('class') or file_line.startswith(' class'): if re.search(r'((?<=class)\s+\w+\(OebuildCommand\))', file_line) and \ - not class_name: + not class_name: class_name = re.search(r'(?<=class)\s+\w+', file_line).group().strip() return def_name, class_name diff --git a/src/oebuild/app/plugins/manifest/manifest.py b/src/oebuild/app/plugins/manifest/manifest.py index 33c1e4c..5facae0 100644 --- a/src/oebuild/app/plugins/manifest/manifest.py +++ b/src/oebuild/app/plugins/manifest/manifest.py @@ -18,7 +18,7 @@ import pathlib import multiprocessing from queue import Queue from threading import Thread -from concurrent.futures import ThreadPoolExecutor,wait,ALL_COMPLETED +from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED import git from git.repo import Repo @@ -29,6 +29,7 @@ from oebuild.configure import Configure import oebuild.util as oebuild_util from oebuild.m_log import logger + class Manifest(OebuildCommand): ''' manifest provides the manifest function of generating dependent @@ -47,7 +48,7 @@ class Manifest(OebuildCommand): source repositories in the build working directory, and can restore relevant source repositories based on the manifest file ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -64,11 +65,11 @@ class Manifest(OebuildCommand): help=''' specify a manifest path to perform the create or restore operation ''' - ) + ) return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): if not self.configure.is_oebuild_dir(): logger.error('Your current directory had not finished init') sys.exit(-1) @@ -117,11 +118,11 @@ class Manifest(OebuildCommand): print(f"Expose progress: {progress}%: ", "▋" * (progress // 2), end="") sys.stdout.flush() print() - manifest_list = dict(sorted(manifest_list.items(),key=lambda s:s[0])) + manifest_list = dict(sorted(manifest_list.items(), key=lambda s: s[0])) oebuild_util.write_yaml( yaml_dir=pathlib.Path(manifest_dir), data={'manifest_list': manifest_list}) - self._add_manifest_banner(manifest_dir = os.path.abspath(manifest_dir)) + self._add_manifest_banner(manifest_dir=os.path.abspath(manifest_dir)) print(f"expose successful, the directory is {os.path.abspath(manifest_dir)}") @@ -161,15 +162,15 @@ class Manifest(OebuildCommand): dserver.start() cpu_count = multiprocessing.cpu_count() - with ThreadPoolExecutor(max_workers = cpu_count) as t_p: + with ThreadPoolExecutor(max_workers=cpu_count) as t_p: src_dir = self.configure.source_dir() all_task = [] for key, value in manifest_list.items(): all_task.append(t_p.submit(self._download_repo, q_e, src_dir, key, value)) - wait(all_task, return_when = ALL_COMPLETED) + wait(all_task, return_when=ALL_COMPLETED) q_e.put("over") - def _download_repo(self, out_q: Queue ,src_dir, key, value): + def _download_repo(self, out_q: Queue, src_dir, key, value): repo_dir = os.path.join(src_dir, key) repo = Repo.init(repo_dir) remote = None @@ -180,10 +181,10 @@ class Manifest(OebuildCommand): continue if remote is None: remote_name = "upstream" - remote = git.Remote.add(repo = repo, name = remote_name, url = value['remote_url']) + remote = git.Remote.add(repo=repo, name=remote_name, url=value['remote_url']) try: repo.git.checkout(value['version']) except GitCommandError: - remote.fetch(value['version'], depth = 1) + remote.fetch(value['version'], depth=1) repo.git.checkout(value['version']) out_q.put('ok') diff --git a/src/oebuild/app/plugins/run_qemu/run_qemu.py b/src/oebuild/app/plugins/run_qemu/run_qemu.py index a73e2b1..5b07801 100644 --- a/src/oebuild/app/plugins/run_qemu/run_qemu.py +++ b/src/oebuild/app/plugins/run_qemu/run_qemu.py @@ -25,10 +25,12 @@ from oebuild.configure import Configure from oebuild.m_log import logger import oebuild.const as oebuild_const + class RunQemu(OebuildCommand): ''' The command for run in qemu platform. ''' + def __init__(self): self.configure = Configure() self.client = None @@ -92,7 +94,7 @@ the container {self.container_id} failed to be destroyed, please run ''' exec qemu ''' - container:Container = self.client.get_container(self.container_id) # type: ignore + container: Container = self.client.get_container(self.container_id) # type: ignore self.bak_bash(container=container) self.init_bash(container=container) content = self._get_bashrc_content(container=container) @@ -145,7 +147,7 @@ now, you can continue run `oebuild runqemu` in compile directory sys.exit(0) return - def deal_env_container(self,docker_image): + def deal_env_container(self, docker_image): ''' This operation realizes the processing of the container, controls how the container is processed by parsing the env @@ -161,14 +163,14 @@ now, you can continue run `oebuild runqemu` in compile directory volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS + " --privileged" - container:Container = self.client.create_container( + container: Container = self.client.create_container( image=docker_image, parameters=parameters, volumes=volumns, command="bash") self.container_id = container.short_id - container:Container = self.client.get_container(self.container_id) + container: Container = self.client.get_container(self.container_id) if not self.client.is_container_running(container): self.client.start_container(container) diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py index 6dcc826..96d8338 100644 --- a/src/oebuild/app/plugins/update/update.py +++ b/src/oebuild/app/plugins/update/update.py @@ -29,11 +29,13 @@ from oebuild.check_docker_tag import CheckDockerTag import oebuild.const as oebuild_const from oebuild.m_log import logger + class Update(OebuildCommand): ''' The update command will prepare the basic environment related to the build, such as container images, build base repositories, etc ''' + def __init__(self): self.configure = Configure() @@ -66,11 +68,11 @@ class Update(OebuildCommand): usage=''' %(prog)s [yocto docker layer] [-tag] ''') - parser.add_argument('-tag','--tag', dest='docker_tag', - help=''' + parser.add_argument('-tag', '--tag', dest='docker_tag', + help=''' with platform will list support archs, with feature will list support features ''' - ) + ) parser.add_argument( 'item', nargs='?', default=None, @@ -78,7 +80,7 @@ class Update(OebuildCommand): return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): ''' update action rely on directory which has initd, so check it first ''' @@ -110,7 +112,6 @@ class Update(OebuildCommand): logger.error('Please run oebuild update [yocto docker layer]') sys.exit(-1) - if update_yocto: self.get_basic_repo() @@ -147,17 +148,17 @@ class Update(OebuildCommand): if repos is None: return - for _ , item in repos.items(): + for _, item in repos.items(): if isinstance(item, OebuildRepo): local_dir = os.path.join(self.configure.source_dir(), item.path) - key_repo = OGit(repo_dir = local_dir, - remote_url = item.url, - branch = item.refspec) + key_repo = OGit(repo_dir=local_dir, + remote_url=item.url, + branch=item.refspec) else: local_dir = os.path.join(self.configure.source_dir(), item['path']) - key_repo = OGit(repo_dir = local_dir, - remote_url = item['url'], - branch = item['refspec']) + key_repo = OGit(repo_dir=local_dir, + remote_url=item['url'], + branch=item['refspec']) key_repo.clone_or_pull_repo() def get_basic_repo(self,): @@ -171,7 +172,7 @@ class Update(OebuildCommand): embedded/src/yocto-meta-openeuler not exists, so just clone from config setting. ''' oebuild_config = self.configure.parse_oebuild_config() - yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] + yocto_config: ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] local_dir = os.path.join(self.configure.source_dir(), yocto_config.path) yocto_repo = OGit(repo_dir=local_dir, @@ -179,7 +180,7 @@ class Update(OebuildCommand): branch=yocto_config.branch) yocto_repo.clone_or_pull_repo() - def docker_image_update(self, docker_tag = None): + def docker_image_update(self, docker_tag=None): ''' The container update logic is to update the corresponding tag container image if tag is specified, otherwise it is determined @@ -189,7 +190,7 @@ class Update(OebuildCommand): ''' oebuild_config = self.configure.parse_oebuild_config() docker_config = oebuild_config.docker - check_docker_tag = CheckDockerTag(docker_tag=docker_tag,configure=self.configure) + check_docker_tag = CheckDockerTag(docker_tag=docker_tag, configure=self.configure) if docker_tag is not None: if check_docker_tag.get_tag() is None or check_docker_tag.get_tag() == "": check_docker_tag.list_image_tag() diff --git a/src/oebuild/bb/utils.py b/src/oebuild/bb/utils.py index 4ac107e..791f385 100644 --- a/src/oebuild/bb/utils.py +++ b/src/oebuild/bb/utils.py @@ -6,6 +6,7 @@ import os import re + def preserved_envvars_exported(): """Variables which are taken from the environment and placed in and exported from the metadata""" @@ -21,6 +22,7 @@ def preserved_envvars_exported(): 'BBSERVER', ] + def preserved_envvars(): """Variables which are taken from the environment and placed in the metadata""" v = [ @@ -31,6 +33,7 @@ def preserved_envvars(): ] return v + preserved_envvars_exported() + def approved_variables(): """ Determine and return the list of whitelisted variables which are approved @@ -50,6 +53,7 @@ def approved_variables(): approved.extend(['BB_ENV_EXTRAWHITE']) return approved + def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): """Edit lines from a recipe or config file and modify one or more specified variable values set in the file using a specified callback @@ -105,9 +109,11 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): override_re = '' for var in variables: if var.endswith('()'): - var_res[var] = re.compile(r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re)) + var_res[var] = re.compile( + r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re)) else: - var_res[var] = re.compile(r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re)) + var_res[var] = re.compile( + r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re)) updated = False varset_start = '' @@ -143,7 +149,8 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): if in_var.endswith('()'): # A function definition if isinstance(newvalue, list): - newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue))) + newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, + ('\n%s' % indentspc).join(newvalue))) else: if not newvalue.startswith('\n'): newvalue = '\n' + newvalue @@ -237,6 +244,7 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): newlines.append(line) return (updated, newlines) + def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): """Edit bblayers.conf, adding and/or removing layers Parameters: @@ -262,6 +270,7 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): return pth approved = approved_variables() + def canonicalise_path(pth): pth = remove_trailing_sep(pth) if 'HOME' in approved and '~' in pth: diff --git a/src/oebuild/bblayers.py b/src/oebuild/bblayers.py index d9e5541..e92770c 100644 --- a/src/oebuild/bblayers.py +++ b/src/oebuild/bblayers.py @@ -14,12 +14,14 @@ import os import oebuild.bb.utils as bb_utils + class BBLayers: ''' The BBlayer class implements the layer added in the container environment in the physical environment, and the add operation references bitbake-related code ''' + def __init__(self, bblayers_dir: str, base_dir: str): self._base_dir = base_dir self._bblayers_dir = bblayers_dir @@ -62,7 +64,7 @@ class BBLayers: bb_utils.edit_bblayers_conf(self.bblayers_dir, add=bblayers, remove=None) - def check_layer_exist(self, layers:str or list): + def check_layer_exist(self, layers: str or list): ''' To check if it is legitimate to add a layer, the main thing is to verify the existence of layer.conf diff --git a/src/oebuild/check_docker_tag.py b/src/oebuild/check_docker_tag.py index 5ba23d4..dc6bda2 100644 --- a/src/oebuild/check_docker_tag.py +++ b/src/oebuild/check_docker_tag.py @@ -18,6 +18,7 @@ from git import Repo from oebuild.configure import Configure import oebuild.util as oebuild_util + class CheckDockerTag: ''' This class is used to synthesize the build environment @@ -27,6 +28,7 @@ class CheckDockerTag: configuration file, the yocto branch name, etc. Judge down in turn, and finally return a suitable docker image version ''' + def __init__(self, docker_tag: str, configure: Configure): self.docker_tag = docker_tag self.configure = configure @@ -68,7 +70,7 @@ the openeuler embedded docker tag can be selected list: return str(self.docker_tag) yocto_dir = self.configure.source_yocto_dir() - env_path = os.path.join(yocto_dir,".oebuild/env.yaml") + env_path = os.path.join(yocto_dir, ".oebuild/env.yaml") if os.path.exists(env_path): env_parse = oebuild_util.read_yaml(pathlib.Path(env_path)) return str(env_parse['docker_tag']) diff --git a/src/oebuild/command.py b/src/oebuild/command.py index 07d22bf..04cf37d 100644 --- a/src/oebuild/command.py +++ b/src/oebuild/command.py @@ -21,6 +21,7 @@ WRN_COLOR = colorama.Fore.LIGHTYELLOW_EX ERR_COLOR = colorama.Fore.LIGHTRED_EX + class OebuildCommand(ABC): '''Abstract superclass for a oebuild command.''' @@ -28,7 +29,7 @@ class OebuildCommand(ABC): self.name = name self.help_msg = help_msg self.description = description - self.parser:argparse.ArgumentParser = None + self.parser: argparse.ArgumentParser = None def run(self, args: argparse.ArgumentParser, unknown: List[str]): ''' @@ -58,13 +59,13 @@ class OebuildCommand(ABC): :param parser_adder: The return value of a call to ``argparse.ArgumentParser.add_subparsers()`` ''' - self.parser:argparse.ArgumentParser = self.do_add_parser(parser_adder) + self.parser: argparse.ArgumentParser = self.do_add_parser(parser_adder) if self.parser is None: raise ValueError('do_add_parser did not return a value') self.parser.add_argument('-h', '--help', dest="help", action="store_true", - help='get help for oebuild or a command') + help='get help for oebuild or a command') return self.parser diff --git a/src/oebuild/configure.py b/src/oebuild/configure.py index 9a71322..25adeb3 100644 --- a/src/oebuild/configure.py +++ b/src/oebuild/configure.py @@ -20,9 +20,11 @@ import oebuild.const as oebuild_const PathType = Union[str, os.PathLike] + class OebuildNotFound(RuntimeError): '''Neither the current directory nor any parent has a oebuild workspace.''' + @dataclass class ConfigContainer: ''' @@ -34,6 +36,7 @@ class ConfigContainer: # tag_mag is for branch to container tag map tag_map: Dict + @dataclass class ConfigBasicRepo: ''' @@ -48,6 +51,7 @@ class ConfigBasicRepo: # branch is for repo's branch branch: str + @dataclass class Config: ''' @@ -57,6 +61,7 @@ class Config: basic_repo: dict + class Configure: ''' Configure object is to contain some generally param or function about oebuild @@ -64,7 +69,7 @@ class Configure: @staticmethod def oebuild_topdir(start: Optional[PathType] = None, - fall_back: bool = True): + fall_back: bool = True): ''' Like oebuild_dir(), but returns the path to the parent directory of the .oebuild/ directory instead, where project repositories are stored @@ -82,7 +87,7 @@ class Configure: return Configure.oebuild_topdir(fall_back=False) raise OebuildNotFound('Could not find a oebuild workspace ' - 'in this or any parent directory') + 'in this or any parent directory') cur_dir = parent_dir @staticmethod @@ -123,7 +128,7 @@ class Configure: ''' config = Configure.parse_oebuild_config() basic_config = config.basic_repo - yocto_config:ConfigBasicRepo = basic_config[oebuild_const.YOCTO_META_OPENEULER] + yocto_config: ConfigBasicRepo = basic_config[oebuild_const.YOCTO_META_OPENEULER] yocto_dir = yocto_config.path return os.path.join(Configure.source_dir(), yocto_dir) @@ -164,7 +169,7 @@ class Configure: ''' config = oebuild_util.read_yaml( - yaml_dir = pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG)) + yaml_dir=pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG)) tag_map = {} for key, value in config['docker']['tag_map'].items(): @@ -199,19 +204,20 @@ class Configure: basic_config = config.basic_repo data['basic_repo'] = {} for key, repo in basic_config.items(): - repo:ConfigBasicRepo = repo + repo: ConfigBasicRepo = repo data['basic_repo'][key] = {'path': repo.path, 'remote_url': repo.remote_url, 'branch': repo.branch} try: oebuild_util.write_yaml( - yaml_dir = pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG), + yaml_dir=pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG), data=data) return True except TypeError: return False + class YoctoEnv: ''' the YoctoEnv class is used for yocto-meta-openeuler/.oebuild/env.yaml @@ -224,7 +230,7 @@ class YoctoEnv: if not os.path.exists(yocto_dir): raise ValueError("the yocto direction is not exists") - env_path = os.path.join(yocto_dir,".oebuild/env.yaml") + env_path = os.path.join(yocto_dir, ".oebuild/env.yaml") if not os.path.exists(env_path): return None diff --git a/src/oebuild/docker_proxy.py b/src/oebuild/docker_proxy.py index f3b00ff..d0e3a03 100644 --- a/src/oebuild/docker_proxy.py +++ b/src/oebuild/docker_proxy.py @@ -22,10 +22,12 @@ from docker.models.containers import Container from oebuild.m_log import logger + class DockerProxy: ''' a object just be wrapper again to run docker command easily ''' + def __init__(self): self._docker = docker.from_env() @@ -53,13 +55,13 @@ class DockerProxy: except NotFound: return False - def pull_image(self, image_name : str): + def pull_image(self, image_name: str): ''' pull image like command 'docker pull' args: image_name (str): docker image name, if no tag, the default tag is latest ''' - repository,tag = self._get_image_name_tag(image_name=image_name) + repository, tag = self._get_image_name_tag(image_name=image_name) self._docker.images.pull(repository=repository, tag=tag) @@ -72,7 +74,7 @@ class DockerProxy: def _get_image_name_tag(self, image_name: str): repository = image_name.split(':')[0] tag = "latest" - if len(image_name.split(':'))==2: + if len(image_name.split(':')) == 2: tag = image_name.split(':')[1] return repository, tag @@ -84,7 +86,6 @@ class DockerProxy: ''' return self._docker.images.get(image_name) - def get_container(self, container_id): ''' get a docker container object @@ -93,7 +94,6 @@ class DockerProxy: ''' return self._docker.containers.get(container_id=container_id) - def get_all_container(self): ''' get all container like command 'docker ps -a' @@ -112,13 +112,13 @@ class DockerProxy: container.stop() @staticmethod - def delete_container(container: Container, is_force:bool = False): + def delete_container(container: Container, is_force: bool = False): ''' rm a container which not running like command 'docker rm' args: container (Container): container object ''' - container.remove(force = is_force) + container.remove(force=is_force) @staticmethod def start_container(container: Container): @@ -130,7 +130,7 @@ class DockerProxy: container.start() @staticmethod - def is_container_running(container : Container): + def is_container_running(container: Container): ''' determize if a container in running state args: @@ -147,10 +147,10 @@ class DockerProxy: args: path_dir (str): the directory that will added to a tar ''' - if os.path.exists(path = path_dir): + if os.path.exists(path=path_dir): pw_tarstream = BytesIO() with tarfile.TarFile(fileobj=pw_tarstream, mode='w') as pw_tar: - pw_tar.add(name = path_dir, arcname=os.path.basename(path_dir)) + pw_tar.add(name=path_dir, arcname=os.path.basename(path_dir)) pw_tarstream.seek(0) return pw_tarstream return None @@ -164,7 +164,7 @@ class DockerProxy: to_path (str): will copy to docker container path ''' tar = self.add_tar(source_path) - return container.put_archive(path = to_path,data = tar) + return container.put_archive(path=to_path, data=tar) def copy_from_container(self, container: Container, from_path, dst_path): ''' @@ -179,17 +179,17 @@ class DockerProxy: for trunk in bits: pw_tarstream.write(trunk) pw_tarstream.seek(0) - with tarfile.open(fileobj = pw_tarstream) as tar: - res = tar.extractall(path = dst_path) + with tarfile.open(fileobj=pw_tarstream) as tar: + res = tar.extractall(path=dst_path) return res is None def container_exec_command(self, container: Container, command: str or list, user: str = '', - work_space = None, - stream = True, - demux = False): + work_space=None, + stream=True, + demux=False): ''' run command like 'docker run exec', other param will be default and just use a little params @@ -208,10 +208,10 @@ class DockerProxy: return res def create_container(self, - image:str, - parameters:str, - volumes:list[str], - command:str) -> Container: + image: str, + parameters: str, + volumes: list[str], + command: str) -> Container: ''' create a new container ''' diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index 1f93ed6..e8be9d2 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -19,20 +19,26 @@ import oebuild.util as oebuild_util from oebuild.m_log import logger import oebuild.const as oebuild_const + class BaseLocalConf(ValueError): ''' basic error about parse_template ''' + + class NativesdkNotExist(BaseLocalConf): ''' nativesdk directory not exist ''' + + class NativesdkNotValid(BaseLocalConf): ''' nativesdk directory not valid ''' -def get_nativesdk_sysroot(nativesdk_dir = oebuild_const.NATIVESDK_DIR): + +def get_nativesdk_sysroot(nativesdk_dir=oebuild_const.NATIVESDK_DIR): ''' return environment initialization shell, if nativesdk directory is not exists or can not find any initialization shell, raise error @@ -70,6 +76,7 @@ def match_and_add(new_str: str, content: str): content = content + '\n' return content + def match_and_replace(pre: str, new_str: str, content: str): ''' math line in content when the new_str exist and replace @@ -85,6 +92,7 @@ def match_and_replace(pre: str, new_str: str, content: str): content = content + '\n' return content + class LocalConf: ''' LocalConf corresponds to the local.conf configuration @@ -94,7 +102,7 @@ class LocalConf: def __init__(self, local_conf_dir: str): self.local_dir = local_conf_dir - def update(self, parse_compile: ParseCompile, src_dir = None): + def update(self, parse_compile: ParseCompile, src_dir=None): ''' update local.conf by ParseCompile ''' @@ -143,40 +151,40 @@ class LocalConf: content = match_and_replace( pre=oebuild_const.OPENEULER_SP_DIR, - new_str= f"{oebuild_const.OPENEULER_SP_DIR} = '{src_dir}'", + new_str=f"{oebuild_const.OPENEULER_SP_DIR} = '{src_dir}'", content=content ) # replace sstate_cache if parse_compile.sstate_cache is not None: if os.path.islink(parse_compile.sstate_cache): - new_str= f"file://.* {parse_compile.sstate_cache}/PATH;downloadfilename=PATH" + new_str = f"file://.* {parse_compile.sstate_cache}/PATH;downloadfilename=PATH" else: if parse_compile.build_in == oebuild_const.BUILD_IN_DOCKER: - new_str= f"file://.* file://{oebuild_const.SSTATE_CACHE}/PATH" + new_str = f"file://.* file://{oebuild_const.SSTATE_CACHE}/PATH" else: - new_str= f"file://.* file://{parse_compile.sstate_cache}/PATH" + new_str = f"file://.* file://{parse_compile.sstate_cache}/PATH" content = match_and_replace( - pre=oebuild_const.SSTATE_MIRRORS, - new_str = f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"', - content=content - ) + pre=oebuild_const.SSTATE_MIRRORS, + new_str=f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"', + content=content + ) # replace sstate_dir if parse_compile.sstate_dir is not None: content = match_and_replace( - pre=oebuild_const.SSTATE_DIR, - new_str = f'{oebuild_const.SSTATE_DIR} = "{parse_compile.sstate_dir}"', - content=content - ) + pre=oebuild_const.SSTATE_DIR, + new_str=f'{oebuild_const.SSTATE_DIR} = "{parse_compile.sstate_dir}"', + content=content + ) # replace tmpdir if parse_compile.tmp_dir is not None: content = match_and_replace( - pre=oebuild_const.TMP_DIR, - new_str = f'{oebuild_const.TMP_DIR} = "{parse_compile.tmp_dir}"', - content=content - ) + pre=oebuild_const.TMP_DIR, + new_str=f'{oebuild_const.TMP_DIR} = "{parse_compile.tmp_dir}"', + content=content + ) content = self.match_lib_param(content=content) @@ -201,12 +209,12 @@ class LocalConf: for key, value in lib_param_list.items(): content = match_and_replace( - pre = key, - new_str = f'{key} = "{value}"', - content = content) + pre=key, + new_str=f'{key} = "{value}"', + content=content) return content - def replace_param(self, parse_compile: ParseCompile, content:str): + def replace_param(self, parse_compile: ParseCompile, content: str): ''' match and replace param by ParseCompile.local_conf ''' @@ -249,4 +257,3 @@ class LocalConf: if not oecore_sysroot_dir.startswith(nativesdk_dir): raise NativesdkNotValid(f"nativesdk directory: {nativesdk_dir} are not valid") return - \ No newline at end of file diff --git a/src/oebuild/oebuild_parser.py b/src/oebuild/oebuild_parser.py index 5d43b89..af6a11c 100644 --- a/src/oebuild/oebuild_parser.py +++ b/src/oebuild/oebuild_parser.py @@ -16,10 +16,12 @@ import shutil from io import StringIO import textwrap + class OebuildHelpAction(argparse.Action): ''' set argparse help is true ''' + def __call__(self, parser, namespace, values, option_string=None): # Just mark that help was requested. namespace.help = True @@ -61,7 +63,7 @@ class OebuildArgumentParser(argparse.ArgumentParser): # one of the subcommand parsers, and we delegate to super. # if not top_level: - # return super(OebuildArgumentParser, self).format_help() + # return super(OebuildArgumentParser, self).format_help() # Format the help to be at most 75 columns wide, the maximum # generally recommended by typographers for readability. @@ -87,7 +89,7 @@ class OebuildArgumentParser(argparse.ArgumentParser): if self.oebuild_app is not None: append('') - for _,command in self.oebuild_app.command_spec.items(): + for _, command in self.oebuild_app.command_spec.items(): self._format_command(append, command, width) if self.epilog: diff --git a/src/oebuild/ogit.py b/src/oebuild/ogit.py index c04d253..5218660 100644 --- a/src/oebuild/ogit.py +++ b/src/oebuild/ogit.py @@ -18,10 +18,12 @@ from git import GitCommandError, RemoteProgress from oebuild.m_log import logger + class OGit: ''' owner git to print progress in clone action ''' + def __init__(self, repo_dir, remote_url, branch) -> None: self._repo_dir = repo_dir self._remote_url = remote_url @@ -99,14 +101,16 @@ class OGit: except TypeError: return "", '' except git.GitError: - return "","" + return "", "" except ValueError: - return "","" + return "", "" + class CustomRemote(git.RemoteProgress): ''' Rewrote RemoteProgress to show the process of code updates ''' + def update(self, op_code, cur_count, max_count=None, message=''): ''' rewrote update function diff --git a/src/oebuild/parse_compile.py b/src/oebuild/parse_compile.py index 3f26112..d4ef81e 100644 --- a/src/oebuild/parse_compile.py +++ b/src/oebuild/parse_compile.py @@ -22,6 +22,7 @@ from oebuild.parse_template import PlatformTemplate, ParseTemplate import oebuild.const as oebuild_const from oebuild.m_log import logger + @dataclass class DockerParam: ''' @@ -60,21 +61,25 @@ class Compile(PlatformTemplate): docker_param: Optional[DockerParam] + class BaseParseCompileError(ValueError): ''' parse compile basic error ''' + class CheckCompileError(BaseParseCompileError): ''' compile.yaml parse check faild error ''' + class ParseCompile: ''' This class is used to parse compile.yaml and download the relevant code repository ''' + def __init__(self, compile_conf_dir): ''' The initialization operation is used to parse the compile.yaml @@ -91,7 +96,7 @@ class ParseCompile: except Exception as e_p: raise e_p - docker_param:DockerParam = None + docker_param: DockerParam = None if "docker_param" in data and data['docker_param'] is not None: try: dparam = data['docker_param'] @@ -253,7 +258,7 @@ class ParseCompile: keys = { "machine": "the key machine is None", "toolchain_type": "the key toolchain_type is None" - } + } for key, value in keys.items(): if key not in data: @@ -266,10 +271,9 @@ class ParseCompile: if "repos" in data: for _, repo in data['repos'].items(): required_keys = { - "url": "the key url is None", - "path": "the key path is None", + "url": "the key url is None", + "path": "the key path is None", "refspec": "the key refspec is None"} for key, value in required_keys.items(): if key not in repo: raise CheckCompileError(value) - \ No newline at end of file diff --git a/src/oebuild/parse_env.py b/src/oebuild/parse_env.py index 98fcc44..f6bd60f 100644 --- a/src/oebuild/parse_env.py +++ b/src/oebuild/parse_env.py @@ -18,6 +18,7 @@ import sys import oebuild.util as oebuild_util from oebuild.m_log import logger + @dataclass class EnvContainer: ''' @@ -25,6 +26,7 @@ class EnvContainer: ''' short_id: Optional[str] + @dataclass class Env: ''' @@ -32,14 +34,16 @@ class Env: ''' container: Optional[EnvContainer] + class ParseEnv: ''' This class is used to parse env.yaml and update env.yaml ''' + def __init__(self, env_dir): self.env_dir = pathlib.Path(env_dir) if isinstance(env_dir, str) else env_dir - self.env:Env = Env(container=None) + self.env: Env = Env(container=None) self._parse_env() @property @@ -60,7 +64,7 @@ class ParseEnv: if "container" in data: env_container = data['container'] try: - self.env.container = EnvContainer( + self.env.container = EnvContainer( short_id=env_container['short_id'] ) except KeyError: diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index a41947b..5055cb0 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -21,6 +21,7 @@ from ruamel.yaml.scalarstring import LiteralScalarString import oebuild.util as oebuild_util import oebuild.const as oebuild_const + @dataclass class OebuildRepo: ''' @@ -40,6 +41,7 @@ class OebuildRepo: refspec: str + @dataclass class Template: ''' @@ -51,6 +53,7 @@ class Template: local_conf: Optional[LiteralScalarString] + @dataclass class PlatformTemplate(Template): ''' @@ -60,6 +63,7 @@ class PlatformTemplate(Template): toolchain_type: LiteralScalarString + @dataclass class FeatureTemplate(Template): ''' @@ -69,36 +73,43 @@ class FeatureTemplate(Template): support: list + class BaseParseTemplate(ValueError): ''' basic error about parse_template ''' + class ConfigPathNotExists(BaseParseTemplate): ''' config path not exists ''' + class PlatformNotAdd(BaseParseTemplate): ''' platform not add first ''' + class FeatureNotSupport(BaseParseTemplate): ''' feature not support ''' + class CommonNotFound(BaseParseTemplate): ''' common param not found ''' + class ParseTemplate: ''' ParseTemplate is to add platform template and feature template and export compile.yaml finially ''' - def __init__(self, yocto_dir:str): + + def __init__(self, yocto_dir: str): self.yocto_dir = yocto_dir self.build_in = None self.platform_template = None @@ -169,16 +180,16 @@ your arch is {self.platform}, the feature is not supported, please check your ap raise e_p def generate_compile_conf(self, - nativesdk_dir = None, - toolchain_dir = None, - build_in: str = oebuild_const.BUILD_IN_DOCKER, - sstate_cache = None, - tmp_dir = None, - datetime = None, - is_disable_fetch = False, - docker_image: str = None, - src_dir: str = None, - compile_dir: str = None): + nativesdk_dir=None, + toolchain_dir=None, + build_in: str = oebuild_const.BUILD_IN_DOCKER, + sstate_cache=None, + tmp_dir=None, + datetime=None, + is_disable_fetch=False, + docker_image: str = None, + src_dir: str = None, + compile_dir: str = None): ''' first param common yaml ''' @@ -193,7 +204,7 @@ your arch is {self.platform}, the feature is not supported, please check your ap data = oebuild_util.read_yaml(common_yaml_dir) repos = {} - if 'repos' in data : + if 'repos' in data: repos.update(data['repos']) layers = [] if 'layers' in data: @@ -217,10 +228,10 @@ your arch is {self.platform}, the feature is not supported, please check your ap layers = self.platform_template.layers if self.platform_template.local_conf is not None: - local_conf = LiteralScalarString(self.platform_template.local_conf + local_conf ) + local_conf = LiteralScalarString(self.platform_template.local_conf + local_conf) for feature in self.feature_template: - feature:FeatureTemplate = feature + feature: FeatureTemplate = feature if feature.repos is not None: for repo_name, oebuild_repo in feature.repos.items(): if repo_name in repos: @@ -270,7 +281,7 @@ your arch is {self.platform}, the feature is not supported, please check your ap volumns.append("/dev/net/tun:/dev/net/tun") volumns.append(src_dir + ':' + oebuild_const.CONTAINER_SRC) volumns.append(compile_dir + ':' + - os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) + os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) if toolchain_dir is not None: volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) if sstate_cache is not None: diff --git a/src/oebuild/spec.py b/src/oebuild/spec.py index 13ef54a..5b19ba7 100644 --- a/src/oebuild/spec.py +++ b/src/oebuild/spec.py @@ -15,10 +15,12 @@ import os from dataclasses import dataclass import sys + class CommandError(RuntimeError): ''' Indicates that a command failed. ''' + def __init__(self, returncode=1): super().__init__() self.returncode = returncode @@ -32,11 +34,11 @@ class ExtensionCommandError(CommandError): '''Exception class indicating an extension command was badly defined and could not be created.''' - def __init__(self, **kwargs): self.hint = kwargs.pop('hint', None) super(ExtensionCommandError, self).__init__(**kwargs) + @dataclass class _CmdFactory: @@ -91,6 +93,7 @@ class OebuildExtCommandSpec: # the command) before constructing it, however. factory: _CmdFactory + @dataclass class _ExtCommand: ''' diff --git a/src/oebuild/util.py b/src/oebuild/util.py index 6ed560a..576bf7d 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -54,7 +54,7 @@ def get_nativesdk_environment(nativesdk_dir=oebuild_const.NATIVESDK_DIR, sys.exit(res.exit_code) list_items = res.output.decode("utf-8").split("\n") for item in list_items: - item:str = item + item: str = item # notice: the item is like format with # "drwxr-xr-x 3 openeuler openeuler 4096 Nov 8 08:10 ." # so we must get the last clip from split with space @@ -68,7 +68,8 @@ def get_nativesdk_environment(nativesdk_dir=oebuild_const.NATIVESDK_DIR, logger.error("can not find any nativesdk environment initialization shell") sys.exit(1) -def read_yaml(yaml_dir : pathlib.Path): + +def read_yaml(yaml_dir: pathlib.Path): ''' read yaml file and parse it to object ''' @@ -84,7 +85,7 @@ def read_yaml(yaml_dir : pathlib.Path): raise e_p -def write_yaml(yaml_dir : pathlib.Path, data): +def write_yaml(yaml_dir: pathlib.Path, data): ''' write data to yaml file ''' @@ -97,14 +98,16 @@ def write_yaml(yaml_dir : pathlib.Path, data): yaml = YAML() yaml.dump(data, w_f) -def get_git_repo_name(remote_url : str): + +def get_git_repo_name(remote_url: str): ''' return repo name ''' - url = remote_url.replace(".git","") + url = remote_url.replace(".git", "") return os.path.basename(url) -def add_git_suffix(remote : str): + +def add_git_suffix(remote: str): ''' add .git suffix to remote if needed ''' @@ -113,36 +116,42 @@ def add_git_suffix(remote : str): return remote + ".git" + def get_base_oebuild(): ''' return oebuild base dir ''' return os.path.abspath(os.path.dirname(__file__)) + def get_config_yaml_dir(): ''' return config yaml dir ''' return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.CONFIG_YAML) + def get_plugins_yaml_path(): ''' return plugin yaml path ''' return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.PLUGINS_YAML) + def get_compile_yaml_dir(): ''' return compile.yaml.sample yaml dir ''' return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.COMPILE_YAML) + def get_upgrade_yaml_dir(): ''' return upgrade yaml dir ''' return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.UPGRADE_YAML) + def generate_random_str(randomlength=16): ''' generate a random string by length @@ -154,18 +163,21 @@ def generate_random_str(randomlength=16): random_str += base_str[random.randint(0, length)] return random_str + def get_time_stamp(): ''' get current timestamp ''' return time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) + def get_oebuild_version(): ''' return oebuild version ''' return __version__ + def check_docker(): ''' check docker had be installed or not @@ -181,12 +193,14 @@ please install docker first, and run follow commands in root: 4, chmod o+rw /var/run/docker.sock ''') from exc + def get_instance(factory): ''' Instantiate a class ''' return factory() + def restore_bashrc_content(old_content): ''' restore .bashrc @@ -199,6 +213,7 @@ def restore_bashrc_content(old_content): new_content = new_content + line + '\n' return new_content + def init_bashrc_content(old_content, init_command: list): ''' init bashrc @@ -210,6 +225,7 @@ def init_bashrc_content(old_content, init_command: list): return new_content + def add_bashrc(content: str, line: str): ''' add command line to bashrc @@ -220,6 +236,7 @@ def add_bashrc(content: str, line: str): return content + def get_host_proxy(proxy_name): ''' get proxy information from host -- Gitee