diff --git a/src/oebuild/app/main.py b/src/oebuild/app/main.py index 88674fc58f8127706fa32018d1a0e967d200f49d..ac10b47da124357e7c9645f42df33cccdabd9bfd 100644 --- a/src/oebuild/app/main.py +++ b/src/oebuild/app/main.py @@ -20,25 +20,27 @@ import colorama import oebuild.util as oebuild_util from oebuild.auto_completion import AutoCompletion from oebuild.version import __version__ -from oebuild.spec import get_spec,_ExtCommand +from oebuild.spec import get_spec, _ExtCommand from oebuild.command import OebuildCommand from oebuild.m_log import logger -from oebuild.oebuild_parser import OebuildArgumentParser,OebuildHelpAction +from oebuild.oebuild_parser import OebuildArgumentParser, OebuildHelpAction APP = "app" + class OebuildApp: ''' The execution body of the oebuild tool, and all oebuild commands are resolved and executed by this body ''' + def __init__(self): self.base_oebuild_dir = oebuild_util.get_base_oebuild() self.oebuild_parser = None self.subparsers = {} self.cmd = None try: - plugins_dir = pathlib.Path(self.base_oebuild_dir,'app/conf','plugins.yaml') + plugins_dir = pathlib.Path(self.base_oebuild_dir, 'app/conf', 'plugins.yaml') self.oebuild_plugins_path = os.path.expanduser('~') + '/.local/oebuild_plugins/' self.append_plugins_dir = pathlib.Path(self.oebuild_plugins_path, 'append_plugins.yaml') self.command_ext = self.get_command_ext(oebuild_util.read_yaml(plugins_dir)['plugins']) @@ -66,11 +68,9 @@ class OebuildApp: path=app['path']) return command_ext - def _load_extension_specs(self, ): self.command_spec = extension_commands(APP, self.command_ext) - def _setup_parsers(self): # Set up and install command-line argument parsers. @@ -109,18 +109,16 @@ class OebuildApp: return parser, subparser_gen - def _check_command(self, args): if args.help or \ - args.command is None or \ - args.command not in self.command_ext or \ - args.command == 'help': + args.command is None or \ + args.command not in self.command_ext or \ + args.command == 'help': self.help() return False return True - def run_command(self, argv): ''' Parse command line arguments and run the OebuildCommand. @@ -135,10 +133,10 @@ class OebuildApp: # Finally, run the command. self._run_extension(args.command, unknown) - def _run_extension(self, name:str, unknown): + def _run_extension(self, name: str, unknown): # Check a program invariant. spec = self.command_spec[name] - cmd:OebuildCommand = oebuild_util.get_instance(spec.factory) + cmd: OebuildCommand = oebuild_util.get_instance(spec.factory) parser = self.subparsers[name] @@ -164,6 +162,7 @@ class OebuildApp: ''' self.oebuild_parser.print_help() + def check_user(): ''' check execute user must in normal user @@ -173,7 +172,8 @@ def check_user(): return False return True -def extension_commands(pre_dir, commandlist:OrderedDict): + +def extension_commands(pre_dir, commandlist: OrderedDict): ''' Get descriptions of available extension commands. The return value is an ordered map from project paths to lists of @@ -187,6 +187,7 @@ def extension_commands(pre_dir, commandlist:OrderedDict): return specs + def main(argv=None): ''' oebuild main entrypoint @@ -199,5 +200,6 @@ def main(argv=None): app = OebuildApp() app.run(argv or sys.argv[1:]) + if __name__ == "__main__": main() diff --git a/src/oebuild/app/plugins/bitbake/base_build.py b/src/oebuild/app/plugins/bitbake/base_build.py index ccfec74c3f55d5d5dd1d9abd011a91d75b5b6596..b0c830db8efc1f9d8da074eaa9f63556339c94fa 100644 --- a/src/oebuild/app/plugins/bitbake/base_build.py +++ b/src/oebuild/app/plugins/bitbake/base_build.py @@ -16,10 +16,12 @@ from oebuild.local_conf import LocalConf from oebuild.bblayers import BBLayers import oebuild.const as oebuild_const + class BaseBuild: ''' The class provides the basic methods that the build class inherits ''' + def _set_tmpfile_content(self, content: str): while True: tmp_file = oebuild_util.generate_random_str(6) @@ -30,7 +32,7 @@ class BaseBuild: break return tmp_file - def replace_local_conf(self, parse_compile, local_dir, src_dir = None): + def replace_local_conf(self, parse_compile, local_dir, src_dir=None): ''' replace some param in local.conf, the LocalConf will be instantiated and exec update diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index 4179271251529f518a47b9f8f3b2872d57d93cf5..512e485c78686956a5bbe214f3ff9de3f0f6a812 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -26,12 +26,14 @@ from oebuild.app.plugins.bitbake.in_host import InHost from oebuild.m_log import logger import oebuild.const as oebuild_const + class Bitbake(OebuildCommand): ''' Bitbake instructions can enter the build interactive environment and then directly run bitbake-related instructions,or run bitbake command directly, for example: `oebuild bitbake busybox` ''' + def __init__(self): self.compile_conf_dir = os.path.join(os.getcwd(), 'compile.yaml') self.configure = Configure() @@ -61,7 +63,7 @@ class Bitbake(OebuildCommand): return parser - def do_run(self, args: argparse.ArgumentParser, unknown = None): + def do_run(self, args: argparse.ArgumentParser, unknown=None): ''' The BitBake execution logic is: the first step is to prepare the code that initializes diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index f750487371ef12a519cdd48076bea136dfd1b79a..7bf8b314d363f21bf83aa18a818f152144eeabce 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -13,7 +13,7 @@ import re import os import sys -from docker.models.containers import Container,ExecResult +from docker.models.containers import Container, ExecResult from oebuild.parse_env import ParseEnv, EnvContainer from oebuild.docker_proxy import DockerProxy @@ -24,6 +24,7 @@ from oebuild.app.plugins.bitbake.base_build import BaseBuild import oebuild.util as oebuild_util import oebuild.const as oebuild_const + class InContainer(BaseBuild): ''' bitbake command execute in container @@ -40,7 +41,7 @@ class InContainer(BaseBuild): ''' logger.info("Bitbake starting ...") - docker_param:DockerParam = None + docker_param: DockerParam = None if parse_compile.docker_param is not None: docker_param = parse_compile.docker_param else: @@ -56,7 +57,7 @@ class InContainer(BaseBuild): `oebuild update docker`''') return - self.deal_env_container(env=parse_env,docker_param=docker_param) + self.deal_env_container(env=parse_env, docker_param=docker_param) self.exec_compile(parse_compile=parse_compile, command=command) def _trans_docker_param(self, @@ -71,7 +72,7 @@ class InContainer(BaseBuild): volumns.append("/dev/net/tun:/dev/net/tun") volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) volumns.append(os.getcwd() + ':' + - os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()))) + os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()))) if toolchain_dir is not None: volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) if sstate_cache is not None: @@ -95,11 +96,11 @@ class InContainer(BaseBuild): directly enable the sleeping container ''' if env.container is None \ - or env.container.short_id is None \ - or not self.client.is_container_exists(env.container.short_id): + or env.container.short_id is None \ + or not self.client.is_container_exists(env.container.short_id): # judge which container - container:Container = self.client.create_container( - image = docker_param.image, + container: Container = self.client.create_container( + image=docker_param.image, parameters=docker_param.parameters, volumes=docker_param.volumns, command=docker_param.command) @@ -109,7 +110,7 @@ class InContainer(BaseBuild): env.export_env() self.container_id = env.container.short_id - container:Container = self.client.get_container(self.container_id) # type: ignore + container: Container = self.client.get_container(self.container_id) # type: ignore if not self.client.is_container_running(container): self.client.start_container(container) @@ -117,7 +118,7 @@ class InContainer(BaseBuild): ''' execute compile task ''' - container:Container = self.client.get_container(self.container_id) # type: ignore + container: Container = self.client.get_container(self.container_id) # type: ignore self.init_bash(container=container, build_dir_name=os.path.basename(os.getcwd())) @@ -146,7 +147,7 @@ class InContainer(BaseBuild): content = self._get_bashrc_content(container=container) new_content = self._add_bashrc(content=content, line=command) self.update_bashrc(container=container, content=new_content) - res:ExecResult = self.client.container_exec_command( + res: ExecResult = self.client.container_exec_command( container=container, command="bash .bashrc", user=oebuild_const.CONTAINER_USER, diff --git a/src/oebuild/app/plugins/bitbake/in_host.py b/src/oebuild/app/plugins/bitbake/in_host.py index 7c258cbbf6293d9d4ec7ff8ea5948d7344e4e1e5..68a696db4ab2f588f6702e7995b4e71c51209792 100644 --- a/src/oebuild/app/plugins/bitbake/in_host.py +++ b/src/oebuild/app/plugins/bitbake/in_host.py @@ -23,6 +23,7 @@ from oebuild.app.plugins.bitbake.base_build import BaseBuild import oebuild.util as oebuild_util import oebuild.const as oebuild_const + class InHost(BaseBuild): ''' bitbake command execute in host @@ -67,13 +68,13 @@ initialization operations''') return if command is not None and command != "": - self._append_build_sh(str_list=[command], build_dir= os.getcwd()) + self._append_build_sh(str_list=[command], build_dir=os.getcwd()) with subprocess.Popen('bash build.sh', - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - text=True) as s_p: + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + text=True) as s_p: if s_p.returncode is not None and s_p.returncode != 0: err_msg = '' if s_p.stderr is not None: @@ -96,9 +97,9 @@ initialization operations''') for b_s in oebuild_const.BASH_BANNER.split('\n'): b_s = f"echo {b_s}{oebuild_const.BASH_END_FLAG}" banner_list.append(b_s) - self._append_build_sh(str_list=banner_list, build_dir= os.getcwd()) + self._append_build_sh(str_list=banner_list, build_dir=os.getcwd()) append_str = f"sed -i '/{oebuild_const.BASH_END_FLAG}/d' $HOME/.bashrc" - self._append_build_sh(str_list = [append_str], build_dir= os.getcwd()) + self._append_build_sh(str_list=[append_str], build_dir=os.getcwd()) build_sh_dir = os.path.join(os.getcwd(), 'build.sh') source_build_str = f"source {build_sh_dir}" @@ -120,7 +121,7 @@ initialization operations''') ps1_command = 'PS1="\\u\\h:\\W> "' self._append_build_sh( - str_list= [init_sdk_command, set_template, init_oe_command, ps1_command], + str_list=[init_sdk_command, set_template, init_oe_command, ps1_command], build_dir=build_dir ) @@ -130,7 +131,7 @@ initialization operations''') os.remove(build_sh_dir) os.mknod(build_sh_dir) - def _append_build_sh(self, str_list:list, build_dir): + def _append_build_sh(self, str_list: list, build_dir): build_sh_dir = os.path.join(build_dir, 'build.sh') if not os.path.exists(build_sh_dir): raise ValueError("build.sh not exists") diff --git a/src/oebuild/app/plugins/clear/clear.py b/src/oebuild/app/plugins/clear/clear.py index c6663f8eae665d1df5ba0cbd114c80c6c5efe1ce..62a5a115a4fdb78c169627d2b2476000d89618e7 100644 --- a/src/oebuild/app/plugins/clear/clear.py +++ b/src/oebuild/app/plugins/clear/clear.py @@ -23,6 +23,7 @@ from oebuild.configure import Configure from oebuild.docker_proxy import DockerProxy from oebuild.m_log import logger + class Clear(OebuildCommand): ''' for some clear task @@ -39,7 +40,7 @@ class Clear(OebuildCommand): will be generated, such as containers,so this command can remove unimportant products, such as containers ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -55,7 +56,7 @@ class Clear(OebuildCommand): return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): # perpare parse help command if self.pre_parse_help(args, unknown): return @@ -80,8 +81,8 @@ class Clear(OebuildCommand): build_list = os.listdir(self.configure.build_dir()) for build_dir in build_list: build_dir = os.path.join(self.configure.build_dir(), build_dir) - if os.path.exists(os.path.join(build_dir,".env")): - env_list.append(os.path.join(build_dir,".env")) + if os.path.exists(os.path.join(build_dir, ".env")): + env_list.append(os.path.join(build_dir, ".env")) # traversal every env file and get container_id, and then try to stop it and rm it for env in env_list: @@ -91,7 +92,7 @@ class Clear(OebuildCommand): container = self.client.get_container(container_id=container_id) DockerProxy().stop_container(container=container) DockerProxy().delete_container(container=container) - logger.info("Delete container: %s successful",container.short_id) + logger.info("Delete container: %s successful", container.short_id) except DockerException: continue except KeyError: diff --git a/src/oebuild/app/plugins/demo/demo.py b/src/oebuild/app/plugins/demo/demo.py index df3b6f3415c82e52c299a5ef8ac1267ca8adbd47..5548ec727e01e811c3a0ef5dcdd3bc87c3618c85 100644 --- a/src/oebuild/app/plugins/demo/demo.py +++ b/src/oebuild/app/plugins/demo/demo.py @@ -8,6 +8,7 @@ from oebuild.configure import Configure logger = logging.getLogger() + class Demo(OebuildCommand): def __init__(self): @@ -18,7 +19,7 @@ class Demo(OebuildCommand): description=textwrap.dedent('''\ this is your description message ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -31,6 +32,6 @@ class Demo(OebuildCommand): return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): args = args.parse_args(unknown) - pass \ No newline at end of file + pass diff --git a/src/oebuild/app/plugins/deploy/com_target.py b/src/oebuild/app/plugins/deploy/com_target.py index 9babf70b322a66b09bdb9e9a8aafc71d3fc6ca64..0066cc0f45f4285c8bf9db28ec9d384751c2754d 100644 --- a/src/oebuild/app/plugins/deploy/com_target.py +++ b/src/oebuild/app/plugins/deploy/com_target.py @@ -30,14 +30,16 @@ logger = logging.getLogger() TARGET_DIR_NAME = "target_dev" TARGET_SCRIPT_NAME = "oebuild_dev" + class ComTarget: ''' The class is used to deploy-target and undeploy-target, this is a main body, the deploy-target and undeploy-target is just a interface and finally go into ComTarget ''' + def __init__(self) -> None: self.configure = Configure() - self.client:DockerProxy = None + self.client: DockerProxy = None self.container_id = None self.work_dir = os.getcwd() self.old_bashrc = None @@ -84,7 +86,7 @@ the container {self.container_id} failed to be destroyed, please run logger.info("Initializing environment, please wait ...") self.deal_env_container(oebuild_const.DEFAULT_DOCKER) - container:Container = self.client.get_container(self.container_id) + container: Container = self.client.get_container(self.container_id) self._make_and_copy_lib(container=container) self.bak_bash(container=container) self.init_bash(container=container) @@ -143,7 +145,7 @@ the container {self.container_id} failed to be destroyed, please run ''' return os.path.exists(self.configure.source_poky_dir()) - def _make_and_copy_lib(self, container:Container): + def _make_and_copy_lib(self, container: Container): # everytime, we should make sure that script is updated, so we make a rm action before copy container.exec_run(f"rm -rf /home/openeuler/{TARGET_DIR_NAME}") # copy package lib to docker @@ -155,7 +157,7 @@ the container {self.container_id} failed to be destroyed, please run to_path="/home/openeuler/") container.exec_run(f"chmod 755 /home/openeuler/{TARGET_DIR_NAME}/{TARGET_SCRIPT_NAME}") - def deal_env_container(self,docker_image:str): + def deal_env_container(self, docker_image: str): ''' This operation realizes the processing of the container, controls how the container is processed by parsing the env @@ -169,18 +171,18 @@ the container {self.container_id} failed to be destroyed, please run volumns.append("/dev/net/tun:/dev/net/tun") volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) volumns.append(os.path.join(self.configure.build_dir(), cwd_name) - + ':' + - os.path.join(oebuild_const.CONTAINER_BUILD, cwd_name)) + + ':' + + os.path.join(oebuild_const.CONTAINER_BUILD, cwd_name)) parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS - container:Container = self.client.create_container( - image=docker_image, - parameters=parameters, - volumes=volumns, - command="bash") + container: Container = self.client.create_container( + image=docker_image, + parameters=parameters, + volumes=volumns, + command="bash") self.container_id = container.short_id - container:Container = self.client.get_container(self.container_id) # type: ignore + container: Container = self.client.get_container(self.container_id) # type: ignore if not self.client.is_container_running(container): self.client.start_container(container) @@ -286,7 +288,7 @@ the container {self.container_id} failed to be destroyed, please run if res.exit_code != 0: raise ValueError("check docker user id faild") - res_cont:str = res.output.decode() + res_cont: str = res.output.decode() cuids = res_cont.split(' ') # get uid from container in default user diff --git a/src/oebuild/app/plugins/deploy/deploy_target.py b/src/oebuild/app/plugins/deploy/deploy_target.py index 83117fef71235dd37e79c636f6f3c90e421c471f..91d0f4d188015b034886d2c68cd9352b5f2f071f 100644 --- a/src/oebuild/app/plugins/deploy/deploy_target.py +++ b/src/oebuild/app/plugins/deploy/deploy_target.py @@ -20,10 +20,12 @@ from oebuild.app.plugins.deploy.com_target import ComTarget logger = logging.getLogger() + class DeployTarget(OebuildCommand): ''' we use package in a ''' + def __init__(self) -> None: super().__init__( '{}', @@ -33,7 +35,7 @@ Deploys a recipe's build output (i.e. the output of the do_install task) to a li overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand. ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -44,7 +46,7 @@ oebuild deploy-target [-h] [-c] [-s] [-n] [-p] [--no-check-space] [-e SSH_EXEC] return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): if '-h' in unknown or '--help' in unknown: self.print_help_msg() return @@ -80,10 +82,12 @@ options: --no-strip Do not strip executables prior to deploy """) + class UnDeployTarget(OebuildCommand): ''' we use package in a ''' + def __init__(self) -> None: super().__init__( '{}', @@ -91,7 +95,7 @@ class UnDeployTarget(OebuildCommand): textwrap.dedent('''\ Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target. ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -102,7 +106,7 @@ oebuild undeploy-target [-h] [-c] [-s] [-a] [-n] [-e SSH_EXEC] [-P PORT] [-I KEY return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): if '-h' in unknown or '--help' in unknown: self.print_help_msg() return diff --git a/src/oebuild/app/plugins/deploy/target_dev/deploy.py b/src/oebuild/app/plugins/deploy/target_dev/deploy.py index f799cb177f59467d7aaff1300daf9bdeb8d2d18b..233e1794f17ff6395955cd2a40562ce5dd4fd355 100644 --- a/src/oebuild/app/plugins/deploy/target_dev/deploy.py +++ b/src/oebuild/app/plugins/deploy/target_dev/deploy.py @@ -22,6 +22,7 @@ logger = logging.getLogger('devtool') deploylist_path = '/.devtool' + def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False): """ Prepare a shell script for running on the target to @@ -79,11 +80,13 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals lines.append('do') lines.append(' checkpath=`dirname "$checkpath"`') lines.append('done') - lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') + lines.append( + r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")') # First line of the file is the total space lines.append('total=`head -n1 $3`') lines.append('if [ $total -gt $freespace ] ; then') - lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') + lines.append( + ' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"') lines.append(' exit 1') lines.append('fi') if not nopreserve: @@ -134,7 +137,6 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals return '\n'.join(lines) - def deploy(args, config, basepath, workspace): """Entry point for the devtool 'deploy' subcommand""" import math @@ -158,12 +160,12 @@ def deploy(args, config, basepath, workspace): rd = tinfoil.parse_recipe(args.recipename) except Exception as e: raise DevtoolError('Exception parsing recipe %s: %s' % - (args.recipename, e)) + (args.recipename, e)) recipe_outdir = rd.getVar('D') if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): raise DevtoolError('No files to deploy - have you built the %s ' - 'recipe? If so, the install step has not installed ' - 'any files.' % args.recipename) + 'recipe? If so, the install step has not installed ' + 'any files.' % args.recipename) if args.strip and not args.dry_run: # Fakeroot copy to new destination @@ -171,10 +173,11 @@ def deploy(args, config, basepath, workspace): recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped') if os.path.isdir(recipe_outdir): exec_fakeroot(rd, "rm -rf %s" % recipe_outdir, shell=True) - exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) + exec_fakeroot(rd, "cp -af %s %s" % + (os.path.join(srcdir, '.'), recipe_outdir), shell=True) os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), - rd.getVar('base_libdir'), rd) + rd.getVar('base_libdir'), rd) filelist = [] inodes = set({}) @@ -230,9 +233,9 @@ def deploy(args, config, basepath, workspace): tmpscript = '/tmp/devtool_deploy.sh' tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') shellscript = _prepare_remote_script(deploy=True, - verbose=args.show_status, - nopreserve=args.no_preserve, - nocheckspace=args.no_check_space) + verbose=args.show_status, + nopreserve=args.no_preserve, + nocheckspace=args.no_check_space) # Write out the script to a file with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: f.write(shellscript) @@ -242,18 +245,20 @@ def deploy(args, config, basepath, workspace): for fpath, fsize in filelist: f.write('%s %d\n' % (fpath, fsize)) # Copy them to the target - ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, + extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) if ret != 0: raise DevtoolError('Failed to copy script to %s - rerun with -s to ' - 'get a complete error message' % args.target) + 'get a complete error message' % args.target) finally: shutil.rmtree(tmpdir) # Now run the script - ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) + ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, + args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) if ret != 0: raise DevtoolError('Deploy failed - rerun with -s to get a complete ' - 'error message') + 'error message') logger.info('Successfully deployed %s' % recipe_outdir) @@ -267,12 +272,15 @@ def deploy(args, config, basepath, workspace): return 0 + def undeploy(args, config, basepath, workspace): """Entry point for the devtool 'undeploy' subcommand""" if args.all and args.recipename: - raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target') + raise argparse_oe.ArgumentUsageError( + 'Cannot specify -a/--all with a recipe name', 'undeploy-target') elif not args.recipename and not args.all: - raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') + raise argparse_oe.ArgumentUsageError( + 'If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target') extraoptions = '' if args.no_host_check: @@ -296,20 +304,23 @@ def undeploy(args, config, basepath, workspace): tmpdir = tempfile.mkdtemp(prefix='devtool') try: tmpscript = '/tmp/devtool_undeploy.sh' - shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all) + shellscript = _prepare_remote_script( + deploy=False, dryrun=args.dry_run, undeployall=args.all) # Write out the script to a file with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: f.write(shellscript) # Copy it to the target - ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) + ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, + extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) if ret != 0: raise DevtoolError('Failed to copy script to %s - rerun with -s to ' - 'get a complete error message' % args.target) + 'get a complete error message' % args.target) finally: shutil.rmtree(tmpdir) # Now run the script - ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True) + ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, + extraoptions, args.target, tmpscript, args.recipename), shell=True) if ret != 0: raise DevtoolError('Undeploy failed - rerun with -s to get a complete ' 'error message') @@ -327,24 +338,33 @@ def register_commands(subparsers, context): description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.', group='testbuild') parser_deploy.add_argument('recipename', help='Recipe to deploy') - parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') - parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') - parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') - parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') - parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true') - parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true') + parser_deploy.add_argument( + 'target', help='Live target machine running an ssh server: user@hostname[:destdir]') + parser_deploy.add_argument('-c', '--no-host-check', + help='Disable ssh host key checking', action='store_true') + parser_deploy.add_argument('-s', '--show-status', + help='Show progress/status output', action='store_true') + parser_deploy.add_argument( + '-n', '--dry-run', help='List files to be deployed only', action='store_true') + parser_deploy.add_argument('-p', '--no-preserve', + help='Do not preserve existing files', action='store_true') + parser_deploy.add_argument( + '--no-check-space', help='Do not check for available space before deploying', action='store_true') parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') - parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') + parser_deploy.add_argument( + '-P', '--port', help='Specify port to use for connection to the target') parser_deploy.add_argument('-I', '--key', help='Specify ssh private key for connection to the target') strip_opts = parser_deploy.add_mutually_exclusive_group(required=False) strip_opts.add_argument('-S', '--strip', - help='Strip executables prior to deploying (default: %(default)s). ' - 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.', - default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')), - action='store_true') - strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') + help='Strip executables prior to deploying (default: %(default)s). ' + 'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.', + default=oe.types.boolean(context.config.get( + 'Deploy', 'strip', default='0')), + action='store_true') + strip_opts.add_argument( + '--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false') parser_deploy.set_defaults(func=deploy) @@ -352,15 +372,22 @@ def register_commands(subparsers, context): help='Undeploy recipe output files in live target machine', description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.', group='testbuild') - parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') - parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname') - parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') - parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') - parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') - parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true') + parser_undeploy.add_argument( + 'recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?') + parser_undeploy.add_argument( + 'target', help='Live target machine running an ssh server: user@hostname') + parser_undeploy.add_argument('-c', '--no-host-check', + help='Disable ssh host key checking', action='store_true') + parser_undeploy.add_argument( + '-s', '--show-status', help='Show progress/status output', action='store_true') + parser_undeploy.add_argument( + '-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true') + parser_undeploy.add_argument( + '-n', '--dry-run', help='List files to be undeployed only', action='store_true') parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh') - parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target') + parser_undeploy.add_argument( + '-P', '--port', help='Specify port to use for connection to the target') parser_undeploy.add_argument('-I', '--key', - help='Specify ssh private key for connection to the target') + help='Specify ssh private key for connection to the target') parser_undeploy.set_defaults(func=undeploy) diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index fa75fa2606a081c8f7b6996da7d3dc243aa047dd..42714211fb074e1a9d6d34c4667f56bb19cd175f 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -21,17 +21,19 @@ from prettytable import PrettyTable from oebuild.command import OebuildCommand import oebuild.util as oebuild_util -from oebuild.parse_compile import ParseCompile,CheckCompileError +from oebuild.parse_compile import ParseCompile, CheckCompileError from oebuild.configure import Configure, YoctoEnv from oebuild.parse_template import BaseParseTemplate, ParseTemplate from oebuild.m_log import logger, INFO_COLOR from oebuild.check_docker_tag import CheckDockerTag import oebuild.const as oebuild_const + class Generate(OebuildCommand): ''' compile.yaml is generated according to different command parameters by generate ''' + def __init__(self): self.configure = Configure() self.nativesdk_dir = None @@ -49,7 +51,7 @@ class Generate(OebuildCommand): way to specify compile.yaml, users can directly specify the file after customizing the build configuration file ''' - )) + )) def do_add_parser(self, parser_adder): parser = self._parser( @@ -59,82 +61,82 @@ class Generate(OebuildCommand): %(prog)s [-p platform] [-f features] [-t toolchain_dir] [-d build_directory] [-l list] [-b_in build_in] ''') - parser.add_argument('-l', '--list', dest='list',action = "store_true", - help=''' + parser.add_argument('-l', '--list', dest='list', action="store_true", + help=''' will list support archs and features ''' - ) + ) parser.add_argument('-p', '--platform', dest='platform', default="qemu-aarch64", - help=''' + help=''' this param is for arch, you can find it in yocto-meta-openeuler/.oebuild/platform ''' - ) + ) - parser.add_argument('-s','--state_cache', dest='sstate_cache', - help=''' + parser.add_argument('-s', '--state_cache', dest='sstate_cache', + help=''' this param is for SSTATE_MIRRORS ''' - ) + ) parser.add_argument('-s_dir', '--sstate_dir', dest='sstate_dir', - help=''' + help=''' this param is for SSTATE_DIR ''' - ) + ) parser.add_argument('-m', '--tmp_dir', dest='tmp_dir', - help=''' + help=''' this param is for tmp directory, the build result will be stored in ''' - ) + ) parser.add_argument('-f', '--features', dest='features', action='append', - help=''' + help=''' this param is feature, it's a reuse command ''' - ) + ) parser.add_argument('-c', '--compile_dir', dest='compile_dir', - help=''' + help=''' this param is for compile.yaml directory ''' - ) + ) parser.add_argument('-d', '--directory', dest='directory', - help=''' + help=''' this param is build directory, the default is same to platform ''' - ) + ) - parser.add_argument('-t', '--toolchain_dir', dest='toolchain_dir', default = '', - help=''' + parser.add_argument('-t', '--toolchain_dir', dest='toolchain_dir', default='', + help=''' this param is for external toolchain dir, if you want use your own toolchain ''' - ) + ) - parser.add_argument('-n', '--nativesdk_dir', dest='nativesdk_dir', default = '', - help=''' + parser.add_argument('-n', '--nativesdk_dir', dest='nativesdk_dir', default='', + help=''' this param is for external nativesdk dir, the param will be useful when you want to build in host ''' - ) + ) - parser.add_argument('-tag', '--docker_tag', dest='docker_tag', default = '', - help=''' + parser.add_argument('-tag', '--docker_tag', dest='docker_tag', default='', + help=''' when build in docker, the param can be point docker image ''' - ) + ) - parser.add_argument('-dt', '--datetime', dest = "datetime", - help=''' + parser.add_argument('-dt', '--datetime', dest="datetime", + help=''' this param is add DATETIME to local.conf, the value format is 20231212010101 ''') parser.add_argument('-df', '--disable_fetch', - dest = "is_disable_fetch", - action = "store_true", - help=''' + dest="is_disable_fetch", + action="store_true", + help=''' this param is set openeuler_fetch in local.conf, the default value is enable, if set -df, the OPENEULER_FETCH will set to 'disable' ''') @@ -142,13 +144,13 @@ class Generate(OebuildCommand): '--build_in', dest='build_in', choices=[oebuild_const.BUILD_IN_DOCKER, oebuild_const.BUILD_IN_HOST], - default = oebuild_const.BUILD_IN_DOCKER, help=''' + default=oebuild_const.BUILD_IN_DOCKER, help=''' This parameter marks the mode at build time, and is built in the container by docker ''') return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): # perpare parse help command if self.pre_parse_help(args, unknown): return @@ -217,8 +219,8 @@ class Generate(OebuildCommand): try: self._add_platform_template(args=args, - yocto_oebuild_dir=yocto_oebuild_dir, - parser_template=parser_template) + yocto_oebuild_dir=yocto_oebuild_dir, + parser_template=parser_template) except BaseParseTemplate as b_t: logger.error(str(b_t)) return @@ -228,8 +230,8 @@ class Generate(OebuildCommand): try: self._add_features_template(args=args, - yocto_oebuild_dir=yocto_oebuild_dir, - parser_template=parser_template) + yocto_oebuild_dir=yocto_oebuild_dir, + parser_template=parser_template) except BaseParseTemplate as b_t: logger.error(str(b_t)) self._list_feature() @@ -238,8 +240,8 @@ class Generate(OebuildCommand): logger.error(str(v_e)) return - if os.path.exists(os.path.join(build_dir,'compile.yaml')): - os.remove(os.path.join(build_dir,'compile.yaml')) + if os.path.exists(os.path.join(build_dir, 'compile.yaml')): + os.remove(os.path.join(build_dir, 'compile.yaml')) docker_image = YoctoEnv().get_docker_image(yocto_dir=yocto_dir) if docker_image is None: @@ -255,7 +257,7 @@ class Generate(OebuildCommand): following container, enter it numerically, and enter q to exit:''') image_list = check_docker_tag.get_tags() - for key,value in enumerate(image_list): + for key, value in enumerate(image_list): print(f"{key}, {oebuild_config.docker.repo_url}:{value}") k = input("please entry number:") if k == "q": @@ -270,20 +272,20 @@ class Generate(OebuildCommand): docker_tag = docker_tag.strip('\n') docker_image = f"{oebuild_config.docker.repo_url}:{docker_tag}" - out_dir = pathlib.Path(os.path.join(build_dir,'compile.yaml')) + out_dir = pathlib.Path(os.path.join(build_dir, 'compile.yaml')) oebuild_util.write_yaml(out_dir, parser_template.generate_compile_conf( - nativesdk_dir= self.nativesdk_dir, - toolchain_dir= self.toolchain_dir, + nativesdk_dir=self.nativesdk_dir, + toolchain_dir=self.toolchain_dir, build_in=build_in, - sstate_cache= self.sstate_cache, - tmp_dir = self.tmp_dir, - datetime = args.datetime, - is_disable_fetch = args.is_disable_fetch, + sstate_cache=self.sstate_cache, + tmp_dir=self.tmp_dir, + datetime=args.datetime, + is_disable_fetch=args.is_disable_fetch, docker_image=docker_image, src_dir=self.configure.source_dir(), compile_dir=build_dir - )) + )) self._print_generate(build_dir=build_dir) @@ -320,8 +322,8 @@ oebuild bitbake try: parser_template.add_template( os.path.join(yocto_oebuild_dir, - 'platform', - args.platform+'.yaml')) + 'platform', + args.platform+'.yaml')) except BaseParseTemplate as e_p: raise e_p else: @@ -335,8 +337,8 @@ wrong platform, please run `oebuild generate -l` to view support platform""") if feature + '.yaml' in os.listdir(os.path.join(yocto_oebuild_dir, 'features')): try: parser_template.add_template(os.path.join(yocto_oebuild_dir, - 'features', - feature+'.yaml')) + 'features', + feature+'.yaml')) except BaseParseTemplate as b_t: raise b_t else: @@ -368,7 +370,7 @@ wrong platform, please run `oebuild generate -l` to view support feature""") print(""" wrong input""") continue - if in_res in ['N','n', 'no']: + if in_res in ['N', 'n', 'no']: return None break if os.path.exists(os.path.join(build_dir, "conf")): @@ -404,13 +406,13 @@ wrong platform, please run `oebuild generate -l` to view support feature""") yocto_oebuild_dir = os.path.join(yocto_dir, ".oebuild") list_feature = os.listdir(os.path.join(yocto_oebuild_dir, 'features')) print("the feature list is:") - table = PrettyTable(['feature name','support arch']) + table = PrettyTable(['feature name', 'support arch']) table.align = "l" for feature in list_feature: if feature.endswith('.yml'): - feature_name = feature.replace('.yml','') + feature_name = feature.replace('.yml', '') if feature.endswith('.yaml'): - feature_name = feature.replace('.yaml','') + feature_name = feature.replace('.yaml', '') feat = oebuild_util.read_yaml(pathlib.Path(os.path.join(yocto_oebuild_dir, 'features', feature))) diff --git a/src/oebuild/app/plugins/init/init.py b/src/oebuild/app/plugins/init/init.py index e342058793dd1925d78cfa2fbb4af22f6e50f84a..f0eefa6f3cccd76272ea7e2f5ddab41c3024d18e 100644 --- a/src/oebuild/app/plugins/init/init.py +++ b/src/oebuild/app/plugins/init/init.py @@ -22,6 +22,7 @@ from oebuild.configure import Configure, ConfigBasicRepo, Config from oebuild.m_log import logger import oebuild.const as oebuild_const + class Init(OebuildCommand): ''' Directory initialization directive, mainly used to initialize @@ -50,7 +51,7 @@ class Init(OebuildCommand): consideration of the build configuration of OEbuild, and can be easily called by third parties ''' - )) + )) def do_add_parser(self, parser_adder): self._parser( @@ -60,11 +61,11 @@ class Init(OebuildCommand): %(prog)s [directory] [-u yocto_remote_url] [-b branch] ''') - parser_adder.add_argument('-u','--yocto_remote_url', dest = 'yocto_remote_url', - help='''Specifies the remote of yocto-meta-openeuler''') + parser_adder.add_argument('-u', '--yocto_remote_url', dest='yocto_remote_url', + help='''Specifies the remote of yocto-meta-openeuler''') - parser_adder.add_argument('-b', '--branch', dest = 'branch', - help='''Specifies the branch of yocto-meta-openeuler''') + parser_adder.add_argument('-b', '--branch', dest='branch', + help='''Specifies the branch of yocto-meta-openeuler''') parser_adder.add_argument( 'directory', nargs='?', default=None, @@ -72,7 +73,7 @@ class Init(OebuildCommand): return parser_adder - def do_run(self, args: argparse.ArgumentParser, unknown = None): + def do_run(self, args: argparse.ArgumentParser, unknown=None): ''' detach target dicrectory if finished init, if inited, just put out err msg and exit ''' @@ -100,9 +101,9 @@ class Init(OebuildCommand): return os.chdir(args.directory) - oebuild_config:Config = self.configure.parse_oebuild_config() + oebuild_config: Config = self.configure.parse_oebuild_config() - yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] + yocto_config: ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] if args.yocto_remote_url is not None: yocto_config.remote_url = args.yocto_remote_url if args.branch is not None: @@ -111,7 +112,7 @@ class Init(OebuildCommand): self.configure.update_oebuild_config(oebuild_config) - logger.info("init %s successful",args.directory) + logger.info("init %s successful", args.directory) format_msg = f''' There is a build configuration example file under {args.directory}/.oebuild/compile.yaml.sample, if you want to block complex generate instructions, you can directly copy a configuration file, @@ -139,7 +140,7 @@ please execute the follow commands next return True @staticmethod - def create_oebuild_directory(updir : str): + def create_oebuild_directory(updir: str): ''' create oebuild config directory ''' @@ -152,7 +153,7 @@ please execute the follow commands next return "" @staticmethod - def create_src_directory(updir : str): + def create_src_directory(updir: str): ''' this is desctiption ''' @@ -165,7 +166,7 @@ please execute the follow commands next return None @staticmethod - def copy_config_file(updir : str): + def copy_config_file(updir: str): ''' copy oebuild config to some directory ''' @@ -176,7 +177,7 @@ please execute the follow commands next logger.error("mkdir config faild") @staticmethod - def copy_compile_file(updir : str): + def copy_compile_file(updir: str): ''' copy oebuild compile.yaml.sample to some directory ''' diff --git a/src/oebuild/app/plugins/m_env/m_env.py b/src/oebuild/app/plugins/m_env/m_env.py index 9552ae11b2da299a2f019ca50829141a1ac6f5ee..4a242e60969ac5fc80f7c618462f4c1e186a5879 100644 --- a/src/oebuild/app/plugins/m_env/m_env.py +++ b/src/oebuild/app/plugins/m_env/m_env.py @@ -28,6 +28,7 @@ from oebuild.configure import Configure import oebuild.util as oebuild_util from oebuild.m_log import logger + class Menv(OebuildCommand): ''' the class is used to manager sdk environment, the sdk environment refers to openEuler @@ -35,6 +36,7 @@ class Menv(OebuildCommand): Embedded system you built. for example, the sdk with qt image can be used to develop apps runned in qt system, the sdk with ros image can be used to develop apps runned in ros system ''' + def __init__(self): self.configure = Configure() self.oebuild_env_path = os.path.expanduser('~') + '/.local/oebuild_env/' diff --git a/src/oebuild/app/plugins/m_plugin/m_plugin.py b/src/oebuild/app/plugins/m_plugin/m_plugin.py index af3403a8d3233cf638938135a38e3842793de0e5..5c4a24526f86e3b270f5e1baf64c4246f161b429 100644 --- a/src/oebuild/app/plugins/m_plugin/m_plugin.py +++ b/src/oebuild/app/plugins/m_plugin/m_plugin.py @@ -31,6 +31,7 @@ class MPlugin(OebuildCommand): manager it, the plugin must be in oebuild plugin standard, when you install the plugin you developped, you can use it through oebuild. """ + def __init__(self): self.configure = Configure() self.oebuild_plugin_commands = ['install', 'list', 'enable', 'disable', 'remove'] @@ -148,10 +149,10 @@ class MPlugin(OebuildCommand): append_command_ext = {} if args.plugin_name in self.command_ext.keys() \ - or args.plugin_name in append_command_ext.keys(): + or args.plugin_name in append_command_ext.keys(): while True: user_input = input( -f'Do you want to overwrite the existing plugin ({args.plugin_name}) in oebuild(Y/N)') + f'Do you want to overwrite the existing plugin ({args.plugin_name}) in oebuild(Y/N)') if user_input.lower() == 'y': break if user_input.lower() == 'n': @@ -293,7 +294,7 @@ f'Do you want to overwrite the existing plugin ({args.plugin_name}) in oebuild(Y def_name += "," if file_line.startswith('class') or file_line.startswith(' class'): if re.search(r'((?<=class)\s+\w+\(OebuildCommand\))', file_line) and \ - not class_name: + not class_name: class_name = re.search(r'(?<=class)\s+\w+', file_line).group().strip() return def_name, class_name diff --git a/src/oebuild/app/plugins/manifest/manifest.py b/src/oebuild/app/plugins/manifest/manifest.py index 33c1e4ca4a3025e98120772854abbda74b6b1f85..5facae0578b36bc543ac1dade05b5f49f10dc4a3 100644 --- a/src/oebuild/app/plugins/manifest/manifest.py +++ b/src/oebuild/app/plugins/manifest/manifest.py @@ -18,7 +18,7 @@ import pathlib import multiprocessing from queue import Queue from threading import Thread -from concurrent.futures import ThreadPoolExecutor,wait,ALL_COMPLETED +from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED import git from git.repo import Repo @@ -29,6 +29,7 @@ from oebuild.configure import Configure import oebuild.util as oebuild_util from oebuild.m_log import logger + class Manifest(OebuildCommand): ''' manifest provides the manifest function of generating dependent @@ -47,7 +48,7 @@ class Manifest(OebuildCommand): source repositories in the build working directory, and can restore relevant source repositories based on the manifest file ''' - )) + )) def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser( @@ -64,11 +65,11 @@ class Manifest(OebuildCommand): help=''' specify a manifest path to perform the create or restore operation ''' - ) + ) return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): if not self.configure.is_oebuild_dir(): logger.error('Your current directory had not finished init') sys.exit(-1) @@ -117,11 +118,11 @@ class Manifest(OebuildCommand): print(f"Expose progress: {progress}%: ", "▋" * (progress // 2), end="") sys.stdout.flush() print() - manifest_list = dict(sorted(manifest_list.items(),key=lambda s:s[0])) + manifest_list = dict(sorted(manifest_list.items(), key=lambda s: s[0])) oebuild_util.write_yaml( yaml_dir=pathlib.Path(manifest_dir), data={'manifest_list': manifest_list}) - self._add_manifest_banner(manifest_dir = os.path.abspath(manifest_dir)) + self._add_manifest_banner(manifest_dir=os.path.abspath(manifest_dir)) print(f"expose successful, the directory is {os.path.abspath(manifest_dir)}") @@ -161,15 +162,15 @@ class Manifest(OebuildCommand): dserver.start() cpu_count = multiprocessing.cpu_count() - with ThreadPoolExecutor(max_workers = cpu_count) as t_p: + with ThreadPoolExecutor(max_workers=cpu_count) as t_p: src_dir = self.configure.source_dir() all_task = [] for key, value in manifest_list.items(): all_task.append(t_p.submit(self._download_repo, q_e, src_dir, key, value)) - wait(all_task, return_when = ALL_COMPLETED) + wait(all_task, return_when=ALL_COMPLETED) q_e.put("over") - def _download_repo(self, out_q: Queue ,src_dir, key, value): + def _download_repo(self, out_q: Queue, src_dir, key, value): repo_dir = os.path.join(src_dir, key) repo = Repo.init(repo_dir) remote = None @@ -180,10 +181,10 @@ class Manifest(OebuildCommand): continue if remote is None: remote_name = "upstream" - remote = git.Remote.add(repo = repo, name = remote_name, url = value['remote_url']) + remote = git.Remote.add(repo=repo, name=remote_name, url=value['remote_url']) try: repo.git.checkout(value['version']) except GitCommandError: - remote.fetch(value['version'], depth = 1) + remote.fetch(value['version'], depth=1) repo.git.checkout(value['version']) out_q.put('ok') diff --git a/src/oebuild/app/plugins/run_qemu/run_qemu.py b/src/oebuild/app/plugins/run_qemu/run_qemu.py index a73e2b1cff4ee045141803fc65370c1931e204f7..5b07801fbde3a8034c254bf1ff379c6881e81b16 100644 --- a/src/oebuild/app/plugins/run_qemu/run_qemu.py +++ b/src/oebuild/app/plugins/run_qemu/run_qemu.py @@ -25,10 +25,12 @@ from oebuild.configure import Configure from oebuild.m_log import logger import oebuild.const as oebuild_const + class RunQemu(OebuildCommand): ''' The command for run in qemu platform. ''' + def __init__(self): self.configure = Configure() self.client = None @@ -92,7 +94,7 @@ the container {self.container_id} failed to be destroyed, please run ''' exec qemu ''' - container:Container = self.client.get_container(self.container_id) # type: ignore + container: Container = self.client.get_container(self.container_id) # type: ignore self.bak_bash(container=container) self.init_bash(container=container) content = self._get_bashrc_content(container=container) @@ -145,7 +147,7 @@ now, you can continue run `oebuild runqemu` in compile directory sys.exit(0) return - def deal_env_container(self,docker_image): + def deal_env_container(self, docker_image): ''' This operation realizes the processing of the container, controls how the container is processed by parsing the env @@ -161,14 +163,14 @@ now, you can continue run `oebuild runqemu` in compile directory volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS + " --privileged" - container:Container = self.client.create_container( + container: Container = self.client.create_container( image=docker_image, parameters=parameters, volumes=volumns, command="bash") self.container_id = container.short_id - container:Container = self.client.get_container(self.container_id) + container: Container = self.client.get_container(self.container_id) if not self.client.is_container_running(container): self.client.start_container(container) diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py index 6dcc8261cfd3e699e6ac99e85f89ada8662d28e8..96d83389dc0a5dc336fd138f168b22cf3a186812 100644 --- a/src/oebuild/app/plugins/update/update.py +++ b/src/oebuild/app/plugins/update/update.py @@ -29,11 +29,13 @@ from oebuild.check_docker_tag import CheckDockerTag import oebuild.const as oebuild_const from oebuild.m_log import logger + class Update(OebuildCommand): ''' The update command will prepare the basic environment related to the build, such as container images, build base repositories, etc ''' + def __init__(self): self.configure = Configure() @@ -66,11 +68,11 @@ class Update(OebuildCommand): usage=''' %(prog)s [yocto docker layer] [-tag] ''') - parser.add_argument('-tag','--tag', dest='docker_tag', - help=''' + parser.add_argument('-tag', '--tag', dest='docker_tag', + help=''' with platform will list support archs, with feature will list support features ''' - ) + ) parser.add_argument( 'item', nargs='?', default=None, @@ -78,7 +80,7 @@ class Update(OebuildCommand): return parser - def do_run(self, args: argparse.Namespace, unknown = None): + def do_run(self, args: argparse.Namespace, unknown=None): ''' update action rely on directory which has initd, so check it first ''' @@ -110,7 +112,6 @@ class Update(OebuildCommand): logger.error('Please run oebuild update [yocto docker layer]') sys.exit(-1) - if update_yocto: self.get_basic_repo() @@ -147,17 +148,17 @@ class Update(OebuildCommand): if repos is None: return - for _ , item in repos.items(): + for _, item in repos.items(): if isinstance(item, OebuildRepo): local_dir = os.path.join(self.configure.source_dir(), item.path) - key_repo = OGit(repo_dir = local_dir, - remote_url = item.url, - branch = item.refspec) + key_repo = OGit(repo_dir=local_dir, + remote_url=item.url, + branch=item.refspec) else: local_dir = os.path.join(self.configure.source_dir(), item['path']) - key_repo = OGit(repo_dir = local_dir, - remote_url = item['url'], - branch = item['refspec']) + key_repo = OGit(repo_dir=local_dir, + remote_url=item['url'], + branch=item['refspec']) key_repo.clone_or_pull_repo() def get_basic_repo(self,): @@ -171,7 +172,7 @@ class Update(OebuildCommand): embedded/src/yocto-meta-openeuler not exists, so just clone from config setting. ''' oebuild_config = self.configure.parse_oebuild_config() - yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] + yocto_config: ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] local_dir = os.path.join(self.configure.source_dir(), yocto_config.path) yocto_repo = OGit(repo_dir=local_dir, @@ -179,7 +180,7 @@ class Update(OebuildCommand): branch=yocto_config.branch) yocto_repo.clone_or_pull_repo() - def docker_image_update(self, docker_tag = None): + def docker_image_update(self, docker_tag=None): ''' The container update logic is to update the corresponding tag container image if tag is specified, otherwise it is determined @@ -189,7 +190,7 @@ class Update(OebuildCommand): ''' oebuild_config = self.configure.parse_oebuild_config() docker_config = oebuild_config.docker - check_docker_tag = CheckDockerTag(docker_tag=docker_tag,configure=self.configure) + check_docker_tag = CheckDockerTag(docker_tag=docker_tag, configure=self.configure) if docker_tag is not None: if check_docker_tag.get_tag() is None or check_docker_tag.get_tag() == "": check_docker_tag.list_image_tag() diff --git a/src/oebuild/bb/utils.py b/src/oebuild/bb/utils.py index 4ac107ea6780f3eebb2345aa686a9240240083a4..791f385714c25d81501de812e552b3b365515ab1 100644 --- a/src/oebuild/bb/utils.py +++ b/src/oebuild/bb/utils.py @@ -6,6 +6,7 @@ import os import re + def preserved_envvars_exported(): """Variables which are taken from the environment and placed in and exported from the metadata""" @@ -21,6 +22,7 @@ def preserved_envvars_exported(): 'BBSERVER', ] + def preserved_envvars(): """Variables which are taken from the environment and placed in the metadata""" v = [ @@ -31,6 +33,7 @@ def preserved_envvars(): ] return v + preserved_envvars_exported() + def approved_variables(): """ Determine and return the list of whitelisted variables which are approved @@ -50,6 +53,7 @@ def approved_variables(): approved.extend(['BB_ENV_EXTRAWHITE']) return approved + def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): """Edit lines from a recipe or config file and modify one or more specified variable values set in the file using a specified callback @@ -105,9 +109,11 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): override_re = '' for var in variables: if var.endswith('()'): - var_res[var] = re.compile(r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re)) + var_res[var] = re.compile( + r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re)) else: - var_res[var] = re.compile(r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re)) + var_res[var] = re.compile( + r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re)) updated = False varset_start = '' @@ -143,7 +149,8 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): if in_var.endswith('()'): # A function definition if isinstance(newvalue, list): - newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue))) + newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, + ('\n%s' % indentspc).join(newvalue))) else: if not newvalue.startswith('\n'): newvalue = '\n' + newvalue @@ -237,6 +244,7 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): newlines.append(line) return (updated, newlines) + def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): """Edit bblayers.conf, adding and/or removing layers Parameters: @@ -262,6 +270,7 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None): return pth approved = approved_variables() + def canonicalise_path(pth): pth = remove_trailing_sep(pth) if 'HOME' in approved and '~' in pth: diff --git a/src/oebuild/bblayers.py b/src/oebuild/bblayers.py index d9e5541a299f532dd011ad9c87118849c2ffe1c5..e92770c1ee2d723b38e9df58306d40bd00a5a412 100644 --- a/src/oebuild/bblayers.py +++ b/src/oebuild/bblayers.py @@ -14,12 +14,14 @@ import os import oebuild.bb.utils as bb_utils + class BBLayers: ''' The BBlayer class implements the layer added in the container environment in the physical environment, and the add operation references bitbake-related code ''' + def __init__(self, bblayers_dir: str, base_dir: str): self._base_dir = base_dir self._bblayers_dir = bblayers_dir @@ -62,7 +64,7 @@ class BBLayers: bb_utils.edit_bblayers_conf(self.bblayers_dir, add=bblayers, remove=None) - def check_layer_exist(self, layers:str or list): + def check_layer_exist(self, layers: str or list): ''' To check if it is legitimate to add a layer, the main thing is to verify the existence of layer.conf diff --git a/src/oebuild/check_docker_tag.py b/src/oebuild/check_docker_tag.py index 5ba23d44e5fcd8cef36b15cdb19f841bcbae2742..dc6bda2976bc1e4678cc826f93ec9c92c1a4aca6 100644 --- a/src/oebuild/check_docker_tag.py +++ b/src/oebuild/check_docker_tag.py @@ -18,6 +18,7 @@ from git import Repo from oebuild.configure import Configure import oebuild.util as oebuild_util + class CheckDockerTag: ''' This class is used to synthesize the build environment @@ -27,6 +28,7 @@ class CheckDockerTag: configuration file, the yocto branch name, etc. Judge down in turn, and finally return a suitable docker image version ''' + def __init__(self, docker_tag: str, configure: Configure): self.docker_tag = docker_tag self.configure = configure @@ -68,7 +70,7 @@ the openeuler embedded docker tag can be selected list: return str(self.docker_tag) yocto_dir = self.configure.source_yocto_dir() - env_path = os.path.join(yocto_dir,".oebuild/env.yaml") + env_path = os.path.join(yocto_dir, ".oebuild/env.yaml") if os.path.exists(env_path): env_parse = oebuild_util.read_yaml(pathlib.Path(env_path)) return str(env_parse['docker_tag']) diff --git a/src/oebuild/command.py b/src/oebuild/command.py index 07d22bf86c44def214ecc9f0b1f2f56163794fe2..04cf37de1f37189a2d33752c48905a6eee1dbe3f 100644 --- a/src/oebuild/command.py +++ b/src/oebuild/command.py @@ -21,6 +21,7 @@ WRN_COLOR = colorama.Fore.LIGHTYELLOW_EX ERR_COLOR = colorama.Fore.LIGHTRED_EX + class OebuildCommand(ABC): '''Abstract superclass for a oebuild command.''' @@ -28,7 +29,7 @@ class OebuildCommand(ABC): self.name = name self.help_msg = help_msg self.description = description - self.parser:argparse.ArgumentParser = None + self.parser: argparse.ArgumentParser = None def run(self, args: argparse.ArgumentParser, unknown: List[str]): ''' @@ -58,13 +59,13 @@ class OebuildCommand(ABC): :param parser_adder: The return value of a call to ``argparse.ArgumentParser.add_subparsers()`` ''' - self.parser:argparse.ArgumentParser = self.do_add_parser(parser_adder) + self.parser: argparse.ArgumentParser = self.do_add_parser(parser_adder) if self.parser is None: raise ValueError('do_add_parser did not return a value') self.parser.add_argument('-h', '--help', dest="help", action="store_true", - help='get help for oebuild or a command') + help='get help for oebuild or a command') return self.parser diff --git a/src/oebuild/configure.py b/src/oebuild/configure.py index 9a713222cc8f069484ab9f7cad41ab8df5c26d88..25adeb367d782a2b17d66854a73d270a82ad0255 100644 --- a/src/oebuild/configure.py +++ b/src/oebuild/configure.py @@ -20,9 +20,11 @@ import oebuild.const as oebuild_const PathType = Union[str, os.PathLike] + class OebuildNotFound(RuntimeError): '''Neither the current directory nor any parent has a oebuild workspace.''' + @dataclass class ConfigContainer: ''' @@ -34,6 +36,7 @@ class ConfigContainer: # tag_mag is for branch to container tag map tag_map: Dict + @dataclass class ConfigBasicRepo: ''' @@ -48,6 +51,7 @@ class ConfigBasicRepo: # branch is for repo's branch branch: str + @dataclass class Config: ''' @@ -57,6 +61,7 @@ class Config: basic_repo: dict + class Configure: ''' Configure object is to contain some generally param or function about oebuild @@ -64,7 +69,7 @@ class Configure: @staticmethod def oebuild_topdir(start: Optional[PathType] = None, - fall_back: bool = True): + fall_back: bool = True): ''' Like oebuild_dir(), but returns the path to the parent directory of the .oebuild/ directory instead, where project repositories are stored @@ -82,7 +87,7 @@ class Configure: return Configure.oebuild_topdir(fall_back=False) raise OebuildNotFound('Could not find a oebuild workspace ' - 'in this or any parent directory') + 'in this or any parent directory') cur_dir = parent_dir @staticmethod @@ -123,7 +128,7 @@ class Configure: ''' config = Configure.parse_oebuild_config() basic_config = config.basic_repo - yocto_config:ConfigBasicRepo = basic_config[oebuild_const.YOCTO_META_OPENEULER] + yocto_config: ConfigBasicRepo = basic_config[oebuild_const.YOCTO_META_OPENEULER] yocto_dir = yocto_config.path return os.path.join(Configure.source_dir(), yocto_dir) @@ -164,7 +169,7 @@ class Configure: ''' config = oebuild_util.read_yaml( - yaml_dir = pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG)) + yaml_dir=pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG)) tag_map = {} for key, value in config['docker']['tag_map'].items(): @@ -199,19 +204,20 @@ class Configure: basic_config = config.basic_repo data['basic_repo'] = {} for key, repo in basic_config.items(): - repo:ConfigBasicRepo = repo + repo: ConfigBasicRepo = repo data['basic_repo'][key] = {'path': repo.path, 'remote_url': repo.remote_url, 'branch': repo.branch} try: oebuild_util.write_yaml( - yaml_dir = pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG), + yaml_dir=pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG), data=data) return True except TypeError: return False + class YoctoEnv: ''' the YoctoEnv class is used for yocto-meta-openeuler/.oebuild/env.yaml @@ -224,7 +230,7 @@ class YoctoEnv: if not os.path.exists(yocto_dir): raise ValueError("the yocto direction is not exists") - env_path = os.path.join(yocto_dir,".oebuild/env.yaml") + env_path = os.path.join(yocto_dir, ".oebuild/env.yaml") if not os.path.exists(env_path): return None diff --git a/src/oebuild/docker_proxy.py b/src/oebuild/docker_proxy.py index f3b00ffaa44cc6f760a95114ecea401bfaadb751..d0e3a0300c8fd33743f7bc2d8ca0d511e6e3b454 100644 --- a/src/oebuild/docker_proxy.py +++ b/src/oebuild/docker_proxy.py @@ -22,10 +22,12 @@ from docker.models.containers import Container from oebuild.m_log import logger + class DockerProxy: ''' a object just be wrapper again to run docker command easily ''' + def __init__(self): self._docker = docker.from_env() @@ -53,13 +55,13 @@ class DockerProxy: except NotFound: return False - def pull_image(self, image_name : str): + def pull_image(self, image_name: str): ''' pull image like command 'docker pull' args: image_name (str): docker image name, if no tag, the default tag is latest ''' - repository,tag = self._get_image_name_tag(image_name=image_name) + repository, tag = self._get_image_name_tag(image_name=image_name) self._docker.images.pull(repository=repository, tag=tag) @@ -72,7 +74,7 @@ class DockerProxy: def _get_image_name_tag(self, image_name: str): repository = image_name.split(':')[0] tag = "latest" - if len(image_name.split(':'))==2: + if len(image_name.split(':')) == 2: tag = image_name.split(':')[1] return repository, tag @@ -84,7 +86,6 @@ class DockerProxy: ''' return self._docker.images.get(image_name) - def get_container(self, container_id): ''' get a docker container object @@ -93,7 +94,6 @@ class DockerProxy: ''' return self._docker.containers.get(container_id=container_id) - def get_all_container(self): ''' get all container like command 'docker ps -a' @@ -112,13 +112,13 @@ class DockerProxy: container.stop() @staticmethod - def delete_container(container: Container, is_force:bool = False): + def delete_container(container: Container, is_force: bool = False): ''' rm a container which not running like command 'docker rm' args: container (Container): container object ''' - container.remove(force = is_force) + container.remove(force=is_force) @staticmethod def start_container(container: Container): @@ -130,7 +130,7 @@ class DockerProxy: container.start() @staticmethod - def is_container_running(container : Container): + def is_container_running(container: Container): ''' determize if a container in running state args: @@ -147,10 +147,10 @@ class DockerProxy: args: path_dir (str): the directory that will added to a tar ''' - if os.path.exists(path = path_dir): + if os.path.exists(path=path_dir): pw_tarstream = BytesIO() with tarfile.TarFile(fileobj=pw_tarstream, mode='w') as pw_tar: - pw_tar.add(name = path_dir, arcname=os.path.basename(path_dir)) + pw_tar.add(name=path_dir, arcname=os.path.basename(path_dir)) pw_tarstream.seek(0) return pw_tarstream return None @@ -164,7 +164,7 @@ class DockerProxy: to_path (str): will copy to docker container path ''' tar = self.add_tar(source_path) - return container.put_archive(path = to_path,data = tar) + return container.put_archive(path=to_path, data=tar) def copy_from_container(self, container: Container, from_path, dst_path): ''' @@ -179,17 +179,17 @@ class DockerProxy: for trunk in bits: pw_tarstream.write(trunk) pw_tarstream.seek(0) - with tarfile.open(fileobj = pw_tarstream) as tar: - res = tar.extractall(path = dst_path) + with tarfile.open(fileobj=pw_tarstream) as tar: + res = tar.extractall(path=dst_path) return res is None def container_exec_command(self, container: Container, command: str or list, user: str = '', - work_space = None, - stream = True, - demux = False): + work_space=None, + stream=True, + demux=False): ''' run command like 'docker run exec', other param will be default and just use a little params @@ -208,10 +208,10 @@ class DockerProxy: return res def create_container(self, - image:str, - parameters:str, - volumes:list[str], - command:str) -> Container: + image: str, + parameters: str, + volumes: list[str], + command: str) -> Container: ''' create a new container ''' diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index 1f93ed6f1d65459766176287c62fc2a486992c80..e8be9d2ef3ce58c11e49147c4758526e9840916b 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -19,20 +19,26 @@ import oebuild.util as oebuild_util from oebuild.m_log import logger import oebuild.const as oebuild_const + class BaseLocalConf(ValueError): ''' basic error about parse_template ''' + + class NativesdkNotExist(BaseLocalConf): ''' nativesdk directory not exist ''' + + class NativesdkNotValid(BaseLocalConf): ''' nativesdk directory not valid ''' -def get_nativesdk_sysroot(nativesdk_dir = oebuild_const.NATIVESDK_DIR): + +def get_nativesdk_sysroot(nativesdk_dir=oebuild_const.NATIVESDK_DIR): ''' return environment initialization shell, if nativesdk directory is not exists or can not find any initialization shell, raise error @@ -70,6 +76,7 @@ def match_and_add(new_str: str, content: str): content = content + '\n' return content + def match_and_replace(pre: str, new_str: str, content: str): ''' math line in content when the new_str exist and replace @@ -85,6 +92,7 @@ def match_and_replace(pre: str, new_str: str, content: str): content = content + '\n' return content + class LocalConf: ''' LocalConf corresponds to the local.conf configuration @@ -94,7 +102,7 @@ class LocalConf: def __init__(self, local_conf_dir: str): self.local_dir = local_conf_dir - def update(self, parse_compile: ParseCompile, src_dir = None): + def update(self, parse_compile: ParseCompile, src_dir=None): ''' update local.conf by ParseCompile ''' @@ -143,40 +151,40 @@ class LocalConf: content = match_and_replace( pre=oebuild_const.OPENEULER_SP_DIR, - new_str= f"{oebuild_const.OPENEULER_SP_DIR} = '{src_dir}'", + new_str=f"{oebuild_const.OPENEULER_SP_DIR} = '{src_dir}'", content=content ) # replace sstate_cache if parse_compile.sstate_cache is not None: if os.path.islink(parse_compile.sstate_cache): - new_str= f"file://.* {parse_compile.sstate_cache}/PATH;downloadfilename=PATH" + new_str = f"file://.* {parse_compile.sstate_cache}/PATH;downloadfilename=PATH" else: if parse_compile.build_in == oebuild_const.BUILD_IN_DOCKER: - new_str= f"file://.* file://{oebuild_const.SSTATE_CACHE}/PATH" + new_str = f"file://.* file://{oebuild_const.SSTATE_CACHE}/PATH" else: - new_str= f"file://.* file://{parse_compile.sstate_cache}/PATH" + new_str = f"file://.* file://{parse_compile.sstate_cache}/PATH" content = match_and_replace( - pre=oebuild_const.SSTATE_MIRRORS, - new_str = f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"', - content=content - ) + pre=oebuild_const.SSTATE_MIRRORS, + new_str=f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"', + content=content + ) # replace sstate_dir if parse_compile.sstate_dir is not None: content = match_and_replace( - pre=oebuild_const.SSTATE_DIR, - new_str = f'{oebuild_const.SSTATE_DIR} = "{parse_compile.sstate_dir}"', - content=content - ) + pre=oebuild_const.SSTATE_DIR, + new_str=f'{oebuild_const.SSTATE_DIR} = "{parse_compile.sstate_dir}"', + content=content + ) # replace tmpdir if parse_compile.tmp_dir is not None: content = match_and_replace( - pre=oebuild_const.TMP_DIR, - new_str = f'{oebuild_const.TMP_DIR} = "{parse_compile.tmp_dir}"', - content=content - ) + pre=oebuild_const.TMP_DIR, + new_str=f'{oebuild_const.TMP_DIR} = "{parse_compile.tmp_dir}"', + content=content + ) content = self.match_lib_param(content=content) @@ -201,12 +209,12 @@ class LocalConf: for key, value in lib_param_list.items(): content = match_and_replace( - pre = key, - new_str = f'{key} = "{value}"', - content = content) + pre=key, + new_str=f'{key} = "{value}"', + content=content) return content - def replace_param(self, parse_compile: ParseCompile, content:str): + def replace_param(self, parse_compile: ParseCompile, content: str): ''' match and replace param by ParseCompile.local_conf ''' @@ -249,4 +257,3 @@ class LocalConf: if not oecore_sysroot_dir.startswith(nativesdk_dir): raise NativesdkNotValid(f"nativesdk directory: {nativesdk_dir} are not valid") return - \ No newline at end of file diff --git a/src/oebuild/oebuild_parser.py b/src/oebuild/oebuild_parser.py index 5d43b89ba40f6db71501afc32b6ca02355b665b1..af6a11c91bfa83d786ed75eb9248ec3ad9eed77b 100644 --- a/src/oebuild/oebuild_parser.py +++ b/src/oebuild/oebuild_parser.py @@ -16,10 +16,12 @@ import shutil from io import StringIO import textwrap + class OebuildHelpAction(argparse.Action): ''' set argparse help is true ''' + def __call__(self, parser, namespace, values, option_string=None): # Just mark that help was requested. namespace.help = True @@ -61,7 +63,7 @@ class OebuildArgumentParser(argparse.ArgumentParser): # one of the subcommand parsers, and we delegate to super. # if not top_level: - # return super(OebuildArgumentParser, self).format_help() + # return super(OebuildArgumentParser, self).format_help() # Format the help to be at most 75 columns wide, the maximum # generally recommended by typographers for readability. @@ -87,7 +89,7 @@ class OebuildArgumentParser(argparse.ArgumentParser): if self.oebuild_app is not None: append('') - for _,command in self.oebuild_app.command_spec.items(): + for _, command in self.oebuild_app.command_spec.items(): self._format_command(append, command, width) if self.epilog: diff --git a/src/oebuild/ogit.py b/src/oebuild/ogit.py index c04d253aef00b651e0d15ed8486ba151b459226a..52186606a5c8807697ed238e44210e50095678dc 100644 --- a/src/oebuild/ogit.py +++ b/src/oebuild/ogit.py @@ -18,10 +18,12 @@ from git import GitCommandError, RemoteProgress from oebuild.m_log import logger + class OGit: ''' owner git to print progress in clone action ''' + def __init__(self, repo_dir, remote_url, branch) -> None: self._repo_dir = repo_dir self._remote_url = remote_url @@ -99,14 +101,16 @@ class OGit: except TypeError: return "", '' except git.GitError: - return "","" + return "", "" except ValueError: - return "","" + return "", "" + class CustomRemote(git.RemoteProgress): ''' Rewrote RemoteProgress to show the process of code updates ''' + def update(self, op_code, cur_count, max_count=None, message=''): ''' rewrote update function diff --git a/src/oebuild/parse_compile.py b/src/oebuild/parse_compile.py index 3f261123ea5f40aea19a11334ab849a331cd8bdf..d4ef81ef256457d05cd029bf0acfa3e1f96fab2a 100644 --- a/src/oebuild/parse_compile.py +++ b/src/oebuild/parse_compile.py @@ -22,6 +22,7 @@ from oebuild.parse_template import PlatformTemplate, ParseTemplate import oebuild.const as oebuild_const from oebuild.m_log import logger + @dataclass class DockerParam: ''' @@ -60,21 +61,25 @@ class Compile(PlatformTemplate): docker_param: Optional[DockerParam] + class BaseParseCompileError(ValueError): ''' parse compile basic error ''' + class CheckCompileError(BaseParseCompileError): ''' compile.yaml parse check faild error ''' + class ParseCompile: ''' This class is used to parse compile.yaml and download the relevant code repository ''' + def __init__(self, compile_conf_dir): ''' The initialization operation is used to parse the compile.yaml @@ -91,7 +96,7 @@ class ParseCompile: except Exception as e_p: raise e_p - docker_param:DockerParam = None + docker_param: DockerParam = None if "docker_param" in data and data['docker_param'] is not None: try: dparam = data['docker_param'] @@ -253,7 +258,7 @@ class ParseCompile: keys = { "machine": "the key machine is None", "toolchain_type": "the key toolchain_type is None" - } + } for key, value in keys.items(): if key not in data: @@ -266,10 +271,9 @@ class ParseCompile: if "repos" in data: for _, repo in data['repos'].items(): required_keys = { - "url": "the key url is None", - "path": "the key path is None", + "url": "the key url is None", + "path": "the key path is None", "refspec": "the key refspec is None"} for key, value in required_keys.items(): if key not in repo: raise CheckCompileError(value) - \ No newline at end of file diff --git a/src/oebuild/parse_env.py b/src/oebuild/parse_env.py index 98fcc44c2ed2354dd606bcce759708d89f2dc8da..f6bd60fcd363e9186b4f25bc7cd150a2e1de8cb3 100644 --- a/src/oebuild/parse_env.py +++ b/src/oebuild/parse_env.py @@ -18,6 +18,7 @@ import sys import oebuild.util as oebuild_util from oebuild.m_log import logger + @dataclass class EnvContainer: ''' @@ -25,6 +26,7 @@ class EnvContainer: ''' short_id: Optional[str] + @dataclass class Env: ''' @@ -32,14 +34,16 @@ class Env: ''' container: Optional[EnvContainer] + class ParseEnv: ''' This class is used to parse env.yaml and update env.yaml ''' + def __init__(self, env_dir): self.env_dir = pathlib.Path(env_dir) if isinstance(env_dir, str) else env_dir - self.env:Env = Env(container=None) + self.env: Env = Env(container=None) self._parse_env() @property @@ -60,7 +64,7 @@ class ParseEnv: if "container" in data: env_container = data['container'] try: - self.env.container = EnvContainer( + self.env.container = EnvContainer( short_id=env_container['short_id'] ) except KeyError: diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index a41947bb9753ab94baffd6ae1abefa5941b11bc0..5055cb06c02a4f7b9a9e4e8eda4ed6f0f01383db 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -21,6 +21,7 @@ from ruamel.yaml.scalarstring import LiteralScalarString import oebuild.util as oebuild_util import oebuild.const as oebuild_const + @dataclass class OebuildRepo: ''' @@ -40,6 +41,7 @@ class OebuildRepo: refspec: str + @dataclass class Template: ''' @@ -51,6 +53,7 @@ class Template: local_conf: Optional[LiteralScalarString] + @dataclass class PlatformTemplate(Template): ''' @@ -60,6 +63,7 @@ class PlatformTemplate(Template): toolchain_type: LiteralScalarString + @dataclass class FeatureTemplate(Template): ''' @@ -69,36 +73,43 @@ class FeatureTemplate(Template): support: list + class BaseParseTemplate(ValueError): ''' basic error about parse_template ''' + class ConfigPathNotExists(BaseParseTemplate): ''' config path not exists ''' + class PlatformNotAdd(BaseParseTemplate): ''' platform not add first ''' + class FeatureNotSupport(BaseParseTemplate): ''' feature not support ''' + class CommonNotFound(BaseParseTemplate): ''' common param not found ''' + class ParseTemplate: ''' ParseTemplate is to add platform template and feature template and export compile.yaml finially ''' - def __init__(self, yocto_dir:str): + + def __init__(self, yocto_dir: str): self.yocto_dir = yocto_dir self.build_in = None self.platform_template = None @@ -169,16 +180,16 @@ your arch is {self.platform}, the feature is not supported, please check your ap raise e_p def generate_compile_conf(self, - nativesdk_dir = None, - toolchain_dir = None, - build_in: str = oebuild_const.BUILD_IN_DOCKER, - sstate_cache = None, - tmp_dir = None, - datetime = None, - is_disable_fetch = False, - docker_image: str = None, - src_dir: str = None, - compile_dir: str = None): + nativesdk_dir=None, + toolchain_dir=None, + build_in: str = oebuild_const.BUILD_IN_DOCKER, + sstate_cache=None, + tmp_dir=None, + datetime=None, + is_disable_fetch=False, + docker_image: str = None, + src_dir: str = None, + compile_dir: str = None): ''' first param common yaml ''' @@ -193,7 +204,7 @@ your arch is {self.platform}, the feature is not supported, please check your ap data = oebuild_util.read_yaml(common_yaml_dir) repos = {} - if 'repos' in data : + if 'repos' in data: repos.update(data['repos']) layers = [] if 'layers' in data: @@ -217,10 +228,10 @@ your arch is {self.platform}, the feature is not supported, please check your ap layers = self.platform_template.layers if self.platform_template.local_conf is not None: - local_conf = LiteralScalarString(self.platform_template.local_conf + local_conf ) + local_conf = LiteralScalarString(self.platform_template.local_conf + local_conf) for feature in self.feature_template: - feature:FeatureTemplate = feature + feature: FeatureTemplate = feature if feature.repos is not None: for repo_name, oebuild_repo in feature.repos.items(): if repo_name in repos: @@ -270,7 +281,7 @@ your arch is {self.platform}, the feature is not supported, please check your ap volumns.append("/dev/net/tun:/dev/net/tun") volumns.append(src_dir + ':' + oebuild_const.CONTAINER_SRC) volumns.append(compile_dir + ':' + - os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) + os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) if toolchain_dir is not None: volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) if sstate_cache is not None: diff --git a/src/oebuild/spec.py b/src/oebuild/spec.py index 13ef54a75e63a938cc0f4795a1cacf01ee6336c9..5b19ba7c1aa03acfd266beac9de8dcff303b2f8d 100644 --- a/src/oebuild/spec.py +++ b/src/oebuild/spec.py @@ -15,10 +15,12 @@ import os from dataclasses import dataclass import sys + class CommandError(RuntimeError): ''' Indicates that a command failed. ''' + def __init__(self, returncode=1): super().__init__() self.returncode = returncode @@ -32,11 +34,11 @@ class ExtensionCommandError(CommandError): '''Exception class indicating an extension command was badly defined and could not be created.''' - def __init__(self, **kwargs): self.hint = kwargs.pop('hint', None) super(ExtensionCommandError, self).__init__(**kwargs) + @dataclass class _CmdFactory: @@ -91,6 +93,7 @@ class OebuildExtCommandSpec: # the command) before constructing it, however. factory: _CmdFactory + @dataclass class _ExtCommand: ''' diff --git a/src/oebuild/util.py b/src/oebuild/util.py index 6ed560a340dfaaecd4582556b5250131b4582df5..576bf7d601897f2778d32b292e36960dc4366f0d 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -54,7 +54,7 @@ def get_nativesdk_environment(nativesdk_dir=oebuild_const.NATIVESDK_DIR, sys.exit(res.exit_code) list_items = res.output.decode("utf-8").split("\n") for item in list_items: - item:str = item + item: str = item # notice: the item is like format with # "drwxr-xr-x 3 openeuler openeuler 4096 Nov 8 08:10 ." # so we must get the last clip from split with space @@ -68,7 +68,8 @@ def get_nativesdk_environment(nativesdk_dir=oebuild_const.NATIVESDK_DIR, logger.error("can not find any nativesdk environment initialization shell") sys.exit(1) -def read_yaml(yaml_dir : pathlib.Path): + +def read_yaml(yaml_dir: pathlib.Path): ''' read yaml file and parse it to object ''' @@ -84,7 +85,7 @@ def read_yaml(yaml_dir : pathlib.Path): raise e_p -def write_yaml(yaml_dir : pathlib.Path, data): +def write_yaml(yaml_dir: pathlib.Path, data): ''' write data to yaml file ''' @@ -97,14 +98,16 @@ def write_yaml(yaml_dir : pathlib.Path, data): yaml = YAML() yaml.dump(data, w_f) -def get_git_repo_name(remote_url : str): + +def get_git_repo_name(remote_url: str): ''' return repo name ''' - url = remote_url.replace(".git","") + url = remote_url.replace(".git", "") return os.path.basename(url) -def add_git_suffix(remote : str): + +def add_git_suffix(remote: str): ''' add .git suffix to remote if needed ''' @@ -113,36 +116,42 @@ def add_git_suffix(remote : str): return remote + ".git" + def get_base_oebuild(): ''' return oebuild base dir ''' return os.path.abspath(os.path.dirname(__file__)) + def get_config_yaml_dir(): ''' return config yaml dir ''' return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.CONFIG_YAML) + def get_plugins_yaml_path(): ''' return plugin yaml path ''' return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.PLUGINS_YAML) + def get_compile_yaml_dir(): ''' return compile.yaml.sample yaml dir ''' return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.COMPILE_YAML) + def get_upgrade_yaml_dir(): ''' return upgrade yaml dir ''' return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.UPGRADE_YAML) + def generate_random_str(randomlength=16): ''' generate a random string by length @@ -154,18 +163,21 @@ def generate_random_str(randomlength=16): random_str += base_str[random.randint(0, length)] return random_str + def get_time_stamp(): ''' get current timestamp ''' return time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) + def get_oebuild_version(): ''' return oebuild version ''' return __version__ + def check_docker(): ''' check docker had be installed or not @@ -181,12 +193,14 @@ please install docker first, and run follow commands in root: 4, chmod o+rw /var/run/docker.sock ''') from exc + def get_instance(factory): ''' Instantiate a class ''' return factory() + def restore_bashrc_content(old_content): ''' restore .bashrc @@ -199,6 +213,7 @@ def restore_bashrc_content(old_content): new_content = new_content + line + '\n' return new_content + def init_bashrc_content(old_content, init_command: list): ''' init bashrc @@ -210,6 +225,7 @@ def init_bashrc_content(old_content, init_command: list): return new_content + def add_bashrc(content: str, line: str): ''' add command line to bashrc @@ -220,6 +236,7 @@ def add_bashrc(content: str, line: str): return content + def get_host_proxy(proxy_name): ''' get proxy information from host