From bd85581ea856c34f33fe6a4054ab5735a4c54bb3 Mon Sep 17 00:00:00 2001 From: alichinese Date: Fri, 24 Mar 2023 07:07:23 +0000 Subject: [PATCH 1/3] manifest: add manifest function * add function manifest * update version to 0.0.11 Signed-off-by: lixinyu --- src/oebuild/app/conf/manifest_banner | 22 +++ src/oebuild/app/conf/plugins.yaml | 3 + src/oebuild/app/plugins/generate/generate.py | 12 +- src/oebuild/app/plugins/manifest/manifest.py | 190 +++++++++++++++++++ src/oebuild/local_conf.py | 2 +- src/oebuild/ogit.py | 6 + src/oebuild/parse_template.py | 12 +- src/oebuild/util.py | 4 +- src/oebuild/version.py | 2 +- 9 files changed, 241 insertions(+), 12 deletions(-) create mode 100644 src/oebuild/app/conf/manifest_banner create mode 100644 src/oebuild/app/plugins/manifest/manifest.py diff --git a/src/oebuild/app/conf/manifest_banner b/src/oebuild/app/conf/manifest_banner new file mode 100644 index 0000000..693b7e9 --- /dev/null +++ b/src/oebuild/app/conf/manifest_banner @@ -0,0 +1,22 @@ +# Copyright (c) 2023 openEuler Embedded +# oebuild is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +# The manifest lists all dependent software manifests and their version numbers, +# relying on which the OpenEuler Embedded build is completed, first go to the +# oebuild working directory, and then do the following: +# 1: oebuild manifest -r -m_dir # for restore or download all repos +# 2: oebuild generate -df -d # generate compile.yaml with disable openeuler_fetch +# 3: Go to the build directory and open compile.yaml, add a line under the build_in +# field, not_use_repos : true, like this: +# build_in: docker +# not_use_repos: true +# platform: x86-64-std +# machine: generic-x86-64 +# 4: oebuild bitbake openeuler-image diff --git a/src/oebuild/app/conf/plugins.yaml b/src/oebuild/app/conf/plugins.yaml index 7b48bb2..4fa8b1e 100644 --- a/src/oebuild/app/conf/plugins.yaml +++ b/src/oebuild/app/conf/plugins.yaml @@ -11,3 +11,6 @@ plugins: - name: bitbake class: Bitbake path: plugins/bitbake/bitbake.py +- name: manifest + class: Manifest + path: plugins/manifest/manifest.py diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index ca23d7a..15e6cf4 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -107,6 +107,11 @@ class Generate(OebuildCommand): this param is add DATETIME to local.conf, the value is getting current time ''') + parser.add_argument('-df', '--disable_fetch', dest = "is_disable_fetch", action = "store_true", + help=''' + this param is set openeuler_fetch in local.conf, the default value is enable, if set -df, the OPENEULER_FETCH will set to 'disable' + ''') + parser.add_argument('-b_in', dest='build_in', choices=[BUILD_IN_DOCKER, BUILD_IN_HOST], default = BUILD_IN_DOCKER, help=''' This parameter marks the mode at build time, and is built in the container by docker @@ -169,8 +174,6 @@ class Generate(OebuildCommand): except ValueError as v_e: log.err(str(v_e)) return - else: - pass try: self._add_features_template(args=args, @@ -183,8 +186,6 @@ class Generate(OebuildCommand): except ValueError as v_e: log.err(str(v_e)) return - else: - pass if os.path.exists(os.path.join(build_dir,'compile.yaml')): os.remove(os.path.join(build_dir,'compile.yaml')) @@ -196,7 +197,8 @@ class Generate(OebuildCommand): build_in=build_in, sstate_cache= self.sstate_cache, tmp_dir = self.tmp_dir, - is_datetime=args.is_datetime + is_datetime = args.is_datetime, + is_disable_fetch = args.is_disable_fetch )) log.info("=============================================") diff --git a/src/oebuild/app/plugins/manifest/manifest.py b/src/oebuild/app/plugins/manifest/manifest.py new file mode 100644 index 0000000..710d778 --- /dev/null +++ b/src/oebuild/app/plugins/manifest/manifest.py @@ -0,0 +1,190 @@ +''' +Copyright (c) 2023 openEuler Embedded +oebuild is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +''' + +import argparse +import textwrap +import sys +import os +import pathlib +import multiprocessing +from queue import Queue +from threading import Thread +from concurrent.futures import ThreadPoolExecutor,wait,ALL_COMPLETED + + +import git +from git.exc import GitCommandError + +from oebuild.command import OebuildCommand +from oebuild.configure import Configure +import oebuild.util as oebuild_util + +from oebuild.my_log import MyLog as log + +class Manifest(OebuildCommand): + ''' + manifest provides the manifest function of generating dependent + source repositories in the build working directory, and can restore + relevant source repositories based on the manifest file + ''' + + def __init__(self): + self.configure = Configure() + super().__init__( + 'manifest', + 'generate manifest from oebuild workspace', + textwrap.dedent('''\ + manifest provides the manifest function of generating dependent + source repositories in the build working directory, and can restore + relevant source repositories based on the manifest file +''' + )) + + def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: + parser = self._parser( + parser_adder, + usage=''' + + %(prog)s [-c CREATE] [-r recover] [-m_dir MANIFEST_DIR] + +''') + + parser.add_argument('-c', + '--create', + dest = "is_create", + action = "store_true", + help=''' + create manifest from oebuild workspace src directory + ''') + + parser.add_argument('-r', + '--recover', + dest = "is_recover", + action = "store_true", + help=''' + restore repo version to oebuild workspace src directory from a manifest + ''') + + parser.add_argument('-m_dir', + dest='manifest_dir', + help=''' + specify a manifest path to perform the create or restore operation + ''' + ) + + return parser + + def do_run(self, args: argparse.Namespace, unknown = None): + args = args.parse_args(unknown) + + if not self.configure.is_oebuild_dir(): + log.err('your current directory had not finishd init') + sys.exit(-1) + + if args.is_create: + self._create_manifest(args.manifest_dir) + elif args.is_recover: + self._restore_manifest(args.manifest_dir) + + def _create_manifest(self, manifest_dir): + src_list = os.listdir(self.configure.source_dir()) + manifest_list = {} + for index, repo_dir in enumerate(src_list): + local_dir = os.path.join(self.configure.source_dir(), repo_dir) + try: + repo = git.Repo(local_dir) + remote_url = repo.remote().url + version = repo.head.commit.hexsha + except git.GitError: + continue + except ValueError: + continue + + manifest_list[repo_dir] = { + 'remote_url': remote_url, + 'version': version + } + print("\r", end="") + progress = int((index + 1) / len(src_list) * 100) + print(f"Expose progress: {progress}%: ", "▋" * (progress // 2), end="") + sys.stdout.flush() + print() + + oebuild_util.write_yaml( + yaml_dir=pathlib.Path(manifest_dir), + data={'manifest_list': manifest_list}) + self._add_manifest_banner(manifest_dir = os.path.abspath(manifest_dir)) + + print(f"expose successful, the directory is {os.path.abspath(manifest_dir)}") + + def _add_manifest_banner(self, manifest_dir): + oebuild_conf_dir = os.path.join(oebuild_util.get_base_oebuild(), 'app/conf') + manifest_banner_dir = os.path.join(oebuild_conf_dir, 'manifest_banner') + + with open(manifest_banner_dir, 'r', encoding='utf-8') as r_f: + banner = r_f.read() + with open(manifest_dir, 'r', encoding='utf-8') as r_f: + manifest_str = r_f.read() + + manifest_content = banner + '\n' + manifest_str + with open(manifest_dir, 'w', encoding='utf-8') as w_f: + w_f.write(manifest_content) + + def _restore_manifest(self, manifest_dir): + manifest_data = oebuild_util.read_yaml(pathlib.Path(manifest_dir)) + manifest_list = manifest_data.get('manifest_list', {}) + + def print_progress(in_q: Queue, length: int): + index = 0 + while True: + data = in_q.get() + if data == "over": + break + index = index + 1 + print("\r", end="") + progress = int((index + 1) / length * 100) + print(f"restore progress: {progress}%: ", "▋" * (progress // 2), end="") + sys.stdout.flush() + print() + print(f"restore successful, the source directory is {os.path.abspath(src_dir)}") + + q_e = Queue() + dserver = Thread(target=print_progress, args=(q_e, len(manifest_list))) + dserver.start() + + cpu_count = multiprocessing.cpu_count() + with ThreadPoolExecutor(max_workers = cpu_count) as t_p: + src_dir = self.configure.source_dir() + all_task = [] + for key, value in manifest_list.items(): + all_task.append(t_p.submit(self._download_repo, q_e, src_dir, key, value)) + wait(all_task, return_when = ALL_COMPLETED) + q_e.put("over") + + def _download_repo(self, out_q: Queue ,src_dir, key, value): + repo_dir = os.path.join(src_dir, key) + repo = git.Repo.init(repo_dir) + remote = None + for item in repo.remotes: + if value['remote_url'] == item.url: + remote = item + else: + continue + if remote is None: + remote_name = "manifest" + remote = git.Remote.add(repo = repo, name = remote_name, url = value['remote_url']) + try: + repo.git.checkout(value['version']) + except GitCommandError: + remote.fetch(value['version'], depth = 1) + repo.git.checkout(value['version']) + out_q.put('ok') diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index 4cf9a51..c1763c6 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -196,7 +196,7 @@ class LocalConf: if ret is not None: content = match_and_add(line, content) continue - ret = re.match(r'^([A-Z0-9a-z_-]+)(\s)', line) + ret = re.match(r'^(([A-Z0-9a-z_-]|[/])+)(\s)', line) if ret is not None: content = match_and_replace(ret.group(), line, content) continue diff --git a/src/oebuild/ogit.py b/src/oebuild/ogit.py index 6e09d7f..f18a1bf 100644 --- a/src/oebuild/ogit.py +++ b/src/oebuild/ogit.py @@ -71,6 +71,8 @@ class OGit: try: repo = git.Repo(self._repo_dir) remote = repo.remote() + if repo.head.is_detached: + repo.git.checkout(self._branch) if repo.active_branch.name != self._branch: log.info(f"Fetching into '{self._repo_dir}'...") remote.fetch(progress=self.clone_process) @@ -135,5 +137,9 @@ class OGit: remote_url = repo.remote().url branch = repo.active_branch.name return remote_url, branch + except TypeError: + return remote_url, '' except git.GitError: return "","" + except ValueError: + return "","" diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 29f2c74..00a6eae 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -169,7 +169,8 @@ class ParseTemplate: build_in: str = BUILD_IN_DOCKER, sstate_cache = None, tmp_dir = None, - is_datetime = False): + is_datetime = False, + is_disable_fetch = False): ''' first param common yaml ''' @@ -225,13 +226,16 @@ class ParseTemplate: layers.extend(feature.layers) if feature.local_conf is not None: - local_conf = LiteralScalarString(feature.local_conf + local_conf) + local_conf = LiteralScalarString(feature.local_conf + '\n' + local_conf) if is_datetime: now_time = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) datetime_str = LiteralScalarString(f'DATETIME = "{now_time}"') - print(datetime_str) - local_conf = LiteralScalarString(local_conf + datetime_str) + local_conf = LiteralScalarString(local_conf + '\n' + datetime_str) + + if is_disable_fetch: + disable_fetch_str = LiteralScalarString('OPENEULER_FETCH = "disable"') + local_conf = LiteralScalarString(local_conf + '\n' + disable_fetch_str) compile_conf = { 'build_in': build_in, diff --git a/src/oebuild/util.py b/src/oebuild/util.py index 328a74b..80f91c0 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -17,9 +17,9 @@ import random import getpass from ruamel.yaml import YAML -from oebuild.docker_proxy import DockerProxy from docker.errors import DockerException +from oebuild.docker_proxy import DockerProxy from oebuild.version import __version__ CONFIG_YAML = 'config.yaml' @@ -46,6 +46,8 @@ def write_yaml(yaml_dir : pathlib.Path, data): write data to yaml file ''' if not os.path.exists(yaml_dir.absolute()): + if not os.path.exists(os.path.dirname(yaml_dir.absolute())): + os.makedirs(os.path.dirname(yaml_dir.absolute())) os.mknod(yaml_dir) with open(yaml_dir, 'w', encoding='utf-8') as w_f: diff --git a/src/oebuild/version.py b/src/oebuild/version.py index c0c53a9..4681d37 100644 --- a/src/oebuild/version.py +++ b/src/oebuild/version.py @@ -10,4 +10,4 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -__version__ = '0.0.6' +__version__ = '0.0.11' -- Gitee From 48fd6ddb15f3b1a87f702810f865bdcbe94bbde0 Mon Sep 17 00:00:00 2001 From: alichinese Date: Wed, 12 Apr 2023 08:59:30 +0000 Subject: [PATCH 2/3] code: optimize code * alter .oebuild/config.yaml about build docker branch * fix bug about feature list showing no complete Signed-off-by: lixinyu --- src/oebuild/app/conf/config.yaml | 3 +-- src/oebuild/app/plugins/generate/generate.py | 8 ++++---- src/oebuild/app/plugins/update/update.py | 5 +++-- src/oebuild/version.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/oebuild/app/conf/config.yaml b/src/oebuild/app/conf/config.yaml index cde0e17..0784271 100644 --- a/src/oebuild/app/conf/config.yaml +++ b/src/oebuild/app/conf/config.yaml @@ -1,8 +1,7 @@ docker: repo_url: swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container tag_map: - openEuler-22.09: "22.09" - openEuler-22.03-lts-sp1: "22.03-lts-sp1" + openEuler-23.03: "23.03" master: "latest" basic_repo: yocto_meta_openeuler: diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index 15e6cf4..46363a6 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -282,9 +282,9 @@ cd {build_dir} log.info("the platform list is:") for platform in list_platform: if platform.endswith('.yml'): - log.info(platform.rstrip('.yml')) + log.info(platform.replace('.yml')) if platform.endswith('.yaml'): - log.info(platform.rstrip('.yaml')) + log.info(platform.replace('.yaml')) def _list_feature(self,): log.info("=============================================") @@ -294,9 +294,9 @@ cd {build_dir} log.info("the feature list is:") for feature in list_feature: if feature.endswith('.yml'): - log.info(feature.rstrip('.yaml')) + log.info(feature.replace('.yml','')) if feature.endswith('.yaml'): - log.info(feature.rstrip('.yaml')) + log.info(feature.replace('.yaml','')) feat = oebuild_util.read_yaml(pathlib.Path(os.path.join(yocto_oebuild_dir, 'features', feature))) diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py index 62e1204..0dee3b0 100644 --- a/src/oebuild/app/plugins/update/update.py +++ b/src/oebuild/app/plugins/update/update.py @@ -16,12 +16,13 @@ import textwrap import sys import shutil +from docker.errors import DockerException + import oebuild.util as oebuild_util from oebuild.command import OebuildCommand from oebuild.configure import Configure, ConfigBasicRepo, YOCTO_META_OPENEULER from oebuild.docker_proxy import DockerProxy from oebuild.ogit import OGit -from docker.errors import DockerException from oebuild.my_log import MyLog as log, INFO_COLOR @@ -139,7 +140,7 @@ class Update(OebuildCommand): if remote_url != yocto_config.remote_url: if not os.path.exists(self.configure.yocto_bak_dir()): os.makedirs(self.configure.yocto_bak_dir()) - bak_dir = os.path.join(self.configure.yocto_bak_dir(), + bak_dir = os.path.join(self.configure.yocto_bak_dir(), yocto_config.path + "_" + oebuild_util.get_time_stamp()) log.warning(f"yocto-meta-openeuler remote is changed, \ bak yocto-meta-openeuler to {bak_dir}") diff --git a/src/oebuild/version.py b/src/oebuild/version.py index 4681d37..66c3b70 100644 --- a/src/oebuild/version.py +++ b/src/oebuild/version.py @@ -10,4 +10,4 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -__version__ = '0.0.11' +__version__ = '0.0.12' -- Gitee From cf80f7b89c9d3e76ba16108cbafde57d8d75aded Mon Sep 17 00:00:00 2001 From: alichinese Date: Tue, 18 Apr 2023 01:46:12 +0000 Subject: [PATCH 3/3] code: optimize code * when use manifest command to generate manifest.yaml and is sorted keys * upgrade version to 0.0.13 Signed-off-by: lixinyu --- src/oebuild/app/plugins/manifest/manifest.py | 3 +-- src/oebuild/version.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/oebuild/app/plugins/manifest/manifest.py b/src/oebuild/app/plugins/manifest/manifest.py index 710d778..6ad9f95 100644 --- a/src/oebuild/app/plugins/manifest/manifest.py +++ b/src/oebuild/app/plugins/manifest/manifest.py @@ -20,7 +20,6 @@ from queue import Queue from threading import Thread from concurrent.futures import ThreadPoolExecutor,wait,ALL_COMPLETED - import git from git.exc import GitCommandError @@ -118,7 +117,7 @@ class Manifest(OebuildCommand): print(f"Expose progress: {progress}%: ", "▋" * (progress // 2), end="") sys.stdout.flush() print() - + manifest_list = dict(sorted(manifest_list.items(),key=lambda s:s[0])) oebuild_util.write_yaml( yaml_dir=pathlib.Path(manifest_dir), data={'manifest_list': manifest_list}) diff --git a/src/oebuild/version.py b/src/oebuild/version.py index 66c3b70..c3d1e7e 100644 --- a/src/oebuild/version.py +++ b/src/oebuild/version.py @@ -10,4 +10,4 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' -__version__ = '0.0.12' +__version__ = '0.0.13' -- Gitee