From 7f0e043eedbcd304afb2e8d64922189e7b7427c9 Mon Sep 17 00:00:00 2001 From: alichinese Date: Fri, 14 Nov 2025 09:43:15 +0800 Subject: [PATCH 1/2] oee-archive: optimize oee-archive operating mechanism * The `oee-archive` class will be separated from `openeuler-fetch`. Inheriting the `oee-archive` class will enable independent downloading of `oee-archive` packages, and this downloading is on-demand. Therefore, in recipes, synchronous downloading of both recipe packages and `oee-archive` packages can be achieved. To use it, simply inherit the `oee-archive` class. By default, the subdirectory of `oee-archive` will be consistent with the recipe. If inconsistency occurs, it can be controlled through the `OEE_ARCHIVE_SUB_DIR` variable. for example: SUB_DIR is same with recipe name aaa.bb(with aaa/xxx.tar.gz): inherit oee-archive SUB_DIR is not same with recipe name aaa.bb(with vvv/xxx.tar.gz): OEE_ARCHIVE_SUB_DIR = "vvv" and SRC_URI anded with "file://xxx.tar.gz" in both same with recipe name or not Signed-off-by: alichinese --- meta-openeuler/classes/oee-archive.bbclass | 96 +++++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/meta-openeuler/classes/oee-archive.bbclass b/meta-openeuler/classes/oee-archive.bbclass index 4984d2168ee..f60e12e13cc 100644 --- a/meta-openeuler/classes/oee-archive.bbclass +++ b/meta-openeuler/classes/oee-archive.bbclass @@ -3,14 +3,106 @@ # oee_archive must be a git repo. # the default repo name is oee_archive, which is used for do_openeuler_fetch -OPENEULER_LOCAL_NAME = "oee_archive" OEE_ARCHIVE_SUB_DIR ?= "${BPN}" # for real file path to search is ${OPENEULER_LOCAL_NAME}/${OEE_ARCHIVE_SUB_DIR}, # not OPENEULER_LOCAL_NAME. -OPENEULER_DL_DIR = "${OPENEULER_SP_DIR}/${OPENEULER_LOCAL_NAME}/${OEE_ARCHIVE_SUB_DIR}" +OEE_ARCHIVE_DIR = "${OPENEULER_SP_DIR}/oee_archive" + +FILESEXTRAPATHS:prepend = "${OEE_ARCHIVE_DIR}/${OEE_ARCHIVE_SUB_DIR}/:" # oee-archive.bbclass is always inherited in .bbappend file, # this will cause the waring of QA "native-last" # add native-last into INSANE_SKIP to avoid this warning. # a better way is to try to inherit oee-archive.bbclass before native.bbclass INSANE_SKIP += "native-last" + +def init_oee_archive_repo_dir(repo_dir): + import git + + repo = git.Repo.init(repo_dir) + + with repo.config_writer() as wr: + wr.set_value('http', 'sslverify', 'false').release() + return repo + +def init_oee_archive_repo_remote(repo, remote_url): + """ + init repo remote + """ + import git + + try: + repo.create_remote("upstream", remote_url) + except git.exc.GitCommandError: + pass + +def check_oee_archive_repo_version(repo, version): + """ + check repo version + """ + import git + from git.exc import GitCommandError + + try: + # Set lfs.fetchexclude to * to skip downloading LFS files + with repo.config_writer() as wr: + wr.set_value('lfs', 'fetchexclude', '*').release() + + # Use the python-git module to perform the fetch operation + repo.git.fetch('upstream', version, '--depth=1') + + # Use the python-git module to perform the checkout operation + repo.git.checkout(version) + + # Restore lfs.fetchexclude setting using the python-git module + with repo.config_writer() as wr: + wr.set_value('lfs', 'fetchexclude', '').release() + except GitCommandError as e: + # Use repo.working_dir to get the repository path + raise Exception("version %s not found in repo %s, error: %s" % (version, repo.working_dir, e)) + +def pull_oee_archive_repo_sub_lfs(repo, sub_dir): + """ + pull repo submodule lfs files + """ + import git + from git.exc import GitCommandError + + try: + # Use the python-git module to execute git lfs pull, including only the specified subdirectory + repo.git.lfs('pull', '--include', '%s/*' % sub_dir) + except GitCommandError: + raise Exception("pull repo submodule lfs files failed in repo %s" % repo_dir) + +python do_download_oee_archive(){ + import os + import subprocess + + # Initialize the oee_archive repo + oee_archive_repo = init_oee_archive_repo_dir(d.getVar('OEE_ARCHIVE_DIR')) + repo_item = d.getVar('MANIFEST_LIST')['oee_archive'] + # Add oee_archive remote + init_oee_archive_repo_remote(oee_archive_repo, repo_item['remote_url']) + # Check oee_archive version + check_oee_archive_repo_version(oee_archive_repo, repo_item['version']) + # Pull oee_archive submodule LFS files + pull_oee_archive_repo_sub_lfs(oee_archive_repo, d.getVar('OEE_ARCHIVE_SUB_DIR')) +} + +do_download_oee_archive[lockfiles] = "/tmp/openeuler/oee_archive.lock" +do_download_oee_archive[network] = "1" +addtask do_download_oee_archive before do_fetch + +# store YAML data within MANIFEST_LIST variable +addhandler parse_manifest +python parse_manifest() { + # used to read YAML file data + def get_manifest(manifest_yaml): + import yaml + + with open(manifest_yaml, 'r' ,encoding="utf-8") as r_f: + return yaml.load(r_f.read(), yaml.Loader)['manifest_list'] + + d.setVar('MANIFEST_LIST', get_manifest(d.getVar("MANIFEST_DIR"))) +} +parse_manifest[eventmask] = "bb.event.ConfigParsed" -- Gitee From 425b7230b81cfd096af9bb8a81cc3b3b4ef00be9 Mon Sep 17 00:00:00 2001 From: alichinese Date: Fri, 14 Nov 2025 09:51:54 +0800 Subject: [PATCH 2/2] oee-archive: adapt oee-archive refs * Adapt packages that reference the oee-archive class Signed-off-by: alichinese --- .oebuild/manifest.yaml | 2 +- .../recipes-bsp/ss928/hieulerpi1-sdk-pkg.bb | 9 ++---- meta-openeuler/classes/openeuler.bbclass | 32 ++----------------- .../perl-cross/perlcross_%.bbappend | 1 + .../pkgconfig/pkgconfig_%.bbappend | 1 + .../recipes-devtools/pseudo/pseudo_%.bbappend | 6 ++-- .../python/python3-build_%.bbappend | 7 ++-- .../python/python3-installer_%.bbappend | 1 + 8 files changed, 17 insertions(+), 42 deletions(-) diff --git a/.oebuild/manifest.yaml b/.oebuild/manifest.yaml index bfc55b1dc7a..068fd1c12bf 100644 --- a/.oebuild/manifest.yaml +++ b/.oebuild/manifest.yaml @@ -1073,7 +1073,7 @@ manifest_list: version: 42e783e24b0a018f81bb6c838ca7532862528a07 oee_archive: remote_url: https://gitee.com/openeuler/oee_archive.git - version: 19f5eea6508ba641169a32c87f57493547cb254a + version: 91e09f91c785e73c3534f290fcb1cc9bfd6ccd05 ompl: remote_url: https://gitee.com/src-openeuler/ompl.git version: 42759f06c9a8d712a5ba86f1f3b22a569ec81232 diff --git a/bsp/meta-hisilicon/recipes-bsp/ss928/hieulerpi1-sdk-pkg.bb b/bsp/meta-hisilicon/recipes-bsp/ss928/hieulerpi1-sdk-pkg.bb index 6b64fd451d6..36648a35d81 100644 --- a/bsp/meta-hisilicon/recipes-bsp/ss928/hieulerpi1-sdk-pkg.bb +++ b/bsp/meta-hisilicon/recipes-bsp/ss928/hieulerpi1-sdk-pkg.bb @@ -3,25 +3,22 @@ SECTION = "base" LICENSE = "Apache-2.0" LIC_FILES_CHKSUM = "file://LICENSE;md5=34d15ab872e1eb3db3292ffb63006766" -inherit module +inherit module oee-archive +OEE_ARCHIVE_SUB_DIR = "mbedtls" DEPENDS = "update-rc.d-native" OPENEULER_LOCAL_NAME = "Hispark-ss928v100-gcc-sdk" -OPENEULER_SRC_URI_REMOVE = "" - SRC_URI = " \ file://Hispark-ss928v100-gcc-sdk \ file://0001-yocto-928-sdk-build-support.patch \ file://0002-fix-928-sdk-cipher-invalid.patch \ file://load_sdk_driver \ - https://gitee.com/openeuler/oee_archive/raw/master/mbedtls/v2.16.10.tar.gz;name=mbedtls;unpack=0 \ + file://v2.16.10.tar.gz;unpack=0 \ file://sdk-fix-mbedtls-err-2.16.10.patch.in \ " -SRC_URI[mbedtls.md5sum] = "35c8002be7088cb9cedb28e9917d7b24" - S = "${WORKDIR}/Hispark-ss928v100-gcc-sdk" INSANE_SKIP:${PN} += "already-stripped" diff --git a/meta-openeuler/classes/openeuler.bbclass b/meta-openeuler/classes/openeuler.bbclass index 888377f4156..2f06667353b 100644 --- a/meta-openeuler/classes/openeuler.bbclass +++ b/meta-openeuler/classes/openeuler.bbclass @@ -220,6 +220,8 @@ python do_openeuler_fetch() { repo_list.append(d.getVar("OPENEULER_LOCAL_NAME")) for repo_name in repo_list: # download code from openEuler + if repo_name == "oee_archive": + continue openeuler_fetch(d, repo_name) } @@ -268,36 +270,6 @@ def download_repo(d, repo_dir, repo_url ,version = None): remote_name = "upstream" remote = git.Remote.add(repo = repo, name = remote_name, url = repo_url) - # This download function is only used for downloading oee_archive which holds tar packages, it can - # download what you want, and only what you need, no more others. In order to do this, we use git - # sparse-checkout, which reduces your working tree to a subset of - # tracked files. You can see more detail by visiting https://git-scm.com/docs/git-sparse-checkout - def oee_archive_download(oee_archive_dir:str, subdir: str): - # if exists subdir and return - if os.path.exists(os.path.join(oee_archive_dir, subdir)): - return - res = subprocess.run("git sparse-checkout init --cone", - shell=True, - stderr=subprocess.PIPE, - text=True, - cwd=oee_archive_dir) - if res.returncode != 0: - bb.fatal(f"in oee_archive run git sparse-checkout init failed, error: {res.stderr}") - res = subprocess.run(f"git sparse-checkout list | grep {subdir}", shell=True, cwd=oee_archive_dir) - if res.returncode == 0: - return - res = subprocess.run(f"git sparse-checkout add {subdir}", - shell=True, - stderr=subprocess.PIPE, - text=True, - cwd=oee_archive_dir) - if res.returncode != 0: - bb.fatal(f"in oee_archive run git sparse-checkout add {subdir} failed, error: {res.stderr}") - - if d.getVar("OEE_ARCHIVE_SUB_DIR") is not None: - sub_dir = d.getVar("OEE_ARCHIVE_SUB_DIR") - oee_archive_download(oee_archive_dir = repo_dir, subdir = sub_dir) - # the function is used to download large file in repo def lfs_download(repo_dir, remote_name, version): res = subprocess.run(f"git lfs fetch {remote_name} {version}", diff --git a/meta-openeuler/recipes-devtools/perl-cross/perlcross_%.bbappend b/meta-openeuler/recipes-devtools/perl-cross/perlcross_%.bbappend index 4c99f8f6d16..099a447e0fe 100644 --- a/meta-openeuler/recipes-devtools/perl-cross/perlcross_%.bbappend +++ b/meta-openeuler/recipes-devtools/perl-cross/perlcross_%.bbappend @@ -5,6 +5,7 @@ PV = "1.5" inherit oee-archive SRC_URI:remove = " \ + https://github.com/arsv/perl-cross/releases/download/1.5/perl-cross-1.5.tar.gz;name=perl-cross \ file://0001-Makefile-check-the-file-if-patched-or-not.patch \ " diff --git a/meta-openeuler/recipes-devtools/pkgconfig/pkgconfig_%.bbappend b/meta-openeuler/recipes-devtools/pkgconfig/pkgconfig_%.bbappend index d75ef943486..df2aadaaff7 100644 --- a/meta-openeuler/recipes-devtools/pkgconfig/pkgconfig_%.bbappend +++ b/meta-openeuler/recipes-devtools/pkgconfig/pkgconfig_%.bbappend @@ -2,6 +2,7 @@ inherit oee-archive PV = "0.29.2" +SRC_URI:remove = "git://gitlab.freedesktop.org/pkg-config/pkg-config.git;branch=master;protocol=https" SRC_URI:prepend = "file://pkg-config-${PV}.tar.gz \ " diff --git a/meta-openeuler/recipes-devtools/pseudo/pseudo_%.bbappend b/meta-openeuler/recipes-devtools/pseudo/pseudo_%.bbappend index 1fecbeae028..9362e9830bc 100644 --- a/meta-openeuler/recipes-devtools/pseudo/pseudo_%.bbappend +++ b/meta-openeuler/recipes-devtools/pseudo/pseudo_%.bbappend @@ -1,10 +1,12 @@ OPENEULER_LOCAL_NAME = "yocto-pseudo" -OPENEULER_REPO_NAMES = "yocto-pseudo oee_archive" +OPENEULER_REPO_NAMES = "yocto-pseudo" + +inherit oee-archive OEE_ARCHIVE_SUB_DIR = "pseudo" SRC_URI:prepend = "file://${BP}.tar.gz \ - file://oee_archive/pseudo/pseudo-prebuilt-2.33.tar.xz;subdir=${BP}/prebuilt;name=prebuilt \ + file://pseudo-prebuilt-2.33.tar.xz;subdir=${BP}/prebuilt;name=prebuilt \ " PV = "df1d1321fb093283485c387e3c933d2d264e509c" diff --git a/meta-openeuler/recipes-devtools/python/python3-build_%.bbappend b/meta-openeuler/recipes-devtools/python/python3-build_%.bbappend index 514d84c8aab..f23d287fbe5 100644 --- a/meta-openeuler/recipes-devtools/python/python3-build_%.bbappend +++ b/meta-openeuler/recipes-devtools/python/python3-build_%.bbappend @@ -5,6 +5,7 @@ inherit oee-archive SRC_URI[sha256sum] = "248a092b06b97a6377ba457264c86c1925a89bbd225da3b03da0c0d42b90974c" # upstream source -SRC_URI:prepend = " \ - file://build-${PV}.zip \ - " +SRC_URI:remove = "https://files.pythonhosted.org/packages/source/b/build/build-0.10.0.tar.gz" +SRC_URI = " \ + file://build-${PV}.zip \ +" diff --git a/meta-openeuler/recipes-devtools/python/python3-installer_%.bbappend b/meta-openeuler/recipes-devtools/python/python3-installer_%.bbappend index cedc7d1a836..757c5afc797 100644 --- a/meta-openeuler/recipes-devtools/python/python3-installer_%.bbappend +++ b/meta-openeuler/recipes-devtools/python/python3-installer_%.bbappend @@ -5,6 +5,7 @@ inherit oee-archive SRC_URI[sha256sum] = "b4df8cf5a649ff6f25cb885a7a93662f38229e05f1859db663f752a6203014f6" # upstream source +SRC_URI:remove = "https://files.pythonhosted.org/packages/source/i/installer/installer-0.6.0.tar.gz" SRC_URI:prepend = " \ file://installer-${PV}.tar.gz \ " -- Gitee