From 688934fd9db1c703e34b09c65766c69a8fd97467 Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Thu, 4 Jul 2024 17:39:46 +0800 Subject: [PATCH] sync cmd-fetch from upstream v0.16.0 [upstream] 024ce76e4 cmd-fetch: use `sudo du` for pkgcache size check 3c8651ca4 tree: address latest ShellCheck errors 2da4dacbe cmd-fetch: support "autolocking" from an arch build 2b376e8d5 cmd-fetch: rename `args` variable 1fdbd66b5 cmd-fetch: document available options d0f638dee cmd-fetch: make `--force-nocache` optional 0ea19847c cmd-fetch: pass --force-nocache --- src/cmd-build | 11 ++++++++++ src/cmd-fetch | 61 ++++++++++++++++++++++++++++++++++++++++++--------- src/cmdlib.sh | 39 ++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/src/cmd-build b/src/cmd-build index 2597d880..fc0c66e2 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -234,6 +234,17 @@ prepare_git_artifacts "${configdir_gitrepo}" "${PWD}/coreos-assembler-config.tar extra_compose_args=() +# Apply autolock from another build for this version if no base lockfile exists. +# Do this before so that overrides come after. Don't do this if in strict mode. +# They're theoretically independent, but in practice it's unlikely that an +# autolockfile will include all the packages needed to satisfy --strict. +if [ ! -f "${manifest_lock}" ] && [ -n "${VERSION}" ] && [ -z "${STRICT}" ]; then + autolockfile=$(generate_autolock "${VERSION}") + if [ -n "${autolockfile}" ]; then + extra_compose_args+=("--ex-lockfile=${autolockfile}") + fi +fi + for lock in "${manifest_lock}" "${manifest_lock_overrides}" "${manifest_lock_arch_overrides}"; do if [ -f "${lock}" ]; then extra_compose_args+=("--ex-lockfile=${lock}") diff --git a/src/cmd-fetch b/src/cmd-fetch index 6813b657..165829b7 100755 --- a/src/cmd-fetch +++ b/src/cmd-fetch @@ -8,7 +8,7 @@ dn=$(dirname "$0") FILE=cache/pkgcache-repo if [ -d "${FILE}" ] then - pkgcachesize=$(du --bytes --max-depth 0 "${FILE}" \ + pkgcachesize=$(sudo du --bytes --max-depth 0 "${FILE}" \ | awk '{print $1; exit}') pkglimit=$((1024 * 1024 * 1024 * 5)) if [[ "${pkgcachesize}" -gt "${pkglimit}" ]] @@ -20,19 +20,31 @@ fi print_help() { cat 1>&2 <<'EOF' Usage: coreos-assembler fetch --help - coreos-assembler fetch [--update-lockfile] [--write-lockfile-to=file] [--with-cosa-overrides] [--strict] + coreos-assembler fetch [OPTIONS]... Fetch and import the latest packages. + + The following options are supported: + + --force Redownload packages even if nothing changed since last build + --strict Only download locked packages when using lockfiles + --update-lockfile Update base lockfile to latest packages + --write-lockfile-to=FILE Write updated base lockfile to separate file + --with-cosa-overrides Don't ignore cosa overrides in `overrides/rpm` + --autolock=VERSION If no base lockfile used, create one from any arch build of `VERSION` + EOF } +AUTOLOCK_VERSION= UPDATE_LOCKFILE= OUTPUT_LOCKFILE= IGNORE_COSA_OVERRIDES_ARG=--ignore-cosa-overrides DRY_RUN= +FORCE_ARG= STRICT= rc=0 -options=$(getopt --options h --longoptions help,update-lockfile,dry-run,with-cosa-overrides,write-lockfile-to:,strict -- "$@") || rc=$? +options=$(getopt --options h --longoptions help,update-lockfile,dry-run,with-cosa-overrides,write-lockfile-to:,strict,force,autolock: -- "$@") || rc=$? [ $rc -eq 0 ] || { print_help exit 1 @@ -61,6 +73,13 @@ while true; do --strict) STRICT=1 ;; + --force) + FORCE_ARG=--force-nocache + ;; + --autolock) + shift; + AUTOLOCK_VERSION=$1 + ;; --) shift break @@ -79,31 +98,53 @@ fi prepare_build -args= +lock_args= +extra_args= + +# Apply autolock from another build for this version if no base lockfile +# exists. Do this before so that overrides come after. +if [ -n "${AUTOLOCK_VERSION}" ]; then + if [ -f "${manifest_lock}" ]; then + fatal "ERROR: Requested --autolock, but base lockfile found" + fi + autolockfile=$(generate_autolock "${AUTOLOCK_VERSION}") + if [ -z "${autolockfile}" ]; then + fatal "ERROR: Requested --autolock, but no generated lockfile found" + fi + lock_args+=" --ex-lockfile=${autolockfile}" +fi + if [ -n "${UPDATE_LOCKFILE}" ]; then # Put this under tmprepo so it gets automatically chown'ed if needed - args="--ex-write-lockfile-to=${tmprepo}/tmp/manifest-lock.json" + extra_args+=" --ex-write-lockfile-to=${tmprepo}/tmp/manifest-lock.json" # Include the overrides in the resulting lockfile here; otherwise, we # might not even be able to get a depsolve solely from the non-lockfile # repos. for lock in "${manifest_lock_overrides}" "${manifest_lock_arch_overrides}"; do if [ -f "${lock}" ]; then - args+=" --ex-lockfile=${lock}" + lock_args+=" --ex-lockfile=${lock}" fi done else for lock in "${manifest_lock}" "${manifest_lock_overrides}" "${manifest_lock_arch_overrides}"; do if [ -f "${lock}" ]; then - args+=" --ex-lockfile=${lock}" + lock_args+=" --ex-lockfile=${lock}" fi done fi if [ -n "${DRY_RUN}" ]; then - args="${args} --dry-run" + extra_args+=" --dry-run" fi + if [ -n "${STRICT}" ]; then - args="${args} --ex-lockfile-strict" + if [ -n "${AUTOLOCK_VERSION}" ]; then + # They're theoretically independent, but in practice it's unlikely + # that an autolockfile will include all the packages needed to satisfy + # --strict. So let's just catch it early. + fatal "ERROR: Can't use --autolock and --strict together" + fi + extra_args+=" --ex-lockfile-strict" fi # By default, we ignore cosa overrides since they're temporary. With @@ -114,7 +155,7 @@ fi prepare_compose_overlays ${IGNORE_COSA_OVERRIDES_ARG} # shellcheck disable=SC2086 -runcompose_tree --download-only ${args} +runcompose_tree --download-only ${lock_args} ${extra_args} ${FORCE_ARG} # This stamp file signifies we successfully fetched once; it's # validated in cmd-build. touch "${fetch_stamp}" diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 8a090447..4621770f 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -519,6 +519,45 @@ strip_out_lockfile_digests() { mv "$1.tmp" "$1" } +# Strip out the arch field from lockfiles to make them archless. +strip_out_lockfile_arches() { + python3 -c ' +import sys, json +lockfile = json.load(sys.stdin) +new_packages = {} +for pkg, meta in lockfile["packages"].items(): + evra = meta.get("evra") + if evra is None or evra.endswith(".noarch"): + new_packages[pkg] = meta + continue + new_packages[pkg] = { + "evr": evra[:evra.rindex(".")] + } +lockfile["packages"] = new_packages +json.dump(lockfile, sys.stdout) +' < "$1" > "$1.tmp" + mv "$1.tmp" "$1" +} + +# Create the autolock for a given version +generate_autolock() { + local version=$1; shift + local autolockfile="${tmprepo}/tmp/manifest-autolock-${version}.json" + if [ ! -f "${autolockfile}" ]; then + # Just use the first arch available. In practice, this'll likely always be + # x86_64, but it should work just as well if not. + local lockfile + lockfile=$(find "${workdir}/builds/${version}/" -name 'manifest-lock.generated.*.json' -print -quit 2>/dev/null) + if [ -z "${lockfile}" ]; then + # no generated lockfiles found; we can't autolock + return + fi + cp "${lockfile}" "${autolockfile}" + strip_out_lockfile_arches "${autolockfile}" + fi + echo "${autolockfile}" +} + json_key() { jq -r ".[\"$1\"]" < "${builddir}/meta.json" } -- Gitee