From c99522152377b3145db9b6eeeacef9dc09d658c5 Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Fri, 12 Jul 2024 17:13:54 +0800 Subject: [PATCH] sync a ton of chores from upstream v0.16.0 [upstream] 04a5f2269 Delete `cosa-runc` 0a36f1ba7 cosalib/build: don't call os.unlink() twice 951d2496f cosalib/build: only take artifact lock when actually building it 786e87b59 cosalib: fix init_build_meta_json for multi-arch ed87ce543 cosalib/cli: drop unused --dump argument 679e8c8c4 *: fix tyops cc605eb2f cp-reflink: re-enable it 453afe6aa src/download-overrides: Fix flake8 CI errors 2f2fec655 scripts: Move download-overrides.py to coreos-assembler/main/src 86943ca02 live-iso: Write kernel (and hmac) to /boot too 0eb25d1c7 grub.cfg: drop `set pager=1` 6f4e34351 src/grub.cfg: Source user.cfg in grub.cfg 3a9cbd58e build: Add support for `rootfs-args` f53bc8fae Simplify `add-root-cas` script and rename ad62159b9 Dockerfile: allow root group to add root certificates --- src/cmd-runc | 50 ------------------------- src/cosalib/build.py | 5 ++- src/cosalib/builds.py | 17 +++++---- src/cosalib/cli.py | 5 +-- src/cp-reflink | 4 +- src/download-overrides.py | 75 ++++++++++++++++++++++++++++++++++++++ src/gf-get-kargs | 2 +- src/gf-mksquashfs | 11 +++++- src/grub.cfg | 7 +++- src/image-default.yaml | 3 +- src/update-ca-trust-unpriv | 17 +++++++++ 11 files changed, 124 insertions(+), 72 deletions(-) delete mode 100644 src/cmd-runc create mode 100755 src/download-overrides.py create mode 100755 src/update-ca-trust-unpriv diff --git a/src/cmd-runc b/src/cmd-runc deleted file mode 100644 index 00c3596b..00000000 --- a/src/cmd-runc +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# Spawn the current build as a container. This can be -# very useful for "let me see the filesystem layout" -# type things or `rpm -q`, however note today that the -# /var/lib/rpm -> /usr/share/rpm symlink is made by systemd-tmpfiles, -# so you'll currently need to do `rpm --dbpath=/usr/share/rpm -q kernel` -# for example. - -dn=$(dirname "$0") -# shellcheck source=src/cmdlib.sh -. "${dn}"/cmdlib.sh - -if ! has_privileges; then - # See https://github.com/kubernetes/enhancements/issues/127 - # but even then what we really want in a pipeline is probably - # more to make a real container image and schedule it as - # a separate pod. - fatal "Must have privileges currently" -fi - -BUILDID=latest -if ! [ -d "builds/${BUILDID}" ]; then - die "No builds/${BUILDID}" -fi -builddir=$(get_build_dir "${BUILDID}") - -commit=$(jq -r '.["ostree-commit"]' < "${builddir}/meta.json") - -tmproot=tmp/run-bwrap -tmprootcommit=tmp/run-bwrap/.commit -if ! [ -f "${tmprootcommit}" ] || ! [ "$(cat ${tmprootcommit})" = "${commit}" ]; then - echo "Checking out ${commit}" - sudo rm "${tmproot}" -rf - sudo ostree --repo=cache/repo-build checkout -UH "${commit}" "${tmproot}" - echo "${commit}" | sudo tee "${tmprootcommit}" -fi -cd "${tmproot}" -if [ "$#" = "0" ]; then - set -- bash -fi -set -x -exec bwrap --unshare-all --dev /dev --proc /proc --chdir / \ - --ro-bind usr /usr --ro-bind usr/etc /etc --dir /tmp \ - --ro-bind / /host \ - --tmpfs /var/tmp --tmpfs /run \ - --symlink usr/lib /lib \ - --symlink usr/lib64 /lib64 \ - --symlink usr/bin /bin \ - --symlink usr/sbin /sbin -- "$@" diff --git a/src/cosalib/build.py b/src/cosalib/build.py index 9f735789..49d5baab 100644 --- a/src/cosalib/build.py +++ b/src/cosalib/build.py @@ -140,8 +140,6 @@ class _Build: self.summary, self.build_name.upper(), self.basearch, self.build_id) - self.set_token() - def __del__(self): try: tmpdir = getattr(self, "_tmpdir", None) @@ -184,6 +182,7 @@ class _Build: tf = getattr(self, "_token_file", None) if tf: os.unlink(tf) + setattr(self, "_token_file", None) @property def workdir(self): @@ -411,7 +410,9 @@ class _Build: :raises: NotImplementedError """ log.info("Processing the build artifacts") + self.set_token() self._build_artifacts(*args, **kwargs) + self.unset_token() log.info("Finished building artifacts") if len(self._found_files.keys()) == 0: log.warn("There were no files found after building") diff --git a/src/cosalib/builds.py b/src/cosalib/builds.py index 5fd70222..4e8ebe41 100644 --- a/src/cosalib/builds.py +++ b/src/cosalib/builds.py @@ -131,14 +131,15 @@ class Builds: # pragma: nocover genver_key = 'coreos-assembler.image-genver' if not self.is_empty(): previous_buildid = parent_build or self.get_latest() - metapath = self.get_build_dir(previous_buildid) + '/meta.json' - with open(metapath) as f: - previous_buildmeta = json.load(f) - previous_commit = previous_buildmeta['ostree-commit'] - previous_image_genver = int(previous_buildmeta.get(genver_key, 0)) - if previous_commit == ostree_commit: - image_genver = previous_image_genver + 1 - buildid = f"{version}-{image_genver}" + if get_basearch() in self.get_build_arches(previous_buildid): + metapath = self.get_build_dir(previous_buildid) + '/meta.json' + with open(metapath) as f: + previous_buildmeta = json.load(f) + previous_commit = previous_buildmeta['ostree-commit'] + previous_image_genver = int(previous_buildmeta.get(genver_key, 0)) + if previous_commit == ostree_commit: + image_genver = previous_image_genver + 1 + buildid = f"{version}-{image_genver}" meta = { 'buildid': buildid, genver_key: image_genver diff --git a/src/cosalib/cli.py b/src/cosalib/cli.py index 707d4056..c55bcaad 100644 --- a/src/cosalib/cli.py +++ b/src/cosalib/cli.py @@ -150,10 +150,7 @@ class BuildCli(Cli): help='Override build id, defaults to latest') self.add_argument( '--buildroot', env_var="BUILD_ROOT", default='builds', - help='Build diretory') - self.add_argument( - '--dump', default=False, action='store_true', - help='Dump the manfiest and exit') + help='Build directory') self.add_argument( '--schema', env_var="META_SCHEMA", default='/usr/lib/coreos-assembler/v1.json', diff --git a/src/cp-reflink b/src/cp-reflink index dbfe1d00..70a7a63b 100755 --- a/src/cp-reflink +++ b/src/cp-reflink @@ -1,4 +1,2 @@ #!/bin/bash -# XXX: disable reflinks for now due to possible corruption: -# https://github.com/coreos/coreos-assembler/pull/935 -exec cp -a --reflink=never "$@" +exec cp -a --reflink=auto "$@" diff --git a/src/download-overrides.py b/src/download-overrides.py new file mode 100755 index 00000000..a984693a --- /dev/null +++ b/src/download-overrides.py @@ -0,0 +1,75 @@ +#!/usr/bin/python3 + +import dnf.subject +import hawkey +import os +import yaml +import subprocess + +arch = os.uname().machine + +# this was partially copied from coreos-koji-tagger + + +def get_rpminfo(string: str) -> str: + form = hawkey.FORM_NEVRA + + # get a hawkey.Subject object for the string + subject = dnf.subject.Subject(string) # returns hawkey.Subject + + # get a list of hawkey.NEVRA objects that are the possibilities + nevras = subject.get_nevra_possibilities(forms=form) + + # return the first hawkey.NEVRA item in the list of possibilities + rpminfo = nevras[0] + return rpminfo + + +def is_override_lockfile(filename: str) -> bool: + return (filename == "manifest-lock.overrides.yaml" or + filename == f'manifest-lock.overrides.{arch}.yaml') + + +def assert_epochs_match(overrides_epoch: int, rpmfile_epoch: str): + # normalize the input into a string + if overrides_epoch is None: + normalized_overrides_epoch = '(none)' # matches rpm -qp --queryformat='%{E}' + else: + normalized_overrides_epoch = str(overrides_epoch) + if normalized_overrides_epoch != rpmfile_epoch: + raise Exception(f"Epoch mismatch between downloaded rpm ({rpmfile_epoch})" + f" and overrides file entry ({overrides_epoch})") + + +assert os.path.isdir("builds"), "Missing builds/ dir; is this a cosa workdir?" + +rpms = set() +os.makedirs('overrides/rpm', exist_ok=True) +for filename in os.listdir(os.path.join("src/config")): + if is_override_lockfile(filename): + with open(f'src/config/{filename}') as f: + lockfile = yaml.safe_load(f) + if lockfile is None or 'packages' not in lockfile: + continue + for pkg, pkgobj in lockfile['packages'].items(): + if 'evr' in pkgobj: + rpminfo = get_rpminfo(f"{pkg}-{pkgobj['evr']}.{arch}") + else: + rpminfo = get_rpminfo(f"{pkg}-{pkgobj['evra']}") + rpmnvra = f"{rpminfo.name}-{rpminfo.version}-{rpminfo.release}.{rpminfo.arch}" + rpms.add(rpmnvra) + subprocess.check_call(['koji', 'download-build', '--rpm', rpmnvra], cwd='overrides/rpm') + # Make sure the epoch matches what was in the overrides file + # otherwise we can get errors: https://github.com/coreos/fedora-coreos-config/pull/293 + cp = subprocess.run(['rpm', '-qp', '--queryformat', '%{E}', f'{rpmnvra}.rpm'], + check=True, + capture_output=True, + cwd='overrides/rpm') + rpmfile_epoch = cp.stdout.decode('utf-8') + assert_epochs_match(rpminfo.epoch, rpmfile_epoch) + +if not rpms: + print("No overrides; exiting.") +else: + for rpm in rpms: + print(f'Downloaded {rpm} to overrides dir') diff --git a/src/gf-get-kargs b/src/gf-get-kargs index 024ec2e0..e325e8e3 100755 --- a/src/gf-get-kargs +++ b/src/gf-get-kargs @@ -21,7 +21,7 @@ fi set -x -coreos_gf_run_mount "${src}" +coreos_gf_run_mount "${src}" --ro coreos_gf glob read-file /boot/loader/entries/ostree*conf | \ sed -e '/^options/!d' -e 's/^options\s*//' diff --git a/src/gf-mksquashfs b/src/gf-mksquashfs index 6fcc8bcf..fc30a76a 100755 --- a/src/gf-mksquashfs +++ b/src/gf-mksquashfs @@ -30,14 +30,21 @@ set -x # https://github.com/coreos/coreos-assembler/pull/394 tmpd=$(mktemp -tdp "$(dirname "${dest}")" gf-mksquashfs.XXXXXX) tmp_dest=${tmpd}/image.squashfs -coreos_gf_run_mount "${src}" +coreos_gf_run_mount "${src}" --ro # Remove the sysroot=readonly flag, see https://github.com/coreos/fedora-coreos-tracker/issues/589 coreos_gf download /ostree/repo/config "${tmpd}/config" grep -v readonly=true "${tmpd}/config" > "${tmpd}/config.new" coreos_gf upload "${tmpd}/config.new" /ostree/repo/config -coreos_gf mksquashfs / "${tmp_dest}" "compress:${compression}" +# And ensure that the kernel binary and hmac file is in the place that dracut +# expects it to be; xref https://issues.redhat.com/browse/OCPBUGS-15843 +kernel_binary=$(coreos_gf glob-expand /boot/ostree/*/vmlinuz*) +kernel_hmac=$(coreos_gf glob-expand /boot/ostree/*/.*.hmac) +coreos_gf ln "${kernel_hmac}" "/boot/$(basename "${kernel_hmac}")" +coreos_gf ln "${kernel_binary}" "/boot/$(basename "${kernel_binary}")" + +coreos_gf mksquashfs / "${tmp_dest}" "compress:${compression}" coreos_gf_shutdown mv "${tmp_dest}" "${dest}" diff --git a/src/grub.cfg b/src/grub.cfg index c1cbc6d1..775289af 100644 --- a/src/grub.cfg +++ b/src/grub.cfg @@ -1,4 +1,3 @@ -set pager=1 # petitboot doesn't support -e and doesn't support an empty path part if [ -d (md/md-boot)/grub2 ]; then # fcct currently creates /boot RAID with superblock 1.0, which allows @@ -85,4 +84,10 @@ if [ -f "/ignition.firstboot" ]; then set ignition_firstboot="ignition.firstboot ${ignition_network_kcmdline}" fi +# Import user defined configuration +# tracker: https://github.com/coreos/fedora-coreos-tracker/issues/805 +if [ -f $prefix/user.cfg ]; then + source $prefix/user.cfg +fi + blscfg diff --git a/src/image-default.yaml b/src/image-default.yaml index 4c2dde04..a3e65567 100644 --- a/src/image-default.yaml +++ b/src/image-default.yaml @@ -2,7 +2,8 @@ bootfs: "ext4" rootfs: "xfs" - +# Add arguments here that will be passed to e.g. mkfs.xfs +rootfs-args: "" # Set to "true" to enable composefs composefs: false diff --git a/src/update-ca-trust-unpriv b/src/update-ca-trust-unpriv new file mode 100755 index 00000000..47bbd3c9 --- /dev/null +++ b/src/update-ca-trust-unpriv @@ -0,0 +1,17 @@ +#!/bin/bash +set -euo pipefail + +# This runs a subset of what `update-ca-trust` does. Unlike the latter, it runs +# fine unprivileged as long as it has write access to /etc/pki/ca-trust/. + +# Compare to: +# https://src.fedoraproject.org/rpms/ca-certificates/blob/3e2443900394/f/update-ca-trust + +DEST=/etc/pki/ca-trust/extracted + +# Prevent p11-kit from reading user configuration files. +export P11_KIT_NO_USER_CONFIG=1 + +# OpenSSL PEM bundle that includes trust flags +/usr/bin/p11-kit extract --format=openssl-bundle --filter=certificates --overwrite --comment $DEST/openssl/ca-bundle.trust.crt +/usr/bin/p11-kit extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose server-auth $DEST/pem/tls-ca-bundle.pem -- Gitee