diff --git a/src/cmd-runc b/src/cmd-runc deleted file mode 100644 index 00c3596bb6b1be8ade7db5a1b2427c52ebf00517..0000000000000000000000000000000000000000 --- a/src/cmd-runc +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# Spawn the current build as a container. This can be -# very useful for "let me see the filesystem layout" -# type things or `rpm -q`, however note today that the -# /var/lib/rpm -> /usr/share/rpm symlink is made by systemd-tmpfiles, -# so you'll currently need to do `rpm --dbpath=/usr/share/rpm -q kernel` -# for example. - -dn=$(dirname "$0") -# shellcheck source=src/cmdlib.sh -. "${dn}"/cmdlib.sh - -if ! has_privileges; then - # See https://github.com/kubernetes/enhancements/issues/127 - # but even then what we really want in a pipeline is probably - # more to make a real container image and schedule it as - # a separate pod. - fatal "Must have privileges currently" -fi - -BUILDID=latest -if ! [ -d "builds/${BUILDID}" ]; then - die "No builds/${BUILDID}" -fi -builddir=$(get_build_dir "${BUILDID}") - -commit=$(jq -r '.["ostree-commit"]' < "${builddir}/meta.json") - -tmproot=tmp/run-bwrap -tmprootcommit=tmp/run-bwrap/.commit -if ! [ -f "${tmprootcommit}" ] || ! [ "$(cat ${tmprootcommit})" = "${commit}" ]; then - echo "Checking out ${commit}" - sudo rm "${tmproot}" -rf - sudo ostree --repo=cache/repo-build checkout -UH "${commit}" "${tmproot}" - echo "${commit}" | sudo tee "${tmprootcommit}" -fi -cd "${tmproot}" -if [ "$#" = "0" ]; then - set -- bash -fi -set -x -exec bwrap --unshare-all --dev /dev --proc /proc --chdir / \ - --ro-bind usr /usr --ro-bind usr/etc /etc --dir /tmp \ - --ro-bind / /host \ - --tmpfs /var/tmp --tmpfs /run \ - --symlink usr/lib /lib \ - --symlink usr/lib64 /lib64 \ - --symlink usr/bin /bin \ - --symlink usr/sbin /sbin -- "$@" diff --git a/src/cosalib/build.py b/src/cosalib/build.py index 9f735789151bb442e770165b781484b1b191d49f..49d5baabca10fad9f39b1d395e547008cb414cf2 100644 --- a/src/cosalib/build.py +++ b/src/cosalib/build.py @@ -140,8 +140,6 @@ class _Build: self.summary, self.build_name.upper(), self.basearch, self.build_id) - self.set_token() - def __del__(self): try: tmpdir = getattr(self, "_tmpdir", None) @@ -184,6 +182,7 @@ class _Build: tf = getattr(self, "_token_file", None) if tf: os.unlink(tf) + setattr(self, "_token_file", None) @property def workdir(self): @@ -411,7 +410,9 @@ class _Build: :raises: NotImplementedError """ log.info("Processing the build artifacts") + self.set_token() self._build_artifacts(*args, **kwargs) + self.unset_token() log.info("Finished building artifacts") if len(self._found_files.keys()) == 0: log.warn("There were no files found after building") diff --git a/src/cosalib/builds.py b/src/cosalib/builds.py index 5fd70222b331a4e8f3c5b5f541d82b454462b6d8..4e8ebe41c757d090d38bf4b9d9647581890bf51b 100644 --- a/src/cosalib/builds.py +++ b/src/cosalib/builds.py @@ -131,14 +131,15 @@ class Builds: # pragma: nocover genver_key = 'coreos-assembler.image-genver' if not self.is_empty(): previous_buildid = parent_build or self.get_latest() - metapath = self.get_build_dir(previous_buildid) + '/meta.json' - with open(metapath) as f: - previous_buildmeta = json.load(f) - previous_commit = previous_buildmeta['ostree-commit'] - previous_image_genver = int(previous_buildmeta.get(genver_key, 0)) - if previous_commit == ostree_commit: - image_genver = previous_image_genver + 1 - buildid = f"{version}-{image_genver}" + if get_basearch() in self.get_build_arches(previous_buildid): + metapath = self.get_build_dir(previous_buildid) + '/meta.json' + with open(metapath) as f: + previous_buildmeta = json.load(f) + previous_commit = previous_buildmeta['ostree-commit'] + previous_image_genver = int(previous_buildmeta.get(genver_key, 0)) + if previous_commit == ostree_commit: + image_genver = previous_image_genver + 1 + buildid = f"{version}-{image_genver}" meta = { 'buildid': buildid, genver_key: image_genver diff --git a/src/cosalib/cli.py b/src/cosalib/cli.py index 707d4056ae6b1f66036b64406483d29b096c0930..c55bcaad526b58da6b0c6e1bf4522ae3119051aa 100644 --- a/src/cosalib/cli.py +++ b/src/cosalib/cli.py @@ -150,10 +150,7 @@ class BuildCli(Cli): help='Override build id, defaults to latest') self.add_argument( '--buildroot', env_var="BUILD_ROOT", default='builds', - help='Build diretory') - self.add_argument( - '--dump', default=False, action='store_true', - help='Dump the manfiest and exit') + help='Build directory') self.add_argument( '--schema', env_var="META_SCHEMA", default='/usr/lib/coreos-assembler/v1.json', diff --git a/src/cp-reflink b/src/cp-reflink index dbfe1d0044de0073cce3555c5390afacd836d35f..70a7a63bec376c8340978de3db389cd32efffb1b 100755 --- a/src/cp-reflink +++ b/src/cp-reflink @@ -1,4 +1,2 @@ #!/bin/bash -# XXX: disable reflinks for now due to possible corruption: -# https://github.com/coreos/coreos-assembler/pull/935 -exec cp -a --reflink=never "$@" +exec cp -a --reflink=auto "$@" diff --git a/src/download-overrides.py b/src/download-overrides.py new file mode 100755 index 0000000000000000000000000000000000000000..a984693a9ad44fd27e25608e0946eae1067ad1af --- /dev/null +++ b/src/download-overrides.py @@ -0,0 +1,75 @@ +#!/usr/bin/python3 + +import dnf.subject +import hawkey +import os +import yaml +import subprocess + +arch = os.uname().machine + +# this was partially copied from coreos-koji-tagger + + +def get_rpminfo(string: str) -> str: + form = hawkey.FORM_NEVRA + + # get a hawkey.Subject object for the string + subject = dnf.subject.Subject(string) # returns hawkey.Subject + + # get a list of hawkey.NEVRA objects that are the possibilities + nevras = subject.get_nevra_possibilities(forms=form) + + # return the first hawkey.NEVRA item in the list of possibilities + rpminfo = nevras[0] + return rpminfo + + +def is_override_lockfile(filename: str) -> bool: + return (filename == "manifest-lock.overrides.yaml" or + filename == f'manifest-lock.overrides.{arch}.yaml') + + +def assert_epochs_match(overrides_epoch: int, rpmfile_epoch: str): + # normalize the input into a string + if overrides_epoch is None: + normalized_overrides_epoch = '(none)' # matches rpm -qp --queryformat='%{E}' + else: + normalized_overrides_epoch = str(overrides_epoch) + if normalized_overrides_epoch != rpmfile_epoch: + raise Exception(f"Epoch mismatch between downloaded rpm ({rpmfile_epoch})" + f" and overrides file entry ({overrides_epoch})") + + +assert os.path.isdir("builds"), "Missing builds/ dir; is this a cosa workdir?" + +rpms = set() +os.makedirs('overrides/rpm', exist_ok=True) +for filename in os.listdir(os.path.join("src/config")): + if is_override_lockfile(filename): + with open(f'src/config/{filename}') as f: + lockfile = yaml.safe_load(f) + if lockfile is None or 'packages' not in lockfile: + continue + for pkg, pkgobj in lockfile['packages'].items(): + if 'evr' in pkgobj: + rpminfo = get_rpminfo(f"{pkg}-{pkgobj['evr']}.{arch}") + else: + rpminfo = get_rpminfo(f"{pkg}-{pkgobj['evra']}") + rpmnvra = f"{rpminfo.name}-{rpminfo.version}-{rpminfo.release}.{rpminfo.arch}" + rpms.add(rpmnvra) + subprocess.check_call(['koji', 'download-build', '--rpm', rpmnvra], cwd='overrides/rpm') + # Make sure the epoch matches what was in the overrides file + # otherwise we can get errors: https://github.com/coreos/fedora-coreos-config/pull/293 + cp = subprocess.run(['rpm', '-qp', '--queryformat', '%{E}', f'{rpmnvra}.rpm'], + check=True, + capture_output=True, + cwd='overrides/rpm') + rpmfile_epoch = cp.stdout.decode('utf-8') + assert_epochs_match(rpminfo.epoch, rpmfile_epoch) + +if not rpms: + print("No overrides; exiting.") +else: + for rpm in rpms: + print(f'Downloaded {rpm} to overrides dir') diff --git a/src/gf-get-kargs b/src/gf-get-kargs index 024ec2e02225f8df1e14a9c1256dcc365a2d52dc..e325e8e36722afde70b55f89a96863034e4eaf73 100755 --- a/src/gf-get-kargs +++ b/src/gf-get-kargs @@ -21,7 +21,7 @@ fi set -x -coreos_gf_run_mount "${src}" +coreos_gf_run_mount "${src}" --ro coreos_gf glob read-file /boot/loader/entries/ostree*conf | \ sed -e '/^options/!d' -e 's/^options\s*//' diff --git a/src/gf-mksquashfs b/src/gf-mksquashfs index 6fcc8bcf03a03e152e5e02dbef09ecd1c6d273c6..fc30a76a478345242a7b80b63378007347a6cb66 100755 --- a/src/gf-mksquashfs +++ b/src/gf-mksquashfs @@ -30,14 +30,21 @@ set -x # https://github.com/coreos/coreos-assembler/pull/394 tmpd=$(mktemp -tdp "$(dirname "${dest}")" gf-mksquashfs.XXXXXX) tmp_dest=${tmpd}/image.squashfs -coreos_gf_run_mount "${src}" +coreos_gf_run_mount "${src}" --ro # Remove the sysroot=readonly flag, see https://github.com/coreos/fedora-coreos-tracker/issues/589 coreos_gf download /ostree/repo/config "${tmpd}/config" grep -v readonly=true "${tmpd}/config" > "${tmpd}/config.new" coreos_gf upload "${tmpd}/config.new" /ostree/repo/config -coreos_gf mksquashfs / "${tmp_dest}" "compress:${compression}" +# And ensure that the kernel binary and hmac file is in the place that dracut +# expects it to be; xref https://issues.redhat.com/browse/OCPBUGS-15843 +kernel_binary=$(coreos_gf glob-expand /boot/ostree/*/vmlinuz*) +kernel_hmac=$(coreos_gf glob-expand /boot/ostree/*/.*.hmac) +coreos_gf ln "${kernel_hmac}" "/boot/$(basename "${kernel_hmac}")" +coreos_gf ln "${kernel_binary}" "/boot/$(basename "${kernel_binary}")" + +coreos_gf mksquashfs / "${tmp_dest}" "compress:${compression}" coreos_gf_shutdown mv "${tmp_dest}" "${dest}" diff --git a/src/grub.cfg b/src/grub.cfg index c1cbc6d1eccdb14346c253e19fcfe48dc90b2d5a..775289aff085b5667b4797fc15e250a89c3b3cf3 100644 --- a/src/grub.cfg +++ b/src/grub.cfg @@ -1,4 +1,3 @@ -set pager=1 # petitboot doesn't support -e and doesn't support an empty path part if [ -d (md/md-boot)/grub2 ]; then # fcct currently creates /boot RAID with superblock 1.0, which allows @@ -85,4 +84,10 @@ if [ -f "/ignition.firstboot" ]; then set ignition_firstboot="ignition.firstboot ${ignition_network_kcmdline}" fi +# Import user defined configuration +# tracker: https://github.com/coreos/fedora-coreos-tracker/issues/805 +if [ -f $prefix/user.cfg ]; then + source $prefix/user.cfg +fi + blscfg diff --git a/src/image-default.yaml b/src/image-default.yaml index 4c2dde046e6c008b1d60162db6cdcd2397720bf0..a3e65567299cbfdf2acdea71514b42123482a454 100644 --- a/src/image-default.yaml +++ b/src/image-default.yaml @@ -2,7 +2,8 @@ bootfs: "ext4" rootfs: "xfs" - +# Add arguments here that will be passed to e.g. mkfs.xfs +rootfs-args: "" # Set to "true" to enable composefs composefs: false diff --git a/src/update-ca-trust-unpriv b/src/update-ca-trust-unpriv new file mode 100755 index 0000000000000000000000000000000000000000..47bbd3c9ade6dc77b3c517a129ae13e1e24c579a --- /dev/null +++ b/src/update-ca-trust-unpriv @@ -0,0 +1,17 @@ +#!/bin/bash +set -euo pipefail + +# This runs a subset of what `update-ca-trust` does. Unlike the latter, it runs +# fine unprivileged as long as it has write access to /etc/pki/ca-trust/. + +# Compare to: +# https://src.fedoraproject.org/rpms/ca-certificates/blob/3e2443900394/f/update-ca-trust + +DEST=/etc/pki/ca-trust/extracted + +# Prevent p11-kit from reading user configuration files. +export P11_KIT_NO_USER_CONFIG=1 + +# OpenSSL PEM bundle that includes trust flags +/usr/bin/p11-kit extract --format=openssl-bundle --filter=certificates --overwrite --comment $DEST/openssl/ca-bundle.trust.crt +/usr/bin/p11-kit extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose server-auth $DEST/pem/tls-ca-bundle.pem