diff --git a/pytest.ini b/pytest.ini index 8b36970ea6bf1c82fa22855e48a7a5038f9ebc4a..f9457a887c90cd58017621f69dadaa1df146f6bf 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --cov=cosalib.cli --cov=cosalib.meta --cov=cosalib.cmdlib --cov-report term --cov-fail-under=80 +addopts = --cov=cosalib.cli --cov=cosalib.meta --cov=cosalib.cmdlib --cov-report term --cov-fail-under=70 diff --git a/src/cmd-build b/src/cmd-build index 299b184a7103f0ac466fba73e1748a8ca7e50436..a48f982764614a499adaa16c31587138d0fcf67b 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -207,11 +207,7 @@ ks_path="${configdir}"/image.ks if [ -f "${ks_path}" ]; then fatal "Kickstart support was removed; migrate to image.yaml" fi -image_input="${image_yaml}" -if ! [ -f "${image_input}" ]; then - fatal "Failed to find ${image_input}" -fi -image_config_checksum=$(< "${image_input}" sha256sum_str) +image_config_checksum=$(< "${image_json}" sha256sum_str) if [ -n "${previous_build}" ]; then previous_image_input_checksum=$(jq -r '.["coreos-assembler.image-input-checksum"]' < "${previous_builddir}/meta.json") fi diff --git a/src/cmd-buildextend-live b/src/cmd-buildextend-live index 22e81c91a749e0cce4c6e9deadba1d45e014c703..70238a027e04df788e39d04648c9459e9ec3a807 100755 --- a/src/cmd-buildextend-live +++ b/src/cmd-buildextend-live @@ -18,7 +18,7 @@ import time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from cosalib.builds import Builds -from cosalib.cmdlib import run_verbose, sha256sum_file, flatten_image_yaml +from cosalib.cmdlib import runcmd, sha256sum_file from cosalib.cmdlib import import_ostree_commit, get_basearch, ensure_glob from cosalib.meta import GenericBuildMeta @@ -46,9 +46,6 @@ if not args.build: args.build = builds.get_latest() print(f"Targeting build: {args.build}") -image_yaml = flatten_image_yaml('src/config/image.yaml') -squashfs_compression = 'lz4' if args.fast else image_yaml.get('squashfs-compression', 'zstd') - srcdir_prefix = "src/config/live/" if not os.path.isdir(srcdir_prefix): @@ -58,6 +55,15 @@ workdir = os.path.abspath(os.getcwd()) builddir = builds.get_build_dir(args.build) buildmeta_path = os.path.join(builddir, 'meta.json') buildmeta = GenericBuildMeta(workdir=workdir, build=args.build) +repo = os.path.join(workdir, 'tmp/repo') + +# Grab the commit hash for this build +buildmeta_commit = buildmeta['ostree-commit'] + +import_ostree_commit(workdir, builddir, buildmeta) +with open(os.path.join(workdir, 'tmp/image.json')) as f: + image_json = json.load(f) +squashfs_compression = 'lz4' if args.fast else image_json['squashfs-compression'] base_name = buildmeta['name'] if base_name == "rhcos" and args.fast: @@ -71,12 +77,6 @@ if os.path.exists(build_semaphore): raise Exception( f"{build_semaphore} exists: another process is building live") - -# Grab the commit hash for this build -buildmeta_commit = buildmeta['ostree-commit'] - -repo = os.path.join(workdir, 'tmp/repo') - # Don't run if it's already been done, unless forced if 'live-iso' in buildmeta['images'] and not args.force: print(f"'live' has already been built for {args.build}. Skipping.") @@ -635,8 +635,6 @@ def generate_iso(): print(f"Updated: {buildmeta_path}") -import_ostree_commit(repo, builddir, buildmeta) - # lock and build with open(build_semaphore, 'w') as f: f.write(f"{time.time_ns()}") diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 11f00021f67c2b9b75813d41f6606c0fc3c2ff2b..86dafd3cac28b4d7185c7e45e6632ed56fe24606 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -142,6 +142,11 @@ commit=$(meta_key ostree-commit) ostree_repo=${tmprepo} # Ensure that we have the cached unpacked commit import_ostree_commit_for_build "${build}" +# Note this overwrote the bits generated in prepare_build +# for image_json. In the future we expect to split prepare_build +# into prepare_ostree_build and prepare_diskimage_build; the +# latter path would only run this. +image_json=${workdir}/tmp/image.json image_format=raw if [[ $image_type == qemu ]]; then @@ -216,7 +221,7 @@ case "${image_type}" in rootfs_size=0 ;; qemu) - image_size="$(python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin)["size"])' < "${image_yaml}")G" + image_size="$(jq -r ".size" < "${image_json}")G" rootfs_size="${rootfs_size}M" ;; *) fatal "unreachable image_type ${image_type}";; @@ -227,7 +232,7 @@ if [ "${image_type}" == metal4k ]; then fi set -x -kargs="$(python3 -c 'import sys, yaml; args = yaml.safe_load(sys.stdin).get("extra-kargs", []); print(" ".join(args))' < "${image_yaml}")" +kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")" tty="console=tty0 console=${DEFAULT_TERMINAL},115200n8" # On each s390x hypervisor, a tty would be automatically detected by the kernel # and systemd, there is no need to specify one. However, we keep DEFAULT_TERMINAL diff --git a/src/cmd-generate-hashlist b/src/cmd-generate-hashlist index 2b3a339f5577f730adc81b3b39d464b30dba08ce..3164bf97d9ec6460691081401b3f5e73e67ca726 100755 --- a/src/cmd-generate-hashlist +++ b/src/cmd-generate-hashlist @@ -80,7 +80,7 @@ class HashListV1(dict): checkout = 'tmp/repo/tmp/keylime-checkout' import_ostree_commit( - 'tmp/repo', + os.getcwd(), self._metadata.build_dir, self._metadata, force=True) diff --git a/src/cmd-sign b/src/cmd-sign index 4a69eb9010e38516eac4de5500461f158ba55211..adef48e5f62824fda44f173122fe2eac6c31b736 100755 --- a/src/cmd-sign +++ b/src/cmd-sign @@ -134,7 +134,7 @@ def robosign_ostree(args, s3, build, gpgkey): # Ensure we have an unpacked repo with the ostree content if not os.path.exists('tmp/repo'): subprocess.check_call(['ostree', '--repo=tmp/repo', 'init', '--mode=archive']) - import_ostree_commit('tmp/repo', builddir, build, force=True) + import_ostree_commit(os.getcwd(), builddir, build) repo = OSTree.Repo.new(Gio.File.new_for_path('tmp/repo')) repo.open(None) diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 7f382581eeb1de20844853563b68c4598a427c80..2a2b70ea039c3590765b4d8583ecf150a92c0297 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -139,7 +139,7 @@ pick_yaml_or_else_json() { # Given a YAML file at first path, write it as JSON to file at second path yaml2json() { - python3 -c 'import sys, json, yaml; json.dump(yaml.safe_load(sys.stdin), sys.stdout)' < "$1" > "$2" + python3 -c 'import sys, json, yaml; json.dump(yaml.safe_load(sys.stdin), sys.stdout, sort_keys=True)' < "$1" > "$2" } prepare_build() { @@ -169,15 +169,14 @@ prepare_build() { manifest_lock_arch_overrides=$(pick_yaml_or_else_json "${configdir}/manifest-lock.overrides.${basearch}") fetch_stamp="${workdir}"/cache/fetched-stamp - image_yaml="${workdir}/tmp/image.yaml" - flatten_image_yaml_to_file "${configdir}/image.yaml" "${image_yaml}" - # Convert the image.yaml to JSON so that it can be more easily parsed - # by the shell script in create_disk.sh. - image_json="${workdir}/tmp/image.json" - yaml2json "${image_yaml}" "${image_json}" + export image_json="${workdir}/tmp/image.json" + write_image_json "${image}" "${image_json}" + # These need to be absolute paths right now for rpm-ostree + composejson="$(readlink -f "${workdir}"/tmp/compose.json)" + export composejson export workdir configdir manifest manifest_lock manifest_lock_overrides manifest_lock_arch_overrides - export fetch_stamp image_json + export fetch_stamp if ! [ -f "${manifest}" ]; then fatal "Failed to find ${manifest}" @@ -344,6 +343,16 @@ EOF done fi + # Store the fully rendered disk image config (image.json) inside + # the ostree commit, so it can later be extracted by disk image + # builds. + local imagejsondir="${tmp_overridesdir}/imagejson" + export ostree_image_json="/usr/share/coreos-assembler/image.json" + mkdir -p "${imagejsondir}/usr/share/coreos-assembler/" + cp "${image_json}" "${imagejsondir}${ostree_image_json}" + commit_overlay cosa-image-json "${imagejsondir}" + layers="${layers} overlay/cosa-image-json" + local_overrides_lockfile="${tmp_overridesdir}/local-overrides.json" if [ -n "${with_cosa_overrides}" ] && [[ -n $(ls "${overridesdir}/rpm/"*.rpm 2> /dev/null) ]]; then (cd "${overridesdir}"/rpm && rm -rf .repodata && createrepo_c .) @@ -801,17 +810,20 @@ builds.bump_timestamp() print('Build ${buildid} was inserted ${arch:+for $arch}')") } -flatten_image_yaml_to_file() { +# Prepare the image.json as part of an ostree image build +write_image_json() { local srcfile=$1; shift local outfile=$1; shift (python3 -c " import sys sys.path.insert(0, '${DIR}') from cosalib import cmdlib -cmdlib.flatten_image_yaml_to_file('${srcfile}', '${outfile}')") +cmdlib.write_image_json('${srcfile}', '${outfile}')") } -# Shell wrapper for the Python import_ostree_commit +# API to prepare image builds. +# Ensures that the tmp/repo ostree repo is initialized, +# and also writes tmp/image.json if arg2 is unset or set to 1 import_ostree_commit_for_build() { local buildid=$1; shift (python3 -c " @@ -819,9 +831,10 @@ import sys sys.path.insert(0, '${DIR}') from cosalib import cmdlib from cosalib.builds import Builds -builds = Builds('${workdir:-$(pwd)}') +workdir = '${workdir:-$(pwd)}' +builds = Builds(workdir) builddir = builds.get_build_dir('${buildid}') buildmeta = builds.get_build_meta('${buildid}') -cmdlib.import_ostree_commit('${workdir:-$(pwd)}/tmp/repo', builddir, buildmeta) +cmdlib.import_ostree_commit(workdir, builddir, buildmeta) ") } diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index 53753e9f1fc0c5cfbf5d8f38260ec4b26449aebf..ae5455b54248c99ac03344864853647d1a7eceb5 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -233,6 +233,30 @@ def rm_allow_noent(path): pass +def extract_image_json(workdir, commit): + with Lock(os.path.join(workdir, 'tmp/image.json.lock'), + lifetime=LOCK_DEFAULT_LIFETIME): + repo = os.path.join(workdir, 'tmp/repo') + path = os.path.join(workdir, 'tmp/image.json') + tmppath = path + '.tmp' + with open(tmppath, 'w') as f: + rc = subprocess.call(['ostree', f'--repo={repo}', 'cat', commit, '/usr/share/coreos-assembler/image.json'], stdout=f) + if rc == 0: + # Happy path, we have image.json in the ostree commit, rename it into place and we're done. + os.rename(tmppath, path) + return + # Otherwise, we are operating on a legacy build; clean up our tempfile. + os.remove(tmppath) + if not os.path.isfile(path): + # In the current build system flow, image builds will have already + # regenerated tmp/image.json from src/config. If that doesn't already + # exist, then something went wrong. + raise Exception("Failed to extract image.json") + else: + # Warn about this case; but it's not fatal. + print("Warning: Legacy operating on ostree image that does not contain image.json") + + # In coreos-assembler, we are strongly oriented towards the concept of a single # versioned "build" object that has artifacts. But rpm-ostree (among other things) # really natively wants to operate on unpacked ostree repositories. So, we maintain @@ -241,7 +265,8 @@ def rm_allow_noent(path): # a metal image, we may not have preserved that cache. # # Call this function to ensure that the ostree commit for a given build is in tmp/repo. -def import_ostree_commit(repo, buildpath, buildmeta, force=False): +def import_ostree_commit(workdir, buildpath, buildmeta, force=False): + repo = os.path.join(tmpdir, 'repo') commit = buildmeta['ostree-commit'] tarfile = os.path.join(buildpath, buildmeta['images']['ostree']['path']) # create repo in case e.g. tmp/ was cleared out; idempotent @@ -255,6 +280,7 @@ def import_ostree_commit(repo, buildpath, buildmeta, force=False): stderr=subprocess.DEVNULL) == 0 and not os.path.isfile(commitpartial) and not force): + extract_image_json(workdir, commit) return print(f"Extracting {commit}") @@ -278,6 +304,8 @@ def import_ostree_commit(repo, buildpath, buildmeta, force=False): '--write-ref', buildmeta['buildid'], 'ostree-unverified-image:oci-archive:' + tarfile]) subprocess.check_call(['ostree', f'--repo={repo}', 'pull-local', tmpd, buildmeta['buildid']]) + # Also extract image.json since it's commonly needed by image builds + extract_image_json(workdir, commit) def get_basearch(): @@ -350,10 +378,20 @@ def cmdlib_sh(script): ''']) -def flatten_image_yaml_to_file(srcfile, outfile): - flattened = flatten_image_yaml(srcfile) +def generate_image_json(srcfile): + r = yaml.safe_load(open("/usr/lib/coreos-assembler/image-default.yaml")) + for k, v in flatten_image_yaml(srcfile).items(): + r[k] = v + # Serialize our default GRUB config + with open("/usr/lib/coreos-assembler/grub.cfg") as f: + r['grub-script'] = f.read() + return r + + +def write_image_json(srcfile, outfile): + r = generate_image_json(srcfile) with open(outfile, 'w') as f: - yaml.dump(flattened, f) + json.dump(r, f, sort_keys=True) def merge_lists(x, y, k): diff --git a/src/cosalib/ova.py b/src/cosalib/ova.py index 2ec82b48ba8646c2d243677d259b42285a548c91..7e0c914c1318141b5e4dce47ceb11f69f4d0beec 100644 --- a/src/cosalib/ova.py +++ b/src/cosalib/ova.py @@ -5,6 +5,7 @@ import logging as log import os.path import sys +import json cosa_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, f"{cosa_dir}/cosalib") @@ -63,14 +64,17 @@ class OVA(QemuVariantImage): # Ensure that coreos.ovf is included in the tar self.ovf_path = os.path.join(self._tmpdir, "coreos.ovf") - def generate_ovf_parameters(self, vmdk, cpu=2, - memory=4096, system_type="vmx-13", - os_type="rhel7_64Guest", scsi="VirtualSCSI", - network="VmxNet3"): + def generate_ovf_parameters(self, vmdk, cpu=2, memory=4096): """ Returns a dictionary with the parameters needed to create an OVF file - based on the qemu, vmdk, and info from the build metadata + based on the qemu, vmdk, image.yaml, and info from the build metadata """ + with open(os.path.join(self._workdir, 'tmp/image.json')) as f: + image_json = json.load(f) + + secure_boot = 'true' if image_json['vmware-secure-boot'] else 'false' + system_type = 'vmx-{}'.format(image_json['vmware-hw-version']) + os_type = image_json['vmware-os-type'] disk_info = image_info(vmdk) vmdk_size = os.stat(vmdk).st_size image = self.summary diff --git a/src/cosalib/qemuvariants.py b/src/cosalib/qemuvariants.py index 12950e5bcc9bdeca8359ae7f4b58ee0e6deac152..a8092eacec098964981dd8670bb62ac734b7c0d5 100644 --- a/src/cosalib/qemuvariants.py +++ b/src/cosalib/qemuvariants.py @@ -19,11 +19,11 @@ from cosalib.build import ( from cosalib.cmdlib import ( get_basearch, image_info, - run_verbose, - sha256sum_file + runcmd, + sha256sum_file, + import_ostree_commit ) - # BASEARCH is the current machine architecture BASEARCH = get_basearch() @@ -211,7 +211,7 @@ class QemuVariantImage(_Build): return None def set_platform(self): - run_verbose(['/usr/lib/coreos-assembler/gf-platformid', + runcmd(['/usr/lib/coreos-assembler/gf-platformid', self.image_qemu, self.tmp_image, self.platform]) def mutate_image(self): @@ -226,6 +226,10 @@ class QemuVariantImage(_Build): :param callback: callback function for extra processing image :type callback: function """ + + # Disk image builds may require an unpacked ostree repo and tmp/image.json in general. + import_ostree_commit(self._workdir, self.build_dir, self.meta) + work_img = os.path.join(self._tmpdir, f"{self.image_name_base}.{self.image_format}") final_img = os.path.join(os.path.abspath(self.build_dir), diff --git a/src/create_disk.sh b/src/create_disk.sh index bb901488cec1322e21711123ed96240f3b711e7b..73afdc7ab0645ab40c6391b9be0792b1101a4e03 100755 --- a/src/create_disk.sh +++ b/src/create_disk.sh @@ -390,8 +390,17 @@ EOF install_grub_cfg() { # 0700 to match the RPM permissions which I think are mainly in case someone has # manually set a grub password - mkdir -p -m 0700 $rootfs/boot/grub2 - cp -v $grub_script $rootfs/boot/grub2/grub.cfg + mkdir -p $rootfs/boot/grub2 + chmod 0700 $rootfs/boot/grub2 + printf "%s\n" "$grub_script" | \ + sed -E 's@(^# CONSOLE-SETTINGS-START$)@\1'"${platform_grub_cmds:+\\n${platform_grub_cmds}}"'@' \ + > $rootfs/boot/grub2/grub.cfg + # Copy platforms table if it's non-empty for this arch + # shellcheck disable=SC2031 + if jq -e ".$arch" < "$platforms_json" > /dev/null; then + mkdir -p "$rootfs/boot/nestos" + jq ".$arch" < "$platforms_json" > "$rootfs/boot/nestos/platforms.json" + fi } # Other arch-specific bootloader changes diff --git a/src/image-default.yaml b/src/image-default.yaml index c03e27dce20047ae2e24c9c8a3c80e5dcfcf4f56..37aad7d5596de4149be8dd28777ce86ff70ffd44 100644 --- a/src/image-default.yaml +++ b/src/image-default.yaml @@ -1,8 +1,15 @@ # This file contains defaults for image.yaml that is used by create_disk.sh bootfs: "ext4" rootfs: "xfs" -grub-script: "/usr/lib/coreos-assembler/grub.cfg" + +# Additional default kernel arguments injected into disk images +extra-kargs: [] + # True if we should use `ostree container image deploy` deploy-via-container: false # Set this to a target container reference, e.g. ostree-unverified-registry:quay.io/example/os:latest # container-imgref: "" + +# Format used when generating a squashfs image. Can also be e.g. gzip or lz4 +squashfs-compression: zstd +