From ce9c22282510901aef1fd039fb700aee9c917882 Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Mon, 29 Jan 2024 18:21:40 +0800 Subject: [PATCH] add --force and secex build support [upstream] 951343601c4550de5cdf6e5d21fa0f181e725d5b 1d33f5bf4d427439577d27a1324c3617c3647a1e d9d6655ad5eee7fd1574147ce9076b9cf29eed19 5e02133e856d00642485e22bd01ddbd959729cc8 3246b72cbbea568d081bf02635799445f93e5ae3 369b3b4e9ceebab102272f7e790c1d3f0359194e --- Makefile | 2 + src/cmd-build | 2 +- src/cmd-buildextend-metal | 75 ++++++-- src/cmd-buildextend-secex | 1 + src/create_disk.sh | 165 ++++++++++++++---- src/deps-s390x.txt | 3 + .../genprotimg-script.sh | 33 ++++ src/secex-genprotimgvm-scripts/post-script.sh | 24 +++ src/secex-genprotimgvm-scripts/runvm.sh | 72 ++++++++ src/vmdeps-s390x.txt | 3 + 10 files changed, 334 insertions(+), 46 deletions(-) create mode 120000 src/cmd-buildextend-secex create mode 100644 src/secex-genprotimgvm-scripts/genprotimg-script.sh create mode 100644 src/secex-genprotimgvm-scripts/post-script.sh create mode 100644 src/secex-genprotimgvm-scripts/runvm.sh diff --git a/Makefile b/Makefile index e74c6c67..6987f271 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,8 @@ install: cp -df -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler/ci $$(find ci/ -maxdepth 1 -type f) install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler/cosalib install -D -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler/cosalib $$(find src/cosalib/ -maxdepth 1 -type f) + install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler/secex-genprotimgvm-scripts + install -D -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler/secex-genprotimgvm-scripts $$(find src/secex-genprotimgvm-scripts/ -maxdepth 1 -type f) install -d $(DESTDIR)$(PREFIX)/bin install bin/coreos-assembler $(DESTDIR)$(PREFIX)/bin/ ln -sf ../lib/coreos-assembler/cp-reflink $(DESTDIR)$(PREFIX)/bin/ diff --git a/src/cmd-build b/src/cmd-build index 43e74cac..6f00d326 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -118,7 +118,7 @@ declare -A targets=( ) for target in "$@"; do if [[ $target != ostree ]]; then case "$target" in - metal|metal4k|qemu) ;; + metal|metal4k|qemu|secex) ;; *) fatal "Unrecognized target: $target" ;; esac targets[$target]=1 diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 6827ddc5..11f00021 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -5,6 +5,10 @@ dn=$(dirname "$0") # shellcheck source=src/cmdlib.sh . "${dn}"/cmdlib.sh +# IBM SecureExecution +secure_execution= +image_suffix= + # This script is used for creating both the bare metal and the canonical VM # image (qemu). `buildextend-qemu` is a symlink to `buildextend-metal`. case "$(basename "$0")" in @@ -12,6 +16,11 @@ case "$(basename "$0")" in "cmd-buildextend-metal4k") image_type=metal4k;; "cmd-buildextend-dasd") image_type=dasd;; "cmd-buildextend-qemu") image_type=qemu;; + "cmd-buildextend-secex") + secure_execution=1 + image_type=qemu + image_suffix=-secex + ;; *) fatal "called as unexpected name $0";; esac @@ -25,9 +34,12 @@ EOF } # Parse options +hostkey= +genprotimgvm=/data.secex/genprotimgvm.qcow2 rc=0 build= -options=$(getopt --options h --longoptions help,build: -- "$@") || rc=$? +force= +options=$(getopt --options h --longoptions help,force,build:,hostkey:,genprotimgvm: -- "$@") || rc=$? [ $rc -eq 0 ] || { print_help exit 1 @@ -39,10 +51,21 @@ while true; do print_help exit 0 ;; + --force) + force=1 + ;; --build) build=$2 shift ;; + --hostkey) + hostkey=$(realpath "$2") + shift + ;; + --genprotimgvm) + genprotimgvm="$2" + shift + ;; --) shift break @@ -98,11 +121,13 @@ touch "${build_semaphore}" trap 'rm -rf ${build_semaphore}' EXIT # check if the image already exists in the meta.json -meta_img=$(meta_key "images.${image_type}.path") -if [ "${meta_img}" != "None" ]; then - echo "${image_type} image already exists:" - echo "$meta_img" - exit 0 +if [ -z "${force}" ]; then + meta_img=$(meta_key "images.${image_type}${image_suffix}.path") + if [ "${meta_img}" != "None" ]; then + echo "${image_type}${image_suffix} image already exists:" + echo "$meta_img" + exit 0 + fi fi # reread these values from the build itself rather than rely on the ones loaded @@ -123,10 +148,11 @@ if [[ $image_type == qemu ]]; then image_format=qcow2 fi -img=${name}-${build}-${image_type}.${basearch}.${image_format} +img=${name}-${build}-${image_type}${image_suffix}.${basearch}.${image_format} path=${PWD}/${img} ignition_platform_id="${image_type}" + # dasd and metal4k are different disk formats, but they're still metal if [ "${image_type}" = dasd ] || [ "${image_type}" = metal4k ]; then ignition_platform_id=metal @@ -152,6 +178,27 @@ if [ "${rootfs_type}" = "ext4verity" ]; then BLKSIZE="$(getconf PAGE_SIZE)" fi +disk_args=() +qemu_args=() +# SecureExecution extra stuff +if [[ $secure_execution -eq "1" ]]; then + ignition_pubkey=$(mktemp -p "${tmp_builddir}") + disk_args+=("--with-secure-execution" "--write-ignition-pubkey-to" "${ignition_pubkey}") + if [ -z "${hostkey}" ]; then + if [ ! -f "${genprotimgvm}" ]; then + fatal "No genprotimgvm provided at ${genprotimgvm}" + fi + genprotimg_img="${PWD}/secex-genprotimg.img" + qemu-img create -f raw "${genprotimg_img}" 512M + mkfs.ext4 "${genprotimg_img}" + qemu_args+=("-drive" "if=none,id=genprotimg,format=raw,file=${genprotimg_img}" \ + "-device" "virtio-blk,serial=genprotimg,drive=genprotimg") + else + qemu_args+=("-drive" "if=none,id=hostkey,format=raw,file=$hostkey,readonly=on" \ + "-device" "virtio-blk,serial=hostkey,drive=hostkey") + fi +fi + echo "Estimating disk size..." # The additional 35% here is obviously a hack, but we can't easily completely fill the filesystem, # and doing so has apparently negative performance implications. @@ -161,8 +208,6 @@ rootfs_size="$(jq '."estimate-mb".final' "$PWD/tmp/ostree-size.json")" image_size="$(( rootfs_size + 513 ))M" echo "Disk size estimated to ${image_size}" -disk_args=() - # For bare metal and dasd images, we use the estimated image size. For IaaS/virt, we get it from # image.yaml because we want a "default" disk size that has some free space. case "${image_type}" in @@ -200,7 +245,7 @@ extra_target_device_opts="" if [[ $image_type == dasd || $image_type == metal4k ]]; then extra_target_device_opts=",physical_block_size=4096,logical_block_size=4096" fi -target_drive=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ +qemu_args+=("-drive" "if=none,id=target,format=${image_format},file=${path}.tmp,cache=unsafe" \ "-device" "virtio-blk,serial=target,drive=target${extra_target_device_opts}") # Generate the JSON describing the disk we want to build @@ -218,18 +263,24 @@ cat >image-dynamic.json << EOF } EOF cat image-configured.json image-dynamic.json | jq -s add > image.json -runvm "${target_drive[@]}" -- \ + runvm "${qemu_args[@]}" -- \ /usr/lib/coreos-assembler/create_disk.sh \ --config "$(pwd)"/image.json \ --kargs "\"${kargs}\"" \ "${disk_args[@]}" + +if [[ $secure_execution -eq "1" && -z "${hostkey}" ]]; then + /usr/lib/coreos-assembler/secex-genprotimgvm-scripts/runvm.sh \ + --genprotimgvm "${genprotimgvm}" -- "${qemu_args[@]}" +fi + /usr/lib/coreos-assembler/finalize-artifact "${path}.tmp" "${path}" sha256=$(sha256sum_str < "${img}") cosa meta --workdir "${workdir}" --build "${build}" --dump | python3 -c " import sys, json j = json.load(sys.stdin) -j['images']['${image_type}'] = { +j['images']['${image_type}${image_suffix}'] = { 'path': '${img}', 'sha256': '${sha256}', 'size': $(stat -c '%s' "${img}") diff --git a/src/cmd-buildextend-secex b/src/cmd-buildextend-secex new file mode 120000 index 00000000..ad07b13c --- /dev/null +++ b/src/cmd-buildextend-secex @@ -0,0 +1 @@ +cmd-buildextend-metal \ No newline at end of file diff --git a/src/create_disk.sh b/src/create_disk.sh index bd0a13c5..bb901488 100755 --- a/src/create_disk.sh +++ b/src/create_disk.sh @@ -18,6 +18,7 @@ uninitialized_gpt_uuid="00000000-0000-4000-a000-000000000001" # https://github.com/coreos/fedora-coreos-tracker/issues/465 bootfs_uuid="96d15588-3596-4b3c-adca-a2ff7279ea63" rootfs_uuid="910678ff-f77e-4a7d-8d53-86f2ac47a823" +deploy_root= usage() { cat <&2; exit 1 ;; esac -mkfs.ext4 ${bootargs} "${disk}${BOOTPN}" -L boot -U "${bootfs_uuid}" + +if [[ ${secure_execution} -eq 1 ]]; then + # Unencrypted partition for sd-boot + # shellcheck disable=SC2086 + mkfs.ext4 ${bootargs} "${disk}${SDPART}" -L se -U random +fi + +# shellcheck disable=SC2086 +mkfs.ext4 ${bootargs} "${boot_dev}" -L boot -U "${bootfs_uuid}" udevtrig if [ ${EFIPN:+x} ]; then @@ -237,7 +264,7 @@ mount -o discard "${root_dev}" ${rootfs} chcon $(matchpathcon -n /) ${rootfs} mkdir ${rootfs}/boot chcon $(matchpathcon -n /boot) $rootfs/boot -mount "${disk}${BOOTPN}" $rootfs/boot +mount "${boot_dev}" $rootfs/boot chcon $(matchpathcon -n /boot) $rootfs/boot # FAT doesn't support SELinux labeling, it uses "genfscon", so we # don't need to give it a label manually. @@ -245,6 +272,10 @@ if [ ${EFIPN:+x} ]; then mkdir $rootfs/boot/efi mount "${disk}${EFIPN}" $rootfs/boot/efi fi +if [[ ${secure_execution} -eq 1 ]]; then + mkdir ${rootfs}/se + chcon "$(matchpathcon -n /boot)" $rootfs/se +fi # Now that we have the basic disk layout, initialize the basic # OSTree layout, load in the ostree commit and deploy it. @@ -389,31 +420,14 @@ ppc64le) ;; s390x) bootloader_backend=zipl - # current zipl expects 'title' to be first line, and no blank lines in BLS file - # see https://github.com/ibm-s390-tools/s390-tools/issues/64 - blsfile=$(find $rootfs/boot/loader/entries/*.conf) - tmpfile=$(mktemp) - for f in title version linux initrd options; do - echo $(grep $f $blsfile) >> $tmpfile - done - cat $tmpfile > $blsfile - # we force firstboot in building base image on s390x, ignition-dracut hook will remove - # this and update zipl for second boot - # this is only a temporary solution until we are able to do firstboot check at bootloader - # stage on s390x, either through zipl->grub2-emu or zipl standalone. - # See https://github.com/coreos/ignition-dracut/issues/84 - # A similar hack is present in https://github.com/coreos/coreos-assembler/blob/main/src/gf-platformid#L55 - echo "$(grep options $blsfile | cut -d' ' -f2-) ignition.firstboot" > $tmpfile - - # ideally we want to invoke zipl with bls and zipl.conf but we might need - # to chroot to $rootfs/ to do so. We would also do that when FCOS boot on its own. - # without chroot we can use --target option in zipl but it requires kernel + initramfs - # pair instead - zipl --verbose \ - --target $rootfs/boot \ - --image $rootfs/boot/"$(grep linux $blsfile | cut -d' ' -f2)" \ - --ramdisk $rootfs/boot/"$(grep initrd $blsfile | cut -d' ' -f2)" \ - --parmfile $tmpfile + rdcore_zipl_args=("--boot-mount=$rootfs/boot" "--append-karg=ignition.firstboot") + # in the secex case, we run zipl at the end; in the non-secex case, we need + # to run it now because zipl wants rw access to the bootfs + if [[ ${secure_execution} -ne 1 ]]; then + # in case builder itself runs with SecureExecution + rdcore_zipl_args+=("--secex-mode=disable") + chroot_run /usr/lib/dracut/modules.d/50rdcore/rdcore zipl "${rdcore_zipl_args[@]}" + fi ;; esac @@ -439,6 +453,91 @@ for fs in $rootfs/boot $rootfs; do mount -o remount,ro $fs xfs_freeze -f $fs done + umount -R $rootfs +create_dmverity() { + local partlabel=$1; shift + local mountpoint=$1; shift + local datapart="/dev/disk/by-partlabel/${partlabel}" + local hashpart="/dev/disk/by-partlabel/${partlabel}hash" + # We have to use 512 here to match the filesystem sector size. It's less + # efficient, but meh, it's for first boot only. Alternatively we could + # change the filesystem sector size higher up. + veritysetup format "${datapart}" "${hashpart}" \ + --root-hash-file "/tmp/${partlabel}-roothash" \ + --data-block-size 512 + veritysetup open "${datapart}" "${partlabel}" "${hashpart}" \ + --root-hash-file "/tmp/${partlabel}-roothash" + mount -o ro "/dev/mapper/${partlabel}" "${mountpoint}" +} + +# Save genprotimg input for later and don't run zipl here +rdcore_replacement() { + local se_kargs_append se_initrd se_kernel se_parmfile + local blsfile kernel initrd + local se_script_dir se_tmp_disk se_tmp_mount se_tmp_boot + + se_kargs_append=("ignition.firstboot") + while [ $# -gt 0 ]; do + se_kargs_append+=("$1") + shift + done + + se_script_dir="/usr/lib/coreos-assembler/secex-genprotimgvm-scripts" + se_tmp_disk=$(realpath /dev/disk/by-id/virtio-genprotimg) + se_tmp_mount=$(mktemp -d /tmp/genprotimg-XXXXXX) + se_tmp_boot="${se_tmp_mount}/genprotimg" + mount "${se_tmp_disk}" "${se_tmp_mount}" + mkdir "${se_tmp_boot}" + + se_initrd="${se_tmp_boot}/initrd.img" + se_kernel="${se_tmp_boot}/vmlinuz" + se_parmfile="${se_tmp_boot}/parmfile" + + # Ignition GPG private key + mkdir -p "${se_tmp_boot}/usr/lib/coreos" + generate_gpgkeys "${se_tmp_boot}/usr/lib/coreos/ignition.asc" + + blsfile=$(find "${rootfs}"/boot/loader/entries/*.conf) + echo "$(grep options "${blsfile}" | cut -d' ' -f2-)" "${se_kargs_append[@]}" > "${se_parmfile}" + kernel="${rootfs}/boot/$(grep linux "${blsfile}" | cut -d' ' -f2)" + initrd="${rootfs}/boot/$(grep initrd "${blsfile}" | cut -d' ' -f2)" + cp "${kernel}" "${se_kernel}" + cp "${initrd}" "${se_initrd}" + + # genprotimg and zipl will be done outside this script + # copy scripts for that step to the tmp disk + cp "${se_script_dir}/genprotimg-script.sh" "${se_script_dir}/post-script.sh" "${se_tmp_mount}/" + + umount "${se_tmp_mount}" + rmdir "${se_tmp_mount}" +} + +if [[ ${secure_execution} -eq 1 ]]; then + # set up dm-verity for the rootfs and bootfs + create_dmverity root $rootfs + create_dmverity boot $rootfs/boot + # We need to run the genprotimg step in a separate step for rhcos release images + if [ ! -e /dev/disk/by-id/virtio-genprotimg ]; then + echo "Building local Secure Execution Image, running zipl and genprotimg" + generate_gpgkeys "/tmp/ignition.asc" + mount --rbind "/tmp/ignition.asc" "${deploy_root}/usr/lib/coreos/ignition.asc" + # run zipl with root hashes as kargs + rdcore_zipl_args+=("--secex-mode=enforce" "--hostkey=/dev/disk/by-id/virtio-hostkey") + rdcore_zipl_args+=("--append-karg=rootfs.roothash=$(cat /tmp/root-roothash)") + rdcore_zipl_args+=("--append-karg=bootfs.roothash=$(cat /tmp/boot-roothash)") + rdcore_zipl_args+=("--append-file=/usr/lib/coreos/ignition.asc") + chroot_run /usr/lib/dracut/modules.d/50rdcore/rdcore zipl "${rdcore_zipl_args[@]}" + else + echo "Building release Secure Execution Image, zipl and genprotimg will be run later" + rdcore_replacement "rootfs.roothash=$(cat /tmp/root-roothash)" "bootfs.roothash=$(cat /tmp/boot-roothash)" + fi + + # unmount and close everything + umount -R $rootfs + veritysetup close boot + veritysetup close root +fi + rmdir $rootfs diff --git a/src/deps-s390x.txt b/src/deps-s390x.txt index 64b89314..0e33e09a 100644 --- a/src/deps-s390x.txt +++ b/src/deps-s390x.txt @@ -3,3 +3,6 @@ lorax xorriso # Deps needed by supermin s390utils-base + +# for Secure Execution +veritysetup diff --git a/src/secex-genprotimgvm-scripts/genprotimg-script.sh b/src/secex-genprotimgvm-scripts/genprotimg-script.sh new file mode 100644 index 00000000..6ea3c999 --- /dev/null +++ b/src/secex-genprotimgvm-scripts/genprotimg-script.sh @@ -0,0 +1,33 @@ +#!bin/bash + +set -exuo pipefail + +echo "Preparing for genprotimg-daemon" + +source="/build/genprotimg" +destination="/genprotimg" +pkey="usr/lib/coreos/ignition.asc" + +trap "rm -f ${source}/${pkey}" EXIT + +# Files need to be named correctly +# genprotimg daemon can only see /genprotimg folder +cp "${source}/vmlinuz" "${source}/initrd.img" "${source}/parmfile" "${destination}/" + +# Append Ignition GPG private key to initramfs +cd "${source}" +echo "${pkey}" | cpio --quiet -H newc -o | gzip -9 -n >> "${destination}/initrd.img" +rm "${pkey}" + +# Signal daemon that it can run genprotimg +touch "${destination}/signal.file" + +# Wait for genprotimg execution +while [ -e "$destination/signal.file" ] && [ ! -e "$destination/error" ]; do + sleep 5 +done +if [ -e "$destination/error" ] || [ ! -e "${destination}/se.img" ]; then + ls -lha $destination + echo "Failed to run genprotimg" + exit 1 +fi diff --git a/src/secex-genprotimgvm-scripts/post-script.sh b/src/secex-genprotimgvm-scripts/post-script.sh new file mode 100644 index 00000000..ced32d57 --- /dev/null +++ b/src/secex-genprotimgvm-scripts/post-script.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -exuo pipefail + +echo "Moving sdboot and executing zipl" + +workdir="/build" +sdboot="/genprotimg/se.img" +genprotimg_dir="${workdir}/genprotimg" +se_boot=$(mktemp -d /tmp/se-XXXXXX) + +disk=$(realpath /dev/disk/by-id/virtio-target) +disk_se="${disk}1" + +mount "${disk_se}" "${se_boot}" +cp "${sdboot}" "${se_boot}/sdboot" +zipl -V -i ${se_boot}/sdboot -t ${se_boot} + +# Disable debug output, the last message should be success +set +x +echo "Success, added sdboot to image and executed zipl" + +umount "${se_boot}" +rm -rf "${se_boot}" diff --git a/src/secex-genprotimgvm-scripts/runvm.sh b/src/secex-genprotimgvm-scripts/runvm.sh new file mode 100644 index 00000000..c90a2c60 --- /dev/null +++ b/src/secex-genprotimgvm-scripts/runvm.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -euo pipefail + +vmrundir="${workdir}/tmp/build.secex" +memory_default=2048 +runvm_console="${vmrundir}/runvm-console.txt" +genprotimgvm= +qemu_args=() +while true; do + case "$1" in + --genprotimgvm) + genprotimgvm="$2" + shift + ;; + --) + shift + break + ;; + -*) + fatal "$0: unrecognized option: $1" + exit 1 + ;; + *) + break + ;; + esac + shift +done +if [ -z "${genprotimgvm}" ]; then + echo "Missing option --genprotimgvm" +fi +while [ $# -gt 0 ]; do + qemu_args+=("$1") + shift +done + +set -x + +[[ -d "${vmrundir}" ]] && rm -rf "${vmrundir}" +mkdir "${vmrundir}" +touch "${runvm_console}" + +kola_args=(kola qemuexec -m "${COSA_SUPERMIN_MEMORY:-${memory_default}}" --auto-cpus -U --workdir none \ + --console-to-file "${runvm_console}") + +base_qemu_args=(-drive "if=none,id=buildvm,format=qcow2,snapshot=on,file=${genprotimgvm},index=1" -device virtio-blk-ccw,drive=buildvm,bootindex=1 \ + -no-reboot -nodefaults -device virtio-serial \ + -device virtserialport,chardev=virtioserial0,name=cosa-cmdout -chardev stdio,id=virtioserial0 + ) + +if [ -z "${GENPROTIMGVM_SE_OFF:-}" ]; then + base_qemu_args+=(-object s390-pv-guest,id=pv0 -machine confidential-guest-support=pv0) +else + echo "No secure execution enabled for build-VM, happy debugging" +fi + +if ! "${kola_args[@]}" -- "${base_qemu_args[@]}" \ + "${qemu_args[@]}" <&-; then # the <&- here closes stdin otherwise qemu waits forever + cat "${runvm_console}" + echo "Failed to run 'kola qemuexec'" + exit 1 +fi + +cat "${runvm_console}" + +if ! grep -q "Success, added sdboot to image and executed zipl" "${runvm_console}"; then + echo "Could not find success message, genprotimg failed." + exit 1 +fi + +exit 0 diff --git a/src/vmdeps-s390x.txt b/src/vmdeps-s390x.txt index e76878bb..c48c5452 100644 --- a/src/vmdeps-s390x.txt +++ b/src/vmdeps-s390x.txt @@ -1 +1,4 @@ s390utils-base + +# for Secure Execution +veritysetup -- Gitee