From f5a7426cea8c73a6d9ba16d433c303361a59055a Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Thu, 2 Sep 2021 03:08:51 -0400 Subject: [PATCH 1/4] buildextend-live: add stub grub.cfg to efiboot.img 05ba85642b4b added an ESP to the ISO's hybrid GPT to help EFI systems that won't boot El Torito from a hard disk. However, when booting from the ESP, GRUB won't read the grub.cfg from the main ISO image, and just drops to a grub prompt. Add a stub grub.cfg that points GRUB to the right place. Fixes: 05ba85642b4b ("buildextend-live: hybridize live ISO for UEFI boot") Fixes: https://github.com/coreos/fedora-coreos-tracker/issues/724 Fixes: https://github.com/coreos/fedora-coreos-tracker/issues/953 (cherry picked from commit 0a3a4da2aa6662318b18368ce2f09326a38457f3) Signed-off-by: zhongling.h --- src/cmd-buildextend-live | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/cmd-buildextend-live b/src/cmd-buildextend-live index 6d271bad..97335bb2 100755 --- a/src/cmd-buildextend-live +++ b/src/cmd-buildextend-live @@ -483,6 +483,30 @@ def generate_iso(): tmpimageefidir = os.path.join(tmpdir, "efi") ostree_extract_efi(repo, buildmeta_commit, tmpimageefidir) + # Inject a stub grub.cfg pointing to the one in the main ISO image. + # + # When booting via El Torito, this stub is not used; GRUB reads + # the ISO image directly using its own ISO support. This + # happens when booting from a CD device, or when the ISO is + # copied to a USB stick and booted on EFI firmware which prefers + # to boot a hard disk from an El Torito image if it has one. + # EDK II in QEMU behaves this way. + # + # This stub is used with EFI firmware which prefers to boot a + # hard disk from an ESP, or which cannot boot a hard disk via El + # Torito at all. In that case, GRUB thinks it booted from a + # partition of the disk (a fake ESP created by isohybrid, + # pointing to efiboot.img) and needs a grub.cfg there. + vendor_ids = [n for n in os.listdir(tmpimageefidir) if n != "BOOT"] + if len(vendor_ids) != 1: + raise Exception(f"did not find exactly one EFI vendor ID: {vendor_ids}") + with open(os.path.join(tmpimageefidir, vendor_ids[0], "grub.cfg"), "w") as fh: + fh.write(f'''search --label "{volid}" --set root --no-floppy +set prefix=($root)/EFI/{vendor_ids[0]} +configfile $prefix/grub.cfg +boot +''') + # Install binaries from boot partition # Manually construct the tarball to ensure proper permissions and ownership efitarfile = tempfile.NamedTemporaryFile(suffix=".tar") -- Gitee From d6793c58e276c72ccead93f7f9a92f6fbae28d18 Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Wed, 8 Sep 2021 17:16:43 -0400 Subject: [PATCH 2/4] cosalib: add glob helper that checks for at least one match (cherry picked from commit b499b8909e18e2d46652a1da8582a1fc1d61610e) Signed-off-by: zhongling.h --- src/cmd-buildextend-live | 6 ++--- src/cosalib/cmdlib.py | 53 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/cmd-buildextend-live b/src/cmd-buildextend-live index 97335bb2..7d644374 100755 --- a/src/cmd-buildextend-live +++ b/src/cmd-buildextend-live @@ -19,8 +19,8 @@ import glob sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from cosalib.builds import Builds -from cosalib.cmdlib import run_verbose, sha256sum_file -from cosalib.cmdlib import import_ostree_commit, get_basearch +from cosalib.cmdlib import run_verbose, sha256sum_file, flatten_image_yaml +from cosalib.cmdlib import import_ostree_commit, get_basearch, ensure_glob from cosalib.meta import GenericBuildMeta @@ -407,7 +407,7 @@ def generate_iso(): elif basearch == "ppc64le": os.makedirs(os.path.join(tmpisoroot, 'boot/grub')) # can be EFI/fedora or EFI/redhat - grubpath = glob.glob(os.path.join(tmpisoroot, 'EFI/*/grub.cfg')) + grubpath = ensure_glob(os.path.join(tmpisoroot, 'EFI/*/grub.cfg')) shutil.move(grubpath[0], os.path.join(tmpisoroot, 'boot/grub/grub.cfg')) # safely remove things we don't need in the final ISO tree diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index 7eaee371..0ba12705 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -2,6 +2,7 @@ """ Houses helper code for python based coreos-assembler commands. """ +import glob import hashlib import json import os @@ -309,3 +310,55 @@ def image_info(image): return out except Exception as e: raise Exception(f"failed to inspect {image} with qemu", e) + +# Hackily run some bash code from cmdlib.sh helpers. +def cmdlib_sh(script): + subprocess.check_call(['bash', '-c', f''' + set -euo pipefail + source {THISDIR}/../cmdlib.sh + {script} + ''']) + + +def flatten_image_yaml_to_file(srcfile, outfile): + flattened = flatten_image_yaml(srcfile) + with open(outfile, 'w') as f: + yaml.dump(flattened, f) + + +def merge_lists(x, y, k): + x[k] = x.get(k, []) + assert type(x[k]) == list + y[k] = y.get(k, []) + assert type(y[k]) == list + x[k].extend(y[k]) + + +def flatten_image_yaml(srcfile, base=None): + if base is None: + base = {} + + with open(srcfile) as f: + srcyaml = yaml.safe_load(f) + + # first, special-case list values + merge_lists(base, srcyaml, 'extra-kargs') + merge_lists(base, srcyaml, 'ignition-network-kcmdline') + + # then handle all the non-list values + base = merge_dicts(base, srcyaml) + + if 'include' not in srcyaml: + return base + + fn = os.path.join(os.path.dirname(srcfile), srcyaml['include']) + del base['include'] + return flatten_image_yaml(fn, base) + + +def ensure_glob(pathname, **kwargs): + '''Call glob.glob(), and fail if there are no results.''' + ret = glob.glob(pathname, **kwargs) + if not ret: + raise Exception(f'No matches for {pathname}') + return ret -- Gitee From 3633b01077d2d7b4b10968ce321d207f50b7fe25 Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Thu, 16 Sep 2021 01:07:49 -0400 Subject: [PATCH 3/4] buildextend-live: drop shim fallback.efi from ISO; simplify EFI image UEFI boots from removable media via the arch-specific default EFI application in /EFI/BOOT. When booted that way, shim chains to fallback.efi if it exists, and fallback.efi creates an EFI boot entry. That's not appropriate for removable media boot, since the media will probably never be present again. If a TPM is present, fallback.efi will additionally reboot the machine, and on some machines this leads to boot loops. Instead of all this, we just want shim to chain directly to GRUB. The EFI boot image currently looks like this: EFI/BOOT/BOOTX64.EFI EFI/BOOT/fbx64.efi EFI/fedora/BOOTX64.CSV EFI/fedora/fonts/ EFI/fedora/grub.cfg EFI/fedora/grubx64.efi EFI/fedora/mmx64.efi EFI/fedora/shim.efi EFI/fedora/shimx64.efi Refactor the EFI boot image under the expectation that we never want to create or boot from an EFI boot entry. Consolidate /EFI/ into /EFI/BOOT and delete things we don't need: fallback.efi, its associated CSV, and extra copies of shim. The result is this: EFI/BOOT/BOOTX64.EFI EFI/BOOT/fonts/ EFI/BOOT/grub.cfg EFI/BOOT/grubx64.efi EFI/BOOT/mmx64.efi This also saves a couple MB of space. Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=2004449 (cherry picked from commit 5bed36f33d92b1b0408e5655717f40deb81e86cd) Signed-off-by: zhongling.h --- src/cmd-buildextend-live | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/cmd-buildextend-live b/src/cmd-buildextend-live index 7d644374..c74580e8 100755 --- a/src/cmd-buildextend-live +++ b/src/cmd-buildextend-live @@ -483,6 +483,34 @@ def generate_iso(): tmpimageefidir = os.path.join(tmpdir, "efi") ostree_extract_efi(repo, buildmeta_commit, tmpimageefidir) + # Find name of vendor directory + vendor_ids = [n for n in os.listdir(tmpimageefidir) if n != "BOOT"] + if len(vendor_ids) != 1: + raise Exception(f"did not find exactly one EFI vendor ID: {vendor_ids}") + + # Delete fallback and its CSV file. Its purpose is to create + # EFI boot variables, which we don't want when booting from + # removable media. + # + # A future shim release will merge fallback.efi into the main + # shim binary and enable the fallback behavior when the CSV + # exists. But for now, fail if fallback.efi is missing. + for path in ensure_glob(os.path.join(tmpimageefidir, "BOOT", "fb*.efi")): + os.unlink(path) + for path in ensure_glob(os.path.join(tmpimageefidir, vendor_ids[0], "BOOT*.CSV")): + os.unlink(path) + + # Drop vendor copies of shim; we already have it in BOOT*.EFI in + # BOOT + for path in ensure_glob(os.path.join(tmpimageefidir, vendor_ids[0], "shim*.efi")): + os.unlink(path) + + # Consolidate remaining files into BOOT. shim needs GRUB to be + # there, and the rest doesn't hurt. + for path in ensure_glob(os.path.join(tmpimageefidir, vendor_ids[0], "*")): + shutil.move(path, os.path.join(tmpimageefidir, "BOOT")) + os.rmdir(os.path.join(tmpimageefidir, vendor_ids[0])) + # Inject a stub grub.cfg pointing to the one in the main ISO image. # # When booting via El Torito, this stub is not used; GRUB reads @@ -497,10 +525,7 @@ def generate_iso(): # Torito at all. In that case, GRUB thinks it booted from a # partition of the disk (a fake ESP created by isohybrid, # pointing to efiboot.img) and needs a grub.cfg there. - vendor_ids = [n for n in os.listdir(tmpimageefidir) if n != "BOOT"] - if len(vendor_ids) != 1: - raise Exception(f"did not find exactly one EFI vendor ID: {vendor_ids}") - with open(os.path.join(tmpimageefidir, vendor_ids[0], "grub.cfg"), "w") as fh: + with open(os.path.join(tmpimageefidir, "BOOT", "grub.cfg"), "w") as fh: fh.write(f'''search --label "{volid}" --set root --no-floppy set prefix=($root)/EFI/{vendor_ids[0]} configfile $prefix/grub.cfg -- Gitee From af09a74b1dfde7ec9004769dddfb95bee8143e95 Mon Sep 17 00:00:00 2001 From: "zhongling.h" Date: Wed, 9 Mar 2022 16:19:46 +0800 Subject: [PATCH 4/4] fix UEFI boot for LifseaOS Signed-off-by: zhongling.h --- src/cmd-buildextend-live | 7 ++++++- src/create_disk.sh | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/cmd-buildextend-live b/src/cmd-buildextend-live index c74580e8..b075c3c0 100755 --- a/src/cmd-buildextend-live +++ b/src/cmd-buildextend-live @@ -525,9 +525,14 @@ def generate_iso(): # Torito at all. In that case, GRUB thinks it booted from a # partition of the disk (a fake ESP created by isohybrid, # pointing to efiboot.img) and needs a grub.cfg there. + # + # IMPORTANT: For successful boot, please check exact path for + # grub.cfg in the first partition and change $prefix here. + # In most of the cases, like Fedora CoreOS, ($root)/EFI/{vendor[0]} + # will give you the right path, but there are exceptions. with open(os.path.join(tmpimageefidir, "BOOT", "grub.cfg"), "w") as fh: fh.write(f'''search --label "{volid}" --set root --no-floppy -set prefix=($root)/EFI/{vendor_ids[0]} +set prefix=($root)/EFI/anolis configfile $prefix/grub.cfg boot ''') diff --git a/src/create_disk.sh b/src/create_disk.sh index d5237e6d..96b90344 100755 --- a/src/create_disk.sh +++ b/src/create_disk.sh @@ -317,15 +317,16 @@ install_grub_cfg() { # Other arch-specific bootloader changes case "$arch" in x86_64) - install_grub_cfg - # And BIOS grub in addition. See also - # https://github.com/coreos/fedora-coreos-tracker/issues/32 - grub2-install \ - --fonts ascii \ - --target i386-pc \ - --install-modules 'test blscfg linux search configfile all_video serial gzio' \ - --boot-directory $rootfs/boot \ - $disk + install_uefi + if [ "${x86_bios_bootloader}" = 1 ]; then + # And BIOS grub in addition. See also + # https://github.com/coreos/fedora-coreos-tracker/issues/32 + grub2-install \ + --target i386-pc \ + --boot-directory $rootfs/boot \ + --modules mdraid1x \ + $disk + fi ;; aarch64) # Our aarch64 is UEFI only. -- Gitee