From 2176d9ca9d37b48d680e3b115f6fa03f3521ecf9 Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Mon, 26 Feb 2024 16:23:41 +0800 Subject: [PATCH] qemu: Add support for full emulation because openEuler only support x86_64 and aarch64, drop the ppc64le and s390x support [upstream] 5cf1cfdc7a9e32afa0258214f49dd9299a743590 --- mantle/cmd/kola/options.go | 3 + mantle/cmd/kola/qemuexec.go | 2 + mantle/platform/machine/qemu/cluster.go | 5 ++ mantle/platform/machine/qemu/flight.go | 1 + mantle/platform/qemu.go | 78 ++++++++++++++++--------- src/deps.txt | 3 + 6 files changed, 63 insertions(+), 29 deletions(-) diff --git a/mantle/cmd/kola/options.go b/mantle/cmd/kola/options.go index 3b544416..fa7801a5 100644 --- a/mantle/cmd/kola/options.go +++ b/mantle/cmd/kola/options.go @@ -298,6 +298,9 @@ func syncOptionsImpl(useCosa bool) error { return err } } + // Currently the `--arch` option is defined in terms of coreos-assembler, but + // we also unconditionally use it for qemu if present. + kola.QEMUOptions.Arch = kola.Options.CosaBuildArch units, _ := root.PersistentFlags().GetStringSlice("debug-systemd-units") for _, unit := range units { diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index 43306882..7bc2915d 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -47,6 +47,8 @@ var ( usernet bool cpuCountHost bool + architecture string + hostname string ignition string butane string diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index 089a971e..b789d725 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -101,6 +101,11 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl builder.ConfigFile = confPath defer builder.Close() builder.UUID = qm.id + if qc.flight.opts.Arch != "" { + if err := builder.SetArchitecture(qc.flight.opts.Arch); err != nil { + return nil, err + } + } if qc.flight.opts.Firmware != "" { builder.Firmware = qc.flight.opts.Firmware } diff --git a/mantle/platform/machine/qemu/flight.go b/mantle/platform/machine/qemu/flight.go index f2bf7ada..5da7f271 100644 --- a/mantle/platform/machine/qemu/flight.go +++ b/mantle/platform/machine/qemu/flight.go @@ -34,6 +34,7 @@ type Options struct { Board string Firmware string Memory string + Arch string NbdDisk bool MultiPathDisk bool diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 135597aa..1c1218b8 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -294,7 +294,10 @@ func (inst *QemuInstance) Destroy() { // is used to boot from the network device (boot once is not supported). For s390x, the boot ordering was not a problem as it // would always read from disk first. For aarch64, the bootindex needs to be switched to boot from disk before a reboot func (inst *QemuInstance) SwitchBootOrder() (err2 error) { - if coreosarch.CurrentRpmArch() != "s390x" && coreosarch.CurrentRpmArch() != "aarch64" { + switch inst.architecture { + case "s390x", "aarch64": + break + default: //Not applicable for other arches return nil } @@ -400,6 +403,8 @@ type QemuBuilder struct { // File to which to redirect the serial console ConsoleFile string + // If set, use QEMU full emulation for the target architecture + architecture string // Memory defaults to 1024 on most architectures, others it may be 2048 Memory int // Processors < 0 means to use host count, unset means 1, values > 1 are directly used @@ -466,6 +471,7 @@ func NewQemuBuilder() *QemuBuilder { Swtpm: true, Pdeathsig: true, Argv: []string{}, + architecture: coreosarch.CurrentRpmArch(), } return &ret } @@ -534,15 +540,15 @@ func (builder *QemuBuilder) AddFd(fd *os.File) string { } // virtio returns a virtio device argument for qemu, which is architecture dependent -func virtio(device, args string) string { +func virtio(arch, device, args string) string { var suffix string - switch coreosarch.CurrentRpmArch() { + switch arch { case "x86_64", "ppc64le", "aarch64": suffix = "pci" case "s390x": suffix = "ccw" default: - panic(fmt.Sprintf("RpmArch %s unhandled in virtio()", coreosarch.CurrentRpmArch())) + panic(fmt.Sprintf("RpmArch %s unhandled in virtio()", arch)) } return fmt.Sprintf("virtio-%s-%s,%s", device, suffix, args) } @@ -582,7 +588,7 @@ func (builder *QemuBuilder) setupNetworking() error { netdev += ",restrict=on" } - builder.Append("-netdev", netdev, "-device", virtio("net", "netdev=eth0")) + builder.Append("-netdev", netdev, "-device", virtio(builder.architecture, "net", "netdev=eth0")) return nil } @@ -595,7 +601,7 @@ func (builder *QemuBuilder) setupAdditionalNetworking() error { macSuffix := fmt.Sprintf("%02x", macCounter) netdev := fmt.Sprintf("user,id=eth%s,dhcpstart=10.0.2.%s", idSuffix, netSuffix) - device := virtio("net", fmt.Sprintf("netdev=eth%s,mac=52:55:00:d1:56:%s", idSuffix, macSuffix)) + device := virtio(builder.architecture, "net", fmt.Sprintf("netdev=eth%s,mac=52:55:00:d1:56:%s", idSuffix, macSuffix)) builder.Append("-netdev", netdev, "-device", device) macCounter++ } @@ -603,6 +609,16 @@ func (builder *QemuBuilder) setupAdditionalNetworking() error { return nil } +// SetArchitecture enables qemu full emulation for the target architecture. +func (builder *QemuBuilder) SetArchitecture(arch string) error { + switch arch { + case "x86_64", "aarch64", "s390x", "ppc64le": + builder.architecture = arch + return nil + } + return fmt.Errorf("architecture %s not supported by coreos-assembler qemu", arch) +} + // Mount9p sets up a mount point from the host to guest. To be replaced // with https://virtio-fs.gitlab.io/ once it lands everywhere. func (builder *QemuBuilder) Mount9p(source, destHint string, readonly bool) { @@ -612,13 +628,13 @@ func (builder *QemuBuilder) Mount9p(source, destHint string, readonly bool) { readonlyStr = ",readonly=on" } builder.Append("--fsdev", fmt.Sprintf("local,id=fs%d,path=%s,security_model=mapped%s", builder.fs9pID, source, readonlyStr)) - builder.Append("-device", virtio("9p", fmt.Sprintf("fsdev=fs%d,mount_tag=%s", builder.fs9pID, destHint))) + builder.Append("-device", virtio(builder.architecture, "9p", fmt.Sprintf("fsdev=fs%d,mount_tag=%s", builder.fs9pID, destHint))) } // supportsFwCfg if the target system supports injecting // Ignition via the qemu -fw_cfg option. func (builder *QemuBuilder) supportsFwCfg() bool { - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "s390x", "ppc64le": return false } @@ -627,7 +643,7 @@ func (builder *QemuBuilder) supportsFwCfg() bool { // supportsSwtpm if the target system supports a virtual TPM device func (builder *QemuBuilder) supportsSwtpm() bool { - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "s390x": // s390x does not support a backend for TPM return false @@ -663,7 +679,7 @@ type coreosGuestfish struct { remote string } -func newGuestfish(diskImagePath string, diskSectorSize int) (*coreosGuestfish, error) { +func newGuestfish(arch, diskImagePath string, diskSectorSize int) (*coreosGuestfish, error) { // Set guestfish backend to direct in order to avoid libvirt as backend. // Using libvirt can lead to permission denied issues if it does not have access // rights to the qcow image @@ -676,7 +692,7 @@ func newGuestfish(diskImagePath string, diskSectorSize int) (*coreosGuestfish, e cmd.Env = append(os.Environ(), "LIBGUESTFS_BACKEND=direct") // Hack to run with a wrapper on older P8 hardware running RHEL7 - switch coreosarch.CurrentRpmArch() { + switch arch { case "ppc64le": u := unix.Utsname{} if err := unix.Uname(&u); err != nil { @@ -744,8 +760,8 @@ func (gf *coreosGuestfish) destroy() { } // setupPreboot performs changes necessary before the disk is booted -func setupPreboot(confPath, firstbootkargs, kargs string, diskImagePath string, diskSectorSize int) error { - gf, err := newGuestfish(diskImagePath, diskSectorSize) +func setupPreboot(arch, confPath, firstbootkargs, kargs string, diskImagePath string, diskSectorSize int) error { + gf, err := newGuestfish(arch, diskImagePath, diskSectorSize) if err != nil { return err } @@ -906,7 +922,7 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { } requiresInjection := builder.ConfigFile != "" && builder.ForceConfigInjection if requiresInjection || builder.AppendFirstbootKernelArgs != "" || builder.AppendKernelArgs != "" { - if err := setupPreboot(builder.ConfigFile, builder.AppendFirstbootKernelArgs, builder.AppendKernelArgs, + if err := setupPreboot(builder.architecture, builder.ConfigFile, builder.AppendFirstbootKernelArgs, builder.AppendKernelArgs, disk.dstFileName, disk.SectorSize); err != nil { return errors.Wrapf(err, "ignition injection with guestfs failed") } @@ -960,13 +976,13 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { wwn := rand.Uint64() var bus string - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "x86_64", "ppc64le", "aarch64": bus = "pci" case "s390x": bus = "ccw" default: - panic(fmt.Sprintf("Mantle doesn't know which bus type to use on %s", coreosarch.CurrentRpmArch())) + panic(fmt.Sprintf("Mantle doesn't know which bus type to use on %s", builder.architecture)) } for i := 0; i < 2; i++ { @@ -992,7 +1008,7 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { disk.dstFileName = "" switch channel { case "virtio": - builder.Append("-device", virtio("blk", fmt.Sprintf("drive=%s%s", id, opts))) + builder.Append("-device", virtio(builder.architecture, "blk", fmt.Sprintf("drive=%s%s", id, opts))) case "nvme": builder.Append("-device", fmt.Sprintf("nvme,drive=%s%s", id, opts)) default: @@ -1075,7 +1091,7 @@ func (builder *QemuBuilder) finalize() { // Then later, other non-x86_64 seemed to just copy that. memory := 1024 - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "aarch64", "s390x", "ppc64le": memory = 2048 } @@ -1091,15 +1107,16 @@ func (builder *QemuBuilder) Append(args ...string) { // baseQemuArgs takes a board and returns the basic qemu // arguments needed for the current architecture. -func baseQemuArgs() []string { +func baseQemuArgs(arch string) ([]string, error) { accel := "accel=kvm" kvm := true - if _, ok := os.LookupEnv("COSA_NO_KVM"); ok { + hostArch := coreosarch.CurrentRpmArch() + if _, ok := os.LookupEnv("COSA_NO_KVM"); ok || hostArch != arch { accel = "accel=tcg" kvm = false } var ret []string - switch coreosarch.CurrentRpmArch() { + switch arch { case "x86_64": ret = []string{ "qemu-system-x86_64", @@ -1121,19 +1138,19 @@ func baseQemuArgs() []string { "-machine", "pseries,kvm-type=HV,vsmt=8,cap-fwnmi=off," + accel, } default: - panic(fmt.Sprintf("RpmArch %s combo not supported for qemu ", coreosarch.CurrentRpmArch())) + return nil, fmt.Errorf("architecture %s not supported for qemu", arch) } if kvm { ret = append(ret, "-cpu", "host") } else { - if coreosarch.CurrentRpmArch() == "x86_64" { + if arch == "x86_64" { // the default qemu64 CPU model does not support x86_64_v2 // causing crashes on EL9+ kernels // see https://bugzilla.redhat.com/show_bug.cgi?id=2060839 ret = append(ret, "-cpu", "Nehalem") } } - return ret + return ret, nil } func (builder *QemuBuilder) setupUefi(secureBoot bool) error { @@ -1297,7 +1314,7 @@ func (builder *QemuBuilder) setupIso() error { } builder.Append("-drive", "file="+builder.iso.path+",format=raw,if=none,readonly=on,id=installiso") if builder.isoAsDisk { - builder.Append("-device", virtio("blk", "drive=installiso"+bootindexStr)) + builder.Append("-device", virtio(builder.architecture, "blk", "drive=installiso"+bootindexStr)) } else { builder.Append("-device", "ide-cd,drive=installiso"+bootindexStr) } @@ -1386,8 +1403,10 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { } }() - argv := baseQemuArgs() - argv = append(argv, "-m", fmt.Sprintf("%d", builder.Memory)) + argv, err := baseQemuArgs(builder.architecture) + if err != nil { + return nil, err + } if builder.Processors < 0 { nproc, err := system.GetProcessors() @@ -1428,7 +1447,7 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { // We always provide a random source argv = append(argv, "-object", "rng-random,filename=/dev/urandom,id=rng0", - "-device", virtio("rng", "rng=rng0")) + "-device", virtio(builder.architecture, "rng", "rng=rng0")) if builder.UUID != "" { argv = append(argv, "-uuid", builder.UUID) } @@ -1530,7 +1549,7 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { } argv = append(argv, "-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s", swtpmSock), "-tpmdev", "emulator,id=tpm0,chardev=chrtpm") // There are different device backends on each architecture - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "x86_64": argv = append(argv, "-device", "tpm-tis,tpmdev=tpm0") case "aarch64": @@ -1572,6 +1591,7 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { argv = append(argv, builder.Argv...) inst.qemu = exec.Command(argv[0], argv[1:]...) + inst.architecture = builder.architecture cmd := inst.qemu.(*exec.ExecCmd) cmd.Stderr = os.Stderr diff --git a/src/deps.txt b/src/deps.txt index d3cfb07a..ac0cdc99 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -30,6 +30,9 @@ make git rpm-build # virt dependencies libguestfs-tools libguestfs-tools-c /usr/bin/qemu-img qemu-kvm swtpm +# And the main arch emulators for cross-arch testing +#qemu-system-aarch64-core qemu-system-ppc-core qemu-system-s390x-core qemu-system-x86-core +qemu-system-aarch64-core qemu-system-x86-core # Useful for moving files around rsync -- Gitee