diff --git a/BinDir.tar.gz b/BinDir.tar.gz index 3154d2aa9392390db53497b06ebe7fdc33033353..326fcb51c3998db1344613bba4f6cdbf0f858e3d 100644 Binary files a/BinDir.tar.gz and b/BinDir.tar.gz differ diff --git a/accel-kvm-Extract-common-KVM-vCPU-creation-parking-c.patch b/accel-kvm-Extract-common-KVM-vCPU-creation-parking-c.patch new file mode 100644 index 0000000000000000000000000000000000000000..d68e7f54f43f58b3be56a880de48e15b7ebd5a2f --- /dev/null +++ b/accel-kvm-Extract-common-KVM-vCPU-creation-parking-c.patch @@ -0,0 +1,147 @@ +From 6999ced63ca3bb05a1cbc4a667bd9fd27eeaeaee Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sat, 9 Sep 2023 00:04:04 +0000 +Subject: [PATCH] accel/kvm: Extract common KVM vCPU {creation,parking} code + +KVM vCPU creation is done once during the initialization of the VM when Qemu +threads are spawned. This is common to all the architectures. If the architecture +supports vCPU hot-{un}plug then this KVM vCPU creation could be deferred to +later point as well. Some architectures might in any case create KVM vCPUs for +the yet-to-be plugged vCPUs (i.e. QoM Object & thread does not exists) during VM +init time and park them. + +Hot-unplug of vCPU results in destruction of the vCPU objects in QOM but +the KVM vCPU objects in the Host KVM are not destroyed and their representative +KVM vCPU objects in Qemu are parked. + +Signed-off-by: Salil Mehta +--- + accel/kvm/kvm-all.c | 61 ++++++++++++++++++++++++++++++++++---------- + include/sysemu/kvm.h | 2 ++ + 2 files changed, 49 insertions(+), 14 deletions(-) + +diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c +index d900df93a4..6d503aa614 100644 +--- a/accel/kvm/kvm-all.c ++++ b/accel/kvm/kvm-all.c +@@ -136,6 +136,7 @@ static QemuMutex kml_slots_lock; + #define kvm_slots_unlock() qemu_mutex_unlock(&kml_slots_lock) + + static void kvm_slot_init_dirty_bitmap(KVMSlot *mem); ++static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id); + + static inline void kvm_resample_fd_remove(int gsi) + { +@@ -324,11 +325,51 @@ err: + return ret; + } + ++void kvm_park_vcpu(CPUState *cpu) ++{ ++ unsigned long vcpu_id = cpu->cpu_index; ++ struct KVMParkedVcpu *vcpu; ++ ++ vcpu = g_malloc0(sizeof(*vcpu)); ++ vcpu->vcpu_id = vcpu_id; ++ vcpu->kvm_fd = cpu->kvm_fd; ++ QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node); ++} ++ ++int kvm_create_vcpu(CPUState *cpu) ++{ ++ unsigned long vcpu_id = cpu->cpu_index; ++ KVMState *s = kvm_state; ++ int ret; ++ ++ DPRINTF("kvm_create_vcpu\n"); ++ ++ /* check if the KVM vCPU already exist but is parked */ ++ ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu)); ++ if (ret > 0) { ++ goto found; ++ } ++ ++ /* create a new KVM vcpu */ ++ ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id); ++ if (ret < 0) { ++ return ret; ++ } ++ ++found: ++ cpu->vcpu_dirty = true; ++ cpu->kvm_fd = ret; ++ cpu->kvm_state = s; ++ cpu->dirty_pages = 0; ++ cpu->throttle_us_per_full = 0; ++ ++ return 0; ++} ++ + static int do_kvm_destroy_vcpu(CPUState *cpu) + { + KVMState *s = kvm_state; + long mmap_size; +- struct KVMParkedVcpu *vcpu = NULL; + int ret = 0; + + DPRINTF("kvm_destroy_vcpu\n"); +@@ -357,10 +398,7 @@ static int do_kvm_destroy_vcpu(CPUState *cpu) + } + } + +- vcpu = g_malloc0(sizeof(*vcpu)); +- vcpu->vcpu_id = kvm_arch_vcpu_id(cpu); +- vcpu->kvm_fd = cpu->kvm_fd; +- QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node); ++ kvm_park_vcpu(cpu); + err: + return ret; + } +@@ -388,7 +426,7 @@ static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id) + } + } + +- return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id); ++ return -1; + } + + int kvm_init_vcpu(CPUState *cpu, Error **errp) +@@ -399,19 +437,14 @@ int kvm_init_vcpu(CPUState *cpu, Error **errp) + + trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu)); + +- ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu)); ++ ret = kvm_create_vcpu(cpu); + if (ret < 0) { +- error_setg_errno(errp, -ret, "kvm_init_vcpu: kvm_get_vcpu failed (%lu)", ++ error_setg_errno(errp, -ret, ++ "kvm_init_vcpu: kvm_create_vcpu failed (%lu)", + kvm_arch_vcpu_id(cpu)); + goto err; + } + +- cpu->kvm_fd = ret; +- cpu->kvm_state = s; +- cpu->vcpu_dirty = true; +- cpu->dirty_pages = 0; +- cpu->throttle_us_per_full = 0; +- + mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); + if (mmap_size < 0) { + ret = mmap_size; +diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h +index b46d6203b4..e534411ddc 100644 +--- a/include/sysemu/kvm.h ++++ b/include/sysemu/kvm.h +@@ -434,6 +434,8 @@ void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len); + + int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr, + hwaddr *phys_addr); ++int kvm_create_vcpu(CPUState *cpu); ++void kvm_park_vcpu(CPUState *cpu); + + #endif /* NEED_CPU_H */ + +-- +2.27.0 + diff --git a/accel-kvm-Use-correct-id-for-parked-vcpu.patch b/accel-kvm-Use-correct-id-for-parked-vcpu.patch new file mode 100644 index 0000000000000000000000000000000000000000..ec759c7697e70a45b61b8c7ba2e27a2be4c35b22 --- /dev/null +++ b/accel-kvm-Use-correct-id-for-parked-vcpu.patch @@ -0,0 +1,32 @@ +From 9de26d69c52db67f48619ad20b8cb9d8ee71e42c Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 15:42:57 +0800 +Subject: [PATCH] accel/kvm: Use correct id for parked vcpu + +kvm_arch_vcpu_id is correct for all platform. + +Signed-off-by: Keqian Zhu +--- + accel/kvm/kvm-all.c | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c +index 6d503aa614..75a3075c14 100644 +--- a/accel/kvm/kvm-all.c ++++ b/accel/kvm/kvm-all.c +@@ -327,11 +327,10 @@ err: + + void kvm_park_vcpu(CPUState *cpu) + { +- unsigned long vcpu_id = cpu->cpu_index; + struct KVMParkedVcpu *vcpu; + + vcpu = g_malloc0(sizeof(*vcpu)); +- vcpu->vcpu_id = vcpu_id; ++ vcpu->vcpu_id = kvm_arch_vcpu_id(cpu); + vcpu->kvm_fd = cpu->kvm_fd; + QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node); + } +-- +2.27.0 + diff --git a/acpi-cpu-Add-cpu_cppc-building-support.patch b/acpi-cpu-Add-cpu_cppc-building-support.patch new file mode 100644 index 0000000000000000000000000000000000000000..2b045f296d183ddf356f8fbc54d8ddd34780f121 --- /dev/null +++ b/acpi-cpu-Add-cpu_cppc-building-support.patch @@ -0,0 +1,72 @@ +From c75a0102a1bb00190b07b06ede8b1f9fa0bdaa3c Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 2 Apr 2024 16:52:10 +0800 +Subject: [PATCH] acpi/cpu: Add cpu_cppc building support + +Signed-off-by: Keqian Zhu +--- + hw/acpi/cpu.c | 8 +++++++- + hw/i386/acpi-build.c | 2 +- + include/hw/acpi/cpu.h | 6 +++++- + 3 files changed, 13 insertions(+), 3 deletions(-) + +diff --git a/hw/acpi/cpu.c b/hw/acpi/cpu.c +index cf0c7e8538..c8c11e51c6 100644 +--- a/hw/acpi/cpu.c ++++ b/hw/acpi/cpu.c +@@ -342,7 +342,9 @@ const VMStateDescription vmstate_cpu_hotplug = { + #define CPU_FW_EJECT_EVENT "CEJF" + + void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, +- build_madt_cpu_fn build_madt_cpu, hwaddr base_addr, ++ build_madt_cpu_fn build_madt_cpu, ++ build_cpu_cppc_fn build_cpu_cppc, ++ hwaddr base_addr, + const char *res_root, + const char *event_handler_method, + AmlRegionSpace rs) +@@ -668,6 +670,10 @@ void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, + aml_append(dev, aml_name_decl("_UID", uid)); + } + ++ if (build_cpu_cppc) { ++ build_cpu_cppc(i, arch_ids->len, dev); ++ } ++ + method = aml_method("_STA", 0, AML_SERIALIZED); + aml_append(method, aml_return(aml_call1(CPU_STS_METHOD, uid))); + aml_append(dev, method); +diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c +index db4ca8a66a..e10799ecc6 100644 +--- a/hw/i386/acpi-build.c ++++ b/hw/i386/acpi-build.c +@@ -1545,7 +1545,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, + .smi_path = pm->smi_on_cpuhp ? "\\_SB.PCI0.SMI0.SMIC" : NULL, + .fw_unplugs_cpu = pm->smi_on_cpu_unplug, + }; +- build_cpus_aml(dsdt, machine, opts, pc_madt_cpu_entry, ++ build_cpus_aml(dsdt, machine, opts, pc_madt_cpu_entry, NULL, + pm->cpu_hp_io_base, "\\_SB.PCI0", "\\_GPE._E02", + AML_SYSTEM_IO); + } +diff --git a/include/hw/acpi/cpu.h b/include/hw/acpi/cpu.h +index 76bc7eb251..b31a2e50d9 100644 +--- a/include/hw/acpi/cpu.h ++++ b/include/hw/acpi/cpu.h +@@ -59,8 +59,12 @@ typedef struct CPUHotplugFeatures { + typedef void (*build_madt_cpu_fn)(int uid, const CPUArchIdList *apic_ids, + GArray *entry, bool force_enabled); + ++typedef void (*build_cpu_cppc_fn)(int uid, int num_cpu, Aml *dev); ++ + void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, +- build_madt_cpu_fn build_madt_cpu, hwaddr base_addr, ++ build_madt_cpu_fn build_madt_cpu, ++ build_cpu_cppc_fn build_cpu_cppc, ++ hwaddr base_addr, + const char *res_root, + const char *event_handler_method, + AmlRegionSpace rs); +-- +2.27.0 + diff --git a/acpi-cpu-Fix-cpu_hotplug_hw_init.patch b/acpi-cpu-Fix-cpu_hotplug_hw_init.patch new file mode 100644 index 0000000000000000000000000000000000000000..bca3afd0faa559c7881fd251b8d164e90e1e5b9f --- /dev/null +++ b/acpi-cpu-Fix-cpu_hotplug_hw_init.patch @@ -0,0 +1,36 @@ +From 14c4062c4acc7d417d163276b65e59073ba18eeb Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 14:51:18 +0800 +Subject: [PATCH] acpi/cpu: Fix cpu_hotplug_hw_init() + +For the present but disabled vCPUs, they will be released after +cpu_hotplug_hw_init(), we should not assign it to AcpiCpuStatus. + +Signed-off-by: Keqian Zhu +--- + hw/acpi/cpu.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/hw/acpi/cpu.c b/hw/acpi/cpu.c +index c922c380aa..b258396e01 100644 +--- a/hw/acpi/cpu.c ++++ b/hw/acpi/cpu.c +@@ -229,7 +229,6 @@ void cpu_hotplug_hw_init(MemoryRegion *as, Object *owner, + for (i = 0; i < id_list->len; i++) { + struct CPUState *cpu = CPU(id_list->cpus[i].cpu); + if (qemu_present_cpu(cpu)) { +- state->devs[i].cpu = cpu; + state->devs[i].is_present = true; + } else { + if (qemu_persistent_cpu(cpu)) { +@@ -240,6 +239,7 @@ void cpu_hotplug_hw_init(MemoryRegion *as, Object *owner, + } + + if (qemu_enabled_cpu(cpu)) { ++ state->devs[i].cpu = cpu; + state->devs[i].is_enabled = true; + } else { + state->devs[i].is_enabled = false; +-- +2.27.0 + diff --git a/acpi-ged-Init-cpu-hotplug-only-when-machine-support-.patch b/acpi-ged-Init-cpu-hotplug-only-when-machine-support-.patch new file mode 100644 index 0000000000000000000000000000000000000000..514292717255b282761e77810dbdcf922f8230d6 --- /dev/null +++ b/acpi-ged-Init-cpu-hotplug-only-when-machine-support-.patch @@ -0,0 +1,47 @@ +From 6e17d32d6df25d4fac1a31da61d89e0bb9c8c7da Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 22:20:20 +0800 +Subject: [PATCH] acpi/ged: Init cpu hotplug only when machine support it + +Signed-off-by: Keqian Zhu +--- + hw/acpi/generic_event_device.c | 16 ++++++++++------ + 1 file changed, 10 insertions(+), 6 deletions(-) + +diff --git a/hw/acpi/generic_event_device.c b/hw/acpi/generic_event_device.c +index 0266733a54..6e4f5f075f 100644 +--- a/hw/acpi/generic_event_device.c ++++ b/hw/acpi/generic_event_device.c +@@ -403,6 +403,7 @@ static void acpi_ged_initfn(Object *obj) + AcpiGedState *s = ACPI_GED(dev); + SysBusDevice *sbd = SYS_BUS_DEVICE(obj); + GEDState *ged_st = &s->ged_state; ++ MachineClass *mc; + + memory_region_init_io(&ged_st->evt, obj, &ged_evt_ops, ged_st, + TYPE_ACPI_GED, ACPI_GED_EVT_SEL_LEN); +@@ -427,12 +428,15 @@ static void acpi_ged_initfn(Object *obj) + TYPE_ACPI_GED "-regs", ACPI_GED_REG_COUNT); + sysbus_init_mmio(sbd, &ged_st->regs); + +- s->cpuhp.device = OBJECT(s); +- memory_region_init(&s->container_cpuhp, OBJECT(dev), "cpuhp container", +- ACPI_CPU_HOTPLUG_REG_LEN); +- sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->container_cpuhp); +- cpu_hotplug_hw_init(&s->container_cpuhp, OBJECT(dev), +- &s->cpuhp_state, 0); ++ mc = MACHINE_GET_CLASS(qdev_get_machine()); ++ if (mc->possible_cpu_arch_ids) { ++ s->cpuhp.device = OBJECT(s); ++ memory_region_init(&s->container_cpuhp, OBJECT(dev), "cpuhp container", ++ ACPI_CPU_HOTPLUG_REG_LEN); ++ sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->container_cpuhp); ++ cpu_hotplug_hw_init(&s->container_cpuhp, OBJECT(dev), ++ &s->cpuhp_state, 0); ++ } + } + + static void acpi_ged_class_init(ObjectClass *class, void *data) +-- +2.27.0 + diff --git a/acpi-ged-Remove-cpuhp-field-of-ged.patch b/acpi-ged-Remove-cpuhp-field-of-ged.patch new file mode 100644 index 0000000000000000000000000000000000000000..760ad92d47604afba039502a7ff07598ac3d83fb --- /dev/null +++ b/acpi-ged-Remove-cpuhp-field-of-ged.patch @@ -0,0 +1,40 @@ +From 7af2722536b4b0d80f6c508066e8e77158869923 Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 23:34:01 +0800 +Subject: [PATCH] acpi/ged: Remove cpuhp field of ged + +It's unused. + +Signed-off-by: Keqian Zhu +--- + hw/acpi/generic_event_device.c | 1 - + include/hw/acpi/generic_event_device.h | 1 - + 2 files changed, 2 deletions(-) + +diff --git a/hw/acpi/generic_event_device.c b/hw/acpi/generic_event_device.c +index 6e4f5f075f..4731a614a3 100644 +--- a/hw/acpi/generic_event_device.c ++++ b/hw/acpi/generic_event_device.c +@@ -430,7 +430,6 @@ static void acpi_ged_initfn(Object *obj) + + mc = MACHINE_GET_CLASS(qdev_get_machine()); + if (mc->possible_cpu_arch_ids) { +- s->cpuhp.device = OBJECT(s); + memory_region_init(&s->container_cpuhp, OBJECT(dev), "cpuhp container", + ACPI_CPU_HOTPLUG_REG_LEN); + sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->container_cpuhp); +diff --git a/include/hw/acpi/generic_event_device.h b/include/hw/acpi/generic_event_device.h +index a803ea818e..90fc41cbb8 100644 +--- a/include/hw/acpi/generic_event_device.h ++++ b/include/hw/acpi/generic_event_device.h +@@ -110,7 +110,6 @@ struct AcpiGedState { + MemoryRegion container_memhp; + CPUHotplugState cpuhp_state; + MemoryRegion container_cpuhp; +- AcpiCpuHotplug cpuhp; + GEDState ged_state; + uint32_t ged_event_bitmap; + qemu_irq irq; +-- +2.27.0 + diff --git a/arm-acpi-Enable-ACPI-support-for-vcpu-hotplug.patch b/arm-acpi-Enable-ACPI-support-for-vcpu-hotplug.patch new file mode 100644 index 0000000000000000000000000000000000000000..0296a6428ac4fb1666e518b9ec80ca20f08fc8a6 --- /dev/null +++ b/arm-acpi-Enable-ACPI-support-for-vcpu-hotplug.patch @@ -0,0 +1,51 @@ +From 37aab238363c8242aa76853396c4f272b5508bca Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Mon, 8 Jun 2020 15:25:35 +0100 +Subject: [PATCH] arm/acpi: Enable ACPI support for vcpu hotplug + +ACPI is required to interface QEMU with the guest. Roughly falls into below +cases, + +1. Convey the possible vcpus config at the machine init time to the guest + using various DSDT tables like MADT etc. +2. Convey vcpu hotplug events to guest(using GED) +3. Assist in evaluation of various ACPI methods(like _EVT, _STA, _OST, _EJ0, + _MAT etc.) +4. Provides ACPI cpu hotplug state and 12 Byte memory mapped cpu hotplug + control register interface to the OSPM/guest corresponding to each possible + vcpu. The register interface consists of various R/W fields and their + handling operations. These are called when ever register fields or memory + regions are accessed(i.e. read or written) by OSPM when ever it evaluates + various ACPI methods. + +Note: lot of this framework code is inherited from the changes already done for + x86 but still some minor changes are required to make it compatible with + ARM64.) + +This patch enables the ACPI support for virtual cpu hotplug. ACPI changes +required will follow in subsequent patches. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/Kconfig | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/hw/arm/Kconfig b/hw/arm/Kconfig +index 3ada335a24..c0a7d0bd58 100644 +--- a/hw/arm/Kconfig ++++ b/hw/arm/Kconfig +@@ -29,6 +29,7 @@ config ARM_VIRT + select ACPI_HW_REDUCED + select ACPI_APEI + select ACPI_VIOT ++ select ACPI_CPU_HOTPLUG + select VIRTIO_MEM_SUPPORTED + select ACPI_CXL + select ACPI_HMAT +-- +2.27.0 + diff --git a/arm-cpu-Some-fixes-for-arm_cpu_unrealizefn.patch b/arm-cpu-Some-fixes-for-arm_cpu_unrealizefn.patch new file mode 100644 index 0000000000000000000000000000000000000000..1b70c456bf22b5738d1f5d172f3ccacd0fc58eb5 --- /dev/null +++ b/arm-cpu-Some-fixes-for-arm_cpu_unrealizefn.patch @@ -0,0 +1,78 @@ +From b394996c99c0af0de870a5d79fff69f01d504b0c Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 14:47:07 +0800 +Subject: [PATCH] arm/cpu: Some fixes for arm_cpu_unrealizefn() + +Some minor fixes for arm_cpu_unrealizefn(). + +Signed-off-by: Keqian Zhu +--- + target/arm/cpu.c | 33 +++++++++++++++++++++------------ + 1 file changed, 21 insertions(+), 12 deletions(-) + +diff --git a/target/arm/cpu.c b/target/arm/cpu.c +index 501f88eb2f..9dd61c10ea 100644 +--- a/target/arm/cpu.c ++++ b/target/arm/cpu.c +@@ -2418,6 +2418,7 @@ static void arm_cpu_unrealizefn(DeviceState *dev) + CPUState *cs = CPU(dev); + bool has_secure; + ++#ifndef CONFIG_USER_ONLY + has_secure = cpu->has_el3 || arm_feature(env, ARM_FEATURE_M_SECURITY); + + /* rock 'n' un-roll, whatever happened in the arm_cpu_realizefn cleanly */ +@@ -2433,30 +2434,38 @@ static void arm_cpu_unrealizefn(DeviceState *dev) + if (has_secure) { + cpu_address_space_destroy(cs, ARMASIdx_S); + } ++#endif + + destroy_cpreg_list(cpu); + arm_cpu_unregister_gdb_regs(cpu); + unregister_cp_regs_for_features(cpu); + ++#ifndef CONFIG_USER_ONLY ++ if (tcg_enabled() && cpu_isar_feature(aa64_rme, cpu)) { ++ arm_unregister_el_change_hooks(cpu); ++ } ++#endif ++ + if (cpu->sau_sregion && arm_feature(env, ARM_FEATURE_M_SECURITY)) { + g_free(env->sau.rbar); + g_free(env->sau.rlar); + } + + if (arm_feature(env, ARM_FEATURE_PMSA) && +- arm_feature(env, ARM_FEATURE_V7) && +- cpu->pmsav7_dregion) { +- if (arm_feature(env, ARM_FEATURE_V8)) { +- g_free(env->pmsav8.rbar[M_REG_NS]); +- g_free(env->pmsav8.rlar[M_REG_NS]); +- if (arm_feature(env, ARM_FEATURE_M_SECURITY)) { +- g_free(env->pmsav8.rbar[M_REG_S]); +- g_free(env->pmsav8.rlar[M_REG_S]); ++ arm_feature(env, ARM_FEATURE_V7)) { ++ if (cpu->pmsav7_dregion) { ++ if (arm_feature(env, ARM_FEATURE_V8)) { ++ g_free(env->pmsav8.rbar[M_REG_NS]); ++ g_free(env->pmsav8.rlar[M_REG_NS]); ++ if (arm_feature(env, ARM_FEATURE_M_SECURITY)) { ++ g_free(env->pmsav8.rbar[M_REG_S]); ++ g_free(env->pmsav8.rlar[M_REG_S]); ++ } ++ } else { ++ g_free(env->pmsav7.drbar); ++ g_free(env->pmsav7.drsr); ++ g_free(env->pmsav7.dracr); + } +- } else { +- g_free(env->pmsav7.drbar); +- g_free(env->pmsav7.drsr); +- g_free(env->pmsav7.dracr); + } + if (cpu->pmsav8r_hdregion) { + g_free(env->pmsav8.hprbar); +-- +2.27.0 + diff --git a/arm-kvm-Set-psci-smccc-filter-only-with-vcpu-hotplug.patch b/arm-kvm-Set-psci-smccc-filter-only-with-vcpu-hotplug.patch new file mode 100644 index 0000000000000000000000000000000000000000..d457c6f33275472cb1e5a546a2822de4b4b979ee --- /dev/null +++ b/arm-kvm-Set-psci-smccc-filter-only-with-vcpu-hotplug.patch @@ -0,0 +1,72 @@ +From 85e8e1ee8560e587845142342f81b218e44cba6a Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 22:07:33 +0800 +Subject: [PATCH] arm/kvm: Set psci smccc filter only with vcpu hotplug + +The smccc filter mechanism is supported by newer Linux kernel, +don't try to do it unconditionaly. + +Signed-off-by: Keqian Zhu +--- + hw/arm/virt.c | 4 +++- + target/arm/kvm.c | 21 ++++++++++++--------- + 2 files changed, 15 insertions(+), 10 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index e60f3431f9..38b5d214a1 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -2366,8 +2366,10 @@ static void machvirt_init(MachineState *machine) + finalize_gic_version(vms); + if (tcg_enabled() || hvf_enabled() || qtest_enabled() || + (vms->gic_version < VIRT_GIC_VERSION_3)) { +- machine->smp.max_cpus = smp_cpus; + mc->has_hotpluggable_cpus = false; ++ } ++ if (!mc->has_hotpluggable_cpus) { ++ machine->smp.max_cpus = smp_cpus; + warn_report("cpu hotplug feature has been disabled"); + } + +diff --git a/target/arm/kvm.c b/target/arm/kvm.c +index 66caf9e5e7..19783d567f 100644 +--- a/target/arm/kvm.c ++++ b/target/arm/kvm.c +@@ -259,6 +259,7 @@ int kvm_arch_get_default_type(MachineState *ms) + + int kvm_arch_init(MachineState *ms, KVMState *s) + { ++ MachineClass *mc = MACHINE_GET_CLASS(ms); + int ret = 0; + + /* For ARM interrupt delivery is always asynchronous, +@@ -316,15 +317,17 @@ int kvm_arch_init(MachineState *ms, KVMState *s) + * filter in the Host KVM. This is required to support features like + * virtual CPU Hotplug on ARM platforms. + */ +- if (kvm_arm_set_smccc_filter(PSCI_0_2_FN64_CPU_ON, +- KVM_SMCCC_FILTER_FWD_TO_USER)) { +- error_report("CPU On PSCI-to-user-space fwd filter install failed"); +- abort(); +- } +- if (kvm_arm_set_smccc_filter(PSCI_0_2_FN_CPU_OFF, +- KVM_SMCCC_FILTER_FWD_TO_USER)) { +- error_report("CPU Off PSCI-to-user-space fwd filter install failed"); +- abort(); ++ if (mc->has_hotpluggable_cpus && ms->smp.max_cpus > ms->smp.cpus) { ++ if (kvm_arm_set_smccc_filter(PSCI_0_2_FN64_CPU_ON, ++ KVM_SMCCC_FILTER_FWD_TO_USER)) { ++ error_report("CPU On PSCI-to-user-space fwd filter install failed"); ++ mc->has_hotpluggable_cpus = false; ++ } ++ if (kvm_arm_set_smccc_filter(PSCI_0_2_FN_CPU_OFF, ++ KVM_SMCCC_FILTER_FWD_TO_USER)) { ++ error_report("CPU Off PSCI-to-user-space fwd filter install failed"); ++ mc->has_hotpluggable_cpus = false; ++ } + } + + kvm_arm_init_debug(s); +-- +2.27.0 + diff --git a/arm-virt-Add-cpu-hotplug-events-to-GED-during-creati.patch b/arm-virt-Add-cpu-hotplug-events-to-GED-during-creati.patch new file mode 100644 index 0000000000000000000000000000000000000000..61f5f9718628156d742d697fe0736400ffe16faf --- /dev/null +++ b/arm-virt-Add-cpu-hotplug-events-to-GED-during-creati.patch @@ -0,0 +1,67 @@ +From f8914ec04d4d892520aa443eaf8018c80516adee Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sun, 6 Aug 2023 16:27:01 +0000 +Subject: [PATCH] arm/virt: Add cpu hotplug events to GED during creation + +Add CPU Hotplug event to the set of supported ged-events during the creation of +GED device during VM init. Also initialize the memory map for CPU Hotplug +control device used in event exchanges between Qemu/VMM and the guest. + +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 5 ++++- + include/hw/arm/virt.h | 1 + + 2 files changed, 5 insertions(+), 1 deletion(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 78ed3c4ba8..155000f22f 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -78,6 +78,7 @@ + #include "hw/mem/pc-dimm.h" + #include "hw/mem/nvdimm.h" + #include "hw/acpi/generic_event_device.h" ++#include "hw/acpi/cpu_hotplug.h" + #include "hw/virtio/virtio-md-pci.h" + #include "hw/virtio/virtio-iommu.h" + #include "hw/char/pl011.h" +@@ -157,6 +158,7 @@ static const MemMapEntry base_memmap[] = { + [VIRT_NVDIMM_ACPI] = { 0x09090000, NVDIMM_ACPI_IO_LEN}, + [VIRT_PVTIME] = { 0x090a0000, 0x00010000 }, + [VIRT_SECURE_GPIO] = { 0x090b0000, 0x00001000 }, ++ [VIRT_CPUHP_ACPI] = { 0x090c0000, ACPI_CPU_HOTPLUG_REG_LEN}, + [VIRT_MMIO] = { 0x0a000000, 0x00000200 }, + [VIRT_CPUFREQ] = { 0x0b000000, 0x00010000 }, + /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */ +@@ -725,7 +727,7 @@ static inline DeviceState *create_acpi_ged(VirtMachineState *vms) + DeviceState *dev; + MachineState *ms = MACHINE(vms); + int irq = vms->irqmap[VIRT_ACPI_GED]; +- uint32_t event = ACPI_GED_PWR_DOWN_EVT; ++ uint32_t event = ACPI_GED_PWR_DOWN_EVT | ACPI_GED_CPU_HOTPLUG_EVT; + + if (ms->ram_slots) { + event |= ACPI_GED_MEM_HOTPLUG_EVT; +@@ -741,6 +743,7 @@ static inline DeviceState *create_acpi_ged(VirtMachineState *vms) + + sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, vms->memmap[VIRT_ACPI_GED].base); + sysbus_mmio_map(SYS_BUS_DEVICE(dev), 1, vms->memmap[VIRT_PCDIMM_ACPI].base); ++ sysbus_mmio_map(SYS_BUS_DEVICE(dev), 3, vms->memmap[VIRT_CPUHP_ACPI].base); + sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, qdev_get_gpio_in(vms->gic, irq)); + + return dev; +diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h +index c2fde0522c..5de0185063 100644 +--- a/include/hw/arm/virt.h ++++ b/include/hw/arm/virt.h +@@ -76,6 +76,7 @@ enum { + VIRT_PCDIMM_ACPI, + VIRT_ACPI_GED, + VIRT_NVDIMM_ACPI, ++ VIRT_CPUHP_ACPI, + VIRT_PVTIME, + VIRT_LOWMEMMAP_LAST, + }; +-- +2.27.0 + diff --git a/arm-virt-Add-update-basic-hot-un-plug-framework.patch b/arm-virt-Add-update-basic-hot-un-plug-framework.patch new file mode 100644 index 0000000000000000000000000000000000000000..ea7c3772eab061766e8df4f16ec75bfb6d399aff --- /dev/null +++ b/arm-virt-Add-update-basic-hot-un-plug-framework.patch @@ -0,0 +1,197 @@ +From 724ab355c047cfb3e970d9ea78577087568eb095 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Fri, 8 May 2020 18:40:19 +0100 +Subject: [PATCH] arm/virt: Add/update basic hot-(un)plug framework + +Add CPU hot-unplug hooks and update hotplug hooks with additional sanity checks +for use in hotplug paths. + +Note, Functional contents of the hooks(now left with TODO comment) shall be +gradually filled in the subsequent patches in an incremental approach to patch +and logic building which would be roughly as follows: +1. (Un-)wiring of interrupts between vCPU<->GIC +2. Sending events to Guest for hot-(un)plug so that guest can take appropriate + actions. +3. Notifying GIC about hot-(un)plug action so that vCPU could be (un-)stitched + to the GIC CPU interface. +4. Updating the Guest with Next boot info for this vCPU in the firmware. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 104 insertions(+) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index bf385a469c..ed354be326 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -83,6 +83,7 @@ + #include "hw/virtio/virtio-iommu.h" + #include "hw/char/pl011.h" + #include "qemu/guest-random.h" ++#include "qapi/qmp/qdict.h" + + #define DEFINE_VIRT_MACHINE_LATEST(major, minor, latest) \ + static void virt_##major##_##minor##_class_init(ObjectClass *oc, \ +@@ -3083,12 +3084,23 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + { + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); + MachineState *ms = MACHINE(hotplug_dev); ++ MachineClass *mc = MACHINE_GET_CLASS(ms); + ARMCPU *cpu = ARM_CPU(dev); + CPUState *cs = CPU(dev); + CPUArchId *cpu_slot; + int32_t min_cpuid = 0; + int32_t max_cpuid; + ++ if (dev->hotplugged && !vms->acpi_dev) { ++ error_setg(errp, "GED acpi device does not exists"); ++ return; ++ } ++ ++ if (dev->hotplugged && !mc->has_hotpluggable_cpus) { ++ error_setg(errp, "CPU hotplug not supported on this machine"); ++ return; ++ } ++ + /* sanity check the cpu */ + if (!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) { + error_setg(errp, "Invalid CPU type, expected cpu type: '%s'", +@@ -3137,6 +3149,22 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + } + virt_cpu_set_properties(OBJECT(cs), cpu_slot, errp); + ++ /* ++ * Fix the GIC for this new vCPU being plugged. The QOM CPU object for the ++ * new vCPU need to be updated in the corresponding QOM GICv3CPUState object ++ * We also need to re-wire the IRQs for this new CPU object. This update ++ * is limited to the QOM only and does not affects the KVM. Later has ++ * already been pre-sized with possible CPU at VM init time. This is a ++ * workaround to the constraints posed by ARM architecture w.r.t supporting ++ * CPU Hotplug. Specification does not exist for the later. ++ * This patch-up is required both for {cold,hot}-plugged vCPUs. Cold-inited ++ * vCPUs have their GIC state initialized during machvit_init(). ++ */ ++ if (vms->acpi_dev) { ++ /* TODO: update GIC about this hotplug change here */ ++ /* TODO: wire the GIC<->CPU irqs */ ++ } ++ + /* + * To give persistent presence view of vCPUs to the guest, ACPI might need + * to fake the presence of the vCPUs to the guest but keep them disabled. +@@ -3148,6 +3176,7 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + Error **errp) + { ++ VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); + MachineState *ms = MACHINE(hotplug_dev); + CPUState *cs = CPU(dev); + CPUArchId *cpu_slot; +@@ -3156,10 +3185,81 @@ static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); + cpu_slot->cpu = OBJECT(dev); + ++ /* ++ * Update the ACPI Hotplug state both for vCPUs being {hot,cold}-plugged. ++ * vCPUs can be cold-plugged using '-device' option. For vCPUs being hot ++ * plugged, guest is also notified. ++ */ ++ if (vms->acpi_dev) { ++ /* TODO: update acpi hotplug state. Send cpu hotplug event to guest */ ++ /* TODO: register cpu for reset & update F/W info for the next boot */ ++ } ++ + cs->disabled = false; + return; + } + ++static void virt_cpu_unplug_request(HotplugHandler *hotplug_dev, ++ DeviceState *dev, Error **errp) ++{ ++ MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine()); ++ VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); ++ ARMCPU *cpu = ARM_CPU(dev); ++ CPUState *cs = CPU(dev); ++ ++ if (!vms->acpi_dev || !dev->realized) { ++ error_setg(errp, "GED does not exists or device is not realized!"); ++ return; ++ } ++ ++ if (!mc->has_hotpluggable_cpus) { ++ error_setg(errp, "CPU hot(un)plug not supported on this machine"); ++ return; ++ } ++ ++ if (cs->cpu_index == first_cpu->cpu_index) { ++ error_setg(errp, "Boot CPU(id%d=%d:%d:%d:%d) hot-unplug not supported", ++ first_cpu->cpu_index, cpu->socket_id, cpu->cluster_id, ++ cpu->core_id, cpu->thread_id); ++ return; ++ } ++ ++ /* TODO: request cpu hotplug from guest */ ++ ++ return; ++} ++ ++static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, ++ Error **errp) ++{ ++ VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); ++ MachineState *ms = MACHINE(hotplug_dev); ++ CPUState *cs = CPU(dev); ++ CPUArchId *cpu_slot; ++ ++ if (!vms->acpi_dev || !dev->realized) { ++ error_setg(errp, "GED does not exists or device is not realized!"); ++ return; ++ } ++ ++ cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); ++ ++ /* TODO: update the acpi cpu hotplug state for cpu hot-unplug */ ++ ++ /* TODO: unwire the gic-cpu irqs here */ ++ /* TODO: update the GIC about this hot unplug change */ ++ ++ /* TODO: unregister cpu for reset & update F/W info for the next boot */ ++ ++ qobject_unref(dev->opts); ++ dev->opts = NULL; ++ ++ cpu_slot->cpu = NULL; ++ cs->disabled = true; ++ ++ return; ++} ++ + static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev, + DeviceState *dev, Error **errp) + { +@@ -3284,6 +3384,8 @@ static void virt_machine_device_unplug_request_cb(HotplugHandler *hotplug_dev, + } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) { + virtio_md_pci_unplug_request(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), + errp); ++ } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { ++ virt_cpu_unplug_request(hotplug_dev, dev, errp); + } else { + error_setg(errp, "device unplug request for unsupported device" + " type: %s", object_get_typename(OBJECT(dev))); +@@ -3297,6 +3399,8 @@ static void virt_machine_device_unplug_cb(HotplugHandler *hotplug_dev, + virt_dimm_unplug(hotplug_dev, dev, errp); + } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) { + virtio_md_pci_unplug(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp); ++ } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { ++ virt_cpu_unplug(hotplug_dev, dev, errp); + } else { + error_setg(errp, "virt: device unplug for unsupported device" + " type: %s", object_get_typename(OBJECT(dev))); +-- +2.27.0 + diff --git a/arm-virt-Changes-to-un-wire-GICC-vCPU-IRQs-during-ho.patch b/arm-virt-Changes-to-un-wire-GICC-vCPU-IRQs-during-ho.patch new file mode 100644 index 0000000000000000000000000000000000000000..61c298d7bec0adbb7fc302343e5b09f94dd947b5 --- /dev/null +++ b/arm-virt-Changes-to-un-wire-GICC-vCPU-IRQs-during-ho.patch @@ -0,0 +1,221 @@ +From a68abeefcbd78daaf7179b922f6b9040b4b63101 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sat, 9 May 2020 15:50:33 +0100 +Subject: [PATCH] arm/virt: Changes to (un)wire GICC<->vCPU IRQs during + hot-(un)plug + +Refactors the existing GIC create code to extract common code to wire the +vcpu<->gic interrupts. This function could be used with cold-plug case and also +used when vCPU is hot-plugged. It also introduces a new function to unwire the +vcpu<->gic interrupts for the vCPU hot-unplug cases. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 138 ++++++++++++++++++++++++++++------------- + hw/core/gpio.c | 2 +- + include/hw/qdev-core.h | 2 + + 3 files changed, 99 insertions(+), 43 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index ed354be326..97bf4cca11 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -798,6 +798,99 @@ static void create_v2m(VirtMachineState *vms) + vms->msi_controller = VIRT_MSI_CTRL_GICV2M; + } + ++/* ++ * Mapping from the output timer irq lines from the CPU to the GIC PPI inputs ++ * we use for the virt board. ++ */ ++const int timer_irq[] = { ++ [GTIMER_PHYS] = ARCH_TIMER_NS_EL1_IRQ, ++ [GTIMER_VIRT] = ARCH_TIMER_VIRT_IRQ, ++ [GTIMER_HYP] = ARCH_TIMER_NS_EL2_IRQ, ++ [GTIMER_SEC] = ARCH_TIMER_S_EL1_IRQ, ++}; ++ ++static void unwire_gic_cpu_irqs(VirtMachineState *vms, CPUState *cs) ++{ ++ MachineState *ms = MACHINE(vms); ++ unsigned int max_cpus = ms->smp.max_cpus; ++ DeviceState *cpudev = DEVICE(cs); ++ DeviceState *gicdev = vms->gic; ++ int cpu = CPU(cs)->cpu_index; ++ int type = vms->gic_version; ++ int irq; ++ ++ for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) { ++ qdev_disconnect_gpio_out_named(cpudev, NULL, irq); ++ } ++ ++ if (type != VIRT_GIC_VERSION_2) { ++ qdev_disconnect_gpio_out_named(cpudev, "gicv3-maintenance-interrupt", ++ 0); ++ } else if (vms->virt) { ++ qdev_disconnect_gpio_out_named(gicdev, SYSBUS_DEVICE_GPIO_IRQ, ++ cpu + 4 * max_cpus); ++ } ++ ++ /* ++ * RFC: Question: This currently does not takes care of intimating the ++ * devices which might be sitting on system bus. Do we need a ++ * sysbus_disconnect_irq() which also does the job of notification beside ++ * disconnection? ++ */ ++ qdev_disconnect_gpio_out_named(cpudev, "pmu-interrupt", 0); ++ qdev_disconnect_gpio_out_named(gicdev, SYSBUS_DEVICE_GPIO_IRQ, cpu); ++ qdev_disconnect_gpio_out_named(gicdev, ++ SYSBUS_DEVICE_GPIO_IRQ, cpu + max_cpus); ++ qdev_disconnect_gpio_out_named(gicdev, SYSBUS_DEVICE_GPIO_IRQ, ++ cpu + 2 * max_cpus); ++ qdev_disconnect_gpio_out_named(gicdev, SYSBUS_DEVICE_GPIO_IRQ, ++ cpu + 3 * max_cpus); ++} ++ ++static void wire_gic_cpu_irqs(VirtMachineState *vms, CPUState *cs) ++{ ++ MachineState *ms = MACHINE(vms); ++ unsigned int max_cpus = ms->smp.max_cpus; ++ DeviceState *cpudev = DEVICE(cs); ++ DeviceState *gicdev = vms->gic; ++ int cpu = CPU(cs)->cpu_index; ++ int type = vms->gic_version; ++ SysBusDevice *gicbusdev; ++ int intidbase; ++ int irq; ++ ++ intidbase = NUM_IRQS + cpu * GIC_INTERNAL; ++ ++ for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) { ++ qdev_connect_gpio_out(cpudev, irq, ++ qdev_get_gpio_in(gicdev, ++ intidbase + timer_irq[irq])); ++ } ++ ++ gicbusdev = SYS_BUS_DEVICE(gicdev); ++ if (type != VIRT_GIC_VERSION_2) { ++ qemu_irq qirq = qdev_get_gpio_in(gicdev, ++ intidbase + ARCH_GIC_MAINT_IRQ); ++ qdev_connect_gpio_out_named(cpudev, "gicv3-maintenance-interrupt", ++ 0, qirq); ++ } else if (vms->virt) { ++ qemu_irq qirq = qdev_get_gpio_in(gicdev, ++ intidbase + ARCH_GIC_MAINT_IRQ); ++ sysbus_connect_irq(gicbusdev, cpu + 4 * max_cpus, qirq); ++ } ++ ++ qdev_connect_gpio_out_named(cpudev, "pmu-interrupt", 0, ++ qdev_get_gpio_in(gicdev, ++ intidbase + VIRTUAL_PMU_IRQ)); ++ sysbus_connect_irq(gicbusdev, cpu, qdev_get_gpio_in(cpudev, ARM_CPU_IRQ)); ++ sysbus_connect_irq(gicbusdev, cpu + max_cpus, ++ qdev_get_gpio_in(cpudev, ARM_CPU_FIQ)); ++ sysbus_connect_irq(gicbusdev, cpu + 2 * max_cpus, ++ qdev_get_gpio_in(cpudev, ARM_CPU_VIRQ)); ++ sysbus_connect_irq(gicbusdev, cpu + 3 * max_cpus, ++ qdev_get_gpio_in(cpudev, ARM_CPU_VFIQ)); ++} ++ + static void create_gic(VirtMachineState *vms, MemoryRegion *mem) + { + MachineState *ms = MACHINE(vms); +@@ -894,46 +987,7 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) + * and the GIC's IRQ/FIQ/VIRQ/VFIQ interrupt outputs to the CPU's inputs. + */ + for (i = 0; i < smp_cpus; i++) { +- DeviceState *cpudev = DEVICE(qemu_get_cpu(i)); +- int intidbase = NUM_IRQS + i * GIC_INTERNAL; +- /* Mapping from the output timer irq lines from the CPU to the +- * GIC PPI inputs we use for the virt board. +- */ +- const int timer_irq[] = { +- [GTIMER_PHYS] = ARCH_TIMER_NS_EL1_IRQ, +- [GTIMER_VIRT] = ARCH_TIMER_VIRT_IRQ, +- [GTIMER_HYP] = ARCH_TIMER_NS_EL2_IRQ, +- [GTIMER_SEC] = ARCH_TIMER_S_EL1_IRQ, +- }; +- +- for (unsigned irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) { +- qdev_connect_gpio_out(cpudev, irq, +- qdev_get_gpio_in(vms->gic, +- intidbase + timer_irq[irq])); +- } +- +- if (vms->gic_version != VIRT_GIC_VERSION_2) { +- qemu_irq irq = qdev_get_gpio_in(vms->gic, +- intidbase + ARCH_GIC_MAINT_IRQ); +- qdev_connect_gpio_out_named(cpudev, "gicv3-maintenance-interrupt", +- 0, irq); +- } else if (vms->virt) { +- qemu_irq irq = qdev_get_gpio_in(vms->gic, +- intidbase + ARCH_GIC_MAINT_IRQ); +- sysbus_connect_irq(gicbusdev, i + 4 * max_cpus, irq); +- } +- +- qdev_connect_gpio_out_named(cpudev, "pmu-interrupt", 0, +- qdev_get_gpio_in(vms->gic, intidbase +- + VIRTUAL_PMU_IRQ)); +- +- sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev, ARM_CPU_IRQ)); +- sysbus_connect_irq(gicbusdev, i + max_cpus, +- qdev_get_gpio_in(cpudev, ARM_CPU_FIQ)); +- sysbus_connect_irq(gicbusdev, i + 2 * max_cpus, +- qdev_get_gpio_in(cpudev, ARM_CPU_VIRQ)); +- sysbus_connect_irq(gicbusdev, i + 3 * max_cpus, +- qdev_get_gpio_in(cpudev, ARM_CPU_VFIQ)); ++ wire_gic_cpu_irqs(vms, qemu_get_cpu(i)); + } + + fdt_add_gic_node(vms); +@@ -3162,7 +3216,7 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + */ + if (vms->acpi_dev) { + /* TODO: update GIC about this hotplug change here */ +- /* TODO: wire the GIC<->CPU irqs */ ++ wire_gic_cpu_irqs(vms, cs); + } + + /* +@@ -3246,7 +3300,7 @@ static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, + + /* TODO: update the acpi cpu hotplug state for cpu hot-unplug */ + +- /* TODO: unwire the gic-cpu irqs here */ ++ unwire_gic_cpu_irqs(vms, cs); + /* TODO: update the GIC about this hot unplug change */ + + /* TODO: unregister cpu for reset & update F/W info for the next boot */ +diff --git a/hw/core/gpio.c b/hw/core/gpio.c +index 80d07a6ec9..abb164d5c0 100644 +--- a/hw/core/gpio.c ++++ b/hw/core/gpio.c +@@ -143,7 +143,7 @@ qemu_irq qdev_get_gpio_out_connector(DeviceState *dev, const char *name, int n) + + /* disconnect a GPIO output, returning the disconnected input (if any) */ + +-static qemu_irq qdev_disconnect_gpio_out_named(DeviceState *dev, ++qemu_irq qdev_disconnect_gpio_out_named(DeviceState *dev, + const char *name, int n) + { + char *propname = g_strdup_printf("%s[%d]", +diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h +index 151d968238..2d3661d6cd 100644 +--- a/include/hw/qdev-core.h ++++ b/include/hw/qdev-core.h +@@ -739,6 +739,8 @@ qemu_irq qdev_get_gpio_out_connector(DeviceState *dev, const char *name, int n); + */ + qemu_irq qdev_intercept_gpio_out(DeviceState *dev, qemu_irq icpt, + const char *name, int n); ++qemu_irq qdev_disconnect_gpio_out_named(DeviceState *dev, ++ const char *name, int n); + + BusState *qdev_get_child_bus(DeviceState *dev, const char *name); + +-- +2.27.0 + diff --git a/arm-virt-Consider-has_ged-when-set-mc-has_hotpluggab.patch b/arm-virt-Consider-has_ged-when-set-mc-has_hotpluggab.patch new file mode 100644 index 0000000000000000000000000000000000000000..27ca6d7ab1b918bffda8a3e78beae1626d19d6fb --- /dev/null +++ b/arm-virt-Consider-has_ged-when-set-mc-has_hotpluggab.patch @@ -0,0 +1,73 @@ +From baa26f2fc075522f91c3e9a332fc4fa3f3b167bf Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 22:55:49 +0800 +Subject: [PATCH] arm/virt: Consider has_ged when set mc->has_hotpluggable_cpus + +Vcpu hotplug relies on ged device. + +Signed-off-by: Keqian Zhu +--- + hw/arm/virt.c | 26 ++++++++++++++++---------- + 1 file changed, 16 insertions(+), 10 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 38b5d214a1..00e57f2d75 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -2357,6 +2357,7 @@ static void machvirt_init(MachineState *machine) + bool has_ged = !vmc->no_ged; + unsigned int smp_cpus = machine->smp.cpus; + unsigned int max_cpus = machine->smp.max_cpus; ++ ObjectClass *cpu_class; + + if (!cpu_type_valid(machine->cpu_type)) { + error_report("mach-virt: CPU type %s not supported", machine->cpu_type); +@@ -2364,14 +2365,6 @@ static void machvirt_init(MachineState *machine) + } + + finalize_gic_version(vms); +- if (tcg_enabled() || hvf_enabled() || qtest_enabled() || +- (vms->gic_version < VIRT_GIC_VERSION_3)) { +- mc->has_hotpluggable_cpus = false; +- } +- if (!mc->has_hotpluggable_cpus) { +- machine->smp.max_cpus = smp_cpus; +- warn_report("cpu hotplug feature has been disabled"); +- } + + possible_cpus = mc->possible_cpu_arch_ids(machine); + +@@ -2501,6 +2494,21 @@ static void machvirt_init(MachineState *machine) + create_fdt(vms); + qemu_log("cpu init start\n"); + ++ cpu_class = object_class_by_name(machine->cpu_type); ++ has_ged = has_ged && firmware_loaded && ++ virt_is_acpi_enabled(vms) && ++ !!object_class_dynamic_cast(cpu_class, TYPE_AARCH64_CPU); ++ if (tcg_enabled() || hvf_enabled() || qtest_enabled() || ++ (vms->gic_version < VIRT_GIC_VERSION_3) || !has_ged) { ++ mc->has_hotpluggable_cpus = false; ++ } ++ if (!mc->has_hotpluggable_cpus) { ++ if (machine->smp.max_cpus > smp_cpus) { ++ warn_report("cpu hotplug feature has been disabled"); ++ } ++ machine->smp.max_cpus = smp_cpus; ++ } ++ + notifier_list_init(&vms->cpuhp_notifiers); + possible_cpus = mc->possible_cpu_arch_ids(machine); + assert(possible_cpus->len == max_cpus); +@@ -2581,8 +2589,6 @@ static void machvirt_init(MachineState *machine) + + create_gic(vms, sysmem); + +- has_ged = has_ged && aarch64 && firmware_loaded && +- virt_is_acpi_enabled(vms); + if (has_ged) { + vms->acpi_dev = create_acpi_ged(vms); + } +-- +2.27.0 + diff --git a/arm-virt-Create-GED-dev-before-disabled-CPU-Objs-are.patch b/arm-virt-Create-GED-dev-before-disabled-CPU-Objs-are.patch new file mode 100644 index 0000000000000000000000000000000000000000..d120fe42a27c540e4c58a345b6425c7278453f26 --- /dev/null +++ b/arm-virt-Create-GED-dev-before-disabled-CPU-Objs-are.patch @@ -0,0 +1,54 @@ +From 028d71744dfeedabfa67d629c71a6ed5e494cc68 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Tue, 29 Aug 2023 00:47:05 +0000 +Subject: [PATCH] arm/virt: Create GED dev before *disabled* CPU Objs are + destroyed + +ACPI CPU hotplug state (is_present=_STA.PRESENT, is_enabled=_STA.ENABLED) for +all the possible vCPUs MUST be initialized during machine init. This is done +during the creation of the GED device. VMM/Qemu MUST expose/fake the ACPI state +of the disabled vCPUs to the Guest kernel as 'present' (_STA.PRESENT) always +i.e. ACPI persistent. if the 'disabled' vCPU objectes are destroyed before the +GED device has been created then their ACPI hotplug state might not get +initialized correctly as acpi_persistent flag is part of the CPUState. This will +expose wrong status of the unplugged vCPUs to the Guest kernel. + +Hence, moving the GED device creation before disabled vCPU objects get destroyed +as part of the post CPU init routine. + +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 10 +++++++--- + 1 file changed, 7 insertions(+), 3 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 155000f22f..818398e753 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -2472,6 +2472,12 @@ static void machvirt_init(MachineState *machine) + + create_gic(vms, sysmem); + ++ has_ged = has_ged && aarch64 && firmware_loaded && ++ virt_is_acpi_enabled(vms); ++ if (has_ged) { ++ vms->acpi_dev = create_acpi_ged(vms); ++ } ++ + virt_cpu_post_init(vms, sysmem); + + fdt_add_pmu_nodes(vms); +@@ -2496,9 +2502,7 @@ static void machvirt_init(MachineState *machine) + + create_pcie(vms); + +- if (has_ged && aarch64 && firmware_loaded && virt_is_acpi_enabled(vms)) { +- vms->acpi_dev = create_acpi_ged(vms); +- } else { ++ if (!has_ged) { + create_gpio_devices(vms, VIRT_GPIO, sysmem); + } + +-- +2.27.0 + diff --git a/arm-virt-Fix-adjudgement-of-core_id-for-vcpu-hotplug.patch b/arm-virt-Fix-adjudgement-of-core_id-for-vcpu-hotplug.patch new file mode 100644 index 0000000000000000000000000000000000000000..30a11521e48cbfda6ec3bbf2d9861bf189398472 --- /dev/null +++ b/arm-virt-Fix-adjudgement-of-core_id-for-vcpu-hotplug.patch @@ -0,0 +1,47 @@ +From 00a78edf572783c18a1d4945758371c0f175e321 Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 15:41:14 +0800 +Subject: [PATCH] arm/virt: Fix adjudgement of core_id for vcpu hotplugged + +The core_id should between 0 and ms->smp.cores - 1. + +Signed-off-by: Keqian Zhu +--- + hw/arm/virt.c | 14 +++----------- + 1 file changed, 3 insertions(+), 11 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 934b0412ef..e60f3431f9 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -3170,8 +3170,6 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + ARMCPU *cpu = ARM_CPU(dev); + CPUState *cs = CPU(dev); + CPUArchId *cpu_slot; +- int32_t min_cpuid = 0; +- int32_t max_cpuid; + + if (dev->hotplugged && !vms->acpi_dev) { + error_setg(errp, "GED acpi device does not exists"); +@@ -3196,15 +3194,9 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + return; + } + +- max_cpuid = ms->possible_cpus->len - 1; +- if (!dev->hotplugged) { +- min_cpuid = vms->acpi_dev ? ms->smp.cpus : 0; +- max_cpuid = vms->acpi_dev ? max_cpuid : ms->smp.cpus - 1; +- } +- +- if ((cpu->core_id < min_cpuid) || (cpu->core_id > max_cpuid)) { +- error_setg(errp, "Invalid core-id %d specified, correct range %d:%d", +- cpu->core_id, min_cpuid, max_cpuid); ++ if ((cpu->core_id < 0) || (cpu->core_id >= ms->smp.cores)) { ++ error_setg(errp, "Invalid core-id %d specified, correct range 0:%u", ++ cpu->core_id, ms->smp.cores - 1); + return; + } + +-- +2.27.0 + diff --git a/arm-virt-Init-PMU-at-host-for-all-possible-vcpus.patch b/arm-virt-Init-PMU-at-host-for-all-possible-vcpus.patch new file mode 100644 index 0000000000000000000000000000000000000000..89fd4ca49fe6115d3ed7d19eb2a265bbe462bc46 --- /dev/null +++ b/arm-virt-Init-PMU-at-host-for-all-possible-vcpus.patch @@ -0,0 +1,71 @@ +From c375e6fdc49f7d3d0232786e4cfd8b792379107c Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Wed, 6 May 2020 14:12:34 +0100 +Subject: [PATCH] arm/virt: Init PMU at host for all possible vcpus + +PMU for all possible vCPUs must be initialized at the VM initialization time. +Refactor existing code to accomodate possible vCPUs. This also assumes that all +processor being used are identical. + +Past discussion for reference: +Link: https://lists.gnu.org/archive/html/qemu-devel/2020-06/msg00131.html + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 12 ++++++++---- + include/hw/arm/virt.h | 1 + + 2 files changed, 9 insertions(+), 4 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 08ba255317..78ed3c4ba8 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -2055,12 +2055,14 @@ static void finalize_gic_version(VirtMachineState *vms) + */ + static void virt_cpu_post_init(VirtMachineState *vms, MemoryRegion *sysmem) + { ++ CPUArchIdList *possible_cpus = vms->parent.possible_cpus; + int max_cpus = MACHINE(vms)->smp.max_cpus; +- bool aarch64, pmu, steal_time; ++ bool aarch64, steal_time; + CPUState *cpu; ++ int n; + + aarch64 = object_property_get_bool(OBJECT(first_cpu), "aarch64", NULL); +- pmu = object_property_get_bool(OBJECT(first_cpu), "pmu", NULL); ++ vms->pmu = object_property_get_bool(OBJECT(first_cpu), "pmu", NULL); + steal_time = object_property_get_bool(OBJECT(first_cpu), + "kvm-steal-time", NULL); + +@@ -2087,8 +2089,10 @@ static void virt_cpu_post_init(VirtMachineState *vms, MemoryRegion *sysmem) + memory_region_add_subregion(sysmem, pvtime_reg_base, pvtime); + } + +- CPU_FOREACH(cpu) { +- if (pmu) { ++ for (n = 0; n < possible_cpus->len; n++) { ++ cpu = qemu_get_possible_cpu(n); ++ ++ if (vms->pmu) { + assert(arm_feature(&ARM_CPU(cpu)->env, ARM_FEATURE_PMU)); + if (kvm_irqchip_in_kernel()) { + kvm_arm_pmu_set_irq(cpu, VIRTUAL_PMU_IRQ); +diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h +index a6977bade5..c2fde0522c 100644 +--- a/include/hw/arm/virt.h ++++ b/include/hw/arm/virt.h +@@ -155,6 +155,7 @@ struct VirtMachineState { + bool ras; + bool mte; + bool dtb_randomness; ++ bool pmu; + OnOffAuto acpi; + VirtGICType gic_version; + VirtIOMMUType iommu; +-- +2.27.0 + diff --git a/arm-virt-Make-ARM-vCPU-present-status-ACPI-persisten.patch b/arm-virt-Make-ARM-vCPU-present-status-ACPI-persisten.patch new file mode 100644 index 0000000000000000000000000000000000000000..c8e661145e7b1b42272971979d463d51a5ee7b4e --- /dev/null +++ b/arm-virt-Make-ARM-vCPU-present-status-ACPI-persisten.patch @@ -0,0 +1,97 @@ +From 3780dddd4fc8f0471525c50893e24846d1474692 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Tue, 8 Aug 2023 00:43:18 +0000 +Subject: [PATCH] arm/virt: Make ARM vCPU *present* status ACPI *persistent* + +ARM arch does not allow CPUs presence to be changed [1] after kernel has booted. +Hence, firmware/ACPI/Qemu must ensure persistent view of the vCPUs to the Guest +kernel even when they are not present in the QoM i.e. are unplugged or are +yet-to-be-plugged + +References: +[1] Check comment 5 in the bugzilla entry + Link: https://bugzilla.tianocore.org/show_bug.cgi?id=4481#c5 + +Signed-off-by: Salil Mehta +--- + cpu-common.c | 6 ++++++ + hw/arm/virt.c | 7 +++++++ + include/hw/core/cpu.h | 20 ++++++++++++++++++++ + 3 files changed, 33 insertions(+) + +diff --git a/cpu-common.c b/cpu-common.c +index d041a351ab..da52e45760 100644 +--- a/cpu-common.c ++++ b/cpu-common.c +@@ -128,6 +128,12 @@ bool qemu_enabled_cpu(CPUState *cpu) + return cpu && !cpu->disabled; + } + ++bool qemu_persistent_cpu(CPUState *cpu) ++{ ++ /* cpu state can be faked to the guest via acpi */ ++ return cpu->acpi_persistent; ++} ++ + uint64_t qemu_get_cpu_archid(int cpu_index) + { + MachineState *ms = MACHINE(qdev_get_machine()); +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 818398e753..91b2653c03 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -3104,6 +3104,13 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + return; + } + virt_cpu_set_properties(OBJECT(cs), cpu_slot, errp); ++ ++ /* ++ * To give persistent presence view of vCPUs to the guest, ACPI might need ++ * to fake the presence of the vCPUs to the guest but keep them disabled. ++ * This shall be used during the init of ACPI Hotplug state and hot-unplug ++ */ ++ cs->acpi_persistent = true; + } + + static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, +diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h +index fdfb952259..0ca778eb75 100644 +--- a/include/hw/core/cpu.h ++++ b/include/hw/core/cpu.h +@@ -550,6 +550,13 @@ struct CPUState { + * By default every CPUState is enabled as of now across all archs. + */ + bool disabled; ++ /* ++ * On certain architectures, to give persistent view of the 'presence' of ++ * vCPUs to the guest, ACPI might need to fake the 'presence' of the vCPUs ++ * but keep them ACPI disabled to the guest. This is done by returning ++ * _STA.PRES=True and _STA.Ena=False for the unplugged vCPUs in QEMU QoM. ++ */ ++ bool acpi_persistent; + /* TODO Move common fields from CPUArchState here. */ + int cpu_index; + int cluster_index; +@@ -957,6 +964,19 @@ bool qemu_present_cpu(CPUState *cpu); + */ + bool qemu_enabled_cpu(CPUState *cpu); + ++/** ++ * qemu_persistent_cpu: ++ * @cpu: The vCPU to check ++ * ++ * Checks if the vCPU state should always be reflected as *present* via ACPI ++ * to the Guest. By default, this is False on all architectures and has to be ++ * explicity set during initialization. ++ * ++ * Returns: True if it is ACPI 'persistent' CPU ++ * ++ */ ++bool qemu_persistent_cpu(CPUState *cpu); ++ + /** + * qemu_get_cpu_archid: + * @cpu_index: possible vCPU for which arch-id needs to be retreived +-- +2.27.0 + diff --git a/arm-virt-Release-objects-for-disabled-possible-vCPUs.patch b/arm-virt-Release-objects-for-disabled-possible-vCPUs.patch new file mode 100644 index 0000000000000000000000000000000000000000..9a1198a9d63b5203f251134500fc03dc3847611a --- /dev/null +++ b/arm-virt-Release-objects-for-disabled-possible-vCPUs.patch @@ -0,0 +1,88 @@ +From 097e3b46a7eede0182a846f7b993e14d3eed83b7 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Tue, 9 Jun 2020 03:01:08 +0100 +Subject: [PATCH] arm/virt: Release objects for *disabled* possible vCPUs after + init + +During machvirt_init(), QOM ARMCPU objects are also pre-created along with the +corresponding KVM vCPUs in the host for all possible vCPUs. This necessary +because of the architectural constraint, KVM restricts the deferred creation of +the KVM vCPUs and VGIC initialization/sizing after VM init. Hence, VGIC is +pre-sized with possible vCPUs. + +After initialization of the machine is complete disabled possible KVM vCPUs are +then parked at the per-virt-machine list "kvm_parked_vcpus" and we release the +QOM ARMCPU objects for the disabled vCPUs. These shall be re-created at the time +when vCPU is hotplugged again. QOM ARMCPU object is then re-attached with +corresponding parked KVM vCPU. + +Alternatively, we could've never released the QOM CPU objects and kept on +reusing. This approach might require some modifications of qdevice_add() +interface to get old ARMCPU object instead of creating a new one for the hotplug +request. + +Each of the above approaches come with their own pros and cons. This prototype +uses the 1st approach.(suggestions are welcome!) + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 32 ++++++++++++++++++++++++++++++++ + 1 file changed, 32 insertions(+) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 91b2653c03..bf385a469c 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -2060,6 +2060,7 @@ static void virt_cpu_post_init(VirtMachineState *vms, MemoryRegion *sysmem) + { + CPUArchIdList *possible_cpus = vms->parent.possible_cpus; + int max_cpus = MACHINE(vms)->smp.max_cpus; ++ MachineState *ms = MACHINE(vms); + bool aarch64, steal_time; + CPUState *cpu; + int n; +@@ -2120,6 +2121,37 @@ static void virt_cpu_post_init(VirtMachineState *vms, MemoryRegion *sysmem) + } + } + } ++ ++ if (kvm_enabled() || tcg_enabled()) { ++ for (n = 0; n < possible_cpus->len; n++) { ++ cpu = qemu_get_possible_cpu(n); ++ ++ /* ++ * Now, GIC has been sized with possible CPUs and we dont require ++ * disabled vCPU objects to be represented in the QOM. Release the ++ * disabled ARMCPU objects earlier used during init for pre-sizing. ++ * ++ * We fake to the guest through ACPI about the presence(_STA.PRES=1) ++ * of these non-existent vCPUs at VMM/qemu and present these as ++ * disabled vCPUs(_STA.ENA=0) so that they cant be used. These vCPUs ++ * can be later added to the guest through hotplug exchanges when ++ * ARMCPU objects are created back again using 'device_add' QMP ++ * command. ++ */ ++ /* ++ * RFC: Question: Other approach could've been to keep them forever ++ * and release it only once when qemu exits as part of finalize or ++ * when new vCPU is hotplugged. In the later old could be released ++ * for the newly created object for the same vCPU? ++ */ ++ if (!qemu_enabled_cpu(cpu)) { ++ CPUArchId *cpu_slot; ++ cpu_slot = virt_find_cpu_slot(ms, cpu->cpu_index); ++ cpu_slot->cpu = NULL; ++ object_unref(OBJECT(cpu)); ++ } ++ } ++ } + } + + static void virt_cpu_set_properties(Object *cpuobj, const CPUArchId *cpu_slot, +-- +2.27.0 + diff --git a/arm-virt-Require-mc-has_hotpluggable_cpus-for-cold-p.patch b/arm-virt-Require-mc-has_hotpluggable_cpus-for-cold-p.patch new file mode 100644 index 0000000000000000000000000000000000000000..d64e1a07a1cdb9de75b21fe11f1cac4b340a596f --- /dev/null +++ b/arm-virt-Require-mc-has_hotpluggable_cpus-for-cold-p.patch @@ -0,0 +1,55 @@ +From 519699c61eeb980bb7d7f443eb95c0406aae82da Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 23:05:39 +0800 +Subject: [PATCH] arm/virt: Require mc->has_hotpluggable_cpus for cold-plugged + vcpu + +Cold-plugged vCPU also need mc->has_hotpluggable_cpus. + +Signed-off-by: Keqian Zhu +--- + hw/arm/virt.c | 21 +++++++++++---------- + 1 file changed, 11 insertions(+), 10 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 00e57f2d75..73b29c7f73 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -3179,16 +3179,6 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + CPUState *cs = CPU(dev); + CPUArchId *cpu_slot; + +- if (dev->hotplugged && !vms->acpi_dev) { +- error_setg(errp, "GED acpi device does not exists"); +- return; +- } +- +- if (dev->hotplugged && !mc->has_hotpluggable_cpus) { +- error_setg(errp, "CPU hotplug not supported on this machine"); +- return; +- } +- + /* sanity check the cpu */ + if (!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) { + error_setg(errp, "Invalid CPU type, expected cpu type: '%s'", +@@ -3222,6 +3212,17 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + + cs->cpu_index = virt_get_cpu_id_from_cpu_topo(ms, dev); + ++ /* Except for cold-booted vCPUs, this should check presence of ACPI GED */ ++ if (cs->cpu_index >= ms->smp.cpus && !vms->acpi_dev) { ++ error_setg(errp, "GED acpi device does not exists"); ++ return; ++ } ++ ++ if (cs->cpu_index >= ms->smp.cpus && !mc->has_hotpluggable_cpus) { ++ error_setg(errp, "CPU [cold|hot]plug not supported on this machine"); ++ return; ++ } ++ + cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); + if (qemu_present_cpu(CPU(cpu_slot->cpu))) { + error_setg(errp, "cpu(id%d=%d:%d:%d:%d) with arch-id %" PRIu64 " exist", +-- +2.27.0 + diff --git a/arm-virt-Update-the-guest-via-GED-about-CPU-hot-un-p.patch b/arm-virt-Update-the-guest-via-GED-about-CPU-hot-un-p.patch new file mode 100644 index 0000000000000000000000000000000000000000..a45f47dea256e42e6e8b8f1817f724c9cac6cb10 --- /dev/null +++ b/arm-virt-Update-the-guest-via-GED-about-CPU-hot-un-p.patch @@ -0,0 +1,123 @@ +From afb71c88d935349cdf9763e8f51f77334ab615ec Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Fri, 8 May 2020 18:54:10 +0100 +Subject: [PATCH] arm/virt: Update the guest(via GED) about CPU hot-(un)plug + events + +During any vCPU hot-(un)plug, running guest VM needs to be intimated about the +new vCPU being added or request the deletion of the vCPU which is already part +of the guest VM. This is done using the ACPI GED event which eventually gets +demultiplexed to a CPU hotplug event and further to specific hot-(un)plug event +of a particular vCPU. + +This change adds the ACPI calls to the existing hot-(un)plug hooks to trigger +ACPI GED events from QEMU to guest VM. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 33 ++++++++++++++++++++++++++++++--- + 1 file changed, 30 insertions(+), 3 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 0312fa366d..60cd560ab9 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -3256,6 +3256,7 @@ static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); + MachineState *ms = MACHINE(hotplug_dev); + CPUState *cs = CPU(dev); ++ Error *local_err = NULL; + CPUArchId *cpu_slot; + + /* insert the cold/hot-plugged vcpu in the slot */ +@@ -3268,12 +3269,20 @@ static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + * plugged, guest is also notified. + */ + if (vms->acpi_dev) { +- /* TODO: update acpi hotplug state. Send cpu hotplug event to guest */ ++ HotplugHandlerClass *hhc; ++ /* update acpi hotplug state and send cpu hotplug event to guest */ ++ hhc = HOTPLUG_HANDLER_GET_CLASS(vms->acpi_dev); ++ hhc->plug(HOTPLUG_HANDLER(vms->acpi_dev), dev, &local_err); ++ if (local_err) { ++ goto fail; ++ } + /* TODO: register cpu for reset & update F/W info for the next boot */ + } + + cs->disabled = false; + return; ++fail: ++ error_propagate(errp, local_err); + } + + static void virt_cpu_unplug_request(HotplugHandler *hotplug_dev, +@@ -3281,8 +3290,10 @@ static void virt_cpu_unplug_request(HotplugHandler *hotplug_dev, + { + MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine()); + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); ++ HotplugHandlerClass *hhc; + ARMCPU *cpu = ARM_CPU(dev); + CPUState *cs = CPU(dev); ++ Error *local_err = NULL; + + if (!vms->acpi_dev || !dev->realized) { + error_setg(errp, "GED does not exists or device is not realized!"); +@@ -3301,9 +3312,16 @@ static void virt_cpu_unplug_request(HotplugHandler *hotplug_dev, + return; + } + +- /* TODO: request cpu hotplug from guest */ ++ /* request cpu hotplug from guest */ ++ hhc = HOTPLUG_HANDLER_GET_CLASS(vms->acpi_dev); ++ hhc->unplug_request(HOTPLUG_HANDLER(vms->acpi_dev), dev, &local_err); ++ if (local_err) { ++ goto fail; ++ } + + return; ++fail: ++ error_propagate(errp, local_err); + } + + static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, +@@ -3311,7 +3329,9 @@ static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, + { + VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); + MachineState *ms = MACHINE(hotplug_dev); ++ HotplugHandlerClass *hhc; + CPUState *cs = CPU(dev); ++ Error *local_err = NULL; + CPUArchId *cpu_slot; + + if (!vms->acpi_dev || !dev->realized) { +@@ -3321,7 +3341,12 @@ static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, + + cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); + +- /* TODO: update the acpi cpu hotplug state for cpu hot-unplug */ ++ /* update the acpi cpu hotplug state for cpu hot-unplug */ ++ hhc = HOTPLUG_HANDLER_GET_CLASS(vms->acpi_dev); ++ hhc->unplug(HOTPLUG_HANDLER(vms->acpi_dev), dev, &local_err); ++ if (local_err) { ++ goto fail; ++ } + + unwire_gic_cpu_irqs(vms, cs); + virt_update_gic(vms, cs); +@@ -3335,6 +3360,8 @@ static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, + cs->disabled = true; + + return; ++fail: ++ error_propagate(errp, local_err); + } + + static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev, +-- +2.27.0 + diff --git a/arm-virt-acpi-Build-CPUs-AML-with-CPU-Hotplug-suppor.patch b/arm-virt-acpi-Build-CPUs-AML-with-CPU-Hotplug-suppor.patch new file mode 100644 index 0000000000000000000000000000000000000000..cde5af36c19f45400e9c75b3755d57fcd19967ba --- /dev/null +++ b/arm-virt-acpi-Build-CPUs-AML-with-CPU-Hotplug-suppor.patch @@ -0,0 +1,43 @@ +From bea23b0f82cedbd860b66c7b9e1f6bb0ca85d1cf Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sun, 6 Aug 2023 17:05:30 +0000 +Subject: [PATCH] arm/virt/acpi: Build CPUs AML with CPU Hotplug support + +Support of vCPU Hotplug requires sequence of ACPI handshakes between Qemu and +Guest kernel when a vCPU is plugged or unplugged. Most of the AML code to +support these handshakes already exists. This AML need to be build during VM +init for ARM architecture as well if the GED support exists. + +Signed-off-by: Salil Mehta +--- + hw/arm/virt-acpi-build.c | 14 +++++++++++++- + 1 file changed, 13 insertions(+), 1 deletion(-) + +diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c +index 084c8abc7c..d88f3cded1 100644 +--- a/hw/arm/virt-acpi-build.c ++++ b/hw/arm/virt-acpi-build.c +@@ -937,7 +937,19 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) + * the RTC ACPI device at all when using UEFI. + */ + scope = aml_scope("\\_SB"); +- acpi_dsdt_add_cpus(scope, vms); ++ /* if GED is enabled then cpus AML shall be added as part build_cpus_aml */ ++ if (vms->acpi_dev) { ++ CPUHotplugFeatures opts = { ++ .acpi_1_compatible = false, ++ .has_legacy_cphp = false ++ }; ++ ++ build_cpus_aml(scope, ms, opts, NULL, virt_acpi_dsdt_cpu_cppc, ++ memmap[VIRT_CPUHP_ACPI].base, ++ "\\_SB", NULL, AML_SYSTEM_MEMORY); ++ } else { ++ acpi_dsdt_add_cpus(scope, vms); ++ } + acpi_dsdt_add_uart(scope, &memmap[VIRT_UART], + (irqmap[VIRT_UART] + ARM_SPI_BASE)); + if (vmc->acpi_expose_flash) { +-- +2.27.0 + diff --git a/arm-virt-acpi-Factor-out-CPPC-building-from-DSDT-CPU.patch b/arm-virt-acpi-Factor-out-CPPC-building-from-DSDT-CPU.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a599cbfe434992133a647dd0b1f0e278a649397 --- /dev/null +++ b/arm-virt-acpi-Factor-out-CPPC-building-from-DSDT-CPU.patch @@ -0,0 +1,76 @@ +From 2d5040ce21af5fc02a8588456be7316fcd5bc2a0 Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 2 Apr 2024 16:36:38 +0800 +Subject: [PATCH] arm/virt/acpi: Factor out CPPC building from DSDT CPU aml + +When CPU hotplug is enabled, we will use build_cpus_aml instead of +acpi_dsdt_add_cpus, so factor out CPPC building to reuse it. + +Signed-off-by: Keqian Zhu +--- + hw/arm/virt-acpi-build.c | 34 ++++++++++++++++++++-------------- + 1 file changed, 20 insertions(+), 14 deletions(-) + +diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c +index 48fc77fb83..084c8abc7c 100644 +--- a/hw/arm/virt-acpi-build.c ++++ b/hw/arm/virt-acpi-build.c +@@ -123,8 +123,23 @@ static void acpi_dsdt_add_cppc(Aml *dev, uint64_t cpu_base, int *regs_offset) + aml_append(dev, aml_name_decl("_CPC", cpc)); + } + +-static void acpi_dsdt_add_cpus(Aml *scope, VirtMachineState *vms, +- const MemMapEntry *cppc_memmap) ++static void virt_acpi_dsdt_cpu_cppc(int ncpu, int num_cpu, Aml *dev) { ++ VirtMachineState *vms = VIRT_MACHINE(qdev_get_machine()); ++ const MemMapEntry *cppc_memmap = &vms->memmap[VIRT_CPUFREQ]; ++ ++ /* ++ * Append _CPC and _PSD to support CPU frequence show ++ * Check CPPC available by DESIRED_PERF register ++ */ ++ if (cppc_regs_offset[DESIRED_PERF] != -1) { ++ acpi_dsdt_add_cppc(dev, ++ cppc_memmap->base + ncpu * CPPC_REG_PER_CPU_STRIDE, ++ cppc_regs_offset); ++ acpi_dsdt_add_psd(dev, num_cpu); ++ } ++} ++ ++static void acpi_dsdt_add_cpus(Aml *scope, VirtMachineState *vms) + { + MachineState *ms = MACHINE(vms); + uint16_t i; +@@ -134,18 +149,9 @@ static void acpi_dsdt_add_cpus(Aml *scope, VirtMachineState *vms, + aml_append(dev, aml_name_decl("_HID", aml_string("ACPI0007"))); + aml_append(dev, aml_name_decl("_UID", aml_int(i))); + +- /* +- * Append _CPC and _PSD to support CPU frequence show +- * Check CPPC available by DESIRED_PERF register +- */ +- if (cppc_regs_offset[DESIRED_PERF] != -1) { +- acpi_dsdt_add_cppc(dev, +- cppc_memmap->base + i * CPPC_REG_PER_CPU_STRIDE, +- cppc_regs_offset); +- acpi_dsdt_add_psd(dev, ms->smp.cpus); +- } ++ virt_acpi_dsdt_cpu_cppc(i, ms->smp.cpus, dev); + +- aml_append(scope, dev); ++ aml_append(scope, dev); + } + } + +@@ -931,7 +937,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) + * the RTC ACPI device at all when using UEFI. + */ + scope = aml_scope("\\_SB"); +- acpi_dsdt_add_cpus(scope, vms, &memmap[VIRT_CPUFREQ]); ++ acpi_dsdt_add_cpus(scope, vms); + acpi_dsdt_add_uart(scope, &memmap[VIRT_UART], + (irqmap[VIRT_UART] + ARM_SPI_BASE)); + if (vmc->acpi_expose_flash) { +-- +2.27.0 + diff --git a/arm-virt-acpi-Require-possible_cpu_arch_ids-for-buil.patch b/arm-virt-acpi-Require-possible_cpu_arch_ids-for-buil.patch new file mode 100644 index 0000000000000000000000000000000000000000..c323c440952a681f11f46ce4f1f3bcfbcb954f91 --- /dev/null +++ b/arm-virt-acpi-Require-possible_cpu_arch_ids-for-buil.patch @@ -0,0 +1,38 @@ +From 0bee56446962676992d11e5879f6fbac57e785e8 Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 23:38:31 +0800 +Subject: [PATCH] arm/virt-acpi: Require possible_cpu_arch_ids for + build_cpus_aml() + +As the acpi_dev requires possible_cpu_arch_ids to support +vcpu hotplug. + +Signed-off-by: Keqian Zhu +--- + hw/arm/virt-acpi-build.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c +index 590afcfa98..46642efac4 100644 +--- a/hw/arm/virt-acpi-build.c ++++ b/hw/arm/virt-acpi-build.c +@@ -1003,6 +1003,7 @@ static void + build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) + { + VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(vms); ++ MachineClass *mc = MACHINE_GET_CLASS(vms); + Aml *scope, *dsdt; + MachineState *ms = MACHINE(vms); + const MemMapEntry *memmap = vms->memmap; +@@ -1020,7 +1021,7 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) + */ + scope = aml_scope("\\_SB"); + /* if GED is enabled then cpus AML shall be added as part build_cpus_aml */ +- if (vms->acpi_dev) { ++ if (mc->has_hotpluggable_cpus) { + CPUHotplugFeatures opts = { + .acpi_1_compatible = false, + .has_legacy_cphp = false +-- +2.27.0 + diff --git a/arm-virt-gicv3-Changes-to-pre-size-GIC-with-possible.patch b/arm-virt-gicv3-Changes-to-pre-size-GIC-with-possible.patch new file mode 100644 index 0000000000000000000000000000000000000000..b2f8c67fce6c0c005d1e5537abc3faddfd22f250 --- /dev/null +++ b/arm-virt-gicv3-Changes-to-pre-size-GIC-with-possible.patch @@ -0,0 +1,225 @@ +From fe61cbaf2dc92b062c8d147b05c3ce213734c24a Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Wed, 6 May 2020 02:20:23 +0100 +Subject: [PATCH] arm/virt,gicv3: Changes to pre-size GIC with possible vcpus + @machine init + +GIC needs to be pre-sized with possible vcpus at the initialization time. This +is necessary because Memory regions and resources associated with GICC/GICR +etc cannot be changed (add/del/modified) after VM has inited. Also, GIC_TYPER +needs to be initialized with mp_affinity and cpu interface number association. +This cannot be changed after GIC has initialized. + +Once all the cpu interfaces of the GIC has been inited it needs to be ensured +that any updates to the GICC during reset only takes place for the present +vcpus and not the disabled ones. Therefore, proper checks are required at +various places. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Jean-Philippe Brucker +[changed the comment in arm_gicv3_icc_reset] +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 13 +++++++------ + hw/intc/arm_gicv3_common.c | 7 +++++-- + hw/intc/arm_gicv3_cpuif.c | 8 ++++++++ + hw/intc/arm_gicv3_kvm.c | 34 +++++++++++++++++++++++++++++++--- + include/hw/arm/virt.h | 2 +- + 5 files changed, 52 insertions(+), 12 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index f10d75366b..08ba255317 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -802,6 +802,7 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) + const char *gictype; + int i; + unsigned int smp_cpus = ms->smp.cpus; ++ unsigned int max_cpus = ms->smp.max_cpus; + uint32_t nb_redist_regions = 0; + int revision; + +@@ -826,7 +827,7 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) + } + vms->gic = qdev_new(gictype); + qdev_prop_set_uint32(vms->gic, "revision", revision); +- qdev_prop_set_uint32(vms->gic, "num-cpu", smp_cpus); ++ qdev_prop_set_uint32(vms->gic, "num-cpu", max_cpus); + /* Note that the num-irq property counts both internal and external + * interrupts; there are always 32 of the former (mandated by GIC spec). + */ +@@ -838,7 +839,7 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) + if (vms->gic_version != VIRT_GIC_VERSION_2) { + QList *redist_region_count; + uint32_t redist0_capacity = virt_redist_capacity(vms, VIRT_GIC_REDIST); +- uint32_t redist0_count = MIN(smp_cpus, redist0_capacity); ++ uint32_t redist0_count = MIN(max_cpus, redist0_capacity); + + nb_redist_regions = virt_gicv3_redist_region_count(vms); + +@@ -915,7 +916,7 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) + } else if (vms->virt) { + qemu_irq irq = qdev_get_gpio_in(vms->gic, + intidbase + ARCH_GIC_MAINT_IRQ); +- sysbus_connect_irq(gicbusdev, i + 4 * smp_cpus, irq); ++ sysbus_connect_irq(gicbusdev, i + 4 * max_cpus, irq); + } + + qdev_connect_gpio_out_named(cpudev, "pmu-interrupt", 0, +@@ -923,11 +924,11 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) + + VIRTUAL_PMU_IRQ)); + + sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev, ARM_CPU_IRQ)); +- sysbus_connect_irq(gicbusdev, i + smp_cpus, ++ sysbus_connect_irq(gicbusdev, i + max_cpus, + qdev_get_gpio_in(cpudev, ARM_CPU_FIQ)); +- sysbus_connect_irq(gicbusdev, i + 2 * smp_cpus, ++ sysbus_connect_irq(gicbusdev, i + 2 * max_cpus, + qdev_get_gpio_in(cpudev, ARM_CPU_VIRQ)); +- sysbus_connect_irq(gicbusdev, i + 3 * smp_cpus, ++ sysbus_connect_irq(gicbusdev, i + 3 * max_cpus, + qdev_get_gpio_in(cpudev, ARM_CPU_VFIQ)); + } + +diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c +index 2ebf880ead..ebd99af610 100644 +--- a/hw/intc/arm_gicv3_common.c ++++ b/hw/intc/arm_gicv3_common.c +@@ -392,10 +392,13 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp) + s->cpu = g_new0(GICv3CPUState, s->num_cpu); + + for (i = 0; i < s->num_cpu; i++) { +- CPUState *cpu = qemu_get_cpu(i); ++ CPUState *cpu = qemu_get_possible_cpu(i); + uint64_t cpu_affid; + +- s->cpu[i].cpu = cpu; ++ if (qemu_enabled_cpu(cpu)) { ++ s->cpu[i].cpu = cpu; ++ } ++ + s->cpu[i].gic = s; + /* Store GICv3CPUState in CPUARMState gicv3state pointer */ + gicv3_set_gicv3state(cpu, &s->cpu[i]); +diff --git a/hw/intc/arm_gicv3_cpuif.c b/hw/intc/arm_gicv3_cpuif.c +index ab1a00508e..0d0eb2f62f 100644 +--- a/hw/intc/arm_gicv3_cpuif.c ++++ b/hw/intc/arm_gicv3_cpuif.c +@@ -934,6 +934,10 @@ void gicv3_cpuif_update(GICv3CPUState *cs) + ARMCPU *cpu = ARM_CPU(cs->cpu); + CPUARMState *env = &cpu->env; + ++ if (!qemu_enabled_cpu(cs->cpu)) { ++ return; ++ } ++ + g_assert(qemu_mutex_iothread_locked()); + + trace_gicv3_cpuif_update(gicv3_redist_affid(cs), cs->hppi.irq, +@@ -1826,6 +1830,10 @@ static void icc_generate_sgi(CPUARMState *env, GICv3CPUState *cs, + for (i = 0; i < s->num_cpu; i++) { + GICv3CPUState *ocs = &s->cpu[i]; + ++ if (!qemu_enabled_cpu(ocs->cpu)) { ++ continue; ++ } ++ + if (irm) { + /* IRM == 1 : route to all CPUs except self */ + if (cs == ocs) { +diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c +index 77eb37e131..db06c75e2b 100644 +--- a/hw/intc/arm_gicv3_kvm.c ++++ b/hw/intc/arm_gicv3_kvm.c +@@ -24,6 +24,7 @@ + #include "hw/intc/arm_gicv3_common.h" + #include "qemu/error-report.h" + #include "qemu/module.h" ++#include "sysemu/cpus.h" + #include "sysemu/kvm.h" + #include "sysemu/runstate.h" + #include "kvm_arm.h" +@@ -458,6 +459,18 @@ static void kvm_arm_gicv3_put(GICv3State *s) + GICv3CPUState *c = &s->cpu[ncpu]; + int num_pri_bits; + ++ /* ++ * To support hotplug of vcpus we need to make sure all gic cpuif/GICC ++ * are initialized at machvirt init time. Once the init is done we ++ * release the ARMCPU object for disabled vcpus but this leg could hit ++ * during reset of GICC later as well i.e. after init has happened and ++ * all of the cases we want to make sure we dont acess the GICC for ++ * the disabled VCPUs. ++ */ ++ if (!qemu_enabled_cpu(c->cpu)) { ++ continue; ++ } ++ + kvm_gicc_access(s, ICC_SRE_EL1, ncpu, &c->icc_sre_el1, true); + kvm_gicc_access(s, ICC_CTLR_EL1, ncpu, + &c->icc_ctlr_el1[GICV3_NS], true); +@@ -616,6 +629,11 @@ static void kvm_arm_gicv3_get(GICv3State *s) + GICv3CPUState *c = &s->cpu[ncpu]; + int num_pri_bits; + ++ /* don't access GICC for the disabled vCPUs. */ ++ if (!qemu_enabled_cpu(c->cpu)) { ++ continue; ++ } ++ + kvm_gicc_access(s, ICC_SRE_EL1, ncpu, &c->icc_sre_el1, false); + kvm_gicc_access(s, ICC_CTLR_EL1, ncpu, + &c->icc_ctlr_el1[GICV3_NS], false); +@@ -695,10 +713,19 @@ static void arm_gicv3_icc_reset(CPUARMState *env, const ARMCPRegInfo *ri) + return; + } + ++ /* ++ * This shall be called even when vcpu is being hotplugged or onlined and ++ * other vcpus might be running. Host kernel KVM code to handle device ++ * access of IOCTLs KVM_{GET|SET}_DEVICE_ATTR might fail due to inability to ++ * grab vcpu locks for all the vcpus. Hence, we need to pause all vcpus to ++ * facilitate locking within host. ++ */ ++ pause_all_vcpus(); + /* Initialize to actual HW supported configuration */ + kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + KVM_VGIC_ATTR(ICC_CTLR_EL1, c->gicr_typer), + &c->icc_ctlr_el1[GICV3_NS], false, &error_abort); ++ resume_all_vcpus(); + + c->icc_ctlr_el1[GICV3_S] = c->icc_ctlr_el1[GICV3_NS]; + } +@@ -808,9 +835,10 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp) + gicv3_init_irqs_and_mmio(s, kvm_arm_gicv3_set_irq, NULL); + + for (i = 0; i < s->num_cpu; i++) { +- ARMCPU *cpu = ARM_CPU(qemu_get_cpu(i)); +- +- define_arm_cp_regs(cpu, gicv3_cpuif_reginfo); ++ CPUState *cs = qemu_get_cpu(i); ++ if (qemu_enabled_cpu(cs)) { ++ define_arm_cp_regs(ARM_CPU(cs), gicv3_cpuif_reginfo); ++ } + } + + /* Try to create the device via the device control API */ +diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h +index 49d1ec8656..a6977bade5 100644 +--- a/include/hw/arm/virt.h ++++ b/include/hw/arm/virt.h +@@ -208,7 +208,7 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms) + + assert(vms->gic_version != VIRT_GIC_VERSION_2); + +- return (MACHINE(vms)->smp.cpus > redist0_capacity && ++ return (MACHINE(vms)->smp.max_cpus > redist0_capacity && + vms->highmem_redists) ? 2 : 1; + } + +-- +2.27.0 + diff --git a/arm-virt-kvm-Pre-create-disabled-possible-vCPUs-mach.patch b/arm-virt-kvm-Pre-create-disabled-possible-vCPUs-mach.patch new file mode 100644 index 0000000000000000000000000000000000000000..b752e1fd85490dc2292692b29bc34652b29d460b --- /dev/null +++ b/arm-virt-kvm-Pre-create-disabled-possible-vCPUs-mach.patch @@ -0,0 +1,221 @@ +From 2669fd26cbc36e24ebfc844c240b45ad831701cc Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Tue, 5 May 2020 18:44:59 +0100 +Subject: [PATCH] arm/virt,kvm: Pre-create disabled possible vCPUs @machine + init + +In ARMv8 architecture, GIC needs all the vCPUs to be created and present when +it is initialized. This is because: +1. GICC and MPIDR association must be fixed at the VM initialization time. + This is represented by register GIC_TYPER(mp_afffinity, proc_num) +2. GICC(cpu interfaces), GICR(redistributors) etc all must be initialized + at the boot time as well. +3. Memory regions associated with GICR etc. cannot be changed(add/del/mod) + after VM has inited. + +This patch adds the support to pre-create all such possible vCPUs within the +host using the KVM interface as part of the virt machine initialization. These +vCPUs could later be attached to QOM/ACPI while they are actually hot plugged +and made present. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Reported-by: Vishnu Pajjuri +[VP: Identified CPU stall issue & suggested probable fix] +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 53 +++++++++++++++++++++++++++++++++++++++++-- + include/hw/core/cpu.h | 1 + + target/arm/cpu64.c | 1 + + target/arm/kvm.c | 32 ++++++++++++++++++++++++++ + target/arm/kvm64.c | 9 +++++++- + target/arm/kvm_arm.h | 11 +++++++++ + 6 files changed, 104 insertions(+), 3 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 2f04bc7666..f10d75366b 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -2389,8 +2389,10 @@ static void machvirt_init(MachineState *machine) + assert(possible_cpus->len == max_cpus); + for (n = 0; n < possible_cpus->len; n++) { + Object *cpuobj; ++ CPUState *cs; + + cpuobj = object_new(possible_cpus->cpus[n].type); ++ cs = CPU(cpuobj); + + aarch64 &= object_property_get_bool(cpuobj, "aarch64", NULL); + object_property_set_int(cpuobj, "socket-id", +@@ -2402,8 +2404,55 @@ static void machvirt_init(MachineState *machine) + object_property_set_int(cpuobj, "thread-id", + virt_get_thread_id(machine, n), NULL); + +- qdev_realize(DEVICE(cpuobj), NULL, &error_fatal); +- object_unref(cpuobj); ++ if (n < smp_cpus) { ++ qdev_realize(DEVICE(cpuobj), NULL, &error_fatal); ++ object_unref(cpuobj); ++ } else { ++ CPUArchId *cpu_slot; ++ ++ /* handling for vcpus which are yet to be hot-plugged */ ++ cs->cpu_index = n; ++ cpu_slot = virt_find_cpu_slot(machine, cs->cpu_index); ++ ++ /* ++ * ARM host vCPU features need to be fixed at the boot time. But as ++ * per current approach this CPU object will be destroyed during ++ * cpu_post_init(). During hotplug of vCPUs these properties are ++ * initialized again. ++ */ ++ virt_cpu_set_properties(cpuobj, cpu_slot, &error_fatal); ++ ++ /* ++ * For KVM, we shall be pre-creating the now disabled/un-plugged ++ * possbile host vcpus and park them till the time they are ++ * actually hot plugged. This is required to pre-size the host ++ * GICC and GICR with the all possible vcpus for this VM. ++ */ ++ if (kvm_enabled()) { ++ kvm_arm_create_host_vcpu(ARM_CPU(cs)); ++ } ++ /* ++ * Add disabled vCPU to CPU slot during the init phase of the virt ++ * machine ++ * 1. We need this ARMCPU object during the GIC init. This object ++ * will facilitate in pre-realizing the GIC. Any info like ++ * mp-affinity(required to derive gicr_type) etc. could still be ++ * fetched while preserving QOM abstraction akin to realized ++ * vCPUs. ++ * 2. Now, after initialization of the virt machine is complete we ++ * could use two approaches to deal with this ARMCPU object: ++ * (i) re-use this ARMCPU object during hotplug of this vCPU. ++ * OR ++ * (ii) defer release this ARMCPU object after gic has been ++ * initialized or during pre-plug phase when a vCPU is ++ * hotplugged. ++ * ++ * We will use the (ii) approach and release the ARMCPU objects ++ * after GIC and machine has been fully initialized during ++ * machine_init_done() phase. ++ */ ++ cpu_slot->cpu = OBJECT(cs); ++ } + } + fdt_add_timer_nodes(vms); + fdt_add_cpu_nodes(vms); +diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h +index c30636a936..fdfb952259 100644 +--- a/include/hw/core/cpu.h ++++ b/include/hw/core/cpu.h +@@ -528,6 +528,7 @@ struct CPUState { + uint32_t kvm_fetch_index; + uint64_t dirty_pages; + int kvm_vcpu_stats_fd; ++ VMChangeStateEntry *vmcse; + + /* Use by accel-block: CPU is executing an ioctl() */ + QemuLockCnt in_ioctl_lock; +diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c +index e226b60b72..5d28838175 100644 +--- a/target/arm/cpu64.c ++++ b/target/arm/cpu64.c +@@ -859,6 +859,7 @@ static void aarch64_cpu_initfn(Object *obj) + * enabled explicitly + */ + cs->disabled = true; ++ cs->thread_id = 0; + } + + static void aarch64_cpu_finalizefn(Object *obj) +diff --git a/target/arm/kvm.c b/target/arm/kvm.c +index f59f4f81b2..70cf15b550 100644 +--- a/target/arm/kvm.c ++++ b/target/arm/kvm.c +@@ -659,6 +659,38 @@ void kvm_arm_reset_vcpu(ARMCPU *cpu) + write_list_to_cpustate(cpu); + } + ++void kvm_arm_create_host_vcpu(ARMCPU *cpu) ++{ ++ CPUState *cs = CPU(cpu); ++ unsigned long vcpu_id = cs->cpu_index; ++ int ret; ++ ++ ret = kvm_create_vcpu(cs); ++ if (ret < 0) { ++ error_report("Failed to create host vcpu %ld", vcpu_id); ++ abort(); ++ } ++ ++ /* ++ * Initialize the vCPU in the host. This will reset the sys regs ++ * for this vCPU and related registers like MPIDR_EL1 etc. also ++ * gets programmed during this call to host. These are referred ++ * later while setting device attributes of the GICR during GICv3 ++ * reset ++ */ ++ ret = kvm_arch_init_vcpu(cs); ++ if (ret < 0) { ++ error_report("Failed to initialize host vcpu %ld", vcpu_id); ++ abort(); ++ } ++ ++ /* ++ * park the created vCPU. shall be used during kvm_get_vcpu() when ++ * threads are created during realization of ARM vCPUs. ++ */ ++ kvm_park_vcpu(cs); ++} ++ + /* + * Update KVM's MP_STATE based on what QEMU thinks it is + */ +diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c +index 3c175c93a7..03ce1e7525 100644 +--- a/target/arm/kvm64.c ++++ b/target/arm/kvm64.c +@@ -562,7 +562,14 @@ int kvm_arch_init_vcpu(CPUState *cs) + return -EINVAL; + } + +- qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cs); ++ /* ++ * Install VM change handler only when vCPU thread has been spawned ++ * i.e. vCPU is being realized ++ */ ++ if (cs->thread_id) { ++ cs->vmcse = qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, ++ cs); ++ } + + /* Determine init features for this CPU */ + memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features)); +diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h +index 051a0da41c..31408499b3 100644 +--- a/target/arm/kvm_arm.h ++++ b/target/arm/kvm_arm.h +@@ -163,6 +163,17 @@ void kvm_arm_cpu_post_load(ARMCPU *cpu); + */ + void kvm_arm_reset_vcpu(ARMCPU *cpu); + ++/** ++ * kvm_arm_create_host_vcpu: ++ * @cpu: ARMCPU ++ * ++ * Called at to pre create all possible kvm vCPUs within the the host at the ++ * virt machine init time. This will also init this pre-created vCPU and ++ * hence result in vCPU reset at host. These pre created and inited vCPUs ++ * shall be parked for use when ARM vCPUs are actually realized. ++ */ ++void kvm_arm_create_host_vcpu(ARMCPU *cpu); ++ + /** + * kvm_arm_init_serror_injection: + * @cs: CPUState +-- +2.27.0 + diff --git a/arm-virt-target-arm-Add-new-ARMCPU-socket-cluster-co.patch b/arm-virt-target-arm-Add-new-ARMCPU-socket-cluster-co.patch new file mode 100644 index 0000000000000000000000000000000000000000..71f2ff037965e861699d5448b53f67381156c7bb --- /dev/null +++ b/arm-virt-target-arm-Add-new-ARMCPU-socket-cluster-co.patch @@ -0,0 +1,153 @@ +From c8e062285078e688e692214baf97b35246fc2552 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Tue, 5 May 2020 23:19:17 +0100 +Subject: [PATCH] arm/virt,target/arm: Add new ARMCPU + {socket,cluster,core,thread}-id property + +This shall be used to store user specified topology{socket,cluster,core,thread} +and shall be converted to a unique 'vcpu-id' which is used as slot-index during +hot(un)plug of vCPU. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ + target/arm/cpu.c | 4 +++ + target/arm/cpu.h | 4 +++ + 3 files changed, 71 insertions(+) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index f4c3d47f30..94481d45d4 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -227,6 +227,11 @@ static const char *valid_cpus[] = { + ARM_CPU_TYPE_NAME("max"), + }; + ++static int virt_get_socket_id(const MachineState *ms, int cpu_index); ++static int virt_get_cluster_id(const MachineState *ms, int cpu_index); ++static int virt_get_core_id(const MachineState *ms, int cpu_index); ++static int virt_get_thread_id(const MachineState *ms, int cpu_index); ++ + static bool cpu_type_valid(const char *cpu) + { + int i; +@@ -2264,6 +2269,14 @@ static void machvirt_init(MachineState *machine) + &error_fatal); + + aarch64 &= object_property_get_bool(cpuobj, "aarch64", NULL); ++ object_property_set_int(cpuobj, "socket-id", ++ virt_get_socket_id(machine, n), NULL); ++ object_property_set_int(cpuobj, "cluster-id", ++ virt_get_cluster_id(machine, n), NULL); ++ object_property_set_int(cpuobj, "core-id", ++ virt_get_core_id(machine, n), NULL); ++ object_property_set_int(cpuobj, "thread-id", ++ virt_get_thread_id(machine, n), NULL); + + if (!vms->secure) { + object_property_set_bool(cpuobj, "has_el3", false, NULL); +@@ -2750,10 +2763,59 @@ static int64_t virt_get_default_cpu_node_id(const MachineState *ms, int idx) + return socket_id % ms->numa_state->num_nodes; + } + ++static int virt_get_socket_id(const MachineState *ms, int cpu_index) ++{ ++ assert(cpu_index >= 0 && cpu_index < ms->possible_cpus->len); ++ ++ return ms->possible_cpus->cpus[cpu_index].props.socket_id; ++} ++ ++static int virt_get_cluster_id(const MachineState *ms, int cpu_index) ++{ ++ assert(cpu_index >= 0 && cpu_index < ms->possible_cpus->len); ++ ++ return ms->possible_cpus->cpus[cpu_index].props.cluster_id; ++} ++ ++static int virt_get_core_id(const MachineState *ms, int cpu_index) ++{ ++ assert(cpu_index >= 0 && cpu_index < ms->possible_cpus->len); ++ ++ return ms->possible_cpus->cpus[cpu_index].props.core_id; ++} ++ ++static int virt_get_thread_id(const MachineState *ms, int cpu_index) ++{ ++ assert(cpu_index >= 0 && cpu_index < ms->possible_cpus->len); ++ ++ return ms->possible_cpus->cpus[cpu_index].props.thread_id; ++} ++ ++static int ++virt_get_cpu_id_from_cpu_topo(const MachineState *ms, DeviceState *dev) ++{ ++ int cpu_id, sock_vcpu_num, clus_vcpu_num, core_vcpu_num; ++ ARMCPU *cpu = ARM_CPU(dev); ++ ++ /* calculate total logical cpus across socket/cluster/core */ ++ sock_vcpu_num = cpu->socket_id * (ms->smp.threads * ms->smp.cores * ++ ms->smp.clusters); ++ clus_vcpu_num = cpu->cluster_id * (ms->smp.threads * ms->smp.cores); ++ core_vcpu_num = cpu->core_id * ms->smp.threads; ++ ++ /* get vcpu-id(logical cpu index) for this vcpu from this topology */ ++ cpu_id = (sock_vcpu_num + clus_vcpu_num + core_vcpu_num) + cpu->thread_id; ++ ++ assert(cpu_id >= 0 && cpu_id < ms->possible_cpus->len); ++ ++ return cpu_id; ++} ++ + static const CPUArchIdList *virt_possible_cpu_arch_ids(MachineState *ms) + { + int n; + unsigned int max_cpus = ms->smp.max_cpus; ++ unsigned int smp_threads = ms->smp.threads; + VirtMachineState *vms = VIRT_MACHINE(ms); + MachineClass *mc = MACHINE_GET_CLASS(vms); + +@@ -2767,6 +2829,7 @@ static const CPUArchIdList *virt_possible_cpu_arch_ids(MachineState *ms) + ms->possible_cpus->len = max_cpus; + for (n = 0; n < ms->possible_cpus->len; n++) { + ms->possible_cpus->cpus[n].type = ms->cpu_type; ++ ms->possible_cpus->cpus[n].vcpus_count = smp_threads; + ms->possible_cpus->cpus[n].arch_id = + virt_cpu_mp_affinity(vms, n); + +diff --git a/target/arm/cpu.c b/target/arm/cpu.c +index efb22a87f9..cce315c18a 100644 +--- a/target/arm/cpu.c ++++ b/target/arm/cpu.c +@@ -2422,6 +2422,10 @@ static Property arm_cpu_properties[] = { + DEFINE_PROP_UINT64("mp-affinity", ARMCPU, + mp_affinity, ARM64_AFFINITY_INVALID), + DEFINE_PROP_INT32("node-id", ARMCPU, node_id, CPU_UNSET_NUMA_NODE_ID), ++ DEFINE_PROP_INT32("socket-id", ARMCPU, socket_id, 0), ++ DEFINE_PROP_INT32("cluster-id", ARMCPU, cluster_id, 0), ++ DEFINE_PROP_INT32("core-id", ARMCPU, core_id, 0), ++ DEFINE_PROP_INT32("thread-id", ARMCPU, thread_id, 0), + DEFINE_PROP_INT32("core-count", ARMCPU, core_count, -1), + DEFINE_PROP_END_OF_LIST() + }; +diff --git a/target/arm/cpu.h b/target/arm/cpu.h +index a0282e0d28..145d3dbf13 100644 +--- a/target/arm/cpu.h ++++ b/target/arm/cpu.h +@@ -1096,6 +1096,10 @@ struct ArchCPU { + QLIST_HEAD(, ARMELChangeHook) el_change_hooks; + + int32_t node_id; /* NUMA node this CPU belongs to */ ++ int32_t socket_id; ++ int32_t cluster_id; ++ int32_t core_id; ++ int32_t thread_id; + + /* Used to synchronize KVM and QEMU in-kernel device levels */ + uint8_t device_irq_level; +-- +2.27.0 + diff --git a/arm-virt-target-arm-Machine-init-time-change-common-.patch b/arm-virt-target-arm-Machine-init-time-change-common-.patch new file mode 100644 index 0000000000000000000000000000000000000000..d8199f7ce05614c00088072be19912fda08e3c13 --- /dev/null +++ b/arm-virt-target-arm-Machine-init-time-change-common-.patch @@ -0,0 +1,328 @@ +From 7cd2d7ef7bb7f6c6a97988d86b97922ff700ab06 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Wed, 6 May 2020 00:13:31 +0100 +Subject: [PATCH] arm/virt,target/arm: Machine init time change common to vCPU + {cold|hot}-plug + +Refactor and introduce the common logic required during the initialization of +both cold and hot plugged vCPUs. Also initialize the *disabled* state of the +vCPUs which shall be used further during init phases of various other components +like GIC, PMU, ACPI etc as part of the virt machine initialization. + +KVM vCPUs corresponding to unplugged/yet-to-be-plugged QOM CPUs are kept in +powered-off state in the KVM Host and do not run the guest code. Plugged vCPUs +are also kept in powered-off state but vCPU threads exist and is kept sleeping. + +TBD: +For the cold booted vCPUs, this change also exists in the arm_load_kernel() +in boot.c but for the hotplugged CPUs this change should still remain part of +the pre-plug phase. We are duplicating the powering-off of the cold booted CPUs. +Shall we remove the duplicate change from boot.c? + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Reported-by: Gavin Shan +[GS: pointed the assertion due to wrong range check] +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 149 ++++++++++++++++++++++++++++++++++++++++----- + target/arm/cpu.c | 7 +++ + target/arm/cpu64.c | 14 +++++ + 3 files changed, 156 insertions(+), 14 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 8f647422d8..2f04bc7666 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -227,6 +227,7 @@ static const char *valid_cpus[] = { + ARM_CPU_TYPE_NAME("max"), + }; + ++static CPUArchId *virt_find_cpu_slot(MachineState *ms, int vcpuid); + static int virt_get_socket_id(const MachineState *ms, int cpu_index); + static int virt_get_cluster_id(const MachineState *ms, int cpu_index); + static int virt_get_core_id(const MachineState *ms, int cpu_index); +@@ -2249,6 +2250,14 @@ static void machvirt_init(MachineState *machine) + exit(1); + } + ++ finalize_gic_version(vms); ++ if (tcg_enabled() || hvf_enabled() || qtest_enabled() || ++ (vms->gic_version < VIRT_GIC_VERSION_3)) { ++ machine->smp.max_cpus = smp_cpus; ++ mc->has_hotpluggable_cpus = false; ++ warn_report("cpu hotplug feature has been disabled"); ++ } ++ + possible_cpus = mc->possible_cpu_arch_ids(machine); + + /* +@@ -2275,11 +2284,6 @@ static void machvirt_init(MachineState *machine) + virt_set_memmap(vms, pa_bits); + } + +- /* We can probe only here because during property set +- * KVM is not available yet +- */ +- finalize_gic_version(vms); +- + sysmem = vms->sysmem = get_system_memory(); + + if (vms->secure) { +@@ -2385,17 +2389,9 @@ static void machvirt_init(MachineState *machine) + assert(possible_cpus->len == max_cpus); + for (n = 0; n < possible_cpus->len; n++) { + Object *cpuobj; +- CPUState *cs; +- +- if (n >= smp_cpus) { +- break; +- } + + cpuobj = object_new(possible_cpus->cpus[n].type); + +- cs = CPU(cpuobj); +- cs->cpu_index = n; +- + aarch64 &= object_property_get_bool(cpuobj, "aarch64", NULL); + object_property_set_int(cpuobj, "socket-id", + virt_get_socket_id(machine, n), NULL); +@@ -2902,6 +2898,50 @@ static const CPUArchIdList *virt_possible_cpu_arch_ids(MachineState *ms) + return ms->possible_cpus; + } + ++static CPUArchId *virt_find_cpu_slot(MachineState *ms, int vcpuid) ++{ ++ VirtMachineState *vms = VIRT_MACHINE(ms); ++ CPUArchId *found_cpu; ++ uint64_t mp_affinity; ++ ++ assert(vcpuid >= 0 && vcpuid < ms->possible_cpus->len); ++ ++ /* ++ * RFC: Question: ++ * TBD: Should mp-affinity be treated as MPIDR? ++ */ ++ mp_affinity = virt_cpu_mp_affinity(vms, vcpuid); ++ found_cpu = &ms->possible_cpus->cpus[vcpuid]; ++ ++ assert(found_cpu->arch_id == mp_affinity); ++ ++ /* ++ * RFC: Question: ++ * Slot-id is the index where vCPU with certain arch-id(=mpidr/ap-affinity) ++ * is plugged. For Host KVM, MPIDR for vCPU is derived using vcpu-id. ++ * As I understand, MPIDR and vcpu-id are property of vCPU but slot-id is ++ * more related to machine? Current code assumes slot-id and vcpu-id are ++ * same i.e. meaning of slot is bit vague. ++ * ++ * Q1: Is there any requirement to clearly represent slot and dissociate it ++ * from vcpu-id? ++ * Q2: Should we make MPIDR within host KVM user configurable? ++ * ++ * +----+----+----+----+----+----+----+----+ ++ * MPIDR ||| Res | Aff2 | Aff1 | Aff0 | ++ * +----+----+----+----+----+----+----+----+ ++ * \ \ \ | | ++ * \ 8bit \ 8bit \ |4bit| ++ * \<------->\<------->\ |<-->| ++ * \ \ \| | ++ * +----+----+----+----+----+----+----+----+ ++ * VCPU-ID | Byte4 | Byte2 | Byte1 | Byte0 | ++ * +----+----+----+----+----+----+----+----+ ++ */ ++ ++ return found_cpu; ++} ++ + static void virt_memory_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + Error **errp) + { +@@ -2945,6 +2985,81 @@ static void virt_memory_plug(HotplugHandler *hotplug_dev, + dev, &error_abort); + } + ++static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, ++ Error **errp) ++{ ++ VirtMachineState *vms = VIRT_MACHINE(hotplug_dev); ++ MachineState *ms = MACHINE(hotplug_dev); ++ ARMCPU *cpu = ARM_CPU(dev); ++ CPUState *cs = CPU(dev); ++ CPUArchId *cpu_slot; ++ int32_t min_cpuid = 0; ++ int32_t max_cpuid; ++ ++ /* sanity check the cpu */ ++ if (!object_dynamic_cast(OBJECT(cpu), ms->cpu_type)) { ++ error_setg(errp, "Invalid CPU type, expected cpu type: '%s'", ++ ms->cpu_type); ++ return; ++ } ++ ++ if ((cpu->thread_id < 0) || (cpu->thread_id >= ms->smp.threads)) { ++ error_setg(errp, "Invalid thread-id %u specified, correct range 0:%u", ++ cpu->thread_id, ms->smp.threads - 1); ++ return; ++ } ++ ++ max_cpuid = ms->possible_cpus->len - 1; ++ if (!dev->hotplugged) { ++ min_cpuid = vms->acpi_dev ? ms->smp.cpus : 0; ++ max_cpuid = vms->acpi_dev ? max_cpuid : ms->smp.cpus - 1; ++ } ++ ++ if ((cpu->core_id < min_cpuid) || (cpu->core_id > max_cpuid)) { ++ error_setg(errp, "Invalid core-id %d specified, correct range %d:%d", ++ cpu->core_id, min_cpuid, max_cpuid); ++ return; ++ } ++ ++ if ((cpu->cluster_id < 0) || (cpu->cluster_id >= ms->smp.clusters)) { ++ error_setg(errp, "Invalid cluster-id %u specified, correct range 0:%u", ++ cpu->cluster_id, ms->smp.clusters - 1); ++ return; ++ } ++ ++ if ((cpu->socket_id < 0) || (cpu->socket_id >= ms->smp.sockets)) { ++ error_setg(errp, "Invalid socket-id %u specified, correct range 0:%u", ++ cpu->socket_id, ms->smp.sockets - 1); ++ return; ++ } ++ ++ cs->cpu_index = virt_get_cpu_id_from_cpu_topo(ms, dev); ++ ++ cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); ++ if (qemu_present_cpu(CPU(cpu_slot->cpu))) { ++ error_setg(errp, "cpu(id%d=%d:%d:%d:%d) with arch-id %" PRIu64 " exist", ++ cs->cpu_index, cpu->socket_id, cpu->cluster_id, cpu->core_id, ++ cpu->thread_id, cpu_slot->arch_id); ++ return; ++ } ++ virt_cpu_set_properties(OBJECT(cs), cpu_slot, errp); ++} ++ ++static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, ++ Error **errp) ++{ ++ MachineState *ms = MACHINE(hotplug_dev); ++ CPUState *cs = CPU(dev); ++ CPUArchId *cpu_slot; ++ ++ /* insert the cold/hot-plugged vcpu in the slot */ ++ cpu_slot = virt_find_cpu_slot(ms, cs->cpu_index); ++ cpu_slot->cpu = OBJECT(dev); ++ ++ cs->disabled = false; ++ return; ++} ++ + static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev, + DeviceState *dev, Error **errp) + { +@@ -2987,6 +3102,8 @@ static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev, + qlist_append_str(reserved_regions, resv_prop_str); + qdev_prop_set_array(dev, "reserved-regions", reserved_regions); + g_free(resv_prop_str); ++ } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { ++ virt_cpu_pre_plug(hotplug_dev, dev, errp); + } + } + +@@ -3008,6 +3125,8 @@ static void virt_machine_device_plug_cb(HotplugHandler *hotplug_dev, + virt_memory_plug(hotplug_dev, dev, errp); + } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) { + virtio_md_pci_plug(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp); ++ } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { ++ virt_cpu_plug(hotplug_dev, dev, errp); + } + + if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_IOMMU_PCI)) { +@@ -3092,7 +3211,8 @@ static HotplugHandler *virt_machine_get_hotplug_handler(MachineState *machine, + if (device_is_dynamic_sysbus(mc, dev) || + object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) || + object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI) || +- object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_IOMMU_PCI)) { ++ object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_IOMMU_PCI) || ++ object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { + return HOTPLUG_HANDLER(machine); + } + return NULL; +@@ -3169,6 +3289,7 @@ static void virt_machine_class_init(ObjectClass *oc, void *data) + #endif + mc->get_default_cpu_node_id = virt_get_default_cpu_node_id; + mc->kvm_type = virt_kvm_type; ++ mc->has_hotpluggable_cpus = true; + assert(!mc->get_hotplug_handler); + mc->get_hotplug_handler = virt_machine_get_hotplug_handler; + hc->pre_plug = virt_machine_device_pre_plug_cb; +diff --git a/target/arm/cpu.c b/target/arm/cpu.c +index cce315c18a..18b8a79c8f 100644 +--- a/target/arm/cpu.c ++++ b/target/arm/cpu.c +@@ -2477,6 +2477,12 @@ static const struct TCGCPUOps arm_tcg_ops = { + }; + #endif /* CONFIG_TCG */ + ++static int64_t arm_cpu_get_arch_id(CPUState *cs) ++{ ++ ARMCPU *cpu = ARM_CPU(cs); ++ return cpu->mp_affinity; ++} ++ + static void arm_cpu_class_init(ObjectClass *oc, void *data) + { + ARMCPUClass *acc = ARM_CPU_CLASS(oc); +@@ -2495,6 +2501,7 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data) + cc->class_by_name = arm_cpu_class_by_name; + cc->has_work = arm_cpu_has_work; + cc->dump_state = arm_cpu_dump_state; ++ cc->get_arch_id = arm_cpu_get_arch_id; + cc->set_pc = arm_cpu_set_pc; + cc->get_pc = arm_cpu_get_pc; + cc->gdb_read_register = arm_cpu_gdb_read_register; +diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c +index 471014b5a9..e226b60b72 100644 +--- a/target/arm/cpu64.c ++++ b/target/arm/cpu64.c +@@ -850,6 +850,17 @@ static void aarch64_cpu_set_aarch64(Object *obj, bool value, Error **errp) + } + } + ++static void aarch64_cpu_initfn(Object *obj) ++{ ++ CPUState *cs = CPU(obj); ++ ++ /* ++ * we start every ARM64 vcpu as disabled possible vCPU. It needs to be ++ * enabled explicitly ++ */ ++ cs->disabled = true; ++} ++ + static void aarch64_cpu_finalizefn(Object *obj) + { + } +@@ -862,7 +873,9 @@ static const gchar *aarch64_gdb_arch_name(CPUState *cs) + static void aarch64_cpu_class_init(ObjectClass *oc, void *data) + { + CPUClass *cc = CPU_CLASS(oc); ++ DeviceClass *dc = DEVICE_CLASS(oc); + ++ dc->user_creatable = true; + cc->gdb_read_register = aarch64_cpu_gdb_read_register; + cc->gdb_write_register = aarch64_cpu_gdb_write_register; + cc->gdb_num_core_regs = 34; +@@ -908,6 +921,7 @@ void aarch64_cpu_register(const ARMCPUInfo *info) + static const TypeInfo aarch64_cpu_type_info = { + .name = TYPE_AARCH64_CPU, + .parent = TYPE_ARM_CPU, ++ .instance_init = aarch64_cpu_initfn, + .instance_finalize = aarch64_cpu_finalizefn, + .abstract = true, + .class_init = aarch64_cpu_class_init, +-- +2.27.0 + diff --git a/arm-virt.c-Convey-local_err-when-set-psci-conduit.patch b/arm-virt.c-Convey-local_err-when-set-psci-conduit.patch new file mode 100644 index 0000000000000000000000000000000000000000..7a2b9ced7bc1c37c4c19bd08d7662f3b63342ee2 --- /dev/null +++ b/arm-virt.c-Convey-local_err-when-set-psci-conduit.patch @@ -0,0 +1,29 @@ +From 25438f2cdb13d07c1bd228fcf4223c21da368548 Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 15:15:31 +0800 +Subject: [PATCH] arm/virt.c: Convey local_err when set psci-conduit + +Signed-off-by: Keqian Zhu +--- + hw/arm/virt.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index ed437ce0e8..934b0412ef 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -2323,7 +2323,10 @@ static void virt_cpu_set_properties(Object *cpuobj, const CPUArchId *cpu_slot, + */ + if (vms->psci_conduit != QEMU_PSCI_CONDUIT_DISABLED) { + object_property_set_int(cpuobj, "psci-conduit", vms->psci_conduit, +- NULL); ++ &local_err); ++ if (local_err) { ++ goto out; ++ } + + /* Secondary CPUs start in PSCI powered-down state */ + if (CPU(cpuobj)->cpu_index > 0) { +-- +2.27.0 + diff --git a/coro-support-live-patch-for-libcare.patch b/coro-support-live-patch-for-libcare.patch new file mode 100644 index 0000000000000000000000000000000000000000..71b83c2df433f74db003e6ceee10a067f7db39af --- /dev/null +++ b/coro-support-live-patch-for-libcare.patch @@ -0,0 +1,116 @@ +From c2b377814e7874811d7eb98462d5153e966281cf Mon Sep 17 00:00:00 2001 +From: Fei Xu +Date: Wed, 3 Apr 2024 18:05:25 +0800 +Subject: [PATCH] coro: support live patch for libcare + +Signed-off-by: Dawei Jiang +--- + include/qemu/coroutine_int.h | 3 ++- + util/coroutine-ucontext.c | 52 ++++++++++++++++++++++++++++++++++++ + util/qemu-coroutine.c | 4 +++ + 3 files changed, 58 insertions(+), 1 deletion(-) + +diff --git a/include/qemu/coroutine_int.h b/include/qemu/coroutine_int.h +index 1da148552f..11b550a0fc 100644 +--- a/include/qemu/coroutine_int.h ++++ b/include/qemu/coroutine_int.h +@@ -73,5 +73,6 @@ Coroutine *qemu_coroutine_new(void); + void qemu_coroutine_delete(Coroutine *co); + CoroutineAction qemu_coroutine_switch(Coroutine *from, Coroutine *to, + CoroutineAction action); +- ++void qemu_coroutine_info_add(const Coroutine *co_); ++void qemu_coroutine_info_delete(const Coroutine *co_); + #endif +diff --git a/util/coroutine-ucontext.c b/util/coroutine-ucontext.c +index 7b304c79d9..650c21846d 100644 +--- a/util/coroutine-ucontext.c ++++ b/util/coroutine-ucontext.c +@@ -80,6 +80,19 @@ union cc_arg { + int i[2]; + }; + ++/** ++ * coroutines list for libcare ++ */ ++struct CoroutineInformation { ++ sigjmp_buf *env; ++ QLIST_ENTRY(CoroutineInformation) next; ++}; ++ ++static QemuMutex coro_mtx; ++QLIST_HEAD(, CoroutineInformation) coro_info_list = QLIST_HEAD_INITIALIZER(pool); ++int coro_env_offset = offsetof(struct CoroutineInformation, env); ++int coro_next_offset = offsetof(struct CoroutineInformation, next); ++ + /* + * QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it. + * always_inline is required to avoid TSan runtime fatal errors. +@@ -340,3 +353,42 @@ bool qemu_in_coroutine(void) + + return self && self->caller; + } ++ ++static void __attribute__((constructor)) coro_mutex_init(void) ++{ ++ qemu_mutex_init(&coro_mtx); ++} ++ ++void qemu_coroutine_info_add(const Coroutine *co_) ++{ ++ CoroutineUContext *co; ++ struct CoroutineInformation *coro_info; ++ ++ /* save coroutine env to coro_info_list */ ++ co = DO_UPCAST(CoroutineUContext, base, co_); ++ coro_info = g_malloc0(sizeof(struct CoroutineInformation)); ++ coro_info->env = &co->env; ++ ++ qemu_mutex_lock(&coro_mtx); ++ QLIST_INSERT_HEAD(&coro_info_list, coro_info, next); ++ qemu_mutex_unlock(&coro_mtx); ++} ++ ++void qemu_coroutine_info_delete(const Coroutine *co_) ++{ ++ CoroutineUContext *co; ++ struct CoroutineInformation *coro_info; ++ ++ /* Remove relative coroutine env info from coro_info_list */ ++ co = DO_UPCAST(CoroutineUContext, base, co_); ++ ++ qemu_mutex_lock(&coro_mtx); ++ QLIST_FOREACH(coro_info, &coro_info_list, next) { ++ if (coro_info->env == &co->env) { ++ QLIST_REMOVE(coro_info, next); ++ g_free(coro_info); ++ break; ++ } ++ } ++ qemu_mutex_unlock(&coro_mtx); ++} +diff --git a/util/qemu-coroutine.c b/util/qemu-coroutine.c +index 5fd2dbaf8b..f550214484 100644 +--- a/util/qemu-coroutine.c ++++ b/util/qemu-coroutine.c +@@ -89,6 +89,8 @@ Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque) + co = qemu_coroutine_new(); + } + ++ qemu_coroutine_info_add(co); ++ + co->entry = entry; + co->entry_arg = opaque; + QSIMPLEQ_INIT(&co->co_queue_wakeup); +@@ -99,6 +101,8 @@ static void coroutine_delete(Coroutine *co) + { + co->caller = NULL; + ++ qemu_coroutine_info_delete(co); ++ + if (IS_ENABLED(CONFIG_COROUTINE_POOL)) { + if (release_pool_size < qatomic_read(&pool_max_size) * 2) { + QSLIST_INSERT_HEAD_ATOMIC(&release_pool, co, pool_next); +-- +2.27.0 + diff --git a/cpus-common-Add-common-CPU-utility-for-possible-vCPU.patch b/cpus-common-Add-common-CPU-utility-for-possible-vCPU.patch new file mode 100644 index 0000000000000000000000000000000000000000..e2148e867fe6d83f1c5b25d7c242b17f02dfd472 --- /dev/null +++ b/cpus-common-Add-common-CPU-utility-for-possible-vCPU.patch @@ -0,0 +1,144 @@ +From 444de91551c1e141a76bf3dae4cebee9dbd57b49 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Wed, 6 May 2020 02:48:49 +0100 +Subject: [PATCH] cpus-common: Add common CPU utility for possible vCPUs + +Adds various utility functions which might be required to fetch or check the +state of the possible vCPUs. This also introduces concept of *disabled* vCPUs, +which are part of the *possible* vCPUs but are not part of the *present* vCPU. +This state shall be used during machine init time to check the presence of +vcpus. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + cpu-common.c | 31 +++++++++++++++++++++++++ + include/hw/core/cpu.h | 53 +++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 84 insertions(+) + +diff --git a/cpu-common.c b/cpu-common.c +index c81fd72d16..d041a351ab 100644 +--- a/cpu-common.c ++++ b/cpu-common.c +@@ -24,6 +24,7 @@ + #include "sysemu/cpus.h" + #include "qemu/lockable.h" + #include "trace/trace-root.h" ++#include "hw/boards.h" + + QemuMutex qemu_cpu_list_lock; + static QemuCond exclusive_cond; +@@ -107,6 +108,36 @@ void cpu_list_remove(CPUState *cpu) + cpu_list_generation_id++; + } + ++CPUState *qemu_get_possible_cpu(int index) ++{ ++ MachineState *ms = MACHINE(qdev_get_machine()); ++ const CPUArchIdList *possible_cpus = ms->possible_cpus; ++ ++ assert((index >= 0) && (index < possible_cpus->len)); ++ ++ return CPU(possible_cpus->cpus[index].cpu); ++} ++ ++bool qemu_present_cpu(CPUState *cpu) ++{ ++ return cpu; ++} ++ ++bool qemu_enabled_cpu(CPUState *cpu) ++{ ++ return cpu && !cpu->disabled; ++} ++ ++uint64_t qemu_get_cpu_archid(int cpu_index) ++{ ++ MachineState *ms = MACHINE(qdev_get_machine()); ++ const CPUArchIdList *possible_cpus = ms->possible_cpus; ++ ++ assert((cpu_index >= 0) && (cpu_index < possible_cpus->len)); ++ ++ return possible_cpus->cpus[cpu_index].arch_id; ++} ++ + CPUState *qemu_get_cpu(int index) + { + CPUState *cpu; +diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h +index c0c8320413..c30636a936 100644 +--- a/include/hw/core/cpu.h ++++ b/include/hw/core/cpu.h +@@ -538,6 +538,17 @@ struct CPUState { + GArray *plugin_mem_cbs; + #endif + ++ /* ++ * Some architectures do not allow *presence* of vCPUs to be changed ++ * after guest has booted using information specified by VMM/firmware ++ * via ACPI MADT at the boot time. Thus to enable vCPU hotplug on these ++ * architectures possible vCPU can have CPUState object in 'disabled' ++ * state or can also not have CPUState object at all. This is possible ++ * when vCPU Hotplug is supported and vCPUs are 'yet-to-be-plugged' in ++ * the QOM or have been hot-unplugged. ++ * By default every CPUState is enabled as of now across all archs. ++ */ ++ bool disabled; + /* TODO Move common fields from CPUArchState here. */ + int cpu_index; + int cluster_index; +@@ -913,6 +924,48 @@ static inline bool cpu_in_exclusive_context(const CPUState *cpu) + */ + CPUState *qemu_get_cpu(int index); + ++/** ++ * qemu_get_possible_cpu: ++ * @index: The CPUState@cpu_index value of the CPU to obtain. ++ * Input index MUST be in range [0, Max Possible CPUs) ++ * ++ * If CPUState object exists,then it gets a CPU matching ++ * @index in the possible CPU array. ++ * ++ * Returns: The possible CPU or %NULL if CPU does not exist. ++ */ ++CPUState *qemu_get_possible_cpu(int index); ++ ++/** ++ * qemu_present_cpu: ++ * @cpu: The vCPU to check ++ * ++ * Checks if the vCPU is amongst the present possible vcpus. ++ * ++ * Returns: True if it is present possible vCPU else false ++ */ ++bool qemu_present_cpu(CPUState *cpu); ++ ++/** ++ * qemu_enabled_cpu: ++ * @cpu: The vCPU to check ++ * ++ * Checks if the vCPU is enabled. ++ * ++ * Returns: True if it is 'enabled' else false ++ */ ++bool qemu_enabled_cpu(CPUState *cpu); ++ ++/** ++ * qemu_get_cpu_archid: ++ * @cpu_index: possible vCPU for which arch-id needs to be retreived ++ * ++ * Fetches the vCPU arch-id from the present possible vCPUs. ++ * ++ * Returns: arch-id of the possible vCPU ++ */ ++uint64_t qemu_get_cpu_archid(int cpu_index); ++ + /** + * cpu_exists: + * @id: Guest-exposed CPU ID to lookup. +-- +2.27.0 + diff --git a/hw-acpi-ACPI-AML-Changes-to-reflect-the-correct-_STA.patch b/hw-acpi-ACPI-AML-Changes-to-reflect-the-correct-_STA.patch new file mode 100644 index 0000000000000000000000000000000000000000..34fa1c91b9d0e375f7f85a4477b3d897b63d936f --- /dev/null +++ b/hw-acpi-ACPI-AML-Changes-to-reflect-the-correct-_STA.patch @@ -0,0 +1,187 @@ +From 19a8fbccbc997110f472df308813ad2d7738065c Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Mon, 14 Nov 2022 02:25:28 +0000 +Subject: [PATCH] hw/acpi: ACPI/AML Changes to reflect the correct + _STA.{PRES,ENA} Bits to Guest + +ACPI AML changes to properly reflect the _STA.PRES and _STA.ENA Bits to the +guest during initialzation, when CPUs are hotplugged and after CPUs are +hot-unplugged. + +Signed-off-by: Salil Mehta +--- + hw/acpi/cpu.c | 49 +++++++++++++++++++++++++++++++--- + hw/acpi/generic_event_device.c | 11 ++++++++ + include/hw/acpi/cpu.h | 2 ++ + 3 files changed, 58 insertions(+), 4 deletions(-) + +diff --git a/hw/acpi/cpu.c b/hw/acpi/cpu.c +index c8c11e51c6..991f1d4181 100644 +--- a/hw/acpi/cpu.c ++++ b/hw/acpi/cpu.c +@@ -64,10 +64,11 @@ static uint64_t cpu_hotplug_rd(void *opaque, hwaddr addr, unsigned size) + cdev = &cpu_st->devs[cpu_st->selector]; + switch (addr) { + case ACPI_CPU_FLAGS_OFFSET_RW: /* pack and return is_* fields */ +- val |= cdev->cpu ? 1 : 0; ++ val |= cdev->is_enabled ? 1 : 0; + val |= cdev->is_inserting ? 2 : 0; + val |= cdev->is_removing ? 4 : 0; + val |= cdev->fw_remove ? 16 : 0; ++ val |= cdev->is_present ? 32 : 0; + trace_cpuhp_acpi_read_flags(cpu_st->selector, val); + break; + case ACPI_CPU_CMD_DATA_OFFSET_RW: +@@ -229,7 +230,21 @@ void cpu_hotplug_hw_init(MemoryRegion *as, Object *owner, + struct CPUState *cpu = CPU(id_list->cpus[i].cpu); + if (qemu_present_cpu(cpu)) { + state->devs[i].cpu = cpu; ++ state->devs[i].is_present = true; ++ } else { ++ if (qemu_persistent_cpu(cpu)) { ++ state->devs[i].is_present = true; ++ } else { ++ state->devs[i].is_present = false; ++ } + } ++ ++ if (qemu_enabled_cpu(cpu)) { ++ state->devs[i].is_enabled = true; ++ } else { ++ state->devs[i].is_enabled = false; ++ } ++ + state->devs[i].arch_id = id_list->cpus[i].arch_id; + } + memory_region_init_io(&state->ctrl_reg, owner, &cpu_hotplug_ops, state, +@@ -262,6 +277,8 @@ void acpi_cpu_plug_cb(HotplugHandler *hotplug_dev, + } + + cdev->cpu = CPU(dev); ++ cdev->is_present = true; ++ cdev->is_enabled = true; + if (dev->hotplugged) { + cdev->is_inserting = true; + acpi_send_event(DEVICE(hotplug_dev), ACPI_CPU_HOTPLUG_STATUS); +@@ -293,6 +310,11 @@ void acpi_cpu_unplug_cb(CPUHotplugState *cpu_st, + return; + } + ++ cdev->is_enabled = false; ++ if (!qemu_persistent_cpu(CPU(dev))) { ++ cdev->is_present = false; ++ } ++ + cdev->cpu = NULL; + } + +@@ -303,6 +325,8 @@ static const VMStateDescription vmstate_cpuhp_sts = { + .fields = (VMStateField[]) { + VMSTATE_BOOL(is_inserting, AcpiCpuStatus), + VMSTATE_BOOL(is_removing, AcpiCpuStatus), ++ VMSTATE_BOOL(is_present, AcpiCpuStatus), ++ VMSTATE_BOOL(is_enabled, AcpiCpuStatus), + VMSTATE_UINT32(ost_event, AcpiCpuStatus), + VMSTATE_UINT32(ost_status, AcpiCpuStatus), + VMSTATE_END_OF_LIST() +@@ -340,6 +364,7 @@ const VMStateDescription vmstate_cpu_hotplug = { + #define CPU_REMOVE_EVENT "CRMV" + #define CPU_EJECT_EVENT "CEJ0" + #define CPU_FW_EJECT_EVENT "CEJF" ++#define CPU_PRESENT "CPRS" + + void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, + build_madt_cpu_fn build_madt_cpu, +@@ -400,7 +425,9 @@ void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, + aml_append(field, aml_named_field(CPU_EJECT_EVENT, 1)); + /* tell firmware to do device eject, write only */ + aml_append(field, aml_named_field(CPU_FW_EJECT_EVENT, 1)); +- aml_append(field, aml_reserved_field(3)); ++ /* 1 if present, read only */ ++ aml_append(field, aml_named_field(CPU_PRESENT, 1)); ++ aml_append(field, aml_reserved_field(2)); + aml_append(field, aml_named_field(CPU_COMMAND, 8)); + aml_append(cpu_ctrl_dev, field); + +@@ -430,6 +457,7 @@ void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, + Aml *ctrl_lock = aml_name("%s.%s", cphp_res_path, CPU_LOCK); + Aml *cpu_selector = aml_name("%s.%s", cphp_res_path, CPU_SELECTOR); + Aml *is_enabled = aml_name("%s.%s", cphp_res_path, CPU_ENABLED); ++ Aml *is_present = aml_name("%s.%s", cphp_res_path, CPU_PRESENT); + Aml *cpu_cmd = aml_name("%s.%s", cphp_res_path, CPU_COMMAND); + Aml *cpu_data = aml_name("%s.%s", cphp_res_path, CPU_DATA); + Aml *ins_evt = aml_name("%s.%s", cphp_res_path, CPU_INSERT_EVENT); +@@ -458,13 +486,26 @@ void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, + { + Aml *idx = aml_arg(0); + Aml *sta = aml_local(0); ++ Aml *ifctx2; ++ Aml *else_ctx; + + aml_append(method, aml_acquire(ctrl_lock, 0xFFFF)); + aml_append(method, aml_store(idx, cpu_selector)); + aml_append(method, aml_store(zero, sta)); +- ifctx = aml_if(aml_equal(is_enabled, one)); ++ ifctx = aml_if(aml_equal(is_present, one)); + { +- aml_append(ifctx, aml_store(aml_int(0xF), sta)); ++ ifctx2 = aml_if(aml_equal(is_enabled, one)); ++ { ++ /* cpu is present and enabled */ ++ aml_append(ifctx2, aml_store(aml_int(0xF), sta)); ++ } ++ aml_append(ifctx, ifctx2); ++ else_ctx = aml_else(); ++ { ++ /* cpu is present but disabled */ ++ aml_append(else_ctx, aml_store(aml_int(0xD), sta)); ++ } ++ aml_append(ifctx, else_ctx); + } + aml_append(method, ifctx); + aml_append(method, aml_release(ctrl_lock)); +diff --git a/hw/acpi/generic_event_device.c b/hw/acpi/generic_event_device.c +index d2fa1d0e4a..b84602b238 100644 +--- a/hw/acpi/generic_event_device.c ++++ b/hw/acpi/generic_event_device.c +@@ -319,6 +319,16 @@ static const VMStateDescription vmstate_memhp_state = { + } + }; + ++static const VMStateDescription vmstate_cpuhp_state = { ++ .name = "acpi-ged/cpuhp", ++ .version_id = 1, ++ .minimum_version_id = 1, ++ .fields = (VMStateField[]) { ++ VMSTATE_CPU_HOTPLUG(cpuhp_state, AcpiGedState), ++ VMSTATE_END_OF_LIST() ++ } ++}; ++ + static const VMStateDescription vmstate_ged_state = { + .name = "acpi-ged-state", + .version_id = 1, +@@ -367,6 +377,7 @@ static const VMStateDescription vmstate_acpi_ged = { + }, + .subsections = (const VMStateDescription * []) { + &vmstate_memhp_state, ++ &vmstate_cpuhp_state, + &vmstate_ghes_state, + NULL + } +diff --git a/include/hw/acpi/cpu.h b/include/hw/acpi/cpu.h +index b31a2e50d9..fced952152 100644 +--- a/include/hw/acpi/cpu.h ++++ b/include/hw/acpi/cpu.h +@@ -23,6 +23,8 @@ typedef struct AcpiCpuStatus { + uint64_t arch_id; + bool is_inserting; + bool is_removing; ++ bool is_present; ++ bool is_enabled; + bool fw_remove; + uint32_t ost_event; + uint32_t ost_status; +-- +2.27.0 + diff --git a/hw-acpi-Add-ACPI-CPU-hotplug-init-stub.patch b/hw-acpi-Add-ACPI-CPU-hotplug-init-stub.patch new file mode 100644 index 0000000000000000000000000000000000000000..072d4eb9bac4ea0a553e14b6e2ce85a6961cc19b --- /dev/null +++ b/hw-acpi-Add-ACPI-CPU-hotplug-init-stub.patch @@ -0,0 +1,34 @@ +From e442d0f8670dc4218ab4beebe645e369f925410d Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sat, 19 Aug 2023 00:26:20 +0000 +Subject: [PATCH] hw/acpi: Add ACPI CPU hotplug init stub + +ACPI CPU hotplug related initialization should only happend if ACPI_CPU_HOTPLUG +support has been enabled for particular architecture. Add cpu_hotplug_hw_init() +stub to avoid compilation break. + +Signed-off-by: Salil Mehta +--- + hw/acpi/acpi-cpu-hotplug-stub.c | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/hw/acpi/acpi-cpu-hotplug-stub.c b/hw/acpi/acpi-cpu-hotplug-stub.c +index 3fc4b14c26..c6c61bb9cd 100644 +--- a/hw/acpi/acpi-cpu-hotplug-stub.c ++++ b/hw/acpi/acpi-cpu-hotplug-stub.c +@@ -19,6 +19,12 @@ void legacy_acpi_cpu_hotplug_init(MemoryRegion *parent, Object *owner, + return; + } + ++void cpu_hotplug_hw_init(MemoryRegion *as, Object *owner, ++ CPUHotplugState *state, hwaddr base_addr) ++{ ++ return; ++} ++ + void acpi_cpu_ospm_status(CPUHotplugState *cpu_st, ACPIOSTInfoList ***list) + { + return; +-- +2.27.0 + diff --git a/hw-acpi-Init-GED-framework-with-cpu-hotplug-events.patch b/hw-acpi-Init-GED-framework-with-cpu-hotplug-events.patch new file mode 100644 index 0000000000000000000000000000000000000000..191328fd094c386d672ca9c32341105fa1cc0b1a --- /dev/null +++ b/hw-acpi-Init-GED-framework-with-cpu-hotplug-events.patch @@ -0,0 +1,81 @@ +From de1c8d6be3de67ff9854e9b008a000e1898aaacb Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Mon, 8 Jun 2020 21:50:08 +0100 +Subject: [PATCH] hw/acpi: Init GED framework with cpu hotplug events + +ACPI GED(as described in the ACPI 6.2 spec) can be used to generate ACPI events +when OSPM/guest receives an interrupt listed in the _CRS object of GED. OSPM +then maps or demultiplexes the event by evaluating _EVT method. + +This change adds the support of cpu hotplug event initialization in the +existing GED framework. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/acpi/generic_event_device.c | 8 ++++++++ + include/hw/acpi/generic_event_device.h | 5 +++++ + 2 files changed, 13 insertions(+) + +diff --git a/hw/acpi/generic_event_device.c b/hw/acpi/generic_event_device.c +index a3d31631fe..d2fa1d0e4a 100644 +--- a/hw/acpi/generic_event_device.c ++++ b/hw/acpi/generic_event_device.c +@@ -25,6 +25,7 @@ static const uint32_t ged_supported_events[] = { + ACPI_GED_MEM_HOTPLUG_EVT, + ACPI_GED_PWR_DOWN_EVT, + ACPI_GED_NVDIMM_HOTPLUG_EVT, ++ ACPI_GED_CPU_HOTPLUG_EVT, + }; + + /* +@@ -400,6 +401,13 @@ static void acpi_ged_initfn(Object *obj) + memory_region_init_io(&ged_st->regs, obj, &ged_regs_ops, ged_st, + TYPE_ACPI_GED "-regs", ACPI_GED_REG_COUNT); + sysbus_init_mmio(sbd, &ged_st->regs); ++ ++ s->cpuhp.device = OBJECT(s); ++ memory_region_init(&s->container_cpuhp, OBJECT(dev), "cpuhp container", ++ ACPI_CPU_HOTPLUG_REG_LEN); ++ sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->container_cpuhp); ++ cpu_hotplug_hw_init(&s->container_cpuhp, OBJECT(dev), ++ &s->cpuhp_state, 0); + } + + static void acpi_ged_class_init(ObjectClass *class, void *data) +diff --git a/include/hw/acpi/generic_event_device.h b/include/hw/acpi/generic_event_device.h +index ba84ce0214..a803ea818e 100644 +--- a/include/hw/acpi/generic_event_device.h ++++ b/include/hw/acpi/generic_event_device.h +@@ -60,6 +60,7 @@ + #define HW_ACPI_GENERIC_EVENT_DEVICE_H + + #include "hw/sysbus.h" ++#include "hw/acpi/cpu_hotplug.h" + #include "hw/acpi/memory_hotplug.h" + #include "hw/acpi/ghes.h" + #include "qom/object.h" +@@ -95,6 +96,7 @@ OBJECT_DECLARE_SIMPLE_TYPE(AcpiGedState, ACPI_GED) + #define ACPI_GED_MEM_HOTPLUG_EVT 0x1 + #define ACPI_GED_PWR_DOWN_EVT 0x2 + #define ACPI_GED_NVDIMM_HOTPLUG_EVT 0x4 ++#define ACPI_GED_CPU_HOTPLUG_EVT 0x8 + + typedef struct GEDState { + MemoryRegion evt; +@@ -106,6 +108,9 @@ struct AcpiGedState { + SysBusDevice parent_obj; + MemHotplugState memhp_state; + MemoryRegion container_memhp; ++ CPUHotplugState cpuhp_state; ++ MemoryRegion container_cpuhp; ++ AcpiCpuHotplug cpuhp; + GEDState ged_state; + uint32_t ged_event_bitmap; + qemu_irq irq; +-- +2.27.0 + diff --git a/hw-acpi-Make-_MAT-method-optional.patch b/hw-acpi-Make-_MAT-method-optional.patch new file mode 100644 index 0000000000000000000000000000000000000000..5695a1981c6efbe5ed71981f97dcf81d8eee5e8e --- /dev/null +++ b/hw-acpi-Make-_MAT-method-optional.patch @@ -0,0 +1,41 @@ +From e9b0d476172e872bf695780a9ffa8072faeb3cd0 Mon Sep 17 00:00:00 2001 +From: Jean-Philippe Brucker +Date: Mon, 25 Apr 2022 17:40:57 +0100 +Subject: [PATCH] hw/acpi: Make _MAT method optional + +The GICC interface on arm64 vCPUs is statically defined in the MADT, and +doesn't require a _MAT entry. Although the GICC is indicated as present +by the MADT entry, it can only be used from vCPU sysregs, which aren't +accessible until hot-add. + +Co-developed-by: Jean-Philippe Brucker +Signed-off-by: Jean-Philippe Brucker +Co-developed-by: Jonathan Cameron +Signed-off-by: Jonathan Cameron +Signed-off-by: Salil Mehta +--- + hw/acpi/cpu.c | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/hw/acpi/cpu.c b/hw/acpi/cpu.c +index 991f1d4181..c922c380aa 100644 +--- a/hw/acpi/cpu.c ++++ b/hw/acpi/cpu.c +@@ -720,9 +720,11 @@ void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, + aml_append(dev, method); + + /* build _MAT object */ +- build_madt_cpu(i, arch_ids, madt_buf, true); /* set enabled flag */ +- aml_append(dev, aml_name_decl("_MAT", +- aml_buffer(madt_buf->len, (uint8_t *)madt_buf->data))); ++ if (build_madt_cpu) { ++ build_madt_cpu(i, arch_ids, madt_buf, true); /* set enabled flag */ ++ aml_append(dev, aml_name_decl("_MAT", ++ aml_buffer(madt_buf->len, (uint8_t *)madt_buf->data))); ++ } + g_array_free(madt_buf, true); + + if (CPU(arch_ids->cpus[i].cpu) != first_cpu) { +-- +2.27.0 + diff --git a/hw-acpi-Move-CPU-ctrl-dev-MMIO-region-len-macro-to-c.patch b/hw-acpi-Move-CPU-ctrl-dev-MMIO-region-len-macro-to-c.patch new file mode 100644 index 0000000000000000000000000000000000000000..9bfb91e15162c90365c621927205f857269bc4c3 --- /dev/null +++ b/hw-acpi-Move-CPU-ctrl-dev-MMIO-region-len-macro-to-c.patch @@ -0,0 +1,52 @@ +From fd6e7e7278e1c0fb08e0a09d9e22157e11b36ece Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sun, 20 Aug 2023 17:11:04 +0000 +Subject: [PATCH] hw/acpi: Move CPU ctrl-dev MMIO region len macro to common + header file + +CPU ctrl-dev MMIO region length could be used in ACPI GED (common ACPI code +across architectures) and various other architecture specific places. To make +these code places independent of compilation order, ACPI_CPU_HOTPLUG_REG_LEN +macro should be moved to a header file. + +Signed-off-by: Salil Mehta +--- + hw/acpi/cpu.c | 2 +- + include/hw/acpi/cpu_hotplug.h | 2 ++ + 2 files changed, 3 insertions(+), 1 deletion(-) + +diff --git a/hw/acpi/cpu.c b/hw/acpi/cpu.c +index 011d2c6c2d..4b24a25003 100644 +--- a/hw/acpi/cpu.c ++++ b/hw/acpi/cpu.c +@@ -1,13 +1,13 @@ + #include "qemu/osdep.h" + #include "migration/vmstate.h" + #include "hw/acpi/cpu.h" ++#include "hw/acpi/cpu_hotplug.h" + #include "hw/core/cpu.h" + #include "qapi/error.h" + #include "qapi/qapi-events-acpi.h" + #include "trace.h" + #include "sysemu/numa.h" + +-#define ACPI_CPU_HOTPLUG_REG_LEN 12 + #define ACPI_CPU_SELECTOR_OFFSET_WR 0 + #define ACPI_CPU_FLAGS_OFFSET_RW 4 + #define ACPI_CPU_CMD_OFFSET_WR 5 +diff --git a/include/hw/acpi/cpu_hotplug.h b/include/hw/acpi/cpu_hotplug.h +index 3b932abbbb..48b291e45e 100644 +--- a/include/hw/acpi/cpu_hotplug.h ++++ b/include/hw/acpi/cpu_hotplug.h +@@ -19,6 +19,8 @@ + #include "hw/hotplug.h" + #include "hw/acpi/cpu.h" + ++#define ACPI_CPU_HOTPLUG_REG_LEN 12 ++ + typedef struct AcpiCpuHotplug { + Object *device; + MemoryRegion io; +-- +2.27.0 + diff --git a/hw-acpi-Update-ACPI-GED-framework-to-support-vCPU-Ho.patch b/hw-acpi-Update-ACPI-GED-framework-to-support-vCPU-Ho.patch new file mode 100644 index 0000000000000000000000000000000000000000..1b2a0a023e0ed032379dd39051804d267edad0f1 --- /dev/null +++ b/hw-acpi-Update-ACPI-GED-framework-to-support-vCPU-Ho.patch @@ -0,0 +1,77 @@ +From 0bdb1861985704af9b82e35053b5ab99f7880eb6 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Thu, 7 May 2020 21:30:09 +0100 +Subject: [PATCH] hw/acpi: Update ACPI GED framework to support vCPU Hotplug + +ACPI GED shall be used to convey to the guest kernel about any CPU hot-(un)plug +events. Therefore, existing ACPI GED framework inside QEMU needs to be enhanced +to support CPU hotplug state and events. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/acpi/generic_event_device.c | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/hw/acpi/generic_event_device.c b/hw/acpi/generic_event_device.c +index ad252e6a91..0266733a54 100644 +--- a/hw/acpi/generic_event_device.c ++++ b/hw/acpi/generic_event_device.c +@@ -12,6 +12,7 @@ + #include "qemu/osdep.h" + #include "qapi/error.h" + #include "hw/acpi/acpi.h" ++#include "hw/acpi/cpu.h" + #include "hw/acpi/generic_event_device.h" + #include "hw/irq.h" + #include "hw/mem/pc-dimm.h" +@@ -239,6 +240,8 @@ static void acpi_ged_device_plug_cb(HotplugHandler *hotplug_dev, + } else { + acpi_memory_plug_cb(hotplug_dev, &s->memhp_state, dev, errp); + } ++ } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { ++ acpi_cpu_plug_cb(hotplug_dev, &s->cpuhp_state, dev, errp); + } else { + error_setg(errp, "virt: device plug request for unsupported device" + " type: %s", object_get_typename(OBJECT(dev))); +@@ -253,6 +256,8 @@ static void acpi_ged_unplug_request_cb(HotplugHandler *hotplug_dev, + if ((object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) && + !(object_dynamic_cast(OBJECT(dev), TYPE_NVDIMM)))) { + acpi_memory_unplug_request_cb(hotplug_dev, &s->memhp_state, dev, errp); ++ } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { ++ acpi_cpu_unplug_request_cb(hotplug_dev, &s->cpuhp_state, dev, errp); + } else { + error_setg(errp, "acpi: device unplug request for unsupported device" + " type: %s", object_get_typename(OBJECT(dev))); +@@ -266,6 +271,8 @@ static void acpi_ged_unplug_cb(HotplugHandler *hotplug_dev, + + if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) { + acpi_memory_unplug_cb(&s->memhp_state, dev, errp); ++ } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) { ++ acpi_cpu_unplug_cb(&s->cpuhp_state, dev, errp); + } else { + error_setg(errp, "acpi: device unplug for unsupported device" + " type: %s", object_get_typename(OBJECT(dev))); +@@ -277,6 +284,7 @@ static void acpi_ged_ospm_status(AcpiDeviceIf *adev, ACPIOSTInfoList ***list) + AcpiGedState *s = ACPI_GED(adev); + + acpi_memory_ospm_status(&s->memhp_state, list); ++ acpi_cpu_ospm_status(&s->cpuhp_state, list); + } + + static void acpi_ged_send_event(AcpiDeviceIf *adev, AcpiEventStatusBits ev) +@@ -291,6 +299,8 @@ static void acpi_ged_send_event(AcpiDeviceIf *adev, AcpiEventStatusBits ev) + sel = ACPI_GED_PWR_DOWN_EVT; + } else if (ev & ACPI_NVDIMM_HOTPLUG_STATUS) { + sel = ACPI_GED_NVDIMM_HOTPLUG_EVT; ++ } else if (ev & ACPI_CPU_HOTPLUG_STATUS) { ++ sel = ACPI_GED_CPU_HOTPLUG_EVT; + } else { + /* Unknown event. Return without generating interrupt. */ + warn_report("GED: Unsupported event %d. No irq injected", ev); +-- +2.27.0 + diff --git a/hw-acpi-Update-CPUs-AML-with-cpu-ctrl-dev-change.patch b/hw-acpi-Update-CPUs-AML-with-cpu-ctrl-dev-change.patch new file mode 100644 index 0000000000000000000000000000000000000000..91d4c4d78e6922d07c7981c2a886c70d07c45d2d --- /dev/null +++ b/hw-acpi-Update-CPUs-AML-with-cpu-ctrl-dev-change.patch @@ -0,0 +1,118 @@ +From 06059c960d863c21c7d9cf4829ad2078692ed9e1 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Fri, 8 May 2020 13:27:57 +0100 +Subject: [PATCH] hw/acpi: Update CPUs AML with cpu-(ctrl)dev change + +CPUs Control device(\\_SB.PCI0) register interface for the x86 arch is based on +PCI and is IO port based and hence existing cpus AML code assumes _CRS objects +would evaluate to a system resource which describes IO Port address. But on ARM +arch CPUs control device(\\_SB.PRES) register interface is memory-mapped hence +_CRS object should evaluate to system resource which describes memory-mapped +base address. + +This cpus AML code change updates the existing inerface of the build cpus AML +function to accept both IO/MEMORY type regions and update the _CRS object +correspondingly. + +NOTE: Beside above CPU scan shall be triggered when OSPM evaluates _EVT method + part of the GED framework which is covered in subsequent patch. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/acpi/cpu.c | 23 ++++++++++++++++------- + hw/i386/acpi-build.c | 3 ++- + include/hw/acpi/cpu.h | 5 +++-- + 3 files changed, 21 insertions(+), 10 deletions(-) + +diff --git a/hw/acpi/cpu.c b/hw/acpi/cpu.c +index cabeb4e86b..cf0c7e8538 100644 +--- a/hw/acpi/cpu.c ++++ b/hw/acpi/cpu.c +@@ -342,9 +342,10 @@ const VMStateDescription vmstate_cpu_hotplug = { + #define CPU_FW_EJECT_EVENT "CEJF" + + void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, +- build_madt_cpu_fn build_madt_cpu, hwaddr io_base, ++ build_madt_cpu_fn build_madt_cpu, hwaddr base_addr, + const char *res_root, +- const char *event_handler_method) ++ const char *event_handler_method, ++ AmlRegionSpace rs) + { + Aml *ifctx; + Aml *field; +@@ -369,13 +370,19 @@ void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, + aml_append(cpu_ctrl_dev, aml_mutex(CPU_LOCK, 0)); + + crs = aml_resource_template(); +- aml_append(crs, aml_io(AML_DECODE16, io_base, io_base, 1, ++ if (rs == AML_SYSTEM_IO) { ++ aml_append(crs, aml_io(AML_DECODE16, base_addr, base_addr, 1, + ACPI_CPU_HOTPLUG_REG_LEN)); ++ } else { ++ aml_append(crs, aml_memory32_fixed(base_addr, ++ ACPI_CPU_HOTPLUG_REG_LEN, AML_READ_WRITE)); ++ } ++ + aml_append(cpu_ctrl_dev, aml_name_decl("_CRS", crs)); + + /* declare CPU hotplug MMIO region with related access fields */ + aml_append(cpu_ctrl_dev, +- aml_operation_region("PRST", AML_SYSTEM_IO, aml_int(io_base), ++ aml_operation_region("PRST", rs, aml_int(base_addr), + ACPI_CPU_HOTPLUG_REG_LEN)); + + field = aml_field("PRST", AML_BYTE_ACC, AML_NOLOCK, +@@ -699,9 +706,11 @@ void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, + aml_append(sb_scope, cpus_dev); + aml_append(table, sb_scope); + +- method = aml_method(event_handler_method, 0, AML_NOTSERIALIZED); +- aml_append(method, aml_call0("\\_SB.CPUS." CPU_SCAN_METHOD)); +- aml_append(table, method); ++ if (event_handler_method) { ++ method = aml_method(event_handler_method, 0, AML_NOTSERIALIZED); ++ aml_append(method, aml_call0("\\_SB.CPUS." CPU_SCAN_METHOD)); ++ aml_append(table, method); ++ } + + g_free(cphp_res_path); + } +diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c +index 80db183b78..db4ca8a66a 100644 +--- a/hw/i386/acpi-build.c ++++ b/hw/i386/acpi-build.c +@@ -1546,7 +1546,8 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, + .fw_unplugs_cpu = pm->smi_on_cpu_unplug, + }; + build_cpus_aml(dsdt, machine, opts, pc_madt_cpu_entry, +- pm->cpu_hp_io_base, "\\_SB.PCI0", "\\_GPE._E02"); ++ pm->cpu_hp_io_base, "\\_SB.PCI0", "\\_GPE._E02", ++ AML_SYSTEM_IO); + } + + if (pcms->memhp_io_base && nr_mem) { +diff --git a/include/hw/acpi/cpu.h b/include/hw/acpi/cpu.h +index 209e1773f8..76bc7eb251 100644 +--- a/include/hw/acpi/cpu.h ++++ b/include/hw/acpi/cpu.h +@@ -60,9 +60,10 @@ typedef void (*build_madt_cpu_fn)(int uid, const CPUArchIdList *apic_ids, + GArray *entry, bool force_enabled); + + void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts, +- build_madt_cpu_fn build_madt_cpu, hwaddr io_base, ++ build_madt_cpu_fn build_madt_cpu, hwaddr base_addr, + const char *res_root, +- const char *event_handler_method); ++ const char *event_handler_method, ++ AmlRegionSpace rs); + + void acpi_cpu_ospm_status(CPUHotplugState *cpu_st, ACPIOSTInfoList ***list); + +-- +2.27.0 + diff --git a/hw-acpi-Update-GED-_EVT-method-AML-with-cpu-scan.patch b/hw-acpi-Update-GED-_EVT-method-AML-with-cpu-scan.patch new file mode 100644 index 0000000000000000000000000000000000000000..a2dceaf31fbf7503d09b1410428b17ffe03f6338 --- /dev/null +++ b/hw-acpi-Update-GED-_EVT-method-AML-with-cpu-scan.patch @@ -0,0 +1,53 @@ +From cfdb0f24431ae0f5115f905a1411509c01a50e88 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Tue, 9 Jun 2020 00:50:36 +0100 +Subject: [PATCH] hw/acpi: Update GED _EVT method AML with cpu scan + +OSPM evaluates _EVT method to map the event. The cpu hotplug event eventually +results in start of the cpu scan. Scan figures out the cpu and the kind of +event(plug/unplug) and notifies it back to the guest. + +The change in this patch updates the GED AML _EVT method with the call to +\\_SB.CPUS.CSCN which will do above. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/acpi/generic_event_device.c | 4 ++++ + include/hw/acpi/cpu_hotplug.h | 2 ++ + 2 files changed, 6 insertions(+) + +diff --git a/hw/acpi/generic_event_device.c b/hw/acpi/generic_event_device.c +index b84602b238..ad252e6a91 100644 +--- a/hw/acpi/generic_event_device.c ++++ b/hw/acpi/generic_event_device.c +@@ -108,6 +108,10 @@ void build_ged_aml(Aml *table, const char *name, HotplugHandler *hotplug_dev, + aml_append(if_ctx, aml_call0(MEMORY_DEVICES_CONTAINER "." + MEMORY_SLOT_SCAN_METHOD)); + break; ++ case ACPI_GED_CPU_HOTPLUG_EVT: ++ aml_append(if_ctx, aml_call0(ACPI_CPU_CONTAINER "." ++ ACPI_CPU_SCAN_METHOD)); ++ break; + case ACPI_GED_PWR_DOWN_EVT: + aml_append(if_ctx, + aml_notify(aml_name(ACPI_POWER_BUTTON_DEVICE), +diff --git a/include/hw/acpi/cpu_hotplug.h b/include/hw/acpi/cpu_hotplug.h +index 48b291e45e..ef631750b4 100644 +--- a/include/hw/acpi/cpu_hotplug.h ++++ b/include/hw/acpi/cpu_hotplug.h +@@ -20,6 +20,8 @@ + #include "hw/acpi/cpu.h" + + #define ACPI_CPU_HOTPLUG_REG_LEN 12 ++#define ACPI_CPU_SCAN_METHOD "CSCN" ++#define ACPI_CPU_CONTAINER "\\_SB.CPUS" + + typedef struct AcpiCpuHotplug { + Object *device; +-- +2.27.0 + diff --git a/hw-acpi-Use-qemu_present_cpu-API-in-ACPI-CPU-hotplug.patch b/hw-acpi-Use-qemu_present_cpu-API-in-ACPI-CPU-hotplug.patch new file mode 100644 index 0000000000000000000000000000000000000000..93772f029c4a74fcc623b2cd8138ac2063e63049 --- /dev/null +++ b/hw-acpi-Use-qemu_present_cpu-API-in-ACPI-CPU-hotplug.patch @@ -0,0 +1,37 @@ +From 576a2a88625978f1befde11f0823f32bbc54cad1 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Mon, 28 Aug 2023 20:00:08 +0000 +Subject: [PATCH] hw/acpi: Use qemu_present_cpu() API in ACPI CPU hotplug init + +ACPI CPU Hotplug code assumes a virtual CPU is unplugged if the CPUState object +is absent in the list of ths possible CPUs(CPUArchIdList *possible_cpus) +maintained on per-machine basis. Use the earlier introduced qemu_present_cpu() +API to check this state. + +This change should have no bearing on the functionality of any architecture and +is mere a representational change. + +Signed-off-by: Salil Mehta +--- + hw/acpi/cpu.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/hw/acpi/cpu.c b/hw/acpi/cpu.c +index 4b24a25003..cabeb4e86b 100644 +--- a/hw/acpi/cpu.c ++++ b/hw/acpi/cpu.c +@@ -226,7 +226,10 @@ void cpu_hotplug_hw_init(MemoryRegion *as, Object *owner, + state->dev_count = id_list->len; + state->devs = g_new0(typeof(*state->devs), state->dev_count); + for (i = 0; i < id_list->len; i++) { +- state->devs[i].cpu = CPU(id_list->cpus[i].cpu); ++ struct CPUState *cpu = CPU(id_list->cpus[i].cpu); ++ if (qemu_present_cpu(cpu)) { ++ state->devs[i].cpu = cpu; ++ } + state->devs[i].arch_id = id_list->cpus[i].arch_id; + } + memory_region_init_io(&state->ctrl_reg, owner, &cpu_hotplug_ops, state, +-- +2.27.0 + diff --git a/hw-arm-Changes-required-for-reset-and-to-support-nex.patch b/hw-arm-Changes-required-for-reset-and-to-support-nex.patch new file mode 100644 index 0000000000000000000000000000000000000000..66816ac14e0fd5e975d63b1786c03028908f8569 --- /dev/null +++ b/hw-arm-Changes-required-for-reset-and-to-support-nex.patch @@ -0,0 +1,111 @@ +From 3e5f043c493fa4765c5637bec66be2bd620bc53f Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sat, 9 May 2020 18:10:24 +0100 +Subject: [PATCH] hw/arm: Changes required for reset and to support next boot + +Updates the firmware config with the next boot cpus information and also +registers the reset callback to be called when guest reboots to reset the cpu. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/boot.c | 2 +- + hw/arm/virt.c | 18 +++++++++++++++--- + include/hw/arm/boot.h | 2 ++ + include/hw/arm/virt.h | 1 + + 4 files changed, 19 insertions(+), 4 deletions(-) + +diff --git a/hw/arm/boot.c b/hw/arm/boot.c +index d1671e1d42..345c7cfa19 100644 +--- a/hw/arm/boot.c ++++ b/hw/arm/boot.c +@@ -683,7 +683,7 @@ fail: + return -1; + } + +-static void do_cpu_reset(void *opaque) ++void do_cpu_reset(void *opaque) + { + ARMCPU *cpu = opaque; + CPUState *cs = CPU(cpu); +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 60cd560ab9..eedff8e525 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -46,6 +46,8 @@ + #include "sysemu/device_tree.h" + #include "sysemu/numa.h" + #include "sysemu/runstate.h" ++#include "sysemu/reset.h" ++#include "sysemu/sysemu.h" + #include "sysemu/tpm.h" + #include "sysemu/tcg.h" + #include "sysemu/kvm.h" +@@ -1453,7 +1455,7 @@ static FWCfgState *create_fw_cfg(const VirtMachineState *vms, AddressSpace *as) + char *nodename; + + fw_cfg = fw_cfg_init_mem_wide(base + 8, base, 8, base + 16, as); +- fw_cfg_add_i16(fw_cfg, FW_CFG_NB_CPUS, (uint16_t)ms->smp.cpus); ++ fw_cfg_add_i16(fw_cfg, FW_CFG_NB_CPUS, vms->boot_cpus); + + nodename = g_strdup_printf("/fw-cfg@%" PRIx64, base); + qemu_fdt_add_subnode(ms->fdt, nodename); +@@ -3276,7 +3278,13 @@ static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + if (local_err) { + goto fail; + } +- /* TODO: register cpu for reset & update F/W info for the next boot */ ++ /* register this cpu for reset & update F/W info for the next boot */ ++ qemu_register_reset(do_cpu_reset, ARM_CPU(cs)); ++ } ++ ++ vms->boot_cpus++; ++ if (vms->fw_cfg) { ++ fw_cfg_modify_i16(vms->fw_cfg, FW_CFG_NB_CPUS, vms->boot_cpus); + } + + cs->disabled = false; +@@ -3351,7 +3359,11 @@ static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, + unwire_gic_cpu_irqs(vms, cs); + virt_update_gic(vms, cs); + +- /* TODO: unregister cpu for reset & update F/W info for the next boot */ ++ qemu_unregister_reset(do_cpu_reset, ARM_CPU(cs)); ++ vms->boot_cpus--; ++ if (vms->fw_cfg) { ++ fw_cfg_modify_i16(vms->fw_cfg, FW_CFG_NB_CPUS, vms->boot_cpus); ++ } + + qobject_unref(dev->opts); + dev->opts = NULL; +diff --git a/include/hw/arm/boot.h b/include/hw/arm/boot.h +index 80c492d742..f81326a1dc 100644 +--- a/include/hw/arm/boot.h ++++ b/include/hw/arm/boot.h +@@ -178,6 +178,8 @@ AddressSpace *arm_boot_address_space(ARMCPU *cpu, + int arm_load_dtb(hwaddr addr, const struct arm_boot_info *binfo, + hwaddr addr_limit, AddressSpace *as, MachineState *ms); + ++void do_cpu_reset(void *opaque); ++ + /* Write a secure board setup routine with a dummy handler for SMCs */ + void arm_write_secure_board_setup_dummy_smc(ARMCPU *cpu, + const struct arm_boot_info *info, +diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h +index 069c9f2a09..ae0f5beb26 100644 +--- a/include/hw/arm/virt.h ++++ b/include/hw/arm/virt.h +@@ -167,6 +167,7 @@ struct VirtMachineState { + MemMapEntry *memmap; + char *pciehb_nodename; + const int *irqmap; ++ uint16_t boot_cpus; + int fdt_size; + uint32_t clock_phandle; + uint32_t gic_phandle; +-- +2.27.0 + diff --git a/hw-arm-MADT-Tbl-change-to-size-the-guest-with-possib.patch b/hw-arm-MADT-Tbl-change-to-size-the-guest-with-possib.patch new file mode 100644 index 0000000000000000000000000000000000000000..75abc4e58ce1f30527638b0c62f94c50b033911b --- /dev/null +++ b/hw-arm-MADT-Tbl-change-to-size-the-guest-with-possib.patch @@ -0,0 +1,98 @@ +From 8e1b8d624128523654786953b381557c82654a57 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Wed, 6 May 2020 18:03:11 +0100 +Subject: [PATCH] hw/arm: MADT Tbl change to size the guest with possible vCPUs + +Changes required during building of MADT Table by QEMU to accomodate disabled +possible vCPUs. This info shall be used by the guest kernel to size up its +resources during boot time. This pre-sizing of the guest kernel done on +possible vCPUs will facilitate hotplug of the disabled vCPUs. + +This change also caters ACPI MADT GIC CPU Interface flag related changes +recently introduced in the UEFI ACPI 6.5 Specification which allows deferred +virtual CPU online'ing in the Guest Kernel. + +Link: https://uefi.org/specs/ACPI/6.5/05_ACPI_Software_Programming_Model.html#gic-cpu-interface-gicc-structure + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/virt-acpi-build.c | 36 ++++++++++++++++++++++++++++++------ + 1 file changed, 30 insertions(+), 6 deletions(-) + +diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c +index d88f3cded1..2870c1ec5a 100644 +--- a/hw/arm/virt-acpi-build.c ++++ b/hw/arm/virt-acpi-build.c +@@ -779,6 +779,29 @@ static void build_append_gicr(GArray *table_data, uint64_t base, uint32_t size) + build_append_int_noprefix(table_data, size, 4); /* Discovery Range Length */ + } + ++static uint32_t virt_acpi_get_gicc_flags(CPUState *cpu) ++{ ++ MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine()); ++ ++ /* can only exist in 'enabled' state */ ++ if (!mc->has_hotpluggable_cpus) { ++ return 1; ++ } ++ ++ /* ++ * ARM GIC CPU Interface can be 'online-capable' or 'enabled' at boot ++ * We MUST set 'online-capable' Bit for all hotpluggable CPUs except the ++ * first/boot CPU. Cold-booted CPUs without 'Id' can also be unplugged. ++ * Though as-of-now this is only used as a debugging feature. ++ * ++ * UEFI ACPI Specification 6.5 ++ * Section: 5.2.12.14. GIC CPU Interface (GICC) Structure ++ * Table: 5.37 GICC CPU Interface Flags ++ * Link: https://uefi.org/specs/ACPI/6.5 ++ */ ++ return cpu && !cpu->cpu_index ? 1 : (1 << 3); ++} ++ + static void + build_madt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) + { +@@ -805,12 +828,13 @@ build_madt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) + build_append_int_noprefix(table_data, vms->gic_version, 1); + build_append_int_noprefix(table_data, 0, 3); /* Reserved */ + +- for (i = 0; i < MACHINE(vms)->smp.cpus; i++) { +- ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(i)); ++ for (i = 0; i < MACHINE(vms)->smp.max_cpus; i++) { ++ CPUState *cpu = qemu_get_possible_cpu(i); + uint64_t physical_base_address = 0, gich = 0, gicv = 0; + uint32_t vgic_interrupt = vms->virt ? ARCH_GIC_MAINT_IRQ : 0; +- uint32_t pmu_interrupt = arm_feature(&armcpu->env, ARM_FEATURE_PMU) ? +- VIRTUAL_PMU_IRQ : 0; ++ uint32_t pmu_interrupt = vms->pmu ? VIRTUAL_PMU_IRQ : 0; ++ uint32_t flags = virt_acpi_get_gicc_flags(cpu); ++ uint64_t mpidr = qemu_get_cpu_archid(i); + + if (vms->gic_version == VIRT_GIC_VERSION_2) { + physical_base_address = memmap[VIRT_GIC_CPU].base; +@@ -825,7 +849,7 @@ build_madt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) + build_append_int_noprefix(table_data, i, 4); /* GIC ID */ + build_append_int_noprefix(table_data, i, 4); /* ACPI Processor UID */ + /* Flags */ +- build_append_int_noprefix(table_data, 1, 4); /* Enabled */ ++ build_append_int_noprefix(table_data, flags, 4); + /* Parking Protocol Version */ + build_append_int_noprefix(table_data, 0, 4); + /* Performance Interrupt GSIV */ +@@ -839,7 +863,7 @@ build_madt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) + build_append_int_noprefix(table_data, vgic_interrupt, 4); + build_append_int_noprefix(table_data, 0, 8); /* GICR Base Address*/ + /* MPIDR */ +- build_append_int_noprefix(table_data, armcpu->mp_affinity, 8); ++ build_append_int_noprefix(table_data, mpidr, 8); + /* Processor Power Efficiency Class */ + build_append_int_noprefix(table_data, 0, 1); + /* Reserved */ +-- +2.27.0 + diff --git a/hw-arm-Support-hotplug-capability-check-using-_OSC-m.patch b/hw-arm-Support-hotplug-capability-check-using-_OSC-m.patch new file mode 100644 index 0000000000000000000000000000000000000000..87e236b6f83accff9b839e2c363e47949feb1f9b --- /dev/null +++ b/hw-arm-Support-hotplug-capability-check-using-_OSC-m.patch @@ -0,0 +1,128 @@ +From c5dfec0bfd78f7e8f84a527a1aa73896f69b2367 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Thu, 10 Aug 2023 01:15:31 +0000 +Subject: [PATCH] hw/arm: Support hotplug capability check using _OSC method + +Physical CPU hotplug results in (un)setting of ACPI _STA.Present bit. AARCH64 +platforms do not support physical CPU hotplug. Virtual CPU hotplug support being +implemented toggles ACPI _STA.Enabled Bit to achieve hotplug functionality. This +is not same as physical CPU hotplug support. + +In future, if ARM architecture supports physical CPU hotplug then the current +design of virtual CPU hotplug can be used unchanged. Hence, there is a need for +firmware/VMM/Qemu to support evaluation of platform wide capabilitiy related to +the *type* of CPU hotplug support present on the platform. OSPM might need this +during boot time to correctly initialize the CPUs and other related components +in the kernel. + +NOTE: This implementation will be improved to add the support of *query* in the +subsequent versions. This is very minimal support to assist kernel. + +ASL for the implemented _OSC method: + +Method (_OSC, 4, NotSerialized) // _OSC: Operating System Capabilities +{ + CreateDWordField (Arg3, Zero, CDW1) + If ((Arg0 == ToUUID ("0811b06e-4a27-44f9-8d60-3cbbc22e7b48") /* Platform-wide Capabilities */)) + { + CreateDWordField (Arg3, 0x04, CDW2) + Local0 = CDW2 /* \_SB_._OSC.CDW2 */ + If ((Arg1 != One)) + { + CDW1 |= 0x08 + } + + Local0 &= 0x00800000 + If ((CDW2 != Local0)) + { + CDW1 |= 0x10 + } + + CDW2 = Local0 + } + Else + { + CDW1 |= 0x04 + } + + Return (Arg3) +} + +Signed-off-by: Salil Mehta +--- + hw/arm/virt-acpi-build.c | 52 ++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 52 insertions(+) + +diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c +index 2870c1ec5a..c402e102c4 100644 +--- a/hw/arm/virt-acpi-build.c ++++ b/hw/arm/virt-acpi-build.c +@@ -940,6 +940,55 @@ static void build_fadt_rev6(GArray *table_data, BIOSLinker *linker, + build_fadt(table_data, linker, &fadt, vms->oem_id, vms->oem_table_id); + } + ++static void build_virt_osc_method(Aml *scope, VirtMachineState *vms) ++{ ++ Aml *if_uuid, *else_uuid, *if_rev, *if_caps_masked, *method; ++ Aml *a_cdw1 = aml_name("CDW1"); ++ Aml *a_cdw2 = aml_local(0); ++ ++ method = aml_method("_OSC", 4, AML_NOTSERIALIZED); ++ aml_append(method, aml_create_dword_field(aml_arg(3), aml_int(0), "CDW1")); ++ ++ /* match UUID */ ++ if_uuid = aml_if(aml_equal( ++ aml_arg(0), aml_touuid("0811B06E-4A27-44F9-8D60-3CBBC22E7B48"))); ++ ++ aml_append(if_uuid, aml_create_dword_field(aml_arg(3), aml_int(4), "CDW2")); ++ aml_append(if_uuid, aml_store(aml_name("CDW2"), a_cdw2)); ++ ++ /* check unknown revision in arg(1) */ ++ if_rev = aml_if(aml_lnot(aml_equal(aml_arg(1), aml_int(1)))); ++ /* set revision error bits, DWORD1 Bit[3] */ ++ aml_append(if_rev, aml_or(a_cdw1, aml_int(0x08), a_cdw1)); ++ aml_append(if_uuid, if_rev); ++ ++ /* ++ * check support for vCPU hotplug type(=enabled) platform-wide capability ++ * in DWORD2 as sepcified in the below ACPI Specification ECR, ++ * # https://bugzilla.tianocore.org/show_bug.cgi?id=4481 ++ */ ++ if (vms->acpi_dev) { ++ aml_append(if_uuid, aml_and(a_cdw2, aml_int(0x800000), a_cdw2)); ++ /* check if OSPM specified hotplug capability bits were masked */ ++ if_caps_masked = aml_if(aml_lnot(aml_equal(aml_name("CDW2"), a_cdw2))); ++ aml_append(if_caps_masked, aml_or(a_cdw1, aml_int(0x10), a_cdw1)); ++ aml_append(if_uuid, if_caps_masked); ++ } ++ aml_append(if_uuid, aml_store(a_cdw2, aml_name("CDW2"))); ++ ++ aml_append(method, if_uuid); ++ else_uuid = aml_else(); ++ ++ /* set unrecognized UUID error bits, DWORD1 Bit[2] */ ++ aml_append(else_uuid, aml_or(a_cdw1, aml_int(4), a_cdw1)); ++ aml_append(method, else_uuid); ++ ++ aml_append(method, aml_return(aml_arg(3))); ++ aml_append(scope, method); ++ ++ return; ++} ++ + /* DSDT */ + static void + build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) +@@ -974,6 +1023,9 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) + } else { + acpi_dsdt_add_cpus(scope, vms); + } ++ ++ build_virt_osc_method(scope, vms); ++ + acpi_dsdt_add_uart(scope, &memmap[VIRT_UART], + (irqmap[VIRT_UART] + ARM_SPI_BASE)); + if (vmc->acpi_expose_flash) { +-- +2.27.0 + diff --git a/hw-arm-gicv3-Changes-to-update-GIC-with-vCPU-hot-plu.patch b/hw-arm-gicv3-Changes-to-update-GIC-with-vCPU-hot-plu.patch new file mode 100644 index 0000000000000000000000000000000000000000..f8097e1973b31092363f156c4ee2df9372884906 --- /dev/null +++ b/hw-arm-gicv3-Changes-to-update-GIC-with-vCPU-hot-plu.patch @@ -0,0 +1,267 @@ +From 8ad397f33f8b7d82c0ef72608ef8dc3e0ecba1c2 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sat, 9 May 2020 14:38:38 +0100 +Subject: [PATCH] hw/arm,gicv3: Changes to update GIC with vCPU hot-plug + notification + +vCPU hot-(un)plug events MUST be notified to the GIC. Introduce a notfication +mechanism to update any such events to GIC so that it can update its vCPU to GIC +CPU interface association. + +This is required to implement a workaround to the limitations posed by the ARM +architecture. For details about the constraints and workarounds please check +below slides: + +Link: https://kvm-forum.qemu.org/2023/talk/9SMPDQ/ + +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 27 +++++++++++++-- + hw/intc/arm_gicv3_common.c | 54 +++++++++++++++++++++++++++++- + hw/intc/arm_gicv3_cpuif_common.c | 5 +++ + hw/intc/gicv3_internal.h | 1 + + include/hw/arm/virt.h | 1 + + include/hw/intc/arm_gicv3_common.h | 22 ++++++++++++ + 6 files changed, 107 insertions(+), 3 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 97bf4cca11..0312fa366d 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -750,6 +750,16 @@ static inline DeviceState *create_acpi_ged(VirtMachineState *vms) + return dev; + } + ++static void virt_add_gic_cpuhp_notifier(VirtMachineState *vms) ++{ ++ MachineClass *mc = MACHINE_GET_CLASS(vms); ++ ++ if (mc->has_hotpluggable_cpus) { ++ Notifier *cpuhp_notifier = gicv3_cpuhp_notifier(vms->gic); ++ notifier_list_add(&vms->cpuhp_notifiers, cpuhp_notifier); ++ } ++} ++ + static void create_its(VirtMachineState *vms) + { + const char *itsclass = its_class_name(); +@@ -997,6 +1007,9 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) + } else if (vms->gic_version == VIRT_GIC_VERSION_2) { + create_v2m(vms); + } ++ ++ /* add GIC CPU hot(un)plug update notifier */ ++ virt_add_gic_cpuhp_notifier(vms); + } + + static void create_uart(const VirtMachineState *vms, int uart, +@@ -2481,6 +2494,8 @@ static void machvirt_init(MachineState *machine) + create_fdt(vms); + qemu_log("cpu init start\n"); + ++ notifier_list_init(&vms->cpuhp_notifiers); ++ possible_cpus = mc->possible_cpu_arch_ids(machine); + assert(possible_cpus->len == max_cpus); + for (n = 0; n < possible_cpus->len; n++) { + Object *cpuobj; +@@ -3133,6 +3148,14 @@ static void virt_memory_plug(HotplugHandler *hotplug_dev, + dev, &error_abort); + } + ++static void virt_update_gic(VirtMachineState *vms, CPUState *cs) ++{ ++ GICv3CPUHotplugInfo gic_info = { .gic = vms->gic, .cpu = cs }; ++ ++ /* notify gic to stitch GICC to this new cpu */ ++ notifier_list_notify(&vms->cpuhp_notifiers, &gic_info); ++} ++ + static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + Error **errp) + { +@@ -3215,7 +3238,7 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + * vCPUs have their GIC state initialized during machvit_init(). + */ + if (vms->acpi_dev) { +- /* TODO: update GIC about this hotplug change here */ ++ virt_update_gic(vms, cs); + wire_gic_cpu_irqs(vms, cs); + } + +@@ -3301,7 +3324,7 @@ static void virt_cpu_unplug(HotplugHandler *hotplug_dev, DeviceState *dev, + /* TODO: update the acpi cpu hotplug state for cpu hot-unplug */ + + unwire_gic_cpu_irqs(vms, cs); +- /* TODO: update the GIC about this hot unplug change */ ++ virt_update_gic(vms, cs); + + /* TODO: unregister cpu for reset & update F/W info for the next boot */ + +diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c +index ebd99af610..fc87fa9369 100644 +--- a/hw/intc/arm_gicv3_common.c ++++ b/hw/intc/arm_gicv3_common.c +@@ -33,7 +33,6 @@ + #include "hw/arm/linux-boot-if.h" + #include "sysemu/kvm.h" + +- + static void gicv3_gicd_no_migration_shift_bug_post_load(GICv3State *cs) + { + if (cs->gicd_no_migration_shift_bug) { +@@ -322,6 +321,56 @@ void gicv3_init_irqs_and_mmio(GICv3State *s, qemu_irq_handler handler, + } + } + ++static int arm_gicv3_get_proc_num(GICv3State *s, CPUState *cpu) ++{ ++ uint64_t mp_affinity; ++ uint64_t gicr_typer; ++ uint64_t cpu_affid; ++ int i; ++ ++ mp_affinity = object_property_get_uint(OBJECT(cpu), "mp-affinity", NULL); ++ /* match the cpu mp-affinity to get the gic cpuif number */ ++ for (i = 0; i < s->num_cpu; i++) { ++ gicr_typer = s->cpu[i].gicr_typer; ++ cpu_affid = (gicr_typer >> 32) & 0xFFFFFF; ++ if (cpu_affid == mp_affinity) { ++ return i; ++ } ++ } ++ ++ return -1; ++} ++ ++static void arm_gicv3_cpu_update_notifier(Notifier *notifier, void * data) ++{ ++ GICv3CPUHotplugInfo *gic_info = (GICv3CPUHotplugInfo *)data; ++ CPUState *cpu = gic_info->cpu; ++ int gic_cpuif_num; ++ GICv3State *s; ++ ++ s = ARM_GICV3_COMMON(gic_info->gic); ++ ++ /* this shall get us mapped gicv3 cpuif corresponding to mpidr */ ++ gic_cpuif_num = arm_gicv3_get_proc_num(s, cpu); ++ if (gic_cpuif_num < 0) { ++ error_report("Failed to associate cpu %d with any GIC cpuif", ++ cpu->cpu_index); ++ abort(); ++ } ++ ++ /* check if update is for vcpu hot-unplug */ ++ if (qemu_enabled_cpu(cpu)) { ++ s->cpu[gic_cpuif_num].cpu = NULL; ++ return; ++ } ++ ++ /* re-stitch the gic cpuif to this new cpu */ ++ gicv3_set_gicv3state(cpu, &s->cpu[gic_cpuif_num]); ++ gicv3_set_cpustate(&s->cpu[gic_cpuif_num], cpu); ++ ++ /* TODO: initialize the registers info for this newly added cpu */ ++} ++ + static void arm_gicv3_common_realize(DeviceState *dev, Error **errp) + { + GICv3State *s = ARM_GICV3_COMMON(dev); +@@ -444,6 +493,8 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp) + s->cpu[cpuidx - 1].gicr_typer |= GICR_TYPER_LAST; + } + ++ s->cpu_update_notifier.notify = arm_gicv3_cpu_update_notifier; ++ + s->itslist = g_ptr_array_new(); + } + +@@ -451,6 +502,7 @@ static void arm_gicv3_finalize(Object *obj) + { + GICv3State *s = ARM_GICV3_COMMON(obj); + ++ notifier_remove(&s->cpu_update_notifier); + g_free(s->redist_region_count); + } + +diff --git a/hw/intc/arm_gicv3_cpuif_common.c b/hw/intc/arm_gicv3_cpuif_common.c +index ff1239f65d..381cf2754b 100644 +--- a/hw/intc/arm_gicv3_cpuif_common.c ++++ b/hw/intc/arm_gicv3_cpuif_common.c +@@ -20,3 +20,8 @@ void gicv3_set_gicv3state(CPUState *cpu, GICv3CPUState *s) + + env->gicv3state = (void *)s; + }; ++ ++void gicv3_set_cpustate(GICv3CPUState *s, CPUState *cpu) ++{ ++ s->cpu = cpu; ++} +diff --git a/hw/intc/gicv3_internal.h b/hw/intc/gicv3_internal.h +index 29d5cdc1b6..9d4c1209bd 100644 +--- a/hw/intc/gicv3_internal.h ++++ b/hw/intc/gicv3_internal.h +@@ -848,5 +848,6 @@ static inline void gicv3_cache_all_target_cpustates(GICv3State *s) + } + + void gicv3_set_gicv3state(CPUState *cpu, GICv3CPUState *s); ++void gicv3_set_cpustate(GICv3CPUState *s, CPUState *cpu); + + #endif /* QEMU_ARM_GICV3_INTERNAL_H */ +diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h +index 5de0185063..069c9f2a09 100644 +--- a/include/hw/arm/virt.h ++++ b/include/hw/arm/virt.h +@@ -180,6 +180,7 @@ struct VirtMachineState { + PCIBus *bus; + char *oem_id; + char *oem_table_id; ++ NotifierList cpuhp_notifiers; + }; + + #define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM : VIRT_PCIE_ECAM) +diff --git a/include/hw/intc/arm_gicv3_common.h b/include/hw/intc/arm_gicv3_common.h +index 4e2fb518e7..97a48f44b9 100644 +--- a/include/hw/intc/arm_gicv3_common.h ++++ b/include/hw/intc/arm_gicv3_common.h +@@ -280,6 +280,7 @@ struct GICv3State { + GICv3CPUState *gicd_irouter_target[GICV3_MAXIRQ]; + uint32_t gicd_nsacr[DIV_ROUND_UP(GICV3_MAXIRQ, 16)]; + ++ Notifier cpu_update_notifier; + GICv3CPUState *cpu; + /* List of all ITSes connected to this GIC */ + GPtrArray *itslist; +@@ -328,6 +329,27 @@ struct ARMGICv3CommonClass { + + void gicv3_init_irqs_and_mmio(GICv3State *s, qemu_irq_handler handler, + const MemoryRegionOps *ops); ++/** ++ * Structure used by GICv3 CPU hotplug notifier ++ */ ++typedef struct GICv3CPUHotplugInfo { ++ DeviceState *gic; /* GICv3State */ ++ CPUState *cpu; ++} GICv3CPUHotplugInfo; ++ ++/** ++ * gicv3_cpuhp_notifier ++ * ++ * Returns CPU hotplug notifier which could be used to update GIC about any ++ * CPU hot(un)plug events. ++ * ++ * Returns: Notifier initialized with CPU Hot(un)plug update function ++ */ ++static inline Notifier *gicv3_cpuhp_notifier(DeviceState *dev) ++{ ++ GICv3State *s = ARM_GICV3_COMMON(dev); ++ return &s->cpu_update_notifier; ++} + + /** + * gicv3_class_name +-- +2.27.0 + diff --git a/hw-arm-virt-Expose-cold-booted-CPUs-as-MADT-GICC-Ena.patch b/hw-arm-virt-Expose-cold-booted-CPUs-as-MADT-GICC-Ena.patch new file mode 100644 index 0000000000000000000000000000000000000000..1c124e92aa8a31f4573ac5b9098d39c925ec6fa3 --- /dev/null +++ b/hw-arm-virt-Expose-cold-booted-CPUs-as-MADT-GICC-Ena.patch @@ -0,0 +1,107 @@ +From 837b04877be49b930a2d437f55e2ae15ff820421 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sat, 23 Sep 2023 22:31:49 +0000 +Subject: [PATCH] hw/arm/virt: Expose cold-booted CPUs as MADT GICC Enabled + +Hotpluggable CPUs MUST be exposed as 'online-capable' as per the new change. But +cold booted CPUs if made 'online-capable' during boot time might not get +detected in the legacy OS. Hence, can cause compatibility problems. + +Original Change Link: https://bugzilla.tianocore.org/show_bug.cgi?id=3706 + +Specification change might take time and hence disabling the support of +unplugging any cold booted CPUs to preserve the compatibility with legacy OS. + +Signed-off-by: Salil Mehta +--- + hw/arm/virt-acpi-build.c | 19 ++++++++++++++----- + hw/arm/virt.c | 16 ++++++++++++++++ + include/hw/core/cpu.h | 2 ++ + 3 files changed, 32 insertions(+), 5 deletions(-) + +diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c +index c402e102c4..590afcfa98 100644 +--- a/hw/arm/virt-acpi-build.c ++++ b/hw/arm/virt-acpi-build.c +@@ -789,17 +789,26 @@ static uint32_t virt_acpi_get_gicc_flags(CPUState *cpu) + } + + /* +- * ARM GIC CPU Interface can be 'online-capable' or 'enabled' at boot +- * We MUST set 'online-capable' Bit for all hotpluggable CPUs except the +- * first/boot CPU. Cold-booted CPUs without 'Id' can also be unplugged. +- * Though as-of-now this is only used as a debugging feature. ++ * ARM GIC CPU Interface can be 'online-capable' or 'enabled' at boot. We ++ * MUST set 'online-capable' bit for all hotpluggable CPUs. ++ * Change Link: https://bugzilla.tianocore.org/show_bug.cgi?id=3706 + * + * UEFI ACPI Specification 6.5 + * Section: 5.2.12.14. GIC CPU Interface (GICC) Structure + * Table: 5.37 GICC CPU Interface Flags + * Link: https://uefi.org/specs/ACPI/6.5 ++ * ++ * Cold-booted CPUs, except for the first/boot CPU, SHOULD be allowed to be ++ * hot(un)plug as well but for this to happen these MUST have ++ * 'online-capable' bit set. Later creates compatibility problem with legacy ++ * OS as it might ignore online-capable' bits during boot time and hence ++ * some CPUs might not get detected. To fix this MADT GIC CPU interface flag ++ * should be allowed to have both bits set i.e. 'online-capable' and ++ * 'Enabled' bits together. This change will require UEFI ACPI standard ++ * change. Till this happens exposing all cold-booted CPUs as 'enabled' only ++ * + */ +- return cpu && !cpu->cpu_index ? 1 : (1 << 3); ++ return cpu && cpu->cold_booted ? 1 : (1 << 3); + } + + static void +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index eedff8e525..ed437ce0e8 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -3250,6 +3250,10 @@ static void virt_cpu_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev, + * This shall be used during the init of ACPI Hotplug state and hot-unplug + */ + cs->acpi_persistent = true; ++ ++ if (!dev->hotplugged) { ++ cs->cold_booted = true; ++ } + } + + static void virt_cpu_plug(HotplugHandler *hotplug_dev, DeviceState *dev, +@@ -3313,6 +3317,18 @@ static void virt_cpu_unplug_request(HotplugHandler *hotplug_dev, + return; + } + ++ /* ++ * UEFI ACPI standard change is required to make both 'enabled' and the ++ * 'online-capable' bit co-exist instead of being mutually exclusive. ++ * check virt_acpi_get_gicc_flags() for more details. ++ * ++ * Disable the unplugging of cold-booted vCPUs as a temporary mitigation. ++ */ ++ if (cs->cold_booted) { ++ error_setg(errp, "Hot-unplug of cold-booted CPU not supported!"); ++ return; ++ } ++ + if (cs->cpu_index == first_cpu->cpu_index) { + error_setg(errp, "Boot CPU(id%d=%d:%d:%d:%d) hot-unplug not supported", + first_cpu->cpu_index, cpu->socket_id, cpu->cluster_id, +diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h +index 6dbe163548..ee04ee44c2 100644 +--- a/include/hw/core/cpu.h ++++ b/include/hw/core/cpu.h +@@ -565,6 +565,8 @@ struct CPUState { + uint32_t halted; + int32_t exception_index; + ++ bool cold_booted; ++ + AccelCPUState *accel; + /* shared by kvm and hvf */ + bool vcpu_dirty; +-- +2.27.0 + diff --git a/hw-arm-virt-Move-setting-of-common-CPU-properties-in.patch b/hw-arm-virt-Move-setting-of-common-CPU-properties-in.patch new file mode 100644 index 0000000000000000000000000000000000000000..ec3b72f02c9fe269e68623de66654906734e83f4 --- /dev/null +++ b/hw-arm-virt-Move-setting-of-common-CPU-properties-in.patch @@ -0,0 +1,311 @@ +From 8daa90ad502b79e232377f831f67df456a743304 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sat, 26 Aug 2023 01:29:37 +0000 +Subject: [PATCH] hw/arm/virt: Move setting of common CPU properties in a + function + +Factor out CPU properties code common for {hot,cold}-plugged CPUs. This allows +code reuse. + +Signed-off-by: Salil Mehta +--- + hw/arm/virt.c | 220 ++++++++++++++++++++++++++---------------- + include/hw/arm/virt.h | 4 + + 2 files changed, 140 insertions(+), 84 deletions(-) + +diff --git a/hw/arm/virt.c b/hw/arm/virt.c +index 94481d45d4..8f647422d8 100644 +--- a/hw/arm/virt.c ++++ b/hw/arm/virt.c +@@ -2113,16 +2113,130 @@ static void virt_cpu_post_init(VirtMachineState *vms, MemoryRegion *sysmem) + } + } + ++static void virt_cpu_set_properties(Object *cpuobj, const CPUArchId *cpu_slot, ++ Error **errp) ++{ ++ MachineState *ms = MACHINE(qdev_get_machine()); ++ VirtMachineState *vms = VIRT_MACHINE(ms); ++ Error *local_err = NULL; ++ VirtMachineClass *vmc; ++ ++ vmc = VIRT_MACHINE_GET_CLASS(ms); ++ ++ /* now, set the cpu object property values */ ++ numa_cpu_pre_plug(cpu_slot, DEVICE(cpuobj), &local_err); ++ if (local_err) { ++ goto out; ++ } ++ ++ object_property_set_int(cpuobj, "mp-affinity", cpu_slot->arch_id, NULL); ++ ++ if (!vms->secure) { ++ object_property_set_bool(cpuobj, "has_el3", false, NULL); ++ } ++ ++ if (!vms->virt && object_property_find(cpuobj, "has_el2")) { ++ object_property_set_bool(cpuobj, "has_el2", false, NULL); ++ } ++ ++ if (vmc->kvm_no_adjvtime && ++ object_property_find(cpuobj, "kvm-no-adjvtime")) { ++ object_property_set_bool(cpuobj, "kvm-no-adjvtime", true, NULL); ++ } ++ ++ if (vmc->no_kvm_steal_time && ++ object_property_find(cpuobj, "kvm-steal-time")) { ++ object_property_set_bool(cpuobj, "kvm-steal-time", false, NULL); ++ } ++ ++ if (vmc->no_pmu && object_property_find(cpuobj, "pmu")) { ++ object_property_set_bool(cpuobj, "pmu", false, NULL); ++ } ++ ++ if (vmc->no_tcg_lpa2 && object_property_find(cpuobj, "lpa2")) { ++ object_property_set_bool(cpuobj, "lpa2", false, NULL); ++ } ++ ++ if (object_property_find(cpuobj, "reset-cbar")) { ++ object_property_set_int(cpuobj, "reset-cbar", ++ vms->memmap[VIRT_CPUPERIPHS].base, ++ &local_err); ++ if (local_err) { ++ goto out; ++ } ++ } ++ ++ /* link already initialized {secure,tag}-memory regions to this cpu */ ++ object_property_set_link(cpuobj, "memory", OBJECT(vms->sysmem), &local_err); ++ if (local_err) { ++ goto out; ++ } ++ ++ if (vms->secure) { ++ object_property_set_link(cpuobj, "secure-memory", ++ OBJECT(vms->secure_sysmem), &local_err); ++ if (local_err) { ++ goto out; ++ } ++ } ++ ++ if (vms->mte) { ++ if (!object_property_find(cpuobj, "tag-memory")) { ++ error_setg(&local_err, "MTE requested, but not supported " ++ "by the guest CPU"); ++ if (local_err) { ++ goto out; ++ } ++ } ++ ++ object_property_set_link(cpuobj, "tag-memory", OBJECT(vms->tag_sysmem), ++ &local_err); ++ if (local_err) { ++ goto out; ++ } ++ ++ if (vms->secure) { ++ object_property_set_link(cpuobj, "secure-tag-memory", ++ OBJECT(vms->secure_tag_sysmem), ++ &local_err); ++ if (local_err) { ++ goto out; ++ } ++ } ++ } ++ ++ /* ++ * RFC: Question: this must only be called for the hotplugged cpus. For the ++ * cold booted secondary cpus this is being taken care in arm_load_kernel() ++ * in boot.c. Perhaps we should remove that code now? ++ */ ++ if (vms->psci_conduit != QEMU_PSCI_CONDUIT_DISABLED) { ++ object_property_set_int(cpuobj, "psci-conduit", vms->psci_conduit, ++ NULL); ++ ++ /* Secondary CPUs start in PSCI powered-down state */ ++ if (CPU(cpuobj)->cpu_index > 0) { ++ object_property_set_bool(cpuobj, "start-powered-off", true, NULL); ++ } ++ } ++ ++out: ++ if (local_err) { ++ error_propagate(errp, local_err); ++ } ++ return; ++} ++ + static void machvirt_init(MachineState *machine) + { + VirtMachineState *vms = VIRT_MACHINE(machine); + VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(machine); + MachineClass *mc = MACHINE_GET_CLASS(machine); + const CPUArchIdList *possible_cpus; +- MemoryRegion *sysmem = get_system_memory(); ++ MemoryRegion *secure_tag_sysmem = NULL; + MemoryRegion *secure_sysmem = NULL; + MemoryRegion *tag_sysmem = NULL; +- MemoryRegion *secure_tag_sysmem = NULL; ++ MemoryRegion *sysmem; + int n, virt_max_cpus; + bool firmware_loaded; + bool aarch64 = true; +@@ -2166,6 +2280,8 @@ static void machvirt_init(MachineState *machine) + */ + finalize_gic_version(vms); + ++ sysmem = vms->sysmem = get_system_memory(); ++ + if (vms->secure) { + /* + * The Secure view of the world is the same as the NonSecure, +@@ -2173,7 +2289,7 @@ static void machvirt_init(MachineState *machine) + * containing the system memory at low priority; any secure-only + * devices go in at higher priority and take precedence. + */ +- secure_sysmem = g_new(MemoryRegion, 1); ++ secure_sysmem = vms->secure_sysmem = g_new(MemoryRegion, 1); + memory_region_init(secure_sysmem, OBJECT(machine), "secure-memory", + UINT64_MAX); + memory_region_add_subregion_overlap(secure_sysmem, 0, sysmem, -1); +@@ -2246,6 +2362,23 @@ static void machvirt_init(MachineState *machine) + exit(1); + } + ++ if (vms->mte) { ++ /* Create the memory region only once, but link to all cpus later */ ++ tag_sysmem = vms->tag_sysmem = g_new(MemoryRegion, 1); ++ memory_region_init(tag_sysmem, OBJECT(machine), ++ "tag-memory", UINT64_MAX / 32); ++ ++ if (vms->secure) { ++ secure_tag_sysmem = vms->secure_tag_sysmem = g_new(MemoryRegion, 1); ++ memory_region_init(secure_tag_sysmem, OBJECT(machine), ++ "secure-tag-memory", UINT64_MAX / 32); ++ ++ /* As with ram, secure-tag takes precedence over tag. */ ++ memory_region_add_subregion_overlap(secure_tag_sysmem, 0, ++ tag_sysmem, -1); ++ } ++ } ++ + create_fdt(vms); + qemu_log("cpu init start\n"); + +@@ -2259,15 +2392,10 @@ static void machvirt_init(MachineState *machine) + } + + cpuobj = object_new(possible_cpus->cpus[n].type); +- object_property_set_int(cpuobj, "mp-affinity", +- possible_cpus->cpus[n].arch_id, NULL); + + cs = CPU(cpuobj); + cs->cpu_index = n; + +- numa_cpu_pre_plug(&possible_cpus->cpus[cs->cpu_index], DEVICE(cpuobj), +- &error_fatal); +- + aarch64 &= object_property_get_bool(cpuobj, "aarch64", NULL); + object_property_set_int(cpuobj, "socket-id", + virt_get_socket_id(machine, n), NULL); +@@ -2278,82 +2406,6 @@ static void machvirt_init(MachineState *machine) + object_property_set_int(cpuobj, "thread-id", + virt_get_thread_id(machine, n), NULL); + +- if (!vms->secure) { +- object_property_set_bool(cpuobj, "has_el3", false, NULL); +- } +- +- if (!vms->virt && object_property_find(cpuobj, "has_el2")) { +- object_property_set_bool(cpuobj, "has_el2", false, NULL); +- } +- +- if (vmc->kvm_no_adjvtime && +- object_property_find(cpuobj, "kvm-no-adjvtime")) { +- object_property_set_bool(cpuobj, "kvm-no-adjvtime", true, NULL); +- } +- +- if (vmc->no_kvm_steal_time && +- object_property_find(cpuobj, "kvm-steal-time")) { +- object_property_set_bool(cpuobj, "kvm-steal-time", false, NULL); +- } +- +- if (vmc->no_pmu && object_property_find(cpuobj, "pmu")) { +- object_property_set_bool(cpuobj, "pmu", false, NULL); +- } +- +- if (vmc->no_tcg_lpa2 && object_property_find(cpuobj, "lpa2")) { +- object_property_set_bool(cpuobj, "lpa2", false, NULL); +- } +- +- if (object_property_find(cpuobj, "reset-cbar")) { +- object_property_set_int(cpuobj, "reset-cbar", +- vms->memmap[VIRT_CPUPERIPHS].base, +- &error_abort); +- } +- +- object_property_set_link(cpuobj, "memory", OBJECT(sysmem), +- &error_abort); +- if (vms->secure) { +- object_property_set_link(cpuobj, "secure-memory", +- OBJECT(secure_sysmem), &error_abort); +- } +- +- if (vms->mte) { +- /* Create the memory region only once, but link to all cpus. */ +- if (!tag_sysmem) { +- /* +- * The property exists only if MemTag is supported. +- * If it is, we must allocate the ram to back that up. +- */ +- if (!object_property_find(cpuobj, "tag-memory")) { +- error_report("MTE requested, but not supported " +- "by the guest CPU"); +- exit(1); +- } +- +- tag_sysmem = g_new(MemoryRegion, 1); +- memory_region_init(tag_sysmem, OBJECT(machine), +- "tag-memory", UINT64_MAX / 32); +- +- if (vms->secure) { +- secure_tag_sysmem = g_new(MemoryRegion, 1); +- memory_region_init(secure_tag_sysmem, OBJECT(machine), +- "secure-tag-memory", UINT64_MAX / 32); +- +- /* As with ram, secure-tag takes precedence over tag. */ +- memory_region_add_subregion_overlap(secure_tag_sysmem, 0, +- tag_sysmem, -1); +- } +- } +- +- object_property_set_link(cpuobj, "tag-memory", OBJECT(tag_sysmem), +- &error_abort); +- if (vms->secure) { +- object_property_set_link(cpuobj, "secure-tag-memory", +- OBJECT(secure_tag_sysmem), +- &error_abort); +- } +- } +- + qdev_realize(DEVICE(cpuobj), NULL, &error_fatal); + object_unref(cpuobj); + } +diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h +index e944d434c4..49d1ec8656 100644 +--- a/include/hw/arm/virt.h ++++ b/include/hw/arm/virt.h +@@ -139,6 +139,10 @@ struct VirtMachineState { + DeviceState *platform_bus_dev; + FWCfgState *fw_cfg; + PFlashCFI01 *flash[2]; ++ MemoryRegion *sysmem; ++ MemoryRegion *secure_sysmem; ++ MemoryRegion *tag_sysmem; ++ MemoryRegion *secure_tag_sysmem; + bool secure; + bool highmem; + bool highmem_compact; +-- +2.27.0 + diff --git a/hw-intc-arm-gicv3-Changes-required-to-re-init-the-vC.patch b/hw-intc-arm-gicv3-Changes-required-to-re-init-the-vC.patch new file mode 100644 index 0000000000000000000000000000000000000000..d551fad998c56d23a82c912ab300dcf261bf381d --- /dev/null +++ b/hw-intc-arm-gicv3-Changes-required-to-re-init-the-vC.patch @@ -0,0 +1,403 @@ +From 4e0a4443b7c36608fc30dcaaf0db120220111dd2 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sat, 9 May 2020 15:26:27 +0100 +Subject: [PATCH] hw/intc/arm-gicv3*: Changes required to (re)init the vCPU + register info + +vCPU register info needs to be re-initialized each time vCPU is hot-plugged. +This has to be done both for emulation/TCG and KVM case. This is done in +context to the GIC update notification for any vCPU hot-(un)plug events. This +change adds that support and re-factors existing to maximize the code re-use. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Signed-off-by: Salil Mehta +--- + hw/intc/arm_gicv3.c | 1 + + hw/intc/arm_gicv3_common.c | 7 +- + hw/intc/arm_gicv3_cpuif.c | 257 +++++++++++++++-------------- + hw/intc/arm_gicv3_kvm.c | 7 +- + hw/intc/gicv3_internal.h | 1 + + include/hw/intc/arm_gicv3_common.h | 1 + + 6 files changed, 150 insertions(+), 124 deletions(-) + +diff --git a/hw/intc/arm_gicv3.c b/hw/intc/arm_gicv3.c +index 0b8f79a122..e1c7c8c4bc 100644 +--- a/hw/intc/arm_gicv3.c ++++ b/hw/intc/arm_gicv3.c +@@ -410,6 +410,7 @@ static void arm_gicv3_class_init(ObjectClass *klass, void *data) + ARMGICv3Class *agc = ARM_GICV3_CLASS(klass); + + agcc->post_load = arm_gicv3_post_load; ++ agcc->init_cpu_reginfo = gicv3_init_cpu_reginfo; + device_class_set_parent_realize(dc, arm_gic_realize, &agc->parent_realize); + } + +diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c +index fc87fa9369..d051024a30 100644 +--- a/hw/intc/arm_gicv3_common.c ++++ b/hw/intc/arm_gicv3_common.c +@@ -345,10 +345,12 @@ static void arm_gicv3_cpu_update_notifier(Notifier *notifier, void * data) + { + GICv3CPUHotplugInfo *gic_info = (GICv3CPUHotplugInfo *)data; + CPUState *cpu = gic_info->cpu; ++ ARMGICv3CommonClass *c; + int gic_cpuif_num; + GICv3State *s; + + s = ARM_GICV3_COMMON(gic_info->gic); ++ c = ARM_GICV3_COMMON_GET_CLASS(s); + + /* this shall get us mapped gicv3 cpuif corresponding to mpidr */ + gic_cpuif_num = arm_gicv3_get_proc_num(s, cpu); +@@ -368,7 +370,10 @@ static void arm_gicv3_cpu_update_notifier(Notifier *notifier, void * data) + gicv3_set_gicv3state(cpu, &s->cpu[gic_cpuif_num]); + gicv3_set_cpustate(&s->cpu[gic_cpuif_num], cpu); + +- /* TODO: initialize the registers info for this newly added cpu */ ++ /* initialize the registers info for this newly added cpu */ ++ if (c->init_cpu_reginfo) { ++ c->init_cpu_reginfo(cpu); ++ } + } + + static void arm_gicv3_common_realize(DeviceState *dev, Error **errp) +diff --git a/hw/intc/arm_gicv3_cpuif.c b/hw/intc/arm_gicv3_cpuif.c +index 0d0eb2f62f..a013510074 100644 +--- a/hw/intc/arm_gicv3_cpuif.c ++++ b/hw/intc/arm_gicv3_cpuif.c +@@ -2782,6 +2782,127 @@ static const ARMCPRegInfo gicv3_cpuif_ich_apxr23_reginfo[] = { + }, + }; + ++void gicv3_init_cpu_reginfo(CPUState *cs) ++{ ++ ARMCPU *cpu = ARM_CPU(cs); ++ GICv3CPUState *gcs = icc_cs_from_env(&cpu->env); ++ ++ /* ++ * If the CPU doesn't define a GICv3 configuration, probably because ++ * in real hardware it doesn't have one, then we use default values ++ * matching the one used by most Arm CPUs. This applies to: ++ * cpu->gic_num_lrs ++ * cpu->gic_vpribits ++ * cpu->gic_vprebits ++ * cpu->gic_pribits ++ */ ++ ++ /* ++ * Note that we can't just use the GICv3CPUState as an opaque pointer ++ * in define_arm_cp_regs_with_opaque(), because when we're called back ++ * it might be with code translated by CPU 0 but run by CPU 1, in ++ * which case we'd get the wrong value. ++ * So instead we define the regs with no ri->opaque info, and ++ * get back to the GICv3CPUState from the CPUARMState. ++ */ ++ define_arm_cp_regs(cpu, gicv3_cpuif_reginfo); ++ ++ /* ++ * The CPU implementation specifies the number of supported ++ * bits of physical priority. For backwards compatibility ++ * of migration, we have a compat property that forces use ++ * of 8 priority bits regardless of what the CPU really has. ++ */ ++ if (gcs->gic->force_8bit_prio) { ++ gcs->pribits = 8; ++ } else { ++ gcs->pribits = cpu->gic_pribits ?: 5; ++ } ++ ++ /* ++ * The GICv3 has separate ID register fields for virtual priority ++ * and preemption bit values, but only a single ID register field ++ * for the physical priority bits. The preemption bit count is ++ * always the same as the priority bit count, except that 8 bits ++ * of priority means 7 preemption bits. We precalculate the ++ * preemption bits because it simplifies the code and makes the ++ * parallels between the virtual and physical bits of the GIC ++ * a bit clearer. ++ */ ++ gcs->prebits = gcs->pribits; ++ if (gcs->prebits == 8) { ++ gcs->prebits--; ++ } ++ /* ++ * Check that CPU code defining pribits didn't violate ++ * architectural constraints our implementation relies on. ++ */ ++ g_assert(gcs->pribits >= 4 && gcs->pribits <= 8); ++ ++ /* ++ * gicv3_cpuif_reginfo[] defines ICC_AP*R0_EL1; add definitions ++ * for ICC_AP*R{1,2,3}_EL1 if the prebits value requires them. ++ */ ++ if (gcs->prebits >= 6) { ++ define_arm_cp_regs(cpu, gicv3_cpuif_icc_apxr1_reginfo); ++ } ++ if (gcs->prebits == 7) { ++ define_arm_cp_regs(cpu, gicv3_cpuif_icc_apxr23_reginfo); ++ } ++ ++ if (arm_feature(&cpu->env, ARM_FEATURE_EL2)) { ++ int j; ++ ++ gcs->num_list_regs = cpu->gic_num_lrs ?: 4; ++ gcs->vpribits = cpu->gic_vpribits ?: 5; ++ gcs->vprebits = cpu->gic_vprebits ?: 5; ++ ++ /* ++ * Check against architectural constraints: getting these ++ * wrong would be a bug in the CPU code defining these, ++ * and the implementation relies on them holding. ++ */ ++ g_assert(gcs->vprebits <= gcs->vpribits); ++ g_assert(gcs->vprebits >= 5 && gcs->vprebits <= 7); ++ g_assert(gcs->vpribits >= 5 && gcs->vpribits <= 8); ++ ++ define_arm_cp_regs(cpu, gicv3_cpuif_hcr_reginfo); ++ ++ for (j = 0; j < gcs->num_list_regs; j++) { ++ /* ++ * Note that the AArch64 LRs are 64-bit; the AArch32 LRs ++ * are split into two cp15 regs, LR (the low part, with the ++ * same encoding as the AArch64 LR) and LRC (the high part). ++ */ ++ ARMCPRegInfo lr_regset[] = { ++ { .name = "ICH_LRn_EL2", .state = ARM_CP_STATE_BOTH, ++ .opc0 = 3, .opc1 = 4, .crn = 12, ++ .crm = 12 + (j >> 3), .opc2 = j & 7, ++ .type = ARM_CP_IO | ARM_CP_NO_RAW, ++ .access = PL2_RW, ++ .readfn = ich_lr_read, ++ .writefn = ich_lr_write, ++ }, ++ { .name = "ICH_LRCn_EL2", .state = ARM_CP_STATE_AA32, ++ .cp = 15, .opc1 = 4, .crn = 12, ++ .crm = 14 + (j >> 3), .opc2 = j & 7, ++ .type = ARM_CP_IO | ARM_CP_NO_RAW, ++ .access = PL2_RW, ++ .readfn = ich_lr_read, ++ .writefn = ich_lr_write, ++ }, ++ }; ++ define_arm_cp_regs(cpu, lr_regset); ++ } ++ if (gcs->vprebits >= 6) { ++ define_arm_cp_regs(cpu, gicv3_cpuif_ich_apxr1_reginfo); ++ } ++ if (gcs->vprebits == 7) { ++ define_arm_cp_regs(cpu, gicv3_cpuif_ich_apxr23_reginfo); ++ } ++ } ++} ++ + static void gicv3_cpuif_el_change_hook(ARMCPU *cpu, void *opaque) + { + GICv3CPUState *cs = opaque; +@@ -2804,131 +2925,23 @@ void gicv3_init_cpuif(GICv3State *s) + + for (i = 0; i < s->num_cpu; i++) { + ARMCPU *cpu = ARM_CPU(qemu_get_cpu(i)); +- GICv3CPUState *cs = &s->cpu[i]; +- +- /* +- * If the CPU doesn't define a GICv3 configuration, probably because +- * in real hardware it doesn't have one, then we use default values +- * matching the one used by most Arm CPUs. This applies to: +- * cpu->gic_num_lrs +- * cpu->gic_vpribits +- * cpu->gic_vprebits +- * cpu->gic_pribits +- */ +- +- /* Note that we can't just use the GICv3CPUState as an opaque pointer +- * in define_arm_cp_regs_with_opaque(), because when we're called back +- * it might be with code translated by CPU 0 but run by CPU 1, in +- * which case we'd get the wrong value. +- * So instead we define the regs with no ri->opaque info, and +- * get back to the GICv3CPUState from the CPUARMState. +- * +- * These CP regs callbacks can be called from either TCG or HVF code. +- */ +- define_arm_cp_regs(cpu, gicv3_cpuif_reginfo); +- +- /* +- * The CPU implementation specifies the number of supported +- * bits of physical priority. For backwards compatibility +- * of migration, we have a compat property that forces use +- * of 8 priority bits regardless of what the CPU really has. +- */ +- if (s->force_8bit_prio) { +- cs->pribits = 8; +- } else { +- cs->pribits = cpu->gic_pribits ?: 5; +- } +- +- /* +- * The GICv3 has separate ID register fields for virtual priority +- * and preemption bit values, but only a single ID register field +- * for the physical priority bits. The preemption bit count is +- * always the same as the priority bit count, except that 8 bits +- * of priority means 7 preemption bits. We precalculate the +- * preemption bits because it simplifies the code and makes the +- * parallels between the virtual and physical bits of the GIC +- * a bit clearer. +- */ +- cs->prebits = cs->pribits; +- if (cs->prebits == 8) { +- cs->prebits--; +- } +- /* +- * Check that CPU code defining pribits didn't violate +- * architectural constraints our implementation relies on. +- */ +- g_assert(cs->pribits >= 4 && cs->pribits <= 8); + +- /* +- * gicv3_cpuif_reginfo[] defines ICC_AP*R0_EL1; add definitions +- * for ICC_AP*R{1,2,3}_EL1 if the prebits value requires them. +- */ +- if (cs->prebits >= 6) { +- define_arm_cp_regs(cpu, gicv3_cpuif_icc_apxr1_reginfo); +- } +- if (cs->prebits == 7) { +- define_arm_cp_regs(cpu, gicv3_cpuif_icc_apxr23_reginfo); +- } +- +- if (arm_feature(&cpu->env, ARM_FEATURE_EL2)) { +- int j; +- +- cs->num_list_regs = cpu->gic_num_lrs ?: 4; +- cs->vpribits = cpu->gic_vpribits ?: 5; +- cs->vprebits = cpu->gic_vprebits ?: 5; +- +- /* Check against architectural constraints: getting these +- * wrong would be a bug in the CPU code defining these, +- * and the implementation relies on them holding. +- */ +- g_assert(cs->vprebits <= cs->vpribits); +- g_assert(cs->vprebits >= 5 && cs->vprebits <= 7); +- g_assert(cs->vpribits >= 5 && cs->vpribits <= 8); +- +- define_arm_cp_regs(cpu, gicv3_cpuif_hcr_reginfo); +- +- for (j = 0; j < cs->num_list_regs; j++) { +- /* Note that the AArch64 LRs are 64-bit; the AArch32 LRs +- * are split into two cp15 regs, LR (the low part, with the +- * same encoding as the AArch64 LR) and LRC (the high part). ++ if (qemu_enabled_cpu(CPU(cpu))) { ++ GICv3CPUState *cs = icc_cs_from_env(&cpu->env); ++ gicv3_init_cpu_reginfo(CPU(cpu)); ++ if (tcg_enabled() || qtest_enabled()) { ++ /* ++ * We can only trap EL changes with TCG. However the GIC ++ * interrupt state only changes on EL changes involving EL2 or ++ * EL3, so for the non-TCG case this is OK, as EL2 and EL3 can't ++ * exist. + */ +- ARMCPRegInfo lr_regset[] = { +- { .name = "ICH_LRn_EL2", .state = ARM_CP_STATE_BOTH, +- .opc0 = 3, .opc1 = 4, .crn = 12, +- .crm = 12 + (j >> 3), .opc2 = j & 7, +- .type = ARM_CP_IO | ARM_CP_NO_RAW, +- .access = PL2_RW, +- .readfn = ich_lr_read, +- .writefn = ich_lr_write, +- }, +- { .name = "ICH_LRCn_EL2", .state = ARM_CP_STATE_AA32, +- .cp = 15, .opc1 = 4, .crn = 12, +- .crm = 14 + (j >> 3), .opc2 = j & 7, +- .type = ARM_CP_IO | ARM_CP_NO_RAW, +- .access = PL2_RW, +- .readfn = ich_lr_read, +- .writefn = ich_lr_write, +- }, +- }; +- define_arm_cp_regs(cpu, lr_regset); +- } +- if (cs->vprebits >= 6) { +- define_arm_cp_regs(cpu, gicv3_cpuif_ich_apxr1_reginfo); +- } +- if (cs->vprebits == 7) { +- define_arm_cp_regs(cpu, gicv3_cpuif_ich_apxr23_reginfo); ++ arm_register_el_change_hook(cpu, gicv3_cpuif_el_change_hook, ++ cs); ++ } else { ++ assert(!arm_feature(&cpu->env, ARM_FEATURE_EL2)); ++ assert(!arm_feature(&cpu->env, ARM_FEATURE_EL3)); + } + } +- if (tcg_enabled() || qtest_enabled()) { +- /* +- * We can only trap EL changes with TCG. However the GIC interrupt +- * state only changes on EL changes involving EL2 or EL3, so for +- * the non-TCG case this is OK, as EL2 and EL3 can't exist. +- */ +- arm_register_el_change_hook(cpu, gicv3_cpuif_el_change_hook, cs); +- } else { +- assert(!arm_feature(&cpu->env, ARM_FEATURE_EL2)); +- assert(!arm_feature(&cpu->env, ARM_FEATURE_EL3)); +- } + } + } +diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c +index db06c75e2b..dd2a60fa20 100644 +--- a/hw/intc/arm_gicv3_kvm.c ++++ b/hw/intc/arm_gicv3_kvm.c +@@ -804,6 +804,10 @@ static void vm_change_state_handler(void *opaque, bool running, + } + } + ++static void kvm_gicv3_init_cpu_reginfo(CPUState *cs) ++{ ++ define_arm_cp_regs(ARM_CPU(cs), gicv3_cpuif_reginfo); ++} + + static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp) + { +@@ -837,7 +841,7 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp) + for (i = 0; i < s->num_cpu; i++) { + CPUState *cs = qemu_get_cpu(i); + if (qemu_enabled_cpu(cs)) { +- define_arm_cp_regs(ARM_CPU(cs), gicv3_cpuif_reginfo); ++ kvm_gicv3_init_cpu_reginfo(cs); + } + } + +@@ -925,6 +929,7 @@ static void kvm_arm_gicv3_class_init(ObjectClass *klass, void *data) + + agcc->pre_save = kvm_arm_gicv3_get; + agcc->post_load = kvm_arm_gicv3_put; ++ agcc->init_cpu_reginfo = kvm_gicv3_init_cpu_reginfo; + device_class_set_parent_realize(dc, kvm_arm_gicv3_realize, + &kgc->parent_realize); + resettable_class_set_parent_phases(rc, NULL, kvm_arm_gicv3_reset_hold, NULL, +diff --git a/hw/intc/gicv3_internal.h b/hw/intc/gicv3_internal.h +index 9d4c1209bd..0bed0f6e2a 100644 +--- a/hw/intc/gicv3_internal.h ++++ b/hw/intc/gicv3_internal.h +@@ -709,6 +709,7 @@ void gicv3_redist_vinvall(GICv3CPUState *cs, uint64_t vptaddr); + + void gicv3_redist_send_sgi(GICv3CPUState *cs, int grp, int irq, bool ns); + void gicv3_init_cpuif(GICv3State *s); ++void gicv3_init_cpu_reginfo(CPUState *cs); + + /** + * gicv3_cpuif_update: +diff --git a/include/hw/intc/arm_gicv3_common.h b/include/hw/intc/arm_gicv3_common.h +index 97a48f44b9..b5f8ba17ff 100644 +--- a/include/hw/intc/arm_gicv3_common.h ++++ b/include/hw/intc/arm_gicv3_common.h +@@ -325,6 +325,7 @@ struct ARMGICv3CommonClass { + + void (*pre_save)(GICv3State *s); + void (*post_load)(GICv3State *s); ++ void (*init_cpu_reginfo)(CPUState *cs); + }; + + void gicv3_init_irqs_and_mmio(GICv3State *s, qemu_irq_handler handler, +-- +2.27.0 + diff --git a/intc-gicv3-Fixes-for-vcpu-hotplug.patch b/intc-gicv3-Fixes-for-vcpu-hotplug.patch new file mode 100644 index 0000000000000000000000000000000000000000..5241a57ddaccffa65e13ff8defd9ffbad9750bc1 --- /dev/null +++ b/intc-gicv3-Fixes-for-vcpu-hotplug.patch @@ -0,0 +1,70 @@ +From 343b61303152b06f9e1ba6d09a405faeaa3fcc98 Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 22:12:58 +0800 +Subject: [PATCH] intc/gicv3: Fixes for vcpu hotplug + +1. Some types of machine don't support possible_cpus +callback. +2. The cpu_update_notifier is register only when machine +support vcpu hotplug, so do notifier_remove() unconditi- +onally is wrong. + +Signed-off-by: Keqian Zhu +--- + cpu-common.c | 4 ++++ + hw/intc/arm_gicv3_common.c | 9 +++++++-- + 2 files changed, 11 insertions(+), 2 deletions(-) + +diff --git a/cpu-common.c b/cpu-common.c +index da52e45760..54e63b3f77 100644 +--- a/cpu-common.c ++++ b/cpu-common.c +@@ -113,6 +113,10 @@ CPUState *qemu_get_possible_cpu(int index) + MachineState *ms = MACHINE(qdev_get_machine()); + const CPUArchIdList *possible_cpus = ms->possible_cpus; + ++ if (possible_cpus == NULL) { ++ return qemu_get_cpu(index); ++ } ++ + assert((index >= 0) && (index < possible_cpus->len)); + + return CPU(possible_cpus->cpus[index].cpu); +diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c +index d051024a30..5667d9f40b 100644 +--- a/hw/intc/arm_gicv3_common.c ++++ b/hw/intc/arm_gicv3_common.c +@@ -25,6 +25,7 @@ + #include "qapi/error.h" + #include "qemu/module.h" + #include "qemu/error-report.h" ++#include "hw/boards.h" + #include "hw/core/cpu.h" + #include "hw/intc/arm_gicv3_common.h" + #include "hw/qdev-properties.h" +@@ -446,7 +447,7 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp) + s->cpu = g_new0(GICv3CPUState, s->num_cpu); + + for (i = 0; i < s->num_cpu; i++) { +- CPUState *cpu = qemu_get_possible_cpu(i); ++ CPUState *cpu = qemu_get_possible_cpu(i) ? : qemu_get_cpu(i); + uint64_t cpu_affid; + + if (qemu_enabled_cpu(cpu)) { +@@ -506,8 +507,12 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp) + static void arm_gicv3_finalize(Object *obj) + { + GICv3State *s = ARM_GICV3_COMMON(obj); ++ Object *ms = qdev_get_machine(); ++ MachineClass *mc = MACHINE_GET_CLASS(ms); + +- notifier_remove(&s->cpu_update_notifier); ++ if (mc->has_hotpluggable_cpus) { ++ notifier_remove(&s->cpu_update_notifier); ++ } + g_free(s->redist_region_count); + } + +-- +2.27.0 + diff --git a/physmem-gdbstub-Common-helping-funcs-changes-to-unre.patch b/physmem-gdbstub-Common-helping-funcs-changes-to-unre.patch new file mode 100644 index 0000000000000000000000000000000000000000..696a9a49357b61cd9d645c657c5aa38830276e88 --- /dev/null +++ b/physmem-gdbstub-Common-helping-funcs-changes-to-unre.patch @@ -0,0 +1,127 @@ +From 8fa5af7de07d9bc2535ea8fab087d509795e3579 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sun, 6 Aug 2023 22:12:52 +0000 +Subject: [PATCH] physmem,gdbstub: Common helping funcs/changes to *unrealize* + vCPU + +Supporting vCPU Hotplug for ARM arch also means introducing new functionality of +unrealizing the ARMCPU. This requires some new common functions. + +Defining them as part of architecture independent change so that this code could +be reused by other interested parties. + +Signed-off-by: Salil Mehta +--- + gdbstub/gdbstub.c | 6 ++++++ + include/exec/cpu-common.h | 8 ++++++++ + include/exec/gdbstub.h | 1 + + include/hw/core/cpu.h | 1 + + system/physmem.c | 25 +++++++++++++++++++++++++ + 5 files changed, 41 insertions(+) + +diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c +index 46d752bbc2..f16006d2a8 100644 +--- a/gdbstub/gdbstub.c ++++ b/gdbstub/gdbstub.c +@@ -582,6 +582,12 @@ void gdb_register_coprocessor(CPUState *cpu, + } + } + ++void gdb_unregister_coprocessor_all(CPUState *cpu) ++{ ++ g_array_free(cpu->gdb_regs, true); ++ cpu->gdb_regs = NULL; ++} ++ + static void gdb_process_breakpoint_remove_all(GDBProcess *p) + { + CPUState *cpu = gdb_get_first_cpu_in_process(p); +diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h +index 41115d8919..2a3d4aa1c8 100644 +--- a/include/exec/cpu-common.h ++++ b/include/exec/cpu-common.h +@@ -139,6 +139,14 @@ size_t qemu_ram_pagesize_largest(void); + */ + void cpu_address_space_init(CPUState *cpu, int asidx, + const char *prefix, MemoryRegion *mr); ++/** ++ * cpu_address_space_destroy: ++ * @cpu: CPU for which address space needs to be destroyed ++ * @asidx: integer index of this address space ++ * ++ * Note that with KVM only one address space is supported. ++ */ ++void cpu_address_space_destroy(CPUState *cpu, int asidx); + + void cpu_physical_memory_rw(hwaddr addr, void *buf, + hwaddr len, bool is_write); +diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h +index d8a3c56fa2..d123b838c2 100644 +--- a/include/exec/gdbstub.h ++++ b/include/exec/gdbstub.h +@@ -39,6 +39,7 @@ typedef int (*gdb_set_reg_cb)(CPUArchState *env, uint8_t *buf, int reg); + void gdb_register_coprocessor(CPUState *cpu, + gdb_get_reg_cb get_reg, gdb_set_reg_cb set_reg, + int num_regs, const char *xml, int g_pos); ++void gdb_unregister_coprocessor_all(CPUState *cpu); + + /** + * gdbserver_start: start the gdb server +diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h +index 0ca778eb75..6dbe163548 100644 +--- a/include/hw/core/cpu.h ++++ b/include/hw/core/cpu.h +@@ -496,6 +496,7 @@ struct CPUState { + QSIMPLEQ_HEAD(, qemu_work_item) work_list; + + CPUAddressSpace *cpu_ases; ++ int cpu_ases_ref_count; + int num_ases; + AddressSpace *as; + MemoryRegion *memory; +diff --git a/system/physmem.c b/system/physmem.c +index 247c252e53..299174ad91 100644 +--- a/system/physmem.c ++++ b/system/physmem.c +@@ -761,6 +761,7 @@ void cpu_address_space_init(CPUState *cpu, int asidx, + + if (!cpu->cpu_ases) { + cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases); ++ cpu->cpu_ases_ref_count = cpu->num_ases; + } + + newas = &cpu->cpu_ases[asidx]; +@@ -774,6 +775,30 @@ void cpu_address_space_init(CPUState *cpu, int asidx, + } + } + ++void cpu_address_space_destroy(CPUState *cpu, int asidx) ++{ ++ CPUAddressSpace *cpuas; ++ ++ assert(asidx < cpu->num_ases); ++ assert(asidx == 0 || !kvm_enabled()); ++ assert(cpu->cpu_ases); ++ ++ cpuas = &cpu->cpu_ases[asidx]; ++ if (tcg_enabled()) { ++ memory_listener_unregister(&cpuas->tcg_as_listener); ++ } ++ ++ address_space_destroy(cpuas->as); ++ g_free_rcu(cpuas->as, rcu); ++ ++ if (cpu->cpu_ases_ref_count == 1) { ++ g_free(cpu->cpu_ases); ++ cpu->cpu_ases = NULL; ++ } ++ ++ cpu->cpu_ases_ref_count--; ++} ++ + AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx) + { + /* Return the AddressSpace corresponding to the specified index */ +-- +2.27.0 + diff --git a/qemu.spec b/qemu.spec index 1a258b1e20cd66cc7a6f829de32f18a651ff236c..e8b6e62f08b632159b3bf2a58188daad10e61cba 100644 --- a/qemu.spec +++ b/qemu.spec @@ -3,7 +3,7 @@ Name: qemu Version: 8.2.0 -Release: 5 +Release: 6 Epoch: 11 Summary: QEMU is a generic and open source machine emulator and virtualizer License: GPLv2 and BSD and MIT and CC-BY-SA-4.0 @@ -179,6 +179,63 @@ Patch0162: arm-acpi-Fix-when-make-qemu-system-aarch64-at-x86_64.patch Patch0163: linux-headers-update-against-5.10-and-manual-clear-v.patch Patch0164: vfio-Maintain-DMA-mapping-range-for-the-container.patch Patch0165: vfio-migration-Add-support-for-manual-clear-vfio-dir.patch +Patch0166: arm-virt-target-arm-Add-new-ARMCPU-socket-cluster-co.patch +Patch0167: cpus-common-Add-common-CPU-utility-for-possible-vCPU.patch +Patch0168: hw-arm-virt-Move-setting-of-common-CPU-properties-in.patch +Patch0169: arm-virt-target-arm-Machine-init-time-change-common-.patch +Patch0170: accel-kvm-Extract-common-KVM-vCPU-creation-parking-c.patch +Patch0171: arm-virt-kvm-Pre-create-disabled-possible-vCPUs-mach.patch +Patch0172: arm-virt-gicv3-Changes-to-pre-size-GIC-with-possible.patch +Patch0173: arm-virt-Init-PMU-at-host-for-all-possible-vcpus.patch +Patch0174: hw-acpi-Move-CPU-ctrl-dev-MMIO-region-len-macro-to-c.patch +Patch0175: arm-acpi-Enable-ACPI-support-for-vcpu-hotplug.patch +Patch0176: hw-acpi-Add-ACPI-CPU-hotplug-init-stub.patch +Patch0177: hw-acpi-Use-qemu_present_cpu-API-in-ACPI-CPU-hotplug.patch +Patch0178: hw-acpi-Init-GED-framework-with-cpu-hotplug-events.patch +Patch0179: arm-virt-Add-cpu-hotplug-events-to-GED-during-creati.patch +Patch0180: arm-virt-Create-GED-dev-before-disabled-CPU-Objs-are.patch +Patch0181: hw-acpi-Update-CPUs-AML-with-cpu-ctrl-dev-change.patch +Patch0182: arm-virt-acpi-Factor-out-CPPC-building-from-DSDT-CPU.patch +Patch0183: acpi-cpu-Add-cpu_cppc-building-support.patch +Patch0184: tests-acpi-bios-tables-test-Allow-changes-to-virt-DS.patch +Patch0185: arm-virt-acpi-Build-CPUs-AML-with-CPU-Hotplug-suppor.patch +Patch0186: arm-virt-Make-ARM-vCPU-present-status-ACPI-persisten.patch +Patch0187: hw-acpi-ACPI-AML-Changes-to-reflect-the-correct-_STA.patch +Patch0188: hw-acpi-Update-GED-_EVT-method-AML-with-cpu-scan.patch +Patch0189: hw-arm-MADT-Tbl-change-to-size-the-guest-with-possib.patch +Patch0190: hw-acpi-Make-_MAT-method-optional.patch +Patch0191: arm-virt-Release-objects-for-disabled-possible-vCPUs.patch +Patch0192: hw-acpi-Update-ACPI-GED-framework-to-support-vCPU-Ho.patch +Patch0193: arm-virt-Add-update-basic-hot-un-plug-framework.patch +Patch0194: arm-virt-Changes-to-un-wire-GICC-vCPU-IRQs-during-ho.patch +Patch0195: hw-arm-gicv3-Changes-to-update-GIC-with-vCPU-hot-plu.patch +Patch0196: hw-intc-arm-gicv3-Changes-required-to-re-init-the-vC.patch +Patch0197: arm-virt-Update-the-guest-via-GED-about-CPU-hot-un-p.patch +Patch0198: hw-arm-Changes-required-for-reset-and-to-support-nex.patch +Patch0199: physmem-gdbstub-Common-helping-funcs-changes-to-unre.patch +Patch0200: target-arm-Add-support-of-unrealize-ARMCPU-during-vC.patch +Patch0201: target-arm-kvm-Write-CPU-state-back-to-KVM-on-reset.patch +Patch0202: target-arm-kvm-tcg-Register-Handle-SMCCC-hypercall-e.patch +Patch0203: hw-arm-Support-hotplug-capability-check-using-_OSC-m.patch +Patch0204: tcg-mttcg-enable-threads-to-unregister-in-tcg_ctxs.patch +Patch0205: hw-arm-virt-Expose-cold-booted-CPUs-as-MADT-GICC-Ena.patch +Patch0206: system-physmem-Fix-possible-double-free-when-destroy.patch +Patch0207: arm-cpu-Some-fixes-for-arm_cpu_unrealizefn.patch +Patch0208: acpi-cpu-Fix-cpu_hotplug_hw_init.patch +Patch0209: system-cpus-Fix-pause_all_vcpus-under-concurrent-env.patch +Patch0210: system-cpus-Fix-resume_all_vcpus-under-vCPU-hotplug-.patch +Patch0211: arm-virt.c-Convey-local_err-when-set-psci-conduit.patch +Patch0212: arm-virt-Fix-adjudgement-of-core_id-for-vcpu-hotplug.patch +Patch0213: accel-kvm-Use-correct-id-for-parked-vcpu.patch +Patch0214: arm-kvm-Set-psci-smccc-filter-only-with-vcpu-hotplug.patch +Patch0215: intc-gicv3-Fixes-for-vcpu-hotplug.patch +Patch0216: acpi-ged-Init-cpu-hotplug-only-when-machine-support-.patch +Patch0217: acpi-ged-Remove-cpuhp-field-of-ged.patch +Patch0218: arm-virt-acpi-Require-possible_cpu_arch_ids-for-buil.patch +Patch0219: arm-virt-Consider-has_ged-when-set-mc-has_hotpluggab.patch +Patch0220: arm-virt-Require-mc-has_hotpluggable_cpus-for-cold-p.patch +Patch0221: tests-acpi-Update-expected-ACPI-tables-for-vcpu-hotp.patch +Patch0222: coro-support-live-patch-for-libcare.patch BuildRequires: flex BuildRequires: gcc @@ -776,6 +833,65 @@ getent passwd qemu >/dev/null || \ %endif %changelog +* Wed Apr 10 2024 Jiabo Feng - 11:8.2.0-6 +- coro: support live patch for libcare +- tests/acpi: Update expected ACPI tables for vcpu hotplug(Update BinDir) +- arm/virt: Require mc->has_hotpluggable_cpus for cold-plugged vcpu +- arm/virt: Consider has_ged when set mc->has_hotpluggable_cpus +- arm/virt-acpi: Require possible_cpu_arch_ids for build_cpus_aml() +- acpi/ged: Remove cpuhp field of ged +- acpi/ged: Init cpu hotplug only when machine support it +- intc/gicv3: Fixes for vcpu hotplug +- arm/kvm: Set psci smccc filter only with vcpu hotplug +- accel/kvm: Use correct id for parked vcpu +- arm/virt: Fix adjudgement of core_id for vcpu hotplugged +- arm/virt.c: Convey local_err when set psci-conduit +- system/cpus: Fix resume_all_vcpus() under vCPU hotplug condition +- system/cpus: Fix pause_all_vcpus() under concurrent environment +- acpi/cpu: Fix cpu_hotplug_hw_init() +- arm/cpu: Some fixes for arm_cpu_unrealizefn() +- system/physmem: Fix possible double free when destroy cpu as +- hw/arm/virt: Expose cold-booted CPUs as MADT GICC Enabled +- tcg/mttcg: enable threads to unregister in tcg_ctxs[] +- hw/arm: Support hotplug capability check using _OSC method +- target/arm/kvm,tcg: Register/Handle SMCCC hypercall exits to VMM/Qemu +- target/arm/kvm: Write CPU state back to KVM on reset +- target/arm: Add support of *unrealize* ARMCPU during vCPU Hot-unplug +- physmem,gdbstub: Common helping funcs/changes to *unrealize* vCPU +- hw/arm: Changes required for reset and to support next boot +- arm/virt: Update the guest(via GED) about CPU hot-(un)plug events +- hw/intc/arm-gicv3*: Changes required to (re)init the vCPU register info +- hw/arm,gicv3: Changes to update GIC with vCPU hot-plug notification +- arm/virt: Changes to (un)wire GICC<->vCPU IRQs during hot-(un)plug +- arm/virt: Add/update basic hot-(un)plug framework +- hw/acpi: Update ACPI GED framework to support vCPU Hotplug +- arm/virt: Release objects for *disabled* possible vCPUs after init +- hw/acpi: Make _MAT method optional +- hw/arm: MADT Tbl change to size the guest with possible vCPUs +- hw/acpi: Update GED _EVT method AML with cpu scan +- hw/acpi: ACPI/AML Changes to reflect the correct _STA.{PRES,ENA} Bits to Guest +- arm/virt: Make ARM vCPU *present* status ACPI *persistent* +- arm/virt/acpi: Build CPUs AML with CPU Hotplug support +- tests/acpi/bios-tables-test: Allow changes to virt/DSDT file +- acpi/cpu: Add cpu_cppc building support +- arm/virt/acpi: Factor out CPPC building from DSDT CPU aml +- hw/acpi: Update CPUs AML with cpu-(ctrl)dev change +- arm/virt: Create GED dev before *disabled* CPU Objs are destroyed +- arm/virt: Add cpu hotplug events to GED during creation +- hw/acpi: Init GED framework with cpu hotplug events +- hw/acpi: Use qemu_present_cpu() API in ACPI CPU hotplug init +- hw/acpi: Add ACPI CPU hotplug init stub +- arm/acpi: Enable ACPI support for vcpu hotplug +- hw/acpi: Move CPU ctrl-dev MMIO region len macro to common header file +- arm/virt: Init PMU at host for all possible vcpus +- arm/virt,gicv3: Changes to pre-size GIC with possible vcpus @machine init +- arm/virt,kvm: Pre-create disabled possible vCPUs @machine init +- accel/kvm: Extract common KVM vCPU {creation,parking} code +- arm/virt,target/arm: Machine init time change common to vCPU {cold|hot}-plug +- hw/arm/virt: Move setting of common CPU properties in a function +- cpus-common: Add common CPU utility for possible vCPUs +- arm/virt,target/arm: Add new ARMCPU {socket,cluster,core,thread}-id property + * Sun Apr 7 2024 Jiabo Feng - 11:8.2.0-5 - vfio/migration: Add support for manual clear vfio dirty log - vfio: Maintain DMA mapping range for the container diff --git a/system-cpus-Fix-pause_all_vcpus-under-concurrent-env.patch b/system-cpus-Fix-pause_all_vcpus-under-concurrent-env.patch new file mode 100644 index 0000000000000000000000000000000000000000..4c1b707d2a181fcc357d04385781add6b274a8f2 --- /dev/null +++ b/system-cpus-Fix-pause_all_vcpus-under-concurrent-env.patch @@ -0,0 +1,91 @@ +From 401e145800134d0310d613f48c4962a108b8ddda Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Sun, 17 Mar 2024 16:37:03 +0800 +Subject: [PATCH] system/cpus: Fix pause_all_vcpus() under concurrent + environment + +Both main loop thread and vCPU thread are allowed to call +pause_all_vcpus(), and in general resume_all_vcpus() is called +after it. Two issues live in pause_all_vcpus(): + +1. There is possibility that during thread T1 waits on +qemu_pause_cond with bql unlocked, other thread has called +pause_all_vcpus() and resume_all_vcpus(), then thread T1 will +stuck, because the condition all_vcpus_paused() is always false. + +2. After all_vcpus_paused() has been checked as true, we will +unlock bql to relock replay_mutex. During the bql was unlocked, +the vcpu's state may has been changed by other thread, so we +must retry. + +Signed-off-by: Keqian Zhu +--- + system/cpus.c | 29 ++++++++++++++++++++++++----- + 1 file changed, 24 insertions(+), 5 deletions(-) + +diff --git a/system/cpus.c b/system/cpus.c +index a444a747f0..7c5369fa9c 100644 +--- a/system/cpus.c ++++ b/system/cpus.c +@@ -551,12 +551,14 @@ static bool all_vcpus_paused(void) + return true; + } + +-void pause_all_vcpus(void) ++static void request_pause_all_vcpus(void) + { + CPUState *cpu; + +- qemu_clock_enable(QEMU_CLOCK_VIRTUAL, false); + CPU_FOREACH(cpu) { ++ if (cpu->stopped) { ++ continue; ++ } + if (qemu_cpu_is_self(cpu)) { + qemu_cpu_stop(cpu, true); + } else { +@@ -564,6 +566,14 @@ void pause_all_vcpus(void) + qemu_cpu_kick(cpu); + } + } ++} ++ ++void pause_all_vcpus(void) ++{ ++ qemu_clock_enable(QEMU_CLOCK_VIRTUAL, false); ++ ++retry: ++ request_pause_all_vcpus(); + + /* We need to drop the replay_lock so any vCPU threads woken up + * can finish their replay tasks +@@ -572,14 +582,23 @@ void pause_all_vcpus(void) + + while (!all_vcpus_paused()) { + qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex); +- CPU_FOREACH(cpu) { +- qemu_cpu_kick(cpu); +- } ++ /* During we waited on qemu_pause_cond the bql was unlocked, ++ * the vcpu's state may has been changed by other thread, so ++ * we must request the pause state on all vcpus again. ++ */ ++ request_pause_all_vcpus(); + } + + qemu_mutex_unlock_iothread(); + replay_mutex_lock(); + qemu_mutex_lock_iothread(); ++ ++ /* During the bql was unlocked, the vcpu's state may has been ++ * changed by other thread, so we must retry. ++ */ ++ if (!all_vcpus_paused()) { ++ goto retry; ++ } + } + + void cpu_resume(CPUState *cpu) +-- +2.27.0 + diff --git a/system-cpus-Fix-resume_all_vcpus-under-vCPU-hotplug-.patch b/system-cpus-Fix-resume_all_vcpus-under-vCPU-hotplug-.patch new file mode 100644 index 0000000000000000000000000000000000000000..f4008abba6fee9de7707a9cdbf1ad33373a58270 --- /dev/null +++ b/system-cpus-Fix-resume_all_vcpus-under-vCPU-hotplug-.patch @@ -0,0 +1,43 @@ +From a29922f76c9b5064ddd2e686fa725b96c435e889 Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Sun, 17 Mar 2024 16:37:04 +0800 +Subject: [PATCH] system/cpus: Fix resume_all_vcpus() under vCPU hotplug + condition + +For vCPU being hotplugged, qemu_init_vcpu() is called. In this +function, we set vcpu state as stopped, and then wait vcpu thread +to be created. + +As the vcpu state is stopped, it will inform us it has been created +and then wait on halt_cond. After we has realized vcpu object, we +will resume the vcpu thread. + +However, during we wait vcpu thread to be created, the bql is +unlocked, and other thread is allowed to call resume_all_vcpus(), +which will resume the un-realized vcpu. + +This fixes the issue by filter out un-realized vcpu during +resume_all_vcpus(). + +Signed-off-by: Keqian Zhu +--- + system/cpus.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/system/cpus.c b/system/cpus.c +index 7c5369fa9c..f2289e9545 100644 +--- a/system/cpus.c ++++ b/system/cpus.c +@@ -618,6 +618,9 @@ void resume_all_vcpus(void) + + qemu_clock_enable(QEMU_CLOCK_VIRTUAL, true); + CPU_FOREACH(cpu) { ++ if (!object_property_get_bool(OBJECT(cpu), "realized", &error_abort)) { ++ continue; ++ } + cpu_resume(cpu); + } + } +-- +2.27.0 + diff --git a/system-physmem-Fix-possible-double-free-when-destroy.patch b/system-physmem-Fix-possible-double-free-when-destroy.patch new file mode 100644 index 0000000000000000000000000000000000000000..a2f3853bc110515d6d30dcb404fee28aef1339eb --- /dev/null +++ b/system-physmem-Fix-possible-double-free-when-destroy.patch @@ -0,0 +1,64 @@ +From 5f7464524d0fb2c25c9bacfb550df92bef9bb3bf Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 26 Mar 2024 14:11:05 +0800 +Subject: [PATCH] system/physmem: Fix possible double free when destroy cpu as + +address_space_destroy() and g_free_rcu() both operate cpuas->as +at rcu thread context asynchronously, each one is a rcu task +that have different callback (the first callback is do_address_ +space_destroy() and the second callback is g_free()). + +It's possible that while the first task is pending and the second +task overwrites the rcu callback (as the second task operates on +the same object). Then the g_free will be called twice on cpuas->as. + +Signed-off-by: Keqian Zhu +--- + include/exec/memory.h | 1 + + system/memory.c | 3 +++ + system/physmem.c | 2 +- + 3 files changed, 5 insertions(+), 1 deletion(-) + +diff --git a/include/exec/memory.h b/include/exec/memory.h +index e131c2682c..91c42c9a6a 100644 +--- a/include/exec/memory.h ++++ b/include/exec/memory.h +@@ -1114,6 +1114,7 @@ struct AddressSpace { + struct rcu_head rcu; + char *name; + MemoryRegion *root; ++ bool free_in_rcu; + + /* Accessed via RCU. */ + struct FlatView *current_map; +diff --git a/system/memory.c b/system/memory.c +index 798b6c0a17..fb817e54bc 100644 +--- a/system/memory.c ++++ b/system/memory.c +@@ -3130,6 +3130,9 @@ static void do_address_space_destroy(AddressSpace *as) + g_free(as->name); + g_free(as->ioeventfds); + memory_region_unref(as->root); ++ if (as->free_in_rcu) { ++ g_free(as); ++ } + } + + void address_space_destroy(AddressSpace *as) +diff --git a/system/physmem.c b/system/physmem.c +index 299174ad91..cbe838f203 100644 +--- a/system/physmem.c ++++ b/system/physmem.c +@@ -788,8 +788,8 @@ void cpu_address_space_destroy(CPUState *cpu, int asidx) + memory_listener_unregister(&cpuas->tcg_as_listener); + } + ++ cpuas->as->free_in_rcu = true; + address_space_destroy(cpuas->as); +- g_free_rcu(cpuas->as, rcu); + + if (cpu->cpu_ases_ref_count == 1) { + g_free(cpu->cpu_ases); +-- +2.27.0 + diff --git a/target-arm-Add-support-of-unrealize-ARMCPU-during-vC.patch b/target-arm-Add-support-of-unrealize-ARMCPU-during-vC.patch new file mode 100644 index 0000000000000000000000000000000000000000..023fe7f49df7f1fa140ac9c9a7ea7dda1511f453 --- /dev/null +++ b/target-arm-Add-support-of-unrealize-ARMCPU-during-vC.patch @@ -0,0 +1,294 @@ +From b311feda2078e7ee8f060531d4d061beccbc2f77 Mon Sep 17 00:00:00 2001 +From: Salil Mehta +Date: Sat, 9 May 2020 20:13:10 +0100 +Subject: [PATCH] target/arm: Add support of *unrealize* ARMCPU during vCPU + Hot-unplug + +vCPU Hot-unplug will result in QOM CPU object unrealization which will do away +with all the vCPU thread creations, allocations, registrations that happened +as part of the realization process. This change introduces the ARM CPU unrealize +function taking care of exactly that. + +Note, initialized KVM vCPUs are not destroyed in host KVM but their Qemu context +is parked at the QEMU KVM layer. + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Keqian Zhu +Signed-off-by: Keqian Zhu +Reported-by: Vishnu Pajjuri +[VP: Identified CPU stall issue & suggested probable fix] +Signed-off-by: Salil Mehta +--- + target/arm/cpu.c | 101 +++++++++++++++++++++++++++++++++++++++++ + target/arm/cpu.h | 14 ++++++ + target/arm/gdbstub.c | 6 +++ + target/arm/helper.c | 25 ++++++++++ + target/arm/internals.h | 3 ++ + target/arm/kvm64.c | 4 ++ + 6 files changed, 153 insertions(+) + +diff --git a/target/arm/cpu.c b/target/arm/cpu.c +index 18b8a79c8f..501f88eb2f 100644 +--- a/target/arm/cpu.c ++++ b/target/arm/cpu.c +@@ -142,6 +142,16 @@ void arm_register_pre_el_change_hook(ARMCPU *cpu, ARMELChangeHookFn *hook, + QLIST_INSERT_HEAD(&cpu->pre_el_change_hooks, entry, node); + } + ++void arm_unregister_pre_el_change_hooks(ARMCPU *cpu) ++{ ++ ARMELChangeHook *entry, *next; ++ ++ QLIST_FOREACH_SAFE(entry, &cpu->pre_el_change_hooks, node, next) { ++ QLIST_REMOVE(entry, node); ++ g_free(entry); ++ } ++} ++ + void arm_register_el_change_hook(ARMCPU *cpu, ARMELChangeHookFn *hook, + void *opaque) + { +@@ -153,6 +163,16 @@ void arm_register_el_change_hook(ARMCPU *cpu, ARMELChangeHookFn *hook, + QLIST_INSERT_HEAD(&cpu->el_change_hooks, entry, node); + } + ++void arm_unregister_el_change_hooks(ARMCPU *cpu) ++{ ++ ARMELChangeHook *entry, *next; ++ ++ QLIST_FOREACH_SAFE(entry, &cpu->el_change_hooks, node, next) { ++ QLIST_REMOVE(entry, node); ++ g_free(entry); ++ } ++} ++ + static void cp_reg_reset(gpointer key, gpointer value, gpointer opaque) + { + /* Reset a single ARMCPRegInfo register */ +@@ -2390,6 +2410,85 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp) + acc->parent_realize(dev, errp); + } + ++static void arm_cpu_unrealizefn(DeviceState *dev) ++{ ++ ARMCPUClass *acc = ARM_CPU_GET_CLASS(dev); ++ ARMCPU *cpu = ARM_CPU(dev); ++ CPUARMState *env = &cpu->env; ++ CPUState *cs = CPU(dev); ++ bool has_secure; ++ ++ has_secure = cpu->has_el3 || arm_feature(env, ARM_FEATURE_M_SECURITY); ++ ++ /* rock 'n' un-roll, whatever happened in the arm_cpu_realizefn cleanly */ ++ cpu_address_space_destroy(cs, ARMASIdx_NS); ++ ++ if (cpu->tag_memory != NULL) { ++ cpu_address_space_destroy(cs, ARMASIdx_TagNS); ++ if (has_secure) { ++ cpu_address_space_destroy(cs, ARMASIdx_TagS); ++ } ++ } ++ ++ if (has_secure) { ++ cpu_address_space_destroy(cs, ARMASIdx_S); ++ } ++ ++ destroy_cpreg_list(cpu); ++ arm_cpu_unregister_gdb_regs(cpu); ++ unregister_cp_regs_for_features(cpu); ++ ++ if (cpu->sau_sregion && arm_feature(env, ARM_FEATURE_M_SECURITY)) { ++ g_free(env->sau.rbar); ++ g_free(env->sau.rlar); ++ } ++ ++ if (arm_feature(env, ARM_FEATURE_PMSA) && ++ arm_feature(env, ARM_FEATURE_V7) && ++ cpu->pmsav7_dregion) { ++ if (arm_feature(env, ARM_FEATURE_V8)) { ++ g_free(env->pmsav8.rbar[M_REG_NS]); ++ g_free(env->pmsav8.rlar[M_REG_NS]); ++ if (arm_feature(env, ARM_FEATURE_M_SECURITY)) { ++ g_free(env->pmsav8.rbar[M_REG_S]); ++ g_free(env->pmsav8.rlar[M_REG_S]); ++ } ++ } else { ++ g_free(env->pmsav7.drbar); ++ g_free(env->pmsav7.drsr); ++ g_free(env->pmsav7.dracr); ++ } ++ if (cpu->pmsav8r_hdregion) { ++ g_free(env->pmsav8.hprbar); ++ g_free(env->pmsav8.hprlar); ++ } ++ } ++ ++ if (arm_feature(env, ARM_FEATURE_PMU)) { ++ if (!kvm_enabled()) { ++ arm_unregister_pre_el_change_hooks(cpu); ++ arm_unregister_el_change_hooks(cpu); ++ } ++ ++#ifndef CONFIG_USER_ONLY ++ if (cpu->pmu_timer) { ++ timer_del(cpu->pmu_timer); ++ } ++#endif ++ } ++ ++ cpu_remove_sync(CPU(dev)); ++ acc->parent_unrealize(dev); ++ ++#ifndef CONFIG_USER_ONLY ++ timer_del(cpu->gt_timer[GTIMER_PHYS]); ++ timer_del(cpu->gt_timer[GTIMER_VIRT]); ++ timer_del(cpu->gt_timer[GTIMER_HYP]); ++ timer_del(cpu->gt_timer[GTIMER_SEC]); ++ timer_del(cpu->gt_timer[GTIMER_HYPVIRT]); ++#endif ++} ++ + static ObjectClass *arm_cpu_class_by_name(const char *cpu_model) + { + ObjectClass *oc; +@@ -2492,6 +2591,8 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data) + + device_class_set_parent_realize(dc, arm_cpu_realizefn, + &acc->parent_realize); ++ device_class_set_parent_unrealize(dc, arm_cpu_unrealizefn, ++ &acc->parent_unrealize); + + device_class_set_props(dc, arm_cpu_properties); + +diff --git a/target/arm/cpu.h b/target/arm/cpu.h +index 145d3dbf13..c51a0e3467 100644 +--- a/target/arm/cpu.h ++++ b/target/arm/cpu.h +@@ -1138,6 +1138,7 @@ struct ARMCPUClass { + + const ARMCPUInfo *info; + DeviceRealize parent_realize; ++ DeviceUnrealize parent_unrealize; + ResettablePhases parent_phases; + }; + +@@ -3359,6 +3360,13 @@ static inline AddressSpace *arm_addressspace(CPUState *cs, MemTxAttrs attrs) + */ + void arm_register_pre_el_change_hook(ARMCPU *cpu, ARMELChangeHookFn *hook, + void *opaque); ++/** ++ * arm_unregister_pre_el_change_hook: ++ * unregister all pre EL change hook functions. Generally called during ++ * unrealize'ing leg ++ */ ++void arm_unregister_pre_el_change_hooks(ARMCPU *cpu); ++ + /** + * arm_register_el_change_hook: + * Register a hook function which will be called immediately after this +@@ -3371,6 +3379,12 @@ void arm_register_pre_el_change_hook(ARMCPU *cpu, ARMELChangeHookFn *hook, + */ + void arm_register_el_change_hook(ARMCPU *cpu, ARMELChangeHookFn *hook, void + *opaque); ++/** ++ * arm_unregister_el_change_hook: ++ * unregister all EL change hook functions. Generally called during ++ * unrealize'ing leg ++ */ ++void arm_unregister_el_change_hooks(ARMCPU *cpu); + + /** + * arm_rebuild_hflags: +diff --git a/target/arm/gdbstub.c b/target/arm/gdbstub.c +index 28f546a5ff..5ba1e28e34 100644 +--- a/target/arm/gdbstub.c ++++ b/target/arm/gdbstub.c +@@ -553,3 +553,9 @@ void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu) + } + #endif /* CONFIG_TCG */ + } ++ ++void arm_cpu_unregister_gdb_regs(ARMCPU *cpu) ++{ ++ CPUState *cs = CPU(cpu); ++ gdb_unregister_coprocessor_all(cs); ++} +diff --git a/target/arm/helper.c b/target/arm/helper.c +index 2746d3fdac..e47498828c 100644 +--- a/target/arm/helper.c ++++ b/target/arm/helper.c +@@ -263,6 +263,19 @@ void init_cpreg_list(ARMCPU *cpu) + g_list_free(keys); + } + ++void destroy_cpreg_list(ARMCPU *cpu) ++{ ++ assert(cpu->cpreg_indexes); ++ assert(cpu->cpreg_values); ++ assert(cpu->cpreg_vmstate_indexes); ++ assert(cpu->cpreg_vmstate_values); ++ ++ g_free(cpu->cpreg_indexes); ++ g_free(cpu->cpreg_values); ++ g_free(cpu->cpreg_vmstate_indexes); ++ g_free(cpu->cpreg_vmstate_values); ++} ++ + /* + * Some registers are not accessible from AArch32 EL3 if SCR.NS == 0. + */ +@@ -9438,6 +9451,18 @@ void register_cp_regs_for_features(ARMCPU *cpu) + #endif + } + ++void unregister_cp_regs_for_features(ARMCPU *cpu) ++{ ++ CPUARMState *env = &cpu->env; ++ if (arm_feature(env, ARM_FEATURE_M)) { ++ /* M profile has no coprocessor registers */ ++ return; ++ } ++ ++ /* empty it all. unregister all the coprocessor registers */ ++ g_hash_table_remove_all(cpu->cp_regs); ++} ++ + /* Sort alphabetically by type name, except for "any". */ + static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b) + { +diff --git a/target/arm/internals.h b/target/arm/internals.h +index 143d57c0fe..c3a7682f05 100644 +--- a/target/arm/internals.h ++++ b/target/arm/internals.h +@@ -187,9 +187,12 @@ void arm_cpu_register(const ARMCPUInfo *info); + void aarch64_cpu_register(const ARMCPUInfo *info); + + void register_cp_regs_for_features(ARMCPU *cpu); ++void unregister_cp_regs_for_features(ARMCPU *cpu); + void init_cpreg_list(ARMCPU *cpu); ++void destroy_cpreg_list(ARMCPU *cpu); + + void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu); ++void arm_cpu_unregister_gdb_regs(ARMCPU *cpu); + void arm_translate_init(void); + + void arm_restore_state_to_opc(CPUState *cs, +diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c +index 03ce1e7525..9c3a35d63a 100644 +--- a/target/arm/kvm64.c ++++ b/target/arm/kvm64.c +@@ -647,6 +647,10 @@ int kvm_arch_init_vcpu(CPUState *cs) + + int kvm_arch_destroy_vcpu(CPUState *cs) + { ++ if (cs->thread_id) { ++ qemu_del_vm_change_state_handler(cs->vmcse); ++ } ++ + return 0; + } + +-- +2.27.0 + diff --git a/target-arm-kvm-Write-CPU-state-back-to-KVM-on-reset.patch b/target-arm-kvm-Write-CPU-state-back-to-KVM-on-reset.patch new file mode 100644 index 0000000000000000000000000000000000000000..0b4922cc702a47bb45f2fa001c86719a8bb9c0c9 --- /dev/null +++ b/target-arm-kvm-Write-CPU-state-back-to-KVM-on-reset.patch @@ -0,0 +1,50 @@ +From a079801cd3ae6484cad6826f20bcf4ecc7e97ead Mon Sep 17 00:00:00 2001 +From: Jean-Philippe Brucker +Date: Wed, 5 May 2021 15:43:27 +0200 +Subject: [PATCH] target/arm/kvm: Write CPU state back to KVM on reset + +When a KVM vCPU is reset following a PSCI CPU_ON call, its power state +is not synchronized with KVM at the moment. Because the vCPU is not +marked dirty, we miss the call to kvm_arch_put_registers() that writes +to KVM's MP_STATE. Force mp_state synchronization. + +Signed-off-by: Jean-Philippe Brucker +Signed-off-by: Salil Mehta +--- + target/arm/kvm.c | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +diff --git a/target/arm/kvm.c b/target/arm/kvm.c +index 70cf15b550..aca652621f 100644 +--- a/target/arm/kvm.c ++++ b/target/arm/kvm.c +@@ -636,11 +636,12 @@ void kvm_arm_cpu_post_load(ARMCPU *cpu) + void kvm_arm_reset_vcpu(ARMCPU *cpu) + { + int ret; ++ CPUState *cs = CPU(cpu); + + /* Re-init VCPU so that all registers are set to + * their respective reset values. + */ +- ret = kvm_arm_vcpu_init(CPU(cpu)); ++ ret = kvm_arm_vcpu_init(cs); + if (ret < 0) { + fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret)); + abort(); +@@ -657,6 +658,12 @@ void kvm_arm_reset_vcpu(ARMCPU *cpu) + * for the same reason we do so in kvm_arch_get_registers(). + */ + write_list_to_cpustate(cpu); ++ ++ /* ++ * Ensure we call kvm_arch_put_registers(). The vCPU isn't marked dirty if ++ * it was parked in KVM and is now booting from a PSCI CPU_ON call. ++ */ ++ cs->vcpu_dirty = true; + } + + void kvm_arm_create_host_vcpu(ARMCPU *cpu) +-- +2.27.0 + diff --git a/target-arm-kvm-tcg-Register-Handle-SMCCC-hypercall-e.patch b/target-arm-kvm-tcg-Register-Handle-SMCCC-hypercall-e.patch new file mode 100644 index 0000000000000000000000000000000000000000..1b4bfbb2d6aa800e60bee7e96d429be813c6271c --- /dev/null +++ b/target-arm-kvm-tcg-Register-Handle-SMCCC-hypercall-e.patch @@ -0,0 +1,407 @@ +From 9c4a7c44c3c9e89c6aeab85b00c72a09a0c13940 Mon Sep 17 00:00:00 2001 +From: Author Salil Mehta +Date: Sat, 27 May 2023 22:13:13 +0200 +Subject: [PATCH] target/arm/kvm,tcg: Register/Handle SMCCC hypercall exits to + VMM/Qemu + +Add registration and Handling of HVC/SMC hypercall exits to VMM + +Co-developed-by: Salil Mehta +Signed-off-by: Salil Mehta +Co-developed-by: Jean-Philippe Brucker +Signed-off-by: Jean-Philippe Brucker +Signed-off-by: Salil Mehta +--- + target/arm/arm-powerctl.c | 51 +++++++++++++++++++++++++++++------- + target/arm/helper.c | 2 +- + target/arm/internals.h | 11 -------- + target/arm/kvm.c | 52 +++++++++++++++++++++++++++++++++++++ + target/arm/kvm64.c | 46 +++++++++++++++++++++++++++++--- + target/arm/kvm_arm.h | 13 ++++++++++ + target/arm/meson.build | 1 + + target/arm/{tcg => }/psci.c | 8 ++++++ + target/arm/tcg/meson.build | 4 --- + 9 files changed, 160 insertions(+), 28 deletions(-) + rename target/arm/{tcg => }/psci.c (97%) + +diff --git a/target/arm/arm-powerctl.c b/target/arm/arm-powerctl.c +index c078849403..fb19b04189 100644 +--- a/target/arm/arm-powerctl.c ++++ b/target/arm/arm-powerctl.c +@@ -16,6 +16,7 @@ + #include "qemu/log.h" + #include "qemu/main-loop.h" + #include "sysemu/tcg.h" ++#include "hw/boards.h" + + #ifndef DEBUG_ARM_POWERCTL + #define DEBUG_ARM_POWERCTL 0 +@@ -28,18 +29,37 @@ + } \ + } while (0) + ++static CPUArchId *arm_get_archid_by_id(uint64_t id) ++{ ++ int n; ++ CPUArchId *arch_id; ++ MachineState *ms = MACHINE(qdev_get_machine()); ++ ++ /* ++ * At this point disabled CPUs don't have a CPUState, but their CPUArchId ++ * exists. ++ * ++ * TODO: Is arch_id == mp_affinity? This needs work. ++ */ ++ for (n = 0; n < ms->possible_cpus->len; n++) { ++ arch_id = &ms->possible_cpus->cpus[n]; ++ ++ if (arch_id->arch_id == id) { ++ return arch_id; ++ } ++ } ++ return NULL; ++} ++ + CPUState *arm_get_cpu_by_id(uint64_t id) + { +- CPUState *cpu; ++ CPUArchId *arch_id; + + DPRINTF("cpu %" PRId64 "\n", id); + +- CPU_FOREACH(cpu) { +- ARMCPU *armcpu = ARM_CPU(cpu); +- +- if (armcpu->mp_affinity == id) { +- return cpu; +- } ++ arch_id = arm_get_archid_by_id(id); ++ if (arch_id && arch_id->cpu) { ++ return CPU(arch_id->cpu); + } + + qemu_log_mask(LOG_GUEST_ERROR, +@@ -97,6 +117,7 @@ int arm_set_cpu_on(uint64_t cpuid, uint64_t entry, uint64_t context_id, + { + CPUState *target_cpu_state; + ARMCPU *target_cpu; ++ CPUArchId *arch_id; + struct CpuOnInfo *info; + + assert(qemu_mutex_iothread_locked()); +@@ -117,12 +138,24 @@ int arm_set_cpu_on(uint64_t cpuid, uint64_t entry, uint64_t context_id, + } + + /* Retrieve the cpu we are powering up */ +- target_cpu_state = arm_get_cpu_by_id(cpuid); +- if (!target_cpu_state) { ++ arch_id = arm_get_archid_by_id(cpuid); ++ if (!arch_id) { + /* The cpu was not found */ + return QEMU_ARM_POWERCTL_INVALID_PARAM; + } + ++ target_cpu_state = CPU(arch_id->cpu); ++ if (!qemu_enabled_cpu(target_cpu_state)) { ++ /* ++ * The cpu is not plugged in or disabled. We should return appropriate ++ * value as introduced in DEN0022E PSCI 1.2 issue E ++ */ ++ qemu_log_mask(LOG_GUEST_ERROR, ++ "[ARM]%s: Denying attempt to online removed/disabled " ++ "CPU%" PRId64"\n", __func__, cpuid); ++ return QEMU_ARM_POWERCTL_IS_OFF; ++ } ++ + target_cpu = ARM_CPU(target_cpu_state); + if (target_cpu->power_state == PSCI_ON) { + qemu_log_mask(LOG_GUEST_ERROR, +diff --git a/target/arm/helper.c b/target/arm/helper.c +index e47498828c..793aa89cc6 100644 +--- a/target/arm/helper.c ++++ b/target/arm/helper.c +@@ -11346,7 +11346,7 @@ void arm_cpu_do_interrupt(CPUState *cs) + env->exception.syndrome); + } + +- if (tcg_enabled() && arm_is_psci_call(cpu, cs->exception_index)) { ++ if (arm_is_psci_call(cpu, cs->exception_index)) { + arm_handle_psci_call(cpu); + qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n"); + return; +diff --git a/target/arm/internals.h b/target/arm/internals.h +index c3a7682f05..20b9c1da38 100644 +--- a/target/arm/internals.h ++++ b/target/arm/internals.h +@@ -314,21 +314,10 @@ vaddr arm_adjust_watchpoint_address(CPUState *cs, vaddr addr, int len); + /* Callback function for when a watchpoint or breakpoint triggers. */ + void arm_debug_excp_handler(CPUState *cs); + +-#if defined(CONFIG_USER_ONLY) || !defined(CONFIG_TCG) +-static inline bool arm_is_psci_call(ARMCPU *cpu, int excp_type) +-{ +- return false; +-} +-static inline void arm_handle_psci_call(ARMCPU *cpu) +-{ +- g_assert_not_reached(); +-} +-#else + /* Return true if the r0/x0 value indicates that this SMC/HVC is a PSCI call. */ + bool arm_is_psci_call(ARMCPU *cpu, int excp_type); + /* Actually handle a PSCI call */ + void arm_handle_psci_call(ARMCPU *cpu); +-#endif + + /** + * arm_clear_exclusive: clear the exclusive monitor +diff --git a/target/arm/kvm.c b/target/arm/kvm.c +index aca652621f..66caf9e5e7 100644 +--- a/target/arm/kvm.c ++++ b/target/arm/kvm.c +@@ -260,6 +260,7 @@ int kvm_arch_get_default_type(MachineState *ms) + int kvm_arch_init(MachineState *ms, KVMState *s) + { + int ret = 0; ++ + /* For ARM interrupt delivery is always asynchronous, + * whether we are using an in-kernel VGIC or not. + */ +@@ -310,6 +311,22 @@ int kvm_arch_init(MachineState *ms, KVMState *s) + } + } + ++ /* ++ * To be able to handle PSCI CPU ON calls in QEMU, we need to install SMCCC ++ * filter in the Host KVM. This is required to support features like ++ * virtual CPU Hotplug on ARM platforms. ++ */ ++ if (kvm_arm_set_smccc_filter(PSCI_0_2_FN64_CPU_ON, ++ KVM_SMCCC_FILTER_FWD_TO_USER)) { ++ error_report("CPU On PSCI-to-user-space fwd filter install failed"); ++ abort(); ++ } ++ if (kvm_arm_set_smccc_filter(PSCI_0_2_FN_CPU_OFF, ++ KVM_SMCCC_FILTER_FWD_TO_USER)) { ++ error_report("CPU Off PSCI-to-user-space fwd filter install failed"); ++ abort(); ++ } ++ + kvm_arm_init_debug(s); + + return ret; +@@ -966,6 +983,38 @@ static int kvm_arm_handle_dabt_nisv(CPUState *cs, uint64_t esr_iss, + return -1; + } + ++static int kvm_arm_handle_hypercall(CPUState *cs, struct kvm_run *run) ++{ ++ ARMCPU *cpu = ARM_CPU(cs); ++ CPUARMState *env = &cpu->env; ++ ++ kvm_cpu_synchronize_state(cs); ++ ++ /* ++ * hard coding immediate to 0 as we dont expect non-zero value as of now ++ * This might change in future versions. Hence, KVM_GET_ONE_REG could be ++ * used in such cases but it must be enhanced then only synchronize will ++ * also fetch ESR_EL2 value. ++ */ ++ if (run->hypercall.flags == KVM_HYPERCALL_EXIT_SMC) { ++ cs->exception_index = EXCP_SMC; ++ env->exception.syndrome = syn_aa64_smc(0); ++ } else { ++ cs->exception_index = EXCP_HVC; ++ env->exception.syndrome = syn_aa64_hvc(0); ++ } ++ env->exception.target_el = 1; ++ qemu_mutex_lock_iothread(); ++ arm_cpu_do_interrupt(cs); ++ qemu_mutex_unlock_iothread(); ++ ++ /* ++ * For PSCI, exit the kvm_run loop and process the work. Especially ++ * important if this was a CPU_OFF command and we can't return to the guest. ++ */ ++ return EXCP_INTERRUPT; ++} ++ + int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) + { + int ret = 0; +@@ -981,6 +1030,9 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) + ret = kvm_arm_handle_dabt_nisv(cs, run->arm_nisv.esr_iss, + run->arm_nisv.fault_ipa); + break; ++ case KVM_EXIT_HYPERCALL: ++ ret = kvm_arm_handle_hypercall(cs, run); ++ break; + default: + qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n", + __func__, run->exit_reason); +diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c +index 9c3a35d63a..00b257bb4b 100644 +--- a/target/arm/kvm64.c ++++ b/target/arm/kvm64.c +@@ -111,6 +111,25 @@ bool kvm_arm_hw_debug_active(CPUState *cs) + return ((cur_hw_wps > 0) || (cur_hw_bps > 0)); + } + ++static bool kvm_arm_set_vm_attr(struct kvm_device_attr *attr, const char *name) ++{ ++ int err; ++ ++ err = kvm_vm_ioctl(kvm_state, KVM_HAS_DEVICE_ATTR, attr); ++ if (err != 0) { ++ error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err)); ++ return false; ++ } ++ ++ err = kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, attr); ++ if (err != 0) { ++ error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err)); ++ return false; ++ } ++ ++ return true; ++} ++ + static bool kvm_arm_set_device_attr(CPUState *cs, struct kvm_device_attr *attr, + const char *name) + { +@@ -181,6 +200,28 @@ void kvm_arm_pvtime_init(CPUState *cs, uint64_t ipa) + } + } + ++int kvm_arm_set_smccc_filter(uint64_t func, uint8_t faction) ++{ ++ struct kvm_smccc_filter filter = { ++ .base = func, ++ .nr_functions = 1, ++ .action = faction, ++ }; ++ struct kvm_device_attr attr = { ++ .group = KVM_ARM_VM_SMCCC_CTRL, ++ .attr = KVM_ARM_VM_SMCCC_FILTER, ++ .flags = 0, ++ .addr = (uintptr_t)&filter, ++ }; ++ ++ if (!kvm_arm_set_vm_attr(&attr, "SMCCC Filter")) { ++ error_report("failed to set SMCCC filter in KVM Host"); ++ return -1; ++ } ++ ++ return 0; ++} ++ + static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id) + { + uint64_t ret; +@@ -629,9 +670,8 @@ int kvm_arch_init_vcpu(CPUState *cs) + } + + /* +- * When KVM is in use, PSCI is emulated in-kernel and not by qemu. +- * Currently KVM has its own idea about MPIDR assignment, so we +- * override our defaults with what we get from KVM. ++ * KVM may emulate PSCI in-kernel. Currently KVM has its own idea about ++ * MPIDR assignment, so we override our defaults with what we get from KVM. + */ + ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr); + if (ret) { +diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h +index 31408499b3..bf4df54c96 100644 +--- a/target/arm/kvm_arm.h ++++ b/target/arm/kvm_arm.h +@@ -388,6 +388,15 @@ void kvm_arm_pvtime_init(CPUState *cs, uint64_t ipa); + + int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level); + ++/** ++ * kvm_arm_set_smccc_filter ++ * @func: funcion ++ * @faction: SMCCC filter action(handle, deny, fwd-to-user) to be deployed ++ * ++ * Sets the ARMs SMC-CC filter in KVM Host for selective hypercall exits ++ */ ++int kvm_arm_set_smccc_filter(uint64_t func, uint8_t faction); ++ + #else + + /* +@@ -462,6 +471,10 @@ static inline uint32_t kvm_arm_sve_get_vls(CPUState *cs) + g_assert_not_reached(); + } + ++static inline int kvm_arm_set_smccc_filter(uint64_t func, uint8_t faction) ++{ ++ g_assert_not_reached(); ++} + #endif + + /** +diff --git a/target/arm/meson.build b/target/arm/meson.build +index 5d04a8e94f..d1dd4932ed 100644 +--- a/target/arm/meson.build ++++ b/target/arm/meson.build +@@ -23,6 +23,7 @@ arm_system_ss.add(files( + 'arm-qmp-cmds.c', + 'cortex-regs.c', + 'machine.c', ++ 'psci.c', + 'ptw.c', + )) + +diff --git a/target/arm/tcg/psci.c b/target/arm/psci.c +similarity index 97% +rename from target/arm/tcg/psci.c +rename to target/arm/psci.c +index 6c1239bb96..a8690a16af 100644 +--- a/target/arm/tcg/psci.c ++++ b/target/arm/psci.c +@@ -21,7 +21,9 @@ + #include "exec/helper-proto.h" + #include "kvm-consts.h" + #include "qemu/main-loop.h" ++#include "qemu/error-report.h" + #include "sysemu/runstate.h" ++#include "sysemu/tcg.h" + #include "internals.h" + #include "arm-powerctl.h" + +@@ -157,6 +159,11 @@ void arm_handle_psci_call(ARMCPU *cpu) + case QEMU_PSCI_0_1_FN_CPU_SUSPEND: + case QEMU_PSCI_0_2_FN_CPU_SUSPEND: + case QEMU_PSCI_0_2_FN64_CPU_SUSPEND: ++ if (!tcg_enabled()) { ++ warn_report("CPU suspend not supported in non-tcg mode"); ++ break; ++ } ++#ifdef CONFIG_TCG + /* Affinity levels are not supported in QEMU */ + if (param[1] & 0xfffe0000) { + ret = QEMU_PSCI_RET_INVALID_PARAMS; +@@ -169,6 +176,7 @@ void arm_handle_psci_call(ARMCPU *cpu) + env->regs[0] = 0; + } + helper_wfi(env, 4); ++#endif + break; + case QEMU_PSCI_1_0_FN_PSCI_FEATURES: + switch (param[1]) { +diff --git a/target/arm/tcg/meson.build b/target/arm/tcg/meson.build +index 6fca38f2cc..ad3cfcb3bd 100644 +--- a/target/arm/tcg/meson.build ++++ b/target/arm/tcg/meson.build +@@ -51,7 +51,3 @@ arm_ss.add(when: 'TARGET_AARCH64', if_true: files( + 'sme_helper.c', + 'sve_helper.c', + )) +- +-arm_system_ss.add(files( +- 'psci.c', +-)) +-- +2.27.0 + diff --git a/tcg-mttcg-enable-threads-to-unregister-in-tcg_ctxs.patch b/tcg-mttcg-enable-threads-to-unregister-in-tcg_ctxs.patch new file mode 100644 index 0000000000000000000000000000000000000000..28e2cf523f3eabc362a585cc6ac48cef43396ed8 --- /dev/null +++ b/tcg-mttcg-enable-threads-to-unregister-in-tcg_ctxs.patch @@ -0,0 +1,98 @@ +From f797e2713a94b48de59324d00c851d89f4438fc0 Mon Sep 17 00:00:00 2001 +From: Miguel Luis +Date: Fri, 3 Feb 2023 12:33:41 -0100 +Subject: [PATCH] tcg/mttcg: enable threads to unregister in tcg_ctxs[] + +[This patch is just for reference. It has problems as it does not takes care of +the TranslationBlocks and their assigned regions during CPU unrealize] + +When using TCG acceleration in a multi-threaded context each vCPU has its own +thread registered in tcg_ctxs[] upon creation and tcg_cur_ctxs stores the current +number of threads that got created. Although, the lack of a mechanism to +unregister these threads is a problem when exercising vCPU hotplug/unplug +due to the fact that tcg_cur_ctxs gets incremented everytime a vCPU gets +hotplugged but never gets decremented everytime a vCPU gets unplugged, therefore +breaking the assert stating tcg_cur_ctxs < tcg_max_ctxs after a certain amount +of vCPU hotplugs. + +Suggested-by: Salil Mehta +[SM: Check Things To Do Section, https://lore.kernel.org/all/20200613213629.21984-1-salil.mehta@huawei.com/] +Signed-off-by: Miguel Luis +--- + accel/tcg/tcg-accel-ops-mttcg.c | 1 + + include/tcg/startup.h | 5 +++++ + tcg/tcg.c | 23 +++++++++++++++++++++++ + 3 files changed, 29 insertions(+) + +diff --git a/accel/tcg/tcg-accel-ops-mttcg.c b/accel/tcg/tcg-accel-ops-mttcg.c +index fac80095bb..73866990ce 100644 +--- a/accel/tcg/tcg-accel-ops-mttcg.c ++++ b/accel/tcg/tcg-accel-ops-mttcg.c +@@ -122,6 +122,7 @@ static void *mttcg_cpu_thread_fn(void *arg) + qemu_mutex_unlock_iothread(); + rcu_remove_force_rcu_notifier(&force_rcu.notifier); + rcu_unregister_thread(); ++ tcg_unregister_thread(); + return NULL; + } + +diff --git a/include/tcg/startup.h b/include/tcg/startup.h +index f71305765c..c6cb1d92a7 100644 +--- a/include/tcg/startup.h ++++ b/include/tcg/startup.h +@@ -45,6 +45,11 @@ void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus); + */ + void tcg_register_thread(void); + ++/** ++ * tcg_register_thread: Unregister this thread with the TCG runtime ++ */ ++void tcg_unregister_thread(void); ++ + /** + * tcg_prologue_init(): Generate the code for the TCG prologue + * +diff --git a/tcg/tcg.c b/tcg/tcg.c +index 896a36caeb..61fcf8597d 100644 +--- a/tcg/tcg.c ++++ b/tcg/tcg.c +@@ -764,6 +764,14 @@ static void alloc_tcg_plugin_context(TCGContext *s) + #endif + } + ++static void free_tcg_plugin_context(TCGContext *s) ++{ ++#ifdef CONFIG_PLUGIN ++ g_ptr_array_unref(s->plugin_tb->insns); ++ g_free(s->plugin_tb); ++#endif ++} ++ + /* + * All TCG threads except the parent (i.e. the one that called tcg_context_init + * and registered the target's TCG globals) must register with this function +@@ -814,6 +822,21 @@ void tcg_register_thread(void) + + tcg_ctx = s; + } ++ ++void tcg_unregister_thread(void) ++{ ++ TCGContext *s = tcg_ctx; ++ unsigned int n; ++ ++ /* Unclaim an entry in tcg_ctxs */ ++ n = qatomic_fetch_dec(&tcg_cur_ctxs); ++ g_assert(n > 1); ++ qatomic_store_release(&tcg_ctxs[n - 1], 0); ++ ++ free_tcg_plugin_context(s); ++ ++ g_free(s); ++} + #endif /* !CONFIG_USER_ONLY */ + + /* pool based memory allocation */ +-- +2.27.0 + diff --git a/tests-acpi-Update-expected-ACPI-tables-for-vcpu-hotp.patch b/tests-acpi-Update-expected-ACPI-tables-for-vcpu-hotp.patch new file mode 100644 index 0000000000000000000000000000000000000000..a6cd45895335b8bbfdcba149b7c30692b9afb0e7 --- /dev/null +++ b/tests-acpi-Update-expected-ACPI-tables-for-vcpu-hotp.patch @@ -0,0 +1,62 @@ +From cecec52ca38fa98a821c2a833e71a5fae1cc735d Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 2 Apr 2024 20:10:51 +0800 +Subject: [PATCH] tests/acpi: Update expected ACPI tables for vcpu hotplug + +Update the ACPI tables for vcpu hotplug. + +Signed-off-by: Keqian Zhu +--- + tests/qtest/bios-tables-test-allowed-diff.h | 40 ------------------ + 1 files changed, 40 deletions(-) + +diff --git a/tests/qtest/bios-tables-test-allowed-diff.h b/tests/qtest/bios-tables-test-allowed-diff.h +index c7406e395a..dfb8523c8b 100644 +--- a/tests/qtest/bios-tables-test-allowed-diff.h ++++ b/tests/qtest/bios-tables-test-allowed-diff.h +@@ -1,41 +1 @@ + /* List of comma-separated changed AML files to ignore */ +-"tests/data/acpi/pc/DSDT", +-"tests/data/acpi/pc/DSDT.acpierst", +-"tests/data/acpi/pc/DSDT.acpihmat", +-"tests/data/acpi/pc/DSDT.bridge", +-"tests/data/acpi/pc/DSDT.cphp", +-"tests/data/acpi/pc/DSDT.dimmpxm", +-"tests/data/acpi/pc/DSDT.hpbridge", +-"tests/data/acpi/pc/DSDT.hpbrroot", +-"tests/data/acpi/pc/DSDT.ipmikcs", +-"tests/data/acpi/pc/DSDT.memhp", +-"tests/data/acpi/pc/DSDT.nohpet", +-"tests/data/acpi/pc/DSDT.numamem", +-"tests/data/acpi/pc/DSDT.roothp", +-"tests/data/acpi/q35/DSDT", +-"tests/data/acpi/q35/DSDT.acpierst", +-"tests/data/acpi/q35/DSDT.acpihmat", +-"tests/data/acpi/q35/DSDT.acpihmat-noinitiator", +-"tests/data/acpi/q35/DSDT.applesmc", +-"tests/data/acpi/q35/DSDT.bridge", +-"tests/data/acpi/q35/DSDT.cphp", +-"tests/data/acpi/q35/DSDT.cxl", +-"tests/data/acpi/q35/DSDT.dimmpxm", +-"tests/data/acpi/q35/DSDT.ipmibt", +-"tests/data/acpi/q35/DSDT.ipmismbus", +-"tests/data/acpi/q35/DSDT.ivrs", +-"tests/data/acpi/q35/DSDT.memhp", +-"tests/data/acpi/q35/DSDT.mmio64", +-"tests/data/acpi/q35/DSDT.multi-bridge", +-"tests/data/acpi/q35/DSDT.noacpihp", +-"tests/data/acpi/q35/DSDT.nohpet", +-"tests/data/acpi/q35/DSDT.numamem", +-"tests/data/acpi/q35/DSDT.pvpanic-isa", +-"tests/data/acpi/q35/DSDT.tis.tpm12", +-"tests/data/acpi/q35/DSDT.tis.tpm2", +-"tests/data/acpi/q35/DSDT.viot", +-"tests/data/acpi/virt/DSDT", +-"tests/data/acpi/virt/DSDT.acpihmatvirt", +-"tests/data/acpi/virt/DSDT.memhp", +-"tests/data/acpi/virt/DSDT.pxb", +-"tests/data/acpi/virt/DSDT.topology", +\ No newline at end of file +-- +2.27.0 + diff --git a/tests-acpi-bios-tables-test-Allow-changes-to-virt-DS.patch b/tests-acpi-bios-tables-test-Allow-changes-to-virt-DS.patch new file mode 100644 index 0000000000000000000000000000000000000000..67d9c444d4f5069d6eeb2a4ee6f4e4cce374ea88 --- /dev/null +++ b/tests-acpi-bios-tables-test-Allow-changes-to-virt-DS.patch @@ -0,0 +1,62 @@ +From 6cfe9afcaceb7d9fb7d54f08b2362fc654b54d12 Mon Sep 17 00:00:00 2001 +From: Keqian Zhu +Date: Tue, 2 Apr 2024 17:23:18 +0800 +Subject: [PATCH] tests/acpi/bios-tables-test: Allow changes to virt/DSDT file + +Prepare to change of cpu aml. + +Signed-off-by: Keqian Zhu +--- + tests/qtest/bios-tables-test-allowed-diff.h | 40 +++++++++++++++++++++ + 1 file changed, 40 insertions(+) + +diff --git a/tests/qtest/bios-tables-test-allowed-diff.h b/tests/qtest/bios-tables-test-allowed-diff.h +index dfb8523c8b..c7406e395a 100644 +--- a/tests/qtest/bios-tables-test-allowed-diff.h ++++ b/tests/qtest/bios-tables-test-allowed-diff.h +@@ -1 +1,41 @@ + /* List of comma-separated changed AML files to ignore */ ++"tests/data/acpi/pc/DSDT", ++"tests/data/acpi/pc/DSDT.acpierst", ++"tests/data/acpi/pc/DSDT.acpihmat", ++"tests/data/acpi/pc/DSDT.bridge", ++"tests/data/acpi/pc/DSDT.cphp", ++"tests/data/acpi/pc/DSDT.dimmpxm", ++"tests/data/acpi/pc/DSDT.hpbridge", ++"tests/data/acpi/pc/DSDT.hpbrroot", ++"tests/data/acpi/pc/DSDT.ipmikcs", ++"tests/data/acpi/pc/DSDT.memhp", ++"tests/data/acpi/pc/DSDT.nohpet", ++"tests/data/acpi/pc/DSDT.numamem", ++"tests/data/acpi/pc/DSDT.roothp", ++"tests/data/acpi/q35/DSDT", ++"tests/data/acpi/q35/DSDT.acpierst", ++"tests/data/acpi/q35/DSDT.acpihmat", ++"tests/data/acpi/q35/DSDT.acpihmat-noinitiator", ++"tests/data/acpi/q35/DSDT.applesmc", ++"tests/data/acpi/q35/DSDT.bridge", ++"tests/data/acpi/q35/DSDT.cphp", ++"tests/data/acpi/q35/DSDT.cxl", ++"tests/data/acpi/q35/DSDT.dimmpxm", ++"tests/data/acpi/q35/DSDT.ipmibt", ++"tests/data/acpi/q35/DSDT.ipmismbus", ++"tests/data/acpi/q35/DSDT.ivrs", ++"tests/data/acpi/q35/DSDT.memhp", ++"tests/data/acpi/q35/DSDT.mmio64", ++"tests/data/acpi/q35/DSDT.multi-bridge", ++"tests/data/acpi/q35/DSDT.noacpihp", ++"tests/data/acpi/q35/DSDT.nohpet", ++"tests/data/acpi/q35/DSDT.numamem", ++"tests/data/acpi/q35/DSDT.pvpanic-isa", ++"tests/data/acpi/q35/DSDT.tis.tpm12", ++"tests/data/acpi/q35/DSDT.tis.tpm2", ++"tests/data/acpi/q35/DSDT.viot", ++"tests/data/acpi/virt/DSDT", ++"tests/data/acpi/virt/DSDT.acpihmatvirt", ++"tests/data/acpi/virt/DSDT.memhp", ++"tests/data/acpi/virt/DSDT.pxb", ++"tests/data/acpi/virt/DSDT.topology", +\ No newline at end of file +-- +2.27.0 +