From 7ce85cd0f4da207986cd48d97527e6f4c942d9a1 Mon Sep 17 00:00:00 2001 From: z00570632 Date: Thu, 19 Dec 2024 10:56:12 +0800 Subject: [PATCH 1/4] uefi support confidential vm --- hw/arm/boot.c | 38 ++++++++++++++++++++++++++++++++++++-- hw/arm/virt.c | 33 ++++++++++++++++++++++++++++++++- include/hw/arm/boot.h | 3 +++ 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/hw/arm/boot.c b/hw/arm/boot.c index 42110b0f18..d92ad13fd9 100644 --- a/hw/arm/boot.c +++ b/hw/arm/boot.c @@ -43,6 +43,9 @@ #define BOOTLOADER_MAX_SIZE (4 * KiB) +#define UEFI_MAX_SIZE 0x8000000 +#define UEFI_LOADER_START 0x0 +#define DTB_MAX 0x200000 AddressSpace *arm_boot_address_space(ARMCPU *cpu, const struct arm_boot_info *info) { @@ -1155,7 +1158,31 @@ static void arm_setup_direct_kernel_boot(ARMCPU *cpu, } } -static void arm_setup_firmware_boot(ARMCPU *cpu, struct arm_boot_info *info) +static void arm_setup_confidential_firmware_boot(ARMCPU *cpu, + struct arm_boot_info *info, + const char *firmware_filename) +{ + ssize_t fw_size; + const char *fname; + AddressSpace *as = arm_boot_address_space(cpu, info); + + fname = qemu_find_file(QEMU_FILE_TYPE_BIOS, firmware_filename); + if (!fname) { + error_report("Could not find firmware image '%s'", firmware_filename); + exit(1); + } + + fw_size = load_image_targphys_as(firmware_filename, + info->firmware_base, + info->firmware_max_size, as); + + if (fw_size <= 0) { + error_report("could not load firmware '%s'", firmware_filename); + exit(1); + } +} + +static void arm_setup_firmware_boot(ARMCPU *cpu, struct arm_boot_info *info, const char *firmware_filename) { /* Set up for booting firmware (which might load a kernel via fw_cfg) */ @@ -1166,6 +1193,8 @@ static void arm_setup_firmware_boot(ARMCPU *cpu, struct arm_boot_info *info) * DTB to the base of RAM for the bootloader to pick up. */ info->dtb_start = info->loader_start; + if (info->confidential) + tmm_add_ram_region(UEFI_LOADER_START, UEFI_MAX_SIZE, info->dtb_start, DTB_MAX , true); } if (info->kernel_filename) { @@ -1206,6 +1235,11 @@ static void arm_setup_firmware_boot(ARMCPU *cpu, struct arm_boot_info *info) } } + if (info->confidential) { + arm_setup_confidential_firmware_boot(cpu, info, firmware_filename); + kvm_load_user_data(UEFI_LOADER_START, UEFI_MAX_SIZE, info->loader_start, info->loader_start + DTB_MAX, info->ram_size, + (struct kvm_numa_info *)info->numa_info); + } /* * We will start from address 0 (typically a boot ROM image) in the * same way as hardware. Leave env->boot_info NULL, so that @@ -1282,7 +1316,7 @@ void arm_load_kernel(ARMCPU *cpu, MachineState *ms, struct arm_boot_info *info) /* Load the kernel. */ if (!info->kernel_filename || info->firmware_loaded) { - arm_setup_firmware_boot(cpu, info); + arm_setup_firmware_boot(cpu, info, ms->firmware); } else { arm_setup_direct_kernel_boot(cpu, info); } diff --git a/hw/arm/virt.c b/hw/arm/virt.c index 8823f2ed1c..6ffb26e7e6 100644 --- a/hw/arm/virt.c +++ b/hw/arm/virt.c @@ -1398,6 +1398,9 @@ static void virt_flash_map1(PFlashCFI01 *flash, qdev_prop_set_uint32(dev, "num-blocks", size / VIRT_FLASH_SECTOR_SIZE); sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal); + if (virtcca_cvm_enabled()) { + return; + } memory_region_add_subregion(sysmem, base, sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0)); @@ -1433,6 +1436,10 @@ static void virt_flash_fdt(VirtMachineState *vms, MachineState *ms = MACHINE(vms); char *nodename; + if (virtcca_cvm_enabled()) { + return; + } + if (sysmem == secure_sysmem) { /* Report both flash devices as a single node in the DT */ nodename = g_strdup_printf("/flash@%" PRIx64, flashbase); @@ -1468,6 +1475,23 @@ static void virt_flash_fdt(VirtMachineState *vms, } } +static bool virt_confidential_firmware_init(VirtMachineState *vms, + MemoryRegion *sysmem) +{ + MemoryRegion *fw_ram; + hwaddr fw_base = vms->memmap[VIRT_FLASH].base; + hwaddr fw_size = vms->memmap[VIRT_FLASH].size; + + if (!MACHINE(vms)->firmware) { + return false; + } + + fw_ram = g_new(MemoryRegion, 1); + memory_region_init_ram(fw_ram, NULL, "fw_ram", fw_size, NULL); + memory_region_add_subregion(sysmem, fw_base, fw_ram); + return true; +} + static bool virt_firmware_init(VirtMachineState *vms, MemoryRegion *sysmem, MemoryRegion *secure_sysmem) @@ -1486,6 +1510,10 @@ static bool virt_firmware_init(VirtMachineState *vms, pflash_blk0 = pflash_cfi01_get_blk(vms->flash[0]); + if (virtcca_cvm_enabled()) { + return virt_confidential_firmware_init(vms, sysmem); + } + bios_name = MACHINE(vms)->firmware; if (bios_name) { char *fname; @@ -2023,7 +2051,7 @@ static void virt_set_memmap(VirtMachineState *vms, int pa_bits) vms->memmap[VIRT_PCIE_MMIO] = (MemMapEntry) { 0x10000000, 0x2edf0000 }; vms->memmap[VIRT_KAE_DEVICE] = (MemMapEntry) { 0x3edf0000, 0x00200000 }; - vms->memmap[VIRT_MEM].base = 3 * GiB; + vms->memmap[VIRT_MEM].base = 1 * GiB; vms->memmap[VIRT_MEM].size = ms->ram_size; info_report("[qemu] fix VIRT_MEM range 0x%llx - 0x%llx\n", (unsigned long long)(vms->memmap[VIRT_MEM].base), (unsigned long long)(vms->memmap[VIRT_MEM].base + ms->ram_size)); @@ -2822,6 +2850,9 @@ static void machvirt_init(MachineState *machine) vms->bootinfo.get_dtb = machvirt_dtb; vms->bootinfo.skip_dtb_autoload = true; vms->bootinfo.firmware_loaded = firmware_loaded; + vms->bootinfo.firmware_base = vms->memmap[VIRT_FLASH].base; + vms->bootinfo.firmware_max_size = vms->memmap[VIRT_FLASH].size; + vms->bootinfo.confidential = virtcca_cvm_enabled(); vms->bootinfo.psci_conduit = vms->psci_conduit; arm_load_kernel(ARM_CPU(first_cpu), machine, &vms->bootinfo); diff --git a/include/hw/arm/boot.h b/include/hw/arm/boot.h index 4491b1f85b..06ca1d90b2 100644 --- a/include/hw/arm/boot.h +++ b/include/hw/arm/boot.h @@ -133,6 +133,9 @@ struct arm_boot_info { bool secure_board_setup; arm_endianness endianness; + hwaddr firmware_base; + hwaddr firmware_max_size; + bool confidential; }; /** -- Gitee From 66708311b7dd5d3ef344519d020e8112f37e0e17 Mon Sep 17 00:00:00 2001 From: z00570632 Date: Tue, 24 Dec 2024 21:00:59 +0800 Subject: [PATCH 2/4] enable iommu for virtio-net-pci --- hw/virtio/virtio-bus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c index 749df6478e..ede98ca377 100644 --- a/hw/virtio/virtio-bus.c +++ b/hw/virtio/virtio-bus.c @@ -90,7 +90,7 @@ void virtio_bus_device_plugged(VirtIODevice *vdev, Error **errp) if (has_iommu) { vdev_has_iommu = virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM); - if (virtcca_cvm_enabled() && (strcmp(vdev->name, "virtio-user-fs") == 0)) { + if (virtcca_cvm_enabled() && ((strcmp(vdev->name, "virtio-user-fs") == 0) || (strcmp(vdev->name, "virtio-net") == 0))) { vdev_has_iommu = true; } -- Gitee From e287a33b99d958eb72a6297afdd71db820254ea3 Mon Sep 17 00:00:00 2001 From: liupingwei Date: Sat, 8 Feb 2025 15:58:52 +0800 Subject: [PATCH 3/4] qemu uefi boot support kae Signed-off-by: liupingwei --- hw/arm/virt-acpi-build.c | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c index 076781423b..f78331d69f 100644 --- a/hw/arm/virt-acpi-build.c +++ b/hw/arm/virt-acpi-build.c @@ -58,6 +58,7 @@ #include "migration/vmstate.h" #include "hw/acpi/ghes.h" #include "hw/acpi/viot.h" +#include "kvm_arm.h" #define ARM_SPI_BASE 32 @@ -405,6 +406,54 @@ static void acpi_dsdt_add_virtio(Aml *scope, } } +static void acpi_dsdt_add_hisi_sec(Aml *scope, + const MemMapEntry *virtio_mmio_memmap, + int dev_id) +{ + hwaddr size = 0x10000; + + /* + * Calculate the base address for the sec device node. + * Each device group contains one sec device and one hpre device,spaced by 2 * size. + */ + hwaddr base = virtio_mmio_memmap->base + dev_id * 2 * size; + + Aml *dev = aml_device("SE%02u", dev_id); + aml_append(dev, aml_name_decl("_HID", aml_string("SEC07"))); + aml_append(dev, aml_name_decl("_UID", aml_int(dev_id))); + aml_append(dev, aml_name_decl("_CCA", aml_int(1))); + + Aml *crs = aml_resource_template(); + + aml_append(crs, aml_memory32_fixed(base, size, AML_READ_WRITE)); + aml_append(dev, aml_name_decl("_CRS", crs)); + aml_append(scope, dev); +} + +static void acpi_dsdt_add_hisi_hpre(Aml *scope, + const MemMapEntry *virtio_mmio_memmap, + int dev_id) +{ + hwaddr size = 0x10000; + + /* + * Calculate the base address for the hpre device node. + * Each hpre device follows the corresponding sec device by an additional offset of size. + */ + hwaddr base = virtio_mmio_memmap->base + dev_id * 2 * size + size; + + Aml *dev = aml_device("HP%02u", dev_id); + aml_append(dev, aml_name_decl("_HID", aml_string("HPRE07"))); + aml_append(dev, aml_name_decl("_UID", aml_int(dev_id))); + aml_append(dev, aml_name_decl("_CCA", aml_int(1))); + + Aml *crs = aml_resource_template(); + + aml_append(crs, aml_memory32_fixed(base, size, AML_READ_WRITE)); + aml_append(dev, aml_name_decl("_CRS", crs)); + aml_append(scope, dev); +} + static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, uint32_t irq, VirtMachineState *vms) { @@ -1201,6 +1250,15 @@ build_dsdt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms) acpi_dsdt_add_virtio(scope, &memmap[VIRT_MMIO], (irqmap[VIRT_MMIO] + ARM_SPI_BASE), NUM_VIRTIO_TRANSPORTS); acpi_dsdt_add_pci(scope, memmap, irqmap[VIRT_PCIE] + ARM_SPI_BASE, vms); + + if (virtcca_cvm_enabled()) { + int kae_num = tmm_get_kae_num(); + for (int i = 0; i < kae_num; i++) { + acpi_dsdt_add_hisi_sec(scope, &memmap[VIRT_KAE_DEVICE], i); + acpi_dsdt_add_hisi_hpre(scope, &memmap[VIRT_KAE_DEVICE], i); + } + } + if (vms->acpi_dev) { build_ged_aml(scope, "\\_SB."GED_DEVICE, HOTPLUG_HANDLER(vms->acpi_dev), -- Gitee From 5431d0374272b5cee9b5e46392ff933dcafeb458 Mon Sep 17 00:00:00 2001 From: nonce <2774337358@qq.com> Date: Thu, 23 Jan 2025 21:03:10 +0800 Subject: [PATCH 4/4] bakcend: VirtCCA:resolve hugepage memory waste issue in vhost-user scenario VirtCCA is based on SWIOTLB to implement virtio and will only allocate Bounce Buffer in the lower address range below 4GB. Therefore, the backend hugepages memory allocated above 4GB will not be used, resulting in significant waste. New address space and memory region are added to manage the backend hugepages memory corresponding to the GPA below 4GB, and there are shared with the vhostuser backend. Signed-off-by: nonce0_0 <2774337358@qq.com> --- backends/hostmem-file.c | 85 +++++++++++++++++++++++++++++++++++ hw/core/numa.c | 20 +++++++++ hw/virtio/vhost.c | 8 +++- include/exec/address-spaces.h | 3 ++ include/exec/cpu-common.h | 1 + include/exec/memory.h | 11 +++++ system/physmem.c | 17 +++++++ system/vl.c | 9 ++++ 8 files changed, 153 insertions(+), 1 deletion(-) diff --git a/backends/hostmem-file.c b/backends/hostmem-file.c index 361d4a8103..891fe4ac4a 100644 --- a/backends/hostmem-file.c +++ b/backends/hostmem-file.c @@ -20,9 +20,13 @@ #include "qom/object.h" #include "qapi/visitor.h" #include "qapi/qapi-visit-common.h" +#include "sysemu/kvm.h" +#include "exec/address-spaces.h" OBJECT_DECLARE_SIMPLE_TYPE(HostMemoryBackendFile, MEMORY_BACKEND_FILE) +bool virtcca_shared_hugepage_mapped = false; +uint64_t virtcca_cvm_ram_size = 0; struct HostMemoryBackendFile { HostMemoryBackend parent_obj; @@ -36,6 +40,83 @@ struct HostMemoryBackendFile { OnOffAuto rom; }; +/* Parse the path of the hugepages memory file used for memory sharing */ +static int virtcca_parse_share_mem_path(char *src, char *dst) +{ + int ret = 0; + char src_copy[PATH_MAX]; + char *token = NULL; + char *last_dir = NULL; + char *second_last_dir = NULL; + static const char delimiter[] = "/"; + + if (src == NULL || dst == NULL || + strlen(src) == 0 || strlen(src) > PATH_MAX - 1) { + error_report("Invalid input: NULL pointer or invalid string length."); + return -1; + } + + strcpy(src_copy, src); + token = strtok(src_copy, delimiter); + + /* Iterate over the path segments to find the second-to-last directory */ + while (token != NULL) { + second_last_dir = last_dir; + last_dir = token; + token = strtok(NULL, delimiter); + } + + /* Check if the second-to-last directory is found */ + if (second_last_dir == NULL) { + error_report("Invalid path: second-to-last directory not found."); + return -1; + } + + /* + * Construct the share memory path by appending the extracted domain name + * to the hugepages memory filesystem prefix + */ + ret = snprintf(dst, PATH_MAX, "/dev/hugepages/libvirt/qemu/%s", + second_last_dir); + + if (ret < 0 || ret >= PATH_MAX) { + error_report("Error: snprintf failed to construct the share mem path"); + return -1; + } + + return 0; +} + +/* + * Create a hugepage memory region in the virtcca scenario + * for sharing with process like vhost-user and others. + */ +static void +virtcca_shared_backend_memory_alloc(char *mem_path, uint32_t ram_flags, Error **errp) +{ + char dst[PATH_MAX]; + uint64_t size = virtcca_cvm_ram_size; + + if (virtcca_parse_share_mem_path(mem_path, dst)) { + error_report("parse virtcca share memory path failed"); + exit(1); + } + if (virtcca_cvm_ram_size >= VIRTCCA_SHARED_HUGEPAGE_MAX_SIZE) { + size = VIRTCCA_SHARED_HUGEPAGE_MAX_SIZE; + } + + virtcca_shared_hugepage = g_new(MemoryRegion, 1); + memory_region_init_ram_from_file(virtcca_shared_hugepage, NULL, + "virtcca_shared_hugepage", size, + VIRTCCA_SHARED_HUGEPAGE_ALIGN, + ram_flags, dst, 0, errp); + if (*errp) { + error_reportf_err(*errp, "cannot init RamBlock for virtcca_shared_hugepage: "); + exit(1); + } + virtcca_shared_hugepage_mapped = true; +} + static void file_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) { @@ -90,6 +171,10 @@ file_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) backend->size, fb->align, ram_flags, fb->mem_path, fb->offset, errp); g_free(name); + + if (virtcca_cvm_enabled() && backend->share && !virtcca_shared_hugepage_mapped) { + virtcca_shared_backend_memory_alloc(fb->mem_path, ram_flags, errp); + } #endif } diff --git a/hw/core/numa.c b/hw/core/numa.c index f08956ddb0..e7c48dab61 100644 --- a/hw/core/numa.c +++ b/hw/core/numa.c @@ -42,6 +42,8 @@ #include "qemu/option.h" #include "qemu/config-file.h" #include "qemu/cutils.h" +#include "exec/address-spaces.h" +#include "sysemu/kvm.h" QemuOptsList qemu_numa_opts = { .name = "numa", @@ -641,6 +643,21 @@ static void numa_init_memdev_container(MachineState *ms, MemoryRegion *ram) } } +/* + * Add virtcca_shared_hugepage as a sub-MR to the root MR of address space + * address_space_memory and address_space_virtcca_shared_memory. + */ +static void virtcca_shared_memory_configuration(MachineState *ms) +{ + MemoryRegion *alias_mr = g_new(MemoryRegion, 1); + + memory_region_add_subregion_overlap(ms->ram, 0, virtcca_shared_hugepage, 1); + memory_region_init_alias(alias_mr, NULL, "alias-mr", virtcca_shared_hugepage, + 0, int128_get64(virtcca_shared_hugepage->size)); + memory_region_add_subregion(address_space_virtcca_shared_memory.root, + VIRTCCA_GPA_START, alias_mr); +} + void numa_complete_configuration(MachineState *ms) { int i; @@ -711,6 +728,9 @@ void numa_complete_configuration(MachineState *ms) memory_region_init(ms->ram, OBJECT(ms), mc->default_ram_id, ms->ram_size); numa_init_memdev_container(ms, ms->ram); + if (virtcca_cvm_enabled() && virtcca_shared_hugepage->ram_block) { + virtcca_shared_memory_configuration(ms); + } } /* QEMU needs at least all unique node pair distances to build * the whole NUMA distance table. QEMU treats the distance table diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index d29075aa04..8b95558013 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -30,6 +30,7 @@ #include "sysemu/dma.h" #include "trace.h" #include "qapi/qapi-commands-migration.h" +#include "sysemu/kvm.h" /* enabled until disconnected backend stabilizes */ #define _VHOST_DEBUG 1 @@ -1616,7 +1617,12 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, hdev->log_size = 0; hdev->log_enabled = false; hdev->started = false; - memory_listener_register(&hdev->memory_listener, &address_space_memory); + if (virtcca_cvm_enabled()) { + memory_listener_register(&hdev->memory_listener, + &address_space_virtcca_shared_memory); + } else { + memory_listener_register(&hdev->memory_listener, &address_space_memory); + } QLIST_INSERT_HEAD(&vhost_devices, hdev, entry); /* diff --git a/include/exec/address-spaces.h b/include/exec/address-spaces.h index 0d0aa61d68..4518b5da86 100644 --- a/include/exec/address-spaces.h +++ b/include/exec/address-spaces.h @@ -33,6 +33,9 @@ MemoryRegion *get_system_io(void); extern AddressSpace address_space_memory; extern AddressSpace address_space_io; +extern AddressSpace address_space_virtcca_shared_memory; + +extern MemoryRegion *virtcca_shared_hugepage; #endif diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index c7fd30d5b9..d21d9990ad 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -28,6 +28,7 @@ typedef uint64_t vaddr; void cpu_exec_init_all(void); void cpu_exec_step_atomic(CPUState *cpu); +void virtcca_shared_memory_address_space_init(void); /* Using intptr_t ensures that qemu_*_page_mask is sign-extended even * when intptr_t is 32-bit and we are aligning a long long. diff --git a/include/exec/memory.h b/include/exec/memory.h index 542c9da918..33778f5c64 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -243,6 +243,17 @@ typedef struct IOMMUTLBEvent { /* RAM FD is opened read-only */ #define RAM_READONLY_FD (1 << 11) +/* The GPA range of the VirtCCA bounce buffer is from 1GB to 4GB. */ +#define VIRTCCA_SHARED_HUGEPAGE_MAX_SIZE 0xc0000000ULL + +/* The VirtCCA shared hugepage memory granularity is 1GB */ +#define VIRTCCA_SHARED_HUGEPAGE_ALIGN 0x40000000ULL + +/* The GPA starting address of the VirtCCA CVM is 1GB */ +#define VIRTCCA_GPA_START 0x40000000ULL + +extern uint64_t virtcca_cvm_ram_size; + static inline void iommu_notifier_init(IOMMUNotifier *n, IOMMUNotify fn, IOMMUNotifierFlag flags, hwaddr start, hwaddr end, diff --git a/system/physmem.c b/system/physmem.c index 250f315bc8..8f4be2d131 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -89,9 +89,17 @@ RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) }; static MemoryRegion *system_memory; static MemoryRegion *system_io; +static MemoryRegion *virtcca_shared_memory; + +/* + * Serves as the sub-MR of the root MR (virtcca_shared_memory) + * and is associated with the RAMBlock. + */ +MemoryRegion *virtcca_shared_hugepage; AddressSpace address_space_io; AddressSpace address_space_memory; +AddressSpace address_space_virtcca_shared_memory; static MemoryRegion io_mem_unassigned; @@ -2586,6 +2594,15 @@ static void memory_map_init(void) address_space_init(&address_space_io, system_io, "I/O"); } +void virtcca_shared_memory_address_space_init(void) +{ + virtcca_shared_memory = g_malloc(sizeof(*virtcca_shared_memory)); + memory_region_init(virtcca_shared_memory, NULL, + "virtcca_shared_memory", UINT64_MAX); + address_space_init(&address_space_virtcca_shared_memory, + virtcca_shared_memory, "virtcca_shared_memory"); +} + MemoryRegion *get_system_memory(void) { return system_memory; diff --git a/system/vl.c b/system/vl.c index a1e5e68773..7c10cd1337 100644 --- a/system/vl.c +++ b/system/vl.c @@ -3784,6 +3784,15 @@ void qemu_init(int argc, char **argv) configure_accelerators(argv[0]); phase_advance(PHASE_ACCEL_CREATED); + /* + * Must run after kvm_init completes, as virtcca_cvm_enabled() + * depends on initialization performed in kvm_init. + */ + if (virtcca_cvm_enabled()) { + virtcca_cvm_ram_size = current_machine->ram_size; + virtcca_shared_memory_address_space_init(); + } + /* * Beware, QOM objects created before this point miss global and * compat properties. -- Gitee