From e73fbf181f9d35c8c9ecfd6a65e801c0385c7feb Mon Sep 17 00:00:00 2001 From: Xin Jiang Date: Thu, 30 Nov 2023 10:08:59 +0800 Subject: [PATCH] Support Hygon CSV3 live migration Like CSV/CSV2 live migration, four new migration commands are added to support CSV3 migration. KVM_CSV_{SEND, RECEIVE}_ENCRYPT_DATA cmds are used to migrate guest's pages. KVM_CSV_{SEND, RECEIVE}_ENCRYPT_CONTEXT cmds are used to migration guest's runtime context. Signed-off-by: Xin Jiang --- ...-map-shared-region-for-CSV-virtual-m.patch | 384 +++++++++++++++ ...ders-update-kernel-headers-to-includ.patch | 79 +++ ...add-support-to-migrate-the-outgoing-.patch | 453 ++++++++++++++++++ ...add-support-to-migrate-the-incoming-.patch | 204 ++++++++ ...add-support-to-migrate-the-outgoing-.patch | 137 ++++++ ...add-support-to-migrate-the-incoming-.patch | 109 +++++ qemu.spec | 17 +- 7 files changed, 1382 insertions(+), 1 deletion(-) create mode 100644 0038-anolis-vfio-only-map-shared-region-for-CSV-virtual-m.patch create mode 100644 0039-anolis-linux-headers-update-kernel-headers-to-includ.patch create mode 100644 0040-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch create mode 100644 0041-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch create mode 100644 0042-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch create mode 100644 0043-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch diff --git a/0038-anolis-vfio-only-map-shared-region-for-CSV-virtual-m.patch b/0038-anolis-vfio-only-map-shared-region-for-CSV-virtual-m.patch new file mode 100644 index 0000000..5c739d2 --- /dev/null +++ b/0038-anolis-vfio-only-map-shared-region-for-CSV-virtual-m.patch @@ -0,0 +1,384 @@ +From ab826b2d84d45ab46b849a0d81b756a43b99f62a Mon Sep 17 00:00:00 2001 +From: liuyafei +Date: Mon, 22 May 2023 20:37:40 +0800 +Subject: [PATCH 38/43] anolis: vfio: only map shared region for CSV virtual + machine + +qemu vfio listener map/unmap all of the virtual machine's memory. +It does not work for CSV virtual machine, as only shared memory +should be accessed by device. + +Change-Id: I3f281c28166a36f2bed8ea193523715af9ff3271 +--- + hw/vfio/common.c | 40 +++++++++- + include/exec/memory.h | 11 +++ + softmmu/memory.c | 18 +++++ + target/i386/csv-sysemu-stub.c | 10 +++ + target/i386/csv.c | 134 ++++++++++++++++++++++++++++++++++ + target/i386/csv.h | 12 +++ + target/i386/kvm/kvm.c | 2 + + 7 files changed, 225 insertions(+), 2 deletions(-) + +diff --git a/hw/vfio/common.c b/hw/vfio/common.c +index 130e5d1dc7..e04473885f 100644 +--- a/hw/vfio/common.c ++++ b/hw/vfio/common.c +@@ -42,6 +42,9 @@ + #include "migration/migration.h" + #include "sysemu/tpm.h" + ++#include "target/i386/sev.h" ++#include "target/i386/csv.h" ++ + VFIOGroupList vfio_group_list = + QLIST_HEAD_INITIALIZER(vfio_group_list); + static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = +@@ -1228,6 +1231,30 @@ static void vfio_listener_log_global_stop(MemoryListener *listener) + vfio_set_dirty_page_tracking(container, false); + } + ++static SharedRegionListener *g_shl; ++static void shared_memory_listener_register(MemoryListener *listener, AddressSpace *as) ++{ ++ SharedRegionListener *shl; ++ ++ shl = g_new0(SharedRegionListener, 1); ++ ++ shl->listener = listener; ++ shl->as = as; ++ ++ shared_region_register_listener(shl); ++ g_shl = shl; ++} ++ ++static void shared_memory_listener_unregister(void) ++{ ++ SharedRegionListener *shl = g_shl; ++ ++ shared_region_unregister_listener(shl); ++ ++ g_free(shl); ++ g_shl = NULL; ++} ++ + static int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova, + uint64_t size, ram_addr_t ram_addr) + { +@@ -1430,7 +1457,12 @@ static const MemoryListener vfio_memory_listener = { + + static void vfio_listener_release(VFIOContainer *container) + { +- memory_listener_unregister(&container->listener); ++ if (csv_enabled()) { ++ shared_memory_listener_unregister(); ++ } else { ++ memory_listener_unregister(&container->listener); ++ } ++ + if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { + memory_listener_unregister(&container->prereg_listener); + } +@@ -2161,7 +2193,11 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, + + container->listener = vfio_memory_listener; + +- memory_listener_register(&container->listener, container->space->as); ++ if (csv_enabled()) { ++ shared_memory_listener_register(&container->listener, container->space->as); ++ } else { ++ memory_listener_register(&container->listener, container->space->as); ++ } + + if (container->error) { + ret = -1; +diff --git a/include/exec/memory.h b/include/exec/memory.h +index 91f8a2395a..b116e88a83 100644 +--- a/include/exec/memory.h ++++ b/include/exec/memory.h +@@ -717,6 +717,17 @@ bool memory_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, + ram_addr_t *ram_addr, bool *read_only, + bool *mr_has_discard_manager); + ++typedef struct SharedRegionListener SharedRegionListener; ++struct SharedRegionListener { ++ MemoryListener *listener; ++ AddressSpace *as; ++ QTAILQ_ENTRY(SharedRegionListener) next; ++}; ++ ++void shared_region_register_listener(SharedRegionListener *shl); ++void shared_region_unregister_listener(SharedRegionListener *shl); ++void *shared_region_listeners_get(void); ++ + typedef struct CoalescedMemoryRange CoalescedMemoryRange; + typedef struct MemoryRegionIoeventfd MemoryRegionIoeventfd; + +diff --git a/softmmu/memory.c b/softmmu/memory.c +index bc0be3f62c..39947c1ee8 100644 +--- a/softmmu/memory.c ++++ b/softmmu/memory.c +@@ -48,6 +48,9 @@ static QTAILQ_HEAD(, MemoryListener) memory_listeners + static QTAILQ_HEAD(, AddressSpace) address_spaces + = QTAILQ_HEAD_INITIALIZER(address_spaces); + ++static QTAILQ_HEAD(, SharedRegionListener) shared_region_listeners ++ = QTAILQ_HEAD_INITIALIZER(shared_region_listeners); ++ + static GHashTable *flat_views; + + typedef struct AddrRange AddrRange; +@@ -2193,6 +2196,21 @@ bool memory_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, + return true; + } + ++void shared_region_register_listener(SharedRegionListener *shl) ++{ ++ QTAILQ_INSERT_TAIL(&shared_region_listeners, shl, next); ++} ++ ++void shared_region_unregister_listener(SharedRegionListener *shl) ++{ ++ QTAILQ_REMOVE(&shared_region_listeners, shl, next); ++} ++ ++void *shared_region_listeners_get(void) ++{ ++ return &shared_region_listeners; ++} ++ + void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) + { + uint8_t mask = 1 << client; +diff --git a/target/i386/csv-sysemu-stub.c b/target/i386/csv-sysemu-stub.c +index a5ce986e3c..96ca055270 100644 +--- a/target/i386/csv-sysemu-stub.c ++++ b/target/i386/csv-sysemu-stub.c +@@ -29,3 +29,13 @@ int csv_launch_encrypt_vmcb(void) + { + g_assert_not_reached(); + } ++ ++int csv_shared_region_dma_map(uint64_t start, uint64_t end) ++{ ++ return 0; ++} ++ ++void csv_shared_region_dma_unmap(uint64_t start, uint64_t end) ++{ ++ ++} +diff --git a/target/i386/csv.c b/target/i386/csv.c +index 161cad39ae..6d05382fb2 100644 +--- a/target/i386/csv.c ++++ b/target/i386/csv.c +@@ -24,6 +24,7 @@ + #include "cpu.h" + #include "sev.h" + #include "csv.h" ++#include "exec/address-spaces.h" + + CsvGuestState csv_guest = { 0 }; + +@@ -63,6 +64,8 @@ csv_init(uint32_t policy, int fd, void *state, struct sev_ops *ops) + csv_guest.state = state; + csv_guest.sev_ioctl = ops->sev_ioctl; + csv_guest.fw_error_to_str = ops->fw_error_to_str; ++ QTAILQ_INIT(&csv_guest.dma_map_regions_list); ++ qemu_mutex_init(&csv_guest.dma_map_regions_list_mutex); + } + return 0; + } +@@ -163,3 +166,134 @@ csv_launch_encrypt_vmcb(void) + err: + return ret; + } ++ ++int csv_shared_region_dma_map(uint64_t start, uint64_t end) ++{ ++ MemoryRegionSection section; ++ AddressSpace *as; ++ QTAILQ_HEAD(, SharedRegionListener) *shared_region_listeners; ++ SharedRegionListener *shl; ++ MemoryListener *listener; ++ uint64_t size; ++ CsvGuestState *s = &csv_guest; ++ struct dma_map_region *region, *pos; ++ int ret = 0; ++ ++ if (!csv_enabled()) ++ return 0; ++ ++ if (end <= start) ++ return 0; ++ ++ shared_region_listeners = shared_region_listeners_get(); ++ if (QTAILQ_EMPTY(shared_region_listeners)) ++ return 0; ++ ++ size = end - start; ++ ++ qemu_mutex_lock(&s->dma_map_regions_list_mutex); ++ QTAILQ_FOREACH(pos, &s->dma_map_regions_list, list) { ++ if (start >= (pos->start + pos->size)) { ++ continue; ++ } else if ((start + size) <= pos->start) { ++ break; ++ } else { ++ goto end; ++ } ++ } ++ QTAILQ_FOREACH(shl, shared_region_listeners, next) { ++ listener = shl->listener; ++ as = shl->as; ++ section = memory_region_find(as->root, start, size); ++ if (!section.mr) { ++ goto end; ++ } ++ ++ if (!memory_region_is_ram(section.mr)) { ++ memory_region_unref(section.mr); ++ goto end; ++ } ++ ++ if (listener->region_add) { ++ listener->region_add(listener, §ion); ++ } ++ memory_region_unref(section.mr); ++ } ++ ++ region = g_malloc0(sizeof(*region)); ++ if (!region) { ++ ret = -1; ++ goto end; ++ } ++ region->start = start; ++ region->size = size; ++ ++ if (pos) { ++ QTAILQ_INSERT_BEFORE(pos, region, list); ++ } else { ++ QTAILQ_INSERT_TAIL(&s->dma_map_regions_list, region, list); ++ } ++ ++end: ++ qemu_mutex_unlock(&s->dma_map_regions_list_mutex); ++ return ret; ++} ++ ++void csv_shared_region_dma_unmap(uint64_t start, uint64_t end) ++{ ++ MemoryRegionSection section; ++ AddressSpace *as; ++ QTAILQ_HEAD(, SharedRegionListener) *shared_region_listeners; ++ SharedRegionListener *shl; ++ MemoryListener *listener; ++ uint64_t size; ++ CsvGuestState *s = &csv_guest; ++ struct dma_map_region *pos, *next_pos; ++ ++ if (!csv_enabled()) ++ return; ++ ++ if (end <= start) ++ return; ++ ++ shared_region_listeners = shared_region_listeners_get(); ++ if (QTAILQ_EMPTY(shared_region_listeners)) ++ return; ++ ++ size = end - start; ++ ++ qemu_mutex_lock(&s->dma_map_regions_list_mutex); ++ QTAILQ_FOREACH_SAFE(pos, &s->dma_map_regions_list, list, next_pos) { ++ uint64_t l, r; ++ uint64_t curr_end = pos->start + pos->size; ++ ++ l = MAX(start, pos->start); ++ r = MIN(start + size, pos->start + pos->size); ++ if (l < r) { ++ if ((start <= pos->start) && (start + size >= pos->start + pos->size)) { ++ QTAILQ_FOREACH(shl, shared_region_listeners, next) { ++ listener = shl->listener; ++ as = shl->as; ++ section = memory_region_find(as->root, pos->start, pos->size); ++ if (!section.mr) { ++ goto end; ++ } ++ if (listener->region_del) { ++ listener->region_del(listener, §ion); ++ } ++ memory_region_unref(section.mr); ++ } ++ ++ QTAILQ_REMOVE(&s->dma_map_regions_list, pos, list); ++ g_free(pos); ++ } ++ break; ++ } ++ if ((start + size) <= curr_end) { ++ break; ++ } ++ } ++end: ++ qemu_mutex_unlock(&s->dma_map_regions_list_mutex); ++ return; ++} +diff --git a/target/i386/csv.h b/target/i386/csv.h +index 986bffd95b..7eb85c46da 100644 +--- a/target/i386/csv.h ++++ b/target/i386/csv.h +@@ -15,6 +15,8 @@ + #define QEMU_CSV_H + + #include "qapi/qapi-commands-misc-target.h" ++#include "qemu/thread.h" ++#include "qemu/queue.h" + + #ifdef CONFIG_SEV + +@@ -55,12 +57,19 @@ bool csv_enabled(void); + #define csv_enabled() 0 + #endif + ++struct dma_map_region { ++ uint64_t start, size; ++ QTAILQ_ENTRY(dma_map_region) list; ++}; ++ + struct CsvGuestState { + uint32_t policy; + int sev_fd; + void *state; + int (*sev_ioctl)(int fd, int cmd, void *data, int *error); + const char *(*fw_error_to_str)(int code); ++ QTAILQ_HEAD(, dma_map_region) dma_map_regions_list; ++ QemuMutex dma_map_regions_list_mutex; + }; + + typedef struct CsvGuestState CsvGuestState; +@@ -71,4 +80,7 @@ extern int csv_launch_encrypt_vmcb(void); + + int csv_load_data(uint64_t gpa, uint8_t *ptr, uint64_t len, Error **errp); + ++int csv_shared_region_dma_map(uint64_t start, uint64_t end); ++void csv_shared_region_dma_unmap(uint64_t start, uint64_t end); ++ + #endif +diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c +index 8ec662e559..26458caab5 100644 +--- a/target/i386/kvm/kvm.c ++++ b/target/i386/kvm/kvm.c +@@ -5049,8 +5049,10 @@ static int kvm_handle_exit_hypercall(X86CPU *cpu, struct kvm_run *run) + + if (enc) { + sev_remove_shared_regions_list(gfn_start, gfn_end); ++ csv_shared_region_dma_unmap(gpa, gfn_end << TARGET_PAGE_BITS); + } else { + sev_add_shared_regions_list(gfn_start, gfn_end); ++ csv_shared_region_dma_map(gpa, gfn_end << TARGET_PAGE_BITS); + } + } + return 0; +-- +2.17.1 + diff --git a/0039-anolis-linux-headers-update-kernel-headers-to-includ.patch b/0039-anolis-linux-headers-update-kernel-headers-to-includ.patch new file mode 100644 index 0000000..b3b5fa9 --- /dev/null +++ b/0039-anolis-linux-headers-update-kernel-headers-to-includ.patch @@ -0,0 +1,79 @@ +From 146ec03e48fb001c3faa7e6f0e00a749cc513ec3 Mon Sep 17 00:00:00 2001 +From: jiangxin +Date: Fri, 17 Jun 2022 09:25:19 +0800 +Subject: [PATCH 39/43] anolis: linux-headers: update kernel headers to include + CSV migration cmds + +Four new migration commands are added to support CSV migration. + +KVM_CSV_SEND_ENCRYPT_DATA/KVM_CSV_RECEIVE_ENCRYPT_DATA cmds are +used to migrate guest's pages. + +KVM_CSV_SEND_ENCRYPT_CONTEXT/KVM_CSV_RECEIVE_ENCRYPT_CONTEXT cmds +are used to migration guest's runtime context. + +Change-Id: Ib3b733c7b5713aa6a6648c65e03cf8c9618ff1af +Signed-off-by: Xin Jiang +--- + linux-headers/linux/kvm.h | 38 ++++++++++++++++++++++++++++++++++++++ + 1 file changed, 38 insertions(+) + +diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h +index c6692b2f6a..534616f560 100644 +--- a/linux-headers/linux/kvm.h ++++ b/linux-headers/linux/kvm.h +@@ -2017,6 +2017,12 @@ enum csv_cmd_id { + KVM_CSV_INIT = KVM_CSV_NR_MIN, + KVM_CSV_LAUNCH_ENCRYPT_DATA, + KVM_CSV_LAUNCH_ENCRYPT_VMCB, ++ KVM_CSV_SEND_ENCRYPT_DATA, ++ KVM_CSV_SEND_ENCRYPT_CONTEXT, ++ KVM_CSV_RECEIVE_ENCRYPT_DATA, ++ KVM_CSV_RECEIVE_ENCRYPT_CONTEXT, ++ ++ KVM_CSV_NR_MAX, + }; + + struct kvm_csv_launch_encrypt_data { +@@ -2040,6 +2046,38 @@ struct kvm_csv_command_batch { + __u64 csv_batch_list_uaddr; + }; + ++struct kvm_csv_send_encrypt_data { ++ __u64 hdr_uaddr; ++ __u32 hdr_len; ++ __u64 guest_addr_data; ++ __u32 guest_addr_len; ++ __u64 trans_uaddr; ++ __u32 trans_len; ++}; ++ ++struct kvm_csv_send_encrypt_context { ++ __u64 hdr_uaddr; ++ __u32 hdr_len; ++ __u64 trans_uaddr; ++ __u32 trans_len; ++}; ++ ++struct kvm_csv_receive_encrypt_data { ++ __u64 hdr_uaddr; ++ __u32 hdr_len; ++ __u64 guest_addr_data; ++ __u32 guest_addr_len; ++ __u64 trans_uaddr; ++ __u32 trans_len; ++}; ++ ++struct kvm_csv_receive_encrypt_context { ++ __u64 hdr_uaddr; ++ __u32 hdr_len; ++ __u64 trans_uaddr; ++ __u32 trans_len; ++}; ++ + #define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0) + #define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1) + #define KVM_DEV_ASSIGN_MASK_INTX (1 << 2) +-- +2.17.1 + diff --git a/0040-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch b/0040-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch new file mode 100644 index 0000000..bed896a --- /dev/null +++ b/0040-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch @@ -0,0 +1,453 @@ +From 996dc2732e69321a6037d1fc4b13c11adf9f4a51 Mon Sep 17 00:00:00 2001 +From: jiangxin +Date: Fri, 17 Jun 2022 09:37:56 +0800 +Subject: [PATCH 40/43] anolis: csv/i386: add support to migrate the outgoing + page + +The csv_send_encrypt_data() provides the method to encrypt the +guest's private pages during migration. The routine is similar to +CSV2's. Usually, it starts with a SEND_START command to create the +migration context. Then SEND_ENCRYPT_DATA command is performed to +encrypt guest pages. After migration is completed, a SEND_FINISH +command is performed to the firmware. + +Change-Id: I804cf322ce756296752a0fa94c5a523a60a8539c +--- + migration/ram.c | 83 ++++++++++++++++++ + target/i386/csv.c | 185 +++++++++++++++++++++++++++++++++++++++ + target/i386/csv.h | 22 +++++ + target/i386/sev.c | 13 ++- + target/i386/sev.h | 1 + + target/i386/trace-events | 1 + + 6 files changed, 304 insertions(+), 1 deletion(-) + +diff --git a/migration/ram.c b/migration/ram.c +index 9586550dc6..3e78efe142 100644 +--- a/migration/ram.c ++++ b/migration/ram.c +@@ -2716,6 +2716,86 @@ ram_save_encrypted_pages_in_batch(RAMState *rs, PageSearchStatus *pss) + } + #endif + ++/** ++ * ram_save_csv_pages - send the given csv VM pages to the stream ++ */ ++static int ram_save_csv_pages(RAMState *rs, PageSearchStatus *pss, ++ bool last_stage) ++{ ++ int ret; ++ int tmppages = 0, pages = 0; ++ RAMBlock *block = pss->block; ++ ram_addr_t offset = 0; ++ hwaddr paddr = RAM_ADDR_INVALID; ++ uint32_t host_len = 0; ++ uint8_t *p; ++ uint64_t bytes_xmit; ++ MachineState *ms = MACHINE(qdev_get_machine()); ++ ConfidentialGuestSupportClass *cgs_class = ++ (ConfidentialGuestSupportClass *) object_get_class(OBJECT(ms->cgs)); ++ struct ConfidentialGuestMemoryEncryptionOps *ops = ++ cgs_class->memory_encryption_ops; ++ ++ if (!csv_enabled()) ++ return 0; ++ ++ do { ++ /* Check the pages is dirty and if it is send it */ ++ if (!migration_bitmap_clear_dirty(rs, block, pss->page)) { ++ pss->page++; ++ continue; ++ } ++ ++ ret = kvm_physical_memory_addr_from_host(kvm_state, ++ block->host + (pss->page << TARGET_PAGE_BITS), &paddr); ++ /* Process ROM or MMIO */ ++ if (paddr == RAM_ADDR_INVALID || memory_region_is_rom(block->mr)) ++ tmppages = ram_save_target_page(rs, pss); ++ else { ++ /* Caculate the offset and host virtual address of the page */ ++ offset = pss->page << TARGET_PAGE_BITS; ++ p = block->host + offset; ++ ++ if (ops->queue_outgoing_page(p, TARGET_PAGE_SIZE, offset)) ++ return -1; ++ ++ tmppages = 1; ++ host_len += TARGET_PAGE_SIZE; ++ ram_counters.normal++; ++ } ++ ++ if (tmppages < 0) { ++ return tmppages; ++ } ++ ++ pages += tmppages; ++ ++ pss->page++; ++ } while (offset_in_ramblock(block, pss->page << TARGET_PAGE_BITS) && ++ host_len < CSV3_OUTGOING_PAGE_WINDOW_SIZE); ++ ++ /* Check if there are any queued pages */ ++ if (host_len != 0) { ++ /* Always set offset as 0 for csv. */ ++ ram_counters.transferred += ++ save_page_header(rs, rs->f, block, 0 | RAM_SAVE_FLAG_ENCRYPTED_DATA); ++ ++ qemu_put_be32(rs->f, RAM_SAVE_ENCRYPTED_PAGE); ++ ram_counters.transferred += 4; ++ /* Process the queued pages in batch */ ++ ret = ops->save_queued_outgoing_pages(rs->f, &bytes_xmit); ++ if (ret) { ++ return -1; ++ } ++ ram_counters.transferred += bytes_xmit; ++ } ++ ++ /* The offset we leave with is the last one we looked at */ ++ pss->page--; ++ ++ return pages; ++} ++ + /** + * ram_save_host_page: save a whole host page + * +@@ -2751,6 +2831,9 @@ static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss) + postcopy_preempt_choose_channel(rs, pss); + } + ++ if (csv_enabled()) ++ return ram_save_csv_pages(rs, pss, rs->last_stage); ++ + #ifdef CONFIG_HYGON_CSV_MIG_ACCEL + /* + * If command_batch function is enabled and memory encryption is enabled +diff --git a/target/i386/csv.c b/target/i386/csv.c +index 6d05382fb2..c5a2bc9924 100644 +--- a/target/i386/csv.c ++++ b/target/i386/csv.c +@@ -13,6 +13,7 @@ + + #include "qemu/osdep.h" + ++#include + #include + #include "qapi/error.h" + +@@ -20,12 +21,30 @@ + #include + #endif + ++#include "migration/blocker.h" ++#include "migration/qemu-file.h" ++#include "migration/misc.h" ++#include "monitor/monitor.h" ++#include "sysemu/kvm.h" ++ + #include "trace.h" + #include "cpu.h" + #include "sev.h" + #include "csv.h" + #include "exec/address-spaces.h" + ++struct ConfidentialGuestMemoryEncryptionOps csv_memory_encryption_ops = { ++ .save_setup = sev_save_setup, ++ .save_outgoing_page = NULL, ++ .is_gfn_in_unshared_region = NULL, ++ .save_outgoing_shared_regions_list = sev_save_outgoing_shared_regions_list, ++ .load_incoming_shared_regions_list = sev_load_incoming_shared_regions_list, ++ .queue_outgoing_page = csv3_queue_outgoing_page, ++ .save_queued_outgoing_pages = csv3_save_queued_outgoing_pages, ++}; ++ ++#define CSV_OUTGOING_PAGE_NUM (CSV3_OUTGOING_PAGE_WINDOW_SIZE/TARGET_PAGE_SIZE) ++ + CsvGuestState csv_guest = { 0 }; + + #define GUEST_POLICY_CSV_BIT (1 << 6) +@@ -66,6 +85,7 @@ csv_init(uint32_t policy, int fd, void *state, struct sev_ops *ops) + csv_guest.fw_error_to_str = ops->fw_error_to_str; + QTAILQ_INIT(&csv_guest.dma_map_regions_list); + qemu_mutex_init(&csv_guest.dma_map_regions_list_mutex); ++ csv_guest.sev_send_start = ops->sev_send_start; + } + return 0; + } +@@ -297,3 +317,168 @@ end: + qemu_mutex_unlock(&s->dma_map_regions_list_mutex); + return; + } ++ ++static inline hwaddr csv_hva_to_gfn(uint8_t *ptr) ++{ ++ ram_addr_t offset = RAM_ADDR_INVALID; ++ ++ kvm_physical_memory_addr_from_host(kvm_state, ptr, &offset); ++ ++ return offset >> TARGET_PAGE_BITS; ++} ++ ++static int ++csv_send_start(QEMUFile *f, uint64_t *bytes_sent) ++{ ++ if (csv_guest.sev_send_start) ++ return csv_guest.sev_send_start(f, bytes_sent); ++ else ++ return -1; ++} ++ ++static int ++csv_send_get_packet_len(int *fw_err) ++{ ++ int ret; ++ struct kvm_csv_send_encrypt_data update = {0}; ++ ++ update.hdr_len = 0; ++ update.trans_len = 0; ++ ret = csv_ioctl(KVM_CSV_SEND_ENCRYPT_DATA, &update, fw_err); ++ if (*fw_err != SEV_RET_INVALID_LEN) { ++ error_report("%s: failed to get session length ret=%d fw_error=%d '%s'", ++ __func__, ret, *fw_err, fw_error_to_str(*fw_err)); ++ ret = 0; ++ goto err; ++ } ++ ++ if (update.hdr_len <= INT_MAX) ++ ret = update.hdr_len; ++ else ++ ret = 0; ++ ++err: ++ return ret; ++} ++ ++static int ++csv_send_encrypt_data(CsvGuestState *s, QEMUFile *f, uint8_t *ptr, uint32_t size, ++ uint64_t *bytes_sent) ++{ ++ int ret, fw_error = 0; ++ guchar *trans; ++ uint32_t guest_addr_entry_num; ++ uint32_t i; ++ struct kvm_csv_send_encrypt_data update = { }; ++ ++ /* ++ * If this is first call then query the packet header bytes and allocate ++ * the packet buffer. ++ */ ++ if (!s->send_packet_hdr) { ++ s->send_packet_hdr_len = csv_send_get_packet_len(&fw_error); ++ if (s->send_packet_hdr_len < 1) { ++ error_report("%s: SEND_UPDATE fw_error=%d '%s'", ++ __func__, fw_error, fw_error_to_str(fw_error)); ++ return 1; ++ } ++ ++ s->send_packet_hdr = g_new(gchar, s->send_packet_hdr_len); ++ } ++ ++ if (!s->guest_addr_len || !s->guest_addr_data) { ++ error_report("%s: invalid host address or size", __func__); ++ return 1; ++ } else { ++ guest_addr_entry_num = s->guest_addr_len / sizeof(struct guest_addr_entry); ++ } ++ ++ /* allocate transport buffer */ ++ trans = g_new(guchar, guest_addr_entry_num * TARGET_PAGE_SIZE); ++ ++ update.hdr_uaddr = (uintptr_t)s->send_packet_hdr; ++ update.hdr_len = s->send_packet_hdr_len; ++ update.guest_addr_data = (uintptr_t)s->guest_addr_data; ++ update.guest_addr_len = s->guest_addr_len; ++ update.trans_uaddr = (uintptr_t)trans; ++ update.trans_len = guest_addr_entry_num * TARGET_PAGE_SIZE; ++ ++ trace_kvm_csv_send_encrypt_data(trans, update.trans_len); ++ ++ ret = csv_ioctl(KVM_CSV_SEND_ENCRYPT_DATA, &update, &fw_error); ++ if (ret) { ++ error_report("%s: SEND_ENCRYPT_DATA ret=%d fw_error=%d '%s'", ++ __func__, ret, fw_error, fw_error_to_str(fw_error)); ++ goto err; ++ } ++ ++ for (i = 0; i < guest_addr_entry_num; i++) { ++ if (s->guest_addr_data[i].share) ++ memcpy(trans + i * TARGET_PAGE_SIZE, (guchar *)s->guest_hva_data[i].hva, ++ TARGET_PAGE_SIZE); ++ } ++ ++ qemu_put_be32(f, update.hdr_len); ++ qemu_put_buffer(f, (uint8_t *)update.hdr_uaddr, update.hdr_len); ++ *bytes_sent = 4 + update.hdr_len; ++ ++ qemu_put_be32(f, update.guest_addr_len); ++ qemu_put_buffer(f, (uint8_t *)update.guest_addr_data, update.guest_addr_len); ++ *bytes_sent = 4 + update.guest_addr_len; ++ ++ qemu_put_be32(f, update.trans_len); ++ qemu_put_buffer(f, (uint8_t *)update.trans_uaddr, update.trans_len); ++ *bytes_sent += (4 + update.trans_len); ++ ++err: ++ s->guest_addr_len = 0; ++ g_free(trans); ++ return ret; ++} ++ ++int ++csv3_queue_outgoing_page(uint8_t *ptr, uint32_t sz, uint64_t addr) ++{ ++ CsvGuestState *s = &csv_guest; ++ uint32_t i = 0; ++ ++ (void) addr; ++ ++ if (!s->guest_addr_data) { ++ s->guest_hva_data = g_new0(struct guest_hva_entry, CSV_OUTGOING_PAGE_NUM); ++ s->guest_addr_data = g_new0(struct guest_addr_entry, CSV_OUTGOING_PAGE_NUM); ++ s->guest_addr_len = 0; ++ } ++ ++ if (s->guest_addr_len >= sizeof(struct guest_addr_entry) * CSV_OUTGOING_PAGE_NUM) { ++ error_report("Failed to queue outgoing page"); ++ return 1; ++ } ++ ++ i = s->guest_addr_len / sizeof(struct guest_addr_entry); ++ s->guest_hva_data[i].hva = (uintptr_t)ptr; ++ s->guest_addr_data[i].share = 0; ++ s->guest_addr_data[i].reserved = 0; ++ s->guest_addr_data[i].gfn = csv_hva_to_gfn(ptr); ++ s->guest_addr_len += sizeof(struct guest_addr_entry); ++ ++ return 0; ++} ++ ++int ++csv3_save_queued_outgoing_pages(QEMUFile *f, uint64_t *bytes_sent) ++{ ++ CsvGuestState *s = &csv_guest; ++ ++ /* ++ * If this is a first buffer then create outgoing encryption context ++ * and write our PDH, policy and session data. ++ */ ++ if (!csv_check_state(SEV_STATE_SEND_UPDATE) && ++ csv_send_start(f, bytes_sent)) { ++ error_report("Failed to create outgoing context"); ++ return 1; ++ } ++ ++ return csv_send_encrypt_data(s, f, NULL, 0, bytes_sent); ++} +diff --git a/target/i386/csv.h b/target/i386/csv.h +index 7eb85c46da..cf7ba95760 100644 +--- a/target/i386/csv.h ++++ b/target/i386/csv.h +@@ -62,6 +62,18 @@ struct dma_map_region { + QTAILQ_ENTRY(dma_map_region) list; + }; + ++#define CSV3_OUTGOING_PAGE_WINDOW_SIZE (512 * TARGET_PAGE_SIZE) ++ ++struct guest_addr_entry { ++ uint64_t share: 1; ++ uint64_t reserved: 11; ++ uint64_t gfn: 52; ++}; ++ ++struct guest_hva_entry { ++ uint64_t hva; ++}; ++ + struct CsvGuestState { + uint32_t policy; + int sev_fd; +@@ -70,6 +82,13 @@ struct CsvGuestState { + const char *(*fw_error_to_str)(int code); + QTAILQ_HEAD(, dma_map_region) dma_map_regions_list; + QemuMutex dma_map_regions_list_mutex; ++ gchar *send_packet_hdr; ++ size_t send_packet_hdr_len; ++ struct guest_hva_entry *guest_hva_data; ++ struct guest_addr_entry *guest_addr_data; ++ size_t guest_addr_len; ++ ++ int (*sev_send_start)(QEMUFile *f, uint64_t *bytes_sent); + }; + + typedef struct CsvGuestState CsvGuestState; +@@ -77,10 +96,13 @@ typedef struct CsvGuestState CsvGuestState; + extern struct CsvGuestState csv_guest; + extern int csv_init(uint32_t policy, int fd, void *state, struct sev_ops *ops); + extern int csv_launch_encrypt_vmcb(void); ++extern struct ConfidentialGuestMemoryEncryptionOps csv_memory_encryption_ops; + + int csv_load_data(uint64_t gpa, uint8_t *ptr, uint64_t len, Error **errp); + + int csv_shared_region_dma_map(uint64_t start, uint64_t end); + void csv_shared_region_dma_unmap(uint64_t start, uint64_t end); ++int csv3_queue_outgoing_page(uint8_t *ptr, uint32_t sz, uint64_t addr); ++int csv3_save_queued_outgoing_pages(QEMUFile *f, uint64_t *bytes_sent); + + #endif +diff --git a/target/i386/sev.c b/target/i386/sev.c +index 65154abbee..334268c46e 100644 +--- a/target/i386/sev.c ++++ b/target/i386/sev.c +@@ -1213,7 +1213,10 @@ int sev_kvm_init(ConfidentialGuestSupport *cgs, Error **errp) + qemu_add_vm_change_state_handler(sev_vm_state_change, sev); + add_migration_state_change_notifier(&sev_migration_state_notify); + +- cgs_class->memory_encryption_ops = &sev_memory_encryption_ops; ++ if (csv_enabled()) ++ cgs_class->memory_encryption_ops = &csv_memory_encryption_ops; ++ else ++ cgs_class->memory_encryption_ops = &sev_memory_encryption_ops; + QTAILQ_INIT(&sev->shared_regions_list); + + /* Determine whether support MSR_AMD64_SEV_ES_GHCB */ +@@ -2592,9 +2595,17 @@ bool sev_add_kernel_loader_hashes(SevKernelLoaderContext *ctx, Error **errp) + return ret; + } + ++static int _sev_send_start(QEMUFile *f, uint64_t *bytes_sent) ++{ ++ SevGuestState *s = sev_guest; ++ ++ return sev_send_start(s, f, bytes_sent); ++} ++ + struct sev_ops sev_ops = { + .sev_ioctl = sev_ioctl, + .fw_error_to_str = fw_error_to_str, ++ .sev_send_start = _sev_send_start, + }; + + static void +diff --git a/target/i386/sev.h b/target/i386/sev.h +index f76f443f6f..6937475a6b 100644 +--- a/target/i386/sev.h ++++ b/target/i386/sev.h +@@ -89,6 +89,7 @@ int csv_load_incoming_cpu_state(QEMUFile *f); + struct sev_ops { + int (*sev_ioctl)(int fd, int cmd, void *data, int *error); + const char *(*fw_error_to_str)(int code); ++ int (*sev_send_start)(QEMUFile *f, uint64_t *bytes_sent); + }; + + #endif +diff --git a/target/i386/trace-events b/target/i386/trace-events +index 60a4609c0f..8db3e36385 100644 +--- a/target/i386/trace-events ++++ b/target/i386/trace-events +@@ -22,3 +22,4 @@ kvm_sev_receive_update_vmsa(uint32_t cpu_id, uint32_t cpu_index, void *src, int + + # csv.c + kvm_csv_launch_encrypt_data(uint64_t gpa, void *addr, uint64_t len) "gpa 0x%" PRIx64 "addr %p len 0x%" PRIu64 ++kvm_csv_send_encrypt_data(void *dst, int len) "trans %p len %d" +-- +2.17.1 + diff --git a/0041-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch b/0041-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch new file mode 100644 index 0000000..0c1c465 --- /dev/null +++ b/0041-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch @@ -0,0 +1,204 @@ +From b7f99a40c6c3a0045d0e92d05bf7ac3a47de6ee7 Mon Sep 17 00:00:00 2001 +From: jiangxin +Date: Fri, 17 Jun 2022 09:45:45 +0800 +Subject: [PATCH 41/43] anolis: csv/i386: add support to migrate the incoming + page + +The csv_receive_encrypt_data() provides the method to read incoming +guest private pages from socket and load them into guest memory. +The routine is similar to CSV2's. Usually, it starts with a RECEIVE +START command to create the migration context. Then RECEIVE ENCRYPT +DATA command is performed to let the firmware load incoming pages +into guest memory. After migration is completed, a RECEIVE FINISH +command is performed to the firmware. + +Change-Id: I53bc350e379fa747c7b3c3b3be9f2fef0bf1af9f +--- + target/i386/csv.c | 87 ++++++++++++++++++++++++++++++++++++++++ + target/i386/csv.h | 2 + + target/i386/sev.c | 8 ++++ + target/i386/sev.h | 1 + + target/i386/trace-events | 1 + + 5 files changed, 99 insertions(+) + +diff --git a/target/i386/csv.c b/target/i386/csv.c +index c5a2bc9924..00ff7d20a5 100644 +--- a/target/i386/csv.c ++++ b/target/i386/csv.c +@@ -36,11 +36,14 @@ + struct ConfidentialGuestMemoryEncryptionOps csv_memory_encryption_ops = { + .save_setup = sev_save_setup, + .save_outgoing_page = NULL, ++ .load_incoming_page = csv3_load_incoming_page, + .is_gfn_in_unshared_region = NULL, + .save_outgoing_shared_regions_list = sev_save_outgoing_shared_regions_list, + .load_incoming_shared_regions_list = sev_load_incoming_shared_regions_list, + .queue_outgoing_page = csv3_queue_outgoing_page, + .save_queued_outgoing_pages = csv3_save_queued_outgoing_pages, ++ .queue_incoming_page = NULL, ++ .load_queued_incoming_pages = NULL, + }; + + #define CSV_OUTGOING_PAGE_NUM (CSV3_OUTGOING_PAGE_WINDOW_SIZE/TARGET_PAGE_SIZE) +@@ -86,6 +89,7 @@ csv_init(uint32_t policy, int fd, void *state, struct sev_ops *ops) + QTAILQ_INIT(&csv_guest.dma_map_regions_list); + qemu_mutex_init(&csv_guest.dma_map_regions_list_mutex); + csv_guest.sev_send_start = ops->sev_send_start; ++ csv_guest.sev_receive_start = ops->sev_receive_start; + } + return 0; + } +@@ -482,3 +486,86 @@ csv3_save_queued_outgoing_pages(QEMUFile *f, uint64_t *bytes_sent) + + return csv_send_encrypt_data(s, f, NULL, 0, bytes_sent); + } ++ ++static int ++csv_receive_start(QEMUFile *f) ++{ ++ if (csv_guest.sev_receive_start) ++ return csv_guest.sev_receive_start(f); ++ else ++ return -1; ++} ++ ++static int csv_receive_encrypt_data(QEMUFile *f, uint8_t *ptr) ++{ ++ int ret = 1, fw_error = 0; ++ uint32_t i, guest_addr_entry_num; ++ gchar *hdr = NULL, *trans = NULL; ++ struct guest_addr_entry *guest_addr_data; ++ struct kvm_csv_receive_encrypt_data update = {}; ++ void *hva = NULL; ++ MemoryRegion *mr = NULL; ++ ++ /* get packet header */ ++ update.hdr_len = qemu_get_be32(f); ++ ++ hdr = g_new(gchar, update.hdr_len); ++ qemu_get_buffer(f, (uint8_t *)hdr, update.hdr_len); ++ update.hdr_uaddr = (uintptr_t)hdr; ++ ++ /* get guest addr data */ ++ update.guest_addr_len = qemu_get_be32(f); ++ ++ guest_addr_data = (struct guest_addr_entry *)g_new(gchar, update.guest_addr_len); ++ qemu_get_buffer(f, (uint8_t *)guest_addr_data, update.guest_addr_len); ++ update.guest_addr_data = (uintptr_t)guest_addr_data; ++ ++ /* get transport buffer */ ++ update.trans_len = qemu_get_be32(f); ++ ++ trans = g_new(gchar, update.trans_len); ++ update.trans_uaddr = (uintptr_t)trans; ++ qemu_get_buffer(f, (uint8_t *)update.trans_uaddr, update.trans_len); ++ ++ /* update share memory. */ ++ guest_addr_entry_num = update.guest_addr_len / sizeof(struct guest_addr_entry); ++ for (i = 0; i < guest_addr_entry_num; i++) { ++ if (guest_addr_data[i].share) { ++ hva = gpa2hva(&mr, ++ ((uint64_t)guest_addr_data[i].gfn << TARGET_PAGE_BITS), ++ TARGET_PAGE_SIZE, ++ NULL); ++ if (hva) ++ memcpy(hva, trans + i * TARGET_PAGE_SIZE, TARGET_PAGE_SIZE); ++ } ++ } ++ ++ trace_kvm_csv_receive_encrypt_data(trans, update.trans_len, hdr, update.hdr_len); ++ ++ ret = csv_ioctl(KVM_CSV_RECEIVE_ENCRYPT_DATA, &update, &fw_error); ++ if (ret) { ++ error_report("Error RECEIVE_ENCRYPT_DATA ret=%d fw_error=%d '%s'", ++ ret, fw_error, fw_error_to_str(fw_error)); ++ goto err; ++ } ++ ++err: ++ g_free(trans); ++ g_free(guest_addr_data); ++ g_free(hdr); ++ return ret; ++} ++ ++int csv3_load_incoming_page(QEMUFile *f, uint8_t *ptr) ++{ ++ /* ++ * If this is first buffer and SEV is not in recieiving state then ++ * use RECEIVE_START command to create a encryption context. ++ */ ++ if (!csv_check_state(SEV_STATE_RECEIVE_UPDATE) && ++ csv_receive_start(f)) { ++ return 1; ++ } ++ ++ return csv_receive_encrypt_data(f, ptr); ++} +diff --git a/target/i386/csv.h b/target/i386/csv.h +index cf7ba95760..da103ec320 100644 +--- a/target/i386/csv.h ++++ b/target/i386/csv.h +@@ -89,6 +89,7 @@ struct CsvGuestState { + size_t guest_addr_len; + + int (*sev_send_start)(QEMUFile *f, uint64_t *bytes_sent); ++ int (*sev_receive_start)(QEMUFile *f); + }; + + typedef struct CsvGuestState CsvGuestState; +@@ -102,6 +103,7 @@ int csv_load_data(uint64_t gpa, uint8_t *ptr, uint64_t len, Error **errp); + + int csv_shared_region_dma_map(uint64_t start, uint64_t end); + void csv_shared_region_dma_unmap(uint64_t start, uint64_t end); ++int csv3_load_incoming_page(QEMUFile *f, uint8_t *ptr); + int csv3_queue_outgoing_page(uint8_t *ptr, uint32_t sz, uint64_t addr); + int csv3_save_queued_outgoing_pages(QEMUFile *f, uint64_t *bytes_sent); + +diff --git a/target/i386/sev.c b/target/i386/sev.c +index 334268c46e..704f441c2a 100644 +--- a/target/i386/sev.c ++++ b/target/i386/sev.c +@@ -2602,10 +2602,18 @@ static int _sev_send_start(QEMUFile *f, uint64_t *bytes_sent) + return sev_send_start(s, f, bytes_sent); + } + ++static int _sev_receive_start(QEMUFile *f) ++{ ++ SevGuestState *s = sev_guest; ++ ++ return sev_receive_start(s, f); ++} ++ + struct sev_ops sev_ops = { + .sev_ioctl = sev_ioctl, + .fw_error_to_str = fw_error_to_str, + .sev_send_start = _sev_send_start, ++ .sev_receive_start = _sev_receive_start, + }; + + static void +diff --git a/target/i386/sev.h b/target/i386/sev.h +index 6937475a6b..4d875fc942 100644 +--- a/target/i386/sev.h ++++ b/target/i386/sev.h +@@ -90,6 +90,7 @@ struct sev_ops { + int (*sev_ioctl)(int fd, int cmd, void *data, int *error); + const char *(*fw_error_to_str)(int code); + int (*sev_send_start)(QEMUFile *f, uint64_t *bytes_sent); ++ int (*sev_receive_start)(QEMUFile *f); + }; + + #endif +diff --git a/target/i386/trace-events b/target/i386/trace-events +index 8db3e36385..1854356fc5 100644 +--- a/target/i386/trace-events ++++ b/target/i386/trace-events +@@ -23,3 +23,4 @@ kvm_sev_receive_update_vmsa(uint32_t cpu_id, uint32_t cpu_index, void *src, int + # csv.c + kvm_csv_launch_encrypt_data(uint64_t gpa, void *addr, uint64_t len) "gpa 0x%" PRIx64 "addr %p len 0x%" PRIu64 + kvm_csv_send_encrypt_data(void *dst, int len) "trans %p len %d" ++kvm_csv_receive_encrypt_data(void *dst, int len, void *hdr, int hdr_len) "trans %p len %d hdr %p hdr_len %d" +-- +2.17.1 + diff --git a/0042-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch b/0042-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch new file mode 100644 index 0000000..13971a1 --- /dev/null +++ b/0042-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch @@ -0,0 +1,137 @@ +From 9e7350dbe464ee6f8743cd4e078f9591b173596d Mon Sep 17 00:00:00 2001 +From: jiangxin +Date: Fri, 17 Jun 2022 09:52:31 +0800 +Subject: [PATCH 42/43] anolis: csv/i386: add support to migrate the outgoing + context + +CSV needs to migrate guest cpu's context pages. Prior to migration +of the context, it should query transfer buffer length and header +data length by SEND ENCRYPT CONTEXT command. New migration flag +RAM_SAVE_ENCRYPTED_CSV_CONTEXT is defined for CSV. + +Change-Id: I1989e76bd5c91c78074fb31eff075deacfa16078 +--- + target/i386/csv.c | 79 ++++++++++++++++++++++++++++++++++++++++ + target/i386/csv.h | 1 + + target/i386/trace-events | 1 + + 3 files changed, 81 insertions(+) + +diff --git a/target/i386/csv.c b/target/i386/csv.c +index 00ff7d20a5..271c48867e 100644 +--- a/target/i386/csv.c ++++ b/target/i386/csv.c +@@ -44,6 +44,7 @@ struct ConfidentialGuestMemoryEncryptionOps csv_memory_encryption_ops = { + .save_queued_outgoing_pages = csv3_save_queued_outgoing_pages, + .queue_incoming_page = NULL, + .load_queued_incoming_pages = NULL, ++ .save_outgoing_cpu_state = csv3_save_outgoing_context, + }; + + #define CSV_OUTGOING_PAGE_NUM (CSV3_OUTGOING_PAGE_WINDOW_SIZE/TARGET_PAGE_SIZE) +@@ -569,3 +570,81 @@ int csv3_load_incoming_page(QEMUFile *f, uint8_t *ptr) + + return csv_receive_encrypt_data(f, ptr); + } ++ ++static int ++csv_send_get_context_len(int *fw_err, int *context_len, int *hdr_len) ++{ ++ int ret = 0; ++ struct kvm_csv_send_encrypt_context update = {}; ++ ++ ret = csv_ioctl(KVM_CSV_SEND_ENCRYPT_CONTEXT, &update, fw_err); ++ if (*fw_err != SEV_RET_INVALID_LEN) { ++ error_report("%s: failed to get context length ret=%d fw_error=%d '%s'", ++ __func__, ret, *fw_err, fw_error_to_str(*fw_err)); ++ ret = -1; ++ goto err; ++ } ++ ++ if (update.trans_len <= INT_MAX && update.hdr_len <= INT_MAX) { ++ *context_len = update.trans_len; ++ *hdr_len = update.hdr_len; ++ } ++ ret = 0; ++err: ++ return ret; ++} ++ ++static int ++csv_send_encrypt_context(CsvGuestState *s, QEMUFile *f) ++{ ++ int ret, fw_error = 0; ++ int context_len = 0; ++ int hdr_len = 0; ++ guchar *trans; ++ guchar *hdr; ++ struct kvm_csv_send_encrypt_context update = { }; ++ ++ ret = csv_send_get_context_len(&fw_error, &context_len, &hdr_len); ++ if (context_len < 1 || hdr_len < 1) { ++ error_report("%s: fail to get context length fw_error=%d '%s'", ++ __func__, fw_error, fw_error_to_str(fw_error)); ++ return 1; ++ } ++ ++ /* allocate transport buffer */ ++ trans = g_new(guchar, context_len); ++ hdr = g_new(guchar, hdr_len); ++ ++ update.hdr_uaddr = (uintptr_t)hdr; ++ update.hdr_len = hdr_len; ++ update.trans_uaddr = (uintptr_t)trans; ++ update.trans_len = context_len; ++ ++ trace_kvm_csv_send_encrypt_context(trans, update.trans_len); ++ ++ ret = csv_ioctl(KVM_CSV_SEND_ENCRYPT_CONTEXT, &update, &fw_error); ++ if (ret) { ++ error_report("%s: SEND_ENCRYPT_CONTEXT ret=%d fw_error=%d '%s'", ++ __func__, ret, fw_error, fw_error_to_str(fw_error)); ++ goto err; ++ } ++ ++ qemu_put_be32(f, update.hdr_len); ++ qemu_put_buffer(f, (uint8_t *)update.hdr_uaddr, update.hdr_len); ++ ++ qemu_put_be32(f, update.trans_len); ++ qemu_put_buffer(f, (uint8_t *)update.trans_uaddr, update.trans_len); ++ ++err: ++ g_free(trans); ++ g_free(hdr); ++ return ret; ++} ++ ++int csv3_save_outgoing_context(QEMUFile *f) ++{ ++ CsvGuestState *s = &csv_guest; ++ ++ /* send csv context. */ ++ return csv_send_encrypt_context(s, f); ++} +diff --git a/target/i386/csv.h b/target/i386/csv.h +index da103ec320..a38ed62193 100644 +--- a/target/i386/csv.h ++++ b/target/i386/csv.h +@@ -103,6 +103,7 @@ int csv_load_data(uint64_t gpa, uint8_t *ptr, uint64_t len, Error **errp); + + int csv_shared_region_dma_map(uint64_t start, uint64_t end); + void csv_shared_region_dma_unmap(uint64_t start, uint64_t end); ++int csv3_save_outgoing_context(QEMUFile *f); + int csv3_load_incoming_page(QEMUFile *f, uint8_t *ptr); + int csv3_queue_outgoing_page(uint8_t *ptr, uint32_t sz, uint64_t addr); + int csv3_save_queued_outgoing_pages(QEMUFile *f, uint64_t *bytes_sent); +diff --git a/target/i386/trace-events b/target/i386/trace-events +index 1854356fc5..08a782ce15 100644 +--- a/target/i386/trace-events ++++ b/target/i386/trace-events +@@ -23,4 +23,5 @@ kvm_sev_receive_update_vmsa(uint32_t cpu_id, uint32_t cpu_index, void *src, int + # csv.c + kvm_csv_launch_encrypt_data(uint64_t gpa, void *addr, uint64_t len) "gpa 0x%" PRIx64 "addr %p len 0x%" PRIu64 + kvm_csv_send_encrypt_data(void *dst, int len) "trans %p len %d" ++kvm_csv_send_encrypt_context(void *dst, int len) "trans %p len %d" + kvm_csv_receive_encrypt_data(void *dst, int len, void *hdr, int hdr_len) "trans %p len %d hdr %p hdr_len %d" +-- +2.17.1 + diff --git a/0043-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch b/0043-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch new file mode 100644 index 0000000..3c36313 --- /dev/null +++ b/0043-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch @@ -0,0 +1,109 @@ +From 5cf8aeb3bdc5c4a0efcf5c6ca4c4cf60b6d7ef75 Mon Sep 17 00:00:00 2001 +From: jiangxin +Date: Fri, 17 Jun 2022 10:00:46 +0800 +Subject: [PATCH 43/43] anolis: csv/i386: add support to migrate the incoming + context + +The csv_load_incoming_context() provides the method to read incoming +guest's context from socket. It loads them into guest private memory. +This is the last step during migration and RECEIVE FINISH command is +performed by then to complete the whole migration. + +Change-Id: I7290e0e9527bb819eee5813038110d981908a880 +--- + target/i386/csv.c | 45 ++++++++++++++++++++++++++++++++++++++++ + target/i386/csv.h | 1 + + target/i386/trace-events | 1 + + 3 files changed, 47 insertions(+) + +diff --git a/target/i386/csv.c b/target/i386/csv.c +index 271c48867e..b0ca16980d 100644 +--- a/target/i386/csv.c ++++ b/target/i386/csv.c +@@ -45,6 +45,7 @@ struct ConfidentialGuestMemoryEncryptionOps csv_memory_encryption_ops = { + .queue_incoming_page = NULL, + .load_queued_incoming_pages = NULL, + .save_outgoing_cpu_state = csv3_save_outgoing_context, ++ .load_incoming_cpu_state = csv3_load_incoming_context, + }; + + #define CSV_OUTGOING_PAGE_NUM (CSV3_OUTGOING_PAGE_WINDOW_SIZE/TARGET_PAGE_SIZE) +@@ -641,6 +642,42 @@ err: + return ret; + } + ++static int ++csv_receive_encrypt_context(CsvGuestState *s, QEMUFile *f) ++{ ++ int ret = 1, fw_error = 0; ++ gchar *hdr = NULL, *trans = NULL; ++ struct kvm_csv_receive_encrypt_context update = {}; ++ ++ /* get packet header */ ++ update.hdr_len = qemu_get_be32(f); ++ ++ hdr = g_new(gchar, update.hdr_len); ++ qemu_get_buffer(f, (uint8_t *)hdr, update.hdr_len); ++ update.hdr_uaddr = (uintptr_t)hdr; ++ ++ /* get transport buffer */ ++ update.trans_len = qemu_get_be32(f); ++ ++ trans = g_new(gchar, update.trans_len); ++ update.trans_uaddr = (uintptr_t)trans; ++ qemu_get_buffer(f, (uint8_t *)update.trans_uaddr, update.trans_len); ++ ++ trace_kvm_csv_receive_encrypt_context(trans, update.trans_len, hdr, update.hdr_len); ++ ++ ret = csv_ioctl(KVM_CSV_RECEIVE_ENCRYPT_CONTEXT, &update, &fw_error); ++ if (ret) { ++ error_report("Error RECEIVE_ENCRYPT_CONTEXT ret=%d fw_error=%d '%s'", ++ ret, fw_error, fw_error_to_str(fw_error)); ++ goto err; ++ } ++ ++err: ++ g_free(trans); ++ g_free(hdr); ++ return ret; ++} ++ + int csv3_save_outgoing_context(QEMUFile *f) + { + CsvGuestState *s = &csv_guest; +@@ -648,3 +685,11 @@ int csv3_save_outgoing_context(QEMUFile *f) + /* send csv context. */ + return csv_send_encrypt_context(s, f); + } ++ ++int csv3_load_incoming_context(QEMUFile *f) ++{ ++ CsvGuestState *s = &csv_guest; ++ ++ /* receive csv context. */ ++ return csv_receive_encrypt_context(s, f); ++} +diff --git a/target/i386/csv.h b/target/i386/csv.h +index a38ed62193..f4aac5fb2a 100644 +--- a/target/i386/csv.h ++++ b/target/i386/csv.h +@@ -104,6 +104,7 @@ int csv_load_data(uint64_t gpa, uint8_t *ptr, uint64_t len, Error **errp); + int csv_shared_region_dma_map(uint64_t start, uint64_t end); + void csv_shared_region_dma_unmap(uint64_t start, uint64_t end); + int csv3_save_outgoing_context(QEMUFile *f); ++int csv3_load_incoming_context(QEMUFile *f); + int csv3_load_incoming_page(QEMUFile *f, uint8_t *ptr); + int csv3_queue_outgoing_page(uint8_t *ptr, uint32_t sz, uint64_t addr); + int csv3_save_queued_outgoing_pages(QEMUFile *f, uint64_t *bytes_sent); +diff --git a/target/i386/trace-events b/target/i386/trace-events +index 08a782ce15..47ab390de6 100644 +--- a/target/i386/trace-events ++++ b/target/i386/trace-events +@@ -25,3 +25,4 @@ kvm_csv_launch_encrypt_data(uint64_t gpa, void *addr, uint64_t len) "gpa 0x%" PR + kvm_csv_send_encrypt_data(void *dst, int len) "trans %p len %d" + kvm_csv_send_encrypt_context(void *dst, int len) "trans %p len %d" + kvm_csv_receive_encrypt_data(void *dst, int len, void *hdr, int hdr_len) "trans %p len %d hdr %p hdr_len %d" ++kvm_csv_receive_encrypt_context(void *dst, int len, void *hdr, int hdr_len) "trans %p len %d hdr %p hdr_len %d" +-- +2.17.1 + diff --git a/qemu.spec b/qemu.spec index 4486b0f..accee61 100644 --- a/qemu.spec +++ b/qemu.spec @@ -1,4 +1,4 @@ -%define anolis_release 4 +%define anolis_release 5 %bcond_with check @@ -300,6 +300,12 @@ Patch0034: 0034-anolis-target-i386-csv-Add-support-for-migrate-VMSA-.patch Patch0035: 0035-anolis-target-i386-get-set-migrate-GHCB-state.patch Patch0036: 0036-anolis-target-i386-kvm-Return-resettable-when-emulat.patch Patch0037: 0037-anolis-kvm-Add-support-for-CSV2-reboot.patch +Patch0038: 0038-anolis-vfio-only-map-shared-region-for-CSV-virtual-m.patch +Patch0039: 0039-anolis-linux-headers-update-kernel-headers-to-includ.patch +Patch0040: 0040-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch +Patch0041: 0041-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch +Patch0042: 0042-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch +Patch0043: 0043-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch ExclusiveArch: x86_64 aarch64 @@ -1868,6 +1874,15 @@ useradd -r -u 107 -g qemu -G kvm -d / -s /sbin/nologin \ %endif %changelog +* Thu Nov 30 2023 Xin Jiang - 15:7.2.6-5 +- Patch0038: 0038-anolis-vfio-only-map-shared-region-for-CSV-virtual-m.patch +- Patch0039: 0039-anolis-linux-headers-update-kernel-headers-to-includ.patch +- Patch0040: 0040-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch +- Patch0041: 0041-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch +- Patch0042: 0042-anolis-csv-i386-add-support-to-migrate-the-outgoing-.patch +- Patch0043: 0043-anolis-csv-i386-add-support-to-migrate-the-incoming-.patch + (Support CSV3 live migration) + * Tue Nov 22 2023 Liyang Han - 15:7.2.6-4 - Patch0010: 0010-doc-update-AMD-SEV-to-include-Live-migration-flow.patch - Patch0011: 0011-migration.json-add-AMD-SEV-specific-migration-parame.patch -- Gitee