From cae8803432f61a4b05bf08b4589b735a3fe401cd Mon Sep 17 00:00:00 2001 From: lihuhua Date: Thu, 30 Jun 2022 16:11:00 +0800 Subject: [PATCH] support vm live migration with vfio devices --- cpus.c | 9 +- hw/pci/pci.c | 5 + hw/pci/pcie.c | 33 ++ hw/vfio/Makefile.objs | 2 +- hw/vfio/common.c | 418 ++++++++++++++- hw/vfio/migration.c | 926 ++++++++++++++++++++++++++++++++++ hw/vfio/pci.c | 137 +++-- hw/vfio/pci.h | 2 +- include/hw/pci/pci.h | 3 + include/hw/vfio/vfio-common.h | 42 ++ linux-headers/linux/vfio.h | 70 +++ memory.c | 15 +- migration/migration.c | 13 + migration/ram.c | 12 + monitor/hmp-cmds.c | 4 + qapi/migration.json | 10 +- 16 files changed, 1640 insertions(+), 61 deletions(-) create mode 100644 hw/vfio/migration.c diff --git a/cpus.c b/cpus.c index ef441bdf62..0c5ee50bfa 100644 --- a/cpus.c +++ b/cpus.c @@ -60,6 +60,8 @@ #include "hw/boards.h" #include "hw/hw.h" +#include "hw/vfio/vfio-common.h" + #ifdef CONFIG_LINUX #include @@ -1024,12 +1026,14 @@ void cpu_synchronize_all_pre_loadvm(void) static int do_vm_stop(RunState state, bool send_stop) { int ret = 0; + int rret = 0; if (runstate_is_running()) { runstate_set(state); cpu_disable_ticks(); pause_all_vcpus(); vm_state_notify(0, state); + rret = set_all_vfio_device(false); if (send_stop) { qapi_event_send_stop(); } @@ -1038,7 +1042,7 @@ static int do_vm_stop(RunState state, bool send_stop) bdrv_drain_all(); ret = bdrv_flush_all(); - return ret; + return rret ? rret : ret; } /* Special vm_stop() variant for terminating the process. Historically clients @@ -2146,7 +2150,8 @@ int vm_prepare_start(void) cpu_enable_ticks(); runstate_set(RUN_STATE_RUNNING); vm_state_notify(1, RUN_STATE_RUNNING); - return 0; + + return set_all_vfio_device(true); } void vm_start(void) diff --git a/hw/pci/pci.c b/hw/pci/pci.c index b5bc842fac..20f2713b84 100644 --- a/hw/pci/pci.c +++ b/hw/pci/pci.c @@ -978,6 +978,10 @@ static PCIReqIDCache pci_req_id_cache_get(PCIDevice *dev) uint16_t pci_requester_id(PCIDevice *dev) { + if (dev->device_bdf > 0 && dev->device_bdf <= UINT16_MAX) { + return dev->device_bdf & UINT16_MAX; + } + return pci_req_id_cache_extract(&dev->requester_id_cache); } @@ -1047,6 +1051,7 @@ static PCIDevice *do_pci_register_device(PCIDevice *pci_dev, pci_dev->devfn = devfn; pci_dev->requester_id_cache = pci_req_id_cache_get(pci_dev); + pci_dev->device_bdf = UINT16_MAX + 1; pstrcpy(pci_dev->name, sizeof(pci_dev->name), name); memory_region_init(&pci_dev->bus_master_container_region, OBJECT(pci_dev), diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c index 0eb3a2a5d2..80ef4ce7d6 100644 --- a/hw/pci/pcie.c +++ b/hw/pci/pcie.c @@ -28,6 +28,8 @@ #include "hw/pci/pcie_regs.h" #include "hw/pci/pcie_port.h" #include "qemu/range.h" +#include "hw/vfio/pci.h" +#include "hw/vfio/vfio-common.h" //#define DEBUG_PCIE #ifdef DEBUG_PCIE @@ -39,6 +41,8 @@ #define PCIE_DEV_PRINTF(dev, fmt, ...) \ PCIE_DPRINTF("%s:%x "fmt, (dev)->name, (dev)->devfn, ## __VA_ARGS__) +#define TYPE_VFIO_PCI "vfio-pci" +#define PCI_VFIO(obj) OBJECT_CHECK(VFIOPCIDevice, obj, TYPE_VFIO_PCI) /*************************************************************************** * pci express capability helper functions @@ -416,6 +420,7 @@ void pcie_cap_slot_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, PCIDevice *hotplug_pdev = PCI_DEVICE(hotplug_dev); uint8_t *exp_cap = hotplug_pdev->config + hotplug_pdev->exp.exp_cap; PCIDevice *pci_dev = PCI_DEVICE(dev); + DeviceClass *dc = DEVICE_GET_CLASS(pci_dev); /* Don't send event when device is enabled during qemu machine creation: * it is present on boot, no hotplug event is necessary. We do send an @@ -444,6 +449,19 @@ void pcie_cap_slot_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, pcie_cap_slot_event(PCI_DEVICE(hotplug_dev), PCI_EXP_HP_EV_PDC | PCI_EXP_HP_EV_ABP); } + + /* + * When device is a migratable vfio device, + * we need to notify vfio device to change device state: running + */ + if (strcmp(dc->desc, "VFIO-based PCI device assignment") == 0) { + VFIOPCIDevice *vdev = PCI_VFIO(pci_dev); + VFIODevice *vbasedev = &vdev->vbasedev; + if (vfio_device_enable(vbasedev)) { + error_setg_errno(errp, EBUSY, + "Enable vfio device %s failed.", vbasedev->name); + } + } } void pcie_cap_slot_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev, @@ -470,6 +488,7 @@ void pcie_cap_slot_unplug_request_cb(HotplugHandler *hotplug_dev, Error *local_err = NULL; PCIDevice *pci_dev = PCI_DEVICE(dev); PCIBus *bus = pci_get_bus(pci_dev); + DeviceClass *dc = DEVICE_GET_CLASS(pci_dev); pcie_cap_slot_plug_common(PCI_DEVICE(hotplug_dev), dev, &local_err); if (local_err) { @@ -477,6 +496,20 @@ void pcie_cap_slot_unplug_request_cb(HotplugHandler *hotplug_dev, return; } + /* + * Before unplug a vfio device, + * we need to notify vfio device to stop and release migration resource. + */ + if (strcmp(dc->desc, "VFIO-based PCI device assignment") == 0) { + VFIOPCIDevice *vdev = PCI_VFIO(pci_dev); + VFIODevice *vbasedev = &vdev->vbasedev; + if (vfio_device_disable(vbasedev)) { + error_setg_errno(errp, EBUSY, + "Disable vfio device %s failed.", vbasedev->name); + return; + } + } + dev->pending_deleted_event = true; /* In case user cancel the operation of multi-function hot-add, diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs index 9bb1c09e84..8b296c889e 100644 --- a/hw/vfio/Makefile.objs +++ b/hw/vfio/Makefile.objs @@ -1,4 +1,4 @@ -obj-y += common.o spapr.o +obj-y += common.o spapr.o migration.o obj-$(CONFIG_VFIO_PCI) += pci.o pci-quirks.o display.o obj-$(CONFIG_VFIO_CCW) += ccw.o obj-$(CONFIG_VFIO_PLATFORM) += platform.o diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 0b3593b3c0..8190baa855 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -27,8 +27,10 @@ #include "hw/vfio/vfio-common.h" #include "hw/vfio/vfio.h" +#include "hw/boards.h" #include "exec/address-spaces.h" #include "exec/memory.h" +#include "exec/ram_addr.h" #include "hw/hw.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" @@ -38,12 +40,20 @@ #include "sysemu/reset.h" #include "trace.h" #include "qapi/error.h" +#include "migration/migration.h" +#include "migration/qemu-file.h" + + VFIOGroupList vfio_group_list = QLIST_HEAD_INITIALIZER(vfio_group_list); static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = QLIST_HEAD_INITIALIZER(vfio_address_spaces); +#define LOG_BUF_DEBUG +#define LOG_BUF_PAGE_OFFSET 12 /* 4K each byte of the log buf */ +#define LOG_BUF_FRAG_SIZE (2 * 1024 * 1024) /* fix to 2M */ + #ifdef CONFIG_KVM /* * We have a single VFIO pseudo device per KVM VM. Once created it lives @@ -813,11 +823,348 @@ static void vfio_listener_region_del(MemoryListener *listener, } } +static int vfio_notify_device(const VFIOContainer *container, unsigned int flags) +{ + int ret = -1; + struct vfio_log_buf_ctl log_buf_ctl = { + .argsz = 0, + .data = NULL, + .flags = flags + }; + + if (container->log_buf.fd <= 0) { + error_report("vfio: error log buf fd invalid"); + return ret; + } + + ret = ioctl(container->log_buf.fd, VFIO_LOG_BUF_CTL, &log_buf_ctl); + if (ret != 0) { + error_report("vfio: error log buf control flags %d failed ret %d", + flags, ret); + return ret; + } + + return 0; +} + +static int vfio_notify_device_log_buf_stop(const VFIOContainer *container) +{ + return vfio_notify_device(container, VFIO_DEVICE_LOG_BUF_FLAG_STOP); +} + +static int vfio_notify_device_log_buf_start(const VFIOContainer *container) +{ + return vfio_notify_device(container, VFIO_DEVICE_LOG_BUF_FLAG_START); +} + +static __u32 vfio_generate_uuid(void) +{ + return (__u32)getpid(); +} + +static void vfio_munmap_log_buf(VFIOContainer *container) +{ + int i, ret; + struct vfio_log_buf_info *log_buf_info = &container->log_buf.info; + + for (i = 0; i < log_buf_info->addrs_size; i++) { + if (log_buf_info->sgevec[i].addr != 0) { + ret = munmap((void *)(log_buf_info->sgevec[i].addr), + log_buf_info->frag_size); + if (ret != 0) { + error_report("vfio: error munmap log buf failed ret %d", ret); + } + memset(&(log_buf_info->sgevec[i]), 0, + sizeof(struct vfio_log_buf_sge)); + } + } +} + +static int vfio_release_log_buf(VFIOContainer *container) +{ + int ret; + + vfio_munmap_log_buf(container); + + ret = vfio_notify_device(container, VFIO_DEVICE_LOG_BUF_FLAG_RELEASE); + if (ret != 0) { + error_report("vfio: error release log buf failed"); + } + + if (container->log_buf.info.sgevec != NULL) { + free(container->log_buf.info.sgevec); + container->log_buf.info.sgevec = NULL; + } + + if (container->log_buf.fd > 0) { + close(container->log_buf.fd); + container->log_buf.fd = 0; + } + + return ret; +} + +static int vfio_map_log_buf(VFIOLogBuf *log_buf) +{ + int i; + + for (i = 0; i < log_buf->info.addrs_size; i++) { + log_buf->info.sgevec[i].addr = (uint64_t)mmap(NULL, + log_buf->info.frag_size, + PROT_READ|PROT_WRITE, + MAP_SHARED, + log_buf->fd, + i * log_buf->info.frag_size); + if (log_buf->info.sgevec[i].addr == 0) { + error_report("vfio: error mmap log buf failed"); + return -1; + } + log_buf->info.sgevec[i].len = log_buf->info.frag_size; + } + + return 0; +} + +static int vfio_driver_setup_log_buf(int fd, + VFIOLogBuf *log_buf, + ram_addr_t log_buf_size) +{ + int ret; + struct vfio_log_buf_info log_buf_info; + + log_buf_info.uuid = vfio_generate_uuid(); + log_buf_info.frag_size = LOG_BUF_FRAG_SIZE; + __u32 addrs_size = log_buf_size / log_buf_info.frag_size; + if (log_buf_size % log_buf_info.frag_size != 0) { + addrs_size++; + } + log_buf_info.addrs_size = addrs_size; + log_buf_info.buffer_size = log_buf_info.addrs_size * log_buf_info.frag_size; + + struct vfio_log_buf_ctl log_buf_ctl; + log_buf_ctl.argsz = sizeof(log_buf_info); + log_buf_ctl.data = &log_buf_info; + log_buf_ctl.flags = VFIO_DEVICE_LOG_BUF_FLAG_SETUP; + + ret = ioctl(fd, VFIO_LOG_BUF_CTL, &log_buf_ctl); + if (ret != 0) { + error_report("vfio: error kernel set up log buf failed"); + return ret; + } + + log_buf->info.uuid = log_buf_info.uuid; + log_buf->info.frag_size = log_buf_info.frag_size; + log_buf->info.addrs_size = log_buf_info.addrs_size; + + return 0; +} + +static hwaddr last_ram_addr(void) +{ + MemoryRegion *root = address_space_memory.root; + MemoryRegion *sub_mr = NULL; + hwaddr last; + + QTAILQ_FOREACH(sub_mr, &root->subregions, subregions_link) { + if (!strcmp(sub_mr->name, "mach-virt.ram")) { + break; + } + } + + if (!sub_mr) { + /* assume there is a memory region which named 'mach-virt.ram', + * otherwith we should check why failed! + */ + error_report("can't find the 'mach-virt.ram' memory region."); + abort(); + } + + last = sub_mr->addr + int128_get64(sub_mr->size); + if (last < sub_mr->addr) { + error_report("the ram range beyond the UINT64_MAX."); + abort(); + } + + return last; +} + +static int vfio_setup_log_buf(VFIOContainer *container) +{ + VFIOLogBuf *log_buf = &container->log_buf; + int ret; + + if (log_buf->info.sgevec != NULL) { + warn_report("vfio: warning log buf already setup"); + return 0; + } + + log_buf->fd = ioctl(container->fd, VFIO_GET_LOG_BUF_FD, NULL); + if (log_buf->fd <= 0) { + error_report("vfio: error get log buf fd failed"); + return log_buf->fd; + } + + ram_addr_t log_buf_size = last_ram_addr() >> LOG_BUF_PAGE_OFFSET; + ret = vfio_driver_setup_log_buf(log_buf->fd, log_buf, log_buf_size); + if (ret != 0) { + goto close_fd_exit; + } + + log_buf->info.sgevec = (struct vfio_log_buf_sge *)malloc( \ + log_buf->info.addrs_size * sizeof(struct vfio_log_buf_sge)); + if (log_buf->info.sgevec == NULL) { + error_report("vfio: error out of memory"); + goto release_log_buf_exit; + } + + ret = vfio_map_log_buf(log_buf); + if (ret != 0) { + goto free_log_buf_exit; + } + + return 0; + +free_log_buf_exit: + vfio_munmap_log_buf(container); + free(log_buf->info.sgevec); + log_buf->info.sgevec = NULL; + +release_log_buf_exit: + ret = vfio_notify_device(container, VFIO_DEVICE_LOG_BUF_FLAG_RELEASE); + if (ret != 0) { + error_report("vfio: error release log buf failed"); + } + +close_fd_exit: + close(log_buf->fd); + log_buf->fd = 0; + + return ret; +} + +static void vfio_listener_log_global_start(MemoryListener *listener) +{ + VFIOContainer *container = container_of(listener, VFIOContainer, listener); + int ret; + + ret = vfio_setup_log_buf(container); + if (ret != 0) { + error_report("vfio: error set up log buf failed"); + qemu_file_set_error(migrate_get_current()->to_dst_file, ret); + return; + } + + ret = vfio_notify_device_log_buf_start(container); + if (ret != 0) { + error_report("vfio: error kernel start log buf failed"); + qemu_file_set_error(migrate_get_current()->to_dst_file, ret); + return; + } +} + +static ram_addr_t target_phys_addr_to_ram_addr(hwaddr target_phys_addr, + uint64_t size) +{ + MemoryRegion *root = address_space_memory.root; + MemoryRegionSection mrs; + hwaddr mr_gpa = 0; + ram_addr_t ram_addr = ~0UL; + + mrs = memory_region_find(root, target_phys_addr, size); + if (!mrs.mr) { + error_report("cant't the related memory region for " + "target_phys_addr: 0x%lx", target_phys_addr); + return ram_addr; + } + + if (!mrs.mr->ram) { + error_report("%s is't a ram memory region", mrs.mr->name); + goto exit; + } + + root = mrs.mr; + do { + mr_gpa += root->addr; + root = root->container; + } while (root); + + ram_addr = target_phys_addr - mr_gpa + mrs.mr->ram_block->offset; + +exit: + memory_region_unref(mrs.mr); + + return ram_addr; +} + +static void vfio_listener_log_sync_full(MemoryListener *listener, + MemoryRegionSection *section) +{ + VFIOContainer *container = container_of(listener, VFIOContainer, listener); + unsigned long i, j; + hwaddr target_phys_addr; + ram_addr_t ram_addr; + uint8_t *val; + int ret; + + ret = vfio_notify_device(container, VFIO_DEVICE_LOG_BUF_FLAG_STATUS_QUERY); + if (ret != 0) { + error_report("vfio: failed to query dirty log status from NIC"); + qemu_file_set_error(migrate_get_current()->to_dst_file, ret); + return; + } + + for (i = 0; i < container->log_buf.info.addrs_size; i++) { + for (j = 0; j < container->log_buf.info.frag_size; j++) { + val = (uint8_t *)container->log_buf.info.sgevec[i].addr + j; + if (*val == 0xff) { + target_phys_addr = (i * container->log_buf.info.frag_size + j) << \ + LOG_BUF_PAGE_OFFSET; + ram_addr = target_phys_addr_to_ram_addr(target_phys_addr, + 1 << LOG_BUF_PAGE_OFFSET); + if (ram_addr == ~0UL) + continue; + cpu_physical_memory_set_dirty_range(ram_addr, + 1 << LOG_BUF_PAGE_OFFSET, + 1 << DIRTY_MEMORY_MIGRATION); + *val = 0; + } + } + } +} + +static void vfio_listener_log_global_stop(MemoryListener *listener) +{ + VFIOContainer *container = container_of(listener, VFIOContainer, listener); + int ret; + + ret = vfio_notify_device_log_buf_stop(container); + if (ret != 0) { + error_report("vfio: error kernel device stop log buf error"); + } + + ret = vfio_release_log_buf(container); + if (ret != 0) { + error_report("vfio: error release log buf error"); + } +} + static const MemoryListener vfio_memory_listener = { .region_add = vfio_listener_region_add, .region_del = vfio_listener_region_del, + .log_global_start = vfio_listener_log_global_start, + .log_sync = vfio_listener_log_sync_full, + .log_global_stop = vfio_listener_log_global_stop, }; +bool vfio_memory_listener_check(MemoryListener * listener) +{ + if (!listener || !listener->log_sync) { + return false; + } + + return listener->log_sync == vfio_listener_log_sync_full; +} + static void vfio_listener_release(VFIOContainer *container) { memory_listener_unregister(&container->listener); @@ -925,6 +1272,14 @@ int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region, return 0; } +static void vfio_subregion_unmap(VFIORegion *region, int index) +{ + memory_region_del_subregion(region->mem, ®ion->mmaps[index].mem); + munmap(region->mmaps[index].mmap, region->mmaps[index].size); + object_unparent(OBJECT(®ion->mmaps[index].mem)); + region->mmaps[index].mmap = NULL; +} + int vfio_region_mmap(VFIORegion *region) { int i, prot = 0; @@ -955,10 +1310,7 @@ int vfio_region_mmap(VFIORegion *region) region->mmaps[i].mmap = NULL; for (i--; i >= 0; i--) { - memory_region_del_subregion(region->mem, ®ion->mmaps[i].mem); - munmap(region->mmaps[i].mmap, region->mmaps[i].size); - object_unparent(OBJECT(®ion->mmaps[i].mem)); - region->mmaps[i].mmap = NULL; + vfio_subregion_unmap(region, i); } return ret; @@ -983,6 +1335,21 @@ int vfio_region_mmap(VFIORegion *region) return 0; } +void vfio_region_unmap(VFIORegion *region) +{ + int i; + + if (!region->mem) { + return; + } + + for (i = 0; i < region->nr_mmaps; i++) { + if (region->mmaps[i].mmap) { + vfio_subregion_unmap(region, i); + } + } +} + void vfio_region_exit(VFIORegion *region) { int i; @@ -1205,6 +1572,31 @@ static int vfio_init_container(VFIOContainer *container, int group_fd, return 0; } +static int vfio_get_iommu_info(VFIOContainer *container, + struct vfio_iommu_type1_info **info) +{ + size_t argsz = sizeof(struct vfio_iommu_type1_info); + + *info = g_new0(struct vfio_iommu_type1_info, 1); + +again: + (*info)->argsz = argsz; + + if (ioctl(container->fd, VFIO_IOMMU_GET_INFO, *info)) { + g_free(*info); + *info = NULL; + return -errno; + } + + if (((*info)->argsz > argsz)) { + argsz = (*info)->argsz; + *info = g_realloc(*info, argsz); + goto again; + } + + return 0; +} + static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, Error **errp) { @@ -1271,6 +1663,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, container->error = NULL; QLIST_INIT(&container->giommu_list); QLIST_INIT(&container->hostwin_list); + memset(&container->log_buf, 0, sizeof(VFIOLogBuf)); ret = vfio_init_container(container, group->fd, errp); if (ret) { @@ -1281,7 +1674,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, case VFIO_TYPE1v2_IOMMU: case VFIO_TYPE1_IOMMU: { - struct vfio_iommu_type1_info info; + struct vfio_iommu_type1_info *info; /* * FIXME: This assumes that a Type1 IOMMU can map any 64-bit @@ -1290,15 +1683,15 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, * existing Type1 IOMMUs generally support any IOVA we're * going to actually try in practice. */ - info.argsz = sizeof(info); - ret = ioctl(fd, VFIO_IOMMU_GET_INFO, &info); - /* Ignore errors */ - if (ret || !(info.flags & VFIO_IOMMU_INFO_PGSIZES)) { + ret = vfio_get_iommu_info(container, &info); + if (ret || !(info->flags & VFIO_IOMMU_INFO_PGSIZES)) { /* Assume 4k IOVA page size */ - info.iova_pgsizes = 4096; + info->iova_pgsizes = 4096; } - vfio_host_win_add(container, 0, (hwaddr)-1, info.iova_pgsizes); - container->pgsizes = info.iova_pgsizes; + vfio_host_win_add(container, 0, (hwaddr)-1, info->iova_pgsizes); + container->pgsizes = info->iova_pgsizes; + + g_free(info); break; } case VFIO_SPAPR_TCE_v2_IOMMU: @@ -1379,7 +1772,6 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, QLIST_INSERT_HEAD(&container->group_list, group, container_next); container->listener = vfio_memory_listener; - memory_listener_register(&container->listener, container->space->as); if (container->error) { diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c new file mode 100644 index 0000000000..b6b978ad18 --- /dev/null +++ b/hw/vfio/migration.c @@ -0,0 +1,926 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. + * Description: support vm live migration with vfio-pci device + */ + +#include +#include +#include "qemu/osdep.h" +#include "qemu/main-loop.h" +#include "qemu/cutils.h" + +#include "sysemu/runstate.h" +#include "hw/vfio/vfio-common.h" +#include "cpu.h" +#include "migration/vmstate.h" +#include "migration/qemu-file.h" +#include "migration/register.h" +#include "migration/blocker.h" +#include "migration/misc.h" +#include "qapi/error.h" +#include "exec/ramlist.h" +#include "exec/ram_addr.h" +#include "pci.h" +#include "trace.h" +#include "hw/hw.h" +#include "migration/migration.h" + +#define VFIO_MIG_FLAG_END_OF_STATE (0xffffffffef100001ULL) +#define VFIO_MIG_FLAG_DEV_CONFIG_STATE (0xffffffffef100002ULL) +#define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL) +#define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL) +#define VFIO_MIG_FLAG_MEM_CHECK (0xffffffffef100005ULL) + +#define LONG_LONG_LEN 8 +#define INT_LEN 4 +#define SHORT_LEN 2 + +#define BSD_CHECKSUM_RIGHT_SHIFT 1 +#define BSD_CHECKSUM_LEFT_SHIFT 15 + +static int64_t g_bytes_transferred; + +void migration_memory_check(void) +{ + MemoryRegion *root_mr = address_space_memory.root; + MemoryRegion *sub_mr = NULL; + hwaddr addr, start; + uint64_t size; + uint16_t sum = 0; + uint8_t val; + bool found = false; + + QTAILQ_FOREACH(sub_mr, &root_mr->subregions, subregions_link) { + if (!strcmp(sub_mr->name, "mach-virt.ram")) { + found = true; + break; + } + } + + if (!found) { + error_report("can't find mach-virt.ram in address_space_memory"); + return; + } + + start = sub_mr->addr; + size = int128_get64(sub_mr->size); + + for (addr = start; addr < (start + size); addr++) { + val = address_space_ldub(&address_space_memory, addr, + MEMTXATTRS_UNSPECIFIED, NULL); + sum = (sum >> BSD_CHECKSUM_RIGHT_SHIFT) | + (sum << BSD_CHECKSUM_LEFT_SHIFT); + sum += val; + } + + info_report("vm memory check: start=%lu, size=%lu, verify code=%05d", + start, size, sum); +} + +static inline int vfio_mig_access(const VFIODevice *vbasedev, + void *val, int count, + off_t off, bool iswrite) +{ + int ret; + + ret = iswrite ? pwrite(vbasedev->fd, val, count, off) : + pread(vbasedev->fd, val, count, off); + if (ret < count) { + error_report("vfio device %s vfio_mig_%s %d byte: failed at offset 0x%" + HWADDR_PRIx", errno: %d", vbasedev->name, + iswrite ? "write" : "read", + count, off, errno); + return (ret < 0) ? ret : -EINVAL; + } + + return 0; +} + +static int vfio_mig_rw(const VFIODevice *vbasedev, + __u8 *buf, size_t count, + off_t off, bool iswrite) +{ + int ret; + int done = 0; + __u8 *tbuf = buf; + + while (count) { + int bytes; + + if (count >= LONG_LONG_LEN && !(off % LONG_LONG_LEN)) { + bytes = LONG_LONG_LEN; + } else if (count >= INT_LEN && !(off % INT_LEN)) { + bytes = INT_LEN; + } else if (count >= SHORT_LEN && !(off % SHORT_LEN)) { + bytes = SHORT_LEN; + } else { + bytes = 1; + } + + ret = vfio_mig_access(vbasedev, tbuf, bytes, off, iswrite); + if (ret != 0) { + return ret; + } + + count -= bytes; + done += bytes; + off += bytes; + tbuf += bytes; + } + + return done; +} + +#define VFIO_MIG_STRUCT_OFFSET(f) offsetof(struct vfio_device_migration_info, f) + +static int vfio_mig_read(const VFIODevice *vbasedev, + void *buf, size_t count, off_t off) +{ + return vfio_mig_rw(vbasedev, (__u8 *)buf, count, off, false); +} + +static int vfio_mig_write(const VFIODevice *vbasedev, + void *buf, size_t count, off_t off) +{ + return vfio_mig_rw(vbasedev, (__u8 *)buf, count, off, true); +} + +static int vfio_migration_set_state(VFIODevice *vbasedev, + uint32_t mask, uint32_t value) +{ + VFIOMigration *migration = vbasedev->migration; + VFIORegion *region = &migration->region; + off_t dev_state_off = region->fd_offset + + VFIO_MIG_STRUCT_OFFSET(device_state); + uint32_t device_state; + int ret; + + ret = vfio_mig_read(vbasedev, &device_state, + sizeof(device_state), dev_state_off); + if (ret < 0) { + return ret; + } + + device_state = (device_state & mask) | value; + + if (!VFIO_DEVICE_STATE_VALID(device_state)) { + return -EINVAL; + } + + ret = vfio_mig_write(vbasedev, &device_state, + sizeof(device_state), dev_state_off); + if (ret < 0) { + int rret; + rret = vfio_mig_read(vbasedev, &device_state, + sizeof(device_state), dev_state_off); + if (rret < 0) + return rret; + + if (VFIO_DEVICE_STATE_IS_ERROR(device_state)) { + error_report("vfio device %s is in error state 0x%x\n", + vbasedev->name, device_state); + return -EIO; + } + return ret; + } + + migration->device_state = device_state; + + return 0; +} + +static int vfio_save_setup(QEMUFile *f, void *opaque) +{ + VFIODevice *vbasedev = opaque; + VFIOMigration *migration = vbasedev->migration; + MigrationState *ms = migrate_get_current(); + int ret; + + qemu_put_be64(f, VFIO_MIG_FLAG_DEV_SETUP_STATE); + + if (migration->region.mmaps) { + /* + * Calling vfio_region_mmap() from migration thread. Memory API called + * from this function require locking the iothread when called from + * outside the main loop thread. + */ + qemu_mutex_lock_iothread(); + ret = vfio_region_mmap(&migration->region); + qemu_mutex_unlock_iothread(); + if (ret != 0) { + error_report("%s: Failed to mmap VFIO migration region, ret=%d", + vbasedev->name, ret); + error_report("%s: Falling back to slow path", vbasedev->name); + } + } + + ret = vfio_migration_set_state(vbasedev, + VFIO_DEVICE_STATE_MASK, + VFIO_DEVICE_STATE_SAVING); + if (ret != 0) { + error_report("%s: Failed to set state SAVING", vbasedev->name); + return ret; + } + + /* migrate the version info of migration driver to dst. */ + qemu_put_be32(f, migration->version_id); + + qemu_put_be64(f, ms->parameters.memory_check == 1 ? + VFIO_MIG_FLAG_MEM_CHECK : VFIO_MIG_FLAG_END_OF_STATE); + + ret = qemu_file_get_error(f); + if (ret != 0) { + return ret; + } + + return ret; +} + +static int vfio_save_device_config_state(QEMUFile *f, void *opaque) +{ + VFIODevice *vbasedev = opaque; + + qemu_put_be64(f, VFIO_MIG_FLAG_DEV_CONFIG_STATE); + if (vbasedev->ops && vbasedev->ops->vfio_save_config) { + vbasedev->ops->vfio_save_config(vbasedev, f); + } + + qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); + + return qemu_file_get_error(f); +} + +static int vfio_update_pending(VFIODevice *vbasedev) +{ + VFIOMigration *migration = vbasedev->migration; + VFIORegion *region = &migration->region; + uint64_t pending_bytes = 0; + int ret; + + ret = vfio_mig_read(vbasedev, &pending_bytes, sizeof(pending_bytes), + region->fd_offset + VFIO_MIG_STRUCT_OFFSET(pending_bytes)); + if (ret < 0) { + migration->pending_bytes = 0; + return ret; + } + + migration->pending_bytes = pending_bytes; + + return 0; +} + +static void *get_data_section_size(VFIORegion *region, uint64_t data_offset, + uint64_t data_size, uint64_t *size) +{ + void *ptr = NULL; + uint64_t limit = 0; + int i; + + if (!region->mmaps) { + if (size) { + *size = MIN(data_size, region->size - data_offset); + } + return ptr; + } + + for (i = 0; i < region->nr_mmaps; i++) { + VFIOMmap *map = region->mmaps + i; + + if ((data_size >= map->offset) && + (data_offset < map->offset + map->size)) { + /* check if data_offset is within sparse mmap areas */ + ptr = map->mmap + data_offset - map->offset; + if (size) { + *size = MIN(data_size, map->offset + map->size - data_offset); + } + break; + } else if ((data_offset < map->offset) && + (!limit || limit > map->offset)) { + /* + * data_offset is not within sparse mmap areas, find size of + * non-mapped area. Check through all list since region->mmaps list + * is not sorted. + */ + limit = map->offset; + } + } + + if (!ptr && size) { + *size = limit ? MIN(data_size, limit - data_offset) : data_size; + } + + return ptr; +} + +static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size) +{ + VFIOMigration *migration = vbasedev->migration; + VFIORegion *region = &migration->region; + uint64_t data_offset = 0; + uint64_t data_size = 0; + uint64_t sz; + int ret; + + ret = vfio_mig_read(vbasedev, &data_offset, sizeof(data_offset), + region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_offset)); + if (ret < 0) { + return ret; + } + + ret = vfio_mig_read(vbasedev, &data_size, sizeof(data_size), + region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_size)); + if (ret < 0) { + return ret; + } + qemu_put_be64(f, data_size); + sz = data_size; + + while (sz) { + void *buf; + uint64_t sec_size; + bool buf_allocated = false; + + buf = get_data_section_size(region, data_offset, sz, &sec_size); + if (!buf) { + /* fall slow path */ + if (sec_size == 0 || (buf = g_try_malloc(sec_size)) == NULL) { + error_report("Error allocating buffer"); + return -ENOMEM; + } + buf_allocated = true; + + ret = vfio_mig_read(vbasedev, buf, sec_size, + region->fd_offset + data_offset); + if (ret < 0) { + g_free(buf); + return ret; + } + } + + qemu_put_buffer(f, buf, sec_size); + if (buf_allocated) { + g_free(buf); + } + sz -= sec_size; + data_offset += sec_size; + } + + ret = qemu_file_get_error(f); + if (!ret && size) { + *size = data_size; + } + + g_bytes_transferred += data_size; + + return ret; +} + +static int vfio_save_complete_precopy(QEMUFile *f, void *opaque) +{ + VFIODevice *vbasedev = opaque; + VFIOMigration *migration = vbasedev->migration; + uint64_t data_size; + int ret; + + ret = vfio_migration_set_state(vbasedev, + ~VFIO_DEVICE_STATE_RUNNING, + VFIO_DEVICE_STATE_SAVING); + if (ret != 0) { + error_report("%s: Failed to set state STOP and SAVING", vbasedev->name); + return ret; + } + + ret = vfio_save_device_config_state(f, opaque); + if (ret != 0) { + return ret; + } + + ret = vfio_update_pending(vbasedev); + if (ret != 0) { + return ret; + } + + while (migration->pending_bytes > 0) { + qemu_put_be64(f, VFIO_MIG_FLAG_DEV_DATA_STATE); + ret = vfio_save_buffer(f, vbasedev, &data_size); + if (ret < 0) { + error_report("%s: Failed to save buffer", vbasedev->name); + return ret; + } + + if (data_size == 0) { + break; + } + + ret = vfio_update_pending(vbasedev); + if (ret != 0) { + return ret; + } + } + + qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE); + + ret = qemu_file_get_error(f); + if (ret != 0) { + return ret; + } + + ret = vfio_migration_set_state(vbasedev, ~VFIO_DEVICE_STATE_SAVING, 0); + if (ret != 0) { + error_report("%s: Failed to set state STOPPED", vbasedev->name); + } + + return ret; +} + +static void vfio_save_cleanup(void *opaque) +{ + VFIODevice *vbasedev = opaque; + VFIOMigration *migration = vbasedev->migration; + + if (migration->region.mmaps) { + vfio_region_unmap(&migration->region); + } +} + +static int vfio_load_setup(QEMUFile *f, void *opaque) +{ + VFIODevice *vbasedev = opaque; + VFIOMigration *migration = vbasedev->migration; + int ret; + + if (migration->region.mmaps) { + ret = vfio_region_mmap(&migration->region); + if (ret != 0) { + error_report("%s: Failed to mmap VFIO migration region %d, ret=%d", + vbasedev->name, migration->region.nr, ret); + error_report("%s: Falling back to slow path", vbasedev->name); + } + } + + ret = vfio_migration_set_state(vbasedev, + ~VFIO_DEVICE_STATE_MASK, + VFIO_DEVICE_STATE_RESUMING); + if (ret != 0) { + error_report("%s: Failed to set state RESUMING", vbasedev->name); + if (migration->region.mmaps) { + vfio_region_unmap(&migration->region); + } + } + + return ret; +} + +static int vfio_load_device_config_state(QEMUFile *f, void *opaque) +{ + VFIODevice *vbasedev = opaque; + int ret; + + if (vbasedev->ops && vbasedev->ops->vfio_load_config) { + ret = vbasedev->ops->vfio_load_config(vbasedev, f); + if (ret != 0) { + error_report("%s: Failed to load device config space", + vbasedev->name); + return ret; + } + } + + return qemu_file_get_error(f); +} + +static int vfio_load_state_data(QEMUFile *f, VFIODevice *vbasedev, + uint64_t data_offset, uint64_t size) +{ + VFIORegion *region = &vbasedev->migration->region; + uint64_t sec_size; + void *buf = NULL; + bool buf_alloc = false; + int ret; + + while (size) { + buf_alloc = false; + buf = get_data_section_size(region, data_offset, size, &sec_size); + if (!buf) { + if (sec_size == 0 || (buf = g_try_malloc(sec_size)) == NULL) { + error_report("Error allocating buffer"); + return -ENOMEM; + } + buf_alloc = true; + } + + qemu_get_buffer(f, buf, sec_size); + + if (buf_alloc) { + ret = vfio_mig_write(vbasedev, buf, sec_size, + region->fd_offset + data_offset); + g_free(buf); + if (ret < 0) { + return ret; + } + } + + size -= sec_size; + data_offset += sec_size; + } + + return 0; +} + +static int vfio_load_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t data_size) +{ + VFIORegion *region = &vbasedev->migration->region; + uint64_t data_offset = 0; + uint64_t size, report_size; + int ret; + + do { + ret = vfio_mig_read(vbasedev, &data_offset, sizeof(data_offset), + region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_offset)); + if (ret < 0) { + return ret; + } + if (data_offset + data_size > region->size) { + report_size = size = region->size - data_offset; + data_size -= size; + } else { + report_size = size = data_size; + data_size = 0; + } + + ret = vfio_load_state_data(f, vbasedev, data_offset, size); + if (ret != 0) { + return ret; + } + + ret = vfio_mig_write(vbasedev, &report_size, sizeof(report_size), + region->fd_offset + VFIO_MIG_STRUCT_OFFSET(data_size)); + if (ret < 0) { + return ret; + } + } while (data_size); + + return 0; +} + +static int migration_driver_version_check(VFIODevice *vbasedev, + uint32_t src_version) +{ + VFIOMigration *migration = vbasedev->migration; + + if (src_version > migration->version_id) { + error_report("Can't support live migration from higher to lower " + "migration driver, src version = %u, dst version = %u.", + src_version, migration->version_id); + return -EINVAL; + } + + return 0; +} + +static int vfio_load_setup_check(QEMUFile *f, VFIODevice *vbasedev) +{ + uint32_t src_version; + uint64_t flag; + MigrationState *ms = migrate_get_current(); + + src_version = qemu_get_be32(f); + flag = qemu_get_be64(f); + if (flag != VFIO_MIG_FLAG_MEM_CHECK && flag != VFIO_MIG_FLAG_END_OF_STATE) { + error_report("Unknown flag 0x%lx in load setup stage", flag); + return -EINVAL; + } + + if (flag == VFIO_MIG_FLAG_MEM_CHECK) { + ms->parameters.memory_check = 1; + } + + return migration_driver_version_check(vbasedev, src_version); +} + +static int vfio_load_state(QEMUFile *f, void *opaque, int version_id) +{ + VFIODevice *vbasedev = opaque; + int ret = 0; + uint64_t data; + + data = qemu_get_be64(f); + while (data != VFIO_MIG_FLAG_END_OF_STATE) { + switch (data) { + case VFIO_MIG_FLAG_DEV_SETUP_STATE: { + return vfio_load_setup_check(f, vbasedev); + break; + } + case VFIO_MIG_FLAG_DEV_CONFIG_STATE: { + ret = vfio_load_device_config_state(f, opaque); + if (ret != 0) { + return ret; + } + data = qemu_get_be64(f); + if (data != VFIO_MIG_FLAG_END_OF_STATE) { + error_report("%s: Failed loading device config space, " + "end flag incorrect 0x%"PRIx64, + vbasedev->name, data); + return -EINVAL; + } + break; + } + case VFIO_MIG_FLAG_DEV_DATA_STATE: { + uint64_t data_size = qemu_get_be64(f); + if (data_size && + (ret = vfio_load_buffer(f, vbasedev, data_size)) < 0) { + return ret; + } + break; + } + default: + error_report("%s: Unknown tag 0x%"PRIx64, vbasedev->name, data); + return -EINVAL; + } + + data = qemu_get_be64(f); + ret = qemu_file_get_error(f); + if (ret != 0) { + return ret; + } + } + + return ret; +} + +static int vfio_load_cleanup(void *opaque) +{ + VFIODevice *vbasedev = opaque; + VFIOMigration *migration = vbasedev->migration; + + if (migration->region.mmaps) { + vfio_region_unmap(&migration->region); + } + return 0; +} + +static SaveVMHandlers g_savevm_vfio_handlers = { + .save_setup = vfio_save_setup, + /* save_live_pending reserved */ + /* save_live_iterate reserved */ + .save_live_complete_precopy = vfio_save_complete_precopy, + .save_cleanup = vfio_save_cleanup, + .load_setup = vfio_load_setup, + .load_state = vfio_load_state, + .load_cleanup = vfio_load_cleanup, +}; + +static int vfio_migration_cancel(VFIODevice *vbasedev) +{ + VFIOMigration *migration = vbasedev->migration; + VFIORegion *region = &migration->region; + VFIOCmd cmd = VFIO_DEVICE_MIGRATION_CANCEL; + int ret; + + ret = vfio_mig_write(vbasedev, &cmd, sizeof(cmd), + region->fd_offset + VFIO_MIG_STRUCT_OFFSET(device_cmd)); + return (ret < 0) ? ret : 0; +} + +static void vfio_migration_state_notifier(Notifier *notifier, void *data) +{ + MigrationState *s = data; + VFIOMigration *migration = container_of(notifier, + VFIOMigration, + migration_state); + VFIODevice *vbasedev = migration->vbasedev; + int ret; + + switch (s->state) { + case MIGRATION_STATUS_CANCELLING: + case MIGRATION_STATUS_CANCELLED: + case MIGRATION_STATUS_FAILED: + g_bytes_transferred = 0; + ret = vfio_migration_cancel(vbasedev); + if (ret != 0) { + error_report("%s: Failed to notify the vifio migration cancel, " + "ret = %d.", vbasedev->name, ret); + return; + } + + ret = vfio_migration_set_state(vbasedev, + ~(VFIO_DEVICE_STATE_SAVING | VFIO_DEVICE_STATE_RESUMING), + VFIO_DEVICE_STATE_RUNNING); + if (ret != 0) { + error_report("%s: Failed to set state RUNNING", vbasedev->name); + } + break; + default: + break; + } +} + +static void vfio_migration_exit(VFIODevice *vbasedev) +{ + VFIOMigration *migration = vbasedev->migration; + + vfio_region_exit(&migration->region); + vfio_region_finalize(&migration->region); + g_free(vbasedev->migration); + vbasedev->migration = NULL; +} + +static int vfio_get_migration_version(VFIODevice *vbasedev) +{ + VFIOMigration *migration = vbasedev->migration; + VFIORegion *region = &migration->region; + uint32_t version_id; + int ret; + + ret = vfio_mig_read(vbasedev, &version_id, sizeof(version_id), + region->fd_offset + VFIO_MIG_STRUCT_OFFSET(version_id)); + if (ret < 0) { + return ret; + } + + migration->version_id = version_id; + return 0; +} + +static int vfio_migration_init(VFIODevice *vbasedev, + struct vfio_region_info *info) +{ + int ret; + Object *obj = NULL; + VFIOMigration *migration = NULL; + g_autofree char *path = NULL; + g_autofree char *oid = NULL; + char id[256] = ""; + + if (!vbasedev->ops->vfio_get_object) { + error_report("vfio device=%s can't find 'vfio_get_object' ops\n", + vbasedev->name); + return -EINVAL; + } + + obj = vbasedev->ops->vfio_get_object(vbasedev); + if (!obj) { + return -EINVAL; + } + + vbasedev->migration = g_new0(VFIOMigration, 1); + ret = vfio_region_setup(obj, vbasedev, + &vbasedev->migration->region, + info->index, "migration"); + if (ret != 0) { + error_report("%s: Failed to setup VFIO migration region %d, ret=%d", + vbasedev->name, info->index, ret); + goto err; + } + + if (!vbasedev->migration->region.size) { + error_report("%s: Invalid zero-sized VFIO migration region %d", + vbasedev->name, info->index); + ret = -EINVAL; + goto err; + } + + ret = vfio_get_migration_version(vbasedev); + if (ret != 0) { + error_report("%s: Failed to get migration driver version: %d", + vbasedev->name, ret); + goto err; + } + + migration = vbasedev->migration; + migration->vbasedev = vbasedev; + + oid = vmstate_if_get_id(VMSTATE_IF(DEVICE(obj))); + if (oid) { + path = g_strdup_printf("%s/vfio", oid); + } else { + path = g_strdup("vfio"); + } + strpadcpy(id, sizeof(id), path, '\0'); + + register_savevm_live(id, VMSTATE_INSTANCE_ID_ANY, 1, + &g_savevm_vfio_handlers, vbasedev); + migration->migration_state.notify = vfio_migration_state_notifier; + add_migration_state_change_notifier(&migration->migration_state); + + migration->vm_running = runstate_is_running() ? 1 : 0; + return 0; + +err: + vfio_migration_exit(vbasedev); + return ret; +} + +int vfio_migration_probe(VFIODevice *vbasedev, Error **errp) +{ + struct vfio_region_info *info = NULL; + Error *local_err = NULL; + int ret = -ENOTSUP; + + if (!vbasedev->enable_migration) { + error_setg(&vbasedev->migration_blocker, + "Current Disable the VFIO device migration"); + return ret; + } + + ret = vfio_get_dev_region_info(vbasedev, VFIO_REGION_TYPE_MIGRATION, + VFIO_REGION_SUBTYPE_MIGRATION, &info); + if (ret != 0) { + error_report("vfio device=%s find migration region from vfio failed\n", + vbasedev->name); + goto add_blocker; + } + + ret = vfio_migration_init(vbasedev, info); + if (ret != 0) { + goto add_blocker; + } + + g_free(info); + return 0; + +add_blocker: + error_setg(&vbasedev->migration_blocker, + "VFIO device=%s doesn't support migration", + vbasedev->name); + g_free(info); + + migrate_add_blocker(vbasedev->migration_blocker, &local_err); + if (local_err) { + error_propagate(errp, local_err); + error_free(vbasedev->migration_blocker); + vbasedev->migration_blocker = NULL; + } + + return ret; +} + +void vfio_migration_finalize(VFIODevice *vbasedev) +{ + Object *obj = NULL; + g_autofree char *path = NULL; + g_autofree char *oid = NULL; + char id[256] = ""; + + if (vbasedev->migration) { + VFIOMigration *migration = vbasedev->migration; + + remove_migration_state_change_notifier(&migration->migration_state); + vfio_migration_exit(vbasedev); + } + + if (vbasedev->migration_blocker) { + migrate_del_blocker(vbasedev->migration_blocker); + error_free(vbasedev->migration_blocker); + vbasedev->migration_blocker = NULL; + } + + obj = vbasedev->ops->vfio_get_object(vbasedev); + if (!obj) + return; + oid = vmstate_if_get_id(VMSTATE_IF(DEVICE(obj))); + path = oid ? g_strdup_printf("%s/vfio", oid) : g_strdup("vfio"); + strpadcpy(id, sizeof(id), path, '\0'); + unregister_savevm(NULL, id, vbasedev); +} + +int vfio_device_disable(VFIODevice *vbasedev) +{ + if (vbasedev->migration_blocker || + (vbasedev->migration->device_state & VFIO_DEVICE_STATE_RUNNING) == 0) { + return 0; + } + + return vfio_migration_set_state(vbasedev, ~VFIO_DEVICE_STATE_RUNNING, 0); +} + +int vfio_device_enable(VFIODevice *vbasedev) +{ + /* + * There is no need to change vfio device state when vm is paused. + */ + if (vbasedev->migration_blocker || + (vbasedev->migration->device_state & VFIO_DEVICE_STATE_RUNNING)) { + return 0; + } + + return vfio_migration_set_state(vbasedev, ~VFIO_DEVICE_STATE_MASK, + VFIO_DEVICE_STATE_RUNNING); +} + +int set_all_vfio_device(bool is_enable) +{ + VFIOGroup *group; + VFIODevice *vbasedev; + int ret = 0; + int rret; + + QLIST_FOREACH(group, &vfio_group_list, next) { + QLIST_FOREACH(vbasedev, &group->device_list, next) { + rret = is_enable ? + vfio_device_enable(vbasedev) : vfio_device_disable(vbasedev); + ret = rret ? rret : ret; + } + } + + return ret; +} \ No newline at end of file diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 5e75a95129..7ab394b3ad 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -41,6 +41,7 @@ #include "trace.h" #include "qapi/error.h" #include "migration/blocker.h" +#include "migration/qemu-file-types.h" #define TYPE_VFIO_PCI "vfio-pci" #define PCI_VFIO(obj) OBJECT_CHECK(VFIOPCIDevice, obj, TYPE_VFIO_PCI) @@ -511,27 +512,30 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr, * for a device if they're not in use, so we shutdown and incrementally * increase them as needed. */ - if (vdev->nr_vectors < nr + 1) { - vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); - vdev->nr_vectors = nr + 1; - ret = vfio_enable_vectors(vdev, true); - if (ret) { - error_report("vfio: failed to enable vectors, %d", ret); - } - } else { - Error *err = NULL; - int32_t fd; - - if (vector->virq >= 0) { - fd = event_notifier_get_fd(&vector->kvm_interrupt); + if ((!vdev->enable_msix_once) || + (vdev->enable_msix_once && (nr == (pdev->msix_entries_nr - 1)))) { + if (vdev->nr_vectors < nr + 1) { + vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX); + vdev->nr_vectors = nr + 1; + ret = vfio_enable_vectors(vdev, true); + if (ret) { + error_report("vfio: failed to enable vectors, %d", ret); + } } else { - fd = event_notifier_get_fd(&vector->interrupt); - } + Error *err = NULL; + int32_t fd; - if (vfio_set_irq_signaling(&vdev->vbasedev, - VFIO_PCI_MSIX_IRQ_INDEX, nr, - VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) { - error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); + if (vector->virq >= 0) { + fd = event_notifier_get_fd(&vector->kvm_interrupt); + } else { + fd = event_notifier_get_fd(&vector->interrupt); + } + + if (vfio_set_irq_signaling(&vdev->vbasedev, + VFIO_PCI_MSIX_IRQ_INDEX, nr, + VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) { + error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); + } } } @@ -601,12 +605,14 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev) */ vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL); vfio_msix_vector_release(&vdev->pdev, 0); + vdev->enable_msix_once = true; if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use, vfio_msix_vector_release, NULL)) { error_report("vfio: msix_set_vector_notifiers failed"); } + vdev->enable_msix_once = false; trace_vfio_msix_enable(vdev->vbasedev.name); } @@ -2407,10 +2413,72 @@ static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev) } } +static bool vfio_msix_present(void *opaque, int version_id) +{ + PCIDevice *pdev = opaque; + + return msix_present(pdev); +} + +static Object *vfio_pci_get_object(VFIODevice *vbasedev) +{ + VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); + + return OBJECT(vdev); +} + +const VMStateDescription vmstate_vfio_pci_config = { + .name = "VFIOPCIDevice", + .version_id = 1, + .minimum_version_id = 1, + .fields = (VMStateField[]) { + VMSTATE_PCI_DEVICE(pdev, VFIOPCIDevice), + VMSTATE_MSIX_TEST(pdev, VFIOPCIDevice, vfio_msix_present), + VMSTATE_END_OF_LIST() + } +}; + +static void vfio_pci_save_config(VFIODevice *vbasedev, QEMUFile *f) +{ + VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); + PCIDevice *pdev = &vdev->pdev; + + vmstate_save_state(f, &vmstate_vfio_pci_config, vdev, NULL); + pdev->device_bdf = pci_get_bdf(pdev); + qemu_put_be32(f, pdev->device_bdf); +} + +static int vfio_pci_load_config(VFIODevice *vbasedev, QEMUFile *f) +{ + VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev); + PCIDevice *pdev = &vdev->pdev; + int ret; + + ret = vmstate_load_state(f, &vmstate_vfio_pci_config, vdev, 1); + if (ret) { + return ret; + } + + pdev->device_bdf = qemu_get_be32(f); + + vfio_pci_write_config(pdev, PCI_COMMAND, + pci_get_word(pdev->config + PCI_COMMAND), 2); + if (msi_enabled(pdev)) { + vfio_msi_enable(vdev); + } else if (msix_enabled(pdev)) { + vfio_msix_enable(vdev); + } + + return ret; +} + static VFIODeviceOps vfio_pci_ops = { .vfio_compute_needs_reset = vfio_pci_compute_needs_reset, .vfio_hot_reset_multi = vfio_pci_hot_reset_multi, .vfio_eoi = vfio_intx_eoi, + .vfio_get_object = vfio_pci_get_object, + .vfio_save_config = vfio_pci_save_config, + .vfio_load_config = vfio_pci_load_config, }; int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp) @@ -2745,18 +2813,6 @@ static void vfio_realize(PCIDevice *pdev, Error **errp) return; } - if (!pdev->failover_pair_id) { - error_setg(&vdev->migration_blocker, - "VFIO device doesn't support migration"); - ret = migrate_add_blocker(vdev->migration_blocker, &err); - if (ret) { - error_propagate(errp, err); - error_free(vdev->migration_blocker); - vdev->migration_blocker = NULL; - return; - } - } - vdev->vbasedev.name = g_path_get_basename(vdev->vbasedev.sysfsdev); vdev->vbasedev.ops = &vfio_pci_ops; vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI; @@ -3024,6 +3080,13 @@ static void vfio_realize(PCIDevice *pdev, Error **errp) } } + if (!pdev->failover_pair_id) { + ret = vfio_migration_probe(&vdev->vbasedev, errp); + if (ret) { + error_report("%s: Migration disabled", vdev->vbasedev.name); + } + } + vfio_register_err_notifier(vdev); vfio_register_req_notifier(vdev); vfio_setup_resetfn_quirk(vdev); @@ -3038,11 +3101,6 @@ out_teardown: vfio_bars_exit(vdev); error: error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.name); - if (vdev->migration_blocker) { - migrate_del_blocker(vdev->migration_blocker); - error_free(vdev->migration_blocker); - vdev->migration_blocker = NULL; - } } static void vfio_instance_finalize(Object *obj) @@ -3054,10 +3112,6 @@ static void vfio_instance_finalize(Object *obj) vfio_bars_finalize(vdev); g_free(vdev->emulated_config_bits); g_free(vdev->rom); - if (vdev->migration_blocker) { - migrate_del_blocker(vdev->migration_blocker); - error_free(vdev->migration_blocker); - } /* * XXX Leaking igd_opregion is not an oversight, we can't remove the * fw_cfg entry therefore leaking this allocation seems like the safest @@ -3085,6 +3139,7 @@ static void vfio_exitfn(PCIDevice *pdev) } vfio_teardown_msi(vdev); vfio_bars_exit(vdev); + vfio_migration_finalize(&vdev->vbasedev); } static void vfio_pci_reset(DeviceState *dev) @@ -3161,6 +3216,8 @@ static Property vfio_pci_dev_properties[] = { VFIO_FEATURE_ENABLE_REQ_BIT, true), DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features, VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false), + DEFINE_PROP_BOOL("x-enable-migration", VFIOPCIDevice, + vbasedev.enable_migration, true), DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false), DEFINE_PROP_BOOL("x-balloon-allowed", VFIOPCIDevice, vbasedev.balloon_allowed, false), diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h index 0da7a20a7e..4d6c983a95 100644 --- a/hw/vfio/pci.h +++ b/hw/vfio/pci.h @@ -167,8 +167,8 @@ typedef struct VFIOPCIDevice { bool no_kvm_ioeventfd; bool no_vfio_ioeventfd; bool enable_ramfb; + bool enable_msix_once; VFIODisplay *dpy; - Error *migration_blocker; Notifier irqchip_change_notifier; } VFIOPCIDevice; diff --git a/include/hw/pci/pci.h b/include/hw/pci/pci.h index cfedf5a995..937f9f2b11 100644 --- a/include/hw/pci/pci.h +++ b/include/hw/pci/pci.h @@ -357,6 +357,9 @@ struct PCIDevice { /* ID of standby device in net_failover pair */ char *failover_pair_id; + + /* bdf after the guest os initialization, only be used for migration. */ + uint32_t device_bdf; }; void pci_register_bar(PCIDevice *pci_dev, int region_num, diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index fd564209ac..d98271cff4 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -57,6 +57,24 @@ typedef struct VFIORegion { uint8_t nr; /* cache the region number for debug */ } VFIORegion; +typedef struct VFIOStateData { + uint8_t *data; + uint64_t data_offset; + uint64_t data_size; +} VFIOStateData; + +typedef struct VFIOMigration { + struct VFIODevice *vbasedev; + VMChangeStateEntry *vm_state; + VFIORegion region; + uint32_t device_state; + uint32_t version_id; + int vm_running; + Notifier migration_state; + uint64_t pending_bytes; + VFIOStateData state_data; +} VFIOMigration; + typedef struct VFIOAddressSpace { AddressSpace *as; QLIST_HEAD(, VFIOContainer) containers; @@ -65,6 +83,11 @@ typedef struct VFIOAddressSpace { struct VFIOGroup; +typedef struct VFIOLogBuf { + struct vfio_log_buf_info info; + int fd; +} VFIOLogBuf; + typedef struct VFIOContainer { VFIOAddressSpace *space; int fd; /* /dev/vfio/vfio, empowered by the attached groups */ @@ -74,6 +97,7 @@ typedef struct VFIOContainer { Error *error; bool initialized; unsigned long pgsizes; + VFIOLogBuf log_buf; QLIST_HEAD(, VFIOGuestIOMMU) giommu_list; QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list; QLIST_HEAD(, VFIOGroup) group_list; @@ -109,16 +133,22 @@ typedef struct VFIODevice { bool needs_reset; bool no_mmap; bool balloon_allowed; + bool enable_migration; VFIODeviceOps *ops; unsigned int num_irqs; unsigned int num_regions; unsigned int flags; + VFIOMigration *migration; + Error *migration_blocker; } VFIODevice; struct VFIODeviceOps { void (*vfio_compute_needs_reset)(VFIODevice *vdev); int (*vfio_hot_reset_multi)(VFIODevice *vdev); void (*vfio_eoi)(VFIODevice *vdev); + Object *(*vfio_get_object)(VFIODevice *vdev); + void (*vfio_save_config)(VFIODevice *vdev, QEMUFile *f); + int (*vfio_load_config)(VFIODevice *vdev, QEMUFile *f); }; typedef struct VFIOGroup { @@ -170,6 +200,7 @@ uint64_t vfio_region_read(void *opaque, int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region, int index, const char *name); int vfio_region_mmap(VFIORegion *region); +void vfio_region_unmap(VFIORegion *region); void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled); void vfio_region_exit(VFIORegion *region); void vfio_region_finalize(VFIORegion *region); @@ -200,4 +231,15 @@ int vfio_spapr_create_window(VFIOContainer *container, int vfio_spapr_remove_window(VFIOContainer *container, hwaddr offset_within_address_space); +int vfio_migration_probe(VFIODevice *vbasedev, Error **errp); +void vfio_migration_finalize(VFIODevice *vbasedev); + +int vfio_device_enable(VFIODevice *vbasedev); +int vfio_device_disable(VFIODevice *vbasedev); +int set_all_vfio_device(bool is_enable); + +void migration_memory_check(void); + +bool vfio_memory_listener_check(MemoryListener * listener); + #endif /* HW_VFIO_VFIO_COMMON_H */ diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h index fb10370d29..2c209da9d6 100644 --- a/linux-headers/linux/vfio.h +++ b/linux-headers/linux/vfio.h @@ -305,9 +305,51 @@ struct vfio_region_info_cap_type { #define VFIO_REGION_TYPE_PCI_VENDOR_MASK (0xffff) #define VFIO_REGION_TYPE_GFX (1) #define VFIO_REGION_TYPE_CCW (2) +#define VFIO_REGION_TYPE_MIGRATION (3) /* sub-types for VFIO_REGION_TYPE_PCI_* */ +/* sub-types for VFIO_REGION_TYPE_MIGRATION */ +#define VFIO_REGION_SUBTYPE_MIGRATION (1) + +typedef enum { + VFIO_DEVICE_STOP = 0xffff0001, + VFIO_DEVICE_CONTINUE, + VFIO_DEVICE_MIGRATION_CANCEL +} VFIOCmd; + +struct vfio_device_migration_info { + __u32 device_state; +#define VFIO_DEVICE_STATE_STOP (0) +#define VFIO_DEVICE_STATE_RUNNING (1 << 0) +#define VFIO_DEVICE_STATE_SAVING (1 << 1) +#define VFIO_DEVICE_STATE_RESUMING (1 << 2) +#define VFIO_DEVICE_STATE_MASK (VFIO_DEVICE_STATE_RUNNING | \ + VFIO_DEVICE_STATE_SAVING | \ + VFIO_DEVICE_STATE_RESUMING) + +#define VFIO_DEVICE_STATE_VALID(state) \ + (state & VFIO_DEVICE_STATE_RESUMING ? \ + (state & VFIO_DEVICE_STATE_MASK) == VFIO_DEVICE_STATE_RESUMING : 1) + +#define VFIO_DEVICE_STATE_IS_ERROR(state) \ + ((state & VFIO_DEVICE_STATE_MASK) == (VFIO_DEVICE_STATE_SAVING | \ + VFIO_DEVICE_STATE_RESUMING)) + +#define VFIO_DEVICE_STATE_SET_ERROR(state) \ + ((state & ~VFIO_DEVICE_STATE_MASK) | VFIO_DEVICE_SATE_SAVING | \ + VFIO_DEVICE_STATE_RESUMING) + + __u32 reserved; + + __u32 device_cmd; + __u32 version_id; + + __u64 pending_bytes; + __u64 data_offset; + __u64 data_size; +}; + /* 8086 vendor PCI sub-types */ #define VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION (1) #define VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG (2) @@ -943,6 +985,34 @@ struct vfio_iommu_spapr_tce_remove { }; #define VFIO_IOMMU_SPAPR_TCE_REMOVE _IO(VFIO_TYPE, VFIO_BASE + 20) +/** + * for log buf control + */ +struct vfio_log_buf_sge { + __u64 len; + __u64 addr; +}; + +struct vfio_log_buf_info { + __u32 uuid; + __u64 buffer_size; + __u64 addrs_size; + __u64 frag_size; + struct vfio_log_buf_sge *sgevec; +}; + +struct vfio_log_buf_ctl { + __u32 argsz; + __u32 flags; + #define VFIO_DEVICE_LOG_BUF_FLAG_SETUP (1 << 0) + #define VFIO_DEVICE_LOG_BUF_FLAG_RELEASE (1 << 1) + #define VFIO_DEVICE_LOG_BUF_FLAG_START (1 << 2) + #define VFIO_DEVICE_LOG_BUF_FLAG_STOP (1 << 3) + #define VFIO_DEVICE_LOG_BUF_FLAG_STATUS_QUERY (1 << 4) + void* data; +}; +#define VFIO_LOG_BUF_CTL _IO(VFIO_TYPE, VFIO_BASE + 21) +#define VFIO_GET_LOG_BUF_FD _IO(VFIO_TYPE, VFIO_BASE + 22) /* ***************************************************************** */ #endif /* VFIO_H */ diff --git a/memory.c b/memory.c index 601b749906..df27b8a538 100644 --- a/memory.c +++ b/memory.c @@ -35,6 +35,8 @@ #include "hw/boards.h" #include "migration/vmstate.h" +#include "hw/vfio/vfio-common.h" + //#define DEBUG_UNASSIGNED static unsigned memory_region_transaction_depth; @@ -1788,7 +1790,9 @@ bool memory_region_is_ram_device(MemoryRegion *mr) uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr) { uint8_t mask = mr->dirty_log_mask; - if (global_dirty_log && mr->ram_block) { + RAMBlock *rb = mr->ram_block; + + if (global_dirty_log && rb && qemu_ram_is_migratable(rb)) { mask |= (1 << DIRTY_MEMORY_MIGRATION); } return mask; @@ -2022,6 +2026,15 @@ static void memory_region_sync_dirty_bitmap(MemoryRegion *mr) if (!listener->log_sync) { continue; } + + /* We need to scan whole dirty page bytemap rather than + * every mrs for vfio memory listener. + */ + if (vfio_memory_listener_check(listener)) { + listener->log_sync(listener, NULL); + continue; + } + as = listener->address_space; view = address_space_get_flatview(as); FOR_EACH_FLAT_RANGE(fr, view) { diff --git a/migration/migration.c b/migration/migration.c index 187ac0410c..77afbce7ea 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -55,6 +55,8 @@ #include "qemu/queue.h" #include "multifd.h" +#include "hw/vfio/vfio-common.h" + #define MAX_THROTTLE (32 << 20) /* Migration transfer speed throttling */ /* Amount of time to allocate to each "chunk" of bandwidth-throttled @@ -448,6 +450,7 @@ static void process_incoming_migration_bh(void *opaque) static void process_incoming_migration_co(void *opaque) { MigrationIncomingState *mis = migration_incoming_get_current(); + MigrationState *ms = migrate_get_current(); PostcopyState ps; int ret; Error *local_err = NULL; @@ -505,6 +508,10 @@ static void process_incoming_migration_co(void *opaque) error_report("load of migration failed: %s", strerror(-ret)); goto fail; } + if (ms->parameters.memory_check == 1) { + migration_memory_check(); + ms->parameters.memory_check = 0; + } mis->bh = qemu_bh_new(process_incoming_migration_bh, mis); qemu_bh_schedule(mis->bh); mis->migration_incoming_co = NULL; @@ -1349,6 +1356,9 @@ static void migrate_params_test_apply(MigrateSetParameters *params, if (params->has_block_incremental) { dest->block_incremental = params->block_incremental; } + if (params->has_memory_check) { + dest->memory_check = params->memory_check; + } if (params->has_multifd_channels) { dest->multifd_channels = params->multifd_channels; } @@ -1452,6 +1462,9 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) if (params->has_block_incremental) { s->parameters.block_incremental = params->block_incremental; } + if (params->has_memory_check) { + s->parameters.memory_check = params->memory_check; + } if (params->has_multifd_channels) { s->parameters.multifd_channels = params->multifd_channels; } diff --git a/migration/ram.c b/migration/ram.c index 04f13feb2e..882f28882d 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -55,6 +55,7 @@ #include "savevm.h" #include "qemu/iov.h" #include "multifd.h" +#include "hw/vfio/vfio-common.h" /***********************************************************/ /* ram save/restore */ @@ -1670,6 +1671,7 @@ static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss, bool last_stage) { int tmppages, pages = 0; + int qemu_file_err; size_t pagesize_bits = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS; @@ -1690,6 +1692,12 @@ static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss, return tmppages; } + if (!last_stage) { + qemu_file_err = qemu_file_get_error(rs->f); + if (qemu_file_err != 0) + return qemu_file_err; + } + pages += tmppages; pss->page++; /* Allow rate limiting to happen in the middle of huge pages */ @@ -2588,6 +2596,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque) { RAMState **temp = opaque; RAMState *rs = *temp; + MigrationState *ms = migrate_get_current(); int ret = 0; WITH_RCU_READ_LOCK_GUARD() { @@ -2622,6 +2631,9 @@ static int ram_save_complete(QEMUFile *f, void *opaque) multifd_send_sync_main(rs->f); qemu_put_be64(f, RAM_SAVE_FLAG_EOS); qemu_fflush(f); + if (ms->parameters.memory_check == 1) { + migration_memory_check(); + } } return ret; diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c index 9b94e67879..351a5aa7a2 100644 --- a/monitor/hmp-cmds.c +++ b/monitor/hmp-cmds.c @@ -1319,6 +1319,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict) p->has_block_incremental = true; visit_type_bool(v, param, &p->block_incremental, &err); break; + case MIGRATION_PARAMETER_MEMORY_CHECK: + p->has_memory_check = true; + visit_type_bool(v, param, &p->memory_check, &err); + break; case MIGRATION_PARAMETER_MULTIFD_CHANNELS: p->has_multifd_channels = true; visit_type_int(v, param, &p->multifd_channels, &err); diff --git a/qapi/migration.json b/qapi/migration.json index eca2981d0a..44e586f889 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -636,7 +636,7 @@ 'multifd-channels', 'xbzrle-cache-size', 'max-postcopy-bandwidth', 'max-cpu-throttle', 'multifd-compression', - 'multifd-zlib-level' ,'multifd-zstd-level' ] } + 'multifd-zlib-level' ,'multifd-zstd-level', 'memory-check' ] } ## # @MigrateSetParameters: @@ -747,6 +747,8 @@ # will consume more CPU. # Defaults to 1. (Since 5.0) # +# @memory-check: whether enable memory check during migration process. +# # Since: 2.4 ## # TODO either fuse back into MigrationParameters, or make @@ -776,7 +778,8 @@ '*max-cpu-throttle': 'int', '*multifd-compression': 'MultiFDCompression', '*multifd-zlib-level': 'int', - '*multifd-zstd-level': 'int' } } + '*multifd-zstd-level': 'int', + '*memory-check': 'bool' } } ## # @migrate-set-parameters: @@ -934,7 +937,8 @@ '*max-cpu-throttle': 'uint8', '*multifd-compression': 'MultiFDCompression', '*multifd-zlib-level': 'uint8', - '*multifd-zstd-level': 'uint8' } } + '*multifd-zstd-level': 'uint8', + '*memory-check': 'bool' } } ## # @query-migrate-parameters: -- Gitee