From c58adb8d74685882167811b628fc5dc32a4b938b Mon Sep 17 00:00:00 2001 From: liuhao365 Date: Fri, 5 Dec 2025 10:14:44 +0800 Subject: [PATCH] virtcca secure cvm supports live migration this patch adds support for live migration in the secure CVM feature of virtcca. It ensures that the security context is maintained during the migration process. Signed-off-by: Hao Liu --- arch/arm64/include/asm/kvm_pgtable.h | 7 + arch/arm64/include/asm/kvm_tmi.h | 92 +- arch/arm64/include/asm/kvm_tmm.h | 382 +++ arch/arm64/include/asm/virtcca_cvm_smc.h | 82 +- arch/arm64/include/uapi/asm/virtcca_cvm_tsi.h | 32 + arch/arm64/kernel/virtcca_cvm_guest.c | 248 ++ arch/arm64/kernel/virtcca_cvm_tsi.c | 153 ++ arch/arm64/kvm/arm.c | 4 + arch/arm64/kvm/hyp/pgtable.c | 11 + arch/arm64/kvm/mmu.c | 16 + arch/arm64/kvm/tmi.c | 185 ++ arch/arm64/kvm/virtcca_cvm.c | 2189 ++++++++++++++++- include/linux/virtcca_cvm_domain.h | 18 + include/uapi/linux/kvm.h | 4 + kernel/dma/swiotlb.c | 1 + tools/include/uapi/linux/kvm.h | 5 + virt/kvm/kvm_main.c | 17 + 17 files changed, 3440 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 0c0ae56e8163..5a640122e241 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -767,3 +767,10 @@ enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte); void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, size_t size); #endif /* __ARM64_KVM_PGTABLE_H__ */ + +#ifdef CONFIG_HISI_VIRTCCA_HOST +int virtcca_stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, + u64 size, kvm_pte_t attr_set, + kvm_pte_t attr_clr, kvm_pte_t *orig_pte, + u32 *level, enum kvm_pgtable_walk_flags flags); +#endif diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index 9ed16e848ab9..c2423578f1b5 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -34,6 +34,8 @@ #define TMI_ERROR_TTT_DESTROY_AGAIN 14 #define TMI_ERROR_STE_CREATED 15 +#define TMI_IMPORT_INCOMPLETE 39 + #define TMI_RETURN_STATUS(ret) ((ret) & 0xFF) #define TMI_RETURN_INDEX(ret) (((ret) >> 8) & 0xFF) @@ -48,9 +50,11 @@ #define TMI_FEATURE_REGISTER_0_HASH_SHA_256 BIT(28) #define TMI_FEATURE_REGISTER_0_HASH_SHA_512 BIT(29) -#define TMI_CVM_PARAM_FLAG_LPA2 BIT(0) +#define TMI_CVM_PARAM_FLAG_LPA2 BIT(0) #define TMI_CVM_PARAM_FLAG_SVE BIT(1) #define TMI_CVM_PARAM_FLAG_PMU BIT(2) +#define TMI_CVM_PARAM_FLAG_MIG BIT(3) +#define TMI_CVM_PARAM_FLAG_MIGVM BIT(4) #define TMI_NOT_RUNNABLE 0 #define TMI_RUNNABLE 1 @@ -246,7 +250,7 @@ struct tmi_tec_run { #define TMI_FNUM_TTT_MAP_RANGE U(0x26D) #define TMI_FNUM_TTT_UNMAP_RANGE U(0x26E) #define TMI_FNUM_TTT_DESTROY U(0x26F) -#define TMI_FNUM_INF_TEST U(0x270) +#define TMI_FNUM_INF_TEST U(0x271) #define TMI_FNUM_KAE_INIT U(0x273) #define TMI_FNUM_KAE_ENABLE U(0x274) #define TMI_FNUM_INFO_SHOW U(0x275) @@ -269,6 +273,11 @@ struct tmi_tec_run { #define TMI_FNUM_DEVICE_CREATE U(0x286) #define TMI_FNUM_DEVICE_DESTROY U(0x287) +/* additional TMI call for migration */ +#define TMI_FNUM_MIG_CONTROL U(0x270) +#define TMI_FNUM_MIG_DATA U(0x272) +#define TMI_FNUM_MIG_ATTESTATION U(0x276) + /* TMI SMC64 PIDs handled by the SPMD */ #define TMI_TMM_VERSION_REQ TMI_FID(SMC_64, TMI_FNUM_VERSION_REQ) #define TMI_TMM_DATA_CREATE TMI_FID(SMC_64, TMI_FNUM_DATA_CREATE) @@ -309,6 +318,41 @@ struct tmi_tec_run { #define TMI_TMM_DEV_CREATE TMI_FID(SMC_64, TMI_FNUM_DEVICE_CREATE) #define TMI_TMM_DEV_DESTROY TMI_FID(SMC_64, TMI_FNUM_DEVICE_DESTROY) +/* additional TMI call for migration */ +#define TMI_TMM_MIG_CONTROL TMI_FID(SMC_64, TMI_FNUM_MIG_CONTROL) +#define TMI_TMM_MIG_DATA TMI_FID(SMC_64, TMI_FNUM_MIG_DATA) +#define TMI_TMM_MIG_ATTESTATION TMI_FID(SMC_64, TMI_FNUM_MIG_ATTESTATION) + +enum tmi_tmm_mig_control_fid_e { + TMI_TMM_GET_MIG_CONFIG, + TMI_TMM_MIG_STREAM_CREATE, + TMI_TMM_SET_TMM_MEMSLOT, + TMI_TMM_MIG_UPDATE_CVM_INFO, + TMI_TMM_MIG_MEM_REGION_PROTECT, + TMI_TMM_MIG_IMPORT_COMMIT, + TMI_TMM_DUMP_CHECKSUM, + TMI_TMM_MIG_EXPORT_ABORT, + TMI_TMM_MIG_EXPORT_PAUSE +}; + +enum tmi_tmm_mig_data_fid_e { + TMI_TMM_MIG_EXPORT_IMMUTABLE, + TMI_TMM_MIG_IMPORT_IMMUTABLE, + TMI_TMM_MIG_EXPORT_TRACK, + TMI_TMM_MIG_IMPORT_TRACK, + TMI_TMM_MIG_EXPORT_MEM, + TMI_TMM_MIG_IMPORT_MEM, + TMI_TMM_MIG_EXPORT_TEC, + TMI_TMM_MIG_IMPORT_TEC, + TMI_TMM_MIG_IS_ZERO_PAGE, + TMI_TMM_MIG_IMPORT_ZERO_PAGE +}; + +enum tmi_tmm_mig_attestation_fid_e { + TMI_TMM_MIG_BIND_CLEAN, + TMI_TMM_MIG_BIND_PEEK +}; + #define TMI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16) #define TMI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF) @@ -333,6 +377,8 @@ struct tmi_tec_run { #define KVM_CAP_ARM_TMM_CFG_DBG 3 #define KVM_CAP_ARM_TMM_CFG_PMU 4 #define KVM_CAP_ARM_TMM_CFG_KAE 5 +#define KVM_CAP_ARM_TMM_CFG_MIG 6 +#define KVM_CAP_ARM_TMM_CFG_MIG_CVM 7 #define KVM_CAP_ARM_TMM_MAX_KAE_VF_NUM 11 @@ -373,6 +419,18 @@ struct kvm_cap_arm_tmm_config_item { __u64 sec_addr[KVM_CAP_ARM_TMM_MAX_KAE_VF_NUM]; __u64 hpre_addr[KVM_CAP_ARM_TMM_MAX_KAE_VF_NUM]; }; + + /* cfg == KVM_CAP_ARM_TMM_CFG_MIG */ + struct { + __u32 mig_enable; + __u32 mig_src; + }; + + /* cfg == KVM_CAP_ARM_TMM_CFG_MIG_CVM */ + struct { + __u32 migration_migvm_cap; + }; + /* Fix the size of the union */ __u8 reserved[256]; }; @@ -454,5 +512,35 @@ int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, unsigned long map_size, enum kvm_pgtable_prot prot, int ret); void virtcca_cvm_set_secure_flag(void *vdev, void *info); bool is_virtcca_available(void); +u64 tmi_mig_stream_create(u64 rd, u64 numa_set); +u64 tmi_get_mig_config(void); +struct arm_smccc_res tmi_export_immutable(uint64_t rd, uint64_t hpa_and_size_pa, + uint64_t page_or_list, uint64_t mig_cmd); +u64 tmi_import_immutable(uint64_t rd, uint64_t hpa_and_size_pa, + uint64_t page_or_list, uint64_t mig_cmd); +struct arm_smccc_res tmi_import_mem(uint64_t rd, uint64_t mig_mem_param); +struct arm_smccc_res tmi_export_mem(uint64_t rd, uint64_t mig_mem_param); +u64 tmi_import_track(uint64_t rd, uint64_t hpa_and_size_pa, uint64_t mig_cmd); +u64 tmi_export_track(uint64_t rd, uint64_t hpa_and_size_pa, uint64_t mig_cmd); +u64 tmi_import_commit(uint64_t rd); +u64 tmi_dump_checksum(uint64_t rd, uint64_t gpa_list_addr, + uint64_t crc_result_addr, uint64_t granularity); +u64 tmi_export_abort(uint64_t rd); +struct arm_smccc_res tmi_is_zero_page(uint64_t rd, uint64_t gpa_list_info_val); +struct arm_smccc_res tmi_import_zero_page(uint64_t rd, uint64_t gpa_list_info_val); +u64 tmi_set_tmm_memslot(uint64_t rd, uint64_t mig_memslot_param); +u64 tmi_import_tec(uint64_t tec_pa, uint64_t mbmd_addr_and_size, + uint64_t page_list_pa, uint64_t stream_info_pa); +struct arm_smccc_res tmi_export_tec(uint64_t tec_pa, uint64_t mbmd_addr_and_size, + uint64_t page_list_pa, uint64_t stream_info_pa); +u64 tmi_update_cvm_info(uint64_t rd, uint64_t cvm_update_info_addr); +void virtcca_set_tmm_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot); +struct arm_smccc_res tmi_export_pause(uint64_t rd); + +/* enable the migcvm ctl */ +int kvm_migcvm_ioctl(struct kvm *kvm, unsigned long arg); +struct arm_smccc_res tmi_mem_region_protect(u64 rd, u64 start, u64 end); +u64 tmi_bind_clean(uint64_t rd); +struct arm_smccc_res tmi_bind_peek(uint64_t rd); #endif #endif diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h index 484940589c7c..540c1a13810d 100644 --- a/arch/arm64/include/asm/kvm_tmm.h +++ b/arch/arm64/include/asm/kvm_tmm.h @@ -14,6 +14,9 @@ enum virtcca_cvm_state { CVM_STATE_DYING }; +#define VIRTCCA_MIG_DST 0 +#define VIRTCCA_MIG_SRC 1 + #define MAX_KAE_VF_NUM 11 /* @@ -38,6 +41,19 @@ struct tmi_cvm_params { u64 kae_vf_num; u64 sec_addr[MAX_KAE_VF_NUM]; u64 hpre_addr[MAX_KAE_VF_NUM]; + u32 mig_enable; /* check the base capability of CVM migration */ + u32 mig_src; /* check the CVM is source or dest*/ + u32 migration_migvm_cap; /* the type of CVM (support migration) */ +}; + +/* the guest cvm and migcvm both use this structure */ +#define KVM_CVM_MIGVM_VERSION 0 +struct mig_cvm { + /* used by guest cvm */ + uint8_t version; /* kvm version of migcvm*/ + uint64_t migvm_cid; /* vsock cid of migvm */ + uint16_t dst_port; /* port of destination cvm */ + char dst_ip[16]; /* ip of destination cvm */ }; struct cvm { @@ -76,6 +92,11 @@ struct virtcca_cvm { struct kvm_numa_info numa_info; struct tmi_cvm_params *params; bool is_mapped; /* Whether the cvm RAM memory is mapped */ + struct virtcca_mig_state *mig_state; + struct mig_cvm *mig_cvm_info; + u64 swiotlb_start; + u64 swiotlb_end; + u64 ipa_start; }; /* @@ -123,6 +144,7 @@ int cvm_psci_complete(struct kvm_vcpu *calling, struct kvm_vcpu *target, unsigne void kvm_cvm_unmap_destroy_range(struct kvm *kvm); int kvm_cvm_map_range(struct kvm *kvm); +int kvm_cvm_mig_map_range(struct kvm *kvm); int virtcca_cvm_arm_smmu_domain_set_kvm(void *group); int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size, uint32_t is_map); @@ -157,6 +179,366 @@ static inline unsigned long cvm_ttt_level_mapsize(int level) return (1UL << CVM_TTT_LEVEL_SHIFT(level)); } + +/* virtcca MIG sub-ioctl() commands. */ +enum kvm_cvm_cmd_id { + /* virtcca MIG migcvm commands. */ + KVM_CVM_MIGCVM_SET_CID = 0, + KVM_CVM_MIGCVM_ATTEST, + KVM_CVM_MIGCVM_ATTEST_DST, + KVM_CVM_GET_BIND_STATUS, + KVM_CVM_MIG_EXPORT_ABORT, + /* virtcca MIG stream commands. */ + KVM_CVM_MIG_STREAM_START, + KVM_CVM_MIG_EXPORT_STATE_IMMUTABLE, + KVM_CVM_MIG_IMPORT_STATE_IMMUTABLE, + KVM_CVM_MIG_EXPORT_MEM, + KVM_CVM_MIG_IMPORT_MEM, + KVM_CVM_MIG_EXPORT_TRACK, + KVM_CVM_MIG_IMPORT_TRACK, + KVM_CVM_MIG_EXPORT_PAUSE, + KVM_CVM_MIG_EXPORT_STATE_TEC, + KVM_CVM_MIG_IMPORT_STATE_TEC, + KVM_CVM_MIG_IMPORT_END, + KVM_CVM_MIG_CRC, + KVM_CVM_MIG_GET_MIG_INFO, + KVM_CVM_MIG_IS_ZERO_PAGE, + KVM_CVM_MIG_IMPORT_ZERO_PAGE, + + KVM_CVM_MIG_CMD_NR_MAX, +}; + +struct kvm_virtcca_mig_cmd { + /* enum kvm_tdx_cmd_id */ + __u32 id; + /* flags for sub-commend. If sub-command doesn't use this, set zero. */ + __u32 flags; + /* + * data for each sub-command. An immediate or a pointer to the actual + * data in process virtual address. If sub-command doesn't use it, + * set zero. + */ + __u64 data; + /* + * Auxiliary error code. The sub-command may return TDX SEAMCALL + * status code in addition to -Exxx. + * Defined for consistency with struct kvm_sev_cmd. + */ + __u64 error; +}; + +/* mig virtcca head*/ +#define KVM_DEV_VIRTCCA_MIG_ATTR 0x1 + +struct kvm_dev_virtcca_mig_attr { +#define KVM_DEV_VIRTCCA_MIG_ATTR_VERSION 0 + __u32 version; +/* 4KB buffer can hold 512 entries at most */ +#define VIRTCCA_MIG_BUF_LIST_PAGES_MAX 512 + __u32 buf_list_pages; + __u32 max_migs; +}; + +#define VIRTCCA_MIG_STREAM_MBMD_MAP_OFFSET 0 +#define VIRTCCA_MIG_STREAM_GPA_LIST_MAP_OFFSET 1 +#define VIRTCCA_MIG_STREAM_MAC_LIST_MAP_OFFSET 2 +#define VIRTCCA_MIG_STREAM_BUF_LIST_MAP_OFFSET 4 + +struct virtcca_bind_info { + int16_t version; + bool premig_done; +}; + +struct virtcca_dst_host_info { + char dst_ip[16]; + uint16_t dst_port; + uint8_t version; +}; + +struct virtcca_mig_mbmd_data { /* both kvm and tmm can access */ + __u16 size; + __u16 mig_version; + __u16 migs_index; /* corresponding stream idx */ + __u8 mb_type; + __u8 rsvd0; /* reserve bit */ + __u32 mb_counter; + __u32 mig_epoch; + __u64 iv_counter; + __u8 type_specific_info[]; +} __packed; + +struct virtcca_mig_mbmd { + struct virtcca_mig_mbmd_data *data; + uint64_t hpa_and_size; /* Host physical address and size of the mbmd */ +}; + +#define VIRTCCA_MIG_EPOCH_START_TOKEN 0xffffffff + +/* + * The buffer list specifies a list of 4KB pages to be used by TDH_EXPORT_MEM + * and TDH_IMPORT_MEM to export and import guest memory pages. Each entry + * is 64-bit and points to a physical address of a 4KB page used as buffer. The + * list itself is a 4KB page, so it can hold up to 512 entries. + */ +union virtcca_mig_buf_list_entry { + uint64_t val; + struct { + uint64_t rsvd0 : 12; + uint64_t pfn : 40; + uint64_t rsvd1 : 11; + uint64_t invalid : 1; + }; +}; + +struct virtcca_mig_buf_list { + union virtcca_mig_buf_list_entry *entries; + // uint64_t *entries; + hpa_t hpa; +}; + +/* + * The page list specifies a list of 4KB pages to be used by the non-memory + * states export and import, i.e. TDH_EXPORT_STATE_* and TDH_IMPORT_STATE_*. + * Each entry is 64-bit and specifies the physical address of a 4KB buffer. + * The list itself is a 4KB page, so it can hold up to 512 entries. + */ +union virtcca_mig_page_list_info { + uint64_t val; + struct { + uint64_t rsvd0 : 12; + uint64_t pfn : 40; + uint64_t rsvd1 : 3; + uint64_t last_entry : 9; + }; +}; + +struct virtcca_mig_page_list { + hpa_t *entries; + union virtcca_mig_page_list_info info; +}; + +union virtcca_mig_gpa_list_entry { + uint64_t val; + struct{ + uint64_t level : 2; /* Bits 1:0: Mapping level */ + uint64_t pending : 1; /* Bit 2: Page is pending */ + uint64_t reserved_0 : 4; /* Bits 6:3 */ + uint64_t l2_map : 3; /* Bits 9:7: L2 mapping flags */ + uint64_t mig_type : 2; /* Bits 11:10: Migration type */ + uint64_t gfn : 40; /* Bits 51:12 */ +#define GPA_LIST_OP_NOP 0 +#define GPA_LIST_OP_EXPORT 1 +#define GPA_LIST_OP_CANCEL 2 + uint64_t operation : 2; /* Bits 53:52 */ + uint64_t reserved_1 : 2; /* Bits 55:54 */ +#define GPA_LIST_S_SUCCESS 0 + uint64_t status : 5; /* Bits 56:52 */ + uint64_t reserved_2 : 3; /* Bits 63:61 */ + }; +}; + +#define TMM_MAX_DIRTY_BITMAP_LEN 8 +/* + * The GPA list specifies a list of GPAs to be used by TDH_EXPORT_MEM and + * TDH_IMPORT_MEM, TDH_EXPORT_BLOCKW, and TDH_EXPORT_RESTORE. The list itself + * is 4KB, so it can hold up to 512 such 64-bit entries. + */ +union virtcca_mig_ipa_list_info { + uint64_t val; + struct { + uint64_t rsvd0 : 3; + uint64_t first_entry: 9; + uint64_t pfn : 40; + uint64_t rsvd1 : 3; + uint64_t last_entry : 9; + }; +}; + +struct virtcca_mig_gpa_list { + union virtcca_mig_gpa_list_entry *entries; + union virtcca_mig_ipa_list_info info; +}; + +/* + * A MAC list specifies a list of MACs over 4KB migrated pages and their GPA + * entries. It is used by TDH_EXPORT_MEM and TDH_IMPORT_MEM. Each entry is + * 128-bit containing a single AES-GMAC-256 of a migrated page. The list itself + * is a 4KB page, so it can hold up to 256 entries. To support the export and + * import of 512 pages, two such MAC lists are needed to be passed to the TDX + * module. + */ +struct virtcca_mig_mac_list { + void *entries; + hpa_t hpa; +}; + +union virtcca_mig_stream_info { + uint64_t val; + struct { + uint64_t index : 16; + uint64_t rsvd : 47; + uint64_t resume : 1; + }; + struct { + uint64_t rsvd1 : 63; + uint64_t in_order : 1; + }; +}; + +struct virtcca_mig_stream { + uint16_t idx; /* stream id */ + uint32_t buf_list_pages; /* ns memory page number of buf_list 5 #include #include +#include #include #include @@ -229,3 +230,250 @@ void virtcca_its_free_shared_pages(void *addr, int order) swiotlb_free(&cvm_alloc_device, (struct page *)addr, (1 << order) * PAGE_SIZE); } + +#define DEVICE_NAME "migvm_queue_mem" + +// IOCTL cmd define +#define QUEUE_IOCTL_MAGIC 'q' +#define MIGVM_CREATE_QUEUE _IOWR(QUEUE_IOCTL_MAGIC, 1, unsigned long) +#define MIGVM_DESTROY_QUEUE _IO(QUEUE_IOCTL_MAGIC, 2) + +#define QUEUE_SIZE (64 * 1024 - 8) // Queue size (64KB - 8B) +#define ALLOC_PAGE (4) + +struct driver_data { + dev_t devno; + struct cdev cdev; + struct class *cls; + struct mig_queue_device *queue_dev; +}; + +struct mig_integrity_share_queue_addr_s { + uint64_t send_buf_ipa; + uint64_t recv_buf_ipa; +}; + +struct mig_queue_device { + struct page *send_page; + struct page *recv_page; + unsigned long rd; + struct mig_integrity_share_queue_addr_s queue_addr; +}; + +static struct driver_data *driver_data_ptr; + +static struct mig_queue_device *migvm_queue_dev_create(unsigned long rd) +{ + struct mig_queue_device *dev = kzalloc(sizeof(*dev), GFP_KERNEL); + + if (!dev) + return NULL; + + /* Allocate send buffer */ + dev->send_page = alloc_pages(GFP_KERNEL, ALLOC_PAGE); + if (!dev->send_page) { + kfree(dev); + return NULL; + } + dev->queue_addr.send_buf_ipa = page_to_phys(dev->send_page); + memset(page_address(dev->send_page), 0, QUEUE_SIZE); + + /* Allocate receive buffer */ + dev->recv_page = alloc_pages(GFP_KERNEL, ALLOC_PAGE); + if (!dev->recv_page) { + __free_page(dev->send_page); + kfree(dev); + return NULL; + } + + dev->queue_addr.recv_buf_ipa = page_to_phys(dev->recv_page); + memset(page_address(dev->recv_page), 0, QUEUE_SIZE); + + if (tsi_mig_integrity_checksum_init(rd, virt_to_phys((void *)&dev->queue_addr))) { + __free_page(dev->send_page); + __free_page(dev->recv_page); + kfree(dev); + return NULL; + } + + dev->rd = rd; + return dev; +} + +static void migvm_queue_dev_destroy(struct mig_queue_device *dev) +{ + if (dev) { + if (dev->recv_page) + __free_page(dev->recv_page); + if (dev->send_page) + __free_page(dev->send_page); + kfree(dev); + } +} + +static int migvm_queue_dev_mmap(struct mig_queue_device *dev, struct vm_area_struct *vma) +{ + if (!dev || !vma) + return -EINVAL; + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long pfn; + + switch (vma->vm_pgoff) { + case 0: + pfn = dev->queue_addr.send_buf_ipa >> PAGE_SHIFT; + break; + case 1: + pfn = dev->queue_addr.recv_buf_ipa >> PAGE_SHIFT; + break; + default: + return -EINVAL; + } + + if (remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot)) + return -EAGAIN; + return 0; +} + +static int migvm_queue_open(struct inode *inode, struct file *filp) +{ + if (!inode) + return -EINVAL; + + struct driver_data *data = container_of(inode->i_cdev, struct driver_data, cdev); + + if (!data || !filp) + return -EINVAL; + data->queue_dev = NULL; + filp->private_data = data; + return 0; +} + +static int migvm_queue_release(struct inode *inode, struct file *filp) +{ + if (!filp) + return -EINVAL; + + struct driver_data *data = filp->private_data; + + if (!data) + return -EINVAL; + if (data->queue_dev) { + migvm_queue_dev_destroy(data->queue_dev); + data->queue_dev = NULL; + } + return 0; +} + +static long migvm_queue_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + if (!filp) + return -EINVAL; + + struct driver_data *data = filp->private_data; + + if (!data) + return -EINVAL; + + switch (cmd) { + case MIGVM_CREATE_QUEUE: + if (!data->queue_dev) { + unsigned long rd; + + if (copy_from_user(&rd, (void __user *)arg, sizeof(unsigned long))) + return -EFAULT; + data->queue_dev = migvm_queue_dev_create(rd); + + if (!data->queue_dev) + return -ENOMEM; + } + break; + case MIGVM_DESTROY_QUEUE: + if (data->queue_dev) { + migvm_queue_dev_destroy(data->queue_dev); + data->queue_dev = NULL; + } + break; + default: + return -ENOTTY; + } + return 0; +} + +static int migvm_queue_mmap(struct file *filp, struct vm_area_struct *vma) +{ + if (!filp || !vma) + return -EINVAL; + + struct driver_data *data = filp->private_data; + + if (!data) + return -EINVAL; + + return migvm_queue_dev_mmap(data->queue_dev, vma); +} + +static const struct file_operations queue_fops = { + .owner = THIS_MODULE, + .open = migvm_queue_open, + .release = migvm_queue_release, + .unlocked_ioctl = migvm_queue_ioctl, + .mmap = migvm_queue_mmap, +}; + +static int __init migvm_queue_driver_init(void) +{ + int ret; + struct driver_data *data; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + ret = alloc_chrdev_region(&data->devno, 0, 1, DEVICE_NAME); + if (ret < 0) + goto fail_alloc_dev; + + cdev_init(&data->cdev, &queue_fops); + data->cdev.owner = THIS_MODULE; + + ret = cdev_add(&data->cdev, data->devno, 1); + if (ret < 0) + goto fail_cdev_add; + + data->cls = class_create(DEVICE_NAME); + if (IS_ERR(data->cls)) { + ret = PTR_ERR(data->cls); + goto fail_class_create; + } + + device_create(data->cls, NULL, data->devno, NULL, DEVICE_NAME); + driver_data_ptr = data; + + pr_info("Migvm queue driver loaded. Major=%d\n", MAJOR(data->devno)); + return 0; + +fail_class_create: + cdev_del(&data->cdev); +fail_cdev_add: + unregister_chrdev_region(data->devno, 1); +fail_alloc_dev: + kfree(data); + return ret; +} + +static void __exit migvm_queue_driver_exit(void) +{ + if (driver_data_ptr) { + device_destroy(driver_data_ptr->cls, driver_data_ptr->devno); + class_destroy(driver_data_ptr->cls); + cdev_del(&driver_data_ptr->cdev); + unregister_chrdev_region(driver_data_ptr->devno, 1); + kfree(driver_data_ptr); + driver_data_ptr = NULL; + } + pr_info("Migvm queue driver unloaded\n"); +} + +module_init(migvm_queue_driver_init); +module_exit(migvm_queue_driver_exit); +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/kernel/virtcca_cvm_tsi.c b/arch/arm64/kernel/virtcca_cvm_tsi.c index dc3238d5fff7..fb62f278c0e8 100644 --- a/arch/arm64/kernel/virtcca_cvm_tsi.c +++ b/arch/arm64/kernel/virtcca_cvm_tsi.c @@ -22,6 +22,8 @@ static long tmm_tsi_ioctl(struct file *file, unsigned int cmd, unsigned long arg static int tmm_get_tsi_version(struct virtcca_cvm_tsi_version __user *arg); static int tmm_get_attestation_token(struct virtcca_cvm_attestation_cmd __user *arg); static int tmm_get_device_cert(struct virtcca_device_cert __user *arg); +static int tmm_get_set_migration_info(struct virtcca_migvm_info __user *arg); +static int tmm_migvm_mem_checksum_loop(unsigned long rd); static const struct file_operations tmm_tsi_fops = { .owner = THIS_MODULE, @@ -68,6 +70,8 @@ static void __exit tmm_tsi_exit(void) static long tmm_tsi_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { int ret; + unsigned long rd; + void __user *argp = (void __user *)arg; switch (cmd) { case TMM_GET_TSI_VERSION: @@ -79,6 +83,16 @@ static long tmm_tsi_ioctl(struct file *file, unsigned int cmd, unsigned long arg case TMM_GET_DEVICE_CERT: ret = tmm_get_device_cert((struct virtcca_device_cert *)arg); break; + case TMM_GET_MIGRATION_INFO: + ret = tmm_get_set_migration_info((struct virtcca_migvm_info *)arg); + break; + case TMM_GET_MIGVM_MEM_CHECKSUM: + if (copy_from_user(&rd, argp, sizeof(unsigned long))) { + pr_err("tmm_tsi: mem checksum copy data from user failed\n"); + return -ENOTTY; + } + ret = tmm_migvm_mem_checksum_loop(rd); + break; default: pr_err("tmm_tsi: unknown ioctl command (0x%x)!\n", cmd); return -ENOTTY; @@ -87,6 +101,15 @@ static long tmm_tsi_ioctl(struct file *file, unsigned int cmd, unsigned long arg return ret; } +static int tmm_migvm_mem_checksum_loop(unsigned long rd) +{ + unsigned long ret; + + ret = tsi_mig_integrity_checksum_loop(rd); + + return ret; +} + static int tmm_get_tsi_version(struct virtcca_cvm_tsi_version __user *arg) { struct virtcca_cvm_tsi_version ver_measured = {0}; @@ -204,6 +227,136 @@ static int tmm_get_device_cert(struct virtcca_device_cert __user *arg) return 0; } +static int tmm_get_set_migration_info(struct virtcca_migvm_info __user *arg) +{ + unsigned long ret = 0; + struct virtcca_migvm_info migvm_info = {0}; + struct pending_guest_rd_s *rdcontent = NULL; + + if (!access_ok(arg, sizeof(*arg))) { + pr_err("tmm_tsi: invalid user pointer\n"); + ret = -EFAULT; + goto out; + } + + ret = copy_from_user(&migvm_info, arg, sizeof(struct virtcca_migvm_info)); + if (ret) { + pr_err("tmm_tsi: copy challenge from user failed (%lu)!\n", ret); + ret = -EFAULT; + goto out; + } + + if (migvm_info.content) { + if (!access_ok(migvm_info.content, migvm_info.size)) { + pr_err("tmm_tsi: invalid content address\n"); + ret = -EFAULT; + goto out; + } + } else { + pr_err("tmm_tsi: invalid content pointer\n"); + goto out; + } + + struct migration_info *kcontent = kmalloc(sizeof(struct migration_info), GFP_KERNEL); + + if (!kcontent) { + ret = -ENOMEM; + goto out; + } + if (sizeof(struct migration_info) != migvm_info.size) { + pr_err("tmm_tsi: size mismatch\n"); + ret = -EINVAL; + goto out; + } + + switch (migvm_info.ops) { + case OP_MIGRATE_GET_ATTR: { + if (copy_from_user(kcontent, migvm_info.content, migvm_info.size)) { + pr_err("tmm_tsi: copy slot value failed\n"); + ret = -EFAULT; + goto out; + } + ret = tsi_migvm_get_attr(migvm_info.guest_rd, kcontent); + if (!ret) { + if (copy_to_user(migvm_info.content, kcontent, migvm_info.size)) { + pr_err("tmm_tsi: copy to user failed\n"); + ret = -EFAULT; + } + pr_info("tmm_tsi: OP_MIGRATE_GET_ATTRT\n"); + } else { + pr_err("tmm_tsi: get attr failed, ret = 0x%lx\n", ret); + } + + break; + } + case OP_MIGRATE_SET_SLOT: { + if (copy_from_user(kcontent, migvm_info.content, migvm_info.size)) { + pr_err("tmm_tsi: copy slot value failed\n"); + ret = -EFAULT; + goto out; + } + ret = tsi_migvm_set_slot(migvm_info.guest_rd, kcontent); + if (ret) { + pr_err("tmm_tsi: set slot failed, ret = %lx\n", ret); + ret = -EINVAL; + } + break; + } + case OP_MIGRATE_PEEK_RDS: { + if (copy_from_user(kcontent, migvm_info.content, migvm_info.size)) { + pr_err("tmm_tsi: copy slot value failed\n"); + ret = -EFAULT; + goto out; + } + + if (kcontent->pending_guest_rds) { + if (!access_ok(kcontent->pending_guest_rds, + sizeof(struct pending_guest_rd_s))) { + pr_err("tmm_tsi: invalid content pending guest rds address\n"); + ret = -EFAULT; + goto out; + } + } else { + pr_err("tmm_tsi: invalid content pending guest rds pointer\n"); + goto out; + } + + rdcontent = kmalloc(sizeof(struct pending_guest_rd_s), GFP_KERNEL); + if (!rdcontent) { + ret = -ENOMEM; + goto out; + } + + if (copy_from_user(rdcontent, kcontent->pending_guest_rds, + sizeof(struct pending_guest_rd_s))) { + pr_err("tmm_tsi: copy slot value failed\n"); + ret = -EFAULT; + goto out; + } + + ret = tsi_peek_binding_list(rdcontent); + if (!ret) { + if (copy_to_user(kcontent->pending_guest_rds, rdcontent, + sizeof(struct pending_guest_rd_s))) { + pr_err("tmm_tsi: copy to user failed\n"); + ret = -EFAULT; + } + } else { + pr_err("tmm_tsi: peek rds failed, ret = 0x%lx\n", ret); + } + break; + } + default: + pr_err("tmm_tsi: invalid operation (%u)!\n", migvm_info.ops); + ret = -EINVAL; + } + +out: + kfree(kcontent); + kfree(rdcontent); + + return ret; +} module_init(tmm_tsi_init); module_exit(tmm_tsi_exit); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e59152ad5c4a..dbb07060fcbf 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2040,6 +2040,10 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) case KVM_LOAD_USER_DATA: { return kvm_load_user_data(kvm, arg); } +/* add the migcvm ioctl*/ + case KVM_CVM_MIG_IOCTL: { + return kvm_migcvm_ioctl(kvm, arg); + } #endif case KVM_CREATE_IRQCHIP: { int ret; diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 874244df723e..82c09d1f045e 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1617,3 +1617,14 @@ void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *p WARN_ON(mm_ops->page_count(pgtable) != 1); mm_ops->put_page(pgtable); } + +#ifdef CONFIG_HISI_VIRTCCA_HOST +int virtcca_stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, + u64 size, kvm_pte_t attr_set, + kvm_pte_t attr_clr, kvm_pte_t *orig_pte, + u32 *level, enum kvm_pgtable_walk_flags flags) +{ + return stage2_update_leaf_attrs(pgt, addr, size, attr_set, + attr_clr, orig_pte, level, flags); +} +#endif diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index cd7589d05064..9db4aaa1083c 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1239,6 +1239,22 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, lockdep_assert_held_write(&kvm->mmu_lock); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (kvm_is_realm(kvm)) { + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + + if (end <= cvm->ipa_start || start >= cvm->ipa_start + cvm->ram_size) + goto handle_ns_mem; + + if (start >= cvm->swiotlb_end || end <= cvm->swiotlb_start) + return; + start = (start < cvm->swiotlb_start) ? cvm->swiotlb_start : start; + end = (end < cvm->swiotlb_end) ? end : cvm->swiotlb_end; + } + +handle_ns_mem: +#endif + stage2_wp_range(&kvm->arch.mmu, start, end); /* diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c index 6ad1a99b0b9a..baf1b14cf3a4 100644 --- a/arch/arm64/kvm/tmi.c +++ b/arch/arm64/kvm/tmi.c @@ -404,3 +404,188 @@ u64 tmi_dev_destroy(u64 dev_num, u64 clean) arm_smccc_1_1_smc(TMI_TMM_DEV_DESTROY, dev_num, clean, &res); return res.a1; } + +/* additional TMI call for migration */ +u64 tmi_get_mig_config(void) +{ + struct arm_smccc_res res; + + /* calculate the max number of these pages(rd,vcpu) */ + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_GET_MIG_CONFIG, &res); + return res.a1; +} + +u64 tmi_mig_stream_create(u64 rd, u64 numa_set) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_STREAM_CREATE, rd, numa_set, &res); + return res.a1; +} + +u64 tmi_set_tmm_memslot(u64 rd, u64 mig_memslot_param) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_SET_TMM_MEMSLOT, + rd, mig_memslot_param, &res); + return res.a1; +} + +u64 tmi_update_cvm_info(u64 rd, u64 cvm_update_info_addr) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_UPDATE_CVM_INFO, rd, + __pa(cvm_update_info_addr), &res); + return res.a1; +} + +struct arm_smccc_res tmi_mem_region_protect(u64 rd, u64 start, u64 end) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_MEM_REGION_PROTECT, + rd, start, end, &res); + return res; +} + +u64 tmi_import_commit(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_IMPORT_COMMIT, rd, &res); + return res.a1; +} + +u64 tmi_dump_checksum(u64 rd, u64 gpa_list_addr, u64 crc_result_addr, u64 granularity) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_DUMP_CHECKSUM, + rd, gpa_list_addr, crc_result_addr, granularity, &res); + return res.a1; +} + +u64 tmi_export_abort(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_EXPORT_ABORT, rd, &res); + return res.a1; +} + +struct arm_smccc_res tmi_export_immutable(u64 rd, u64 hpa_and_size_pa, + u64 page_or_list, u64 mig_cmd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_EXPORT_IMMUTABLE, + rd, hpa_and_size_pa, page_or_list, mig_cmd, &res); + return res; +} + +u64 tmi_import_immutable(u64 rd, u64 hpa_and_size_pa, + u64 page_or_list, u64 mig_cmd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_IMMUTABLE, + rd, hpa_and_size_pa, page_or_list, mig_cmd, &res); + return res.a1; +} + +u64 tmi_export_track(u64 rd, u64 hpa_and_size_pa, u64 mig_cmd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_EXPORT_TRACK, + rd, hpa_and_size_pa, mig_cmd, &res); + return res.a1; +} + +u64 tmi_import_track(u64 rd, u64 hpa_and_size_pa, u64 mig_cmd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_TRACK, + rd, hpa_and_size_pa, mig_cmd, &res); + return res.a1; +} + +struct arm_smccc_res tmi_import_mem(u64 rd, u64 mig_mem_param) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_MEM, rd, mig_mem_param, &res); + return res; +} + +struct arm_smccc_res tmi_export_mem(u64 rd, u64 mig_mem_param) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_EXPORT_MEM, + rd, mig_mem_param, &res); + return res; +} + +struct arm_smccc_res tmi_export_tec(u64 tec_pa, u64 mbmd_addr_and_size, + u64 page_list_pa, u64 stream_info_pa) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_EXPORT_TEC, + tec_pa, mbmd_addr_and_size, page_list_pa, stream_info_pa, &res); + return res; +} + +u64 tmi_import_tec(u64 tec_pa, u64 mbmd_addr_and_size, u64 page_list_pa, u64 stream_info_pa) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_TEC, + tec_pa, mbmd_addr_and_size, page_list_pa, stream_info_pa, &res); + return res.a1; +} + +struct arm_smccc_res tmi_is_zero_page(u64 rd, u64 gpa_list_info_val) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IS_ZERO_PAGE, + rd, gpa_list_info_val, &res); + return res; +} + +struct arm_smccc_res tmi_import_zero_page(u64 rd, u64 gpa_list_info_val) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_DATA, TMI_TMM_MIG_IMPORT_ZERO_PAGE, + rd, gpa_list_info_val, &res); + return res; +} + +struct arm_smccc_res tmi_export_pause(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_CONTROL, TMI_TMM_MIG_EXPORT_PAUSE, rd, &res); + return res; +} + +u64 tmi_bind_clean(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_ATTESTATION, TMI_TMM_MIG_BIND_CLEAN, rd, &res); + return res.a1; +} +struct arm_smccc_res tmi_bind_peek(u64 rd) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_MIG_ATTESTATION, TMI_TMM_MIG_BIND_PEEK, rd, &res); + return res; +} diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index c270f33ce933..4fffcf30b4be 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -7,17 +7,22 @@ #include #include #include +#include +#include #include #include #include #include +#include +#include #include #include -#include +#include #include #include - -#include +#include +#include +#include /* Protects access to cvm_vmid_bitmap */ static DEFINE_SPINLOCK(cvm_vmid_lock); @@ -29,6 +34,32 @@ static bool virtcca_vtimer_adjust; #define UEFI_DTB_START 0x40000000 #define DTB_MAX_SIZE 0x200000 +#define SEC_CRC_PATH "/tmp/sec_memory_check" +#define NS_CRC_PATH "/tmp/ns_memory_check" +#define CRC_DUMP_CHUNK_SIZE 512 +#define FILE_NAME_LEN 256 + +#define DEFAULT_IPA_START 0x40000000 +#define CRC_POLYNOMIAL 0xEDB88320 +#define CRC_LEN 512 +#define CRC_SHIFT 8 +#define MAX_MAC_PAGES_PER_ARR 256 +#define MAX_BUF_PAGES 512 +/* migvm vsock retry times */ +#define SEND_RETRY_LIMIT 5 +#define RECV_RETRY_LIMIT 5 +#define CONNECT_RETRY_LIMIT 3 +#define TMI_IMPORT_TIMEOUT_MS 600000 +#define TMI_TRACK_TIMEOUT_MS 20 + +static struct virtcca_mig_capabilities g_virtcca_mig_caps; +static struct migcvm_agent_listen_cids g_migcvm_agent_listen_cid; + +static struct crc_config g_crc_configs[2] = { + [0] = { .is_secure = false, .enabled = false }, /* non-secure mem */ + [1] = { .is_secure = true, .enabled = false } /* secure mem */ +}; + bool is_virtcca_available(void) { return static_key_enabled(&virtcca_cvm_is_enable); @@ -175,6 +206,39 @@ int kvm_arm_create_cvm(struct kvm *kvm) goto out; } + if (cvm->params->mig_enable) { + ret = kvm_virtcca_mig_stream_ops_init(); /* init the migration main struct */ + if (ret) { + kvm_err("KVM support migstream ops init failed: %d\n", cvm->cvm_vmid); + ret = -ENOMEM; + goto out; + } + + ret = virtcca_mig_capabilities_setup(cvm); + if (ret) { + kvm_err("KVM support migration cap setup failed: %d\n", cvm->cvm_vmid); + ret = -ENOMEM; + goto out; + } + + /* this state might along with the protected memory */ + ret = virtcca_mig_state_create(cvm); + if (ret) { + kvm_err("KVM support mig state create failed: %d\n", cvm->cvm_vmid); + ret = -ENOMEM; + goto out; + } + } else { + pr_warn("Migration Capability is not set\n"); + } + + if (cvm->params->migration_migvm_cap) { + cvm->mig_cvm_info = kzalloc(sizeof(struct mig_cvm), GFP_KERNEL_ACCOUNT); + if (!cvm->mig_cvm_info) + return -ENOMEM; + pr_info("This CVM is Mig-CVM\n"); + } + WRITE_ONCE(cvm->state, CVM_STATE_NEW); ret = 0; out: @@ -200,6 +264,18 @@ void kvm_destroy_cvm(struct kvm *kvm) if (!cvm) return; + /* disable mig config and clean binding state*/ + if (cvm->mig_state) { + virtcca_mig_state_release(cvm); + kvm_virtcca_mig_stream_ops_exit(); + ret = tmi_bind_clean(cvm->rd); + if (ret) + pr_err("KVM destroy cVM mig tmi_bind_clean failed\n"); + kfree(cvm->mig_state); + } + + kfree(cvm->mig_cvm_info); + #ifdef CONFIG_HISI_VIRTCCA_CODA /* Unmap the cvm with arm smmu domain */ kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list); @@ -404,6 +480,9 @@ int kvm_finalize_vcpu_tec(struct kvm_vcpu *vcpu) struct virtcca_cvm *cvm = vcpu->kvm->arch.virtcca_cvm; struct virtcca_cvm_tec *tec = &vcpu->arch.tec; + if (tec->tec_created) + return 0; + mutex_lock(&vcpu->kvm->lock); tec->run = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!tec->run) { @@ -521,6 +600,32 @@ static int config_cvm_kae(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *c return 0; } +/* Get the qemu's transport migration config */ +static int config_cvm_migration(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *cfg) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct tmi_cvm_params *params; + + pr_info("calling %s\n", __func__); + params = cvm->params; + params->mig_enable = cfg->mig_enable; + params->mig_src = cfg->mig_src; + params->flags |= TMI_CVM_PARAM_FLAG_MIG; + return 0; +} + +static int config_cvm_migvm(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *cfg) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct tmi_cvm_params *params; + + params = cvm->params; + + params->migration_migvm_cap = cfg->migration_migvm_cap; + params->flags |= TMI_CVM_PARAM_FLAG_MIGVM; + return 0; +} + static int kvm_tmm_config_cvm(struct kvm *kvm, struct kvm_enable_cap *cap) { struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; @@ -546,6 +651,12 @@ static int kvm_tmm_config_cvm(struct kvm *kvm, struct kvm_enable_cap *cap) case KVM_CAP_ARM_TMM_CFG_KAE: r = config_cvm_kae(kvm, &cfg); break; + case KVM_CAP_ARM_TMM_CFG_MIG: /* enable the mig config of cvm */ + r = config_cvm_migration(kvm, &cfg); + break; + case KVM_CAP_ARM_TMM_CFG_MIG_CVM: + r = config_cvm_migvm(kvm, &cfg); + break; default: r = -EINVAL; @@ -590,6 +701,54 @@ int kvm_cvm_map_range(struct kvm *kvm) return ret; } + +int kvm_cvm_mig_map_range(struct kvm *kvm) +{ + int ret = 0; + u64 curr_numa_set; + int idx; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct kvm_numa_info *numa_info = &cvm->numa_info; + struct kvm_numa_node *numa_node; + gpa_t gpa; + + curr_numa_set = kvm_get_first_binded_numa_set(kvm); + /* uefi boot */ + if (cvm->ipa_start == UEFI_LOADER_START) { + gpa = cvm->ipa_start; + numa_node = &numa_info->numa_nodes[0]; + ret = tmi_ttt_map_range(cvm->rd, gpa, + UEFI_SIZE, + curr_numa_set, numa_node->host_numa_nodes[0]); + if (ret) { + kvm_err("tmi_ttt_map_range failed: %d.\n", ret); + return ret; + } + } + + for (idx = 0; idx < numa_info->numa_cnt; idx++) { + numa_node = &numa_info->numa_nodes[idx]; + gpa = numa_node->ipa_start; + if (gpa >= numa_node->ipa_start && + gpa < numa_node->ipa_start + numa_node->ipa_size) { + ret = tmi_ttt_map_range(cvm->rd, gpa, + numa_node->ipa_size, + curr_numa_set, numa_node->host_numa_nodes[0]); + if (ret) { + kvm_err("tmi_ttt_map_range failed: %d.\n", ret); + return ret; + } + } + } + /* Vfio driver will pin memory in advance, + * if the ram already mapped, activate cvm + * does not need to map twice + */ + cvm->is_mapped = true; + return ret; +} + + static int kvm_activate_cvm(struct kvm *kvm) { #ifdef CONFIG_HISI_VIRTCCA_CODA @@ -599,6 +758,19 @@ static int kvm_activate_cvm(struct kvm *kvm) #endif struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + if (cvm->mig_state) { + kvm_info("%s: vm->mig_state->mig_src = %d", __func__, cvm->mig_state->mig_src); + if (cvm->mig_state->mig_src == VIRTCCA_MIG_DST) { + kvm_info("%s: vm->mig_state->mig_src == VIRTCCA_MIG_DST", __func__); + return 0; + } + } + + if (virtcca_cvm_state(kvm) == CVM_STATE_ACTIVE) { + kvm_info("cVM%d is already activated!\n", cvm->cvm_vmid); + return 0; + } + if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) return -EINVAL; @@ -656,6 +828,18 @@ static int kvm_populate_ipa_cvm_range(struct kvm *kvm, u64 l2_granule = cvm_granule_size(TMM_TTT_LEVEL_2); phys_addr_t ipa_base1, ipa_end2; + /* + * if cvm comes from live migraion, mem is already maped. + * Set mig_state to VIRTCCA_MIG_SRC, init live migration structs. + */ + if (cvm->mig_state) { + if (cvm->mig_state->mig_src == VIRTCCA_MIG_DST) { + cvm->mig_state->mig_src = VIRTCCA_MIG_SRC; + kvm_info("the ipa range is populated before migraion\n"); + return 0; + } + } + if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) return -EINVAL; if (!IS_ALIGNED(args->populate_ipa_base1, PAGE_SIZE) || @@ -673,6 +857,8 @@ static int kvm_populate_ipa_cvm_range(struct kvm *kvm, ipa_base1 = round_down(args->populate_ipa_base1, l2_granule); ipa_end2 = round_up(args->populate_ipa_base2 + args->populate_ipa_size2, l2_granule); + cvm->ipa_start = ipa_base1; + /* uefi boot, uefi image and uefi ram from 0 to 128M */ if (ipa_base1 == UEFI_LOADER_START) { phys_addr_t ipa_base2 = round_down(args->populate_ipa_base2, l2_granule); @@ -898,6 +1084,40 @@ static inline bool is_dtb_info_has_extend_data(u64 dtb_info) return dtb_info & 0x1; } +int kvm_migcvm_ioctl(struct kvm *kvm, unsigned long arg) +{ + struct kvm_virtcca_mig_cmd cvm_cmd; + int ret = 0; + void __user *argp = (void __user *)arg; + + if (copy_from_user(&cvm_cmd, argp, sizeof(struct kvm_virtcca_mig_cmd))) + return -EINVAL; + + if (cvm_cmd.id < KVM_CVM_MIGCVM_SET_CID || cvm_cmd.id >= KVM_CVM_MIG_STREAM_START) + return -EINVAL; + + switch (cvm_cmd.id) { + case KVM_CVM_MIGCVM_SET_CID: + ret = virtcca_save_migvm_cid(kvm, &cvm_cmd); + break; + case KVM_CVM_MIGCVM_ATTEST: + ret = virtcca_migvm_agent_ratstls(kvm, &cvm_cmd); + break; + case KVM_CVM_MIGCVM_ATTEST_DST: + ret = virtcca_migvm_agent_ratstls_dst(kvm, &cvm_cmd); + break; + case KVM_CVM_GET_BIND_STATUS: + ret = virtcca_get_bind_info(kvm, &cvm_cmd); + break; + case KVM_CVM_MIG_EXPORT_ABORT: + ret = virtcca_mig_export_abort(kvm); + break; + default: + return -EINVAL; + } + return ret; +} + int kvm_load_user_data(struct kvm *kvm, unsigned long arg) { struct kvm_user_data user_data; @@ -1317,6 +1537,14 @@ int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, if (!is_virtcca_cvm_enable() || !kvm_is_realm(kvm)) return ret; + if (kvm->arch.virtcca_cvm->mig_state && + kvm->arch.virtcca_cvm->mig_state->mig_src == VIRTCCA_MIG_SRC) { + if (ipa >= kvm->arch.virtcca_cvm->swiotlb_start && + ipa < kvm->arch.virtcca_cvm->swiotlb_end) { + return ret; + } + } + struct page *dst_page = pfn_to_page(pfn); phys_addr_t dst_phys = page_to_phys(dst_page); @@ -1379,4 +1607,1959 @@ int virtcca_cvm_arm_smmu_domain_set_kvm(void *group) (void *)NULL, cvm_arm_smmu_domain_set_kvm); return ret; } + +/* now bypass the migCVM, config staightly 1 is source, 2 is dest*/ +bool virtcca_is_migration_source(struct virtcca_cvm *cvm) +{ + if (!cvm || !cvm->mig_state) { + pr_err("Error: cvm or cvm->params is NULL\n"); + return false; + } + + if (cvm->mig_state->mig_src == VIRTCCA_MIG_SRC) + return true; + + return false; +} + +/* read the max-migs , max of rd/tec pages support */ +int virtcca_mig_capabilities_setup(struct virtcca_cvm *cvm) +{ + uint64_t res; + uint16_t immutable_state_pages, rd_state_pages, tec_state_pages; + + res = tmi_get_mig_config(); + crc32_init(); + + g_virtcca_mig_caps.max_migs = (uint32_t)(res >> 48) & 0xFFFF; + + immutable_state_pages = (uint32_t)(res >> 32) & 0xFFFF; + + rd_state_pages = (uint32_t)(res >> 16) & 0xFFFF; + + tec_state_pages = (uint32_t)res & 0xFFFF; + /* + * The minimal number of pages required. It hould be large enough to + * store all the non-memory states. + */ + g_virtcca_mig_caps.nonmem_state_pages = max3(immutable_state_pages, + rd_state_pages, tec_state_pages); + + return 0; +} + +static void virtcca_mig_stream_get_virtcca_mig_attr(struct virtcca_mig_stream *stream, + struct kvm_dev_virtcca_mig_attr *attr) +{ + attr->version = KVM_DEV_VIRTCCA_MIG_ATTR_VERSION; + attr->max_migs = g_virtcca_mig_caps.max_migs; + attr->buf_list_pages = stream->buf_list_pages; +} + +static int virtcca_mig_stream_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + struct virtcca_mig_stream *stream = dev->private; + u64 __user *uaddr = (u64 __user *)(long)attr->addr; + + switch (attr->group) { + case KVM_DEV_VIRTCCA_MIG_ATTR: { + struct kvm_dev_virtcca_mig_attr virtcca_mig_attr; + + if (attr->attr != sizeof(struct kvm_dev_virtcca_mig_attr)) { + pr_err("Incompatible kvm_dev_get_tdx_mig_attr\n"); + return -EINVAL; + } + + virtcca_mig_stream_get_virtcca_mig_attr(stream, &virtcca_mig_attr); + if (copy_to_user(uaddr, &virtcca_mig_attr, sizeof(virtcca_mig_attr))) + return -EFAULT; + break; + } + default: + return -EINVAL; + } + + return 0; +} +/* this func is to check and cut the max page num of a stream */ +static int virtcca_mig_stream_set_virtcca_mig_attr(struct virtcca_mig_stream *stream, + struct kvm_dev_virtcca_mig_attr *attr) +{ + uint32_t req_pages = attr->buf_list_pages; + uint32_t min_pages = g_virtcca_mig_caps.nonmem_state_pages; + + if (req_pages > VIRTCCA_MIG_BUF_LIST_PAGES_MAX) { + stream->buf_list_pages = VIRTCCA_MIG_BUF_LIST_PAGES_MAX; + pr_warn("Cut the buf_list_npages to the max supported num\n"); + } else if (req_pages < min_pages) { + stream->buf_list_pages = min_pages; + } else { + stream->buf_list_pages = req_pages; + } + + return 0; +} + +static uint32_t crc32_table[CRC_LEN]; + +void crc32_init(void) +{ + for (uint32_t i = 0; i < CRC_LEN; i++) { + uint32_t c = i; + + for (size_t j = 0; j < CRC_SHIFT; j++) { + if (c & 1) + c = CRC_POLYNOMIAL ^ (c >> 1); + else + c >>= 1; + } + crc32_table[i] = c; + } +} + +uint32_t crc32_compute(const uint8_t *data, size_t len) +{ + uint32_t crc = 0xFFFFFFFF; + + for (size_t i = 0; i < len; i++) { + uint8_t index = (crc ^ data[i]) & 0xFF; + + crc = crc32_table[index] ^ (crc >> CRC_SHIFT); + } + return crc ^ 0xFFFFFFFF; +} + +int virtcca_config_crc(uint64_t crc_addr_start, uint64_t crc_addr_end, + uint64_t crc_granularity, bool is_secure) +{ + struct crc_config *config = &g_crc_configs[is_secure]; + const char *mem_type = is_secure ? SEC_MEM : NON_SEC_MEM; + + /* disable crc check */ + if (crc_granularity == 0) { + memset(config, 0, sizeof(*config)); + pr_info("Virtcca migration %s memory crc check disabled", mem_type); + return 0; + } + + if (crc_addr_start >= crc_addr_end || + (crc_granularity != SZ_2M && crc_granularity != SZ_4K) || + crc_addr_end - crc_addr_start < crc_granularity) { + pr_err("%s: invalid input parameters", __func__); + return -EINVAL; + } + + config->ipa_start = ALIGN(crc_addr_start, crc_granularity); + config->ipa_end = ALIGN_DOWN(crc_addr_end, crc_granularity); + config->granularity = crc_granularity; + config->enabled = true; + + pr_info("Virtcca migration %s memory crc check enabled", mem_type); + return 0; +} +EXPORT_SYMBOL_GPL(virtcca_config_crc); + +bool is_valid_crc_params_for_cvm(struct kvm *kvm, bool is_secure) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct crc_config *config = &g_crc_configs[is_secure]; + uint64_t cvm_addr_start = is_secure ? cvm->ipa_start : cvm->swiotlb_start; + uint64_t cvm_addr_end = is_secure ? (DEFAULT_IPA_START + cvm->ram_size) : cvm->swiotlb_end; + + if (config->ipa_start < cvm_addr_start || config->ipa_end > cvm_addr_end) + return false; + return true; +} + +static int virtcca_prepare_crc_file(struct virtcca_cvm *cvm, char *file_name, + size_t name_size, bool is_secure) +{ + struct file *file_p = NULL; + loff_t pos = 0; + int ret = 0; + const char *crc_file_path = is_secure ? SEC_CRC_PATH : NS_CRC_PATH; + + if (!file_name) { + pr_err("Invalid file name"); + return -EINVAL; + } + + snprintf(file_name, name_size, "%s_%u", crc_file_path, cvm->cvm_vmid); + file_p = filp_open(file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (IS_ERR(file_p)) { + ret = PTR_ERR(file_p); + pr_err("Failed to open file %s: %d\n", file_name, ret); + return -EIO; + } + + ret = kernel_write(file_p, "=== crc check start ===\n", + strlen("=== crc check start ===\n"), &pos); + if (ret < 0) + pr_err("Failed to write file header: %d\n", ret); + + filp_close(file_p, NULL); + return ret; +} + +int virtcca_dump_array_to_file(uint64_t *gpa_list, uint64_t *crc_result, + int gpa_nums, char *file_name) +{ + loff_t pos = 0; + char *buf = NULL; + int i, len; + int ret = 0; + struct file *file_p = NULL; + + if (file_name == NULL) { + pr_err("dump_array_to_file error: invalid input."); + return -EINVAL; + } + + file_p = filp_open(file_name, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (IS_ERR(file_p)) { + pr_err("dump_array_to_file: failed to open file"); + return -EIO; + } + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto cleanup; + } + + for (i = 0; i < gpa_nums; i++) { + len = snprintf(buf, PAGE_SIZE, "gpa = 0x%llx crc = 0x%llx\n", + gpa_list[i], crc_result[i]); + ret = kernel_write(file_p, buf, len, &pos); + if (ret < 0) { + pr_err("dump_array_to_file: write error at %d\n", i); + ret = -EIO; + goto cleanup; + } + } + +cleanup: + kfree(buf); + if (file_p) + filp_close(file_p, NULL); + return ret; +} + +uint32_t __execute_ns_crc_dump(struct kvm *kvm, uint64_t target_ipa, + unsigned char *crc_buf, uint64_t crc_granularity) +{ + uint64_t crc_buf_offset = 0; + int ret; + + while (crc_buf_offset < crc_granularity) { + gfn_t gfn = target_ipa >> PAGE_SHIFT; + /* The default granularity of the swiotlb range is 4K. */ + ret = kvm_read_guest_page(kvm, gfn, crc_buf + crc_buf_offset, 0, SZ_4K); + if (ret < 0) { + pr_err("read swiotlb page failed, ret = %d", ret); + return 0; + } + crc_buf_offset += SZ_4K; + } + + return crc32_compute((uint8_t *)crc_buf, crc_granularity); +} + +static int virtcca_execute_crc_dump(struct kvm *kvm, struct crc_config *config, char *file_name) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + uint64_t crc_granularity = config->granularity; + uint64_t gpa_start = config->ipa_start; + uint64_t gpa_end = config->ipa_end; + uint64_t crc_addr = gpa_start; + uint64_t *gpa_list = NULL; + uint64_t *crc_result = NULL; + unsigned char *crc_buf = NULL; + uint64_t actual_chunk_size = 0; + uint64_t valid_count = 0; + int ret = 0; + + if (!file_name || !config) { + pr_err("execute_crc_dump_new: invalid input"); + return -EINVAL; + } + + gpa_list = kcalloc(CRC_DUMP_CHUNK_SIZE, sizeof(uint64_t), GFP_KERNEL); + crc_result = kcalloc(CRC_DUMP_CHUNK_SIZE, sizeof(uint64_t), GFP_KERNEL); + crc_buf = kzalloc(crc_granularity, GFP_KERNEL); + if (!crc_result || !gpa_list || !crc_buf) { + pr_err("execute_crc_dump_new: memory allocation failed"); + ret = -ENOMEM; + goto cleanup; + } + + while (crc_addr < gpa_end) { + valid_count = 0; + actual_chunk_size = min_t(uint64_t, CRC_DUMP_CHUNK_SIZE, + (gpa_end - crc_addr) / crc_granularity); + if (actual_chunk_size <= 0) + break; + if (config->is_secure) { + for (int i = 0; i < actual_chunk_size; i++) { + uint64_t addr = crc_addr + i * crc_granularity; + + if (addr >= UEFI_SIZE && addr < DEFAULT_IPA_START) + continue; /* skip the uefi reversed area */ + gpa_list[valid_count++] = addr; + } + + if (valid_count == 0) { + crc_addr += actual_chunk_size * crc_granularity; + continue; + } + ret = tmi_dump_checksum(cvm->rd, virt_to_phys(gpa_list), + virt_to_phys(crc_result), crc_granularity); + if (ret) { + pr_err("tmi_dump_checksum failed: %d", ret); + ret = -EIO; + goto cleanup; + } + } else { + for (int i = 0; i < actual_chunk_size; i++) { + gpa_list[i] = crc_addr + i * crc_granularity; + crc_result[i] = __execute_ns_crc_dump(kvm, gpa_list[i], + crc_buf, crc_granularity); + valid_count++; + } + } + + ret = virtcca_dump_array_to_file(gpa_list, crc_result, valid_count, file_name); + if (ret < 0) { + pr_err("dump crc to file failed: %d", ret); + ret = -EIO; + goto cleanup; + } + + memset(gpa_list, 0, actual_chunk_size * sizeof(uint64_t)); + memset(crc_result, 0, actual_chunk_size * sizeof(uint64_t)); + crc_addr += actual_chunk_size * crc_granularity; + touch_softlockup_watchdog(); + } + + ret = 0; +cleanup: + kfree(crc_result); + kfree(gpa_list); + kfree(crc_buf); + return ret; +} + +int virtcca_dump_crc(struct kvm *kvm, bool is_secure) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct crc_config *config = &g_crc_configs[is_secure]; + char file_name[FILE_NAME_LEN]; + const char *mem_type = is_secure ? SEC_MEM : NON_SEC_MEM; + int ret = 0; + + if (!cvm->mig_state) { + pr_err("%s: invalid mig_state", __func__); + return -EINVAL; + } + + if (!config->enabled) { + pr_err("%s disabled!", __func__); + return 0; + } + + if (!is_valid_crc_params_for_cvm(kvm, config->is_secure)) { + pr_err("%s: invalid input parameters", __func__); + return -EINVAL; + } + + ret = virtcca_prepare_crc_file(cvm, file_name, sizeof(file_name), config->is_secure); + if (ret < 0) { + pr_err("%s: create file failed", __func__); + return -EIO; + } + + ret = virtcca_execute_crc_dump(kvm, config, file_name); + if (ret) { + pr_err("%s: CRC dump execution failed", __func__); + return -EIO; + } + + pr_info("virtcca dump %s crc success", mem_type); + return 0; +} + +static int virtcca_mig_stream_mbmd_setup(struct virtcca_mig_mbmd *mbmd) +{ + struct page *page; + unsigned long mbmd_size = PAGE_SIZE; + int order = get_order(mbmd_size); + + page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, order); + if (!page) + return -ENOMEM; + + mbmd->data = page_address(page); + mbmd->hpa_and_size = page_to_phys(page) | (mbmd_size - 1) << 52; + + return 0; +} + +static void virtcca_mig_stream_buf_list_cleanup(struct virtcca_mig_buf_list *buf_list) +{ + int i; + kvm_pfn_t pfn; + struct page *page; + + if (!buf_list->entries) + return; + + for (i = 0; i < MAX_BUF_PAGES; i++) { + pfn = buf_list->entries[i].pfn; + if (!pfn) + break; + page = pfn_to_page(pfn); + __free_page(page); + } + free_page((unsigned long)buf_list->entries); +} + +static int virtcca_mig_stream_buf_list_alloc(struct virtcca_mig_buf_list *buf_list) +{ + struct page *page; + + /* + * Allocate the buf list page, which has 512 entries pointing to up to + * 512 pages used as buffers to export/import migration data. + */ + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) + return -ENOMEM; + + buf_list->entries = page_address(page); + buf_list->hpa = page_to_phys(page); + + return 0; +} + +static int virtcca_mig_stream_buf_list_setup(struct virtcca_mig_buf_list *buf_list, uint32_t npages) +{ + int i; + struct page *page; + + if (!npages) { + pr_err("Userspace should set_attr on the device first\n"); + return -EINVAL; + } + + if (virtcca_mig_stream_buf_list_alloc(buf_list)) + return -ENOMEM; + + for (i = 0; i < npages; i++) { + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) { + virtcca_mig_stream_buf_list_cleanup(buf_list); + return -ENOMEM; + } + buf_list->entries[i].pfn = page_to_pfn(page); + } + + /* Mark unused entries as invalid */ + for (i = npages; i < MAX_BUF_PAGES; i++) + buf_list->entries[i].invalid = true; + + return 0; +} + +static int +virtcca_mig_stream_page_list_setup(struct virtcca_mig_page_list *page_list, + struct virtcca_mig_buf_list *buf_list, uint32_t npages) +{ + struct page *page; + uint32_t i; + + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) + return -ENOMEM; + + page_list->entries = page_address(page); + page_list->info.pfn = page_to_pfn(page); + + /* Reuse the buffers from the buffer list for pages list */ + for (i = 0; i < npages; i++) + page_list->entries[i] = __pfn_to_phys(buf_list->entries[i].pfn); + page_list->info.last_entry = npages - 1; + + return 0; +} + +/* this function is used to setup the page list for migration */ +static int virtcca_mig_stream_gpa_list_setup(struct virtcca_mig_gpa_list *gpa_list) +{ + struct page *page; + + page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!page) + return -ENOMEM; + + gpa_list->info.pfn = page_to_pfn(page); + gpa_list->entries = page_address(page); + + return 0; +} + +static int virtcca_mig_stream_mac_list_setup(struct virtcca_mig_mac_list *mac_list) +{ + struct page *page; + + page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, 0); + if (!page) + return -ENOMEM; + + mac_list->entries = page_address(page); + mac_list->hpa = page_to_phys(page); + + return 0; +} + +static int virtcca_mig_stream_setup(struct virtcca_mig_stream *stream, bool mig_src) +{ + int ret; + + ret = virtcca_mig_stream_mbmd_setup(&stream->mbmd); + if (ret) + goto err_mbmd; + + ret = virtcca_mig_stream_buf_list_setup(&stream->mem_buf_list, stream->buf_list_pages); + if (ret) + goto err_mem_buf_list; + + ret = virtcca_mig_stream_page_list_setup(&stream->page_list, + &stream->mem_buf_list, stream->buf_list_pages); + if (ret) + goto err_page_list; + + ret = virtcca_mig_stream_gpa_list_setup(&stream->gpa_list); + if (ret) + goto err_gpa_list; + + ret = virtcca_mig_stream_mac_list_setup(&stream->mac_list[0]); + if (ret) + goto err_mac_list0; + /* + * The 2nd mac list is needed only when the buf list uses more than + * 256 entries + */ + if (stream->buf_list_pages > MAX_MAC_PAGES_PER_ARR) { + ret = virtcca_mig_stream_mac_list_setup(&stream->mac_list[1]); + if (ret) + goto err_mac_list1; + } + + /* The lists used by the destination rd only */ + if (!mig_src) { + ret = virtcca_mig_stream_buf_list_alloc(&stream->dst_buf_list); + if (ret) + goto err_dst_buf_list; + ret = virtcca_mig_stream_buf_list_alloc(&stream->import_mem_buf_list); + if (ret) + goto err_import_mem_buf_list; + } + + return 0; +err_import_mem_buf_list: + free_page((unsigned long)stream->dst_buf_list.entries); +err_dst_buf_list: + if (stream->mac_list[1].entries) + free_page((unsigned long)stream->mac_list[1].entries); +err_mac_list1: + free_page((unsigned long)stream->mac_list[0].entries); +err_mac_list0: + free_page((unsigned long)stream->gpa_list.entries); +err_gpa_list: + free_page((unsigned long)stream->page_list.entries); +err_page_list: + virtcca_mig_stream_buf_list_cleanup(&stream->mem_buf_list); +err_mem_buf_list: + free_page((unsigned long)stream->mbmd.data); +err_mbmd: + pr_err("%s failed\n", __func__); + return ret; +} + +/* check the attr is enough */ +static int virtcca_mig_stream_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + struct virtcca_cvm *cvm = dev->kvm->arch.virtcca_cvm; + struct virtcca_mig_stream *stream = dev->private; + u64 __user *uaddr = (u64 __user *)(long)attr->addr; + int ret; + + switch (attr->group) { + case KVM_DEV_VIRTCCA_MIG_ATTR: { + struct kvm_dev_virtcca_mig_attr virtcca_mig_attr; + + if (copy_from_user(&virtcca_mig_attr, uaddr, sizeof(virtcca_mig_attr))) + return -EFAULT; + + if (virtcca_mig_attr.version != KVM_DEV_VIRTCCA_MIG_ATTR_VERSION) + return -EINVAL; + + ret = virtcca_mig_stream_set_virtcca_mig_attr(stream, &virtcca_mig_attr); + if (ret) + break; + + ret = virtcca_mig_stream_setup(stream, + virtcca_is_migration_source(cvm)); + break; + } + default: + return -EINVAL; + } + + return ret; +} + +static bool virtcca_mig_stream_in_mig_buf_list(uint32_t i, uint32_t max_pages) +{ + if (i >= VIRTCCA_MIG_STREAM_BUF_LIST_MAP_OFFSET && + i < VIRTCCA_MIG_STREAM_BUF_LIST_MAP_OFFSET + max_pages) + return true; + + return false; +} + +static vm_fault_t virtcca_mig_stream_fault(struct vm_fault *vmf) +{ + struct kvm_device *dev = vmf->vma->vm_file->private_data; + struct virtcca_mig_stream *stream = dev->private; + struct page *page; + kvm_pfn_t pfn; + uint32_t i; + + /* See linear_page_index for pgoff */ + if (vmf->pgoff == VIRTCCA_MIG_STREAM_MBMD_MAP_OFFSET) { + page = virt_to_page(stream->mbmd.data); + } else if (vmf->pgoff == VIRTCCA_MIG_STREAM_GPA_LIST_MAP_OFFSET) { + page = virt_to_page(stream->gpa_list.entries); + } else if (vmf->pgoff == VIRTCCA_MIG_STREAM_MAC_LIST_MAP_OFFSET || + vmf->pgoff == VIRTCCA_MIG_STREAM_MAC_LIST_MAP_OFFSET + 1) { + i = vmf->pgoff - VIRTCCA_MIG_STREAM_MAC_LIST_MAP_OFFSET; + if (stream->mac_list[i].entries) { + page = virt_to_page(stream->mac_list[i].entries); + } else { + pr_err("%s: mac list page %d not allocated\n", + __func__, i); + return VM_FAULT_SIGBUS; + } + } else if (virtcca_mig_stream_in_mig_buf_list(vmf->pgoff, stream->buf_list_pages)) { + i = vmf->pgoff - VIRTCCA_MIG_STREAM_BUF_LIST_MAP_OFFSET; + pfn = stream->mem_buf_list.entries[i].pfn; + page = pfn_to_page(pfn); + } else { + pr_err("%s: VM_FAULT_SIGBUS\n", __func__); + return VM_FAULT_SIGBUS; + } + + get_page(page); + vmf->page = page; + return 0; +} + +static const struct vm_operations_struct virtcca_mig_stream_ops = { + .fault = virtcca_mig_stream_fault, +}; + +static int virtcca_mig_stream_mmap(struct kvm_device *dev, struct vm_area_struct *vma) +{ + vma->vm_ops = &virtcca_mig_stream_ops; + return 0; +} + +static int virtcca_mig_export_state_immutable(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct virtcca_mig_page_list *page_list = &stream->page_list; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct arm_smccc_res ret; + + ret = tmi_export_immutable(cvm->rd, stream->mbmd.hpa_and_size, + page_list->info.val, stream_info.val); + if (ret.a1 == TMI_SUCCESS) { + stream->idx = stream->mbmd.data->migs_index; + if (copy_to_user(data, &ret.a2, sizeof(uint64_t))) + return -EFAULT; + } else { + pr_err("%s: failed, err=%lx\n", __func__, ret.a1); + return -EIO; + } + return 0; +} + +static int virtcca_mig_import_state_immutable(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct virtcca_mig_page_list *page_list = &stream->page_list; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct mig_cvm_update_info *update_info = NULL; + uint64_t ret, npages; + int res = 0; + + if (copy_from_user(&npages, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + page_list->info.last_entry = npages - 1; + + ret = tmi_import_immutable(cvm->rd, stream->mbmd.hpa_and_size, + page_list->info.val, stream_info.val); + if (ret == TMI_SUCCESS) { + stream->idx = stream->mbmd.data->migs_index; + } else { + pr_err("%s: failed, err=%llx\n", __func__, ret); + return -EIO; + } + + update_info = kmalloc(sizeof(struct mig_cvm_update_info), GFP_KERNEL); + if (!update_info) + return -ENOMEM; + + ret = tmi_update_cvm_info(cvm->rd, (uint64_t)update_info); + if (ret) { + pr_err("tmi_update_cvm_info failed, err=%llx", ret); + res = -EIO; + goto out; + } + + cvm->swiotlb_start = update_info->swiotlb_start; + cvm->swiotlb_end = update_info->swiotlb_end; + cvm->ipa_start = update_info->ipa_start; + + ret = kvm_cvm_mig_map_range(kvm); + if (ret) { + pr_err("kvm_cvm_mig_map_range: failed, err=%llx\n", ret); + res = -EIO; + } + +out: + kfree(update_info); + return res; +} + +static void virtcca_mig_buf_list_set_valid(struct virtcca_mig_buf_list *mem_buf_list, + uint64_t num) +{ + int i; + + for (i = 0; i < num; i++) + mem_buf_list->entries[i].invalid = false; + + for (i = num; i < MAX_BUF_PAGES; i++) { + if (!mem_buf_list->entries[i].invalid) + mem_buf_list->entries[i].invalid = true; + else + break; + } +} + +static int virtcca_mig_mem_param_setup(struct tmi_mig_mem *mig_mem_param) +{ + struct page *page; + unsigned long mig_mem_param_size = PAGE_SIZE; + int order = get_order(mig_mem_param_size); + + page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, order); + if (!page) + return -ENOMEM; + + mig_mem_param->data = page_address(page); + mig_mem_param->addr_and_size = page_to_phys(page) | (mig_mem_param_size - 1) << 52; + + return 0; +} + +static int64_t virtcca_mig_stream_export_mem(struct kvm *kvm, + struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_mig_state *mig_state = cvm->mig_state; + struct virtcca_mig_gpa_list *gpa_list = &stream->gpa_list; + union virtcca_mig_stream_info stream_info = {.val = 0}; + + struct tmi_mig_mem mig_mem_param = {0}; + struct tmi_mig_mem_data *mig_mem_param_data; + uint64_t npages, gpa_list_info_val; + struct arm_smccc_res tmi_res = { 0 }; + int ret; + + if (mig_state->bugged) + return -EBADF; + + if (copy_from_user(&npages, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + if (npages > stream->buf_list_pages) + return -EINVAL; + + ret = virtcca_mig_mem_param_setup(&mig_mem_param); + if (ret) + goto out; + + mig_mem_param_data = mig_mem_param.data; + if (!mig_mem_param_data) { + ret = -ENOMEM; + goto out; + } + + gpa_list->info.first_entry = 0; + gpa_list->info.last_entry = npages - 1; + virtcca_mig_buf_list_set_valid(&stream->mem_buf_list, npages); + stream_info.index = stream->idx; + + mig_mem_param_data->gpa_list_info = gpa_list->info.val; + mig_mem_param_data->mig_buff_list_pa = stream->mem_buf_list.hpa; + mig_mem_param_data->mig_cmd = stream_info.val; + mig_mem_param_data->mbmd_hpa_and_size = stream->mbmd.hpa_and_size; + mig_mem_param_data->mac_pa0 = stream->mac_list[0].hpa; + mig_mem_param_data->mac_pa1 = stream->mac_list[1].hpa; + + tmi_res = tmi_export_mem(cvm->rd, mig_mem_param.addr_and_size); + + ret = tmi_res.a1; + gpa_list_info_val = tmi_res.a2; + + if (ret == TMI_SUCCESS) { + if (copy_to_user(data, &gpa_list_info_val, sizeof(uint64_t))) + return -EFAULT; + } else { + pr_err("%s: err=%d, gfn=%llx\n", + __func__, ret, (uint64_t)gpa_list->entries[0].gfn); + return -EIO; + } + +out: + if (mig_mem_param.data) + free_pages((unsigned long)mig_mem_param.data, get_order(PAGE_SIZE)); + + return ret; +} + +static int virtcca_mig_stream_import_mem(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_mig_state *mig_state = cvm->mig_state; + struct virtcca_mig_gpa_list *gpa_list = &stream->gpa_list; + union virtcca_mig_stream_info stream_info = {.val = 0}; + + struct tmi_mig_mem mig_mem_param = {0}; + struct tmi_mig_mem_data *mig_mem_param_data; + + uint64_t npages = 0; + uint64_t gpa_list_info_val = 0; + uint64_t ret = 0; + struct arm_smccc_res tmi_res; + + if (mig_state->bugged) + return -EBADF; + + if (copy_from_user(&npages, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + if (npages > stream->buf_list_pages) + return -EINVAL; + + ret = virtcca_mig_mem_param_setup(&mig_mem_param); + if (ret) + goto out; + + mig_mem_param_data = mig_mem_param.data; + if (!mig_mem_param_data) { + ret = -ENOMEM; + goto out; + } + + gpa_list->info.first_entry = 0; + gpa_list->info.last_entry = npages - 1; + virtcca_mig_buf_list_set_valid(&stream->mem_buf_list, npages); + stream_info.index = stream->idx; + + mig_mem_param_data->gpa_list_info = gpa_list->info.val; + mig_mem_param_data->mig_buff_list_pa = stream->mem_buf_list.hpa; + mig_mem_param_data->mig_cmd = stream_info.val; + mig_mem_param_data->mbmd_hpa_and_size = stream->mbmd.hpa_and_size; + mig_mem_param_data->mac_pa0 = stream->mac_list[0].hpa; + mig_mem_param_data->mac_pa1 = stream->mac_list[1].hpa; + + tmi_res = tmi_import_mem(cvm->rd, mig_mem_param.addr_and_size); + + ret = tmi_res.a1; + gpa_list_info_val = tmi_res.a2; + + if (ret == TMI_SUCCESS) { + if (copy_to_user(data, &gpa_list_info_val, sizeof(uint64_t))) + return -EFAULT; + } else { + pr_err("%s: err=%llx, gfn=%llx\n", + __func__, ret, (uint64_t)gpa_list->entries[0].gfn); + return -EIO; + } + +out: + if (mig_mem_param.data) + free_pages((unsigned long)mig_mem_param.data, get_order(PAGE_SIZE)); + + return ret; +} + +static int virtcca_mig_memslot_param_setup(struct tmi_mig_memslot *mig_mem_param) +{ + struct page *page; + unsigned long mig_mem_param_size = PAGE_SIZE; + int order = get_order(mig_mem_param_size); + + page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, order); + if (!page) + return -ENOMEM; + + mig_mem_param->data = page_address(page); + mig_mem_param->addr_and_size = page_to_phys(page) | (mig_mem_param_size - 1) << 52; + + return 0; +} + +void virtcca_set_tmm_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct tmi_mig_memslot mig_memslot_param = {0}; + struct tmi_mig_memslot_data *mig_memslot_param_data; + struct page *dirty_bitmap_page; + unsigned int dirty_bitmap_list_len; + uint64_t dirty_bitmap_addr; + int ret; + + if (memslot->base_gfn << PAGE_SHIFT < cvm->ipa_start) + return; + + ret = virtcca_mig_memslot_param_setup(&mig_memslot_param); + if (ret) + return; + + mig_memslot_param_data = mig_memslot_param.data; + if (!mig_memslot_param_data) { + ret = -ENOMEM; + return; + } + + unsigned long bitmap_size_bytes = kvm_dirty_bitmap_bytes(memslot); + + dirty_bitmap_list_len = DIV_ROUND_UP(bitmap_size_bytes, SZ_2M); + + dirty_bitmap_addr = (uint64_t)memslot->dirty_bitmap; + for (int i = 0; i < dirty_bitmap_list_len; i++) { + dirty_bitmap_page = vmalloc_to_page((uint64_t *)dirty_bitmap_addr); + mig_memslot_param_data->dirty_bitmap_list[i] = page_to_phys(dirty_bitmap_page); + dirty_bitmap_addr += SZ_2M; + } + mig_memslot_param_data->base_gfn = memslot->base_gfn; + mig_memslot_param_data->npages = memslot->npages; + mig_memslot_param_data->memslot_id = memslot->id; + + tmi_set_tmm_memslot(cvm->rd, mig_memslot_param.addr_and_size); +} + +static int virtcca_mig_export_track(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + unsigned long timeout = jiffies + msecs_to_jiffies(TMI_IMPORT_TIMEOUT_MS); + uint64_t in_order, ret; + + if (copy_from_user(&in_order, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + /* + * Set the in_order bit if userspace requests to generate a start + * token by sending a non-0 value through tdx_cmd.data. + */ + stream_info.in_order = !!in_order; + do { + ret = tmi_export_track(cvm->rd, stream->mbmd.hpa_and_size, stream_info.val); + msleep(TMI_TRACK_TIMEOUT_MS); + + if (time_after(jiffies, timeout)) { + pr_err("tmi_export_track timeout (%d ms)", TMI_IMPORT_TIMEOUT_MS); + ret = ETIMEDOUT; + break; + } + } while (ret == TMI_IMPORT_INCOMPLETE); + + if (ret != TMI_SUCCESS) { + pr_err("%s: failed, err=%llx\n", __func__, ret); + return -EIO; + } + return 0; +} + +static inline bool +virtcca_mig_epoch_is_start_token(struct virtcca_mig_mbmd_data *data) +{ + return data->mig_epoch == VIRTCCA_MIG_EPOCH_START_TOKEN; +} + +static int virtcca_mig_import_track(struct kvm *kvm, + struct virtcca_mig_stream *stream) +{ + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + uint64_t ret; + + ret = tmi_import_track(cvm->rd, stream->mbmd.hpa_and_size, stream_info.val); + if (ret != TMI_SUCCESS) { + pr_err("tmi_import_track failed, err=%llx\n", ret); + return -EIO; + } + return 0; +} + +static int virtcca_mig_import_end(struct kvm *kvm) +{ + uint64_t ret; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + unsigned long timeout = jiffies + msecs_to_jiffies(TMI_IMPORT_TIMEOUT_MS); + + if (!cvm) { + pr_err("%s: cvm is not initialized\n", __func__); + return -EINVAL; + } + + do { + ret = tmi_import_commit(cvm->rd); + msleep(TMI_TRACK_TIMEOUT_MS); + + if (time_after(jiffies, timeout)) { + pr_err("tmi_import_commit timeout (%d ms)", TMI_IMPORT_TIMEOUT_MS); + ret = ETIMEDOUT; + break; + } + } while (ret == TMI_IMPORT_INCOMPLETE); + + if (ret != TMI_SUCCESS) { + pr_err("%s: failed, err=%llx\n", __func__, ret); + return -EIO; + } + + virtcca_mig_state_release(cvm); + + WRITE_ONCE(cvm->state, CVM_STATE_ACTIVE); + + return 0; +} + +static int virtcca_mig_export_state_tec(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct kvm_vcpu *vcpu; + struct virtcca_cvm_tec *tec; + struct virtcca_mig_state *mig_state = kvm->arch.virtcca_cvm->mig_state; + union virtcca_mig_stream_info stream_info = {.val = 0}; + struct arm_smccc_res ret; + + if (mig_state->vcpu_export_next_idx >= atomic_read(&kvm->online_vcpus)) { + pr_err("%s: vcpu_export_next_idx %d >= online_vcpus %d\n", + __func__, mig_state->vcpu_export_next_idx, + atomic_read(&kvm->online_vcpus)); + return -EINVAL; + } + + vcpu = kvm_get_vcpu(kvm, mig_state->vcpu_export_next_idx); + tec = &vcpu->arch.tec; + + stream_info.index = stream->idx; + + + ret = tmi_export_tec(tec->tec, stream->mbmd.hpa_and_size, + stream->page_list.info.val, stream_info.val); + + if (ret.a1 == TMI_SUCCESS) { + mig_state->vcpu_export_next_idx++; + if (copy_to_user(data, &(ret.a2), sizeof(uint64_t))) + return -EFAULT; + } else { + pr_err("%s: failed, err=%lx\n", __func__, ret.a1); + return -EIO; + } + + return 0; +} + +static uint16_t tdx_mig_mbmd_get_vcpu_idx(struct virtcca_mig_mbmd_data *data) +{ + return *(uint16_t *)data->type_specific_info; +} + +static int virtcca_mig_import_state_tec(struct kvm *kvm, struct virtcca_mig_stream *stream, + uint64_t __user *data) +{ + struct kvm_vcpu *vcpu; + struct virtcca_cvm_tec *tec; + union virtcca_mig_stream_info stream_info = {.val = 0}; + uint64_t ret; + uint64_t npages; + uint16_t vcpu_idx; + + if (copy_from_user(&npages, (void __user *)data, sizeof(uint64_t))) + return -EFAULT; + + stream->page_list.info.last_entry = npages - 1; + + vcpu_idx = tdx_mig_mbmd_get_vcpu_idx(stream->mbmd.data); + vcpu = kvm_get_vcpu(kvm, vcpu_idx); + tec = &vcpu->arch.tec; + + ret = tmi_import_tec(tec->tec, stream->mbmd.hpa_and_size, + stream->page_list.info.val, stream_info.val); + if (ret != TMI_SUCCESS) { + pr_err("%s: failed, err=%llx\n", __func__, ret); + return -EIO; + } + + return 0; +} + +static int virtcca_mig_get_mig_info(struct kvm *kvm, uint64_t __user *data) +{ + struct virtCCAMigInfo migInfo; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + + migInfo.swiotlb_start = cvm->swiotlb_start; + migInfo.swiotlb_end = cvm->swiotlb_end; + + if (copy_to_user(data, &(migInfo), sizeof(struct virtCCAMigInfo))) + return -EFAULT; + + return 0; +} + +static int virtcca_mig_is_zero_page(struct kvm *kvm, + struct virtcca_mig_stream *stream, uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_mig_gpa_list *gpa_list = &stream->gpa_list; + + int ret; + struct arm_smccc_res tmi_res = { 0 }; + bool is_zero_page = false; + + tmi_res = tmi_is_zero_page(cvm->rd, gpa_list->info.val); + + ret = tmi_res.a1; + if (tmi_res.a2) + is_zero_page = true; + + if (ret == TMI_SUCCESS) { + if (copy_to_user(data, &is_zero_page, sizeof(bool))) + return -EFAULT; + } else { + pr_err("%s: err=%d, gfn=%llx\n", + __func__, ret, (uint64_t)gpa_list->entries[0].gfn); + return -EIO; + } + + return ret; +} + +static int virtcca_mig_import_zero_page(struct kvm *kvm, + struct virtcca_mig_stream *stream, uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + int ret; + struct arm_smccc_res tmi_res = { 0 }; + uint64_t gpa; + + gpa = (uint64_t)data; + + tmi_res = tmi_import_zero_page(cvm->rd, gpa); + + ret = tmi_res.a1; + + if (ret) { + pr_err("%s: err=%d\n", + __func__, ret); + return -EIO; + } + + return ret; +} + +static int virtcca_mig_export_pause(struct kvm *kvm, + struct virtcca_mig_stream *stream, uint64_t __user *data) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct arm_smccc_res tmi_res = { 0 }; + + tmi_res = tmi_export_pause(cvm->rd); + if (tmi_res.a1) { + pr_err("%s: err=%lu\n", + __func__, tmi_res.a1); + return -EIO; + } + + return tmi_res.a1; +} + +/* add qemu ioctl struct to fit this func */ +static long virtcca_mig_stream_ioctl(struct kvm_device *dev, unsigned int ioctl, unsigned long arg) +{ + struct kvm *kvm = dev->kvm; + struct virtcca_mig_stream *stream = dev->private; + void __user *argp = (void __user *)arg; + struct kvm_virtcca_mig_cmd cvm_cmd; + int r; + + if (copy_from_user(&cvm_cmd, argp, sizeof(struct kvm_virtcca_mig_cmd))) + return -EFAULT; + + if (ioctl != KVM_CVM_MIG_IOCTL) + return -EINVAL; + + switch (cvm_cmd.id) { + case KVM_CVM_MIG_EXPORT_STATE_IMMUTABLE: + r = virtcca_mig_export_state_immutable(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_STATE_IMMUTABLE: + r = virtcca_mig_import_state_immutable(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_EXPORT_MEM: + r = virtcca_mig_stream_export_mem(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_MEM: + r = virtcca_mig_stream_import_mem(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_EXPORT_TRACK: + r = virtcca_mig_export_track(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_TRACK: + r = virtcca_mig_import_track(kvm, stream); + break; + case KVM_CVM_MIG_EXPORT_STATE_TEC: + r = virtcca_mig_export_state_tec(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_STATE_TEC: + r = virtcca_mig_import_state_tec(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_END: + r = virtcca_mig_import_end(kvm); + break; + case KVM_CVM_MIG_CRC: + r = 0; + virtcca_dump_crc(kvm, true); + virtcca_dump_crc(kvm, false); + break; + case KVM_CVM_MIG_GET_MIG_INFO: + r = virtcca_mig_get_mig_info(kvm, (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IS_ZERO_PAGE: + r = virtcca_mig_is_zero_page(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_IMPORT_ZERO_PAGE: + r = virtcca_mig_import_zero_page(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + case KVM_CVM_MIG_EXPORT_PAUSE: + r = virtcca_mig_export_pause(kvm, stream, + (uint64_t __user *)cvm_cmd.data); + break; + default: + r = -EINVAL; + } + + return r; +} + +static int virtcca_mig_do_stream_create(struct kvm *kvm, + struct virtcca_mig_stream *stream, hpa_t *migsc_addr) +{ + u64 numa_set = kvm_get_host_numa_set_by_vcpu(0, kvm); + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + hpa_t migsc_pa = 0; + struct mig_cvm_update_info *update_info = NULL; + uint64_t ret = 0; + + /* + * This migration stream has been created, e.g. the previous migration + * session is aborted and the migration stream is retained during the + * TD guest lifecycle (required by the TDX migration architecture for + * later re-migration). No need to proceed to the creation in this + * case. + */ + if (!migsc_addr) { + pr_err("invalid migsc_addr!"); + return -1; + } + + /* now just create stream in tmm */ + migsc_pa = tmi_mig_stream_create(cvm->rd, numa_set); + if (!migsc_pa) + kvm_err("virtcca mig stream create failed!\n"); + + *migsc_addr = migsc_pa; + + update_info = kmalloc(sizeof(struct mig_cvm_update_info), GFP_KERNEL); + if (!update_info) + return -ENOMEM; + + ret = tmi_update_cvm_info(cvm->rd, (uint64_t)update_info); + if (ret) { + pr_err("tmi_update_cvm_info failed, err=%llx", ret); + kfree(update_info); + return -EIO; + } + cvm->swiotlb_start = update_info->swiotlb_start; + cvm->swiotlb_end = update_info->swiotlb_end; + + kfree(update_info); + return 0; +} + +static int virtcca_mig_session_init(struct kvm *kvm) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_mig_state *mig_state = cvm->mig_state; + struct virtcca_mig_gpa_list *blockw_gpa_list = &mig_state->blockw_gpa_list; + int ret = 0; + + if (virtcca_mig_do_stream_create(kvm, &mig_state->backward_stream, + &mig_state->backward_migsc_paddr)) + return -EIO; + + if (virtcca_is_migration_source(cvm)) + ret = virtcca_mig_stream_gpa_list_setup(blockw_gpa_list); + + return ret; +} + +static void virtcca_mig_session_exit(struct virtcca_mig_state *mig_state) +{ + if (mig_state->blockw_gpa_list.entries) { + free_pages((uint64_t)mig_state->blockw_gpa_list.entries, 0); + mig_state->blockw_gpa_list.entries = NULL; + mig_state->blockw_gpa_list.info.pfn = 0; + } +} + +static int virtcca_mig_stream_create(struct kvm_device *dev, u32 type) +{ + struct kvm *kvm = dev->kvm; + struct virtcca_cvm *cvm = dev->kvm->arch.virtcca_cvm; + struct virtcca_mig_state *mig_state = cvm->mig_state; + struct virtcca_mig_stream *stream; + int ret; + + stream = kzalloc(sizeof(struct virtcca_mig_stream), GFP_KERNEL_ACCOUNT); + if (!stream) + return -ENOMEM; + + dev->private = stream; + /* set the stream idx of the cvm */ + stream->idx = atomic_inc_return(&mig_state->streams_created) - 1; + + if (!stream->idx) { + ret = virtcca_mig_session_init(kvm); /* if is the first stream, call this func */ + if (ret) + goto err_mig_session_init; + mig_state->default_stream = stream; + } + + ret = virtcca_mig_do_stream_create(kvm, stream, &mig_state->migsc_paddrs[stream->idx]); + if (ret) + goto err_stream_create; + + return 0; +err_stream_create: + virtcca_mig_session_exit(mig_state); +err_mig_session_init: + atomic_dec(&mig_state->streams_created); + kfree(stream); + return ret; +} + +void virtcca_mig_state_release(struct virtcca_cvm *cvm) +{ + struct virtcca_mig_state *mig_state = cvm->mig_state; + + if (!mig_state) + return; + + mig_state->vcpu_export_next_idx = 0; + mig_state->backward_migsc_paddr = 0; + + atomic_dec(&mig_state->streams_created); + if (!atomic_read(&mig_state->streams_created)) + virtcca_mig_session_exit(mig_state); +} + + +static void virtcca_mig_stream_release(struct kvm_device *dev) +{ + struct virtcca_mig_stream *stream = dev->private; + + free_page((unsigned long)stream->mbmd.data); + virtcca_mig_stream_buf_list_cleanup(&stream->mem_buf_list); + free_page((unsigned long)stream->page_list.entries); + free_page((unsigned long)stream->gpa_list.entries); + free_page((unsigned long)stream->mac_list[0].entries); + /* + * The 2nd mac list page is allocated conditionally when + * stream->buf_list_pages is larger than 256. + */ + if (stream->mac_list[1].entries) + free_page((unsigned long)stream->mac_list[1].entries); + if (stream->dst_buf_list.entries) + free_page((unsigned long)stream->dst_buf_list.entries); + if (stream->import_mem_buf_list.entries) + free_page((unsigned long)stream->import_mem_buf_list.entries); + /*print the elements of the stream*/ + kfree(stream); +} + +int virtcca_mig_state_create(struct virtcca_cvm *cvm) +{ + struct virtcca_mig_state *mig_state = cvm->mig_state; + hpa_t *migsc_paddrs = NULL; + + mig_state = NULL; + cvm->mig_cvm_info = NULL; + mig_state = kzalloc(sizeof(struct virtcca_mig_state), GFP_KERNEL_ACCOUNT); + if (!mig_state) + goto out; + + migsc_paddrs = kcalloc(g_virtcca_mig_caps.max_migs, sizeof(hpa_t), GFP_KERNEL_ACCOUNT); + if (!migsc_paddrs) + goto out; + + cvm->mig_cvm_info = kzalloc(sizeof(struct mig_cvm), GFP_KERNEL_ACCOUNT); + mig_state->mig_src = cvm->params->mig_src; + if (!cvm->mig_cvm_info) + goto out; + + mig_state->migsc_paddrs = migsc_paddrs; + cvm->mig_state = mig_state; + + return 0; + +out: + pr_err("%s failed", __func__); + kfree(mig_state); + kfree(migsc_paddrs); + kfree(cvm->mig_cvm_info); + return -ENOMEM; +} + +int virtcca_save_migvm_cid(struct kvm *kvm, struct kvm_virtcca_mig_cmd *cmd) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct mig_cvm *mig_cvm_usr; + struct mig_cvm *mig_cvm_info = cvm->mig_cvm_info; + + pr_info("calling %s\n", __func__); + if (mig_cvm_info == NULL) { + pr_info("guest_mig_cvm_info is NULL\n"); + return -EINVAL; + } + + mig_cvm_usr = kmalloc(sizeof(struct mig_cvm), GFP_KERNEL); + if (!mig_cvm_usr) + return -ENOMEM; + if (copy_from_user(mig_cvm_usr, (void __user *)cmd->data, + sizeof(struct mig_cvm))) { + kfree(mig_cvm_usr); + return -EFAULT; + } + + if (cmd->flags || mig_cvm_usr->version != KVM_CVM_MIGVM_VERSION) { + kfree(mig_cvm_usr); + return -EINVAL; + } + + memcpy(&mig_cvm_info->migvm_cid, &mig_cvm_usr->migvm_cid, sizeof(uint64_t)); + g_migcvm_agent_listen_cid.cid = mig_cvm_info->migvm_cid; + + kfree(mig_cvm_usr); + return 0; +} + +static int send_and_wait_ack(struct socket *sock, struct bind_msg_s *req_msg) +{ + int ret; + struct kvec vec; + struct msghdr hdr; + struct bind_msg_s ack_msg = {0}; + int retry = 0; + + memset(&hdr, 0, sizeof(hdr)); + vec.iov_base = req_msg; + vec.iov_len = sizeof(*req_msg); + /* set vsock hdr */ + hdr.msg_flags = MSG_NOSIGNAL; + iov_iter_kvec(&hdr.msg_iter, WRITE, &vec, 1, vec.iov_len); + + if (req_msg->payload_len > MAX_PAYLOAD_SIZE) { + pr_err("Payload size %u exceeds limit\n", req_msg->payload_len); + ret = -EINVAL; + goto out; + } + + /* send request to migcvm agent, expect ack from migcvm agent */ + retry = 0; + do { + ret = kernel_sendmsg(sock, &hdr, &vec, 1, vec.iov_len); + if (ret == -EINTR) { + pr_warn("sendmsg interrupted by signal, retry %d\n", retry); + retry++; + continue; + } + break; + } while (retry < SEND_RETRY_LIMIT); + + if (ret < 0) { + pr_err("Failed to send request, ret=%d\n", ret); + goto out; + } else if (ret != sizeof(*req_msg)) { + pr_err("Partial send, ret=%d\n", ret); + ret = -EIO; + goto out; + } + + /* reset ack buffer */ + vec.iov_base = &ack_msg; + vec.iov_len = sizeof(ack_msg); + + retry = 0; + do { + ret = kernel_recvmsg(sock, &hdr, &vec, 1, sizeof(ack_msg), hdr.msg_flags); + if (ret == -EINTR) { + pr_warn("recvmsg interrupted by signal, retry %d\n", retry); + retry++; + continue; + } + break; + } while (retry < RECV_RETRY_LIMIT); + + if (ret < 0) { + pr_err("Failed to recv ack, ret=%d\n", ret); + goto out; + } else if (ret != sizeof(ack_msg)) { + pr_err("Partial ack recv ret=%d\n", ret); + ret = -EIO; + goto out; + } + + /* validate ack message */ + if (ack_msg.payload_type != VSOCK_MSG_ACK || + ack_msg.session_id != req_msg->session_id || + ack_msg.success == 0) { + pr_err("ACK validation failed, the payload_type=%d, session_id=%llu, success=%d\n", + ack_msg.payload_type, ack_msg.session_id, ack_msg.success); + ret = -EPROTO; + goto out; + } + pr_info("ACK validation passed\n"); + ret = 0; + +out: + return ret; +} + +/* vsock connection*/ +/* step 1: send to mig-cvm agent: the migrated rd, the destination platform ip*/ +/* step 2: wait for mig-cvm agent's response */ +static int notify_migcvm_agent(uint64_t cid, struct virtcca_dst_host_info *dst_host_info, + uint64_t guest_rd, bool is_src) +{ + struct socket *sock = NULL; + int ret = 0; + int retry_count = 3; + int error = 0, len = sizeof(error); + int connect_retry = 0; + long old_sndtimeo = 0, old_rcvtimeo = 0; + const unsigned long timeout = 5 * HZ; + + struct sockaddr_vm sa = { + .svm_family = AF_VSOCK, + .svm_cid = cid, + .svm_port = is_src ? MIGCVM_AGENT_PORT_SRC : MIGCVM_AGENT_PORT_DST + }; + struct bind_msg_s bind_msg = {0}; + + pr_info("calling %s, cid=%llu, port=%d\n", __func__, cid, sa.svm_port); + ret = sock_create_kern(&init_net, AF_VSOCK, SOCK_STREAM, 0, &sock); + if (ret < 0) { + pr_err("Failed to create socket, error: %d\n", ret); + return ret; + } + + /* save original receive timeout, and set 5s timeout for migcvm ack */ + if (sock->sk) { + old_sndtimeo = sock->sk->sk_sndtimeo; + old_rcvtimeo = sock->sk->sk_rcvtimeo; + sock->sk->sk_sndtimeo = timeout; + sock->sk->sk_rcvtimeo = timeout; + } + + connect_retry = 0; + do { + ret = kernel_connect(sock, (struct sockaddr *)&sa, sizeof(sa), O_NONBLOCK); + if (ret == -EINTR) { + pr_warn("connect interrupted by signal, retry %d\n", connect_retry); + schedule_timeout_uninterruptible(HZ / 10); + connect_retry++; + continue; + } + break; + } while (connect_retry < CONNECT_RETRY_LIMIT); + + if (ret < 0) { + if (sock->ops && sock->ops->getsockopt) + sock->ops->getsockopt(sock, SOL_SOCKET, SO_ERROR, (char *)&error, &len); + if (error) { + pr_err("Connect failed (cid=%llu, port=%d, err=%d)\n", + cid, sa.svm_port, error); + ret = -error; + goto cleanup; + } + } + + if (strscpy(bind_msg.cmd, is_src ? "START_CLIENT" : "START_SERVER", + sizeof(bind_msg.cmd)) < 0) { + pr_err("Command string too long\n"); + ret = -EINVAL; + goto cleanup; + } + + bind_msg.session_id = get_jiffies_64(); + bind_msg.payload_type = is_src ? PAYLOAD_TYPE_ALL : PAYLOAD_TYPE_ULL; + bind_msg.payload.ull_payload = guest_rd; + bind_msg.payload_len = MAX_PAYLOAD_SIZE; + if (is_src) { + if (!dst_host_info) { + pr_err("No destination host info provided for source\n"); + ret = -EINVAL; + goto cleanup; + } + bind_msg.payload_len = strlen(dst_host_info->dst_ip) + 1; + if (strscpy(bind_msg.payload.char_payload, dst_host_info->dst_ip, + sizeof(bind_msg.payload.char_payload)) < 0) { + pr_err("Destination IP too long\n"); + ret = -EINVAL; + goto cleanup; + } + } + + do { + ret = send_and_wait_ack(sock, &bind_msg); + if (!ret) + break; + pr_warn("Send/ACK failed, retrying (%d left)\n", retry_count - 1); + retry_count--; + /* a delay time */ + schedule_timeout_uninterruptible(HZ / 10); + } while (retry_count > 0); + + if (ret) + pr_err("Failed to get bind info after retries, error=%d\n", ret); + +cleanup: + if (sock) { + if (sock->sk) { + sock->sk->sk_sndtimeo = old_sndtimeo; + sock->sk->sk_rcvtimeo = old_rcvtimeo; + } + if (ret == 0) + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); + } + return ret; +} + +int virtcca_migvm_agent_ratstls_dst(struct kvm *kvm, struct kvm_virtcca_mig_cmd *cmd) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_dst_host_info dst_info; + struct arm_smccc_res tmi_ret; + int ret = 0; + + if (g_migcvm_agent_listen_cid.cid == 0) { + pr_err("there is no cid of migcvm, cannot migrate virtCCA cVM\n"); + return -EINVAL; + } + + if (copy_from_user(&dst_info, (void __user *)cmd->data, + sizeof(struct virtcca_dst_host_info))) { + return -EFAULT; + } + + if (cmd->flags || dst_info.version != KVM_CVM_MIGVM_VERSION) { + pr_err("invalid flags or version, flags is %x, version is %x\n", + cmd->flags, dst_info.version); + return -EINVAL; + } + pr_info("calling tmi_bind_peek"); + /* check if the slot is binded*/ + tmi_ret = tmi_bind_peek(cvm->rd); + if (tmi_ret.a1 == TMI_SUCCESS) { + if (tmi_ret.a2 <= SLOT_NOT_BINDED) { + pr_err("%s: failed, err=%lx\n", __func__, tmi_ret.a1); + return -EINVAL; + } + } else { + pr_err("%s: failed, err=%lx\n", __func__, tmi_ret.a1); + return -EINVAL; + } + + ret = notify_migcvm_agent(g_migcvm_agent_listen_cid.cid, NULL, cvm->rd, false); + if (ret != 0) { + pr_err("%s: notify_migcvm_agent failed, ret=%d\n", __func__, ret); + return -EINVAL; + } + + return ret; +} + +int virtcca_migvm_agent_ratstls(struct kvm *kvm, struct kvm_virtcca_mig_cmd *cmd) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_dst_host_info dst_info; + struct arm_smccc_res tmi_ret; + int ret = 0; + + if (g_migcvm_agent_listen_cid.cid == 0) { + pr_err("there is no cid of migcvm, cannot migrate virtCCA cVM\n"); + return -EINVAL; + } + + if (copy_from_user(&dst_info, (void __user *)cmd->data, + sizeof(struct virtcca_dst_host_info))) { + return -EFAULT; + } + + if (cmd->flags || dst_info.version != KVM_CVM_MIGVM_VERSION) { + pr_err("invalid flags or version, flags is %x, version is %x\n", + cmd->flags, dst_info.version); + return -EINVAL; + } + + /* now the dst ip is none, and dst port is 0, + * it should be add check into this (after qemu input) + */ + if (strscpy(cvm->mig_cvm_info->dst_ip, dst_info.dst_ip, + sizeof(cvm->mig_cvm_info->dst_ip)) < 0) { + pr_err("save dst_ip failed\n"); + return -EINVAL; + } + cvm->mig_cvm_info->dst_port = dst_info.dst_port; + /* check if the slot is binded*/ + tmi_ret = tmi_bind_peek(cvm->rd); + if (tmi_ret.a1 == TMI_SUCCESS) { + if (tmi_ret.a2 <= SLOT_NOT_BINDED) { + pr_err("%s: failed, err=%lx\n", __func__, tmi_ret.a1); + return -EINVAL; + } + } else { + pr_err("%s: failed, err=%lx\n", __func__, tmi_ret.a1); + return -EINVAL; + } + + ret = notify_migcvm_agent(g_migcvm_agent_listen_cid.cid, &dst_info, cvm->rd, true); + if (ret != 0) { + pr_info("%s: notify_migcvm_agent failed, ret=%d\n", __func__, ret); + return -EINVAL; + } + return ret; +} + +int virtcca_get_bind_info(struct kvm *kvm, struct kvm_virtcca_mig_cmd *cmd) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + struct virtcca_bind_info info; + struct arm_smccc_res ret; + + if (copy_from_user(&info, (void __user *)cmd->data, + sizeof(struct virtcca_bind_info))) { + return -EFAULT; + } + + if (cmd->flags || info.version != KVM_CVM_MIGVM_VERSION) + return -EINVAL; + + ret = tmi_bind_peek(cvm->rd); + if (ret.a1 == TMI_SUCCESS) { + if (ret.a2 == SLOT_IS_READY) + info.premig_done = true; + else + info.premig_done = false; + if (copy_to_user((void __user *)cmd->data, &info, + sizeof(struct virtcca_bind_info))) { + return -EFAULT; + } + return ret.a1; + } + pr_err("%s: failed, err=%lx\n", __func__, ret.a1); + return -EIO; +} + +static struct kvm_device_ops kvm_virtcca_mig_stream_ops = { + .name = "kvm-virtcca-mig-stream", + .get_attr = virtcca_mig_stream_get_attr, + .set_attr = virtcca_mig_stream_set_attr, + .mmap = virtcca_mig_stream_mmap, + .ioctl = virtcca_mig_stream_ioctl, + .create = virtcca_mig_stream_create, + .destroy = virtcca_mig_stream_release, +}; + +static atomic_t g_mig_streams_used = ATOMIC_INIT(0); + +int kvm_virtcca_mig_stream_ops_init(void) +{ + int ret = 0; + + if (!atomic_read(&g_mig_streams_used)) + ret = kvm_register_device_ops(&kvm_virtcca_mig_stream_ops, + KVM_DEV_TYPE_VIRTCCA_MIG_STREAM); + + if (!ret) + atomic_inc(&g_mig_streams_used); + + return ret; +} + +void kvm_virtcca_mig_stream_ops_exit(void) +{ + atomic_dec(&g_mig_streams_used); + if (!atomic_read(&g_mig_streams_used)) + kvm_unregister_device_ops(KVM_DEV_TYPE_VIRTCCA_MIG_STREAM); +} + +void virtcca_enable_log_dirty(struct kvm *kvm, uint64_t start, uint64_t end) +{ + struct arm_smccc_res res; + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + uint64_t s_start = cvm->ipa_start; + uint64_t s_end = cvm->ipa_start + cvm->ram_size; + + if (end <= s_start || start >= s_end) + return; + + res = tmi_mem_region_protect(cvm->rd, start, end); + if (res.a1 != 0) + pr_err("tmi_mem_region_protect failed!\n"); +} + +int virtcca_mig_export_abort(struct kvm *kvm) +{ + struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; + phys_addr_t target_ipa = cvm->swiotlb_start; + struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; + int level; + uint64_t granule; + uint64_t ret = 0; + + while (target_ipa != 0 && target_ipa < cvm->swiotlb_end) { + ret = kvm_pgtable_get_leaf(pgt, target_ipa, NULL, &level); + if (ret) { + pr_err("%s: err=%llx\n", __func__, ret); + ret = -EIO; + } + + granule = kvm_granule_size(level); + ret = virtcca_stage2_update_leaf_attrs(pgt, target_ipa, granule, + KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R | KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W, 0, NULL, NULL, 0); + if (ret) { + pr_err("%s: err=%llx\n", __func__, ret); + ret = -EIO; + } + + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, target_ipa, level); + target_ipa += granule; + } + + ret = tmi_export_abort(cvm->rd); + if (ret) { + pr_err("%s: err=%llx\n", __func__, ret); + ret = -EIO; + } + + virtcca_mig_state_release(cvm); + return ret; +} + +static int virtcca_kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot) +{ + unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(memslot); + unsigned long dirty_bitmap_size = ALIGN(dirty_bytes, SZ_2M); + int current_node; + + current_node = numa_node_id(); + if (current_node < 0) { + pr_err("Failed to get current NUMA node."); + return -EINVAL; + } + + memslot->dirty_bitmap = __vmalloc_node_range(dirty_bitmap_size * 2, SZ_2M, + VMALLOC_START, VMALLOC_END, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, + VM_ALLOW_HUGE_VMAP, current_node, __builtin_return_address(0)); + if (!memslot->dirty_bitmap) + return -ENOMEM; + + return 0; +} + +int virtcca_kvm_prepare_dirty_bitmap(struct kvm *kvm, void *new) +{ + int r; + struct kvm_memory_slot *memslot = (struct kvm_memory_slot *)new; + + r = virtcca_kvm_alloc_dirty_bitmap(memslot); + if (r) + return r; + + virtcca_set_tmm_memslot(kvm, memslot); + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) + bitmap_set(memslot->dirty_bitmap, 0, memslot->npages); + return 0; +} + +void virtcca_kvm_enable_log_dirty(struct kvm *kvm, void *new, bool *flush) +{ + struct kvm_memory_slot *memslot = (struct kvm_memory_slot *)new; + + if (!kvm_is_realm(kvm)) + return; + + spin_lock((spinlock_t *)&(kvm)->mmu_lock); + + phys_addr_t start = (memslot->base_gfn) << PAGE_SHIFT; + phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + + *flush = true; + virtcca_enable_log_dirty(kvm, start, end); + + spin_unlock((spinlock_t *)&(kvm)->mmu_lock); +} + +bool kvm_check_realm(struct kvm *kvm) +{ + return kvm_is_realm(kvm); +} #endif diff --git a/include/linux/virtcca_cvm_domain.h b/include/linux/virtcca_cvm_domain.h index c5d7779b1ffc..588b07030b8b 100644 --- a/include/linux/virtcca_cvm_domain.h +++ b/include/linux/virtcca_cvm_domain.h @@ -55,6 +55,9 @@ void virtcca_dev_destroy(u64 dev_num, u64 clean); bool is_virtcca_pci_cc_dev(struct device *dev); int virtcca_create_vdev(struct device *dev); bool is_virtcca_secure_vf(struct device *dev, struct device_driver *drv); +int virtcca_kvm_prepare_dirty_bitmap(struct kvm *kvm, void *new); +void virtcca_kvm_enable_log_dirty(struct kvm *kvm, void *new, bool *flush); +bool kvm_check_realm(struct kvm *kvm); #else static inline size_t virtcca_pci_get_rom_size(void *pdev, void __iomem *rom, @@ -89,5 +92,20 @@ static inline bool is_virtcca_secure_vf(struct device *dev, struct device_driver { return false; } + +static inline int virtcca_kvm_prepare_dirty_bitmap(struct kvm *kvm, void *new) +{ + return 0; +} + +static inline void virtcca_kvm_enable_log_dirty(struct kvm *kvm, void *new, bool *flush) +{ + ; +} + +static inline bool kvm_check_realm(struct kvm *kvm) +{ + return false; +} #endif /* CONFIG_HISI_VIRTCCA_CODA */ #endif /* __VIRTCCA_CVM_DOMAIN_H */ diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 0036cfaf5d69..609c24bef381 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1499,6 +1499,8 @@ enum kvm_device_type { #define KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_RISCV_AIA, #define KVM_DEV_TYPE_RISCV_AIA KVM_DEV_TYPE_RISCV_AIA + KVM_DEV_TYPE_VIRTCCA_MIG_STREAM = 0x00C, +#define KVM_DEV_TYPE_VIRTCCA_MIG_STREAM KVM_DEV_TYPE_VIRTCCA_MIG_STREAM KVM_DEV_TYPE_LOONGARCH_IPI, #define KVM_DEV_TYPE_LOONGARCH_IPI KVM_DEV_TYPE_LOONGARCH_IPI KVM_DEV_TYPE_LOONGARCH_EIOINTC, @@ -1551,6 +1553,8 @@ struct kvm_numa_info { #define KVM_SET_IDENTITY_MAP_ADDR _IOW(KVMIO, 0x48, __u64) #define KVM_LOAD_USER_DATA _IOW(KVMIO, 0x49, struct kvm_user_data) +/*virtcca migration*/ +#define KVM_CVM_MIG_IOCTL _IOWR(KVMIO, 0xf2, struct kvm_virtcca_mig_cmd) #define KVM_CAP_ARM_TMM 300 /* FIXME: Large number to prevent conflicts */ diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index b44ab7919428..f252cc39495f 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1804,6 +1804,7 @@ void __init swiotlb_cvm_update_mem_attributes(void) bytes = PAGE_ALIGN(io_tlb_default_mem.defpool.nslabs << IO_TLB_SHIFT); set_cvm_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT); memset(vaddr, 0, bytes); + swiotlb_unmap_notify(io_tlb_default_mem.defpool.start, bytes); io_tlb_default_mem.for_alloc = true; } #endif diff --git a/tools/include/uapi/linux/kvm.h b/tools/include/uapi/linux/kvm.h index bd1a496b5448..34f3243767c1 100644 --- a/tools/include/uapi/linux/kvm.h +++ b/tools/include/uapi/linux/kvm.h @@ -1448,6 +1448,8 @@ enum kvm_device_type { #define KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_ARM_PV_TIME KVM_DEV_TYPE_RISCV_AIA, #define KVM_DEV_TYPE_RISCV_AIA KVM_DEV_TYPE_RISCV_AIA + KVM_DEV_TYPE_VIRTCCA_MIG_STREAM = 0x00C, +#define KVM_DEV_TYPE_VIRTCCA_MIG_STREAM KVM_DEV_TYPE_VIRTCCA_MIG_STREAM KVM_DEV_TYPE_LA_IOAPIC = 0x100, #define KVM_DEV_TYPE_LA_IOAPIC KVM_DEV_TYPE_LA_IOAPIC KVM_DEV_TYPE_LA_IPI, @@ -1668,6 +1670,9 @@ struct kvm_enc_region { __u64 size; }; +/*virtcca migration*/ +#define KVM_CVM_MIG_IOCTL _IOWR(KVMIO, 0xf2, struct kvm_virtcca_mig_cmd) + #define KVM_MEMORY_ENCRYPT_REG_REGION _IOR(KVMIO, 0xbb, struct kvm_enc_region) #define KVM_MEMORY_ENCRYPT_UNREG_REGION _IOR(KVMIO, 0xbc, struct kvm_enc_region) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index d8cb9c60db20..830d445dad84 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1703,6 +1703,13 @@ static int kvm_prepare_memory_region(struct kvm *kvm, if (change != KVM_MR_DELETE) { if (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) new->dirty_bitmap = NULL; +#ifdef CONFIG_HISI_VIRTCCA_HOST + else if (kvm_check_realm(kvm) && kvm_use_dirty_bitmap(kvm)) { + r = virtcca_kvm_prepare_dirty_bitmap(kvm, (void *)new); + if (r) + return r; + } +#endif else if (old && old->dirty_bitmap) new->dirty_bitmap = old->dirty_bitmap; else if (kvm_use_dirty_bitmap(kvm)) { @@ -2241,6 +2248,12 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) n = kvm_dirty_bitmap_bytes(memslot); flush = false; + +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (!kvm_check_realm(kvm)) + kvm->manual_dirty_log_protect = false; +#endif + if (kvm->manual_dirty_log_protect) { /* * Unlike kvm_get_dirty_log, we always return false in *flush, @@ -2274,6 +2287,10 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) KVM_MMU_UNLOCK(kvm); } +#ifdef CONFIG_HISI_VIRTCCA_HOST + virtcca_kvm_enable_log_dirty(kvm, (void *)memslot, &flush); +#endif + if (flush) kvm_flush_remote_tlbs_memslot(kvm, memslot); -- Gitee