From 1cf4caf10f1deda33ae1a0b9be4de899928bd928 Mon Sep 17 00:00:00 2001 From: Guoqi Xie Date: Fri, 29 Nov 2024 10:34:36 +0800 Subject: [PATCH 1/6] Log: add zvm log Adding boot log for zvm. Signed-off-by: Guoqi Xie --- samples/_zvm/src/main.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/samples/_zvm/src/main.c b/samples/_zvm/src/main.c index 4500d7089..aa39d3d2b 100644 --- a/samples/_zvm/src/main.c +++ b/samples/_zvm/src/main.c @@ -1,8 +1,9 @@ /* - * Copyright 2021-2022 HNU-ESNL - * - * SPDX-License-Identifier: Apache-2.0 - */ + * Copyright 2021-2022 HNU-ESNL + * + * SPDX-License-Identifier: Apache-2.0 + */ + #include #include #include @@ -13,13 +14,22 @@ #include #include #include -#include -/* For test time slice vm change */ -int main(int argc, char **argv) +static void zvm_print_log(void) { - ZVM_LOG_INFO("--Init zvm successful! --\n"); - ZVM_LOG_INFO("--Ready to input shell cmd to create and build vm!--\n"); + printk("\n"); + printk("\n"); + printk("█████████╗ ██╗ ██╗ ███╗ ███╗ \n"); + printk("╚════███╔╝ ██║ ██║ ████╗ ████║ \n"); + printk(" ███╔╝ ╚██╗ ██╔╝ ██╔ ████╔██║ \n"); + printk(" ██╔╝ ╚██ ██╔╝ ██║ ╚██╔╝██║ \n"); + printk("█████████╗ ╚████╔╝ ██║ ╚═╝ ██║ \n"); + printk("╚════════╝ ╚═╝ ╚═╝ ╚═╝ \n"); + +} +int main(int argc, char **argv) +{ + zvm_print_log(); return 0; } -- Gitee From ebee46991c5f077670a8c5d1e7d7896ff621bb56 Mon Sep 17 00:00:00 2001 From: Guoqi Xie Date: Sun, 1 Dec 2024 16:57:05 +0800 Subject: [PATCH 2/6] vgic: add pending operation in vgic In some VM, vm may operate vgic's pending status, we should emulate it. Signed-off-by: Guoqi Xie --- include/virtualization/vdev/vgic_common.h | 5 +- include/virtualization/vdev/vgic_v3.h | 49 +++++++++++++++++ include/virtualization/vm.h | 5 +- include/virtualization/zvm.h | 2 +- subsys/virtualization/vdev/vgic_common.c | 67 ++++++++++++++++++++++- subsys/virtualization/vdev/vgic_v3.c | 13 ++++- subsys/virtualization/vm_cpu.c | 1 + 7 files changed, 135 insertions(+), 7 deletions(-) diff --git a/include/virtualization/vdev/vgic_common.h b/include/virtualization/vdev/vgic_common.h index 63fc549a3..1d62d8ce5 100644 --- a/include/virtualization/vdev/vgic_common.h +++ b/include/virtualization/vdev/vgic_common.h @@ -147,14 +147,15 @@ int vgic_vdev_mem_read(struct virt_dev *vdev, uint64_t addr, uint64_t *value, ui int vgic_vdev_mem_write(struct virt_dev *vdev, uint64_t addr, uint64_t *value, uint16_t size); /** - * @brief send a virt irq signal to a vcpu. + * @brief set/unset a virt irq signal to a vcpu. */ int set_virq_to_vcpu(struct vcpu *vcpu, uint32_t virq_num); /** - * @brief send a virq to vm that bind to this vcpu. + * @brief set/unset a virt irq to vm. */ int set_virq_to_vm(struct vm *vm, uint32_t virq_num); +int unset_virq_to_vm(struct vm *vm, uint32_t virq_num); int virt_irq_sync_vgic(struct vcpu *vcpu); int virt_irq_flush_vgic(struct vcpu *vcpu); diff --git a/include/virtualization/vdev/vgic_v3.h b/include/virtualization/vdev/vgic_v3.h index c6dc224ba..a947c8ea6 100644 --- a/include/virtualization/vdev/vgic_v3.h +++ b/include/virtualization/vdev/vgic_v3.h @@ -63,6 +63,7 @@ #define VGICR_ISENABLER0 0x0100 #define VGICR_ICENABLER0 0x0180 #define VGICR_SGI_PENDING 0x0200 +#define VGICR_SGI_ICPENDING 0x0280 #define VGICR_PIDR2 0xFFE8 /* list register test and set */ @@ -274,6 +275,54 @@ static ALWAYS_INLINE void vgic_test_and_set_enable_bit(struct vcpu *vcpu, uint32 } } +/** + * @brief When VM write ispending or icpending flag, we + * should set/unset irq signal to VM. +*/ +static ALWAYS_INLINE void vgic_test_and_set_pending_bit(struct vcpu *vcpu, uint32_t spi_nr_count, + uint32_t *value, uint32_t bit_size, bool enable, void *vgic_priv) +{ + int bit; + uint32_t reg_mem_addr = (uint64_t)value; + struct virt_gic_gicd *gicd = NULL; + struct virt_gic_gicr *gicr = NULL; + + for (bit=0; bit= VM_GLOBAL_VIRQ_NR) { + /* spi num is too big. */ + return; + } + set_virq_to_vm(vcpu->vm, spi_nr_count + bit); + if(spi_nr_count < VM_LOCAL_VIRQ_NR){ + gicr = (struct virt_gic_gicr *)vgic_priv; + vgic_sysreg_write32(vgic_sysreg_read32(gicr->gicr_sgi_reg_base, VGICR_SGI_PENDING) | BIT(bit),\ + gicr->gicr_sgi_reg_base, VGICR_SGI_PENDING); + }else{ + gicd = (struct virt_gic_gicd *)vgic_priv; + vgic_sysreg_write32(vgic_sysreg_read32(gicd->gicd_regs_base, VGICD_ISPENDRn) | BIT(bit),\ + gicd->gicd_regs_base, VGICD_ISPENDRn); + } + } else { + if (spi_nr_count + bit >= VM_GLOBAL_VIRQ_NR) { + return; + } + unset_virq_to_vm(vcpu->vm, spi_nr_count + bit); + if(spi_nr_count < VM_LOCAL_VIRQ_NR){ + gicr = (struct virt_gic_gicr *)vgic_priv; + vgic_sysreg_write32(vgic_sysreg_read32(gicr->gicr_sgi_reg_base, VGICR_SGI_ICPENDING) & ~BIT(bit),\ + gicr->gicr_sgi_reg_base, VGICR_SGI_ICPENDING); + }else{ + gicd = (struct virt_gic_gicd *)vgic_priv; + vgic_sysreg_write32(vgic_sysreg_read32(gicd->gicd_regs_base, VGICD_ICPENDRn) | ~BIT(bit),\ + gicd->gicd_regs_base, VGICD_ICPENDRn); + } + } + } + } +} + static ALWAYS_INLINE uint64_t gicv3_read_lr(uint8_t register_id) { switch (register_id) { diff --git a/include/virtualization/vm.h b/include/virtualization/vm.h index 46778b83e..96069710b 100644 --- a/include/virtualization/vm.h +++ b/include/virtualization/vm.h @@ -39,8 +39,9 @@ /** * @brief VM return values. */ -#define VM_IRQ_TO_VM_SUCCESS (1) -#define VM_IRQ_TO_VCPU_SUCCESS (2) +#define SET_IRQ_TO_VM_SUCCESS (1) +#define UNSET_IRQ_TO_VM_SUCCESS (2) +#define VM_IRQ_TO_VCPU_SUCCESS (3) #define VM_IRQ_NUM_OUT (99) #define _VCPU_STATE_READY (BIT(0)) diff --git a/include/virtualization/zvm.h b/include/virtualization/zvm.h index 092c56240..709774caa 100644 --- a/include/virtualization/zvm.h +++ b/include/virtualization/zvm.h @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 HNU + * Copyright 2021-2022 HNU-ESNL * * SPDX-License-Identifier: Apache-2.0 */ diff --git a/subsys/virtualization/vdev/vgic_common.c b/subsys/virtualization/vdev/vgic_common.c index 0431268d6..a258b4c59 100644 --- a/subsys/virtualization/vdev/vgic_common.c +++ b/subsys/virtualization/vdev/vgic_common.c @@ -171,6 +171,34 @@ static int vgic_set_virq(struct vcpu *vcpu, struct virt_irq_desc *desc) return 0; } +static int vgic_unset_virq(struct vcpu *vcpu, struct virt_irq_desc *desc) +{ + uint8_t lr_state; + k_spinlock_key_t key; + struct vcpu_virt_irq_block *vb = &vcpu->virq_block; + + if (!is_vm_irq_valid(vcpu->vm, desc->virq_flags)) { + ZVM_LOG_WARN("VM can not recieve virq signal, \ + VM's name: %s.", vcpu->vm->vm_name); + return -EVIRQ; + } + + key = k_spin_lock(&vb->spinlock); + lr_state = desc->virq_states; + + desc->virq_flags &= ~VIRQ_PENDING_FLAG; + desc->virq_flags &= ~VIRQ_ACTIVED_FLAG; + + if (sys_dnode_is_linked(&desc->desc_node)) { + sys_dlist_remove(&desc->desc_node); + vb->virq_pending_counts--; + } + + k_spin_unlock(&vb->spinlock, key); + + return 0; +} + /** * @brief set sgi interrupt to vm, which usually used on vcpu * communication. @@ -247,6 +275,18 @@ static int vgic_gicd_mem_write(struct vcpu *vcpu, struct virt_gic_gicd *gicd, y = x * 32; vgic_test_and_set_enable_bit(vcpu, y, value, 32, 0, gicd); break; + case GICD_ISPENDRn...(GICD_ICPENDRn - 1): + /* Set virt irq to vm. */ + x = (offset - GICD_ISPENDRn) / 4; + y = x * 32; + vgic_test_and_set_pending_bit(vcpu, y, value, 32, 1, gicd); + break; + case GICD_ICPENDRn...(GICD_ISACTIVERn - 1): + /* Unset virt irq to vm. */ + x = (offset - GICD_ICPENDRn) / 4; + y = x * 32; + vgic_test_and_set_pending_bit(vcpu, y, value, 32, 0, gicd); + break; case GICD_IPRIORITYRn...(GIC_DIST_BASE + 0x07f8 - 1): t = *value; x = (offset - GICD_IPRIORITYRn) / 4; @@ -414,7 +454,32 @@ int set_virq_to_vm(struct vm *vm, uint32_t virq_num) target_vcpu = vm->vcpus[desc->vcpu_id]; ret = vgic_set_virq(target_vcpu, desc); if (ret >= 0) { - return VM_IRQ_TO_VM_SUCCESS; + return SET_IRQ_TO_VM_SUCCESS; + } + + return ret; +} + +int unset_virq_to_vm(struct vm *vm, uint32_t virq_num) +{ + uint32_t ret = 0; + struct virt_irq_desc *desc; + struct vcpu *vcpu, *target_vcpu; + vcpu = vm->vcpus[DEFAULT_VCPU]; + + if (virq_num < VM_LOCAL_VIRQ_NR) { + desc = &vcpu->virq_block.vcpu_virt_irq_desc[virq_num]; + } else if (virq_num <= VM_GLOBAL_VIRQ_NR) { + desc = &vm->vm_irq_block.vm_virt_irq_desc[virq_num - VM_LOCAL_VIRQ_NR]; + } else { + ZVM_LOG_WARN("The spi num that ready to allocate is too big."); + return -ENODEV; + } + + target_vcpu = vm->vcpus[desc->vcpu_id]; + ret = vgic_unset_virq(target_vcpu, desc); + if (ret >= 0) { + return UNSET_IRQ_TO_VM_SUCCESS; } return ret; diff --git a/subsys/virtualization/vdev/vgic_v3.c b/subsys/virtualization/vdev/vgic_v3.c index ab9343a87..2ba86d38c 100644 --- a/subsys/virtualization/vdev/vgic_v3.c +++ b/subsys/virtualization/vdev/vgic_v3.c @@ -405,6 +405,7 @@ int vgicv3_raise_sgi(struct vcpu *vcpu, unsigned long sgi_value) int i, bit, sgi_num=0; uint32_t sgi_id, sgi_mode; uint32_t target_list, aff1, aff2, aff3, tmp_id; + uint32_t target_vcpu_list = 0; struct vcpu *target; struct vm *vm = vcpu->vm; k_spinlock_key_t key; @@ -424,6 +425,7 @@ int vgicv3_raise_sgi(struct vcpu *vcpu, unsigned long sgi_value) target->vcpuipi_count ++; k_spin_unlock(&target->vcpu_lock, key); } + arch_sched_ipi(); } else if (sgi_mode == SGI_SIG_TO_LIST) { target_list = sgi_value & 0xffff; aff1 = (sgi_value & (uint64_t)(0xffUL << 16)) >> 16; @@ -433,6 +435,7 @@ int vgicv3_raise_sgi(struct vcpu *vcpu, unsigned long sgi_value) if (sys_test_bit((uintptr_t)&target_list, bit)) { /*Each cluster has CONFIG_MP_NUM_CPUS*/ tmp_id = aff1 * CONFIG_MP_NUM_CPUS + bit; + sys_set_bits((uintptr_t)&target_vcpu_list, BIT(tmp_id)); /*@TODO: May need modified to vm->vcpu_num. */ if(++sgi_num > CONFIG_MAX_VCPU_PER_VM || tmp_id >= CONFIG_MAX_VCPU_PER_VM) { ZVM_LOG_WARN("The target cpu list is too long."); @@ -445,11 +448,19 @@ int vgicv3_raise_sgi(struct vcpu *vcpu, unsigned long sgi_value) k_spin_unlock(&target->vcpu_lock, key); } } + if(target_vcpu_list & BIT(tmp_id)) { + set_virq_to_vm(vcpu->vm, sgi_id); + /* Set vcpu flag include itself */ + if(target_vcpu_list & ~BIT(tmp_id)) { + arch_sched_ipi(); + } + } else { + arch_sched_ipi(); + } } else { ZVM_LOG_WARN("Unsupported sgi signal."); return -EVIRQ; } - arch_sched_ipi(); return 0; } diff --git a/subsys/virtualization/vm_cpu.c b/subsys/virtualization/vm_cpu.c index 740ac4b79..3950408be 100644 --- a/subsys/virtualization/vm_cpu.c +++ b/subsys/virtualization/vm_cpu.c @@ -365,6 +365,7 @@ struct vcpu *vm_vcpu_init(struct vm *vm, uint16_t vcpu_id, char *vcpu_name) sys_dlist_init(&vcpu->virq_block.active_irqs); ZVM_SPINLOCK_INIT(&vcpu->virq_block.spinlock); init_vcpu_virt_irq_desc(&vcpu->virq_block); + ZVM_SPINLOCK_INIT(&vcpu->vcpu_lock); if (vm->is_rtos) { vm_prio = VCPU_RT_PRIO; -- Gitee From 36149a3b80aaf54c2fef75c430c46199764122cc Mon Sep 17 00:00:00 2001 From: Guoqi Xie Date: Sun, 1 Dec 2024 20:20:59 +0800 Subject: [PATCH 3/6] mmu: remove stage-2 page table for vm Remove vm page table when delete vm. Signed-off-by: Guoqi Xie --- arch/arm64/core/virtualization/mm.c | 60 +++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/arch/arm64/core/virtualization/mm.c b/arch/arm64/core/virtualization/mm.c index f3c08d2ad..99bacc602 100644 --- a/arch/arm64/core/virtualization/mm.c +++ b/arch/arm64/core/virtualization/mm.c @@ -28,10 +28,10 @@ LOG_MODULE_DECLARE(ZVM_MODULE_NAME); static uint64_t vm_zephyr_xlat_tables[ZVM_ZEPHYR_VM_NUM][CONFIG_ZVM_ZEPHYR_MAX_XLAT_TABLES * Ln_XLAT_NUM_ENTRIES]\ __aligned(Ln_XLAT_NUM_ENTRIES * sizeof(uint64_t)); -uint16_t vm_zephyr_xlat_use_count[ZVM_ZEPHYR_VM_NUM][CONFIG_ZVM_ZEPHYR_MAX_XLAT_TABLES]; +static int vm_zephyr_xlat_use_count[ZVM_ZEPHYR_VM_NUM][CONFIG_ZVM_ZEPHYR_MAX_XLAT_TABLES]; static uint64_t vm_linux_xlat_tables[ZVM_LINUX_VM_NUM][CONFIG_ZVM_LINUX_MAX_XLAT_TABLES * Ln_XLAT_NUM_ENTRIES]\ __aligned(Ln_XLAT_NUM_ENTRIES * sizeof(uint64_t)); -uint16_t vm_linux_xlat_use_count[ZVM_LINUX_VM_NUM][CONFIG_ZVM_LINUX_MAX_XLAT_TABLES]; +static int vm_linux_xlat_use_count[ZVM_LINUX_VM_NUM][CONFIG_ZVM_LINUX_MAX_XLAT_TABLES]; static struct k_spinlock vm_xlat_lock; /** @@ -255,6 +255,13 @@ static int vm_table_usage(uint64_t *table, int adjustment, uint32_t vmid) return table_use; } +static inline void vm_dec_table_ref(uint64_t *table, uint32_t vmid) +{ + int ref_unit = 0xFFFFFFFF; + + vm_table_usage(table, -ref_unit, vmid); +} + static inline bool vm_is_table_unused(uint64_t *table, uint32_t vmid) { return vm_table_usage(table, 0, vmid) == 1; @@ -264,8 +271,9 @@ static uint64_t *vm_expand_to_table(uint64_t *pte, unsigned int level, uint32_t { uint64_t *table; - if(level >= XLAT_LAST_LEVEL) + if(level >= XLAT_LAST_LEVEL) { __ASSERT(level < XLAT_LAST_LEVEL, "can't expand last level"); + } table = vm_new_table(vmid); @@ -394,6 +402,43 @@ move_on: return ret; } +static void vm_del_mapping(uint64_t *table, uintptr_t virt, size_t size, + unsigned int level, uint32_t vmid) +{ + size_t step, level_size = 1ULL << LEVEL_TO_VA_SIZE_SHIFT(level); + uint64_t *pte, *subtable; + + for ( ; size; virt += step, size -= step) { + step = level_size - (virt & (level_size - 1)); + if (step > size) { + step = size; + } + pte = &table[XLAT_TABLE_VA_IDX(virt, level)]; + + if (vm_is_free_desc(*pte)) { + continue; + } + + if (step != level_size && vm_is_block_desc(*pte)) { + /* need to split this block mapping */ + vm_expand_to_table(pte, level, vmid); + } + + if (vm_is_table_desc(*pte, level)) { + subtable = vm_pte_desc_table(*pte); + vm_del_mapping(subtable, virt, step, level + 1, vmid); + if (!vm_is_table_unused(subtable, vmid)) { + continue; + } + vm_dec_table_ref(subtable, vmid); + } + + /* free this entry */ + *pte = 0; + vm_table_usage(pte, -1, vmid); + } +} + /** * @brief un_map the vm's page table entry. */ @@ -457,13 +502,13 @@ static int vm_add_map(struct arm_mmu_ptables *ptables, const char *name, } static int vm_remove_map(struct arm_mmu_ptables *ptables, const char *name, - uintptr_t virt,size_t size,uint32_t vmid) + uintptr_t virt, size_t size, uint32_t vmid) { k_spinlock_key_t key; - int ret; + int ret = 0; key = k_spin_lock(&vm_xlat_lock); - ret = vm_set_mapping(ptables,virt,size,0,true,vmid); + vm_del_mapping(ptables->base_xlat_table, virt, size, BASE_XLAT_LEVEL, vmid); k_spin_unlock(&vm_xlat_lock,key); return ret; } @@ -529,8 +574,7 @@ int arch_vm_mem_domain_partition_remove(struct k_mem_domain *domain, struct arm_mmu_ptables *domain_ptables = &domain->arch.ptables; struct k_mem_partition *ptn = &domain->partitions[partition_id]; - ret = vm_remove_map(domain_ptables,"vm-mmio-space",ptn->start,ptn->size,vmid); - + ret = vm_remove_map(domain_ptables, "vm-mmio-space", ptn->start, ptn->size, vmid); return ret; } -- Gitee From 2b3e4c365293c7b0b3f5bcdef7a520c77558647c Mon Sep 17 00:00:00 2001 From: Guoqi Xie Date: Sun, 1 Dec 2024 20:23:32 +0800 Subject: [PATCH 4/6] Mem: replace vm memory for qemu platform When delete vm, we need store the vm imgae and copy it to vm boot memory address. Signed-off-by: Guoqi Xie --- auto_zvm.sh | 10 +- include/virtualization/os/os_zephyr.h | 2 + .../_zvm/boards/qemu_cortex_max_smp.overlay | 95 +++++++++---------- samples/_zvm/boards/roc_rk3568_pc_smp.overlay | 5 + subsys/virtualization/os/os_zephyr.c | 19 ++++ zvm_config/qemu_platform/linux-qemu-virt.dts | 4 +- 6 files changed, 80 insertions(+), 55 deletions(-) diff --git a/auto_zvm.sh b/auto_zvm.sh index c0a0b4850..a1f35ae98 100755 --- a/auto_zvm.sh +++ b/auto_zvm.sh @@ -63,13 +63,15 @@ elif [ "$OPS" = "${ops_array[1]}" ]; then -cpu max -m 8G -nographic -machine virt,virtualization=on,gic-version=3 \ -net none -pidfile qemu.pid -chardev stdio,id=con,mux=on \ -serial chardev:con -mon chardev=con,mode=readline -serial pty -serial pty -smp cpus=4 \ - -device loader,file=$(pwd)/zvm_config/qemu_platform/hub/zephyr.bin,addr=0xc8000000,force-raw=on \ - -device loader,file=$(pwd)/zvm_config/qemu_platform/hub/Image_rfss,addr=0xe0000000,force-raw=on \ - -device loader,file=$(pwd)/zvm_config/qemu_platform/hub/linux-qemu-virt.dtb,addr=0xf2a00000 \ - -device loader,file=$(pwd)/zvm_config/qemu_platform/hub/debian.cpio.gz,addr=0xe4000000 \ + -device loader,file=$(pwd)/zvm_config/qemu_platform/hub/zephyr.bin,addr=0x60000000,force-raw=on \ + -device loader,file=$(pwd)/zvm_config/qemu_platform/hub/Image_rfss,addr=0x70000000,force-raw=on \ + -device loader,file=$(pwd)/zvm_config/qemu_platform/hub/linux-qemu-virt.dtb,addr=0xf2000000 \ + -device loader,file=$(pwd)/zvm_config/qemu_platform/hub/debian.cpio.gz,addr=0x90000000 \ -kernel $(pwd)/build/zephyr/zvm_host.elf ### using gdb to connect it: # gdb-multiarch -q -ex 'file ./build/zephyr/zvm_host.elf' -ex 'target remote localhost:1234' +### using trace to record qemu info when boot qemu +# strace -o qemu_bug.txt elif [ "$PLAT" = "${plat_array[1]}" ]; then ### export lib for Foundation_Platform, sometimes some lib may not be found diff --git a/include/virtualization/os/os_zephyr.h b/include/virtualization/os/os_zephyr.h index 393d77ee7..13b58786c 100644 --- a/include/virtualization/os/os_zephyr.h +++ b/include/virtualization/os/os_zephyr.h @@ -20,6 +20,8 @@ #define ZEPHYR_VM_IMAGE_SIZE DT_REG_SIZE(DT_NODELABEL(zephyr_ddr)) #define ZEPHYR_VMSYS_BASE DT_PROP(DT_NODELABEL(zephyr_ddr), vm_reg_base) #define ZEPHYR_VMSYS_SIZE DT_PROP(DT_NODELABEL(zephyr_ddr), vm_reg_size) +#define ZEPHYR_VMCPY_BASE DT_REG_ADDR(DT_ALIAS(zephyrcpy)) +#define ZEPHYR_VMCPY_SIZE DT_REG_SIZE(DT_ALIAS(zephyrcpy)) #define ZEPHYR_VM_VCPU_NUM DT_PROP(DT_INST(0, zephyr_vm), vcpu_num) int load_zephyr_image(struct vm_mem_domain *vmem_domain); diff --git a/samples/_zvm/boards/qemu_cortex_max_smp.overlay b/samples/_zvm/boards/qemu_cortex_max_smp.overlay index 351b2efb4..8b398b150 100644 --- a/samples/_zvm/boards/qemu_cortex_max_smp.overlay +++ b/samples/_zvm/boards/qemu_cortex_max_smp.overlay @@ -8,8 +8,9 @@ aliases { vmvirtio1 = "/soc/virtio_mmio@a000000"; vmvirtio2 = "/soc/virtio_mmio@a001000"; - linuxcpy = "/soc/linux_cpy@e0000000"; - linuxrfs = "/soc/linux_rootfs@e4000000"; + zephyrcpy = "/soc/zephyr_cpy@60000000"; + linuxcpy = "/soc/linux_cpy@70000000"; + linuxrfs = "/soc/linux_rootfs@90000000"; /*Passthrough device.*/ ptdevice1 = "/soc/pass_through_device/uart@9001000"; ptdevice2 = "/soc/pass_through_device/uart@9002000"; @@ -58,36 +59,6 @@ status = "okay"; }; - pass_through_device { - - #address-cells = <2>; - #size-cells = <2>; - ranges; - - uart1: uart@9001000 { - compatible = "arm,pl011"; - reg = <0x00 0x09001000 0x00 0x1000>; - interrupts = ; - interrupt-names = "irq_0"; - clocks = <&uartclk>; - status = "reserved"; - current-speed = <115200>; - label = "UART_1"; - }; - - uart2: uart@9002000 { - compatible = "arm,pl011"; - reg = <0x00 0x09002000 0x00 0x1000>; - interrupts = ; - interrupt-names = "irq_0"; - clocks = <&uartclk>; - status = "reserved"; - current-speed = <115200>; - label = "UART_2"; - }; - - }; - pcie@4010000000 { compatible = "pci-host-ecam-generic"; interrupt-map-mask = <0x1800 0x00 0x00 0x07>; @@ -120,22 +91,48 @@ compatible = "pci-host-ecam-generic"; }; - shmem_rw@bd000000 { - compatible = "shmem,rw"; - device_type = "memory"; - reg = <0x0 0xbd000000 0x0 0x10>; - interrupts = ; - label = "VM_SHMEMRW"; - status = "okay"; + zephyr_cpy@60000000 { + reg = <0x0 0x60000000 0x0 DT_SIZE_M(2)>; + }; + + linux_cpy@70000000 { + reg = <0x0 0x70000000 0x0 DT_SIZE_M(64)>; }; - linux_cpy@e0000000 { - reg = <0x0 0xe0000000 0x0 DT_SIZE_M(64)>; + linux_rootfs@90000000 { + reg = <0x0 0x90000000 0x0 DT_SIZE_M(180)>; }; - linux_rootfs@e4000000 { - reg = <0x0 0xe4000000 0x0 DT_SIZE_M(180)>; + pass_through_device { + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uart1: uart@9001000 { + compatible = "arm,pl011"; + reg = <0x00 0x09001000 0x00 0x1000>; + interrupts = ; + interrupt-names = "irq_0"; + clocks = <&uartclk>; + status = "reserved"; + current-speed = <115200>; + label = "UART_1"; + }; + + uart2: uart@9002000 { + compatible = "arm,pl011"; + reg = <0x00 0x09002000 0x00 0x1000>; + interrupts = ; + interrupt-names = "irq_0"; + clocks = <&uartclk>; + status = "reserved"; + current-speed = <115200>; + label = "UART_2"; + }; + }; + }; vm_zephyr_space { @@ -146,13 +143,13 @@ dtb_size = ; vcpu_num = <0x01>; - zephyr_ddr: memory@c8000000 { + zephyr_ddr: memory@80000000 { compatible = "vm-dram"; memmap_type = "direct"; address_type = "normal_memory"; vm_reg_base = <0x40000000>; vm_reg_size = ; - reg = <0x0 0xc8000000 0x0 DT_SIZE_M(1)>; + reg = <0x0 0x80000000 0x0 DT_SIZE_M(1)>; label = "VM0_MEM"; }; }; @@ -161,18 +158,18 @@ #address-cells = <2>; #size-cells = <2>; compatible = "linux-vm"; - dtb_address = <0xf2a00000>; + dtb_address = <0xf2000000>; dtb_size = ; vcpu_num = <0x01>; - rootfs_address = <0xf8000000>; + rootfs_address = <0xc9000000>; - linux_ddr: memory@f3000000 { + linux_ddr: memory@c0000000 { compatible = "vm-dram"; memmap_type = "direct"; address_type = "normal_memory"; vm_reg_base = <0x40000000>; vm_reg_size = ; - reg = <0x0 0xf3000000 0x0 DT_SIZE_M(64)>; + reg = <0x0 0xc0000000 0x0 DT_SIZE_M(64)>; label = "VM1_MEM"; }; diff --git a/samples/_zvm/boards/roc_rk3568_pc_smp.overlay b/samples/_zvm/boards/roc_rk3568_pc_smp.overlay index 38d99bb95..af18fef75 100644 --- a/samples/_zvm/boards/roc_rk3568_pc_smp.overlay +++ b/samples/_zvm/boards/roc_rk3568_pc_smp.overlay @@ -18,6 +18,7 @@ linuxdebugger = "/soc/fiq-debugger"; vmvirtmem = "/soc/virtmem@bf000000"; vmshmemrw = "/soc/shmem_rw@bd000000"; + zephyrcpy = "/soc/zephyr_cpy@5000000"; linuxcpy = "/soc/linux_cpy@e0000000"; linuxrfs = "/soc/linux_rootfs@bf000000"; }; @@ -155,6 +156,10 @@ status = "okay"; }; + zephyr_cpy@5000000 { + reg = <0x0 0x05000000 0x0 DT_SIZE_M(2)>; + }; + linux_cpy@e0000000 { reg = <0x0 0xe0000000 0x0 DT_SIZE_M(64)>; }; diff --git a/subsys/virtualization/os/os_zephyr.c b/subsys/virtualization/os/os_zephyr.c index 3207ea93b..bbb3a5959 100644 --- a/subsys/virtualization/os/os_zephyr.c +++ b/subsys/virtualization/os/os_zephyr.c @@ -52,6 +52,25 @@ int load_zephyr_image(struct vm_mem_domain *vmem_domain) ARG_UNUSED(blk); ARG_UNUSED(this_vm); + uint64_t *src_hva, des_hva; + uint64_t num_m = ZEPHYR_VM_IMAGE_SIZE / (1024 * 1024); + uint64_t src_hpa = ZEPHYR_VMCPY_BASE; + uint64_t des_hpa = ZEPHYR_VM_IMAGE_BASE; + uint64_t per_size = 1048576; //1M + + ZVM_LOG_INFO("1 image_num_m = %ld\n", num_m); + ZVM_LOG_INFO("1 image_src_hpa = 0x%lx\n", src_hpa); + ZVM_LOG_INFO("1 image_des_hpa = 0x%lx\n", des_hpa); + + while(num_m){ + z_phys_map((uint8_t **)&src_hva, (uintptr_t)src_hpa, per_size, K_MEM_CACHE_NONE | K_MEM_PERM_RW); + z_phys_map((uint8_t **)&des_hva, (uintptr_t)des_hpa, per_size, K_MEM_CACHE_NONE | K_MEM_PERM_RW); + memcpy(des_hva, src_hva, per_size); + des_hpa += per_size; + src_hpa += per_size; + num_m--; + } + #ifndef CONFIG_VM_DYNAMIC_MEMORY ARG_UNUSED(this_vm); return ret; diff --git a/zvm_config/qemu_platform/linux-qemu-virt.dts b/zvm_config/qemu_platform/linux-qemu-virt.dts index f944d6dea..11cd8e81a 100644 --- a/zvm_config/qemu_platform/linux-qemu-virt.dts +++ b/zvm_config/qemu_platform/linux-qemu-virt.dts @@ -103,8 +103,8 @@ chosen { bootargs = "root=/dev/ram0 init=/sbin/init ramdisk_size=10000000 rootfstype=cpio rw"; stdout-path = "/pl011@9000000"; - linux,initrd-start = <0x45000000>; - linux,initrd-end = <0x4FA75E3D>; + linux,initrd-start = <0x49000000>; + linux,initrd-end = <0x54000000>; kaslr-seed = <0xfe5f4802 0xd861995f>; }; }; -- Gitee From ffede4f7822cf379d3b5b73c6298d567e39de195 Mon Sep 17 00:00:00 2001 From: Guoqi Xie Date: Tue, 3 Dec 2024 18:50:44 +0800 Subject: [PATCH 5/6] vcpu:release vcpu bit when delete vm When delete vm,vcpu set bit need to be removed. Signed-off-by: Guoqi Xie --- arch/arm64/core/mmu.c | 6 ++++- dts/arm64/qemu-virt/qemu-virt-max.dtsi | 4 +++- include/virtualization/vm_cpu.h | 5 ++++ include/virtualization/zvm.h | 25 ++++++++++++++++++++ samples/_zvm/boards/qemu_cortex_max_smp.conf | 4 ++-- subsys/virtualization/os/os_linux.c | 12 +++++----- subsys/virtualization/os/os_zephyr.c | 6 ++--- subsys/virtualization/vm.c | 17 +++++++------ subsys/virtualization/vm_cpu.c | 9 ++++--- subsys/virtualization/vm_manager.c | 8 +------ 10 files changed, 64 insertions(+), 32 deletions(-) diff --git a/arch/arm64/core/mmu.c b/arch/arm64/core/mmu.c index 693bed511..898222256 100644 --- a/arch/arm64/core/mmu.c +++ b/arch/arm64/core/mmu.c @@ -419,6 +419,8 @@ static int privatize_page_range(struct arm_mmu_ptables *dst_pt, return ret; } +static uint32_t miss_table = 0; + static void discard_table(uint64_t *table, unsigned int level) { unsigned int i; @@ -426,7 +428,9 @@ static void discard_table(uint64_t *table, unsigned int level) for (i = 0U; Ln_XLAT_NUM_ENTRIES; i++) { if (is_table_desc(table[i], level)) { table_usage(pte_desc_table(table[i]), -1); - discard_table(pte_desc_table(table[i]), level + 1); + if (miss_table > 0xFFFFFFFF) { + discard_table(pte_desc_table(table[i]), level + 1); + } } if (!is_free_desc(table[i])) { table[i] = 0U; diff --git a/dts/arm64/qemu-virt/qemu-virt-max.dtsi b/dts/arm64/qemu-virt/qemu-virt-max.dtsi index c0d3f7907..8e3f4aa20 100644 --- a/dts/arm64/qemu-virt/qemu-virt-max.dtsi +++ b/dts/arm64/qemu-virt/qemu-virt-max.dtsi @@ -86,11 +86,13 @@ #interrupt-cells = <4>; label = "GIC"; status = "okay"; + #address-cells = <2>; + #size-cells = <2>; gic_ist: its@8080000 { compatible = "arm,gic-v3-its"; phandle = <0x8006>; - reg = <0x00 0x8080000 0x20000>; + reg = <0x00 0x8080000 0x00 0x20000>; label = "GIC_ITS"; msi-controller; }; diff --git a/include/virtualization/vm_cpu.h b/include/virtualization/vm_cpu.h index c28c07960..1bc3706f0 100644 --- a/include/virtualization/vm_cpu.h +++ b/include/virtualization/vm_cpu.h @@ -65,4 +65,9 @@ void do_asm_vcpu_swap(struct k_thread *new_thread, struct k_thread *old_thread); */ int vcpu_ipi_scheduler(uint32_t cpu_mask, uint32_t timeout); +/** + * @brief reset the vm cpu to idle. + */ +void vm_cpu_reset(uint16_t cpu_id); + #endif /* ZEPHYR_INCLUDE_ZVM_VM_CPU_H_ */ diff --git a/include/virtualization/zvm.h b/include/virtualization/zvm.h index 709774caa..fe9ef2fb8 100644 --- a/include/virtualization/zvm.h +++ b/include/virtualization/zvm.h @@ -288,6 +288,31 @@ static ALWAYS_INLINE int get_static_idle_cpu(void) { return -ESRCH; } +static ALWAYS_INLINE void reset_idle_cpu(uint16_t cpu_id) { + k_spinlock_key_t key; + +#ifdef CONFIG_SMP + /* In SMP, _current is a field read from _current_cpu, which + * can race with preemption before it is read. We must lock + * local interrupts when reading it. + */ + unsigned int k = arch_irq_lock(); +#endif + k_tid_t tid = _kernel.cpus[cpu_id].current; +#ifdef CONFIG_SMP + arch_irq_unlock(k); +#endif + + int prio = k_thread_priority_get(tid); + + if (prio == K_IDLE_PRIO && (used_cpus & (1 << cpu_id))) { + key = k_spin_lock(&cpu_mask_lock); + used_cpus &= ~(1 << cpu_id); + k_spin_unlock(&cpu_mask_lock, key); + } + isb(); +} + static ALWAYS_INLINE bool is_vmid_full(void) { return zvm_overall_info->alloced_vmid == BIT_MASK(CONFIG_MAX_VM_NUM); diff --git a/samples/_zvm/boards/qemu_cortex_max_smp.conf b/samples/_zvm/boards/qemu_cortex_max_smp.conf index 98d9e91c4..c9855810d 100644 --- a/samples/_zvm/boards/qemu_cortex_max_smp.conf +++ b/samples/_zvm/boards/qemu_cortex_max_smp.conf @@ -33,8 +33,8 @@ CONFIG_ZVM_LINUX_MAX_XLAT_TABLES=10240 # enable vm device below: CONFIG_VM_VIRTIO_MMIO=y CONFIG_VM_VIRTIO_BLOCK=y -CONFIG_VM_SHMEM=y -CONFIG_VM_SHMEMRW=y +CONFIG_VM_SHMEM=n +CONFIG_VM_SHMEMRW=n CONFIG_MEM_INIT_PRIORITY=59 CONFIG_MEMRW_INIT_PRIORITY=60 diff --git a/subsys/virtualization/os/os_linux.c b/subsys/virtualization/os/os_linux.c index 2c75c60e1..26f36af56 100644 --- a/subsys/virtualization/os/os_linux.c +++ b/subsys/virtualization/os/os_linux.c @@ -47,9 +47,9 @@ int load_linux_image(struct vm_mem_domain *vmem_domain) uint64_t des_hpa = LINUX_VM_IMAGE_BASE; uint64_t per_size = 1048576; //1M - ZVM_LOG_INFO("1 image_num_m = %ld\n", num_m); - ZVM_LOG_INFO("1 image_src_hpa = 0x%lx\n", src_hpa); - ZVM_LOG_INFO("1 image_des_hpa = 0x%lx\n", des_hpa); + ZVM_LOG_INFO("1 image_num_m = %lld\n", num_m); + ZVM_LOG_INFO("1 image_src_hpa = 0x%llx\n", src_hpa); + ZVM_LOG_INFO("1 image_des_hpa = 0x%llx\n", des_hpa); while(num_m){ z_phys_map((uint8_t **)&src_hva, (uintptr_t)src_hpa, per_size, K_MEM_CACHE_NONE | K_MEM_PERM_RW); @@ -64,9 +64,9 @@ int load_linux_image(struct vm_mem_domain *vmem_domain) src_hpa = LINUX_VMRFS_BASE; des_hpa = LINUX_VMRFS_PHY_BASE; - ZVM_LOG_INFO("1 rf_num_m = %ld\n", num_m); - ZVM_LOG_INFO("1 rf_src_hpa = 0x%lx\n", src_hpa); - ZVM_LOG_INFO("1 rf_des_hpa = 0x%lx\n", des_hpa); + ZVM_LOG_INFO("1 rf_num_m = %lld\n", num_m); + ZVM_LOG_INFO("1 rf_src_hpa = 0x%llx\n", src_hpa); + ZVM_LOG_INFO("1 rf_des_hpa = 0x%llx\n", des_hpa); while(num_m){ z_phys_map((uint8_t **)&src_hva, (uintptr_t)src_hpa, per_size, K_MEM_CACHE_NONE | K_MEM_PERM_RW); diff --git a/subsys/virtualization/os/os_zephyr.c b/subsys/virtualization/os/os_zephyr.c index bbb3a5959..74b84f2b6 100644 --- a/subsys/virtualization/os/os_zephyr.c +++ b/subsys/virtualization/os/os_zephyr.c @@ -58,9 +58,9 @@ int load_zephyr_image(struct vm_mem_domain *vmem_domain) uint64_t des_hpa = ZEPHYR_VM_IMAGE_BASE; uint64_t per_size = 1048576; //1M - ZVM_LOG_INFO("1 image_num_m = %ld\n", num_m); - ZVM_LOG_INFO("1 image_src_hpa = 0x%lx\n", src_hpa); - ZVM_LOG_INFO("1 image_des_hpa = 0x%lx\n", des_hpa); + ZVM_LOG_INFO("1 image_num_m = %lld\n", num_m); + ZVM_LOG_INFO("1 image_src_hpa = 0x%llx\n", src_hpa); + ZVM_LOG_INFO("1 image_des_hpa = 0x%llx\n", des_hpa); while(num_m){ z_phys_map((uint8_t **)&src_hva, (uintptr_t)src_hpa, per_size, K_MEM_CACHE_NONE | K_MEM_PERM_RW); diff --git a/subsys/virtualization/vm.c b/subsys/virtualization/vm.c index 511ba1e31..4a0b7284b 100644 --- a/subsys/virtualization/vm.c +++ b/subsys/virtualization/vm.c @@ -457,7 +457,12 @@ int vm_delete(struct vm *vm) /* delete vcpu struct */ for(int i = 0; i < vm->vcpu_num; i++){ vcpu = vm->vcpus[i]; - if(!vcpu) continue; + if(!vcpu) { + continue; + } + /* release the used physical cpu*/ + vm_cpu_reset(vcpu->cpu); + vwork = vcpu->work; if(vwork){ k_free(vwork->vcpu_thread); @@ -542,14 +547,8 @@ int z_parse_info_vm_args(size_t argc, char **argv, struct getopt_state *state) int z_list_vms_info(uint16_t vmid) { - /* if vmid equal to CONFIG_MAX_VM_NUM, list all vm */ - if (vmid == CONFIG_MAX_VM_NUM) { - z_list_all_vms_info(); - } else { - printk("\n|*********************** VM INFO *********************|\n"); - printk("|***vmid name \t vcpus vmem(M)\tstatus ***|\n"); - z_list_vm_info(vmid); - } + /* list all vm */ + z_list_all_vms_info(); printk("|*****************************************************|\n"); return 0; } diff --git a/subsys/virtualization/vm_cpu.c b/subsys/virtualization/vm_cpu.c index 3950408be..ac71394d9 100644 --- a/subsys/virtualization/vm_cpu.c +++ b/subsys/virtualization/vm_cpu.c @@ -410,9 +410,7 @@ struct vcpu *vm_vcpu_init(struct vm *vm, uint16_t vcpu_id, char *vcpu_name) } /* Just work on 4 cores system now */ - if(++created_vm_num == CONFIG_MP_NUM_CPUS-1){ - pcpu_num = CONFIG_MP_NUM_CPUS-1; - } + created_vm_num++; k_thread_cpu_mask_enable(tid, pcpu_num); vcpu->cpu = pcpu_num; #else @@ -466,3 +464,8 @@ int vm_vcpu_reset(struct vcpu *vcpu) { return vcpu_state_switch(vcpu->work->vcpu_thread, _VCPU_STATE_RESET); } + +void vm_cpu_reset(uint16_t cpu_id) +{ + reset_idle_cpu(cpu_id); +} diff --git a/subsys/virtualization/vm_manager.c b/subsys/virtualization/vm_manager.c index fa21f40d8..0005b049f 100644 --- a/subsys/virtualization/vm_manager.c +++ b/subsys/virtualization/vm_manager.c @@ -227,16 +227,10 @@ int zvm_delete_guest(size_t argc, char **argv) int zvm_info_guest(size_t argc, char **argv) { - uint16_t vm_id; int ret = 0; - vm_id = z_parse_info_vm_args(argc, argv, state); - if (!(BIT(vm_id) & zvm_overall_info->alloced_vmid) && vm_id != CONFIG_MAX_VM_NUM) { - ZVM_LOG_WARN("This vm is not exist!\n Please input zvm info to list vms! \n"); - return -ENODEV; - } if (zvm_overall_info->vm_total_num > 0) { - ret = z_list_vms_info(vm_id); + ret = z_list_vms_info(0); }else{ ret = -ENODEV; } -- Gitee From 7b004a14039c413f8f57536ad4b03c69fdbd3db2 Mon Sep 17 00:00:00 2001 From: Guoqi Xie Date: Tue, 3 Dec 2024 19:01:19 +0800 Subject: [PATCH 6/6] Doc: update readme Update readme file. Signed-off-by: Guoqi Xie --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1890c783b..8e23a0f97 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ ZVM作为Zephyr实时操作系统生态在国内的关键一环，致力于构 研发团队依托 [嵌入式与网络计算湖南省重点实验室](http://esnl.hnu.edu.cn/index.htm) 及 [车网智能融合技术研创中心](http://cyy.hnu.edu.cn/yjly1/cwznrhjs1.htm) 等平台，核心成员如下： -**谢国琪（ZVM项目负责人）**，邮箱：xgqman@hnu.edu.cn, [个人主页](http://csee.hnu.edu.cn/people/xieguoqi) +**谢国琪（ZVM项目创始人）**，邮箱：xgqman@hnu.edu.cn, [个人主页](http://csee.hnu.edu.cn/people/xieguoqi) **熊程来（openEuler SIG-Zephyr maintainer）**，邮箱：xiongcl@hnu.edu.cn @@ -129,14 +129,18 @@ ZVM作为Zephyr实时操作系统生态在国内的关键一环，致力于构 **王中甲**，邮箱：zjwang@hnu.edu.cn -**韦东升**，邮箱：weidongsheng@hnu.edu.cn - **赵思蓉**，邮箱：zhaosr@hnu.edu.cn **温浩**，邮箱：wenhao@stu.ynu.edu.cn **胡宇昊**，邮箱：ahui@hun.edu.cn +**王清桥**，邮箱：qingqiaowang@hnu.edu.cn + +**黄鹤**，邮箱：huanghe@phytium.com.cn + +**任慰**，邮箱：dfrd-renw@dfmc.com.cn + 欢迎大家反馈开发中遇到的问题，可以联系上面邮箱或者加入技术交流群。 ## 版权与许可证 -- Gitee