From aba3be1a420577d751418678bed61d0c2740f11a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E9=87=91=E7=94=9F?= Date: Fri, 21 Jul 2023 17:26:10 +0800 Subject: [PATCH] target/arm: enable kvm instance to live migrate on arm Enable vm start on kvm with custom arm cpu. Check kvm-no-adjvtime before add kvm properties, Set cortex-aXX kvm_target=host.kvm_target when kvm is enabled. Set extra regs off on S2500 and Ft-2000-plus by default. Custom cpu is forbidden if kvm not support setting features. ReadOnly reg is ignore when error to set. This enables kvm instance which started with Tengyun-S2500 or FT2000+ to live migrate within Kunpeng920, Tengyun-S2500 and FT2000+. Signed-off-by: jx8zjs --- target/arm/cpu.c | 4 +- target/arm/cpu.h | 2 + target/arm/cpu64.c | 112 ++++++++++++++++++++++++++++++++++--------- target/arm/helper.c | 25 ++++++++++ target/arm/kvm.c | 77 +++++++++++++++++++++++------ target/arm/kvm_arm.h | 19 ++++++++ 6 files changed, 201 insertions(+), 38 deletions(-) diff --git a/target/arm/cpu.c b/target/arm/cpu.c index d550022f18..3dd8e432ac 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -1635,8 +1635,8 @@ static void arm_cpu_set_bit_prop(Object *obj, Visitor *v, const char *name, bool value; if (!kvm_arm_cpu_feature_supported()) { - warn_report("KVM doesn't support to set CPU feature in arm. " - "Setting to `%s` is ignored.", name); + error_setg(errp, "KVM doesn't support to set CPU feature in arm. " + "Setting `%s` feature error.", name); return; } if (dev->realized) { diff --git a/target/arm/cpu.h b/target/arm/cpu.h index eb804dffaa..b665b9dfc0 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -3012,6 +3012,8 @@ bool write_list_to_cpustate(ARMCPU *cpu); */ bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync); +bool is_readonly_reg(const ARMCPRegInfo *ri); + #define ARM_CPUID_TI915T 0x54029152 #define ARM_CPUID_TI925T 0x54029252 diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index 3ec788fc29..541cd41880 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -91,6 +91,19 @@ static const ARMCPRegInfo cortex_a72_a57_a53_cp_reginfo[] = { REGINFO_SENTINEL }; +static void check_kvm_feature_support_setting(ARMCPU *cpu) +{ + if (kvm_enabled()) { + if (!kvm_arm_cpu_feature_supported()) { + error_setg(&error_abort, "KVM doesn't support to set CPU feature" + " in arm. Error setting custom cpu with" + " kvm."); + return; + } + kvm_arm_set_cpu_kvm_target_from_host(cpu); + } +} + static void aarch64_a57_initfn(Object *obj) { ARMCPU *cpu = ARM_CPU(obj); @@ -105,6 +118,7 @@ static void aarch64_a57_initfn(Object *obj) set_feature(&cpu->env, ARM_FEATURE_EL3); set_feature(&cpu->env, ARM_FEATURE_PMU); cpu->kvm_target = QEMU_KVM_ARM_TARGET_CORTEX_A57; + check_kvm_feature_support_setting(cpu); cpu->midr = 0x411fd070; cpu->revidr = 0x00000000; cpu->reset_fpsid = 0x41034070; @@ -158,6 +172,7 @@ static void aarch64_a53_initfn(Object *obj) set_feature(&cpu->env, ARM_FEATURE_EL3); set_feature(&cpu->env, ARM_FEATURE_PMU); cpu->kvm_target = QEMU_KVM_ARM_TARGET_CORTEX_A53; + check_kvm_feature_support_setting(cpu); cpu->midr = 0x410fd034; cpu->revidr = 0x00000000; cpu->reset_fpsid = 0x41034070; @@ -203,6 +218,7 @@ static void aarch64_a72_initfn(Object *obj) cpu->dtb_compatible = "arm,cortex-a72"; cpu->kvm_target = QEMU_KVM_ARM_TARGET_GENERIC_V8; + check_kvm_feature_support_setting(cpu); set_feature(&cpu->env, ARM_FEATURE_V8); set_feature(&cpu->env, ARM_FEATURE_NEON); set_feature(&cpu->env, ARM_FEATURE_GENERIC_TIMER); @@ -702,26 +718,77 @@ static void aarch64_max_ft2000plus_initfn(Object *obj) { ARMCPU *cpu = ARM_CPU(obj); - if (kvm_enabled()) { - kvm_arm_set_cpu_features_from_host(cpu); - kvm_arm_add_vcpu_properties(obj); - } else { - aarch64_a72_initfn(obj); - cpu->midr = 0x70186622; - } + aarch64_a72_initfn(obj); + + cpu->midr = 0x70186622; + + /* + * Disable atomics asimddp asimdfhm asimdrdm fphp + * jscvt fcma dcpop asimdhp aes pmull sha1 sha2 crc32 + */ + uint64_t t; + t = cpu->isar.regs[ID_AA64ISAR0]; + t = FIELD_DP64(t, ID_AA64ISAR0, ATOMIC, 0); + t = FIELD_DP64(t, ID_AA64ISAR0, DP, 0); + t = FIELD_DP64(t, ID_AA64ISAR0, FHM, 0); + t = FIELD_DP64(t, ID_AA64ISAR0, RDM, 0); + cpu->isar.regs[ID_AA64ISAR0] = t; + + t = cpu->isar.regs[ID_AA64PFR0]; + t = FIELD_DP64(t, ID_AA64PFR0, FP, 0); + cpu->isar.regs[ID_AA64PFR0] = t; + + t = cpu->isar.regs[ID_AA64ISAR1]; + t = FIELD_DP64(t, ID_AA64ISAR1, JSCVT, 0); + t = FIELD_DP64(t, ID_AA64ISAR1, FCMA, 0); + t = FIELD_DP64(t, ID_AA64ISAR1, DPB, 0); + cpu->isar.regs[ID_AA64ISAR1] = t; + + t = cpu->isar.regs[ID_AA64PFR0]; + t = FIELD_DP64(t, ID_AA64PFR0, ADVSIMD, 0); + cpu->isar.regs[ID_AA64PFR0] = t; + + t = FIELD_DP64(t, ID_AA64ISAR0, AES, 0); + t = FIELD_DP64(t, ID_AA64ISAR0, SHA1, 0); + t = FIELD_DP64(t, ID_AA64ISAR0, SHA2, 0); + t = FIELD_DP64(t, ID_AA64ISAR0, CRC32, 1); + + cpu->isar.regs[ID_AA64ISAR0] = t; } static void aarch64_max_tengyun_s2500_initfn(Object *obj) { ARMCPU *cpu = ARM_CPU(obj); - if (kvm_enabled()) { - kvm_arm_set_cpu_features_from_host(cpu); - kvm_arm_add_vcpu_properties(obj); - } else { - aarch64_a72_initfn(obj); - cpu->midr = 0x70186632; - } + aarch64_a72_initfn(obj); + + cpu->midr = 0x70186632; + + /* + * Disable atomics asimddp asimdfhm asimdrdm fphp + * jscvt fcma dcpop asimdhp + */ + uint64_t t; + t = cpu->isar.regs[ID_AA64ISAR0]; + t = FIELD_DP64(t, ID_AA64ISAR0, ATOMIC, 0); + t = FIELD_DP64(t, ID_AA64ISAR0, DP, 0); + t = FIELD_DP64(t, ID_AA64ISAR0, FHM, 0); + t = FIELD_DP64(t, ID_AA64ISAR0, RDM, 0); + cpu->isar.regs[ID_AA64ISAR0] = t; + + t = cpu->isar.regs[ID_AA64PFR0]; + t = FIELD_DP64(t, ID_AA64PFR0, FP, 0); + cpu->isar.regs[ID_AA64PFR0] = t; + + t = cpu->isar.regs[ID_AA64ISAR1]; + t = FIELD_DP64(t, ID_AA64ISAR1, JSCVT, 0); + t = FIELD_DP64(t, ID_AA64ISAR1, FCMA, 0); + t = FIELD_DP64(t, ID_AA64ISAR1, DPB, 0); + cpu->isar.regs[ID_AA64ISAR1] = t; + + t = cpu->isar.regs[ID_AA64PFR0]; + t = FIELD_DP64(t, ID_AA64PFR0, ADVSIMD, 0); + cpu->isar.regs[ID_AA64PFR0] = t; } /* -cpu max: if KVM is enabled, like -cpu host (best possible with this host); @@ -922,6 +989,7 @@ static void aarch64_a64fx_initfn(Object *obj) set_feature(&cpu->env, ARM_FEATURE_EL2); set_feature(&cpu->env, ARM_FEATURE_EL3); set_feature(&cpu->env, ARM_FEATURE_PMU); + check_kvm_feature_support_setting(cpu); cpu->midr = 0x461f0010; cpu->revidr = 0x00000000; cpu->ctr = 0x86668006; @@ -958,14 +1026,14 @@ static void aarch64_a64fx_initfn(Object *obj) } static const ARMCPUInfo aarch64_cpus[] = { - { .name = "cortex-a57", .initfn = aarch64_a57_initfn }, - { .name = "cortex-a53", .initfn = aarch64_a53_initfn }, - { .name = "cortex-a72", .initfn = aarch64_a72_initfn }, - { .name = "Kunpeng-920", .initfn = aarch64_kunpeng_920_initfn}, - { .name = "FT-2000+", .initfn = aarch64_max_ft2000plus_initfn }, - { .name = "Tengyun-S2500", .initfn = aarch64_max_tengyun_s2500_initfn }, - { .name = "a64fx", .initfn = aarch64_a64fx_initfn }, - { .name = "max", .initfn = aarch64_max_initfn }, + { .name = "cortex-a57", .initfn = aarch64_a57_initfn }, + { .name = "cortex-a53", .initfn = aarch64_a53_initfn }, + { .name = "cortex-a72", .initfn = aarch64_a72_initfn }, + { .name = "Kunpeng-920", .initfn = aarch64_kunpeng_920_initfn}, + { .name = "FT-2000+", .initfn = aarch64_max_ft2000plus_initfn }, + { .name = "Tengyun-S2500", .initfn = aarch64_max_tengyun_s2500_initfn }, + { .name = "a64fx", .initfn = aarch64_a64fx_initfn }, + { .name = "max", .initfn = aarch64_max_initfn }, }; static bool aarch64_cpu_get_aarch64(Object *obj, Error **errp) diff --git a/target/arm/helper.c b/target/arm/helper.c index 1854c65863..9712e8d153 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -144,6 +144,31 @@ static bool is_id_reg(const ARMCPRegInfo *ri) ri->crm > 0 && ri->crm < 8; } +bool is_readonly_reg(const ARMCPRegInfo *ri) +{ + if (ri->opc0 == 3) { + if (ri->opc1 == 0 && ri->crn == 0 && ri->crm == 0 && + (ri->opc2 == 0 || ri->opc2 == 5 || + ri->opc2 == 6)) { + return true; + } else if (ri->opc1 == 0 && ri->crn == 12 && ri->crm == 1 && + ri->opc2 == 0) { + return true; + } else if (ri->opc1 == 1 && ri->crn == 0 && ri->crm == 0 && + (ri->opc2 == 0 || ri->opc2 == 1 || + ri->opc2 == 7)) { + return true; + } else if (ri->opc1 == 3 && ri->crn == 0 && ri->crm == 0 && + (ri->opc2 == 1 || ri->opc2 == 7)) { + return true; + } else if (ri->opc1 == 6 && ri->crn == 12 && ri->crm == 0 && + ri->opc2 == 1) { + return true; + } + } + return false; +} + bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync) { /* Write the coprocessor state from cpu->env to the (index,value) list. */ diff --git a/target/arm/kvm.c b/target/arm/kvm.c index 22ac5bcb97..d510b31d6a 100644 --- a/target/arm/kvm.c +++ b/target/arm/kvm.c @@ -165,6 +165,19 @@ void kvm_arm_destroy_scratch_host_vcpu(int *fdarray) } } +void kvm_arm_set_cpu_kvm_target_from_host(ARMCPU *cpu) +{ + if (!arm_host_cpu_features.dtb_compatible) { + if (!kvm_enabled() || + !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) { + cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE; + cpu->host_cpu_probe_failed = true; + return; + } + } + cpu->kvm_target = arm_host_cpu_features.target; +} + void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu) { CPUARMState *env = &cpu->env; @@ -215,19 +228,23 @@ void kvm_arm_add_vcpu_properties(Object *obj) if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) { cpu->kvm_adjvtime = true; - object_property_add_bool(obj, "kvm-no-adjvtime", kvm_no_adjvtime_get, - kvm_no_adjvtime_set); - object_property_set_description(obj, "kvm-no-adjvtime", - "Set on to disable the adjustment of " - "the virtual counter. VM stopped time " - "will be counted."); - } - - cpu->kvm_steal_time = ON_OFF_AUTO_AUTO; - object_property_add_bool(obj, "kvm-steal-time", kvm_steal_time_get, - kvm_steal_time_set); - object_property_set_description(obj, "kvm-steal-time", - "Set off to disable KVM steal time."); + if (object_property_find(obj, "kvm-no-adjvtime") == NULL) { + object_property_add_bool(obj, "kvm-no-adjvtime", + kvm_no_adjvtime_get, + kvm_no_adjvtime_set); + object_property_set_description(obj, "kvm-no-adjvtime", + "Set on to disable the adjustment " + "of the virtual counter. VM " + "stopped time will be counted."); + } + } + if (object_property_find(obj, "kvm-steal-time") == NULL) { + cpu->kvm_steal_time = ON_OFF_AUTO_AUTO; + object_property_add_bool(obj, "kvm-steal-time", kvm_steal_time_get, + kvm_steal_time_set); + object_property_set_description(obj, "kvm-steal-time", + "Set off to disable KVM steal time."); + } } bool kvm_arm_pmu_supported(void) @@ -622,7 +639,39 @@ bool write_list_to_kvmstate(ARMCPU *cpu, int level) * "you tried to set a register which is constant with * a different value from what it actually contains". */ - ok = false; + const ARMCPRegInfo *ri; + uint32_t cpreg_idx = kvm_to_cpreg_id(regidx); + ri = get_arm_cp_reginfo(cpu->cp_regs, cpreg_idx); + if (ri != NULL) { + if (is_readonly_reg(ri)) { + fprintf(stderr, "%s: failed to set register:%s (%lx) " + "to (%llx) %d/%s, since reg is readonly\n", + __func__, ri->name, regidx, r.addr, ret, + strerror(-ret)); + continue; + } else { + fprintf(stderr, "%s: failed to set register:%s (%lx) " + "to (%llx) %d/%s, which is a RW register " + "but is unknown on this host cpu\n", + __func__, ri->name, regidx, r.addr, ret, + strerror(-ret)); + ok = false; + } + } else { + /* TODO Check these regs whether can be skipped */ + if ((regidx >= 0x6020000000110000 && + regidx <= 0x6020000000110002) + || regidx == 0x6030000000140001 + || regidx == 0x6030000000140002) { + fprintf(stderr, "skip 0x%lx : %u %xu\n", regidx, + cpreg_idx, cpreg_idx); + continue; + } + fprintf(stderr, "ri is NULL, and ri is not valid, " + "regidx=(%lx)\n", regidx); + ok = false; + break; + } } } return ok; diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h index 8b644b3924..18e3e190b9 100644 --- a/target/arm/kvm_arm.h +++ b/target/arm/kvm_arm.h @@ -247,6 +247,15 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf); */ void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map); +/** + * kvm_arm_set_cpu_kvm_target_from_host: + * @cpu: ARMCPU to set the features for + * + * Set up the ARMCPU struct kvm_target to match the information probed + * from the host CPU. + */ +void kvm_arm_set_cpu_kvm_target_from_host(ARMCPU *cpu); + /** * kvm_arm_set_cpu_features_from_host: * @cpu: ARMCPU to set the features for @@ -411,6 +420,11 @@ static inline bool kvm_arm_cpu_feature_supported(void) return false; } +static inline bool kvm_arm_cpu_feature_supported(void) +{ + return false; +} + /* * These functions should never actually be called without KVM support. */ @@ -419,6 +433,11 @@ static inline void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu) g_assert_not_reached(); } +static inline void kvm_arm_set_cpu_kvm_target_from_host(ARMCPU *cpu) +{ + g_assert_not_reached(); +} + static inline void kvm_arm_add_vcpu_properties(Object *obj) { g_assert_not_reached(); -- Gitee