diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index f509501323e749630c21e2fdc3601246087cd8d9..033251f8f716d8c6cd1b544c3874c0b2e4fbf992 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -663,7 +663,8 @@ static __always_inline bool system_uses_irq_prio_masking(void)
 static __always_inline bool system_uses_nmi(void)
 {
 	return IS_ENABLED(CONFIG_ARM64_NMI) &&
-		cpus_have_const_cap(ARM64_USES_NMI);
+		cpus_have_const_cap(ARM64_USES_NMI) &&
+		!system_uses_irq_prio_masking();
 }
 
 static inline bool system_has_prio_mask_debugging(void)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 01886b83d1206afac90f61bb89b96cc719ecc1f4..209ac1b5e1b37e2f7d653a9d8c4f557a46276f68 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -73,6 +73,8 @@ struct kvm_arch {
 	/* VTCR_EL2 value for this VM */
 	u64    vtcr;
 
+	u8 pfr1_nmi;
+
 	/* The last vcpu id that ran on each physical CPU */
 	int __percpu *last_vcpu_ran;
 
@@ -84,6 +86,19 @@ struct kvm_arch {
 
 	/* Mandated version of PSCI */
 	u32 psci_version;
+
+	/*
+	* Emulated CPU ID registers per VM
+	* (Op0, Op1, CRn, CRm, Op2) of the ID registers to be saved in it
+	* is (3, 0, 0, crm, op2), where 1<=crm<8, 0<=op2<8.
+	*
+	* These emulated idregs are VM-wide, but accessed from the context of a vCPU.
+	* Atomic access to multiple idregs are guarded by kvm_arch.config_lock.
+	*/
+	#define IDREG_IDX(id)		(((sys_reg_CRm(id) - 1) << 3) | sys_reg_Op2(id))
+	#define IDREG(kvm, id)		((kvm)->arch.id_regs[IDREG_IDX(id)])
+	#define KVM_ARM_ID_REG_NUM	(IDREG_IDX(sys_reg(3, 0, 0, 7, 7)) + 1)
+		u64 id_regs[KVM_ARM_ID_REG_NUM];
 };
 
 #define KVM_NR_MEM_OBJS     40
@@ -249,10 +264,13 @@ struct kvm_vcpu_arch {
 	void *sve_state;
 	unsigned int sve_max_vl;
 
-	/* HYP configuration */
+	/* Values of trap registers for the guest. */
 	u64 hcr_el2;
 	u32 mdcr_el2;
 
+	/* Values of trap registers for the host before guest entry. */
+	u64 mdcr_el2_host;
+
 	/* Exception Information */
 	struct kvm_vcpu_fault_info fault;
 
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 66d4c5e35932b7a4d916529e04e14c0b2632660b..6fdf37cf5209ea2560b770ae34bf4419956935e2 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -79,7 +79,7 @@ void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
 void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
 
 void activate_traps_vhe_load(struct kvm_vcpu *vcpu);
-void deactivate_traps_vhe_put(void);
+void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu);
 
 u64 __guest_enter(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host_ctxt);
 void __noreturn __hyp_do_panic(unsigned long, ...);
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 761b26417a5d3151201047f0931ac45cddcc0976..13ebfb31ae2229df5c22acc066bd2b6a63b6c3d4 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -204,6 +204,8 @@
 
 #define SYS_ALLINT			sys_reg(3, 0, 4, 3, 0)
 
+#define SYS_ALLINT			sys_reg(3, 0, 4, 3, 0)
+
 #define SYS_ICC_PMR_EL1			sys_reg(3, 0, 4, 6, 0)
 #define SYS_ICC_NMIAR1_EL1		sys_reg(3, 0, 12, 9, 5)
 
@@ -923,5 +925,6 @@
 } while (0)
 
 #endif
+#define ICH_LR_NMI		(1ULL << 59)
 
 #endif	/* __ASM_SYSREG_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 753d6a6e96f31dacaaa3e08883c32466e89950d2..188ef18393904196635124cb99b8fa350c4af545 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1295,20 +1295,24 @@ static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry,
 }
 #endif
 
-#ifdef CONFIG_ARM64_NMI
 static bool use_nmi(const struct arm64_cpu_capabilities *entry, int scope)
 {
 	if (!has_cpuid_feature(entry, scope))
 		return false;
 
 	/*
+	 * NMI support was not enabled in the kernel, but can still be
+	 * used by guests. Let the world know.
+	 *
 	 * Having both real and pseudo NMIs enabled simultaneously is
 	 * likely to cause confusion.  Since pseudo NMIs must be
 	 * enabled with an explicit command line option, if the user
 	 * has set that option on a system with real NMIs for some
 	 * reason assume they know what they're doing.
 	 */
-	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && enable_pseudo_nmi) {
+	if (!IS_ENABLED(CONFIG_ARM64_NMI))
+		pr_info("CONFIG_ARM64_NMI disabled, using NMIs for guests only\n");
+	else if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && enable_pseudo_nmi) {
 		pr_info("Pseudo NMI enabled, not using architected NMI\n");
 		return false;
 	}
@@ -1316,6 +1320,7 @@ static bool use_nmi(const struct arm64_cpu_capabilities *entry, int scope)
 	return true;
 }
 
+#ifdef CONFIG_ARM64_NMI
 static void nmi_enable(const struct arm64_cpu_capabilities *__unused)
 {
 	/*
@@ -1710,7 +1715,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.field_pos = ID_AA64PFR1_NMI_SHIFT,
 		.min_field_value = ID_AA64PFR1_NMI_IMP_DEF,
 		.matches = use_nmi,
+#ifdef CONFIG_ARM64_NMI
 		.cpu_enable = nmi_enable,
+#endif
 	},
 #endif
 	{
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 624e5c83a4972bae0b993cd60ee3c753eb575c25..fac08802c432ea0ff32fd947edd4e288ef45481f 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -88,15 +88,20 @@ static void __hyp_text __activate_traps_common(struct kvm_vcpu *vcpu)
 	write_sysreg(0, pmselr_el0);
 	write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
 
-	if (cpus_have_final_cap(ARM64_HAS_NMI))
+	if (cpus_have_final_cap(ARM64_HAS_NMI) &&
+		!kern_hyp_va(vcpu->kvm)->arch.pfr1_nmi)
 		sysreg_clear_set_s(SYS_HCRX_EL2, 0, HCRX_EL2_TALLINT);
 
+	vcpu->arch.mdcr_el2_host = read_sysreg(mdcr_el2);
 	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
 }
 
-static void __hyp_text __deactivate_traps_common(void)
+static void __hyp_text __deactivate_traps_common(struct kvm_vcpu *vcpu)
 {
-	if (cpus_have_final_cap(ARM64_HAS_NMI))
+	write_sysreg(vcpu->arch.mdcr_el2_host, mdcr_el2);
+
+	if (cpus_have_final_cap(ARM64_HAS_NMI) &&
+	    !kern_hyp_va(vcpu->kvm)->arch.pfr1_nmi)
 		sysreg_clear_set_s(SYS_HCRX_EL2, HCRX_EL2_TALLINT, 0);
 
 	write_sysreg(0, hstr_el2);
@@ -178,16 +183,13 @@ static void deactivate_traps_vhe(void)
 }
 NOKPROBE_SYMBOL(deactivate_traps_vhe);
 
-static void __hyp_text __deactivate_traps_nvhe(void)
+static void __hyp_text __deactivate_traps_nvhe(struct kvm_vcpu *vcpu)
 {
-	u64 mdcr_el2 = read_sysreg(mdcr_el2);
-
-	__deactivate_traps_common();
+	vcpu->arch.mdcr_el2_host &= MDCR_EL2_HPMN_MASK |
+				    MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
 
-	mdcr_el2 &= MDCR_EL2_HPMN_MASK;
-	mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
+	__deactivate_traps_common(vcpu);
 
-	write_sysreg(mdcr_el2, mdcr_el2);
 	write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2);
 	write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
 }
@@ -208,7 +210,7 @@ static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
 	if (has_vhe())
 		deactivate_traps_vhe();
 	else
-		__deactivate_traps_nvhe();
+		__deactivate_traps_nvhe(vcpu);
 }
 
 void activate_traps_vhe_load(struct kvm_vcpu *vcpu)
@@ -216,17 +218,13 @@ void activate_traps_vhe_load(struct kvm_vcpu *vcpu)
 	__activate_traps_common(vcpu);
 }
 
-void deactivate_traps_vhe_put(void)
+void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu)
 {
-	u64 mdcr_el2 = read_sysreg(mdcr_el2);
-
-	mdcr_el2 &= MDCR_EL2_HPMN_MASK |
-		    MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT |
-		    MDCR_EL2_TPMS;
-
-	write_sysreg(mdcr_el2, mdcr_el2);
+	vcpu->arch.mdcr_el2_host &= MDCR_EL2_HPMN_MASK |
+				MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT |
+				MDCR_EL2_TPMS;
 
-	__deactivate_traps_common();
+	__deactivate_traps_common(vcpu);
 }
 
 static void __hyp_text __activate_vm(struct kvm *kvm)
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index 7ddbc849b58001fe8ed40ff0cd23d2a20c8664fe..ad83adbf4172eb9fdf566974e5c6c76221ab47d0 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -287,7 +287,7 @@ void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu)
 	if (!has_vhe())
 		return;
 
-	deactivate_traps_vhe_put();
+	deactivate_traps_vhe_put(vcpu);
 
 	__sysreg_save_el1_state(guest_ctxt);
 	__sysreg_save_user_state(guest_ctxt);
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 4fad598eb3dd580664b26a42f2c377d5c72b9415..8a8647bb564f41f409f5de1f7633ef619ed02177 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -1013,6 +1013,14 @@ static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
 	{ SYS_DESC(SYS_PMEVTYPERn_EL0(n)),					\
 	  access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }
 
+static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
+	const struct sys_reg_desc *r)
+{
+	kvm_inject_undefined(vcpu);
+
+	return false;
+}
+
 static bool trap_ptrauth(struct kvm_vcpu *vcpu,
 			 struct sys_reg_params *p,
 			 const struct sys_reg_desc *rd)
@@ -1097,11 +1105,18 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu,
 			 (0xfUL << ID_AA64ISAR1_GPI_SHIFT));
 	} else if (id == SYS_ID_AA64PFR1_EL1) {
 		val &= ~ID_AA64PFR1_NMI_MASK;
+		val |= FIELD_PREP(ID_AA64PFR1_NMI_MASK,
+				vcpu->kvm->arch.pfr1_nmi);
 	}
 
 	return val;
 }
 
+void read_id_reg_reset(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
+{
+	read_id_reg(vcpu, r, false);
+}
+
 /* cpufeature ID register access trap handlers */
 
 static bool __access_id_reg(struct kvm_vcpu *vcpu,
@@ -1229,6 +1244,36 @@ static int __set_id_reg(const struct kvm_vcpu *vcpu,
 	return 0;
 }
 
+static int set_id_aa64pfr1_el1(struct kvm_vcpu *vcpu,
+	const struct sys_reg_desc *rd,
+	const struct kvm_one_reg *reg, void __user *uaddr)
+{
+	const u64 id = sys_reg_to_index(rd);
+	int err;
+	u64 val;
+	u8 nmi;
+	u64 r;
+
+	err = reg_from_user(&val, uaddr, id);
+	if (err)
+		return err;
+
+	nmi = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR1_NMI_SHIFT);
+	if (nmi > ID_AA64PFR1_NMI_IMP_DEF || (nmi && !system_uses_nmi()))
+		return -EINVAL;
+
+	/* We can only differ with NMI, and anything else is an error */
+	r = reg_to_encoding(rd);
+	val ^= IDREG(vcpu->kvm, r);
+	val &= ~ID_AA64PFR1_NMI_MASK;
+	if (val)
+		return -EINVAL;
+
+	vcpu->kvm->arch.pfr1_nmi = nmi;
+
+	return 0;
+}
+
 static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 		      const struct kvm_one_reg *reg, void __user *uaddr)
 {
@@ -1361,6 +1406,8 @@ static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
  * more demanding guest...
  */
 static const struct sys_reg_desc sys_reg_descs[] = {
+	{ SYS_DESC(SYS_ALLINT_CLR), undef_access },
+	{ SYS_DESC(SYS_ALLINT_SET), undef_access },
 	{ SYS_DESC(SYS_DC_ISW), access_dcsw },
 	{ SYS_DESC(SYS_DC_CSW), access_dcsw },
 	{ SYS_DESC(SYS_DC_CISW), access_dcsw },
@@ -1441,7 +1488,9 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	/* AArch64 ID registers */
 	/* CRm=4 */
 	ID_SANITISED(ID_AA64PFR0_EL1),
-	ID_SANITISED(ID_AA64PFR1_EL1),
+	{ SYS_DESC(SYS_ID_AA64PFR1_EL1), .access = access_id_reg,
+		.get_user = get_id_reg, .set_user = set_id_aa64pfr1_el1,
+		.reset = read_id_reg_reset, },
 	ID_UNALLOCATED(4,2),
 	ID_UNALLOCATED(4,3),
 	{ SYS_DESC(SYS_ID_AA64ZFR0_EL1), access_id_aa64zfr0_el1, .get_user = get_id_aa64zfr0_el1, .set_user = set_id_aa64zfr0_el1, },
@@ -1492,6 +1541,8 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	PTRAUTH_KEY(APDB),
 	PTRAUTH_KEY(APGA),
 
+	{ SYS_DESC(SYS_ALLINT), undef_access },
+
 	{ SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
 	{ SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
 	{ SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
@@ -1526,6 +1577,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	{ SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
 	{ SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
 	{ SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
+	{ SYS_DESC(SYS_ICC_NMIAR1_EL1), undef_access },
 	{ SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
 	{ SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
 	{ SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
diff --git a/arch/arm64/kvm/vgic-sys-reg-v3.c b/arch/arm64/kvm/vgic-sys-reg-v3.c
index e7d1ea92095ddd796354e3536577604a2f8851b4..59fedc2488cea0278294ee914734d21df704ed6a 100644
--- a/arch/arm64/kvm/vgic-sys-reg-v3.c
+++ b/arch/arm64/kvm/vgic-sys-reg-v3.c
@@ -184,17 +184,18 @@ static void vgic_v3_access_apr_reg(struct kvm_vcpu *vcpu,
 				   struct sys_reg_params *p, u8 apr, u8 idx)
 {
 	struct vgic_v3_cpu_if *vgicv3 = &vcpu->arch.vgic_cpu.vgic_v3;
-	uint32_t *ap_reg;
 
-	if (apr)
-		ap_reg = &vgicv3->vgic_ap1r[idx];
-	else
-		ap_reg = &vgicv3->vgic_ap0r[idx];
-
-	if (p->is_write)
-		*ap_reg = p->regval;
-	else
-		p->regval = *ap_reg;
+	if (apr) {
+		if (p->is_write)
+			vgicv3->vgic_ap1r[idx] = p->regval;
+		else
+			p->regval = vgicv3->vgic_ap1r[idx];
+	} else {
+		if (p->is_write)
+			vgicv3->vgic_ap0r[idx] = p->regval;
+		else
+			p->regval = vgicv3->vgic_ap0r[idx];
+	}
 }
 
 static bool access_gic_aprn(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 9e421c7f1c074060827e09d4c3a9e1253bc13591..c57925533c8a03bc3a444e860327c68739ae1f42 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -672,6 +672,7 @@ struct its_cmd_desc {
 			u8 sgi;
 			u8 priority;
 			bool enable;
+			bool nmi;
 			bool group;
 			bool clear;
 		} its_vsgi_cmd;
@@ -842,6 +843,11 @@ static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
 	its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
 }
 
+static void its_encode_sgi_nmi(struct its_cmd_block *cmd, bool nmi)
+{
+	its_mask_encode(&cmd->raw_cmd[0], nmi, 11, 11);
+}
+
 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
 {
 	its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
@@ -1247,6 +1253,7 @@ static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
 	its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
 #endif
 	its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
+	its_encode_sgi_nmi(cmd, desc->its_vsgi_cmd.nmi);
 	its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
 	its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
 	its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
@@ -4521,6 +4528,7 @@ static void its_configure_sgi(struct irq_data *d, bool clear)
 	desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
 	desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
 	desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
+	desc.its_vsgi_cmd.nmi = vpe->sgi_config[d->hwirq].nmi;
 	desc.its_vsgi_cmd.clear = clear;
 
 	/*
@@ -4679,6 +4687,7 @@ static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
 	case PROP_UPDATE_VSGI:
 		vpe->sgi_config[d->hwirq].priority = info->priority;
 		vpe->sgi_config[d->hwirq].group = info->group;
+		vpe->sgi_config[d->hwirq].nmi = info->nmi;
 		its_configure_sgi(d, false);
 		return 0;
 
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index e83b24e2fad84386b2c02533c62d993794084d3d..bc8dcdda7898d99a08cb84b26464fbbe2da080ed 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -110,11 +110,21 @@ static inline bool has_v3_3_nmi(void)
 {
 	return gic_data.has_nmi && system_uses_nmi();
 }
+
+static bool system_is_nmi_capable(void)
+{
+	return gic_data.has_nmi && cpus_have_const_cap(ARM64_HAS_NMI);
+}
 #else
 static inline bool has_v3_3_nmi(void)
 {
 	return false;
 }
+
+static bool system_is_nmi_capable(void)
+{
+	return false;
+}
 #endif
 
 #ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS
@@ -1967,6 +1977,7 @@ static void __init gic_of_setup_kvm_info(struct device_node *node)
 
 	gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
 	gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
+	gic_v3_kvm_info.has_nmi = system_is_nmi_capable();
 #ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS
        gic_v3_kvm_info.has_vtimer = gic_data.rdists.has_vtimer;
 #endif
@@ -2286,9 +2297,10 @@ static void __init gic_acpi_setup_kvm_info(void)
 
 	gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
 	gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
+	gic_v3_kvm_info.has_nmi = system_is_nmi_capable();
 #ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS
        gic_v3_kvm_info.has_vtimer = gic_data.rdists.has_vtimer;
-#endif	
+#endif
 	gic_set_kvm_info(&gic_v3_kvm_info);
 }
 
diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c
index 21b822e201ce29a2aab3c284aa0c2efa295af37d..7941da37fc80e99310b4e63e93385cc508f8024e 100644
--- a/drivers/irqchip/irq-gic-v4.c
+++ b/drivers/irqchip/irq-gic-v4.c
@@ -398,13 +398,14 @@ int its_prop_update_vlpi(int irq, u8 config, bool inv)
 	return irq_set_vcpu_affinity(irq, &info);
 }
 
-int its_prop_update_vsgi(int irq, u8 priority, bool group)
+int its_prop_update_vsgi(int irq, u8 priority, bool group, bool nmi)
 {
 	struct its_cmd_info info = {
 		.cmd_type = PROP_UPDATE_VSGI,
 		{
 			.priority	= priority,
 			.group		= group,
+			.nmi		= nmi,
 		},
 	};
 
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index 317afc404441b2262f0be333a89eeaeab4fa3d30..8a0d1a9b06e441975af5b8b6af6765d06edcdf77 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -99,6 +99,9 @@ struct vgic_global {
 	bool			has_gicv4;
 	bool			has_gicv4_1;
 
+	/* NMI */
+	bool                    has_nmi;
+
 #ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS
 /*
 	* Hardware (HiSilicon implementation) has vtimer interrupt
@@ -148,6 +151,7 @@ struct vgic_irq {
 	bool active;			/* not used for LPIs */
 	bool enabled;
 	bool hw;			/* Tied to HW IRQ */
+	bool nmi;			/* Configured as NMI */
 	struct kref refcount;		/* Used for LPIs */
 	u32 hwintid;			/* HW INTID number */
 	unsigned int host_irq;		/* linux irq corresponding to hwintid */
@@ -249,6 +253,10 @@ struct vgic_dist {
 
 	/* Implementation revision as reported in the GICD_IIDR */
 	u32			implementation_rev;
+#define KVM_VGIC_IMP_REV_2	2 /* GICv2 restorable groups */
+#define KVM_VGIC_IMP_REV_3	3 /* GICv3 GICR_CTLR.{IW,CES,RWP} */
+#define KVM_VGIC_IMP_REV_4	4 /* GICv3 NMI */
+#define KVM_VGIC_IMP_REV_LATEST	KVM_VGIC_IMP_REV_4
 
 	/* Userspace can write to GICv2 IGROUPR */
 	bool			v2_groups_user_writable;
@@ -281,6 +289,7 @@ struct vgic_dist {
 
 	struct vgic_io_device	dist_iodev;
 
+	bool                    has_nmi;
 	bool			has_its;
 
 	/*
@@ -328,7 +337,7 @@ struct vgic_v3_cpu_if {
 	u32		vgic_vmcr;
 	u32		vgic_sre;	/* Restored only, change ignored */
 	u32		vgic_ap0r[4];
-	u32		vgic_ap1r[4];
+	u64		vgic_ap1r[4];
 	u64		vgic_lr[VGIC_V3_MAX_LRS];
 
 	/*
@@ -372,11 +381,11 @@ struct vgic_cpu {
 	 */
 	struct vgic_io_device	rd_iodev;
 	struct vgic_redist_region *rdreg;
-
+	atomic_t syncr_busy;
 	/* Contains the attributes and gpa of the LPI pending tables. */
 	u64 pendbaser;
-
-	bool lpis_enabled;
+	/* GICR_CTLR.{ENABLE_LPIS,RWP} */
+	atomic_t ctlr;
 
 	/* Cache guest priority bits */
 	u32 num_pri_bits;
diff --git a/include/linux/irqchip/arm-gic-common.h b/include/linux/irqchip/arm-gic-common.h
index 408261a606db3337bb970e87d41517d86e6f52cf..4e29767d3aa71d2efd6e77b4cec46093dd1f76e0 100644
--- a/include/linux/irqchip/arm-gic-common.h
+++ b/include/linux/irqchip/arm-gic-common.h
@@ -34,6 +34,8 @@ struct gic_kvm_info {
 	bool		has_v4;
 	/* rvpeid support */
 	bool		has_v4_1;
+	/* NMI support */
+	bool            has_nmi;
 #ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS
 	/* vtimer irqbypass support */
     bool        has_vtimer;
diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h
index 9bf8c0c8b5d5d79c56a25ca514079d31e9cfa2b7..a24ac1ec17160fe73da7259c4d392e3ea5b7468f 100644
--- a/include/linux/irqchip/arm-gic-v3.h
+++ b/include/linux/irqchip/arm-gic-v3.h
@@ -134,6 +134,8 @@
 #define GICR_PIDR2			GICD_PIDR2
 
 #define GICR_CTLR_ENABLE_LPIS		(1UL << 0)
+#define GICR_CTLR_CES			(1UL << 1)
+#define GICR_CTLR_IR			(1UL << 2)
 #define GICR_CTLR_RWP			(1UL << 3)
 
 #define GICR_TYPER_CPU_NUMBER(r)	(((r) >> 8) & 0xffff)
diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h
index 07e8db2aa449f46086375daeb685fea11e249296..d31c085662b275f265a3a107a2e4ce5b4930f331 100644
--- a/include/linux/irqchip/arm-gic-v4.h
+++ b/include/linux/irqchip/arm-gic-v4.h
@@ -57,6 +57,7 @@ struct its_vpe {
 				u8	priority;
 				bool	enabled;
 				bool	group;
+				bool    nmi;
 #ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS
 			}			sgi_config[32];
 			int nr_irqs;
@@ -131,6 +132,7 @@ struct its_cmd_info {
 		struct {
 			u8		priority;
 			bool		group;
+			bool		nmi;
 		};
 	};
 };
@@ -145,7 +147,7 @@ int its_map_vlpi(int irq, struct its_vlpi_map *map);
 int its_get_vlpi(int irq, struct its_vlpi_map *map);
 int its_unmap_vlpi(int irq);
 int its_prop_update_vlpi(int irq, u8 config, bool inv);
-int its_prop_update_vsgi(int irq, u8 priority, bool group);
+int its_prop_update_vsgi(int irq, u8 priority, bool group, bool nmi);
 
 struct irq_domain_ops;
 int its_init_v4(struct irq_domain *domain,
diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 28cdd2f4d1eff766e66d9a56d198133c182b863c..7b2b4f8b6988045a3489666fce8efb7b97d7c483 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -135,6 +135,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	kvm->arch.max_vcpus = vgic_present ?
 				kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
 
+	if (cpus_have_const_cap(ARM64_HAS_NMI) &&
+			!static_branch_unlikely(&vgic_v3_cpuif_trap))
+		kvm->arch.pfr1_nmi = ID_AA64PFR1_NMI_IMP_DEF;
+
 	return ret;
 out_free_stage2_pgd:
 	kvm_free_stage2_pgd(kvm);
diff --git a/virt/kvm/arm/hyp/vgic-v3-sr.c b/virt/kvm/arm/hyp/vgic-v3-sr.c
index ccf1fde9836c1e2a5ae041899a0e7675e8837cc2..3ffbd5dbb3b0d9f2f836db36c816c2790f3f7ede 100644
--- a/virt/kvm/arm/hyp/vgic-v3-sr.c
+++ b/virt/kvm/arm/hyp/vgic-v3-sr.c
@@ -128,7 +128,11 @@ static void __hyp_text __vgic_v3_write_ap0rn(u32 val, int n)
 	}
 }
 
-static void __hyp_text __vgic_v3_write_ap1rn(u32 val, int n)
+/*
+ * Contrary to ICH_AP0Rn_EL2, ICH_AP1R0_EL2 is 64bit, thanks to the
+ * NMI bit stuck at [63]. Isn't that fun?
+ */
+static void __vgic_v3_write_ap1rn(u64 val, int n)
 {
 	switch (n) {
 	case 0:
@@ -170,9 +174,10 @@ static u32 __hyp_text __vgic_v3_read_ap0rn(int n)
 	return val;
 }
 
-static u32 __hyp_text __vgic_v3_read_ap1rn(int n)
+/* Same remark about the 64bit-ness of AP1R0 */
+static u64 __vgic_v3_read_ap1rn(int n)
 {
-	u32 val;
+	u64 val;
 
 	switch (n) {
 	case 0:
@@ -1028,6 +1033,9 @@ int __hyp_text __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
 			return 0;
 		fn = __vgic_v3_read_iar;
 		break;
+	case SYS_ICC_NMIAR1_EL1:
+	/* Here's an UNDEF for you */
+	return 0;
 	case SYS_ICC_EOIR0_EL1:
 	case SYS_ICC_EOIR1_EL1:
 		if (unlikely(is_read))
diff --git a/virt/kvm/arm/vgic/vgic-debug.c b/virt/kvm/arm/vgic/vgic-debug.c
index 19f5e0179b5187db0f28e72798d57a6589d17f3b..afbb1af13f459c55c52aa376908d8392edaf354c 100644
--- a/virt/kvm/arm/vgic/vgic-debug.c
+++ b/virt/kvm/arm/vgic/vgic-debug.c
@@ -155,7 +155,7 @@ static void print_dist_state(struct seq_file *s, struct vgic_dist *dist)
 
 	seq_printf(s, "P=pending_latch, L=line_level, A=active\n");
 	seq_printf(s, "E=enabled, H=hw, C=config (level=1, edge=0)\n");
-	seq_printf(s, "G=group\n");
+	seq_puts(s, "G=group, N=NMI\n");
 }
 
 static void print_header(struct seq_file *s, struct vgic_irq *irq,
@@ -170,8 +170,9 @@ static void print_header(struct seq_file *s, struct vgic_irq *irq,
 	}
 
 	seq_printf(s, "\n");
-	seq_printf(s, "%s%2d TYP   ID TGT_ID PLAEHCG     HWID   TARGET SRC PRI VCPU_ID\n", hdr, id);
-	seq_printf(s, "----------------------------------------------------------------\n");
+	seq_printf(s, "%s%2d TYP   ID TGT_ID PLAEHCGN     HWID   TARGET SRC PRI VCPU_ID\n",
+		hdr, id);
+	seq_puts(s, "-----------------------------------------------------------------\n");
 }
 
 static void print_irq_state(struct seq_file *s, struct vgic_irq *irq,
@@ -204,7 +205,7 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq,
 
 	seq_printf(s, "       %s %4d "
 		      "    %2d "
-		      "%d%d%d%d%d%d%d "
+		      "%d%d%d%d%d%d%d%d "
 		      "%8d "
 		      "%8x "
 		      " %2x "
@@ -220,6 +221,7 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq,
 			irq->hw,
 			irq->config == VGIC_CONFIG_LEVEL,
 			irq->group,
+			irq->nmi,
 			irq->hwintid,
 			irq->mpidr,
 			irq->source,
diff --git a/virt/kvm/arm/vgic/vgic-init.c b/virt/kvm/arm/vgic/vgic-init.c
index 3d04b3c539840c7e7097f9ff25b5a3ba1d7e710d..3365140f026968dd14c886b1cb56e92f2511302d 100644
--- a/virt/kvm/arm/vgic/vgic-init.c
+++ b/virt/kvm/arm/vgic/vgic-init.c
@@ -331,7 +331,19 @@ int vgic_init(struct kvm *kvm)
 
 	vgic_debug_init(kvm);
 
-	dist->implementation_rev = 2;
+	/*
+	 * If userspace didn't set the GIC implementation revision,
+	 * default to the latest and greatest. You know want it.
+	 */
+	if (!dist->implementation_rev) {
+		dist->implementation_rev = KVM_VGIC_IMP_REV_LATEST;
+		/*
+		 * Advertise NMI if available. Userspace that explicitly
+		 * doesn't want NMI will have written to GICD_{IIDR,TYPER}
+		 * to set the implementation and the NMI support status.
+		 */
+		dist->has_nmi = kvm_vgic_global_state.has_nmi;
+	}
 	dist->initialized = true;
 
 out:
diff --git a/virt/kvm/arm/vgic/vgic-its.c b/virt/kvm/arm/vgic/vgic-its.c
index a72f584254d088f8843dad7bbdbc11934b456de7..cd09a41c87606b347b09f460cec82d348d2c451b 100644
--- a/virt/kvm/arm/vgic/vgic-its.c
+++ b/virt/kvm/arm/vgic/vgic-its.c
@@ -683,7 +683,7 @@ int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
 	if (!vcpu)
 		return E_ITS_INT_UNMAPPED_INTERRUPT;
 
-	if (!vcpu->arch.vgic_cpu.lpis_enabled)
+	if (!vgic_lpis_enabled(vcpu))
 		return -EBUSY;
 
 	vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
@@ -1273,6 +1273,11 @@ static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
 	return 0;
 }
 
+int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq)
+{
+	return update_lpi_config(kvm, irq, NULL, true);
+}
+
 /*
  * The INV command syncs the configuration bits from the memory table.
  * Must be called with the its_lock mutex held.
@@ -1289,9 +1294,43 @@ static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
 	if (!ite)
 		return E_ITS_INV_UNMAPPED_INTERRUPT;
 
-	return update_lpi_config(kvm, ite->irq, NULL, true);
+	return vgic_its_inv_lpi(kvm, ite->irq);
 }
 
+/**
+ * vgic_its_invall - invalidate all LPIs targetting a given vcpu
+ * @vcpu: the vcpu for which the RD is targetted by an invalidation
+ *
+ * Contrary to the INVALL command, this targets a RD instead of a
+ * collection, and we don't need to hold the its_lock, since no ITS is
+ * involved here.
+ */
+int vgic_its_invall(struct kvm_vcpu *vcpu)
+{
+	struct kvm *kvm = vcpu->kvm;
+	int irq_count, i = 0;
+	u32 *intids;
+
+	irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids);
+	if (irq_count < 0)
+		return irq_count;
+
+	for (i = 0; i < irq_count; i++) {
+		struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intids[i]);
+		if (!irq)
+			continue;
+		update_lpi_config(kvm, irq, vcpu, false);
+		vgic_put_irq(kvm, irq);
+	}
+
+	kfree(intids);
+
+	if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
+		its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
+
+	return 0;
+ }
+
 /*
  * The INVALL command requests flushing of all IRQ data in this collection.
  * Find the VCPU mapped to that collection, then iterate over the VM's list
@@ -1306,9 +1345,6 @@ static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
 	u32 coll_id = its_cmd_get_collection(its_cmd);
 	struct its_collection *collection;
 	struct kvm_vcpu *vcpu;
-	struct vgic_irq *irq;
-	u32 *intids;
-	int irq_count, i;
 
 	collection = find_collection(its, coll_id);
 	if (!its_is_collection_mapped(collection))
@@ -1316,22 +1352,7 @@ static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
 
 	vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 
-	irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids);
-	if (irq_count < 0)
-		return irq_count;
-
-	for (i = 0; i < irq_count; i++) {
-		irq = vgic_get_irq(kvm, NULL, intids[i]);
-		if (!irq)
-			continue;
-		update_lpi_config(kvm, irq, vcpu, false);
-		vgic_put_irq(kvm, irq);
-	}
-
-	kfree(intids);
-
-	if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
-		its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
+	vgic_its_invall(vcpu);
 
 	return 0;
 }
diff --git a/virt/kvm/arm/vgic/vgic-mmio-v2.c b/virt/kvm/arm/vgic/vgic-mmio-v2.c
index d63881f60e1a51b6e3d52b5ea5f6df7be28ff73a..613d7244224454695d90221a934188ec66560d43 100644
--- a/virt/kvm/arm/vgic/vgic-mmio-v2.c
+++ b/virt/kvm/arm/vgic/vgic-mmio-v2.c
@@ -73,9 +73,13 @@ static int vgic_mmio_uaccess_write_v2_misc(struct kvm_vcpu *vcpu,
 					   gpa_t addr, unsigned int len,
 					   unsigned long val)
 {
+	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+	u32 reg;
+
 	switch (addr & 0x0c) {
 	case GIC_DIST_IIDR:
-		if (val != vgic_mmio_read_v2_misc(vcpu, addr, len))
+		reg = vgic_mmio_read_v2_misc(vcpu, addr, len);
+		if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
 			return -EINVAL;
 
 		/*
@@ -87,8 +91,17 @@ static int vgic_mmio_uaccess_write_v2_misc(struct kvm_vcpu *vcpu,
 		 * migration from old kernels to new kernels with legacy
 		 * userspace.
 		 */
-		vcpu->kvm->arch.vgic.v2_groups_user_writable = true;
-		return 0;
+		reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
+		switch (reg) {
+		case KVM_VGIC_IMP_REV_2:
+		case KVM_VGIC_IMP_REV_3:
+		case KVM_VGIC_IMP_REV_4:
+			vcpu->kvm->arch.vgic.v2_groups_user_writable = true;
+			dist->implementation_rev = reg;
+			return 0;
+		default:
+			return -EINVAL;
+		}
 	}
 
 	vgic_mmio_write_v2_misc(vcpu, addr, len, val);
diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c
index 9f58d383dbc3ade3a92fac43c25f4e66ad594a7c..357bbda35459e30bddfdd0aa264f22dc4f6fe256 100644
--- a/virt/kvm/arm/vgic/vgic-mmio-v3.c
+++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c
@@ -77,6 +77,8 @@ static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
 	case GICD_TYPER:
 		value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS;
 		value = (value >> 5) - 1;
+		if (vgic->has_nmi)
+			value |= GICD_TYPER_NMI;
 		if (vgic_has_its(vcpu->kvm)) {
 			value |= (INTERRUPT_ID_BITS_ITS - 1) << 19;
 			value |= GICD_TYPER_LPIS;
@@ -154,13 +156,38 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
 					   unsigned long val)
 {
 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
+	u32 reg;
 
 	switch (addr & 0x0c) {
+	case GICD_TYPER:
+	if (dist->implementation_rev >= KVM_VGIC_IMP_REV_4 &&
+		kvm_vgic_global_state.has_nmi)
+		dist->has_nmi = val & GICD_TYPER_NMI;
+	else if (val & GICD_TYPER_NMI)
+		return -EINVAL;
+	return 0;
 	case GICD_TYPER2:
-	case GICD_IIDR:
 		if (val != vgic_mmio_read_v3_misc(vcpu, addr, len))
 			return -EINVAL;
 		return 0;
+	case GICD_IIDR:
+		reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
+		if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
+			return -EINVAL;
+
+		reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
+		switch (reg) {
+		case KVM_VGIC_IMP_REV_2:
+		case KVM_VGIC_IMP_REV_3:
+			/* Disable NMI on selecting an older revision */
+			dist->has_nmi = false;
+			fallthrough;
+		case KVM_VGIC_IMP_REV_4:
+			dist->implementation_rev = reg;
+			return 0;
+		default:
+			return -EINVAL;
+		}
 	case GICD_CTLR:
 		/* Not a GICv4.1? No HW SGIs */
 		if (!kvm_vgic_global_state.has_gicv4_1)
@@ -171,7 +198,7 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
 		return 0;
 	}
 
-	vgic_mmio_write_v3_misc(vcpu, addr, len, val);
+	/* Not reachable... */
 	return 0;
 }
 
@@ -220,12 +247,23 @@ static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu,
 	vgic_put_irq(vcpu->kvm, irq);
 }
 
+bool vgic_lpis_enabled(struct kvm_vcpu *vcpu)
+{
+	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+
+	return atomic_read(&vgic_cpu->ctlr) == GICR_CTLR_ENABLE_LPIS;
+}
+
 static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu,
 					     gpa_t addr, unsigned int len)
 {
 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+	unsigned long val;
 
-	return vgic_cpu->lpis_enabled ? GICR_CTLR_ENABLE_LPIS : 0;
+	val = atomic_read(&vgic_cpu->ctlr);
+	if (vgic_get_implementation_rev(vcpu) >= KVM_VGIC_IMP_REV_3)
+		val |= GICR_CTLR_IR | GICR_CTLR_CES;
+	return val;
 }
 
 
@@ -234,20 +272,33 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
 				     unsigned long val)
 {
 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
-	bool was_enabled = vgic_cpu->lpis_enabled;
+	u32 ctlr;
 
 	if (!vgic_has_its(vcpu->kvm))
 		return;
 
-	vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS;
+	if (!(val & GICR_CTLR_ENABLE_LPIS)) {
+		/*
+		* Don't disable if RWP is set, as there already an
+		* ongoing disable. Funky guest...
+		*/
+		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr,
+							GICR_CTLR_ENABLE_LPIS,
+							GICR_CTLR_RWP);
+		if (ctlr != GICR_CTLR_ENABLE_LPIS)
+			return;
 
-	if (was_enabled && !vgic_cpu->lpis_enabled) {
 		vgic_flush_pending_lpis(vcpu);
 		vgic_its_invalidate_cache(vcpu->kvm);
-	}
+		atomic_set_release(&vgic_cpu->ctlr, 0);
+	} else {
+		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0,
+							GICR_CTLR_ENABLE_LPIS);
+		if (ctlr != 0)
+			return;
 
-	if (!was_enabled && vgic_cpu->lpis_enabled)
 		vgic_enable_lpis(vcpu);
+	}
 }
 
 static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu,
@@ -485,11 +536,10 @@ static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu,
 				     unsigned long val)
 {
 	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
-	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 	u64 old_propbaser, propbaser;
 
 	/* Storing a value with LPIs already enabled is undefined */
-	if (vgic_cpu->lpis_enabled)
+	if (vgic_lpis_enabled(vcpu))
 		return;
 
 	do {
@@ -517,7 +567,7 @@ static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu,
 	u64 old_pendbaser, pendbaser;
 
 	/* Storing a value with LPIs already enabled is undefined */
-	if (vgic_cpu->lpis_enabled)
+	if (vgic_lpis_enabled(vcpu))
 		return;
 
 	do {
@@ -529,6 +579,113 @@ static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu,
 			   pendbaser) != old_pendbaser);
 }
 
+static unsigned long vgic_mmio_read_sync(struct kvm_vcpu *vcpu,
+					 gpa_t addr, unsigned int len)
+{
+	return !!atomic_read(&vcpu->arch.vgic_cpu.syncr_busy);
+}
+
+static void vgic_set_rdist_busy(struct kvm_vcpu *vcpu, bool busy)
+{
+	if (busy) {
+		atomic_inc(&vcpu->arch.vgic_cpu.syncr_busy);
+		smp_mb__after_atomic();
+	} else {
+		smp_mb__before_atomic();
+		atomic_dec(&vcpu->arch.vgic_cpu.syncr_busy);
+	}
+}
+
+static void vgic_mmio_write_invlpi(struct kvm_vcpu *vcpu,
+				   gpa_t addr, unsigned int len,
+				   unsigned long val)
+{
+	struct vgic_irq *irq;
+
+	/*
+	 * If the guest wrote only to the upper 32bit part of the
+	 * register, drop the write on the floor, as it is only for
+	 * vPEs (which we don't support for obvious reasons).
+	 *
+	 * Also discard the access if LPIs are not enabled.
+	 */
+	if ((addr & 4) || !vgic_lpis_enabled(vcpu))
+		return;
+
+	vgic_set_rdist_busy(vcpu, true);
+
+	irq = vgic_get_irq(vcpu->kvm, NULL, lower_32_bits(val));
+	if (irq) {
+		vgic_its_inv_lpi(vcpu->kvm, irq);
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+
+	vgic_set_rdist_busy(vcpu, false);
+}
+
+static void vgic_mmio_write_invall(struct kvm_vcpu *vcpu,
+				   gpa_t addr, unsigned int len,
+				   unsigned long val)
+{
+	/* See vgic_mmio_write_invlpi() for the early return rationale */
+	if ((addr & 4) || !vgic_lpis_enabled(vcpu))
+		return;
+
+	vgic_set_rdist_busy(vcpu, true);
+	vgic_its_invall(vcpu);
+	vgic_set_rdist_busy(vcpu, false);
+}
+
+
+static unsigned long vgic_mmio_read_nmi(struct kvm_vcpu *vcpu,
+					gpa_t addr, unsigned int len)
+{
+	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
+	u32 value = 0;
+	int i;
+
+	/* Loop over all IRQs affected by this read */
+	for (i = 0; i < len * 8; i++) {
+		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
+
+		if (irq->nmi)
+			value |= (1U << i);
+
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+
+	return value;
+}
+
+static void vgic_mmio_write_nmi(struct kvm_vcpu *vcpu, gpa_t addr,
+				unsigned int len, unsigned long val)
+{
+	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
+	unsigned long flags;
+	int i;
+
+	if (!vcpu->kvm->arch.vgic.has_nmi)
+		return;
+
+	for (i = 0; i < len * 8; i++) {
+		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
+		bool was_nmi;
+
+		raw_spin_lock_irqsave(&irq->irq_lock, flags);
+
+		was_nmi = irq->nmi;
+		irq->nmi = (val & BIT(i));
+
+		if (irq->hw && vgic_irq_is_sgi(irq->intid) &&
+			was_nmi != irq->nmi)
+			vgic_update_vsgi(irq);
+
+		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
+
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+}
+
 /*
  * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the
  * redistributors, while SPIs are covered by registers in the distributor
@@ -600,6 +757,9 @@ static const struct vgic_register_region vgic_v3_dist_registers[] = {
 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGRPMODR,
 		vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 1,
 		VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_INMIR,
+		vgic_mmio_read_nmi, vgic_mmio_write_nmi, NULL, NULL, 1,
+		VGIC_ACCESS_32bit),
 	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IROUTER,
 		vgic_mmio_read_irouter, vgic_mmio_write_irouter, NULL, NULL, 64,
 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
@@ -632,6 +792,15 @@ static const struct vgic_register_region vgic_v3_rd_registers[] = {
 	REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER,
 		vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8,
 		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICR_INVLPIR,
+		vgic_mmio_read_raz, vgic_mmio_write_invlpi, 8,
+		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICR_INVALLR,
+		vgic_mmio_read_raz, vgic_mmio_write_invall, 8,
+		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICR_SYNCR,
+		vgic_mmio_read_sync, vgic_mmio_write_wi, 4,
+		VGIC_ACCESS_32bit),
 	REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
 		vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
 		VGIC_ACCESS_32bit),
@@ -673,6 +842,9 @@ static const struct vgic_register_region vgic_v3_rd_registers[] = {
 	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR,
 		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
 		VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_INMIR0,
+		vgic_mmio_read_nmi, vgic_mmio_write_nmi, 4,
+		VGIC_ACCESS_32bit),
 };
 
 unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev)
diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c
index 949408af409cb3837fa8d9d4fdfa085c17052e9f..b5698c5620561daea828d01c090c656b95978aa1 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.c
+++ b/virt/kvm/arm/vgic/vgic-mmio.c
@@ -61,9 +61,11 @@ unsigned long vgic_mmio_read_group(struct kvm_vcpu *vcpu,
 	return value;
 }
 
-static void vgic_update_vsgi(struct vgic_irq *irq)
+void vgic_update_vsgi(struct vgic_irq *irq)
 {
-	WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group));
+	WARN_ON(its_prop_update_vsgi(irq->host_irq,
+					irq->nmi ? 0 : irq->priority,
+					irq->group, irq->nmi));
 }
 
 void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr,
@@ -423,7 +425,7 @@ static unsigned long __vgic_mmio_read_active(struct kvm_vcpu *vcpu,
 		 */
 #ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS
 		if (state)
-#else		
+#else
 		if (irq->active)
 #endif
 			value |= (1U << i);
@@ -598,13 +600,17 @@ unsigned long vgic_mmio_read_priority(struct kvm_vcpu *vcpu,
 				      gpa_t addr, unsigned int len)
 {
 	u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
+	unsigned long flags;
 	int i;
 	u64 val = 0;
 
 	for (i = 0; i < len; i++) {
 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-		val |= (u64)irq->priority << (i * 8);
+		raw_spin_lock_irqsave(&irq->irq_lock, flags);
+		if (!irq->nmi)
+			val |= (u64)irq->priority << (i * 8);
+		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
 		vgic_put_irq(vcpu->kvm, irq);
 	}
@@ -631,10 +637,16 @@ void vgic_mmio_write_priority(struct kvm_vcpu *vcpu,
 		struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
 		raw_spin_lock_irqsave(&irq->irq_lock, flags);
-		/* Narrow the priority range to what we actually support */
-		irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);
-		if (vgic_direct_sgi_or_ppi(irq))
-			vgic_update_vsgi(irq);
+		if (!irq->nmi) {
+			/*
+			 * Narrow the priority range to what we
+			 * actually support
+			 */
+			irq->priority = (val >> (i * 8)) &
+				GENMASK(7, 8 - VGIC_PRI_BITS);
+			if (vgic_direct_sgi_or_ppi(irq))
+				vgic_update_vsgi(irq);
+		}
 		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
 		vgic_put_irq(vcpu->kvm, irq);
diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c
index 909c5934a664521cb1a5e06e8099390160b84043..90c18f1eed11d7ff57205ef309f9325dea819b18 100644
--- a/virt/kvm/arm/vgic/vgic-v3.c
+++ b/virt/kvm/arm/vgic/vgic-v3.c
@@ -194,7 +194,11 @@ void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr)
 	if (irq->group)
 		val |= ICH_LR_GROUP;
 
-	val |= (u64)irq->priority << ICH_LR_PRIORITY_SHIFT;
+	if (vcpu->kvm->arch.pfr1_nmi == ID_AA64PFR1_NMI_IMP_DEF &&
+		irq->nmi)
+		val |= ICH_LR_NMI;
+	else
+		val |= (u64)irq->priority << ICH_LR_PRIORITY_SHIFT;
 
 	vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr] = val;
 }
@@ -706,6 +710,15 @@ int vgic_v3_probe(const struct gic_kvm_info *info)
 		static_branch_enable(&vgic_v3_cpuif_trap);
 	}
 
+
+	if (info->has_nmi) {
+		kvm_vgic_global_state.has_nmi =
+				!static_branch_unlikely(&vgic_v3_cpuif_trap);
+		kvm_info("GICv3 NMI support %s\n",
+			 kvm_vgic_global_state.has_nmi ? "enabled" :
+					"disabled due to trapping");
+	}
+
 	kvm_vgic_global_state.vctrl_base = NULL;
 	kvm_vgic_global_state.type = VGIC_V3;
 	kvm_vgic_global_state.max_gic_vcpus = VGIC_V3_MAX_CPUS;
diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c
index 60c758b54fac018346f6936c3df0c0e08e6e0b9c..b47104e011cd6e5c8ae7742ca823250b588d3b9a 100644
--- a/virt/kvm/arm/vgic/vgic-v4.c
+++ b/virt/kvm/arm/vgic/vgic-v4.c
@@ -110,6 +110,7 @@ static void vgic_v4_sync_sgi_config(struct its_vpe *vpe, struct vgic_irq *irq)
 	vpe->sgi_config[irq->intid].enabled	= irq->enabled;
 	vpe->sgi_config[irq->intid].group 	= irq->group;
 	vpe->sgi_config[irq->intid].priority	= irq->priority;
+	vpe->sgi_config[irq->intid].nmi		= irq->nmi;
 }
 
 static void vgic_v4_enable_vsgis(struct kvm_vcpu *vcpu)
diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c
index 804e9cc9867fba9a9b3c1965e5ce9f106d21d9a3..bac9feb52e368f694a4503a00592c731d45bc913 100644
--- a/virt/kvm/arm/vgic/vgic.c
+++ b/virt/kvm/arm/vgic/vgic.c
@@ -251,6 +251,7 @@ static struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq)
  *
  * Otherwise things should be sorted by the priority field and the GIC
  * hardware support will take care of preemption of priority groups etc.
+ * NMI acts as a super-priority.
  *
  * Return negative if "a" sorts before "b", 0 to preserve order, and positive
  * to sort "b" before "a".
@@ -285,7 +286,11 @@ static int vgic_irq_cmp(void *priv, struct list_head *a, struct list_head *b)
 		goto out;
 	}
 
-	/* Both pending and enabled, sort by priority */
+	/* Both pending and enabled, sort by NMI and then priority */
+	if (irqa->nmi != irqb->nmi) {
+		ret = (int)irqb->nmi - (int)irqa->nmi;
+		goto out;
+	}
 	ret = irqa->priority - irqb->priority;
 out:
 	raw_spin_unlock(&irqb->irq_lock);
diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h
index 9d98a4345212c6e6963fee95c25f074f8a2f4edf..7592f04c5234337423a26772f3c5176c9b78780b 100644
--- a/virt/kvm/arm/vgic/vgic.h
+++ b/virt/kvm/arm/vgic/vgic.h
@@ -98,6 +98,11 @@
 #define DEBUG_SPINLOCK_BUG_ON(p)
 #endif
 
+static inline u32 vgic_get_implementation_rev(struct kvm_vcpu *vcpu)
+{
+	return vcpu->kvm->arch.vgic.implementation_rev;
+}
+
 /* Requires the irq_lock to be held by the caller. */
 static inline bool irq_is_pending(struct vgic_irq *irq)
 {
@@ -317,6 +322,7 @@ static inline bool vgic_dist_overlap(struct kvm *kvm, gpa_t base, size_t size)
 		(base < d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE);
 }
 
+bool vgic_lpis_enabled(struct kvm_vcpu *vcpu);
 int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr);
 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
 			 u32 devid, u32 eventid, struct vgic_irq **irq);
@@ -326,10 +332,16 @@ void vgic_lpi_translation_cache_init(struct kvm *kvm);
 void vgic_lpi_translation_cache_destroy(struct kvm *kvm);
 void vgic_its_invalidate_cache(struct kvm *kvm);
 
+/* GICv4.1 MMIO interface */
+int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq);
+int vgic_its_invall(struct kvm_vcpu *vcpu);
+
+
 bool vgic_supports_direct_msis(struct kvm *kvm);
 int vgic_v4_init(struct kvm *kvm);
 void vgic_v4_teardown(struct kvm *kvm);
 void vgic_v4_configure_vsgis(struct kvm *kvm);
+void vgic_update_vsgi(struct vgic_irq *irq);
 void vgic_v4_get_vlpi_state(struct vgic_irq *irq, bool *val);
 int vgic_v4_request_vpe_irq(struct kvm_vcpu *vcpu, int irq);