diff --git a/arch/arm64/Kconfig.turbo b/arch/arm64/Kconfig.turbo index 76ad4ea567c3a52a50c117ad1fd4d2d1bcbb9760..c4a8e4e889aa4ae81c7c9ed5a739400d30ca8c9c 100644 --- a/arch/arm64/Kconfig.turbo +++ b/arch/arm64/Kconfig.turbo @@ -63,6 +63,7 @@ config SECURITY_FEATURE_BYPASS config ACTLR_XCALL_XINT bool "Hardware XCALL and Xint support" + depends on FAST_SYSCALL default n help Use the 0x600 as the offset to the exception vector base address for diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index ad688e157c9bed3663f74c427558f870b7159565..d69f0e6d53f82b487b8a21c9a9c8acc8f35b5fac 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -77,4 +77,9 @@ void do_serror(struct pt_regs *regs, unsigned long esr); void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags); void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far); + +#ifdef CONFIG_ACTLR_XCALL_XINT +asmlinkage void el0t_64_xint_handler(struct pt_regs *regs); +asmlinkage void el0t_64_xcall_handler(struct pt_regs *regs); +#endif #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index a6fb325424e7a990313dc6419a198155e5205b5c..39595fa034913801c27603e4e993b691ef09f77d 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -24,6 +24,9 @@ #include #include #include +#ifdef CONFIG_ACTLR_XCALL_XINT +#include +#endif extern bool rodata_full; @@ -264,6 +267,10 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, if (prev != next) __switch_mm(next); +#ifdef CONFIG_ACTLR_XCALL_XINT + cpu_switch_xcall_entry(tsk); +#endif + /* * Update the saved TTBR0_EL1 of the scheduled-in task as the previous * value may have not been initialised yet (activate_mm caller) or the diff --git a/arch/arm64/include/asm/xcall.h b/arch/arm64/include/asm/xcall.h new file mode 100644 index 0000000000000000000000000000000000000000..a00d710478e35995556519121c971aab57dbd33a --- /dev/null +++ b/arch/arm64/include/asm/xcall.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_XCALL_H +#define __ASM_XCALL_H + +#include +#include +#include +#include + +#include +DECLARE_STATIC_KEY_FALSE(xcall_enable); +DECLARE_STATIC_KEY_FALSE(has_hwcap_actlr_xcall); + +struct xcall_info { + /* Must be first! */ + DECLARE_BITMAP(xcall_enable, __NR_syscalls); +}; + +#define TASK_XINFO(p) ((struct xcall_info *)p->xinfo) + +int xcall_init_task(struct task_struct *p, struct task_struct *orig); +void xcall_task_free(struct task_struct *p); + +#ifdef CONFIG_ACTLR_XCALL_XINT +struct hw_xcall_info { + /* Must be first! */ + void *xcall_entry[__NR_syscalls + 1]; + bool xcall_scno_enabled; +}; + +#define TASK_HW_XINFO(p) ((struct hw_xcall_info *)p->xinfo) +#define XCALL_ENTRY_SIZE (sizeof(unsigned long) * (__NR_syscalls + 1)) + +DECLARE_PER_CPU(void *, __cpu_xcall_entry); +extern void xcall_entry(void); +extern void no_xcall_entry(void); + +static inline bool is_xcall_entry(struct hw_xcall_info *xinfo, unsigned int sc_no) +{ + return xinfo->xcall_entry[sc_no] == xcall_entry; +} + +static inline int has_xcall_scno_enabled(struct hw_xcall_info *xinfo) +{ + unsigned int i; + + for (i = 0; i < __NR_syscalls; i++) { + if (is_xcall_entry(xinfo, i)) + return true; + } + + return false; +} + +static inline int set_xcall_entry(struct hw_xcall_info *xinfo, unsigned int sc_no) +{ + xinfo->xcall_entry[sc_no] = xcall_entry; + xinfo->xcall_scno_enabled = true; + + return 0; +} + +static inline int set_no_xcall_entry(struct hw_xcall_info *xinfo, unsigned int sc_no) +{ + xinfo->xcall_entry[sc_no] = no_xcall_entry; + if (!has_xcall_scno_enabled(xinfo)) + xinfo->xcall_scno_enabled = false; + + return 0; +} + +static inline void cpu_enable_arch_xcall(void) +{ + u64 el = read_sysreg(CurrentEL); + + if (el == CurrentEL_EL2) + write_sysreg(read_sysreg(actlr_el2) | ACTLR_ELx_XCALL, actlr_el2); + else + write_sysreg(read_sysreg(actlr_el1) | ACTLR_ELx_XCALL, actlr_el1); +} + +static inline void cpu_switch_xcall_entry(struct task_struct *tsk) +{ + if (!static_branch_unlikely(&has_hwcap_actlr_xcall) || !tsk->xinfo) + return; + + if (TASK_HW_XINFO(tsk)->xcall_scno_enabled) { + __this_cpu_write(__cpu_xcall_entry, TASK_HW_XINFO(tsk)->xcall_entry); + cpu_enable_arch_xcall(); + } +} +#endif /* CONFIG_ACTLR_XCALL_XINT */ + +#endif /*__ASM_XCALL_H*/ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 3d404a2cc961ae52657da727e3123d679061eea5..20b8c646696541246c559661ee22b807040ad1ed 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -79,6 +79,7 @@ obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o obj-$(CONFIG_ARM64_ILP32) += vdso-ilp32/ +obj-$(CONFIG_FAST_SYSCALL) += xcall/ obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o obj-$(CONFIG_IPI_AS_NMI) += ipi_nmi.o obj-$(CONFIG_HISI_VIRTCCA_GUEST) += virtcca_cvm_guest.o virtcca_cvm_tsi.o diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index f20918eb36bc34e1751e71f6e50ed7bff302204e..bc8b380046f4cc76cd203b460895b01205b31b8e 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -29,7 +29,7 @@ int main(void) { #ifdef CONFIG_FAST_SYSCALL - DEFINE(TSK_XCALL, offsetof(struct task_struct, xcall_enable)); + DEFINE(TSK_XCALL, offsetof(struct task_struct, xinfo)); #endif DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); BLANK(); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index b138586688772e1e2b22ef02ae0cf90ebeeff1ba..0052e5007de60c8b22364663000a63403be2ce60 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2430,7 +2430,11 @@ static void mpam_extra_caps(void) } #ifdef CONFIG_FAST_SYSCALL +#include static bool is_xcall_support; +DEFINE_STATIC_KEY_FALSE(xcall_enable); +DEFINE_STATIC_KEY_FALSE(has_hwcap_actlr_xcall); + static int __init xcall_setup(char *str) { is_xcall_support = true; @@ -2438,14 +2442,17 @@ static int __init xcall_setup(char *str) } __setup("xcall", xcall_setup); -bool fast_syscall_enabled(void) -{ - return is_xcall_support; -} - static bool has_xcall_support(const struct arm64_cpu_capabilities *entry, int __unused) { - return is_xcall_support; + if (static_branch_unlikely(&has_hwcap_actlr_xcall)) + return false; + + if (is_xcall_support) { + static_branch_enable(&xcall_enable); + return true; + } + + return false; } #endif @@ -2476,7 +2483,12 @@ static bool has_arch_xcall_xint_support(const struct arm64_cpu_capabilities *ent { /* sentinel */ } }; - return is_midr_in_range_list(read_cpuid_id(), xcall_xint_cpus); + if (is_midr_in_range_list(read_cpuid_id(), xcall_xint_cpus)) { + static_branch_enable(&has_hwcap_actlr_xcall); + return true; + } + + return false; } static void enable_xcall_xint_vectors(void) @@ -2507,7 +2519,7 @@ static void enable_xcall_xint_vectors(void) isb(); } -static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__unused) +static void cpu_enable_arch_xint(void) { int cpu = smp_processor_id(); u64 actlr_el1, actlr_el2; @@ -2520,20 +2532,24 @@ static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__un */ write_sysreg_s(read_sysreg_s(SYS_HCR_EL2) | HCR_TACR, SYS_HCR_EL2); actlr_el2 = read_sysreg(actlr_el2); - actlr_el2 |= (ACTLR_ELx_XINT | ACTLR_ELx_XCALL); + actlr_el2 |= ACTLR_ELx_XINT; write_sysreg(actlr_el2, actlr_el2); isb(); actlr_el2 = read_sysreg(actlr_el2); pr_info("actlr_el2: %llx, cpu:%d\n", actlr_el2, cpu); } else { actlr_el1 = read_sysreg(actlr_el1); - actlr_el1 |= (ACTLR_ELx_XINT | ACTLR_ELx_XCALL); + actlr_el1 |= ACTLR_ELx_XINT; write_sysreg(actlr_el1, actlr_el1); isb(); actlr_el1 = read_sysreg(actlr_el1); pr_info("actlr_el1: %llx, cpu:%d\n", actlr_el1, cpu); } +} +static void cpu_enable_arch_xcall_xint(const struct arm64_cpu_capabilities *__unused) +{ + cpu_enable_arch_xint(); enable_xcall_xint_vectors(); } #endif diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index a9023134675169b0570b06d9b808a4b4443ce72f..1e8171c1efe76a1873849c1c34bb73c577b9cdef 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -1054,7 +1054,7 @@ UNHANDLED(el0t, 32, error) #ifdef CONFIG_ACTLR_XCALL_XINT asmlinkage void noinstr el0t_64_xcall_handler(struct pt_regs *regs) { - el0_svc(regs); + el0_xcall(regs); } asmlinkage void noinstr el0t_64_xint_handler(struct pt_regs *regs) { diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 117be8a7d52941243f65502cb3be4d94c6d79fbf..a7bdaa0fb8b95739f4ed9a0e1641ed11d1af9c90 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -581,6 +581,8 @@ SYM_CODE_START(vectors) SYM_CODE_END(vectors) #ifdef CONFIG_ACTLR_XCALL_XINT +#include "xcall/entry.S" + .align 11 SYM_CODE_START(vectors_xcall_xint) kernel_ventry 1, t, 64, sync // Synchronous EL1t @@ -598,7 +600,7 @@ SYM_CODE_START(vectors_xcall_xint) kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0 kernel_ventry 0, t, 64, error // Error 64-bit EL0 - kernel_ventry 0, t, 64, xcall // XCALL synchronous 64-bit EL0 + xcall_ventry // XCALL synchronous 64-bit EL0 kernel_ventry 0, t, 64, xint // XINT 64-bit EL0 kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0 kernel_ventry 0, t, 32, error // Error 32-bit EL0 @@ -647,10 +649,12 @@ SYM_CODE_END(__bad_stack) cmp x8, __NR_syscalls .endm + /* + * x21 = task_struct->xinfo->xcall_enable + */ .macro check_xcall_enable - /* x21 = task_struct->xcall_enable */ - ldr_this_cpu x20, __entry_task, x21 - ldr x21, [x20, #TSK_XCALL] + /* is xcall enabled */ + str x8, [sp, #16] /* x20 = sc_no / 8 */ lsr x20, x8, 3 ldr x21, [x21, x20] @@ -660,6 +664,7 @@ SYM_CODE_END(__bad_stack) lsl x20, x20, x8 and x21, x21, x20 cmp x21, 0 + ldr x8, [sp, #16] .endm .macro check_xcall_pre_kernel_entry @@ -670,10 +675,12 @@ SYM_CODE_END(__bad_stack) /* x8 >= __NR_syscalls */ check_syscall_nr bhs .Lskip_xcall\@ - str x8, [sp, #16] - /* is xcall enabled */ + + ldr_this_cpu x20, __entry_task, x21 + ldr x21, [x20, #TSK_XCALL] + /* is task_struct->xinfo == NULL? */ + cbz x21, .Lskip_xcall\@ check_xcall_enable - ldr x8, [sp, #16] beq .Lskip_xcall\@ ldp x20, x21, [sp, #0] /* do xcall */ diff --git a/arch/arm64/kernel/xcall/Makefile b/arch/arm64/kernel/xcall/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..0168bd1907939373ff26689f3e22d9b95a5dfe70 --- /dev/null +++ b/arch/arm64/kernel/xcall/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-y += xcall.o diff --git a/arch/arm64/kernel/xcall/entry.S b/arch/arm64/kernel/xcall/entry.S new file mode 100644 index 0000000000000000000000000000000000000000..60eac5f61426b157b4ff5c73859719535b7554bc --- /dev/null +++ b/arch/arm64/kernel/xcall/entry.S @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Low-level exception handling code + * + * Copyright (C) 2012 ARM Ltd. + * Authors: Catalin Marinas + * Will Deacon + */ + + .macro hw_xcall_entry + stp x0, x1, [sp, #16 * 0] // save x0~x29 + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + clear_gp_regs // clear x0~x29 + mrs x21, sp_el0 + ldr_this_cpu tsk, __entry_task, x20 + msr sp_el0, tsk + +#ifdef CONFIG_ARM64_PTR_AUTH +alternative_if ARM64_HAS_ADDRESS_AUTH + /* + * Enable IA for in-kernel PAC if the task had it disabled. Although + * this could be implemented with an unconditional MRS which would avoid + * a load, this was measured to be slower on Cortex-A75 and Cortex-A76. + * + * Install the kernel IA key only if IA was enabled in the task. If IA + * was disabled on kernel exit then we would have left the kernel IA + * installed so there is no need to install it again. + */ + ldr x0, [tsk, THREAD_SCTLR_USER] + + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f + __ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23 + b 2f +1: + mrs x0, sctlr_el1 + orr x0, x0, SCTLR_ELx_ENIA + msr sctlr_el1, x0 +2: +alternative_else_nop_endif +#endif + +alternative_if ARM64_HAS_ADDRESS_AUTH + isb +alternative_else_nop_endif + + mrs x22, elr_el1 + mrs x23, spsr_el1 + stp lr, x21, [sp, #S_LR] // save LR,USER SP + + stp xzr, xzr, [sp, #S_STACKFRAME] + add x29, sp, #S_STACKFRAME // calc FP + + stp x22, x23, [sp, #S_PC] // save ELR,SPSR + + mov w21, #NO_SYSCALL + str w21, [sp, #S_SYSCALLNO] + + +#ifdef CONFIG_ARM64_PSEUDO_NMI +alternative_if_not ARM64_HAS_GIC_PRIO_MASKING + b .Lskip_pmr_save\@ +alternative_else_nop_endif + + mrs_s x20, SYS_ICC_PMR_EL1 + str x20, [sp, #S_PMR_SAVE] + mov x20, #GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET + msr_s SYS_ICC_PMR_EL1, x20 + +.Lskip_pmr_save\@: +#endif + .endm + + .macro hw_xcall_exit +#ifdef CONFIG_ARM64_PSEUDO_NMI +alternative_if_not ARM64_HAS_GIC_PRIO_MASKING + b .Lskip_pmr_restore\@ +alternative_else_nop_endif + + ldr x20, [sp, #S_PMR_SAVE] + msr_s SYS_ICC_PMR_EL1, x20 + + /* Ensure priority change is seen by redistributor */ +alternative_if_not ARM64_HAS_GIC_PRIO_RELAXED_SYNC + dsb sy +alternative_else_nop_endif + +.Lskip_pmr_restore\@: +#endif + + ldp x21, x22, [sp, #S_PC] + ldr x23, [sp, #S_SP] + msr sp_el0, x23 // restore USER SP + +#ifdef CONFIG_ARM64_PTR_AUTH +alternative_if ARM64_HAS_ADDRESS_AUTH + /* + * IA was enabled for in-kernel PAC. Disable it now if needed, or + * alternatively install the user's IA. All other per-task keys and + * SCTLR bits were updated on task switch. + * + * No kernel C function calls after this. + */ + ldr x0, [tsk, THREAD_SCTLR_USER] + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f + __ptrauth_keys_install_user tsk, x0, x1, x2 + b 2f +1: + mrs x0, sctlr_el1 + bic x0, x0, SCTLR_ELx_ENIA + msr sctlr_el1, x0 +2: +alternative_else_nop_endif +#endif + + msr elr_el1, x21 // restore ELR + msr spsr_el1, x22 // restore SPSR + + ldp x0, x1, [sp, #16 * 0] // restore x0~x29 + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + + ldr lr, [sp, #S_LR] + add sp, sp, #PT_REGS_SIZE // restore sp + eret + sb + .endm + +SYM_CODE_START(no_xcall_entry) + ldp x20, x21, [sp, #0] + kernel_entry 0, 64 + mov x0, sp + bl el0t_64_sync_handler + b ret_to_user +SYM_CODE_END(no_xcall_entry) + +SYM_CODE_START(xcall_entry) + ldp x20, x21, [sp, #0] + hw_xcall_entry + mov x0, sp + bl el0t_64_xcall_handler + hw_xcall_exit +SYM_CODE_END(xcall_entry) + + .macro check_hw_xcall_pre_kernel_entry + /* x8 >= __NR_syscalls */ + cmp x8, __NR_syscalls + + stp x20, x21, [sp, #0] + ldr_this_cpu x21, __cpu_xcall_entry, x20 + mov x20, __NR_syscalls + csel x20, x8, x20, lt + ldr x21, [x21, x20, lsl #3] + br x21 + .endm + +SYM_CODE_START_LOCAL(el0t_64_hw_xcall) + check_hw_xcall_pre_kernel_entry +SYM_CODE_END(el0t_64_hw_xcall) + + .macro xcall_ventry + .align 7 +.Lventry_start\@: + /* + * This must be the first instruction of the EL0 vector entries. It is + * skipped by the trampoline vectors, to trigger the cleanup. + */ + b .Lskip_tramp_vectors_cleanup\@ + mrs x30, tpidrro_el0 + msr tpidrro_el0, xzr +.Lskip_tramp_vectors_cleanup\@: + + sub sp, sp, #PT_REGS_SIZE + b el0t_64_hw_xcall +.org .Lventry_start\@ + 128 // Did we overflow the ventry slot? + .endm diff --git a/arch/arm64/kernel/xcall/xcall.c b/arch/arm64/kernel/xcall/xcall.c new file mode 100644 index 0000000000000000000000000000000000000000..c44111e06448dfefc08d53da4d2e3b061ca8977c --- /dev/null +++ b/arch/arm64/kernel/xcall/xcall.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * xcall related code + * + * Copyright (C) 2025 Huawei Ltd. + */ + +#include +#include +#include +#include +#include + +static inline int sw_xcall_init_task(struct task_struct *p, struct task_struct *orig) +{ + p->xinfo = kzalloc(sizeof(struct xcall_info), GFP_KERNEL); + if (!p->xinfo) + return -ENOMEM; + + if (orig->xinfo) { + bitmap_copy(TASK_XINFO(p)->xcall_enable, TASK_XINFO(orig)->xcall_enable, + __NR_syscalls); + } + + return 0; +} + +#ifdef CONFIG_ACTLR_XCALL_XINT +static void *default_syscall_table[__NR_syscalls + 1] = { + [0 ... __NR_syscalls] = no_xcall_entry, +}; + +asmlinkage DEFINE_PER_CPU(void *, __cpu_xcall_entry) = default_syscall_table; +static inline int hw_xcall_init_task(struct task_struct *p, struct task_struct *orig) +{ + p->xinfo = kzalloc(sizeof(struct hw_xcall_info), GFP_KERNEL); + if (!p->xinfo) + return -ENOMEM; + + if (orig->xinfo) { + memcpy(p->xinfo, orig->xinfo, XCALL_ENTRY_SIZE); + TASK_HW_XINFO(p)->xcall_scno_enabled = TASK_HW_XINFO(orig)->xcall_scno_enabled; + } else { + memcpy(p->xinfo, default_syscall_table, XCALL_ENTRY_SIZE); + TASK_HW_XINFO(p)->xcall_scno_enabled = false; + } + + return 0; +} +#endif + +int xcall_init_task(struct task_struct *p, struct task_struct *orig) +{ +#ifdef CONFIG_ACTLR_XCALL_XINT + if (static_branch_unlikely(&has_hwcap_actlr_xcall)) + return hw_xcall_init_task(p, orig); +#endif + if (static_branch_unlikely(&xcall_enable)) + return sw_xcall_init_task(p, orig); + + return 0; +} + +void xcall_task_free(struct task_struct *p) +{ + if (!static_branch_unlikely(&has_hwcap_actlr_xcall) && + !static_branch_unlikely(&xcall_enable)) + return; + + kfree(p->xinfo); +} diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index ae0268822d61c37390296ced9a43ec85cb6c9053..4fc5ce93f3ea5d43864ca87c96e58ee55ca7c4ab 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -108,11 +108,11 @@ WORKAROUND_SPECULATIVE_UNPRIV_LOAD WORKAROUND_HISILICON_ERRATUM_162100125 WORKAROUND_HISI_HIP08_RU_PREFETCH WORKAROUND_HISILICON_1980005 +HAS_HW_XCALL_XINT HAS_XCALL HAS_XINT HAS_LS64 HAS_LS64_V -HAS_HW_XCALL_XINT KABI_RESERVE_6 KABI_RESERVE_7 KABI_RESERVE_8 diff --git a/fs/proc/Makefile b/fs/proc/Makefile index daa43e10b40b60edd7b9751ce051b3acdeccca6c..b17f9b9eb6520aa63ecc1fca7b0b6bb9c34e85e3 100644 --- a/fs/proc/Makefile +++ b/fs/proc/Makefile @@ -38,3 +38,4 @@ proc-$(CONFIG_MEMORY_RELIABLE) += mem_reliable.o obj-$(CONFIG_ETMEM_SCAN) += etmem_scan.o obj-$(CONFIG_ETMEM_SWAP) += etmem_swap.o proc-${CONFIG_ETMEM} += etmem_proc.o +proc-$(CONFIG_FAST_SYSCALL) += proc_xcall.o diff --git a/fs/proc/base.c b/fs/proc/base.c index e62bb31bcbb1486645c80ea43b11e0d0872c5b3b..62ec367d7f943cad7ff7ae799d9cb5a34ca54e47 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3454,115 +3454,6 @@ static const struct file_operations proc_pid_sg_level_operations = { }; #endif -#ifdef CONFIG_FAST_SYSCALL -bool fast_syscall_enabled(void); - -static int xcall_show(struct seq_file *m, void *v) -{ - struct inode *inode = m->private; - struct task_struct *p; - unsigned int rs, re; - - if (!fast_syscall_enabled()) - return -EACCES; - - p = get_proc_task(inode); - if (!p) - return -ESRCH; - - if (!p->xcall_enable) - goto out; - - seq_printf(m, "Enabled Total[%d/%d]:", bitmap_weight(p->xcall_enable, __NR_syscalls), - __NR_syscalls); - - for (rs = 0, bitmap_next_set_region(p->xcall_enable, &rs, &re, __NR_syscalls); - rs < re; rs = re + 1, - bitmap_next_set_region(p->xcall_enable, &rs, &re, __NR_syscalls)) { - rs == (re - 1) ? seq_printf(m, "%d,", rs) : - seq_printf(m, "%d-%d,", rs, re - 1); - } - seq_puts(m, "\n"); -out: - put_task_struct(p); - - return 0; -} - -static int xcall_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, xcall_show, inode); -} - -static int xcall_enable_one(struct task_struct *p, unsigned int sc_no) -{ - bitmap_set(p->xcall_enable, sc_no, 1); - return 0; -} - -static int xcall_disable_one(struct task_struct *p, unsigned int sc_no) -{ - bitmap_clear(p->xcall_enable, sc_no, 1); - return 0; -} - -static ssize_t xcall_write(struct file *file, const char __user *buf, - size_t count, loff_t *offset) -{ - struct inode *inode = file_inode(file); - struct task_struct *p; - char buffer[TASK_COMM_LEN]; - const size_t maxlen = sizeof(buffer) - 1; - unsigned int sc_no = __NR_syscalls; - int ret = 0; - int is_clear = 0; - - if (!fast_syscall_enabled()) - return -EACCES; - - memset(buffer, 0, sizeof(buffer)); - if (!count || copy_from_user(buffer, buf, count > maxlen ? maxlen : count)) - return -EFAULT; - - p = get_proc_task(inode); - if (!p || !p->xcall_enable) - return -ESRCH; - - if (buffer[0] == '!') - is_clear = 1; - - if (kstrtouint(buffer + is_clear, 10, &sc_no)) { - ret = -EINVAL; - goto out; - } - - if (sc_no >= __NR_syscalls) { - ret = -EINVAL; - goto out; - } - - if (!is_clear && !test_bit(sc_no, p->xcall_enable)) - ret = xcall_enable_one(p, sc_no); - else if (is_clear && test_bit(sc_no, p->xcall_enable)) - ret = xcall_disable_one(p, sc_no); - else - ret = -EINVAL; - -out: - put_task_struct(p); - - return ret ? ret : count; -} - -static const struct file_operations proc_pid_xcall_operations = { - .open = xcall_open, - .read = seq_read, - .write = xcall_write, - .llseek = seq_lseek, - .release = single_release, -}; -#endif - /* * Thread groups */ diff --git a/fs/proc/internal.h b/fs/proc/internal.h index a86dba479ffff5813d15d5574d45d9b7df3969d7..d5509cb5019c781ea8c469a2c3339f0f9668533e 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -334,3 +334,7 @@ static inline void pde_force_lookup(struct proc_dir_entry *pde) /* /proc/net/ entries can be changed under us by setns(CLONE_NEWNET) */ pde->proc_dops = &proc_net_dentry_ops; } + +#ifdef CONFIG_FAST_SYSCALL +extern const struct file_operations proc_pid_xcall_operations; +#endif diff --git a/fs/proc/proc_xcall.c b/fs/proc/proc_xcall.c new file mode 100644 index 0000000000000000000000000000000000000000..765a8f75e22996de13e5752166d1fde5e9c5b56f --- /dev/null +++ b/fs/proc/proc_xcall.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * xcall related proc code + * + * Copyright (C) 2025 Huawei Ltd. + */ +#include +#include +#include +#include +#include "internal.h" + +#ifdef CONFIG_ACTLR_XCALL_XINT +static void hw_xcall_show(struct task_struct *p, struct seq_file *m) +{ + struct hw_xcall_info *hw_xinfo = TASK_HW_XINFO(p); + unsigned int i, start = 0, end = 0; + bool in_range = false; + + if (!hw_xinfo) + return; + + for (i = 0; i < __NR_syscalls; i++) { + bool scno_xcall_enable = is_xcall_entry(hw_xinfo, i); + + if (scno_xcall_enable && !in_range) { + in_range = true; + start = i; + } + + if ((!scno_xcall_enable || i == __NR_syscalls - 1) && in_range) { + in_range = false; + end = scno_xcall_enable ? i : i - 1; + if (i == start + 1) + seq_printf(m, "%u,", start); + else + seq_printf(m, "%u-%u,", start, end); + } + } + seq_puts(m, "\n"); +} +#endif + +static int xcall_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct task_struct *p; + unsigned int rs, re; + struct xcall_info *xinfo; + + if (!static_branch_unlikely(&has_hwcap_actlr_xcall) && + !static_branch_unlikely(&xcall_enable)) + return -EACCES; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + +#ifdef CONFIG_ACTLR_XCALL_XINT + if (static_branch_unlikely(&has_hwcap_actlr_xcall)) { + hw_xcall_show(p, m); + goto out; + } +#endif + + xinfo = TASK_XINFO(p); + if (!xinfo) + goto out; + + for (rs = 0, bitmap_next_set_region(xinfo->xcall_enable, &rs, &re, __NR_syscalls); + rs < re; rs = re + 1, + bitmap_next_set_region(xinfo->xcall_enable, &rs, &re, __NR_syscalls)) { + if (rs == (re - 1)) + seq_printf(m, "%d,", rs); + else + seq_printf(m, "%d-%d,", rs, re - 1); + } + seq_puts(m, "\n"); +out: + put_task_struct(p); + + return 0; +} + +static int xcall_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, xcall_show, inode); +} + +static int xcall_enable_one(struct xcall_info *xinfo, unsigned int sc_no) +{ + test_and_set_bit(sc_no, xinfo->xcall_enable); + return 0; +} + +static int xcall_disable_one(struct xcall_info *xinfo, unsigned int sc_no) +{ + test_and_clear_bit(sc_no, xinfo->xcall_enable); + return 0; +} + +#ifdef CONFIG_ACTLR_XCALL_XINT +static int set_hw_xcall(struct task_struct *p, unsigned int sc_no, bool is_clear) +{ + struct hw_xcall_info *hw_xinfo = TASK_HW_XINFO(p); + + if (!is_clear && !is_xcall_entry(hw_xinfo, sc_no)) + return set_xcall_entry(hw_xinfo, sc_no); + + if (is_clear && is_xcall_entry(hw_xinfo, sc_no)) + return set_no_xcall_entry(hw_xinfo, sc_no); + + return -EINVAL; +} +#endif + +static ssize_t xcall_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct inode *inode = file_inode(file); + struct task_struct *p; + char buffer[5]; + const size_t maxlen = sizeof(buffer) - 1; + unsigned int sc_no = __NR_syscalls; + int ret = 0; + int is_clear = 0; + struct xcall_info *xinfo; + + if (!static_branch_unlikely(&has_hwcap_actlr_xcall) && + !static_branch_unlikely(&xcall_enable)) + return -EACCES; + + memset(buffer, 0, sizeof(buffer)); + if (!count || copy_from_user(buffer, buf, count > maxlen ? maxlen : count)) + return -EFAULT; + + p = get_proc_task(inode); + if (!p || !p->xinfo) + return -ESRCH; + + if (buffer[0] == '!') + is_clear = 1; + + if (kstrtouint(buffer + is_clear, 10, &sc_no)) { + ret = -EINVAL; + goto out; + } + + if (sc_no >= __NR_syscalls) { + ret = -EINVAL; + goto out; + } + +#ifdef CONFIG_ACTLR_XCALL_XINT + if (static_branch_unlikely(&has_hwcap_actlr_xcall)) { + ret = set_hw_xcall(p, sc_no, is_clear); + goto out; + } +#endif + + xinfo = TASK_XINFO(p); + if (!is_clear && !test_bit(sc_no, xinfo->xcall_enable)) + ret = xcall_enable_one(xinfo, sc_no); + else if (is_clear && test_bit(sc_no, xinfo->xcall_enable)) + ret = xcall_disable_one(xinfo, sc_no); + else + ret = -EINVAL; + +out: + put_task_struct(p); + + return ret ? ret : count; +} + +const struct file_operations proc_pid_xcall_operations = { + .open = xcall_open, + .read = seq_read, + .write = xcall_write, + .llseek = seq_lseek, + .release = single_release, +}; diff --git a/include/linux/sched.h b/include/linux/sched.h index 3979c34e9b83d09343468dd5fa406de289098875..b6bc8d72309af4551faa7bff8c221d8cd2010ab1 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1615,7 +1615,7 @@ struct task_struct { randomized_struct_fields_end #if defined(CONFIG_FAST_SYSCALL) - KABI_USE(1, unsigned long *xcall_enable) + KABI_USE(1, void *xinfo) #else KABI_RESERVE(1) #endif diff --git a/kernel/fork.c b/kernel/fork.c index 96c6a9e446ac01de782450b563ba52cc3bc794b3..7f7c297d5f48f0ea804e6137f58c1f57cf073308 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -110,6 +110,9 @@ #include #include #include +#ifdef CONFIG_FAST_SYSCALL +#include +#endif #include @@ -639,8 +642,7 @@ void free_task(struct task_struct *tsk) sched_grid_qos_free(tsk); #endif #ifdef CONFIG_FAST_SYSCALL - if (tsk->xcall_enable) - bitmap_free(tsk->xcall_enable); + xcall_task_free(tsk); #endif free_task_struct(tsk); } @@ -1273,7 +1275,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #endif #ifdef CONFIG_FAST_SYSCALL - tsk->xcall_enable = NULL; + tsk->xinfo = NULL; #endif return tsk; @@ -2442,12 +2444,9 @@ __latent_entropy struct task_struct *copy_process( rt_mutex_init_task(p); #ifdef CONFIG_FAST_SYSCALL - p->xcall_enable = bitmap_zalloc(__NR_syscalls, GFP_KERNEL); - if (!p->xcall_enable) + retval = xcall_init_task(p, current); + if (retval) goto bad_fork_free; - - if (current->xcall_enable) - bitmap_copy(p->xcall_enable, current->xcall_enable, __NR_syscalls); #endif #ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY