diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 7c546c3487c9fccbc65fc911f6ae8402c9f24c6d..b5d112cc2197f85cd8697d35488ac62d5d28ea33 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -40,6 +40,7 @@ #include #include #include +#include /* * TASK_SIZE - the maximum size of a user space task. @@ -125,6 +126,9 @@ struct cpu_context { unsigned long fp; unsigned long sp; unsigned long pc; +#ifdef CONFIG_ARM64_PTR_AUTH + unsigned long pac_hash; +#endif }; struct thread_struct { @@ -195,9 +199,11 @@ void tls_preserve_current_state(void); static inline void start_thread_common(struct pt_regs *regs, unsigned long pc) { s32 previous_syscall = regs->syscallno; + sign_exception_context_start(regs); memset(regs, 0, sizeof(*regs)); - regs->syscallno = previous_syscall; regs->pc = pc; + sign_exception_context_end(regs); + regs->syscallno = previous_syscall; if (system_uses_irq_prio_masking()) regs->pmr_save = GIC_PRIO_IRQON; @@ -207,26 +213,45 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { start_thread_common(regs, pc); +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, PSR_MODE_EL0t); +#else regs->pstate = PSR_MODE_EL0t; +#endif spectre_v4_enable_task_mitigation(current); +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_SP, sp); +#else regs->sp = sp; +#endif } #ifdef CONFIG_COMPAT static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { + u64 pstate; + start_thread_common(regs, pc); - regs->pstate = PSR_AA32_MODE_USR; + pstate = PSR_AA32_MODE_USR; if (pc & 1) - regs->pstate |= PSR_AA32_T_BIT; + pstate |= PSR_AA32_T_BIT; #ifdef __AARCH64EB__ - regs->pstate |= PSR_AA32_E_BIT; + pstate |= PSR_AA32_E_BIT; #endif +#ifdef CONFIG_ARM64_PTR_AUTH + set_compat_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif spectre_v4_enable_task_mitigation(current); +#ifdef CONFIG_ARM64_PTR_AUTH + set_compat_exception_context_register(regs, REGS_SP, sp); +#else regs->compat_sp = sp; +#endif } #endif diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index d3106f5e121f9c746bdb6de59181da40a3ad6a9f..c65f26bd5bed0c03dc57b62671f9e1bdb0c0df58 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -188,6 +188,9 @@ struct pt_regs { s32 syscallno; u32 unused2; #endif +#ifdef CONFIG_ARM64_PTR_AUTH + u64 pac_hash; +#endif u64 orig_addr_limit; /* Only valid when ARM64_HAS_IRQ_PRIO_MASKING is enabled. */ @@ -252,6 +255,13 @@ extern int regs_query_register_offset(const char *name); extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n); +/* + * Not at the top of the file due to a direct #include cycle between + * and . Deferring this #include + * ensures that contents of ptrace.h are visible to pointer_auth_context.h. + */ +#include + /** * regs_get_register() - get register value from its offset * @regs: pt_regs from which register value is gotten @@ -304,8 +314,13 @@ static inline unsigned long pt_regs_read_reg(const struct pt_regs *regs, int r) static inline void pt_regs_write_reg(struct pt_regs *regs, int r, unsigned long val) { - if (r != 31) + if (r != 31) { +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register_index(regs, r, val); +#else regs->regs[r] = val; +#endif + } } /* Valid only for Kernel mode traps. */ @@ -367,7 +382,11 @@ static inline unsigned long instruction_pointer(struct pt_regs *regs) static inline void instruction_pointer_set(struct pt_regs *regs, unsigned long val) { +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PC, val); +#else regs->pc = val; +#endif } static inline unsigned long frame_pointer(struct pt_regs *regs) @@ -380,7 +399,11 @@ static inline unsigned long frame_pointer(struct pt_regs *regs) static inline void procedure_link_pointer_set(struct pt_regs *regs, unsigned long val) { +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register_index(regs, 30, val); +#else procedure_link_pointer(regs) = val; +#endif } extern unsigned long profile_pc(struct pt_regs *regs); diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 34ef70877de45abb1e2e9d9a93ce7e54b41df08d..f4bbe3c7dc59d973f6ce1d96c548de4ee012f306 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -44,9 +44,12 @@ int main(void) #endif BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context)); + DEFINE(CPU_CONTEXT_SP, offsetof(struct cpu_context, sp)); + DEFINE(CPU_CONTEXT_PC, offsetof(struct cpu_context, pc)); #ifdef CONFIG_ARM64_PTR_AUTH DEFINE(THREAD_KEYS_USER, offsetof(struct task_struct, thread.keys_user)); DEFINE(THREAD_KEYS_KERNEL, offsetof(struct task_struct, thread.keys_kernel)); + DEFINE(CPU_CONTEXT_PAC_HASH, offsetof(struct cpu_context, pac_hash)); #endif BLANK(); DEFINE(S_X0, offsetof(struct pt_regs, regs[0])); @@ -56,8 +59,10 @@ int main(void) DEFINE(S_X8, offsetof(struct pt_regs, regs[8])); DEFINE(S_X10, offsetof(struct pt_regs, regs[10])); DEFINE(S_X12, offsetof(struct pt_regs, regs[12])); + DEFINE(S_X13, offsetof(struct pt_regs, regs[13])); DEFINE(S_X14, offsetof(struct pt_regs, regs[14])); DEFINE(S_X16, offsetof(struct pt_regs, regs[16])); + DEFINE(S_X17, offsetof(struct pt_regs, regs[17])); DEFINE(S_X18, offsetof(struct pt_regs, regs[18])); DEFINE(S_X20, offsetof(struct pt_regs, regs[20])); DEFINE(S_X22, offsetof(struct pt_regs, regs[22])); @@ -74,8 +79,13 @@ int main(void) DEFINE(S_PMR_SAVE, offsetof(struct pt_regs, pmr_save)); DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe)); DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs)); +#ifdef CONFIG_ARM64_PTR_AUTH + DEFINE(S_PAC_HASH, offsetof(struct pt_regs, pac_hash)); +#endif BLANK(); #ifdef CONFIG_COMPAT + DEFINE(S_COMPAT_SP, offsetof(struct pt_regs, regs[13])); + DEFINE(S_COMPAT_LR, offsetof(struct pt_regs, regs[14])); DEFINE(COMPAT_SIGFRAME_REGS_OFFSET, offsetof(struct compat_sigframe, uc.uc_mcontext.arm_r0)); DEFINE(COMPAT_RT_SIGFRAME_REGS_OFFSET, offsetof(struct compat_rt_sigframe, sig.uc.uc_mcontext.arm_r0)); BLANK(); diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index fa76151de6ff148372963f91521af354931fc115..a33a1eecbd850a279ac1056a14ad3b2f1c52c0b7 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -23,6 +23,7 @@ #include #include #include +#include /* Determine debug architecture. */ u8 debug_monitors_arch(void) @@ -143,13 +144,23 @@ postcore_initcall(debug_monitors_init); */ static void set_user_regs_spsr_ss(struct user_pt_regs *regs) { - regs->pstate |= DBG_SPSR_SS; + u64 pstate = regs->pstate | DBG_SPSR_SS; +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif } NOKPROBE_SYMBOL(set_user_regs_spsr_ss); static void clear_user_regs_spsr_ss(struct user_pt_regs *regs) { - regs->pstate &= ~DBG_SPSR_SS; + u64 pstate = regs->pstate & (~DBG_SPSR_SS); +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif } NOKPROBE_SYMBOL(clear_user_regs_spsr_ss); diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index d5bc1dbdd2fda84cd1e6e9508c07be5b41fae793..95e7678c8f932f110ecfea2a6915e9029ad81a79 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -28,6 +28,7 @@ #include #include #include +#include /* * Context tracking and irqflag tracing need to instrument transitions between @@ -201,6 +202,7 @@ alternative_else_nop_endif stp x24, x25, [sp, #16 * 12] stp x26, x27, [sp, #16 * 13] stp x28, x29, [sp, #16 * 14] + sign_pt_regs \el .if \el == 0 clear_gp_regs @@ -335,9 +337,6 @@ alternative_else_nop_endif 3: scs_save tsk, x0 - /* No kernel C function calls after this as user keys are set. */ - ptrauth_keys_install_user tsk, x0, x1, x2 - apply_ssbd 0, x0, x1 .endif @@ -356,6 +355,12 @@ alternative_else_nop_endif ldp x20, x21, [sp, #16 * 10] ldp x22, x23, [sp, #16 * 11] ldp x24, x25, [sp, #16 * 12] + auth_pt_regs \el + + .if \el == 0 + /* No kernel C function calls after this as user keys are set. */ + ptrauth_keys_install_user tsk, x26, x27, x29 + .endif ldp x26, x27, [sp, #16 * 13] ldp x28, x29, [sp, #16 * 14] @@ -1013,6 +1018,7 @@ SYM_FUNC_START(cpu_switch_to) stp x27, x28, [x8], #16 stp x29, x9, [x8], #16 str lr, [x8] + sign_cpu_context add x8, x1, x10 ldp x19, x20, [x8], #16 // restore callee-saved registers ldp x21, x22, [x8], #16 @@ -1021,6 +1027,7 @@ SYM_FUNC_START(cpu_switch_to) ldp x27, x28, [x8], #16 ldp x29, x9, [x8], #16 ldr lr, [x8] + auth_cpu_context mov sp, x9 msr sp_el0, x1 ptrauth_keys_install_kernel x1, x8, x9, x10 diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 798c3e78b84bba5fbb68175fb5e4d2b4743fad8c..dc89a03548d29b6155096c18216995eeb53b32e7 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "decode-insn.h" @@ -166,15 +167,27 @@ static void __kprobes set_current_kprobe(struct kprobe *p) static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, struct pt_regs *regs) { + u64 pstate; kcb->saved_irqflag = regs->pstate & DAIF_MASK; - regs->pstate |= DAIF_MASK; + pstate = regs->pstate | DAIF_MASK; +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif } static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, struct pt_regs *regs) { - regs->pstate &= ~DAIF_MASK; - regs->pstate |= kcb->saved_irqflag; + u64 pstate = regs->pstate; + pstate &= ~DAIF_MASK; + pstate |= kcb->saved_irqflag; +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif } static void __kprobes @@ -454,7 +467,11 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, ri->fp = (void *)kernel_stack_pointer(regs); /* replace return addr (x30) with trampoline */ +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register_index(regs, 30, (long)&kretprobe_trampoline); +#else regs->regs[30] = (long)&kretprobe_trampoline; +#endif } int __kprobes arch_trampoline_kprobe(struct kprobe *p) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 22275d8518eb39fa8bd4287deaad832453cfa0fb..16d0aaff6b80bb26334abbe9c9ec73a4dd2fc333 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -57,6 +57,7 @@ #include #include #include +#include #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include @@ -414,6 +415,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, else childregs->sp = stack_start; } + sign_exception_context(childregs); /* * If a TLS pointer was passed to clone, use it for the new @@ -428,6 +430,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, cpus_have_const_cap(ARM64_HAS_UAO)) childregs->pstate |= PSR_UAO_BIT; + sign_exception_context(childregs); spectre_v4_enable_task_mitigation(p); if (system_uses_irq_prio_masking()) @@ -438,6 +441,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, } p->thread.cpu_context.pc = (unsigned long)ret_from_fork; p->thread.cpu_context.sp = (unsigned long)childregs; + sign_thread_context(&p->thread.cpu_context); ptrace_hw_copy_thread(p); diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 0efc58f5f1bb3299bf8f77b22f8fe05e20afda71..8d09bac3b0530c48a9baedb3b748deb7f8780be1 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -30,6 +30,7 @@ #include #include #include +#include /* * We try to ensure that the mitigation state can never change as the result of @@ -539,13 +540,21 @@ bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope) static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr) { + u64 pstate = regs->pstate; + if (user_mode(regs)) return 1; if (instr & BIT(PSTATE_Imm_shift)) - regs->pstate |= PSR_SSBS_BIT; + pstate |= PSR_SSBS_BIT; else - regs->pstate &= ~PSR_SSBS_BIT; + pstate &= ~PSR_SSBS_BIT; + +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif arm64_skip_faulting_instruction(regs, 4); return 0; @@ -672,11 +681,18 @@ void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused) static void __update_pstate_ssbs(struct pt_regs *regs, bool state) { u64 bit = compat_user_mode(regs) ? PSR_AA32_SSBS_BIT : PSR_SSBS_BIT; + u64 pstate; - if (state) - regs->pstate |= bit; - else - regs->pstate &= ~bit; + if (state) { + pstate = regs->pstate | bit; + } else { + pstate = regs->pstate & (~bit); + } +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif } void spectre_v4_enable_task_mitigation(struct task_struct *tsk) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 2817e39881fee304b39c9ff9ef8566f7541d8de7..32770d9fba52023ec23d1585a343751f2479d2e9 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -40,6 +40,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -1239,6 +1240,7 @@ static int compat_gpr_set(struct task_struct *target, struct pt_regs newregs; int ret = 0; unsigned int i, start, num_regs; + struct pt_regs *regs = task_pt_regs(target); /* Calculate the number of AArch32 registers contained in count */ num_regs = count / regset->size; @@ -1249,7 +1251,7 @@ static int compat_gpr_set(struct task_struct *target, if (start + num_regs > regset->n) return -EIO; - newregs = *task_pt_regs(target); + newregs = *regs; for (i = 0; i < num_regs; ++i) { unsigned int idx = start + i; @@ -1285,10 +1287,14 @@ static int compat_gpr_set(struct task_struct *target, } - if (valid_user_regs(&newregs.user_regs, target)) - *task_pt_regs(target) = newregs; - else + sign_compat_exception_context(&newregs.user_regs); + if (valid_user_regs(&newregs.user_regs, target)) { + resign_compat_exception_context_start(regs); + *regs = newregs; + resign_compat_exception_context_end(regs); + } else { ret = -EINVAL; + } return ret; } @@ -1482,7 +1488,8 @@ static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off, static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off, compat_ulong_t val) { - struct pt_regs newregs = *task_pt_regs(tsk); + struct pt_regs *regs = task_pt_regs(tsk); + struct pt_regs newregs = *regs; unsigned int idx = off / 4; if (off & 3 || off >= COMPAT_USER_SZ) @@ -1505,10 +1512,13 @@ static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off, newregs.regs[idx] = val; } + sign_compat_exception_context(&newregs.user_regs); if (!valid_user_regs(&newregs.user_regs, tsk)) return -EINVAL; - *task_pt_regs(tsk) = newregs; + resign_compat_exception_context_start(regs); + *regs = newregs; + resign_compat_exception_context_end(regs); return 0; } @@ -1849,19 +1859,25 @@ void syscall_trace_exit(struct pt_regs *regs) static int valid_compat_regs(struct user_pt_regs *regs) { - regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS; + u64 pstate = regs->pstate; + pstate &= ~SPSR_EL1_AARCH32_RES0_BITS; if (!system_supports_mixed_endian_el0()) { if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) - regs->pstate |= PSR_AA32_E_BIT; + pstate |= PSR_AA32_E_BIT; else - regs->pstate &= ~PSR_AA32_E_BIT; + pstate &= ~PSR_AA32_E_BIT; } - if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) && - (regs->pstate & PSR_AA32_A_BIT) == 0 && - (regs->pstate & PSR_AA32_I_BIT) == 0 && - (regs->pstate & PSR_AA32_F_BIT) == 0) { + if (user_mode(regs) && (pstate & PSR_MODE32_BIT) && + (pstate & PSR_AA32_A_BIT) == 0 && + (pstate & PSR_AA32_I_BIT) == 0 && + (pstate & PSR_AA32_F_BIT) == 0) { +#ifdef CONFIG_ARM64_PTR_AUTH + set_compat_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif return 1; } @@ -1869,30 +1885,46 @@ static int valid_compat_regs(struct user_pt_regs *regs) * Force PSR to a valid 32-bit EL0t, preserving the same bits as * arch/arm. */ - regs->pstate &= PSR_AA32_N_BIT | PSR_AA32_Z_BIT | + pstate &= PSR_AA32_N_BIT | PSR_AA32_Z_BIT | PSR_AA32_C_BIT | PSR_AA32_V_BIT | PSR_AA32_Q_BIT | PSR_AA32_IT_MASK | PSR_AA32_GE_MASK | PSR_AA32_E_BIT | PSR_AA32_T_BIT; - regs->pstate |= PSR_MODE32_BIT; + pstate |= PSR_MODE32_BIT; +#ifdef CONFIG_ARM64_PTR_AUTH + set_compat_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif return 0; } static int valid_native_regs(struct user_pt_regs *regs) { - regs->pstate &= ~SPSR_EL1_AARCH64_RES0_BITS; + u64 pstate = regs->pstate; + pstate &= ~SPSR_EL1_AARCH64_RES0_BITS; - if (user_mode(regs) && !(regs->pstate & PSR_MODE32_BIT) && - (regs->pstate & PSR_D_BIT) == 0 && - (regs->pstate & PSR_A_BIT) == 0 && - (regs->pstate & PSR_I_BIT) == 0 && - (regs->pstate & PSR_F_BIT) == 0) { + if (user_mode(regs) && !(pstate & PSR_MODE32_BIT) && + (pstate & PSR_D_BIT) == 0 && + (pstate & PSR_A_BIT) == 0 && + (pstate & PSR_I_BIT) == 0 && + (pstate & PSR_F_BIT) == 0) { +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif return 1; } /* Force PSR to a valid 64-bit EL0t */ - regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT; + pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT; +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif return 0; } diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 0dab5679a97d58ab746868c9dc8cdfdc9731fc73..a92d0963efb4020f89c21ef38d79597c5aa1bdf2 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -33,6 +33,7 @@ #include #include #include +#include /* * Do a signal return; undo the signal stack. These are aligned to 128-bit. @@ -494,12 +495,14 @@ static int restore_sigframe(struct pt_regs *regs, if (err == 0) set_current_blocked(&set); + resign_exception_context_start(regs); for (i = 0; i < 31; i++) __get_user_error(regs->regs[i], &sf->uc.uc_mcontext.regs[i], err); __get_user_error(regs->sp, &sf->uc.uc_mcontext.sp, err); __get_user_error(regs->pc, &sf->uc.uc_mcontext.pc, err); __get_user_error(regs->pstate, &sf->uc.uc_mcontext.pstate, err); + resign_exception_context_end(regs); /* * Avoid sys_rt_sigreturn() restarting. @@ -731,6 +734,7 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka, { __sigrestore_t sigtramp; + resign_exception_context_start(regs); regs->regs[0] = usig; regs->sp = (unsigned long)user->sigframe; regs->regs[29] = (unsigned long)&user->next_frame->fp; @@ -761,6 +765,7 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka, sigtramp = VDSO_SYMBOL(current->mm->context.vdso, sigtramp); regs->regs[30] = (unsigned long)sigtramp; + resign_exception_context_end(regs); } static int setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set, @@ -873,7 +878,11 @@ static void do_signal(struct pt_regs *regs) case -ERESTARTNOINTR: case -ERESTART_RESTARTBLOCK: regs->regs[0] = regs->orig_x0; +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PC, restart_addr); +#else regs->pc = restart_addr; +#endif break; } } @@ -894,7 +903,11 @@ static void do_signal(struct pt_regs *regs) (retval == -ERESTARTSYS && !(ksig.ka.sa.sa_flags & SA_RESTART)))) { syscall_set_return_value(current, regs, -EINTR, 0); +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PC, continue_addr); +#else regs->pc = continue_addr; +#endif } handle_signal(&ksig, regs); diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c index 2f507f565c48ae0fe8a27c3cf2b8e286e038e940..977a50a39255fe0fcb5961382dc8962ab694e4e4 100644 --- a/arch/arm64/kernel/signal32.c +++ b/arch/arm64/kernel/signal32.c @@ -19,6 +19,7 @@ #include #include #include +#include struct compat_vfp_sigframe { compat_ulong_t magic; @@ -195,6 +196,7 @@ static int compat_restore_sigframe(struct pt_regs *regs, set_current_blocked(&set); } + resign_compat_exception_context_start(regs); __get_user_error(regs->regs[0], &sf->uc.uc_mcontext.arm_r0, err); __get_user_error(regs->regs[1], &sf->uc.uc_mcontext.arm_r1, err); __get_user_error(regs->regs[2], &sf->uc.uc_mcontext.arm_r2, err); @@ -214,6 +216,7 @@ static int compat_restore_sigframe(struct pt_regs *regs, __get_user_error(psr, &sf->uc.uc_mcontext.arm_cpsr, err); regs->pstate = compat_psr_to_pstate(psr); + resign_compat_exception_context_end(regs); /* * Avoid compat_sys_sigreturn() restarting. @@ -351,11 +354,13 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka, (idx << 2) + thumb; } + resign_compat_exception_context_start(regs); regs->regs[0] = usig; regs->compat_sp = ptr_to_compat(frame); regs->compat_lr = retcode; regs->pc = handler; regs->pstate = spsr; + resign_compat_exception_context_end(regs); } static int compat_setup_sigframe(struct compat_sigframe __user *sf, diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 2059d8f43f55f05d9bd93ca3515043f4ae0dcf44..6ff517027a8355df13ccdfba75d84d863b5e4a46 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -45,6 +45,7 @@ #include #include #include +#include static const char *handler[]= { "Synchronous Abort", @@ -267,6 +268,7 @@ static void advance_itstate(struct pt_regs *regs) void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size) { + resign_exception_context_start(regs); regs->pc += size; /* @@ -280,6 +282,7 @@ void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size) advance_itstate(regs); else regs->pstate &= ~PSR_BTYPE_MASK; + resign_exception_context_end(regs); } static LIST_HEAD(undef_hook); diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c index aa0060178343a88a4d25f362ec8b0db2d2bda878..d767f63e5a9294bf48d9b662dbc6e1d499dd2c99 100644 --- a/arch/arm64/mm/extable.c +++ b/arch/arm64/mm/extable.c @@ -5,10 +5,12 @@ #include #include +#include int fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *fixup; + u64 pc; fixup = search_exception_tables(instruction_pointer(regs)); if (!fixup) @@ -17,6 +19,11 @@ int fixup_exception(struct pt_regs *regs) if (in_bpf_jit(regs)) return arm64_bpf_fixup_exception(fixup, regs); - regs->pc = (unsigned long)&fixup->fixup + fixup->fixup; + pc = (unsigned long)&fixup->fixup + fixup->fixup; +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PC, pc); +#else + regs->pc = pc; +#endif return 1; } diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 795d224f184ff2ce63d8427f5e70d4a9a89fd4ef..8fe48c2bbccac0064e5534e5e3c2ca68c8aa0cfb 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -38,6 +38,7 @@ #include #include #include +#include struct fault_info { int (*fn)(unsigned long addr, unsigned int esr, @@ -807,6 +808,7 @@ DECLARE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) { + u64 pstate; if (user_mode(regs)) return 0; @@ -820,7 +822,12 @@ static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) * masked so that we can safely restore the mdscr and get on with * handling the syscall. */ - regs->pstate |= PSR_D_BIT; + pstate = regs->pstate | PSR_D_BIT; +#ifdef CONFIG_ARM64_PTR_AUTH + set_exception_context_register(regs, REGS_PSTATE, pstate); +#else + regs->pstate = pstate; +#endif return 1; } #else diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 18627cbd6da4ef45a90a549fd4f06a0fbce8dc6f..db76775032422668419d15283d98150f142d194d 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "bpf_jit.h" @@ -365,8 +366,10 @@ int arm64_bpf_fixup_exception(const struct exception_table_entry *ex, off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); + resign_exception_context_start(regs); regs->regs[dst_reg] = 0; regs->pc = (unsigned long)&ex->fixup - offset; + resign_exception_context_end(regs); return 1; } diff --git a/drivers/misc/lkdtm/Makefile b/drivers/misc/lkdtm/Makefile index 0d768a13e2bb705ff645df6cf1e3d1c7a0055726..2cb0933b667761dcd6e971dd81e84af7950d7ae8 100644 --- a/drivers/misc/lkdtm/Makefile +++ b/drivers/misc/lkdtm/Makefile @@ -10,6 +10,7 @@ lkdtm-$(CONFIG_LKDTM) += rodata_objcopy.o lkdtm-$(CONFIG_LKDTM) += usercopy.o lkdtm-$(CONFIG_LKDTM) += stackleak.o lkdtm-$(CONFIG_LKDTM) += cfi.o +lkdtm-$(CONFIG_LKDTM) += pac.o KASAN_SANITIZE_stackleak.o := n KCOV_INSTRUMENT_rodata.o := n diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c index 32b3d77368e37d734ad541e1cda80997fa733345..a478626c1a18d43883b4a83f056a4ea67e456bac 100644 --- a/drivers/misc/lkdtm/core.c +++ b/drivers/misc/lkdtm/core.c @@ -174,6 +174,11 @@ static const struct crashtype crashtypes[] = { CRASHTYPE(STACKLEAK_ERASING), CRASHTYPE(CFI_FORWARD_PROTO), CRASHTYPE(DOUBLE_FAULT), + CRASHTYPE(AUTH_AND_SIGN_CPU_CONTEXT), + CRASHTYPE(SET_EXCEPTION_CONTEXT_REGISTER), + CRASHTYPE(AUTH_AND_SIGN_EXCEPTION_CONTEXT), + CRASHTYPE(SET_COMPAT_EXCEPTION_CONTEXT_REGISTER), + CRASHTYPE(AUTH_AND_SIGN_COMPAT_EXCEPTION_CONTEXT), }; diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h index 6dec4c9b442ff34e9c516e54ee8d4706af7ad0bd..cb77b6800ef9f0e7b0aba39a1527ad33b55323f1 100644 --- a/drivers/misc/lkdtm/lkdtm.h +++ b/drivers/misc/lkdtm/lkdtm.h @@ -102,4 +102,11 @@ void lkdtm_STACKLEAK_ERASING(void); /* cfi.c */ void lkdtm_CFI_FORWARD_PROTO(void); +/* pac.c */ +void lkdtm_AUTH_AND_SIGN_CPU_CONTEXT(void); +void lkdtm_SET_EXCEPTION_CONTEXT_REGISTER(void); +void lkdtm_AUTH_AND_SIGN_EXCEPTION_CONTEXT(void); +void lkdtm_SET_COMPAT_EXCEPTION_CONTEXT_REGISTER(void); +void lkdtm_AUTH_AND_SIGN_COMPAT_EXCEPTION_CONTEXT(void); + #endif diff --git a/drivers/misc/lkdtm/pac.c b/drivers/misc/lkdtm/pac.c new file mode 100644 index 0000000000000000000000000000000000000000..8cc8fcb54c5efb971ff38e0431153d19b04418e0 --- /dev/null +++ b/drivers/misc/lkdtm/pac.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This is for all the tests relating directly to Control Flow Integrity. + */ +#include "lkdtm.h" + +#include +#include +#include +#include +#include +#include + +void lkdtm_AUTH_AND_SIGN_CPU_CONTEXT(void) +{ + struct task_struct *p = kmalloc(sizeof(struct task_struct), GFP_KERNEL); + if (p == NULL) { + pr_info("No memory...\n"); + return; + } + p->thread.cpu_context.sp = 0xffff800012412346; + p->thread.cpu_context.pc = 0xffff800012789012; + + pr_info("Sign cpu_context...\n"); + sign_thread_context(&p->thread.cpu_context); + + pr_info("Check cpu_context...\n"); + auth_thread_context(&p->thread.cpu_context); + + pr_info("Modify sp...\n"); + p->thread.cpu_context.sp = 0xffff800012412412; + + pr_info("Resign cpu_context...\n"); + sign_thread_context(&p->thread.cpu_context); + + pr_info("Recheck cpu_context...\n"); + auth_thread_context(&p->thread.cpu_context); + pr_info("Sucess to sign and check cpu_context!\n"); + + pr_info("Modify pc...\n"); + p->thread.cpu_context.pc = 0xffff800012111412; + + pr_info("Resign cpu_context...\n"); + sign_thread_context(&p->thread.cpu_context); + + pr_info("Recheck cpu_context...\n"); + auth_thread_context(&p->thread.cpu_context); + pr_info("Sucess to sign and check cpu_context!\n"); + + pr_info("Test that sp of cpu_context is modefied will result in panic!\n"); + p->thread.cpu_context.sp = 0xff8000189012; + auth_thread_context(&p->thread.cpu_context); + + pr_info("Test that pc of cpu_context is modefied will result in panic!\n"); + sign_thread_context(&p->thread.cpu_context); + p->thread.cpu_context.pc = 0xff8111189012; + auth_thread_context(&p->thread.cpu_context); + + pr_info("Test successful. There are 2 panics here!\n"); +} + +void lkdtm_SET_EXCEPTION_CONTEXT_REGISTER(void) +{ + int ret; + struct pt_regs regs; + regs.regs[16] = 0xffff800012423412; + regs.regs[17] = 0xffff800012412412; + regs.regs[30] = 0xffff800078912456; + regs.sp = 0xffff800011111012; + regs.pc = 0xffff800012789012; + regs.pstate = 0x12412412; + + pr_info("Sign regs...\n"); + sign_exception_context(®s); + + pr_info("Modify x16, check and resign regs...\n"); + (void)set_exception_context_register(®s, REGS_X16, 0x1234432); + regs.regs[16] = 0x1234432; + pr_info("Check regs...\n"); + auth_exception_context(®s); + + pr_info("Modify x17, check and resign regs...\n"); + (void)set_exception_context_register(®s, REGS_X17, 0x123965412); + regs.regs[17] = 0x123965412; + pr_info("Check regs...\n"); + auth_exception_context(®s); + + pr_info("Modify x30, check and resign regs...\n"); + (void)set_exception_context_register(®s, REGS_LR, 0x9586482362); + regs.regs[30] = 0x9586482362; + pr_info("Check regs...\n"); + auth_exception_context(®s); + + pr_info("Modify SP, check and resign regs...\n"); + (void)set_exception_context_register(®s, REGS_SP, 0x9586482111); + regs.sp = 0x9586482111; + pr_info("Check regs...\n"); + auth_exception_context(®s); + + pr_info("Modify PC, check and resign regs...\n"); + (void)set_exception_context_register(®s, REGS_PC, 0x136541298562); + regs.pc = 0x136541298562; + pr_info("Check regs...\n"); + auth_exception_context(®s); + + pr_info("Modify PSTATE, check and resign regs...\n"); + (void)set_exception_context_register(®s, REGS_PSTATE, 0x65946853652); + regs.pstate = 0x65946853652; + pr_info("Check regs...\n"); + auth_exception_context(®s); + + pr_info("Modify x16, check and resign regs...\n"); + set_exception_context_register_index(®s, 16, 0x1234432); + regs.regs[16] = 0x1234432; + pr_info("Check regs...\n"); + auth_exception_context(®s); + + pr_info("Modify x17, check and resign regs...\n"); + set_exception_context_register_index(®s, 17, 0x123965412); + regs.regs[17] = 0x123965412; + pr_info("Check regs...\n"); + auth_exception_context(®s); + + pr_info("Modify x30, check and resign regs...\n"); + set_exception_context_register_index(®s, 30, 0x9586482362); + regs.regs[30] = 0x9586482362; + pr_info("Check regs...\n"); + auth_exception_context(®s); + + pr_info("Test invalid pamameter...\n"); + ret = set_exception_context_register(®s, REGS_X16 - 8, 0x1234432); + if (ret == -EINVAL) { + pr_info("The 2nd parameter: %d is invalid...\n", REGS_X16 - 8); + } + + ret = set_exception_context_register(®s, REGS_PSTATE + 8, 0x1234432); + if (ret == -EINVAL) { + pr_info("The 2nd parameter: %d is invalid...\n", REGS_PSTATE + 8); + } + + pr_info("Test that x16 of pt_regs is modefied will be panic!\n"); + regs.regs[16] = 0x8000000ddd; + auth_exception_context(®s); + + pr_info("Test that x17 of pt_regs is modefied will be panic!\n"); + sign_exception_context(®s); + regs.regs[17] = 0x8000000ddd; + auth_exception_context(®s); + + pr_info("Test that x30 of pt_regs is modefied will be panic!\n"); + sign_exception_context(®s); + regs.regs[30] = 0x8000000ddd; + auth_exception_context(®s); + + pr_info("Test that sp of pt_regs is modefied will be panic!\n"); + sign_exception_context(®s); + regs.sp = 0x8000000ddd; + auth_exception_context(®s); + + pr_info("Test that pc of pt_regs is modefied will be panic!\n"); + sign_exception_context(®s); + regs.pc = 0x8000000ddd; + auth_exception_context(®s); + + pr_info("Test that pstate of pt_regs is modefied will be panic!\n"); + sign_exception_context(®s); + regs.pstate = 0x8000000ddd; + auth_exception_context(®s); + + pr_info("Test successful. There are 6 panics here!\n"); +} + +void lkdtm_AUTH_AND_SIGN_EXCEPTION_CONTEXT(void) +{ + struct pt_regs regs1; + struct pt_regs regs2; + regs1.regs[16] = 0xffff800012423412; + regs1.regs[17] = 0xffff800012412412; + regs1.regs[30] = 0xffff800078912456; + regs1.sp = 0xffff800011119012; + regs1.pc = 0xffff800012789012; + regs1.pstate = 0x12412412; + + pr_info("Sign regs1...\n"); + sign_exception_context(®s1); + + memcpy(®s2, ®s1, sizeof(struct pt_regs)); + regs2.pc = 0x8000000ddd; + + pr_info("Check regs1 and sign regs2...\n"); + auth_exception_context(®s1); + sign_exception_context(®s2); + + pr_info("Check regs2...\n"); + auth_exception_context(®s2); + + pr_info("Sucess to check and sign!\n"); + + pr_info("Test that one parameter of pt_regs is modefied will result in panic!\n"); + regs2.pc = 0x8000000dd; + auth_exception_context(®s2); +} + +void lkdtm_SET_COMPAT_EXCEPTION_CONTEXT_REGISTER(void) +{ + int ret; + struct pt_regs regs; + regs.compat_lr = 0xffff800078912456; + regs.compat_sp = 0xffff800011111012; + regs.pc = 0xffff800012789012; + regs.pstate = 0x12412412; + + pr_info("Sign regs...\n"); + sign_compat_exception_context(®s); + + pr_info("Modify compat_lr, check and resign regs...\n"); + (void)set_compat_exception_context_register(®s, REGS_LR, 0x9586482362); + regs.compat_lr = 0x9586482362; + pr_info("Check regs...\n"); + auth_compat_exception_context(®s); + + pr_info("Modify compat_sp, check and resign regs...\n"); + (void)set_compat_exception_context_register(®s, REGS_SP, 0x9586482111); + regs.regs[13] = 0x9586482111; + pr_info("Check regs...\n"); + auth_compat_exception_context(®s); + + pr_info("Modify PC, check and resign regs...\n"); + (void)set_compat_exception_context_register(®s, REGS_PC, 0x136541298562); + regs.pc = 0x136541298562; + pr_info("Check regs...\n"); + auth_compat_exception_context(®s); + + pr_info("Modify PSTATE, check and resign regs...\n"); + (void)set_compat_exception_context_register(®s, REGS_PSTATE, 0x65946853652); + regs.pstate = 0x65946853652; + pr_info("Check regs...\n"); + auth_compat_exception_context(®s); + + pr_info("Modify x14->compat_lr, check and resign regs...\n"); + set_compat_exception_context_register_index(®s, 14, 0x9586482362); + regs.compat_lr = 0x9586482362; + pr_info("Check regs...\n"); + auth_compat_exception_context(®s); + + pr_info("Test invalid pamameter...\n"); + ret = set_compat_exception_context_register(®s, REGS_PC + 8, 0x1234432); + if (ret == -EINVAL) { + pr_info("The 2nd parameter: %d is invalid...\n", REGS_PC + 8); + } + + ret = set_compat_exception_context_register(®s, REGS_PSTATE + 8, 0x1234432); + if (ret == -EINVAL) { + pr_info("The 2nd parameter: %d is invalid...\n", REGS_PSTATE + 8); + } + + pr_info("Test that compat_lr of pt_regs is modefied will be panic!\n"); + regs.compat_lr = 0x8000000ddd; + auth_compat_exception_context(®s); + + pr_info("Test that compat_sp of pt_regs is modefied will be panic!\n"); + sign_compat_exception_context(®s); + regs.compat_sp = 0x8000000ddd; + auth_compat_exception_context(®s); + + pr_info("Test that pc of pt_regs is modefied will be panic!\n"); + sign_compat_exception_context(®s); + regs.pc = 0x8000000ddd; + auth_compat_exception_context(®s); + + pr_info("Test that pstate of pt_regs is modefied will be panic!\n"); + sign_compat_exception_context(®s); + regs.pstate = 0x8000000ddd; + auth_compat_exception_context(®s); + + pr_info("Test successful. There are 4 panics here!\n"); +} + +void lkdtm_AUTH_AND_SIGN_COMPAT_EXCEPTION_CONTEXT(void) +{ + struct pt_regs regs1; + struct pt_regs regs2; + regs1.compat_lr = 0xffff800078912456; + regs1.compat_sp = 0xffff800078911111; + regs1.pc = 0xffff800012789012; + regs1.pstate = 0x12412412; + + pr_info("Sign regs1...\n"); + sign_compat_exception_context(®s1); + + memcpy(®s2, ®s1, sizeof(struct pt_regs)); + regs2.compat_lr = 0x8000000ddd; + + pr_info("Check regs1 and sign regs2...\n"); + auth_compat_exception_context(®s1); + sign_compat_exception_context(®s2); + + pr_info("Check regs2...\n"); + auth_compat_exception_context(®s2); + + pr_info("Sucess to check and sign!\n"); + + pr_info("Test that one parameter of pt_regs is modefied will result in panic!\n"); + regs2.regs[13] = 0x8000000dd; + auth_compat_exception_context(®s2); +} +