cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

process.c (19962B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Based on arch/arm/kernel/process.c
      4 *
      5 * Original Copyright (C) 1995  Linus Torvalds
      6 * Copyright (C) 1996-2000 Russell King - Converted to ARM.
      7 * Copyright (C) 2012 ARM Ltd.
      8 */
      9#include <linux/compat.h>
     10#include <linux/efi.h>
     11#include <linux/elf.h>
     12#include <linux/export.h>
     13#include <linux/sched.h>
     14#include <linux/sched/debug.h>
     15#include <linux/sched/task.h>
     16#include <linux/sched/task_stack.h>
     17#include <linux/kernel.h>
     18#include <linux/mman.h>
     19#include <linux/mm.h>
     20#include <linux/nospec.h>
     21#include <linux/stddef.h>
     22#include <linux/sysctl.h>
     23#include <linux/unistd.h>
     24#include <linux/user.h>
     25#include <linux/delay.h>
     26#include <linux/reboot.h>
     27#include <linux/interrupt.h>
     28#include <linux/init.h>
     29#include <linux/cpu.h>
     30#include <linux/elfcore.h>
     31#include <linux/pm.h>
     32#include <linux/tick.h>
     33#include <linux/utsname.h>
     34#include <linux/uaccess.h>
     35#include <linux/random.h>
     36#include <linux/hw_breakpoint.h>
     37#include <linux/personality.h>
     38#include <linux/notifier.h>
     39#include <trace/events/power.h>
     40#include <linux/percpu.h>
     41#include <linux/thread_info.h>
     42#include <linux/prctl.h>
     43#include <linux/stacktrace.h>
     44
     45#include <asm/alternative.h>
     46#include <asm/compat.h>
     47#include <asm/cpufeature.h>
     48#include <asm/cacheflush.h>
     49#include <asm/exec.h>
     50#include <asm/fpsimd.h>
     51#include <asm/mmu_context.h>
     52#include <asm/mte.h>
     53#include <asm/processor.h>
     54#include <asm/pointer_auth.h>
     55#include <asm/stacktrace.h>
     56#include <asm/switch_to.h>
     57#include <asm/system_misc.h>
     58
     59#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
     60#include <linux/stackprotector.h>
     61unsigned long __stack_chk_guard __ro_after_init;
     62EXPORT_SYMBOL(__stack_chk_guard);
     63#endif
     64
     65/*
     66 * Function pointers to optional machine specific functions
     67 */
     68void (*pm_power_off)(void);
     69EXPORT_SYMBOL_GPL(pm_power_off);
     70
     71#ifdef CONFIG_HOTPLUG_CPU
     72void arch_cpu_idle_dead(void)
     73{
     74       cpu_die();
     75}
     76#endif
     77
     78/*
     79 * Called by kexec, immediately prior to machine_kexec().
     80 *
     81 * This must completely disable all secondary CPUs; simply causing those CPUs
     82 * to execute e.g. a RAM-based pin loop is not sufficient. This allows the
     83 * kexec'd kernel to use any and all RAM as it sees fit, without having to
     84 * avoid any code or data used by any SW CPU pin loop. The CPU hotplug
     85 * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this.
     86 */
     87void machine_shutdown(void)
     88{
     89	smp_shutdown_nonboot_cpus(reboot_cpu);
     90}
     91
     92/*
     93 * Halting simply requires that the secondary CPUs stop performing any
     94 * activity (executing tasks, handling interrupts). smp_send_stop()
     95 * achieves this.
     96 */
     97void machine_halt(void)
     98{
     99	local_irq_disable();
    100	smp_send_stop();
    101	while (1);
    102}
    103
    104/*
    105 * Power-off simply requires that the secondary CPUs stop performing any
    106 * activity (executing tasks, handling interrupts). smp_send_stop()
    107 * achieves this. When the system power is turned off, it will take all CPUs
    108 * with it.
    109 */
    110void machine_power_off(void)
    111{
    112	local_irq_disable();
    113	smp_send_stop();
    114	do_kernel_power_off();
    115}
    116
    117/*
    118 * Restart requires that the secondary CPUs stop performing any activity
    119 * while the primary CPU resets the system. Systems with multiple CPUs must
    120 * provide a HW restart implementation, to ensure that all CPUs reset at once.
    121 * This is required so that any code running after reset on the primary CPU
    122 * doesn't have to co-ordinate with other CPUs to ensure they aren't still
    123 * executing pre-reset code, and using RAM that the primary CPU's code wishes
    124 * to use. Implementing such co-ordination would be essentially impossible.
    125 */
    126void machine_restart(char *cmd)
    127{
    128	/* Disable interrupts first */
    129	local_irq_disable();
    130	smp_send_stop();
    131
    132	/*
    133	 * UpdateCapsule() depends on the system being reset via
    134	 * ResetSystem().
    135	 */
    136	if (efi_enabled(EFI_RUNTIME_SERVICES))
    137		efi_reboot(reboot_mode, NULL);
    138
    139	/* Now call the architecture specific reboot code. */
    140	do_kernel_restart(cmd);
    141
    142	/*
    143	 * Whoops - the architecture was unable to reboot.
    144	 */
    145	printk("Reboot failed -- System halted\n");
    146	while (1);
    147}
    148
    149#define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = str
    150static const char *const btypes[] = {
    151	bstr(NONE, "--"),
    152	bstr(  JC, "jc"),
    153	bstr(   C, "-c"),
    154	bstr(  J , "j-")
    155};
    156#undef bstr
    157
    158static void print_pstate(struct pt_regs *regs)
    159{
    160	u64 pstate = regs->pstate;
    161
    162	if (compat_user_mode(regs)) {
    163		printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c %cDIT %cSSBS)\n",
    164			pstate,
    165			pstate & PSR_AA32_N_BIT ? 'N' : 'n',
    166			pstate & PSR_AA32_Z_BIT ? 'Z' : 'z',
    167			pstate & PSR_AA32_C_BIT ? 'C' : 'c',
    168			pstate & PSR_AA32_V_BIT ? 'V' : 'v',
    169			pstate & PSR_AA32_Q_BIT ? 'Q' : 'q',
    170			pstate & PSR_AA32_T_BIT ? "T32" : "A32",
    171			pstate & PSR_AA32_E_BIT ? "BE" : "LE",
    172			pstate & PSR_AA32_A_BIT ? 'A' : 'a',
    173			pstate & PSR_AA32_I_BIT ? 'I' : 'i',
    174			pstate & PSR_AA32_F_BIT ? 'F' : 'f',
    175			pstate & PSR_AA32_DIT_BIT ? '+' : '-',
    176			pstate & PSR_AA32_SSBS_BIT ? '+' : '-');
    177	} else {
    178		const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >>
    179					       PSR_BTYPE_SHIFT];
    180
    181		printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cSSBS BTYPE=%s)\n",
    182			pstate,
    183			pstate & PSR_N_BIT ? 'N' : 'n',
    184			pstate & PSR_Z_BIT ? 'Z' : 'z',
    185			pstate & PSR_C_BIT ? 'C' : 'c',
    186			pstate & PSR_V_BIT ? 'V' : 'v',
    187			pstate & PSR_D_BIT ? 'D' : 'd',
    188			pstate & PSR_A_BIT ? 'A' : 'a',
    189			pstate & PSR_I_BIT ? 'I' : 'i',
    190			pstate & PSR_F_BIT ? 'F' : 'f',
    191			pstate & PSR_PAN_BIT ? '+' : '-',
    192			pstate & PSR_UAO_BIT ? '+' : '-',
    193			pstate & PSR_TCO_BIT ? '+' : '-',
    194			pstate & PSR_DIT_BIT ? '+' : '-',
    195			pstate & PSR_SSBS_BIT ? '+' : '-',
    196			btype_str);
    197	}
    198}
    199
    200void __show_regs(struct pt_regs *regs)
    201{
    202	int i, top_reg;
    203	u64 lr, sp;
    204
    205	if (compat_user_mode(regs)) {
    206		lr = regs->compat_lr;
    207		sp = regs->compat_sp;
    208		top_reg = 12;
    209	} else {
    210		lr = regs->regs[30];
    211		sp = regs->sp;
    212		top_reg = 29;
    213	}
    214
    215	show_regs_print_info(KERN_DEFAULT);
    216	print_pstate(regs);
    217
    218	if (!user_mode(regs)) {
    219		printk("pc : %pS\n", (void *)regs->pc);
    220		printk("lr : %pS\n", (void *)ptrauth_strip_insn_pac(lr));
    221	} else {
    222		printk("pc : %016llx\n", regs->pc);
    223		printk("lr : %016llx\n", lr);
    224	}
    225
    226	printk("sp : %016llx\n", sp);
    227
    228	if (system_uses_irq_prio_masking())
    229		printk("pmr_save: %08llx\n", regs->pmr_save);
    230
    231	i = top_reg;
    232
    233	while (i >= 0) {
    234		printk("x%-2d: %016llx", i, regs->regs[i]);
    235
    236		while (i-- % 3)
    237			pr_cont(" x%-2d: %016llx", i, regs->regs[i]);
    238
    239		pr_cont("\n");
    240	}
    241}
    242
    243void show_regs(struct pt_regs *regs)
    244{
    245	__show_regs(regs);
    246	dump_backtrace(regs, NULL, KERN_DEFAULT);
    247}
    248
    249static void tls_thread_flush(void)
    250{
    251	write_sysreg(0, tpidr_el0);
    252	if (system_supports_tpidr2())
    253		write_sysreg_s(0, SYS_TPIDR2_EL0);
    254
    255	if (is_compat_task()) {
    256		current->thread.uw.tp_value = 0;
    257
    258		/*
    259		 * We need to ensure ordering between the shadow state and the
    260		 * hardware state, so that we don't corrupt the hardware state
    261		 * with a stale shadow state during context switch.
    262		 */
    263		barrier();
    264		write_sysreg(0, tpidrro_el0);
    265	}
    266}
    267
    268static void flush_tagged_addr_state(void)
    269{
    270	if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI))
    271		clear_thread_flag(TIF_TAGGED_ADDR);
    272}
    273
    274void flush_thread(void)
    275{
    276	fpsimd_flush_thread();
    277	tls_thread_flush();
    278	flush_ptrace_hw_breakpoint(current);
    279	flush_tagged_addr_state();
    280}
    281
    282void release_thread(struct task_struct *dead_task)
    283{
    284}
    285
    286void arch_release_task_struct(struct task_struct *tsk)
    287{
    288	fpsimd_release_task(tsk);
    289}
    290
    291int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
    292{
    293	if (current->mm)
    294		fpsimd_preserve_current_state();
    295	*dst = *src;
    296
    297	/* We rely on the above assignment to initialize dst's thread_flags: */
    298	BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK));
    299
    300	/*
    301	 * Detach src's sve_state (if any) from dst so that it does not
    302	 * get erroneously used or freed prematurely.  dst's copies
    303	 * will be allocated on demand later on if dst uses SVE.
    304	 * For consistency, also clear TIF_SVE here: this could be done
    305	 * later in copy_process(), but to avoid tripping up future
    306	 * maintainers it is best not to leave TIF flags and buffers in
    307	 * an inconsistent state, even temporarily.
    308	 */
    309	dst->thread.sve_state = NULL;
    310	clear_tsk_thread_flag(dst, TIF_SVE);
    311
    312	/*
    313	 * In the unlikely event that we create a new thread with ZA
    314	 * enabled we should retain the ZA state so duplicate it here.
    315	 * This may be shortly freed if we exec() or if CLONE_SETTLS
    316	 * but it's simpler to do it here. To avoid confusing the rest
    317	 * of the code ensure that we have a sve_state allocated
    318	 * whenever za_state is allocated.
    319	 */
    320	if (thread_za_enabled(&src->thread)) {
    321		dst->thread.sve_state = kzalloc(sve_state_size(src),
    322						GFP_KERNEL);
    323		if (!dst->thread.sve_state)
    324			return -ENOMEM;
    325		dst->thread.za_state = kmemdup(src->thread.za_state,
    326					       za_state_size(src),
    327					       GFP_KERNEL);
    328		if (!dst->thread.za_state) {
    329			kfree(dst->thread.sve_state);
    330			dst->thread.sve_state = NULL;
    331			return -ENOMEM;
    332		}
    333	} else {
    334		dst->thread.za_state = NULL;
    335		clear_tsk_thread_flag(dst, TIF_SME);
    336	}
    337
    338	/* clear any pending asynchronous tag fault raised by the parent */
    339	clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT);
    340
    341	return 0;
    342}
    343
    344asmlinkage void ret_from_fork(void) asm("ret_from_fork");
    345
    346int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
    347{
    348	unsigned long clone_flags = args->flags;
    349	unsigned long stack_start = args->stack;
    350	unsigned long tls = args->tls;
    351	struct pt_regs *childregs = task_pt_regs(p);
    352
    353	memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context));
    354
    355	/*
    356	 * In case p was allocated the same task_struct pointer as some
    357	 * other recently-exited task, make sure p is disassociated from
    358	 * any cpu that may have run that now-exited task recently.
    359	 * Otherwise we could erroneously skip reloading the FPSIMD
    360	 * registers for p.
    361	 */
    362	fpsimd_flush_task_state(p);
    363
    364	ptrauth_thread_init_kernel(p);
    365
    366	if (likely(!args->fn)) {
    367		*childregs = *current_pt_regs();
    368		childregs->regs[0] = 0;
    369
    370		/*
    371		 * Read the current TLS pointer from tpidr_el0 as it may be
    372		 * out-of-sync with the saved value.
    373		 */
    374		*task_user_tls(p) = read_sysreg(tpidr_el0);
    375		if (system_supports_tpidr2())
    376			p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0);
    377
    378		if (stack_start) {
    379			if (is_compat_thread(task_thread_info(p)))
    380				childregs->compat_sp = stack_start;
    381			else
    382				childregs->sp = stack_start;
    383		}
    384
    385		/*
    386		 * If a TLS pointer was passed to clone, use it for the new
    387		 * thread.  We also reset TPIDR2 if it's in use.
    388		 */
    389		if (clone_flags & CLONE_SETTLS) {
    390			p->thread.uw.tp_value = tls;
    391			p->thread.tpidr2_el0 = 0;
    392		}
    393	} else {
    394		/*
    395		 * A kthread has no context to ERET to, so ensure any buggy
    396		 * ERET is treated as an illegal exception return.
    397		 *
    398		 * When a user task is created from a kthread, childregs will
    399		 * be initialized by start_thread() or start_compat_thread().
    400		 */
    401		memset(childregs, 0, sizeof(struct pt_regs));
    402		childregs->pstate = PSR_MODE_EL1h | PSR_IL_BIT;
    403
    404		p->thread.cpu_context.x19 = (unsigned long)args->fn;
    405		p->thread.cpu_context.x20 = (unsigned long)args->fn_arg;
    406	}
    407	p->thread.cpu_context.pc = (unsigned long)ret_from_fork;
    408	p->thread.cpu_context.sp = (unsigned long)childregs;
    409	/*
    410	 * For the benefit of the unwinder, set up childregs->stackframe
    411	 * as the final frame for the new task.
    412	 */
    413	p->thread.cpu_context.fp = (unsigned long)childregs->stackframe;
    414
    415	ptrace_hw_copy_thread(p);
    416
    417	return 0;
    418}
    419
    420void tls_preserve_current_state(void)
    421{
    422	*task_user_tls(current) = read_sysreg(tpidr_el0);
    423	if (system_supports_tpidr2() && !is_compat_task())
    424		current->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0);
    425}
    426
    427static void tls_thread_switch(struct task_struct *next)
    428{
    429	tls_preserve_current_state();
    430
    431	if (is_compat_thread(task_thread_info(next)))
    432		write_sysreg(next->thread.uw.tp_value, tpidrro_el0);
    433	else if (!arm64_kernel_unmapped_at_el0())
    434		write_sysreg(0, tpidrro_el0);
    435
    436	write_sysreg(*task_user_tls(next), tpidr_el0);
    437	if (system_supports_tpidr2())
    438		write_sysreg_s(next->thread.tpidr2_el0, SYS_TPIDR2_EL0);
    439}
    440
    441/*
    442 * Force SSBS state on context-switch, since it may be lost after migrating
    443 * from a CPU which treats the bit as RES0 in a heterogeneous system.
    444 */
    445static void ssbs_thread_switch(struct task_struct *next)
    446{
    447	/*
    448	 * Nothing to do for kernel threads, but 'regs' may be junk
    449	 * (e.g. idle task) so check the flags and bail early.
    450	 */
    451	if (unlikely(next->flags & PF_KTHREAD))
    452		return;
    453
    454	/*
    455	 * If all CPUs implement the SSBS extension, then we just need to
    456	 * context-switch the PSTATE field.
    457	 */
    458	if (cpus_have_const_cap(ARM64_SSBS))
    459		return;
    460
    461	spectre_v4_enable_task_mitigation(next);
    462}
    463
    464/*
    465 * We store our current task in sp_el0, which is clobbered by userspace. Keep a
    466 * shadow copy so that we can restore this upon entry from userspace.
    467 *
    468 * This is *only* for exception entry from EL0, and is not valid until we
    469 * __switch_to() a user task.
    470 */
    471DEFINE_PER_CPU(struct task_struct *, __entry_task);
    472
    473static void entry_task_switch(struct task_struct *next)
    474{
    475	__this_cpu_write(__entry_task, next);
    476}
    477
    478/*
    479 * ARM erratum 1418040 handling, affecting the 32bit view of CNTVCT.
    480 * Ensure access is disabled when switching to a 32bit task, ensure
    481 * access is enabled when switching to a 64bit task.
    482 */
    483static void erratum_1418040_thread_switch(struct task_struct *next)
    484{
    485	if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040) ||
    486	    !this_cpu_has_cap(ARM64_WORKAROUND_1418040))
    487		return;
    488
    489	if (is_compat_thread(task_thread_info(next)))
    490		sysreg_clear_set(cntkctl_el1, ARCH_TIMER_USR_VCT_ACCESS_EN, 0);
    491	else
    492		sysreg_clear_set(cntkctl_el1, 0, ARCH_TIMER_USR_VCT_ACCESS_EN);
    493}
    494
    495static void erratum_1418040_new_exec(void)
    496{
    497	preempt_disable();
    498	erratum_1418040_thread_switch(current);
    499	preempt_enable();
    500}
    501
    502/*
    503 * __switch_to() checks current->thread.sctlr_user as an optimisation. Therefore
    504 * this function must be called with preemption disabled and the update to
    505 * sctlr_user must be made in the same preemption disabled block so that
    506 * __switch_to() does not see the variable update before the SCTLR_EL1 one.
    507 */
    508void update_sctlr_el1(u64 sctlr)
    509{
    510	/*
    511	 * EnIA must not be cleared while in the kernel as this is necessary for
    512	 * in-kernel PAC. It will be cleared on kernel exit if needed.
    513	 */
    514	sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr);
    515
    516	/* ISB required for the kernel uaccess routines when setting TCF0. */
    517	isb();
    518}
    519
    520/*
    521 * Thread switching.
    522 */
    523__notrace_funcgraph __sched
    524struct task_struct *__switch_to(struct task_struct *prev,
    525				struct task_struct *next)
    526{
    527	struct task_struct *last;
    528
    529	fpsimd_thread_switch(next);
    530	tls_thread_switch(next);
    531	hw_breakpoint_thread_switch(next);
    532	contextidr_thread_switch(next);
    533	entry_task_switch(next);
    534	ssbs_thread_switch(next);
    535	erratum_1418040_thread_switch(next);
    536	ptrauth_thread_switch_user(next);
    537
    538	/*
    539	 * Complete any pending TLB or cache maintenance on this CPU in case
    540	 * the thread migrates to a different CPU.
    541	 * This full barrier is also required by the membarrier system
    542	 * call.
    543	 */
    544	dsb(ish);
    545
    546	/*
    547	 * MTE thread switching must happen after the DSB above to ensure that
    548	 * any asynchronous tag check faults have been logged in the TFSR*_EL1
    549	 * registers.
    550	 */
    551	mte_thread_switch(next);
    552	/* avoid expensive SCTLR_EL1 accesses if no change */
    553	if (prev->thread.sctlr_user != next->thread.sctlr_user)
    554		update_sctlr_el1(next->thread.sctlr_user);
    555
    556	/* the actual thread switch */
    557	last = cpu_switch_to(prev, next);
    558
    559	return last;
    560}
    561
    562struct wchan_info {
    563	unsigned long	pc;
    564	int		count;
    565};
    566
    567static bool get_wchan_cb(void *arg, unsigned long pc)
    568{
    569	struct wchan_info *wchan_info = arg;
    570
    571	if (!in_sched_functions(pc)) {
    572		wchan_info->pc = pc;
    573		return false;
    574	}
    575	return wchan_info->count++ < 16;
    576}
    577
    578unsigned long __get_wchan(struct task_struct *p)
    579{
    580	struct wchan_info wchan_info = {
    581		.pc = 0,
    582		.count = 0,
    583	};
    584
    585	if (!try_get_task_stack(p))
    586		return 0;
    587
    588	arch_stack_walk(get_wchan_cb, &wchan_info, p, NULL);
    589
    590	put_task_stack(p);
    591
    592	return wchan_info.pc;
    593}
    594
    595unsigned long arch_align_stack(unsigned long sp)
    596{
    597	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
    598		sp -= get_random_int() & ~PAGE_MASK;
    599	return sp & ~0xf;
    600}
    601
    602#ifdef CONFIG_COMPAT
    603int compat_elf_check_arch(const struct elf32_hdr *hdr)
    604{
    605	if (!system_supports_32bit_el0())
    606		return false;
    607
    608	if ((hdr)->e_machine != EM_ARM)
    609		return false;
    610
    611	if (!((hdr)->e_flags & EF_ARM_EABI_MASK))
    612		return false;
    613
    614	/*
    615	 * Prevent execve() of a 32-bit program from a deadline task
    616	 * if the restricted affinity mask would be inadmissible on an
    617	 * asymmetric system.
    618	 */
    619	return !static_branch_unlikely(&arm64_mismatched_32bit_el0) ||
    620	       !dl_task_check_affinity(current, system_32bit_el0_cpumask());
    621}
    622#endif
    623
    624/*
    625 * Called from setup_new_exec() after (COMPAT_)SET_PERSONALITY.
    626 */
    627void arch_setup_new_exec(void)
    628{
    629	unsigned long mmflags = 0;
    630
    631	if (is_compat_task()) {
    632		mmflags = MMCF_AARCH32;
    633
    634		/*
    635		 * Restrict the CPU affinity mask for a 32-bit task so that
    636		 * it contains only 32-bit-capable CPUs.
    637		 *
    638		 * From the perspective of the task, this looks similar to
    639		 * what would happen if the 64-bit-only CPUs were hot-unplugged
    640		 * at the point of execve(), although we try a bit harder to
    641		 * honour the cpuset hierarchy.
    642		 */
    643		if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
    644			force_compatible_cpus_allowed_ptr(current);
    645	} else if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) {
    646		relax_compatible_cpus_allowed_ptr(current);
    647	}
    648
    649	current->mm->context.flags = mmflags;
    650	ptrauth_thread_init_user();
    651	mte_thread_init_user();
    652	erratum_1418040_new_exec();
    653
    654	if (task_spec_ssb_noexec(current)) {
    655		arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS,
    656					 PR_SPEC_ENABLE);
    657	}
    658}
    659
    660#ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
    661/*
    662 * Control the relaxed ABI allowing tagged user addresses into the kernel.
    663 */
    664static unsigned int tagged_addr_disabled;
    665
    666long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg)
    667{
    668	unsigned long valid_mask = PR_TAGGED_ADDR_ENABLE;
    669	struct thread_info *ti = task_thread_info(task);
    670
    671	if (is_compat_thread(ti))
    672		return -EINVAL;
    673
    674	if (system_supports_mte())
    675		valid_mask |= PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC \
    676			| PR_MTE_TAG_MASK;
    677
    678	if (arg & ~valid_mask)
    679		return -EINVAL;
    680
    681	/*
    682	 * Do not allow the enabling of the tagged address ABI if globally
    683	 * disabled via sysctl abi.tagged_addr_disabled.
    684	 */
    685	if (arg & PR_TAGGED_ADDR_ENABLE && tagged_addr_disabled)
    686		return -EINVAL;
    687
    688	if (set_mte_ctrl(task, arg) != 0)
    689		return -EINVAL;
    690
    691	update_ti_thread_flag(ti, TIF_TAGGED_ADDR, arg & PR_TAGGED_ADDR_ENABLE);
    692
    693	return 0;
    694}
    695
    696long get_tagged_addr_ctrl(struct task_struct *task)
    697{
    698	long ret = 0;
    699	struct thread_info *ti = task_thread_info(task);
    700
    701	if (is_compat_thread(ti))
    702		return -EINVAL;
    703
    704	if (test_ti_thread_flag(ti, TIF_TAGGED_ADDR))
    705		ret = PR_TAGGED_ADDR_ENABLE;
    706
    707	ret |= get_mte_ctrl(task);
    708
    709	return ret;
    710}
    711
    712/*
    713 * Global sysctl to disable the tagged user addresses support. This control
    714 * only prevents the tagged address ABI enabling via prctl() and does not
    715 * disable it for tasks that already opted in to the relaxed ABI.
    716 */
    717
    718static struct ctl_table tagged_addr_sysctl_table[] = {
    719	{
    720		.procname	= "tagged_addr_disabled",
    721		.mode		= 0644,
    722		.data		= &tagged_addr_disabled,
    723		.maxlen		= sizeof(int),
    724		.proc_handler	= proc_dointvec_minmax,
    725		.extra1		= SYSCTL_ZERO,
    726		.extra2		= SYSCTL_ONE,
    727	},
    728	{ }
    729};
    730
    731static int __init tagged_addr_init(void)
    732{
    733	if (!register_sysctl("abi", tagged_addr_sysctl_table))
    734		return -EINVAL;
    735	return 0;
    736}
    737
    738core_initcall(tagged_addr_init);
    739#endif	/* CONFIG_ARM64_TAGGED_ADDR_ABI */
    740
    741#ifdef CONFIG_BINFMT_ELF
    742int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state,
    743			 bool has_interp, bool is_interp)
    744{
    745	/*
    746	 * For dynamically linked executables the interpreter is
    747	 * responsible for setting PROT_BTI on everything except
    748	 * itself.
    749	 */
    750	if (is_interp != has_interp)
    751		return prot;
    752
    753	if (!(state->flags & ARM64_ELF_BTI))
    754		return prot;
    755
    756	if (prot & PROT_EXEC)
    757		prot |= PROT_BTI;
    758
    759	return prot;
    760}
    761#endif