cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

futex.h (4357B)

      1/* SPDX-License-Identifier: GPL-2.0 */
      2#ifndef _ASM_ARM_FUTEX_H
      3#define _ASM_ARM_FUTEX_H
      4
      5#ifdef __KERNEL__
      6
      7#include <linux/futex.h>
      8#include <linux/uaccess.h>
      9#include <asm/errno.h>
     10
     11#define __futex_atomic_ex_table(err_reg)			\
     12	"3:\n"							\
     13	"	.pushsection __ex_table,\"a\"\n"		\
     14	"	.align	3\n"					\
     15	"	.long	1b, 4f, 2b, 4f\n"			\
     16	"	.popsection\n"					\
     17	"	.pushsection .text.fixup,\"ax\"\n"		\
     18	"	.align	2\n"					\
     19	"4:	mov	%0, " err_reg "\n"			\
     20	"	b	3b\n"					\
     21	"	.popsection"
     22
     23#ifdef CONFIG_SMP
     24
     25#define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg)	\
     26({								\
     27	unsigned int __ua_flags;				\
     28	smp_mb();						\
     29	prefetchw(uaddr);					\
     30	__ua_flags = uaccess_save_and_enable();			\
     31	__asm__ __volatile__(					\
     32	"1:	ldrex	%1, [%3]\n"				\
     33	"	" insn "\n"					\
     34	"2:	strex	%2, %0, [%3]\n"				\
     35	"	teq	%2, #0\n"				\
     36	"	bne	1b\n"					\
     37	"	mov	%0, #0\n"				\
     38	__futex_atomic_ex_table("%5")				\
     39	: "=&r" (ret), "=&r" (oldval), "=&r" (tmp)		\
     40	: "r" (uaddr), "r" (oparg), "Ir" (-EFAULT)		\
     41	: "cc", "memory");					\
     42	uaccess_restore(__ua_flags);				\
     43})
     44
     45static inline int
     46futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
     47			      u32 oldval, u32 newval)
     48{
     49	unsigned int __ua_flags;
     50	int ret;
     51	u32 val;
     52
     53	if (!access_ok(uaddr, sizeof(u32)))
     54		return -EFAULT;
     55
     56	smp_mb();
     57	/* Prefetching cannot fault */
     58	prefetchw(uaddr);
     59	__ua_flags = uaccess_save_and_enable();
     60	__asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
     61	"1:	ldrex	%1, [%4]\n"
     62	"	teq	%1, %2\n"
     63	"	ite	eq	@ explicit IT needed for the 2b label\n"
     64	"2:	strexeq	%0, %3, [%4]\n"
     65	"	movne	%0, #0\n"
     66	"	teq	%0, #0\n"
     67	"	bne	1b\n"
     68	__futex_atomic_ex_table("%5")
     69	: "=&r" (ret), "=&r" (val)
     70	: "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT)
     71	: "cc", "memory");
     72	uaccess_restore(__ua_flags);
     73	smp_mb();
     74
     75	*uval = val;
     76	return ret;
     77}
     78
     79#else /* !SMP, we can work around lack of atomic ops by disabling preemption */
     80
     81#include <linux/preempt.h>
     82#include <asm/domain.h>
     83
     84#define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg)	\
     85({								\
     86	unsigned int __ua_flags = uaccess_save_and_enable();	\
     87	__asm__ __volatile__(					\
     88	"1:	" TUSER(ldr) "	%1, [%3]\n"			\
     89	"	" insn "\n"					\
     90	"2:	" TUSER(str) "	%0, [%3]\n"			\
     91	"	mov	%0, #0\n"				\
     92	__futex_atomic_ex_table("%5")				\
     93	: "=&r" (ret), "=&r" (oldval), "=&r" (tmp)		\
     94	: "r" (uaddr), "r" (oparg), "Ir" (-EFAULT)		\
     95	: "cc", "memory");					\
     96	uaccess_restore(__ua_flags);				\
     97})
     98
     99static inline int
    100futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
    101			      u32 oldval, u32 newval)
    102{
    103	unsigned int __ua_flags;
    104	int ret = 0;
    105	u32 val;
    106
    107	if (!access_ok(uaddr, sizeof(u32)))
    108		return -EFAULT;
    109
    110	preempt_disable();
    111	__ua_flags = uaccess_save_and_enable();
    112	__asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
    113	"	.syntax unified\n"
    114	"1:	" TUSER(ldr) "	%1, [%4]\n"
    115	"	teq	%1, %2\n"
    116	"	it	eq	@ explicit IT needed for the 2b label\n"
    117	"2:	" TUSERCOND(str, eq) "	%3, [%4]\n"
    118	__futex_atomic_ex_table("%5")
    119	: "+r" (ret), "=&r" (val)
    120	: "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT)
    121	: "cc", "memory");
    122	uaccess_restore(__ua_flags);
    123
    124	*uval = val;
    125	preempt_enable();
    126
    127	return ret;
    128}
    129
    130#endif /* !SMP */
    131
    132static inline int
    133arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
    134{
    135	int oldval = 0, ret, tmp;
    136
    137	if (!access_ok(uaddr, sizeof(u32)))
    138		return -EFAULT;
    139
    140#ifndef CONFIG_SMP
    141	preempt_disable();
    142#endif
    143
    144	switch (op) {
    145	case FUTEX_OP_SET:
    146		__futex_atomic_op("mov	%0, %4", ret, oldval, tmp, uaddr, oparg);
    147		break;
    148	case FUTEX_OP_ADD:
    149		__futex_atomic_op("add	%0, %1, %4", ret, oldval, tmp, uaddr, oparg);
    150		break;
    151	case FUTEX_OP_OR:
    152		__futex_atomic_op("orr	%0, %1, %4", ret, oldval, tmp, uaddr, oparg);
    153		break;
    154	case FUTEX_OP_ANDN:
    155		__futex_atomic_op("and	%0, %1, %4", ret, oldval, tmp, uaddr, ~oparg);
    156		break;
    157	case FUTEX_OP_XOR:
    158		__futex_atomic_op("eor	%0, %1, %4", ret, oldval, tmp, uaddr, oparg);
    159		break;
    160	default:
    161		ret = -ENOSYS;
    162	}
    163
    164#ifndef CONFIG_SMP
    165	preempt_enable();
    166#endif
    167
    168	/*
    169	 * Store unconditionally. If ret != 0 the extra store is the least
    170	 * of the worries but GCC cannot figure out that __futex_atomic_op()
    171	 * is either setting ret to -EFAULT or storing the old value in
    172	 * oldval which results in a uninitialized warning at the call site.
    173	 */
    174	*oval = oldval;
    175
    176	return ret;
    177}
    178
    179#endif /* __KERNEL__ */
    180#endif /* _ASM_ARM_FUTEX_H */