cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

rseq-arm.h (23128B)


      1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
      2/*
      3 * rseq-arm.h
      4 *
      5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
      6 */
      7
      8/*
      9 * - ARM little endian
     10 *
     11 * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
     12 * value 0x5de3. This traps if user-space reaches this instruction by mistake,
     13 * and the uncommon operand ensures the kernel does not move the instruction
     14 * pointer to attacker-controlled code on rseq abort.
     15 *
     16 * The instruction pattern in the A32 instruction set is:
     17 *
     18 * e7f5def3    udf    #24035    ; 0x5de3
     19 *
     20 * This translates to the following instruction pattern in the T16 instruction
     21 * set:
     22 *
     23 * little endian:
     24 * def3        udf    #243      ; 0xf3
     25 * e7f5        b.n    <7f5>
     26 *
     27 * - ARMv6+ big endian (BE8):
     28 *
     29 * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
     30 * code and big-endian data. The data value of the signature needs to have its
     31 * byte order reversed to generate the trap instruction:
     32 *
     33 * Data: 0xf3def5e7
     34 *
     35 * Translates to this A32 instruction pattern:
     36 *
     37 * e7f5def3    udf    #24035    ; 0x5de3
     38 *
     39 * Translates to this T16 instruction pattern:
     40 *
     41 * def3        udf    #243      ; 0xf3
     42 * e7f5        b.n    <7f5>
     43 *
     44 * - Prior to ARMv6 big endian (BE32):
     45 *
     46 * Prior to ARMv6, -mbig-endian generates big-endian code and data
     47 * (which match), so the endianness of the data representation of the
     48 * signature should not be reversed. However, the choice between BE32
     49 * and BE8 is done by the linker, so we cannot know whether code and
     50 * data endianness will be mixed before the linker is invoked. So rather
     51 * than try to play tricks with the linker, the rseq signature is simply
     52 * data (not a trap instruction) prior to ARMv6 on big endian. This is
     53 * why the signature is expressed as data (.word) rather than as
     54 * instruction (.inst) in assembler.
     55 */
     56
     57#ifdef __ARMEB__
     58#define RSEQ_SIG    0xf3def5e7      /* udf    #24035    ; 0x5de3 (ARMv6+) */
     59#else
     60#define RSEQ_SIG    0xe7f5def3      /* udf    #24035    ; 0x5de3 */
     61#endif
     62
     63#define rseq_smp_mb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
     64#define rseq_smp_rmb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
     65#define rseq_smp_wmb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
     66
     67#define rseq_smp_load_acquire(p)					\
     68__extension__ ({							\
     69	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
     70	rseq_smp_mb();							\
     71	____p1;								\
     72})
     73
     74#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
     75
     76#define rseq_smp_store_release(p, v)					\
     77do {									\
     78	rseq_smp_mb();							\
     79	RSEQ_WRITE_ONCE(*p, v);						\
     80} while (0)
     81
     82#ifdef RSEQ_SKIP_FASTPATH
     83#include "rseq-skip.h"
     84#else /* !RSEQ_SKIP_FASTPATH */
     85
     86#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,	\
     87				post_commit_offset, abort_ip)		\
     88		".pushsection __rseq_cs, \"aw\"\n\t"			\
     89		".balign 32\n\t"					\
     90		__rseq_str(label) ":\n\t"					\
     91		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
     92		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
     93		".popsection\n\t"					\
     94		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
     95		".word " __rseq_str(label) "b, 0x0\n\t"			\
     96		".popsection\n\t"
     97
     98#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
     99	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
    100				(post_commit_ip - start_ip), abort_ip)
    101
    102/*
    103 * Exit points of a rseq critical section consist of all instructions outside
    104 * of the critical section where a critical section can either branch to or
    105 * reach through the normal course of its execution. The abort IP and the
    106 * post-commit IP are already part of the __rseq_cs section and should not be
    107 * explicitly defined as additional exit points. Knowing all exit points is
    108 * useful to assist debuggers stepping over the critical section.
    109 */
    110#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
    111		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
    112		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
    113		".popsection\n\t"
    114
    115#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
    116		RSEQ_INJECT_ASM(1)					\
    117		"adr r0, " __rseq_str(cs_label) "\n\t"			\
    118		"str r0, %[" __rseq_str(rseq_cs) "]\n\t"		\
    119		__rseq_str(label) ":\n\t"
    120
    121#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
    122		RSEQ_INJECT_ASM(2)					\
    123		"ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t"	\
    124		"cmp %[" __rseq_str(cpu_id) "], r0\n\t"		\
    125		"bne " __rseq_str(label) "\n\t"
    126
    127#define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,		\
    128				abort_label, version, flags,		\
    129				start_ip, post_commit_offset, abort_ip)	\
    130		".balign 32\n\t"					\
    131		__rseq_str(table_label) ":\n\t"				\
    132		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
    133		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
    134		".word " __rseq_str(RSEQ_SIG) "\n\t"			\
    135		__rseq_str(label) ":\n\t"				\
    136		teardown						\
    137		"b %l[" __rseq_str(abort_label) "]\n\t"
    138
    139#define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
    140			      start_ip, post_commit_ip, abort_ip)	\
    141	__RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,		\
    142				abort_label, 0x0, 0x0, start_ip,	\
    143				(post_commit_ip - start_ip), abort_ip)
    144
    145#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
    146		__rseq_str(label) ":\n\t"				\
    147		teardown						\
    148		"b %l[" __rseq_str(cmpfail_label) "]\n\t"
    149
    150static inline __attribute__((always_inline))
    151int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
    152{
    153	RSEQ_INJECT_C(9)
    154
    155	__asm__ __volatile__ goto (
    156		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
    157		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    158#ifdef RSEQ_COMPARE_TWICE
    159		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    160		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    161#endif
    162		/* Start rseq by storing table entry pointer into rseq_cs. */
    163		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
    164		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    165		RSEQ_INJECT_ASM(3)
    166		"ldr r0, %[v]\n\t"
    167		"cmp %[expect], r0\n\t"
    168		"bne %l[cmpfail]\n\t"
    169		RSEQ_INJECT_ASM(4)
    170#ifdef RSEQ_COMPARE_TWICE
    171		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    172		"ldr r0, %[v]\n\t"
    173		"cmp %[expect], r0\n\t"
    174		"bne %l[error2]\n\t"
    175#endif
    176		/* final store */
    177		"str %[newv], %[v]\n\t"
    178		"2:\n\t"
    179		RSEQ_INJECT_ASM(5)
    180		"b 5f\n\t"
    181		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
    182		"5:\n\t"
    183		: /* gcc asm goto does not allow outputs */
    184		: [cpu_id]		"r" (cpu),
    185		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
    186		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
    187		  [v]			"m" (*v),
    188		  [expect]		"r" (expect),
    189		  [newv]		"r" (newv)
    190		  RSEQ_INJECT_INPUT
    191		: "r0", "memory", "cc"
    192		  RSEQ_INJECT_CLOBBER
    193		: abort, cmpfail
    194#ifdef RSEQ_COMPARE_TWICE
    195		  , error1, error2
    196#endif
    197	);
    198	rseq_after_asm_goto();
    199	return 0;
    200abort:
    201	rseq_after_asm_goto();
    202	RSEQ_INJECT_FAILED
    203	return -1;
    204cmpfail:
    205	rseq_after_asm_goto();
    206	return 1;
    207#ifdef RSEQ_COMPARE_TWICE
    208error1:
    209	rseq_after_asm_goto();
    210	rseq_bug("cpu_id comparison failed");
    211error2:
    212	rseq_after_asm_goto();
    213	rseq_bug("expected value comparison failed");
    214#endif
    215}
    216
    217static inline __attribute__((always_inline))
    218int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
    219			       long voffp, intptr_t *load, int cpu)
    220{
    221	RSEQ_INJECT_C(9)
    222
    223	__asm__ __volatile__ goto (
    224		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
    225		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    226#ifdef RSEQ_COMPARE_TWICE
    227		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    228		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    229#endif
    230		/* Start rseq by storing table entry pointer into rseq_cs. */
    231		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
    232		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    233		RSEQ_INJECT_ASM(3)
    234		"ldr r0, %[v]\n\t"
    235		"cmp %[expectnot], r0\n\t"
    236		"beq %l[cmpfail]\n\t"
    237		RSEQ_INJECT_ASM(4)
    238#ifdef RSEQ_COMPARE_TWICE
    239		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    240		"ldr r0, %[v]\n\t"
    241		"cmp %[expectnot], r0\n\t"
    242		"beq %l[error2]\n\t"
    243#endif
    244		"str r0, %[load]\n\t"
    245		"add r0, %[voffp]\n\t"
    246		"ldr r0, [r0]\n\t"
    247		/* final store */
    248		"str r0, %[v]\n\t"
    249		"2:\n\t"
    250		RSEQ_INJECT_ASM(5)
    251		"b 5f\n\t"
    252		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
    253		"5:\n\t"
    254		: /* gcc asm goto does not allow outputs */
    255		: [cpu_id]		"r" (cpu),
    256		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
    257		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
    258		  /* final store input */
    259		  [v]			"m" (*v),
    260		  [expectnot]		"r" (expectnot),
    261		  [voffp]		"Ir" (voffp),
    262		  [load]		"m" (*load)
    263		  RSEQ_INJECT_INPUT
    264		: "r0", "memory", "cc"
    265		  RSEQ_INJECT_CLOBBER
    266		: abort, cmpfail
    267#ifdef RSEQ_COMPARE_TWICE
    268		  , error1, error2
    269#endif
    270	);
    271	rseq_after_asm_goto();
    272	return 0;
    273abort:
    274	rseq_after_asm_goto();
    275	RSEQ_INJECT_FAILED
    276	return -1;
    277cmpfail:
    278	rseq_after_asm_goto();
    279	return 1;
    280#ifdef RSEQ_COMPARE_TWICE
    281error1:
    282	rseq_after_asm_goto();
    283	rseq_bug("cpu_id comparison failed");
    284error2:
    285	rseq_after_asm_goto();
    286	rseq_bug("expected value comparison failed");
    287#endif
    288}
    289
    290static inline __attribute__((always_inline))
    291int rseq_addv(intptr_t *v, intptr_t count, int cpu)
    292{
    293	RSEQ_INJECT_C(9)
    294
    295	__asm__ __volatile__ goto (
    296		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
    297#ifdef RSEQ_COMPARE_TWICE
    298		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    299#endif
    300		/* Start rseq by storing table entry pointer into rseq_cs. */
    301		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
    302		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    303		RSEQ_INJECT_ASM(3)
    304#ifdef RSEQ_COMPARE_TWICE
    305		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    306#endif
    307		"ldr r0, %[v]\n\t"
    308		"add r0, %[count]\n\t"
    309		/* final store */
    310		"str r0, %[v]\n\t"
    311		"2:\n\t"
    312		RSEQ_INJECT_ASM(4)
    313		"b 5f\n\t"
    314		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
    315		"5:\n\t"
    316		: /* gcc asm goto does not allow outputs */
    317		: [cpu_id]		"r" (cpu),
    318		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
    319		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
    320		  [v]			"m" (*v),
    321		  [count]		"Ir" (count)
    322		  RSEQ_INJECT_INPUT
    323		: "r0", "memory", "cc"
    324		  RSEQ_INJECT_CLOBBER
    325		: abort
    326#ifdef RSEQ_COMPARE_TWICE
    327		  , error1
    328#endif
    329	);
    330	rseq_after_asm_goto();
    331	return 0;
    332abort:
    333	rseq_after_asm_goto();
    334	RSEQ_INJECT_FAILED
    335	return -1;
    336#ifdef RSEQ_COMPARE_TWICE
    337error1:
    338	rseq_after_asm_goto();
    339	rseq_bug("cpu_id comparison failed");
    340#endif
    341}
    342
    343static inline __attribute__((always_inline))
    344int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
    345				 intptr_t *v2, intptr_t newv2,
    346				 intptr_t newv, int cpu)
    347{
    348	RSEQ_INJECT_C(9)
    349
    350	__asm__ __volatile__ goto (
    351		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
    352		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    353#ifdef RSEQ_COMPARE_TWICE
    354		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    355		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    356#endif
    357		/* Start rseq by storing table entry pointer into rseq_cs. */
    358		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
    359		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    360		RSEQ_INJECT_ASM(3)
    361		"ldr r0, %[v]\n\t"
    362		"cmp %[expect], r0\n\t"
    363		"bne %l[cmpfail]\n\t"
    364		RSEQ_INJECT_ASM(4)
    365#ifdef RSEQ_COMPARE_TWICE
    366		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    367		"ldr r0, %[v]\n\t"
    368		"cmp %[expect], r0\n\t"
    369		"bne %l[error2]\n\t"
    370#endif
    371		/* try store */
    372		"str %[newv2], %[v2]\n\t"
    373		RSEQ_INJECT_ASM(5)
    374		/* final store */
    375		"str %[newv], %[v]\n\t"
    376		"2:\n\t"
    377		RSEQ_INJECT_ASM(6)
    378		"b 5f\n\t"
    379		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
    380		"5:\n\t"
    381		: /* gcc asm goto does not allow outputs */
    382		: [cpu_id]		"r" (cpu),
    383		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
    384		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
    385		  /* try store input */
    386		  [v2]			"m" (*v2),
    387		  [newv2]		"r" (newv2),
    388		  /* final store input */
    389		  [v]			"m" (*v),
    390		  [expect]		"r" (expect),
    391		  [newv]		"r" (newv)
    392		  RSEQ_INJECT_INPUT
    393		: "r0", "memory", "cc"
    394		  RSEQ_INJECT_CLOBBER
    395		: abort, cmpfail
    396#ifdef RSEQ_COMPARE_TWICE
    397		  , error1, error2
    398#endif
    399	);
    400	rseq_after_asm_goto();
    401	return 0;
    402abort:
    403	rseq_after_asm_goto();
    404	RSEQ_INJECT_FAILED
    405	return -1;
    406cmpfail:
    407	rseq_after_asm_goto();
    408	return 1;
    409#ifdef RSEQ_COMPARE_TWICE
    410error1:
    411	rseq_after_asm_goto();
    412	rseq_bug("cpu_id comparison failed");
    413error2:
    414	rseq_after_asm_goto();
    415	rseq_bug("expected value comparison failed");
    416#endif
    417}
    418
    419static inline __attribute__((always_inline))
    420int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
    421					 intptr_t *v2, intptr_t newv2,
    422					 intptr_t newv, int cpu)
    423{
    424	RSEQ_INJECT_C(9)
    425
    426	__asm__ __volatile__ goto (
    427		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
    428		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    429#ifdef RSEQ_COMPARE_TWICE
    430		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    431		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    432#endif
    433		/* Start rseq by storing table entry pointer into rseq_cs. */
    434		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
    435		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    436		RSEQ_INJECT_ASM(3)
    437		"ldr r0, %[v]\n\t"
    438		"cmp %[expect], r0\n\t"
    439		"bne %l[cmpfail]\n\t"
    440		RSEQ_INJECT_ASM(4)
    441#ifdef RSEQ_COMPARE_TWICE
    442		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    443		"ldr r0, %[v]\n\t"
    444		"cmp %[expect], r0\n\t"
    445		"bne %l[error2]\n\t"
    446#endif
    447		/* try store */
    448		"str %[newv2], %[v2]\n\t"
    449		RSEQ_INJECT_ASM(5)
    450		"dmb\n\t"	/* full mb provides store-release */
    451		/* final store */
    452		"str %[newv], %[v]\n\t"
    453		"2:\n\t"
    454		RSEQ_INJECT_ASM(6)
    455		"b 5f\n\t"
    456		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
    457		"5:\n\t"
    458		: /* gcc asm goto does not allow outputs */
    459		: [cpu_id]		"r" (cpu),
    460		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
    461		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
    462		  /* try store input */
    463		  [v2]			"m" (*v2),
    464		  [newv2]		"r" (newv2),
    465		  /* final store input */
    466		  [v]			"m" (*v),
    467		  [expect]		"r" (expect),
    468		  [newv]		"r" (newv)
    469		  RSEQ_INJECT_INPUT
    470		: "r0", "memory", "cc"
    471		  RSEQ_INJECT_CLOBBER
    472		: abort, cmpfail
    473#ifdef RSEQ_COMPARE_TWICE
    474		  , error1, error2
    475#endif
    476	);
    477	rseq_after_asm_goto();
    478	return 0;
    479abort:
    480	rseq_after_asm_goto();
    481	RSEQ_INJECT_FAILED
    482	return -1;
    483cmpfail:
    484	rseq_after_asm_goto();
    485	return 1;
    486#ifdef RSEQ_COMPARE_TWICE
    487error1:
    488	rseq_after_asm_goto();
    489	rseq_bug("cpu_id comparison failed");
    490error2:
    491	rseq_after_asm_goto();
    492	rseq_bug("expected value comparison failed");
    493#endif
    494}
    495
    496static inline __attribute__((always_inline))
    497int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
    498			      intptr_t *v2, intptr_t expect2,
    499			      intptr_t newv, int cpu)
    500{
    501	RSEQ_INJECT_C(9)
    502
    503	__asm__ __volatile__ goto (
    504		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
    505		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    506#ifdef RSEQ_COMPARE_TWICE
    507		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    508		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    509		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
    510#endif
    511		/* Start rseq by storing table entry pointer into rseq_cs. */
    512		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
    513		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    514		RSEQ_INJECT_ASM(3)
    515		"ldr r0, %[v]\n\t"
    516		"cmp %[expect], r0\n\t"
    517		"bne %l[cmpfail]\n\t"
    518		RSEQ_INJECT_ASM(4)
    519		"ldr r0, %[v2]\n\t"
    520		"cmp %[expect2], r0\n\t"
    521		"bne %l[cmpfail]\n\t"
    522		RSEQ_INJECT_ASM(5)
    523#ifdef RSEQ_COMPARE_TWICE
    524		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
    525		"ldr r0, %[v]\n\t"
    526		"cmp %[expect], r0\n\t"
    527		"bne %l[error2]\n\t"
    528		"ldr r0, %[v2]\n\t"
    529		"cmp %[expect2], r0\n\t"
    530		"bne %l[error3]\n\t"
    531#endif
    532		/* final store */
    533		"str %[newv], %[v]\n\t"
    534		"2:\n\t"
    535		RSEQ_INJECT_ASM(6)
    536		"b 5f\n\t"
    537		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
    538		"5:\n\t"
    539		: /* gcc asm goto does not allow outputs */
    540		: [cpu_id]		"r" (cpu),
    541		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
    542		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
    543		  /* cmp2 input */
    544		  [v2]			"m" (*v2),
    545		  [expect2]		"r" (expect2),
    546		  /* final store input */
    547		  [v]			"m" (*v),
    548		  [expect]		"r" (expect),
    549		  [newv]		"r" (newv)
    550		  RSEQ_INJECT_INPUT
    551		: "r0", "memory", "cc"
    552		  RSEQ_INJECT_CLOBBER
    553		: abort, cmpfail
    554#ifdef RSEQ_COMPARE_TWICE
    555		  , error1, error2, error3
    556#endif
    557	);
    558	rseq_after_asm_goto();
    559	return 0;
    560abort:
    561	rseq_after_asm_goto();
    562	RSEQ_INJECT_FAILED
    563	return -1;
    564cmpfail:
    565	rseq_after_asm_goto();
    566	return 1;
    567#ifdef RSEQ_COMPARE_TWICE
    568error1:
    569	rseq_after_asm_goto();
    570	rseq_bug("cpu_id comparison failed");
    571error2:
    572	rseq_after_asm_goto();
    573	rseq_bug("1st expected value comparison failed");
    574error3:
    575	rseq_after_asm_goto();
    576	rseq_bug("2nd expected value comparison failed");
    577#endif
    578}
    579
    580static inline __attribute__((always_inline))
    581int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
    582				 void *dst, void *src, size_t len,
    583				 intptr_t newv, int cpu)
    584{
    585	uint32_t rseq_scratch[3];
    586
    587	RSEQ_INJECT_C(9)
    588
    589	__asm__ __volatile__ goto (
    590		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
    591		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    592#ifdef RSEQ_COMPARE_TWICE
    593		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    594		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    595#endif
    596		"str %[src], %[rseq_scratch0]\n\t"
    597		"str %[dst], %[rseq_scratch1]\n\t"
    598		"str %[len], %[rseq_scratch2]\n\t"
    599		/* Start rseq by storing table entry pointer into rseq_cs. */
    600		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
    601		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    602		RSEQ_INJECT_ASM(3)
    603		"ldr r0, %[v]\n\t"
    604		"cmp %[expect], r0\n\t"
    605		"bne 5f\n\t"
    606		RSEQ_INJECT_ASM(4)
    607#ifdef RSEQ_COMPARE_TWICE
    608		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
    609		"ldr r0, %[v]\n\t"
    610		"cmp %[expect], r0\n\t"
    611		"bne 7f\n\t"
    612#endif
    613		/* try memcpy */
    614		"cmp %[len], #0\n\t" \
    615		"beq 333f\n\t" \
    616		"222:\n\t" \
    617		"ldrb %%r0, [%[src]]\n\t" \
    618		"strb %%r0, [%[dst]]\n\t" \
    619		"adds %[src], #1\n\t" \
    620		"adds %[dst], #1\n\t" \
    621		"subs %[len], #1\n\t" \
    622		"bne 222b\n\t" \
    623		"333:\n\t" \
    624		RSEQ_INJECT_ASM(5)
    625		/* final store */
    626		"str %[newv], %[v]\n\t"
    627		"2:\n\t"
    628		RSEQ_INJECT_ASM(6)
    629		/* teardown */
    630		"ldr %[len], %[rseq_scratch2]\n\t"
    631		"ldr %[dst], %[rseq_scratch1]\n\t"
    632		"ldr %[src], %[rseq_scratch0]\n\t"
    633		"b 8f\n\t"
    634		RSEQ_ASM_DEFINE_ABORT(3, 4,
    635				      /* teardown */
    636				      "ldr %[len], %[rseq_scratch2]\n\t"
    637				      "ldr %[dst], %[rseq_scratch1]\n\t"
    638				      "ldr %[src], %[rseq_scratch0]\n\t",
    639				      abort, 1b, 2b, 4f)
    640		RSEQ_ASM_DEFINE_CMPFAIL(5,
    641					/* teardown */
    642					"ldr %[len], %[rseq_scratch2]\n\t"
    643					"ldr %[dst], %[rseq_scratch1]\n\t"
    644					"ldr %[src], %[rseq_scratch0]\n\t",
    645					cmpfail)
    646#ifdef RSEQ_COMPARE_TWICE
    647		RSEQ_ASM_DEFINE_CMPFAIL(6,
    648					/* teardown */
    649					"ldr %[len], %[rseq_scratch2]\n\t"
    650					"ldr %[dst], %[rseq_scratch1]\n\t"
    651					"ldr %[src], %[rseq_scratch0]\n\t",
    652					error1)
    653		RSEQ_ASM_DEFINE_CMPFAIL(7,
    654					/* teardown */
    655					"ldr %[len], %[rseq_scratch2]\n\t"
    656					"ldr %[dst], %[rseq_scratch1]\n\t"
    657					"ldr %[src], %[rseq_scratch0]\n\t",
    658					error2)
    659#endif
    660		"8:\n\t"
    661		: /* gcc asm goto does not allow outputs */
    662		: [cpu_id]		"r" (cpu),
    663		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
    664		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
    665		  /* final store input */
    666		  [v]			"m" (*v),
    667		  [expect]		"r" (expect),
    668		  [newv]		"r" (newv),
    669		  /* try memcpy input */
    670		  [dst]			"r" (dst),
    671		  [src]			"r" (src),
    672		  [len]			"r" (len),
    673		  [rseq_scratch0]	"m" (rseq_scratch[0]),
    674		  [rseq_scratch1]	"m" (rseq_scratch[1]),
    675		  [rseq_scratch2]	"m" (rseq_scratch[2])
    676		  RSEQ_INJECT_INPUT
    677		: "r0", "memory", "cc"
    678		  RSEQ_INJECT_CLOBBER
    679		: abort, cmpfail
    680#ifdef RSEQ_COMPARE_TWICE
    681		  , error1, error2
    682#endif
    683	);
    684	rseq_after_asm_goto();
    685	return 0;
    686abort:
    687	rseq_after_asm_goto();
    688	RSEQ_INJECT_FAILED
    689	return -1;
    690cmpfail:
    691	rseq_after_asm_goto();
    692	return 1;
    693#ifdef RSEQ_COMPARE_TWICE
    694error1:
    695	rseq_after_asm_goto();
    696	rseq_bug("cpu_id comparison failed");
    697error2:
    698	rseq_after_asm_goto();
    699	rseq_bug("expected value comparison failed");
    700#endif
    701}
    702
    703static inline __attribute__((always_inline))
    704int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
    705					 void *dst, void *src, size_t len,
    706					 intptr_t newv, int cpu)
    707{
    708	uint32_t rseq_scratch[3];
    709
    710	RSEQ_INJECT_C(9)
    711
    712	__asm__ __volatile__ goto (
    713		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
    714		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
    715#ifdef RSEQ_COMPARE_TWICE
    716		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
    717		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
    718#endif
    719		"str %[src], %[rseq_scratch0]\n\t"
    720		"str %[dst], %[rseq_scratch1]\n\t"
    721		"str %[len], %[rseq_scratch2]\n\t"
    722		/* Start rseq by storing table entry pointer into rseq_cs. */
    723		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
    724		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
    725		RSEQ_INJECT_ASM(3)
    726		"ldr r0, %[v]\n\t"
    727		"cmp %[expect], r0\n\t"
    728		"bne 5f\n\t"
    729		RSEQ_INJECT_ASM(4)
    730#ifdef RSEQ_COMPARE_TWICE
    731		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
    732		"ldr r0, %[v]\n\t"
    733		"cmp %[expect], r0\n\t"
    734		"bne 7f\n\t"
    735#endif
    736		/* try memcpy */
    737		"cmp %[len], #0\n\t" \
    738		"beq 333f\n\t" \
    739		"222:\n\t" \
    740		"ldrb %%r0, [%[src]]\n\t" \
    741		"strb %%r0, [%[dst]]\n\t" \
    742		"adds %[src], #1\n\t" \
    743		"adds %[dst], #1\n\t" \
    744		"subs %[len], #1\n\t" \
    745		"bne 222b\n\t" \
    746		"333:\n\t" \
    747		RSEQ_INJECT_ASM(5)
    748		"dmb\n\t"	/* full mb provides store-release */
    749		/* final store */
    750		"str %[newv], %[v]\n\t"
    751		"2:\n\t"
    752		RSEQ_INJECT_ASM(6)
    753		/* teardown */
    754		"ldr %[len], %[rseq_scratch2]\n\t"
    755		"ldr %[dst], %[rseq_scratch1]\n\t"
    756		"ldr %[src], %[rseq_scratch0]\n\t"
    757		"b 8f\n\t"
    758		RSEQ_ASM_DEFINE_ABORT(3, 4,
    759				      /* teardown */
    760				      "ldr %[len], %[rseq_scratch2]\n\t"
    761				      "ldr %[dst], %[rseq_scratch1]\n\t"
    762				      "ldr %[src], %[rseq_scratch0]\n\t",
    763				      abort, 1b, 2b, 4f)
    764		RSEQ_ASM_DEFINE_CMPFAIL(5,
    765					/* teardown */
    766					"ldr %[len], %[rseq_scratch2]\n\t"
    767					"ldr %[dst], %[rseq_scratch1]\n\t"
    768					"ldr %[src], %[rseq_scratch0]\n\t",
    769					cmpfail)
    770#ifdef RSEQ_COMPARE_TWICE
    771		RSEQ_ASM_DEFINE_CMPFAIL(6,
    772					/* teardown */
    773					"ldr %[len], %[rseq_scratch2]\n\t"
    774					"ldr %[dst], %[rseq_scratch1]\n\t"
    775					"ldr %[src], %[rseq_scratch0]\n\t",
    776					error1)
    777		RSEQ_ASM_DEFINE_CMPFAIL(7,
    778					/* teardown */
    779					"ldr %[len], %[rseq_scratch2]\n\t"
    780					"ldr %[dst], %[rseq_scratch1]\n\t"
    781					"ldr %[src], %[rseq_scratch0]\n\t",
    782					error2)
    783#endif
    784		"8:\n\t"
    785		: /* gcc asm goto does not allow outputs */
    786		: [cpu_id]		"r" (cpu),
    787		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
    788		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
    789		  /* final store input */
    790		  [v]			"m" (*v),
    791		  [expect]		"r" (expect),
    792		  [newv]		"r" (newv),
    793		  /* try memcpy input */
    794		  [dst]			"r" (dst),
    795		  [src]			"r" (src),
    796		  [len]			"r" (len),
    797		  [rseq_scratch0]	"m" (rseq_scratch[0]),
    798		  [rseq_scratch1]	"m" (rseq_scratch[1]),
    799		  [rseq_scratch2]	"m" (rseq_scratch[2])
    800		  RSEQ_INJECT_INPUT
    801		: "r0", "memory", "cc"
    802		  RSEQ_INJECT_CLOBBER
    803		: abort, cmpfail
    804#ifdef RSEQ_COMPARE_TWICE
    805		  , error1, error2
    806#endif
    807	);
    808	rseq_after_asm_goto();
    809	return 0;
    810abort:
    811	rseq_after_asm_goto();
    812	RSEQ_INJECT_FAILED
    813	return -1;
    814cmpfail:
    815	rseq_after_asm_goto();
    816	return 1;
    817#ifdef RSEQ_COMPARE_TWICE
    818error1:
    819	rseq_after_asm_goto();
    820	rseq_bug("cpu_id comparison failed");
    821error2:
    822	rseq_after_asm_goto();
    823	rseq_bug("expected value comparison failed");
    824#endif
    825}
    826
    827#endif /* !RSEQ_SKIP_FASTPATH */