rseq-riscv.h (20366B)
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike 4 * other architectures, the ebreak instruction has no immediate field for 5 * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG. 6 * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it 7 * is an uncommon instruction and will raise an illegal instruction 8 * exception when executed in all modes. 9 */ 10#include <endian.h> 11 12#if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN) 13#define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */ 14#else 15#error "Currently, RSEQ only supports Little-Endian version" 16#endif 17 18#if __riscv_xlen == 64 19#define __REG_SEL(a, b) a 20#elif __riscv_xlen == 32 21#define __REG_SEL(a, b) b 22#endif 23 24#define REG_L __REG_SEL("ld ", "lw ") 25#define REG_S __REG_SEL("sd ", "sw ") 26 27#define RISCV_FENCE(p, s) \ 28 __asm__ __volatile__ ("fence " #p "," #s : : : "memory") 29#define rseq_smp_mb() RISCV_FENCE(rw, rw) 30#define rseq_smp_rmb() RISCV_FENCE(r, r) 31#define rseq_smp_wmb() RISCV_FENCE(w, w) 32#define RSEQ_ASM_TMP_REG_1 "t6" 33#define RSEQ_ASM_TMP_REG_2 "t5" 34#define RSEQ_ASM_TMP_REG_3 "t4" 35#define RSEQ_ASM_TMP_REG_4 "t3" 36 37#define rseq_smp_load_acquire(p) \ 38__extension__ ({ \ 39 __typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ 40 RISCV_FENCE(r, rw) \ 41 ____p1; \ 42}) 43 44#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 45 46#define rseq_smp_store_release(p, v) \ 47do { \ 48 RISCV_FENCE(rw, w); \ 49 RSEQ_WRITE_ONCE(*(p), v); \ 50} while (0) 51 52#ifdef RSEQ_SKIP_FASTPATH 53#include "rseq-skip.h" 54#else /* !RSEQ_SKIP_FASTPATH */ 55 56#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 57 post_commit_offset, abort_ip) \ 58 ".pushsection __rseq_cs, \"aw\"\n" \ 59 ".balign 32\n" \ 60 __rseq_str(label) ":\n" \ 61 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ 62 ".quad " __rseq_str(start_ip) ", " \ 63 __rseq_str(post_commit_offset) ", " \ 64 __rseq_str(abort_ip) "\n" \ 65 ".popsection\n\t" \ 66 ".pushsection __rseq_cs_ptr_array, \"aw\"\n" \ 67 ".quad " __rseq_str(label) "b\n" \ 68 ".popsection\n" 69 70#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 71 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 72 ((post_commit_ip) - (start_ip)), abort_ip) 73 74/* 75 * Exit points of a rseq critical section consist of all instructions outside 76 * of the critical section where a critical section can either branch to or 77 * reach through the normal course of its execution. The abort IP and the 78 * post-commit IP are already part of the __rseq_cs section and should not be 79 * explicitly defined as additional exit points. Knowing all exit points is 80 * useful to assist debuggers stepping over the critical section. 81 */ 82#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 83 ".pushsection __rseq_exit_point_array, \"aw\"\n" \ 84 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \ 85 ".popsection\n" 86 87#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 88 RSEQ_INJECT_ASM(1) \ 89 "la "RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \ 90 REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \ 91 __rseq_str(label) ":\n" 92 93#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 94 "j 222f\n" \ 95 ".balign 4\n" \ 96 ".long " __rseq_str(RSEQ_SIG) "\n" \ 97 __rseq_str(label) ":\n" \ 98 "j %l[" __rseq_str(abort_label) "]\n" \ 99 "222:\n" 100 101#define RSEQ_ASM_OP_STORE(value, var) \ 102 REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" 103 104#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ 105 REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ 106 "bne "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ 107 __rseq_str(label) "\n" 108 109#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ 110 "lw "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ 111 "bne "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ 112 __rseq_str(label) "\n" 113 114#define RSEQ_ASM_OP_CMPNE(var, expect, label) \ 115 REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ 116 "beq "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ 117 __rseq_str(label) "\n" 118 119#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 120 RSEQ_INJECT_ASM(2) \ 121 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label) 122 123#define RSEQ_ASM_OP_R_LOAD(var) \ 124 REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" 125 126#define RSEQ_ASM_OP_R_STORE(var) \ 127 REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" 128 129#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \ 130 "add "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \ 131 RSEQ_ASM_TMP_REG_1 "\n" \ 132 REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n" 133 134#define RSEQ_ASM_OP_R_ADD(count) \ 135 "add "RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \ 136 ", %[" __rseq_str(count) "]\n" 137 138#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ 139 RSEQ_ASM_OP_STORE(value, var) \ 140 __rseq_str(post_commit_label) ":\n" 141 142#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \ 143 "fence rw, w\n" \ 144 RSEQ_ASM_OP_STORE(value, var) \ 145 __rseq_str(post_commit_label) ":\n" 146 147#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ 148 REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ 149 __rseq_str(post_commit_label) ":\n" 150 151#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \ 152 "beqz %[" __rseq_str(len) "], 333f\n" \ 153 "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n" \ 154 "mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n" \ 155 "mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n" \ 156 "222:\n" \ 157 "lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n" \ 158 "sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n" \ 159 "addi " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n" \ 160 "addi " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n" \ 161 "addi " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n" \ 162 "bnez " RSEQ_ASM_TMP_REG_1 ", 222b\n" \ 163 "333:\n" 164 165#define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label) \ 166 "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n" \ 167 RSEQ_ASM_OP_R_ADD(off) \ 168 REG_L RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \ 169 RSEQ_ASM_OP_R_ADD(inc) \ 170 __rseq_str(post_commit_label) ":\n" 171 172static inline __always_inline 173int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 174{ 175 RSEQ_INJECT_C(9) 176 177 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 178 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 179#ifdef RSEQ_COMPARE_TWICE 180 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 181 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 182#endif 183 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 184 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 185 RSEQ_INJECT_ASM(3) 186 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 187 RSEQ_INJECT_ASM(4) 188#ifdef RSEQ_COMPARE_TWICE 189 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 190 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 191#endif 192 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 193 RSEQ_INJECT_ASM(5) 194 RSEQ_ASM_DEFINE_ABORT(4, abort) 195 : /* gcc asm goto does not allow outputs */ 196 : [cpu_id] "r" (cpu), 197 [current_cpu_id] "m" (__rseq_abi.cpu_id), 198 [rseq_cs] "m" (__rseq_abi.rseq_cs), 199 [v] "m" (*v), 200 [expect] "r" (expect), 201 [newv] "r" (newv) 202 RSEQ_INJECT_INPUT 203 : "memory", RSEQ_ASM_TMP_REG_1 204 RSEQ_INJECT_CLOBBER 205 : abort, cmpfail 206#ifdef RSEQ_COMPARE_TWICE 207 , error1, error2 208#endif 209 ); 210 211 return 0; 212abort: 213 RSEQ_INJECT_FAILED 214 return -1; 215cmpfail: 216 return 1; 217#ifdef RSEQ_COMPARE_TWICE 218error1: 219 rseq_bug("cpu_id comparison failed"); 220error2: 221 rseq_bug("expected value comparison failed"); 222#endif 223} 224 225static inline __always_inline 226int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 227 off_t voffp, intptr_t *load, int cpu) 228{ 229 RSEQ_INJECT_C(9) 230 231 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 232 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 233#ifdef RSEQ_COMPARE_TWICE 234 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 235 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 236#endif 237 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 238 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 239 RSEQ_INJECT_ASM(3) 240 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]") 241 RSEQ_INJECT_ASM(4) 242#ifdef RSEQ_COMPARE_TWICE 243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 244 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]") 245#endif 246 RSEQ_ASM_OP_R_LOAD(v) 247 RSEQ_ASM_OP_R_STORE(load) 248 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 249 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 250 RSEQ_INJECT_ASM(5) 251 RSEQ_ASM_DEFINE_ABORT(4, abort) 252 : /* gcc asm goto does not allow outputs */ 253 : [cpu_id] "r" (cpu), 254 [current_cpu_id] "m" (__rseq_abi.cpu_id), 255 [rseq_cs] "m" (__rseq_abi.rseq_cs), 256 [v] "m" (*v), 257 [expectnot] "r" (expectnot), 258 [load] "m" (*load), 259 [voffp] "r" (voffp) 260 RSEQ_INJECT_INPUT 261 : "memory", RSEQ_ASM_TMP_REG_1 262 RSEQ_INJECT_CLOBBER 263 : abort, cmpfail 264#ifdef RSEQ_COMPARE_TWICE 265 , error1, error2 266#endif 267 ); 268 return 0; 269abort: 270 RSEQ_INJECT_FAILED 271 return -1; 272cmpfail: 273 return 1; 274#ifdef RSEQ_COMPARE_TWICE 275error1: 276 rseq_bug("cpu_id comparison failed"); 277error2: 278 rseq_bug("expected value comparison failed"); 279#endif 280} 281 282static inline __always_inline 283int rseq_addv(intptr_t *v, intptr_t count, int cpu) 284{ 285 RSEQ_INJECT_C(9) 286 287 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 288#ifdef RSEQ_COMPARE_TWICE 289 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 290#endif 291 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 292 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 293 RSEQ_INJECT_ASM(3) 294#ifdef RSEQ_COMPARE_TWICE 295 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 296#endif 297 RSEQ_ASM_OP_R_LOAD(v) 298 RSEQ_ASM_OP_R_ADD(count) 299 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 300 RSEQ_INJECT_ASM(4) 301 RSEQ_ASM_DEFINE_ABORT(4, abort) 302 : /* gcc asm goto does not allow outputs */ 303 : [cpu_id] "r" (cpu), 304 [current_cpu_id] "m" (__rseq_abi.cpu_id), 305 [rseq_cs] "m" (__rseq_abi.rseq_cs), 306 [v] "m" (*v), 307 [count] "r" (count) 308 RSEQ_INJECT_INPUT 309 : "memory", RSEQ_ASM_TMP_REG_1 310 RSEQ_INJECT_CLOBBER 311 : abort 312#ifdef RSEQ_COMPARE_TWICE 313 , error1 314#endif 315 ); 316 return 0; 317abort: 318 RSEQ_INJECT_FAILED 319 return -1; 320#ifdef RSEQ_COMPARE_TWICE 321error1: 322 rseq_bug("cpu_id comparison failed"); 323#endif 324} 325 326static inline __always_inline 327int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 328 intptr_t *v2, intptr_t newv2, 329 intptr_t newv, int cpu) 330{ 331 RSEQ_INJECT_C(9) 332 333 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 334 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 335#ifdef RSEQ_COMPARE_TWICE 336 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 337 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 338#endif 339 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 340 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 341 RSEQ_INJECT_ASM(3) 342 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 343 RSEQ_INJECT_ASM(4) 344#ifdef RSEQ_COMPARE_TWICE 345 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 346 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 347#endif 348 RSEQ_ASM_OP_STORE(newv2, v2) 349 RSEQ_INJECT_ASM(5) 350 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 351 RSEQ_INJECT_ASM(6) 352 RSEQ_ASM_DEFINE_ABORT(4, abort) 353 : /* gcc asm goto does not allow outputs */ 354 : [cpu_id] "r" (cpu), 355 [current_cpu_id] "m" (__rseq_abi.cpu_id), 356 [rseq_cs] "m" (__rseq_abi.rseq_cs), 357 [expect] "r" (expect), 358 [v] "m" (*v), 359 [newv] "r" (newv), 360 [v2] "m" (*v2), 361 [newv2] "r" (newv2) 362 RSEQ_INJECT_INPUT 363 : "memory", RSEQ_ASM_TMP_REG_1 364 RSEQ_INJECT_CLOBBER 365 : abort, cmpfail 366#ifdef RSEQ_COMPARE_TWICE 367 , error1, error2 368#endif 369 ); 370 371 return 0; 372abort: 373 RSEQ_INJECT_FAILED 374 return -1; 375cmpfail: 376 return 1; 377#ifdef RSEQ_COMPARE_TWICE 378error1: 379 rseq_bug("cpu_id comparison failed"); 380error2: 381 rseq_bug("expected value comparison failed"); 382#endif 383} 384 385static inline __always_inline 386int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 387 intptr_t *v2, intptr_t newv2, 388 intptr_t newv, int cpu) 389{ 390 RSEQ_INJECT_C(9) 391 392 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 393 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 394#ifdef RSEQ_COMPARE_TWICE 395 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 396 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 397#endif 398 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 399 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 400 RSEQ_INJECT_ASM(3) 401 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 402 RSEQ_INJECT_ASM(4) 403#ifdef RSEQ_COMPARE_TWICE 404 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 405 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 406#endif 407 RSEQ_ASM_OP_STORE(newv2, v2) 408 RSEQ_INJECT_ASM(5) 409 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 410 RSEQ_INJECT_ASM(6) 411 RSEQ_ASM_DEFINE_ABORT(4, abort) 412 : /* gcc asm goto does not allow outputs */ 413 : [cpu_id] "r" (cpu), 414 [current_cpu_id] "m" (__rseq_abi.cpu_id), 415 [rseq_cs] "m" (__rseq_abi.rseq_cs), 416 [expect] "r" (expect), 417 [v] "m" (*v), 418 [newv] "r" (newv), 419 [v2] "m" (*v2), 420 [newv2] "r" (newv2) 421 RSEQ_INJECT_INPUT 422 : "memory", RSEQ_ASM_TMP_REG_1 423 RSEQ_INJECT_CLOBBER 424 : abort, cmpfail 425#ifdef RSEQ_COMPARE_TWICE 426 , error1, error2 427#endif 428 ); 429 430 return 0; 431abort: 432 RSEQ_INJECT_FAILED 433 return -1; 434cmpfail: 435 return 1; 436#ifdef RSEQ_COMPARE_TWICE 437error1: 438 rseq_bug("cpu_id comparison failed"); 439error2: 440 rseq_bug("expected value comparison failed"); 441#endif 442} 443 444static inline __always_inline 445int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 446 intptr_t *v2, intptr_t expect2, 447 intptr_t newv, int cpu) 448{ 449 RSEQ_INJECT_C(9) 450 451 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 452 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 453#ifdef RSEQ_COMPARE_TWICE 454 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 455 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 456 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]") 457#endif 458 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 459 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 460 RSEQ_INJECT_ASM(3) 461 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 462 RSEQ_INJECT_ASM(4) 463 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]") 464 RSEQ_INJECT_ASM(5) 465#ifdef RSEQ_COMPARE_TWICE 466 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 467 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 468 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]") 469#endif 470 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 471 RSEQ_INJECT_ASM(6) 472 RSEQ_ASM_DEFINE_ABORT(4, abort) 473 : /* gcc asm goto does not allow outputs */ 474 : [cpu_id] "r" (cpu), 475 [current_cpu_id] "m" (__rseq_abi.cpu_id), 476 [rseq_cs] "m" (__rseq_abi.rseq_cs), 477 [v] "m" (*v), 478 [expect] "r" (expect), 479 [v2] "m" (*v2), 480 [expect2] "r" (expect2), 481 [newv] "r" (newv) 482 RSEQ_INJECT_INPUT 483 : "memory", RSEQ_ASM_TMP_REG_1 484 RSEQ_INJECT_CLOBBER 485 : abort, cmpfail 486#ifdef RSEQ_COMPARE_TWICE 487 , error1, error2, error3 488#endif 489 ); 490 491 return 0; 492abort: 493 RSEQ_INJECT_FAILED 494 return -1; 495cmpfail: 496 return 1; 497#ifdef RSEQ_COMPARE_TWICE 498error1: 499 rseq_bug("cpu_id comparison failed"); 500error2: 501 rseq_bug("expected value comparison failed"); 502error3: 503 rseq_bug("2nd expected value comparison failed"); 504#endif 505} 506 507static inline __always_inline 508int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 509 void *dst, void *src, size_t len, 510 intptr_t newv, int cpu) 511{ 512 RSEQ_INJECT_C(9) 513 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 514 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 515#ifdef RSEQ_COMPARE_TWICE 516 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 517 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 518#endif 519 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 520 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 521 RSEQ_INJECT_ASM(3) 522 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 523 RSEQ_INJECT_ASM(4) 524#ifdef RSEQ_COMPARE_TWICE 525 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 526 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 527#endif 528 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 529 RSEQ_INJECT_ASM(5) 530 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 531 RSEQ_INJECT_ASM(6) 532 RSEQ_ASM_DEFINE_ABORT(4, abort) 533 : /* gcc asm goto does not allow outputs */ 534 : [cpu_id] "r" (cpu), 535 [current_cpu_id] "m" (__rseq_abi.cpu_id), 536 [rseq_cs] "m" (__rseq_abi.rseq_cs), 537 [expect] "r" (expect), 538 [v] "m" (*v), 539 [newv] "r" (newv), 540 [dst] "r" (dst), 541 [src] "r" (src), 542 [len] "r" (len) 543 RSEQ_INJECT_INPUT 544 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, 545 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 546 RSEQ_INJECT_CLOBBER 547 : abort, cmpfail 548#ifdef RSEQ_COMPARE_TWICE 549 , error1, error2 550#endif 551 ); 552 553 return 0; 554abort: 555 RSEQ_INJECT_FAILED 556 return -1; 557cmpfail: 558 return 1; 559#ifdef RSEQ_COMPARE_TWICE 560error1: 561 rseq_bug("cpu_id comparison failed"); 562error2: 563 rseq_bug("expected value comparison failed"); 564#endif 565} 566 567static inline __always_inline 568int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 569 void *dst, void *src, size_t len, 570 intptr_t newv, int cpu) 571{ 572 RSEQ_INJECT_C(9) 573 574 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 575 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 576#ifdef RSEQ_COMPARE_TWICE 577 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 578 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 579#endif 580 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 581 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 582 RSEQ_INJECT_ASM(3) 583 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 584 RSEQ_INJECT_ASM(4) 585#ifdef RSEQ_COMPARE_TWICE 586 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 587 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 588#endif 589 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 590 RSEQ_INJECT_ASM(5) 591 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 592 RSEQ_INJECT_ASM(6) 593 RSEQ_ASM_DEFINE_ABORT(4, abort) 594 : /* gcc asm goto does not allow outputs */ 595 : [cpu_id] "r" (cpu), 596 [current_cpu_id] "m" (__rseq_abi.cpu_id), 597 [rseq_cs] "m" (__rseq_abi.rseq_cs), 598 [expect] "r" (expect), 599 [v] "m" (*v), 600 [newv] "r" (newv), 601 [dst] "r" (dst), 602 [src] "r" (src), 603 [len] "r" (len) 604 RSEQ_INJECT_INPUT 605 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, 606 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 607 RSEQ_INJECT_CLOBBER 608 : abort, cmpfail 609#ifdef RSEQ_COMPARE_TWICE 610 , error1, error2 611#endif 612 ); 613 614 return 0; 615abort: 616 RSEQ_INJECT_FAILED 617 return -1; 618cmpfail: 619 return 1; 620#ifdef RSEQ_COMPARE_TWICE 621error1: 622 rseq_bug("cpu_id comparison failed"); 623error2: 624 rseq_bug("expected value comparison failed"); 625#endif 626} 627 628#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 629 630/* 631 * pval = *(ptr+off) 632 * *pval += inc; 633 */ 634static inline __always_inline 635int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu) 636{ 637 RSEQ_INJECT_C(9) 638 639 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 640#ifdef RSEQ_COMPARE_TWICE 641 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 642#endif 643 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 644 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 645 RSEQ_INJECT_ASM(3) 646#ifdef RSEQ_COMPARE_TWICE 647 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 648#endif 649 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3) 650 RSEQ_INJECT_ASM(4) 651 RSEQ_ASM_DEFINE_ABORT(4, abort) 652 : /* gcc asm goto does not allow outputs */ 653 : [cpu_id] "r" (cpu), 654 [current_cpu_id] "m" (__rseq_abi.cpu_id), 655 [rseq_cs] "m" (__rseq_abi.rseq_cs), 656 [ptr] "r" (ptr), 657 [off] "er" (off), 658 [inc] "er" (inc) 659 RSEQ_INJECT_INPUT 660 : "memory", RSEQ_ASM_TMP_REG_1 661 RSEQ_INJECT_CLOBBER 662 : abort 663#ifdef RSEQ_COMPARE_TWICE 664 , error1 665#endif 666 ); 667 return 0; 668abort: 669 RSEQ_INJECT_FAILED 670 return -1; 671#ifdef RSEQ_COMPARE_TWICE 672error1: 673 rseq_bug("cpu_id comparison failed"); 674#endif 675} 676 677#endif /* !RSEQ_SKIP_FASTPATH */