cmpxchg.h (9262B)
1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2014 Regents of the University of California 4 */ 5 6#ifndef _ASM_RISCV_CMPXCHG_H 7#define _ASM_RISCV_CMPXCHG_H 8 9#include <linux/bug.h> 10 11#include <asm/barrier.h> 12#include <asm/fence.h> 13 14#define __xchg_relaxed(ptr, new, size) \ 15({ \ 16 __typeof__(ptr) __ptr = (ptr); \ 17 __typeof__(new) __new = (new); \ 18 __typeof__(*(ptr)) __ret; \ 19 switch (size) { \ 20 case 4: \ 21 __asm__ __volatile__ ( \ 22 " amoswap.w %0, %2, %1\n" \ 23 : "=r" (__ret), "+A" (*__ptr) \ 24 : "r" (__new) \ 25 : "memory"); \ 26 break; \ 27 case 8: \ 28 __asm__ __volatile__ ( \ 29 " amoswap.d %0, %2, %1\n" \ 30 : "=r" (__ret), "+A" (*__ptr) \ 31 : "r" (__new) \ 32 : "memory"); \ 33 break; \ 34 default: \ 35 BUILD_BUG(); \ 36 } \ 37 __ret; \ 38}) 39 40#define arch_xchg_relaxed(ptr, x) \ 41({ \ 42 __typeof__(*(ptr)) _x_ = (x); \ 43 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ 44 _x_, sizeof(*(ptr))); \ 45}) 46 47#define __xchg_acquire(ptr, new, size) \ 48({ \ 49 __typeof__(ptr) __ptr = (ptr); \ 50 __typeof__(new) __new = (new); \ 51 __typeof__(*(ptr)) __ret; \ 52 switch (size) { \ 53 case 4: \ 54 __asm__ __volatile__ ( \ 55 " amoswap.w %0, %2, %1\n" \ 56 RISCV_ACQUIRE_BARRIER \ 57 : "=r" (__ret), "+A" (*__ptr) \ 58 : "r" (__new) \ 59 : "memory"); \ 60 break; \ 61 case 8: \ 62 __asm__ __volatile__ ( \ 63 " amoswap.d %0, %2, %1\n" \ 64 RISCV_ACQUIRE_BARRIER \ 65 : "=r" (__ret), "+A" (*__ptr) \ 66 : "r" (__new) \ 67 : "memory"); \ 68 break; \ 69 default: \ 70 BUILD_BUG(); \ 71 } \ 72 __ret; \ 73}) 74 75#define arch_xchg_acquire(ptr, x) \ 76({ \ 77 __typeof__(*(ptr)) _x_ = (x); \ 78 (__typeof__(*(ptr))) __xchg_acquire((ptr), \ 79 _x_, sizeof(*(ptr))); \ 80}) 81 82#define __xchg_release(ptr, new, size) \ 83({ \ 84 __typeof__(ptr) __ptr = (ptr); \ 85 __typeof__(new) __new = (new); \ 86 __typeof__(*(ptr)) __ret; \ 87 switch (size) { \ 88 case 4: \ 89 __asm__ __volatile__ ( \ 90 RISCV_RELEASE_BARRIER \ 91 " amoswap.w %0, %2, %1\n" \ 92 : "=r" (__ret), "+A" (*__ptr) \ 93 : "r" (__new) \ 94 : "memory"); \ 95 break; \ 96 case 8: \ 97 __asm__ __volatile__ ( \ 98 RISCV_RELEASE_BARRIER \ 99 " amoswap.d %0, %2, %1\n" \ 100 : "=r" (__ret), "+A" (*__ptr) \ 101 : "r" (__new) \ 102 : "memory"); \ 103 break; \ 104 default: \ 105 BUILD_BUG(); \ 106 } \ 107 __ret; \ 108}) 109 110#define arch_xchg_release(ptr, x) \ 111({ \ 112 __typeof__(*(ptr)) _x_ = (x); \ 113 (__typeof__(*(ptr))) __xchg_release((ptr), \ 114 _x_, sizeof(*(ptr))); \ 115}) 116 117#define __xchg(ptr, new, size) \ 118({ \ 119 __typeof__(ptr) __ptr = (ptr); \ 120 __typeof__(new) __new = (new); \ 121 __typeof__(*(ptr)) __ret; \ 122 switch (size) { \ 123 case 4: \ 124 __asm__ __volatile__ ( \ 125 " amoswap.w.aqrl %0, %2, %1\n" \ 126 : "=r" (__ret), "+A" (*__ptr) \ 127 : "r" (__new) \ 128 : "memory"); \ 129 break; \ 130 case 8: \ 131 __asm__ __volatile__ ( \ 132 " amoswap.d.aqrl %0, %2, %1\n" \ 133 : "=r" (__ret), "+A" (*__ptr) \ 134 : "r" (__new) \ 135 : "memory"); \ 136 break; \ 137 default: \ 138 BUILD_BUG(); \ 139 } \ 140 __ret; \ 141}) 142 143#define arch_xchg(ptr, x) \ 144({ \ 145 __typeof__(*(ptr)) _x_ = (x); \ 146 (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \ 147}) 148 149#define xchg32(ptr, x) \ 150({ \ 151 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 152 arch_xchg((ptr), (x)); \ 153}) 154 155#define xchg64(ptr, x) \ 156({ \ 157 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 158 arch_xchg((ptr), (x)); \ 159}) 160 161/* 162 * Atomic compare and exchange. Compare OLD with MEM, if identical, 163 * store NEW in MEM. Return the initial value in MEM. Success is 164 * indicated by comparing RETURN with OLD. 165 */ 166#define __cmpxchg_relaxed(ptr, old, new, size) \ 167({ \ 168 __typeof__(ptr) __ptr = (ptr); \ 169 __typeof__(*(ptr)) __old = (old); \ 170 __typeof__(*(ptr)) __new = (new); \ 171 __typeof__(*(ptr)) __ret; \ 172 register unsigned int __rc; \ 173 switch (size) { \ 174 case 4: \ 175 __asm__ __volatile__ ( \ 176 "0: lr.w %0, %2\n" \ 177 " bne %0, %z3, 1f\n" \ 178 " sc.w %1, %z4, %2\n" \ 179 " bnez %1, 0b\n" \ 180 "1:\n" \ 181 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 182 : "rJ" ((long)__old), "rJ" (__new) \ 183 : "memory"); \ 184 break; \ 185 case 8: \ 186 __asm__ __volatile__ ( \ 187 "0: lr.d %0, %2\n" \ 188 " bne %0, %z3, 1f\n" \ 189 " sc.d %1, %z4, %2\n" \ 190 " bnez %1, 0b\n" \ 191 "1:\n" \ 192 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 193 : "rJ" (__old), "rJ" (__new) \ 194 : "memory"); \ 195 break; \ 196 default: \ 197 BUILD_BUG(); \ 198 } \ 199 __ret; \ 200}) 201 202#define arch_cmpxchg_relaxed(ptr, o, n) \ 203({ \ 204 __typeof__(*(ptr)) _o_ = (o); \ 205 __typeof__(*(ptr)) _n_ = (n); \ 206 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \ 207 _o_, _n_, sizeof(*(ptr))); \ 208}) 209 210#define __cmpxchg_acquire(ptr, old, new, size) \ 211({ \ 212 __typeof__(ptr) __ptr = (ptr); \ 213 __typeof__(*(ptr)) __old = (old); \ 214 __typeof__(*(ptr)) __new = (new); \ 215 __typeof__(*(ptr)) __ret; \ 216 register unsigned int __rc; \ 217 switch (size) { \ 218 case 4: \ 219 __asm__ __volatile__ ( \ 220 "0: lr.w %0, %2\n" \ 221 " bne %0, %z3, 1f\n" \ 222 " sc.w %1, %z4, %2\n" \ 223 " bnez %1, 0b\n" \ 224 RISCV_ACQUIRE_BARRIER \ 225 "1:\n" \ 226 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 227 : "rJ" ((long)__old), "rJ" (__new) \ 228 : "memory"); \ 229 break; \ 230 case 8: \ 231 __asm__ __volatile__ ( \ 232 "0: lr.d %0, %2\n" \ 233 " bne %0, %z3, 1f\n" \ 234 " sc.d %1, %z4, %2\n" \ 235 " bnez %1, 0b\n" \ 236 RISCV_ACQUIRE_BARRIER \ 237 "1:\n" \ 238 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 239 : "rJ" (__old), "rJ" (__new) \ 240 : "memory"); \ 241 break; \ 242 default: \ 243 BUILD_BUG(); \ 244 } \ 245 __ret; \ 246}) 247 248#define arch_cmpxchg_acquire(ptr, o, n) \ 249({ \ 250 __typeof__(*(ptr)) _o_ = (o); \ 251 __typeof__(*(ptr)) _n_ = (n); \ 252 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ 253 _o_, _n_, sizeof(*(ptr))); \ 254}) 255 256#define __cmpxchg_release(ptr, old, new, size) \ 257({ \ 258 __typeof__(ptr) __ptr = (ptr); \ 259 __typeof__(*(ptr)) __old = (old); \ 260 __typeof__(*(ptr)) __new = (new); \ 261 __typeof__(*(ptr)) __ret; \ 262 register unsigned int __rc; \ 263 switch (size) { \ 264 case 4: \ 265 __asm__ __volatile__ ( \ 266 RISCV_RELEASE_BARRIER \ 267 "0: lr.w %0, %2\n" \ 268 " bne %0, %z3, 1f\n" \ 269 " sc.w %1, %z4, %2\n" \ 270 " bnez %1, 0b\n" \ 271 "1:\n" \ 272 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 273 : "rJ" ((long)__old), "rJ" (__new) \ 274 : "memory"); \ 275 break; \ 276 case 8: \ 277 __asm__ __volatile__ ( \ 278 RISCV_RELEASE_BARRIER \ 279 "0: lr.d %0, %2\n" \ 280 " bne %0, %z3, 1f\n" \ 281 " sc.d %1, %z4, %2\n" \ 282 " bnez %1, 0b\n" \ 283 "1:\n" \ 284 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 285 : "rJ" (__old), "rJ" (__new) \ 286 : "memory"); \ 287 break; \ 288 default: \ 289 BUILD_BUG(); \ 290 } \ 291 __ret; \ 292}) 293 294#define arch_cmpxchg_release(ptr, o, n) \ 295({ \ 296 __typeof__(*(ptr)) _o_ = (o); \ 297 __typeof__(*(ptr)) _n_ = (n); \ 298 (__typeof__(*(ptr))) __cmpxchg_release((ptr), \ 299 _o_, _n_, sizeof(*(ptr))); \ 300}) 301 302#define __cmpxchg(ptr, old, new, size) \ 303({ \ 304 __typeof__(ptr) __ptr = (ptr); \ 305 __typeof__(*(ptr)) __old = (old); \ 306 __typeof__(*(ptr)) __new = (new); \ 307 __typeof__(*(ptr)) __ret; \ 308 register unsigned int __rc; \ 309 switch (size) { \ 310 case 4: \ 311 __asm__ __volatile__ ( \ 312 "0: lr.w %0, %2\n" \ 313 " bne %0, %z3, 1f\n" \ 314 " sc.w.rl %1, %z4, %2\n" \ 315 " bnez %1, 0b\n" \ 316 " fence rw, rw\n" \ 317 "1:\n" \ 318 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 319 : "rJ" ((long)__old), "rJ" (__new) \ 320 : "memory"); \ 321 break; \ 322 case 8: \ 323 __asm__ __volatile__ ( \ 324 "0: lr.d %0, %2\n" \ 325 " bne %0, %z3, 1f\n" \ 326 " sc.d.rl %1, %z4, %2\n" \ 327 " bnez %1, 0b\n" \ 328 " fence rw, rw\n" \ 329 "1:\n" \ 330 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 331 : "rJ" (__old), "rJ" (__new) \ 332 : "memory"); \ 333 break; \ 334 default: \ 335 BUILD_BUG(); \ 336 } \ 337 __ret; \ 338}) 339 340#define arch_cmpxchg(ptr, o, n) \ 341({ \ 342 __typeof__(*(ptr)) _o_ = (o); \ 343 __typeof__(*(ptr)) _n_ = (n); \ 344 (__typeof__(*(ptr))) __cmpxchg((ptr), \ 345 _o_, _n_, sizeof(*(ptr))); \ 346}) 347 348#define arch_cmpxchg_local(ptr, o, n) \ 349 (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) 350 351#define arch_cmpxchg64(ptr, o, n) \ 352({ \ 353 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 354 arch_cmpxchg((ptr), (o), (n)); \ 355}) 356 357#define arch_cmpxchg64_local(ptr, o, n) \ 358({ \ 359 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 360 arch_cmpxchg_relaxed((ptr), (o), (n)); \ 361}) 362 363#endif /* _ASM_RISCV_CMPXCHG_H */