atomic.h (8749B)
1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _ALPHA_ATOMIC_H 3#define _ALPHA_ATOMIC_H 4 5#include <linux/types.h> 6#include <asm/barrier.h> 7#include <asm/cmpxchg.h> 8 9/* 10 * Atomic operations that C can't guarantee us. Useful for 11 * resource counting etc... 12 * 13 * But use these as seldom as possible since they are much slower 14 * than regular operations. 15 */ 16 17/* 18 * To ensure dependency ordering is preserved for the _relaxed and 19 * _release atomics, an smp_mb() is unconditionally inserted into the 20 * _relaxed variants, which are used to build the barriered versions. 21 * Avoid redundant back-to-back fences in the _acquire and _fence 22 * versions. 23 */ 24#define __atomic_acquire_fence() 25#define __atomic_post_full_fence() 26 27#define ATOMIC64_INIT(i) { (i) } 28 29#define arch_atomic_read(v) READ_ONCE((v)->counter) 30#define arch_atomic64_read(v) READ_ONCE((v)->counter) 31 32#define arch_atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) 33#define arch_atomic64_set(v,i) WRITE_ONCE((v)->counter, (i)) 34 35/* 36 * To get proper branch prediction for the main line, we must branch 37 * forward to code at the end of this object's .text section, then 38 * branch back to restart the operation. 39 */ 40 41#define ATOMIC_OP(op, asm_op) \ 42static __inline__ void arch_atomic_##op(int i, atomic_t * v) \ 43{ \ 44 unsigned long temp; \ 45 __asm__ __volatile__( \ 46 "1: ldl_l %0,%1\n" \ 47 " " #asm_op " %0,%2,%0\n" \ 48 " stl_c %0,%1\n" \ 49 " beq %0,2f\n" \ 50 ".subsection 2\n" \ 51 "2: br 1b\n" \ 52 ".previous" \ 53 :"=&r" (temp), "=m" (v->counter) \ 54 :"Ir" (i), "m" (v->counter)); \ 55} \ 56 57#define ATOMIC_OP_RETURN(op, asm_op) \ 58static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ 59{ \ 60 long temp, result; \ 61 __asm__ __volatile__( \ 62 "1: ldl_l %0,%1\n" \ 63 " " #asm_op " %0,%3,%2\n" \ 64 " " #asm_op " %0,%3,%0\n" \ 65 " stl_c %0,%1\n" \ 66 " beq %0,2f\n" \ 67 ".subsection 2\n" \ 68 "2: br 1b\n" \ 69 ".previous" \ 70 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 71 :"Ir" (i), "m" (v->counter) : "memory"); \ 72 smp_mb(); \ 73 return result; \ 74} 75 76#define ATOMIC_FETCH_OP(op, asm_op) \ 77static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ 78{ \ 79 long temp, result; \ 80 __asm__ __volatile__( \ 81 "1: ldl_l %2,%1\n" \ 82 " " #asm_op " %2,%3,%0\n" \ 83 " stl_c %0,%1\n" \ 84 " beq %0,2f\n" \ 85 ".subsection 2\n" \ 86 "2: br 1b\n" \ 87 ".previous" \ 88 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 89 :"Ir" (i), "m" (v->counter) : "memory"); \ 90 smp_mb(); \ 91 return result; \ 92} 93 94#define ATOMIC64_OP(op, asm_op) \ 95static __inline__ void arch_atomic64_##op(s64 i, atomic64_t * v) \ 96{ \ 97 s64 temp; \ 98 __asm__ __volatile__( \ 99 "1: ldq_l %0,%1\n" \ 100 " " #asm_op " %0,%2,%0\n" \ 101 " stq_c %0,%1\n" \ 102 " beq %0,2f\n" \ 103 ".subsection 2\n" \ 104 "2: br 1b\n" \ 105 ".previous" \ 106 :"=&r" (temp), "=m" (v->counter) \ 107 :"Ir" (i), "m" (v->counter)); \ 108} \ 109 110#define ATOMIC64_OP_RETURN(op, asm_op) \ 111static __inline__ s64 \ 112arch_atomic64_##op##_return_relaxed(s64 i, atomic64_t * v) \ 113{ \ 114 s64 temp, result; \ 115 __asm__ __volatile__( \ 116 "1: ldq_l %0,%1\n" \ 117 " " #asm_op " %0,%3,%2\n" \ 118 " " #asm_op " %0,%3,%0\n" \ 119 " stq_c %0,%1\n" \ 120 " beq %0,2f\n" \ 121 ".subsection 2\n" \ 122 "2: br 1b\n" \ 123 ".previous" \ 124 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 125 :"Ir" (i), "m" (v->counter) : "memory"); \ 126 smp_mb(); \ 127 return result; \ 128} 129 130#define ATOMIC64_FETCH_OP(op, asm_op) \ 131static __inline__ s64 \ 132arch_atomic64_fetch_##op##_relaxed(s64 i, atomic64_t * v) \ 133{ \ 134 s64 temp, result; \ 135 __asm__ __volatile__( \ 136 "1: ldq_l %2,%1\n" \ 137 " " #asm_op " %2,%3,%0\n" \ 138 " stq_c %0,%1\n" \ 139 " beq %0,2f\n" \ 140 ".subsection 2\n" \ 141 "2: br 1b\n" \ 142 ".previous" \ 143 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 144 :"Ir" (i), "m" (v->counter) : "memory"); \ 145 smp_mb(); \ 146 return result; \ 147} 148 149#define ATOMIC_OPS(op) \ 150 ATOMIC_OP(op, op##l) \ 151 ATOMIC_OP_RETURN(op, op##l) \ 152 ATOMIC_FETCH_OP(op, op##l) \ 153 ATOMIC64_OP(op, op##q) \ 154 ATOMIC64_OP_RETURN(op, op##q) \ 155 ATOMIC64_FETCH_OP(op, op##q) 156 157ATOMIC_OPS(add) 158ATOMIC_OPS(sub) 159 160#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 161#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 162#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 163#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 164 165#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 166#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 167#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 168#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 169 170#define arch_atomic_andnot arch_atomic_andnot 171#define arch_atomic64_andnot arch_atomic64_andnot 172 173#undef ATOMIC_OPS 174#define ATOMIC_OPS(op, asm) \ 175 ATOMIC_OP(op, asm) \ 176 ATOMIC_FETCH_OP(op, asm) \ 177 ATOMIC64_OP(op, asm) \ 178 ATOMIC64_FETCH_OP(op, asm) 179 180ATOMIC_OPS(and, and) 181ATOMIC_OPS(andnot, bic) 182ATOMIC_OPS(or, bis) 183ATOMIC_OPS(xor, xor) 184 185#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 186#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed 187#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 188#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 189 190#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 191#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 192#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 193#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 194 195#undef ATOMIC_OPS 196#undef ATOMIC64_FETCH_OP 197#undef ATOMIC64_OP_RETURN 198#undef ATOMIC64_OP 199#undef ATOMIC_FETCH_OP 200#undef ATOMIC_OP_RETURN 201#undef ATOMIC_OP 202 203#define arch_atomic64_cmpxchg(v, old, new) \ 204 (arch_cmpxchg(&((v)->counter), old, new)) 205#define arch_atomic64_xchg(v, new) \ 206 (arch_xchg(&((v)->counter), new)) 207 208#define arch_atomic_cmpxchg(v, old, new) \ 209 (arch_cmpxchg(&((v)->counter), old, new)) 210#define arch_atomic_xchg(v, new) \ 211 (arch_xchg(&((v)->counter), new)) 212 213/** 214 * arch_atomic_fetch_add_unless - add unless the number is a given value 215 * @v: pointer of type atomic_t 216 * @a: the amount to add to v... 217 * @u: ...unless v is equal to u. 218 * 219 * Atomically adds @a to @v, so long as it was not @u. 220 * Returns the old value of @v. 221 */ 222static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 223{ 224 int c, new, old; 225 smp_mb(); 226 __asm__ __volatile__( 227 "1: ldl_l %[old],%[mem]\n" 228 " cmpeq %[old],%[u],%[c]\n" 229 " addl %[old],%[a],%[new]\n" 230 " bne %[c],2f\n" 231 " stl_c %[new],%[mem]\n" 232 " beq %[new],3f\n" 233 "2:\n" 234 ".subsection 2\n" 235 "3: br 1b\n" 236 ".previous" 237 : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c) 238 : [mem] "m"(*v), [a] "rI"(a), [u] "rI"((long)u) 239 : "memory"); 240 smp_mb(); 241 return old; 242} 243#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 244 245/** 246 * arch_atomic64_fetch_add_unless - add unless the number is a given value 247 * @v: pointer of type atomic64_t 248 * @a: the amount to add to v... 249 * @u: ...unless v is equal to u. 250 * 251 * Atomically adds @a to @v, so long as it was not @u. 252 * Returns the old value of @v. 253 */ 254static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 255{ 256 s64 c, new, old; 257 smp_mb(); 258 __asm__ __volatile__( 259 "1: ldq_l %[old],%[mem]\n" 260 " cmpeq %[old],%[u],%[c]\n" 261 " addq %[old],%[a],%[new]\n" 262 " bne %[c],2f\n" 263 " stq_c %[new],%[mem]\n" 264 " beq %[new],3f\n" 265 "2:\n" 266 ".subsection 2\n" 267 "3: br 1b\n" 268 ".previous" 269 : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c) 270 : [mem] "m"(*v), [a] "rI"(a), [u] "rI"(u) 271 : "memory"); 272 smp_mb(); 273 return old; 274} 275#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 276 277/* 278 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive 279 * @v: pointer of type atomic_t 280 * 281 * The function returns the old value of *v minus 1, even if 282 * the atomic variable, v, was not decremented. 283 */ 284static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 285{ 286 s64 old, tmp; 287 smp_mb(); 288 __asm__ __volatile__( 289 "1: ldq_l %[old],%[mem]\n" 290 " subq %[old],1,%[tmp]\n" 291 " ble %[old],2f\n" 292 " stq_c %[tmp],%[mem]\n" 293 " beq %[tmp],3f\n" 294 "2:\n" 295 ".subsection 2\n" 296 "3: br 1b\n" 297 ".previous" 298 : [old] "=&r"(old), [tmp] "=&r"(tmp) 299 : [mem] "m"(*v) 300 : "memory"); 301 smp_mb(); 302 return old - 1; 303} 304#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 305 306#endif /* _ALPHA_ATOMIC_H */