atomic-arch-fallback.h (65218B)
1// SPDX-License-Identifier: GPL-2.0 2 3// Generated by scripts/atomic/gen-atomic-fallback.sh 4// DO NOT MODIFY THIS FILE DIRECTLY 5 6#ifndef _LINUX_ATOMIC_FALLBACK_H 7#define _LINUX_ATOMIC_FALLBACK_H 8 9#include <linux/compiler.h> 10 11#ifndef arch_xchg_relaxed 12#define arch_xchg_acquire arch_xchg 13#define arch_xchg_release arch_xchg 14#define arch_xchg_relaxed arch_xchg 15#else /* arch_xchg_relaxed */ 16 17#ifndef arch_xchg_acquire 18#define arch_xchg_acquire(...) \ 19 __atomic_op_acquire(arch_xchg, __VA_ARGS__) 20#endif 21 22#ifndef arch_xchg_release 23#define arch_xchg_release(...) \ 24 __atomic_op_release(arch_xchg, __VA_ARGS__) 25#endif 26 27#ifndef arch_xchg 28#define arch_xchg(...) \ 29 __atomic_op_fence(arch_xchg, __VA_ARGS__) 30#endif 31 32#endif /* arch_xchg_relaxed */ 33 34#ifndef arch_cmpxchg_relaxed 35#define arch_cmpxchg_acquire arch_cmpxchg 36#define arch_cmpxchg_release arch_cmpxchg 37#define arch_cmpxchg_relaxed arch_cmpxchg 38#else /* arch_cmpxchg_relaxed */ 39 40#ifndef arch_cmpxchg_acquire 41#define arch_cmpxchg_acquire(...) \ 42 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) 43#endif 44 45#ifndef arch_cmpxchg_release 46#define arch_cmpxchg_release(...) \ 47 __atomic_op_release(arch_cmpxchg, __VA_ARGS__) 48#endif 49 50#ifndef arch_cmpxchg 51#define arch_cmpxchg(...) \ 52 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) 53#endif 54 55#endif /* arch_cmpxchg_relaxed */ 56 57#ifndef arch_cmpxchg64_relaxed 58#define arch_cmpxchg64_acquire arch_cmpxchg64 59#define arch_cmpxchg64_release arch_cmpxchg64 60#define arch_cmpxchg64_relaxed arch_cmpxchg64 61#else /* arch_cmpxchg64_relaxed */ 62 63#ifndef arch_cmpxchg64_acquire 64#define arch_cmpxchg64_acquire(...) \ 65 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) 66#endif 67 68#ifndef arch_cmpxchg64_release 69#define arch_cmpxchg64_release(...) \ 70 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) 71#endif 72 73#ifndef arch_cmpxchg64 74#define arch_cmpxchg64(...) \ 75 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) 76#endif 77 78#endif /* arch_cmpxchg64_relaxed */ 79 80#ifndef arch_try_cmpxchg_relaxed 81#ifdef arch_try_cmpxchg 82#define arch_try_cmpxchg_acquire arch_try_cmpxchg 83#define arch_try_cmpxchg_release arch_try_cmpxchg 84#define arch_try_cmpxchg_relaxed arch_try_cmpxchg 85#endif /* arch_try_cmpxchg */ 86 87#ifndef arch_try_cmpxchg 88#define arch_try_cmpxchg(_ptr, _oldp, _new) \ 89({ \ 90 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 91 ___r = arch_cmpxchg((_ptr), ___o, (_new)); \ 92 if (unlikely(___r != ___o)) \ 93 *___op = ___r; \ 94 likely(___r == ___o); \ 95}) 96#endif /* arch_try_cmpxchg */ 97 98#ifndef arch_try_cmpxchg_acquire 99#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \ 100({ \ 101 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 102 ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \ 103 if (unlikely(___r != ___o)) \ 104 *___op = ___r; \ 105 likely(___r == ___o); \ 106}) 107#endif /* arch_try_cmpxchg_acquire */ 108 109#ifndef arch_try_cmpxchg_release 110#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \ 111({ \ 112 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 113 ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \ 114 if (unlikely(___r != ___o)) \ 115 *___op = ___r; \ 116 likely(___r == ___o); \ 117}) 118#endif /* arch_try_cmpxchg_release */ 119 120#ifndef arch_try_cmpxchg_relaxed 121#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ 122({ \ 123 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 124 ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \ 125 if (unlikely(___r != ___o)) \ 126 *___op = ___r; \ 127 likely(___r == ___o); \ 128}) 129#endif /* arch_try_cmpxchg_relaxed */ 130 131#else /* arch_try_cmpxchg_relaxed */ 132 133#ifndef arch_try_cmpxchg_acquire 134#define arch_try_cmpxchg_acquire(...) \ 135 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) 136#endif 137 138#ifndef arch_try_cmpxchg_release 139#define arch_try_cmpxchg_release(...) \ 140 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) 141#endif 142 143#ifndef arch_try_cmpxchg 144#define arch_try_cmpxchg(...) \ 145 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__) 146#endif 147 148#endif /* arch_try_cmpxchg_relaxed */ 149 150#ifndef arch_try_cmpxchg64_relaxed 151#ifdef arch_try_cmpxchg64 152#define arch_try_cmpxchg64_acquire arch_try_cmpxchg64 153#define arch_try_cmpxchg64_release arch_try_cmpxchg64 154#define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64 155#endif /* arch_try_cmpxchg64 */ 156 157#ifndef arch_try_cmpxchg64 158#define arch_try_cmpxchg64(_ptr, _oldp, _new) \ 159({ \ 160 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 161 ___r = arch_cmpxchg64((_ptr), ___o, (_new)); \ 162 if (unlikely(___r != ___o)) \ 163 *___op = ___r; \ 164 likely(___r == ___o); \ 165}) 166#endif /* arch_try_cmpxchg64 */ 167 168#ifndef arch_try_cmpxchg64_acquire 169#define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ 170({ \ 171 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 172 ___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \ 173 if (unlikely(___r != ___o)) \ 174 *___op = ___r; \ 175 likely(___r == ___o); \ 176}) 177#endif /* arch_try_cmpxchg64_acquire */ 178 179#ifndef arch_try_cmpxchg64_release 180#define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \ 181({ \ 182 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 183 ___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \ 184 if (unlikely(___r != ___o)) \ 185 *___op = ___r; \ 186 likely(___r == ___o); \ 187}) 188#endif /* arch_try_cmpxchg64_release */ 189 190#ifndef arch_try_cmpxchg64_relaxed 191#define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ 192({ \ 193 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 194 ___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ 195 if (unlikely(___r != ___o)) \ 196 *___op = ___r; \ 197 likely(___r == ___o); \ 198}) 199#endif /* arch_try_cmpxchg64_relaxed */ 200 201#else /* arch_try_cmpxchg64_relaxed */ 202 203#ifndef arch_try_cmpxchg64_acquire 204#define arch_try_cmpxchg64_acquire(...) \ 205 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) 206#endif 207 208#ifndef arch_try_cmpxchg64_release 209#define arch_try_cmpxchg64_release(...) \ 210 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) 211#endif 212 213#ifndef arch_try_cmpxchg64 214#define arch_try_cmpxchg64(...) \ 215 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__) 216#endif 217 218#endif /* arch_try_cmpxchg64_relaxed */ 219 220#ifndef arch_atomic_read_acquire 221static __always_inline int 222arch_atomic_read_acquire(const atomic_t *v) 223{ 224 int ret; 225 226 if (__native_word(atomic_t)) { 227 ret = smp_load_acquire(&(v)->counter); 228 } else { 229 ret = arch_atomic_read(v); 230 __atomic_acquire_fence(); 231 } 232 233 return ret; 234} 235#define arch_atomic_read_acquire arch_atomic_read_acquire 236#endif 237 238#ifndef arch_atomic_set_release 239static __always_inline void 240arch_atomic_set_release(atomic_t *v, int i) 241{ 242 if (__native_word(atomic_t)) { 243 smp_store_release(&(v)->counter, i); 244 } else { 245 __atomic_release_fence(); 246 arch_atomic_set(v, i); 247 } 248} 249#define arch_atomic_set_release arch_atomic_set_release 250#endif 251 252#ifndef arch_atomic_add_return_relaxed 253#define arch_atomic_add_return_acquire arch_atomic_add_return 254#define arch_atomic_add_return_release arch_atomic_add_return 255#define arch_atomic_add_return_relaxed arch_atomic_add_return 256#else /* arch_atomic_add_return_relaxed */ 257 258#ifndef arch_atomic_add_return_acquire 259static __always_inline int 260arch_atomic_add_return_acquire(int i, atomic_t *v) 261{ 262 int ret = arch_atomic_add_return_relaxed(i, v); 263 __atomic_acquire_fence(); 264 return ret; 265} 266#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire 267#endif 268 269#ifndef arch_atomic_add_return_release 270static __always_inline int 271arch_atomic_add_return_release(int i, atomic_t *v) 272{ 273 __atomic_release_fence(); 274 return arch_atomic_add_return_relaxed(i, v); 275} 276#define arch_atomic_add_return_release arch_atomic_add_return_release 277#endif 278 279#ifndef arch_atomic_add_return 280static __always_inline int 281arch_atomic_add_return(int i, atomic_t *v) 282{ 283 int ret; 284 __atomic_pre_full_fence(); 285 ret = arch_atomic_add_return_relaxed(i, v); 286 __atomic_post_full_fence(); 287 return ret; 288} 289#define arch_atomic_add_return arch_atomic_add_return 290#endif 291 292#endif /* arch_atomic_add_return_relaxed */ 293 294#ifndef arch_atomic_fetch_add_relaxed 295#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add 296#define arch_atomic_fetch_add_release arch_atomic_fetch_add 297#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add 298#else /* arch_atomic_fetch_add_relaxed */ 299 300#ifndef arch_atomic_fetch_add_acquire 301static __always_inline int 302arch_atomic_fetch_add_acquire(int i, atomic_t *v) 303{ 304 int ret = arch_atomic_fetch_add_relaxed(i, v); 305 __atomic_acquire_fence(); 306 return ret; 307} 308#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire 309#endif 310 311#ifndef arch_atomic_fetch_add_release 312static __always_inline int 313arch_atomic_fetch_add_release(int i, atomic_t *v) 314{ 315 __atomic_release_fence(); 316 return arch_atomic_fetch_add_relaxed(i, v); 317} 318#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release 319#endif 320 321#ifndef arch_atomic_fetch_add 322static __always_inline int 323arch_atomic_fetch_add(int i, atomic_t *v) 324{ 325 int ret; 326 __atomic_pre_full_fence(); 327 ret = arch_atomic_fetch_add_relaxed(i, v); 328 __atomic_post_full_fence(); 329 return ret; 330} 331#define arch_atomic_fetch_add arch_atomic_fetch_add 332#endif 333 334#endif /* arch_atomic_fetch_add_relaxed */ 335 336#ifndef arch_atomic_sub_return_relaxed 337#define arch_atomic_sub_return_acquire arch_atomic_sub_return 338#define arch_atomic_sub_return_release arch_atomic_sub_return 339#define arch_atomic_sub_return_relaxed arch_atomic_sub_return 340#else /* arch_atomic_sub_return_relaxed */ 341 342#ifndef arch_atomic_sub_return_acquire 343static __always_inline int 344arch_atomic_sub_return_acquire(int i, atomic_t *v) 345{ 346 int ret = arch_atomic_sub_return_relaxed(i, v); 347 __atomic_acquire_fence(); 348 return ret; 349} 350#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire 351#endif 352 353#ifndef arch_atomic_sub_return_release 354static __always_inline int 355arch_atomic_sub_return_release(int i, atomic_t *v) 356{ 357 __atomic_release_fence(); 358 return arch_atomic_sub_return_relaxed(i, v); 359} 360#define arch_atomic_sub_return_release arch_atomic_sub_return_release 361#endif 362 363#ifndef arch_atomic_sub_return 364static __always_inline int 365arch_atomic_sub_return(int i, atomic_t *v) 366{ 367 int ret; 368 __atomic_pre_full_fence(); 369 ret = arch_atomic_sub_return_relaxed(i, v); 370 __atomic_post_full_fence(); 371 return ret; 372} 373#define arch_atomic_sub_return arch_atomic_sub_return 374#endif 375 376#endif /* arch_atomic_sub_return_relaxed */ 377 378#ifndef arch_atomic_fetch_sub_relaxed 379#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub 380#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub 381#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub 382#else /* arch_atomic_fetch_sub_relaxed */ 383 384#ifndef arch_atomic_fetch_sub_acquire 385static __always_inline int 386arch_atomic_fetch_sub_acquire(int i, atomic_t *v) 387{ 388 int ret = arch_atomic_fetch_sub_relaxed(i, v); 389 __atomic_acquire_fence(); 390 return ret; 391} 392#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire 393#endif 394 395#ifndef arch_atomic_fetch_sub_release 396static __always_inline int 397arch_atomic_fetch_sub_release(int i, atomic_t *v) 398{ 399 __atomic_release_fence(); 400 return arch_atomic_fetch_sub_relaxed(i, v); 401} 402#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release 403#endif 404 405#ifndef arch_atomic_fetch_sub 406static __always_inline int 407arch_atomic_fetch_sub(int i, atomic_t *v) 408{ 409 int ret; 410 __atomic_pre_full_fence(); 411 ret = arch_atomic_fetch_sub_relaxed(i, v); 412 __atomic_post_full_fence(); 413 return ret; 414} 415#define arch_atomic_fetch_sub arch_atomic_fetch_sub 416#endif 417 418#endif /* arch_atomic_fetch_sub_relaxed */ 419 420#ifndef arch_atomic_inc 421static __always_inline void 422arch_atomic_inc(atomic_t *v) 423{ 424 arch_atomic_add(1, v); 425} 426#define arch_atomic_inc arch_atomic_inc 427#endif 428 429#ifndef arch_atomic_inc_return_relaxed 430#ifdef arch_atomic_inc_return 431#define arch_atomic_inc_return_acquire arch_atomic_inc_return 432#define arch_atomic_inc_return_release arch_atomic_inc_return 433#define arch_atomic_inc_return_relaxed arch_atomic_inc_return 434#endif /* arch_atomic_inc_return */ 435 436#ifndef arch_atomic_inc_return 437static __always_inline int 438arch_atomic_inc_return(atomic_t *v) 439{ 440 return arch_atomic_add_return(1, v); 441} 442#define arch_atomic_inc_return arch_atomic_inc_return 443#endif 444 445#ifndef arch_atomic_inc_return_acquire 446static __always_inline int 447arch_atomic_inc_return_acquire(atomic_t *v) 448{ 449 return arch_atomic_add_return_acquire(1, v); 450} 451#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire 452#endif 453 454#ifndef arch_atomic_inc_return_release 455static __always_inline int 456arch_atomic_inc_return_release(atomic_t *v) 457{ 458 return arch_atomic_add_return_release(1, v); 459} 460#define arch_atomic_inc_return_release arch_atomic_inc_return_release 461#endif 462 463#ifndef arch_atomic_inc_return_relaxed 464static __always_inline int 465arch_atomic_inc_return_relaxed(atomic_t *v) 466{ 467 return arch_atomic_add_return_relaxed(1, v); 468} 469#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed 470#endif 471 472#else /* arch_atomic_inc_return_relaxed */ 473 474#ifndef arch_atomic_inc_return_acquire 475static __always_inline int 476arch_atomic_inc_return_acquire(atomic_t *v) 477{ 478 int ret = arch_atomic_inc_return_relaxed(v); 479 __atomic_acquire_fence(); 480 return ret; 481} 482#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire 483#endif 484 485#ifndef arch_atomic_inc_return_release 486static __always_inline int 487arch_atomic_inc_return_release(atomic_t *v) 488{ 489 __atomic_release_fence(); 490 return arch_atomic_inc_return_relaxed(v); 491} 492#define arch_atomic_inc_return_release arch_atomic_inc_return_release 493#endif 494 495#ifndef arch_atomic_inc_return 496static __always_inline int 497arch_atomic_inc_return(atomic_t *v) 498{ 499 int ret; 500 __atomic_pre_full_fence(); 501 ret = arch_atomic_inc_return_relaxed(v); 502 __atomic_post_full_fence(); 503 return ret; 504} 505#define arch_atomic_inc_return arch_atomic_inc_return 506#endif 507 508#endif /* arch_atomic_inc_return_relaxed */ 509 510#ifndef arch_atomic_fetch_inc_relaxed 511#ifdef arch_atomic_fetch_inc 512#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc 513#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc 514#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc 515#endif /* arch_atomic_fetch_inc */ 516 517#ifndef arch_atomic_fetch_inc 518static __always_inline int 519arch_atomic_fetch_inc(atomic_t *v) 520{ 521 return arch_atomic_fetch_add(1, v); 522} 523#define arch_atomic_fetch_inc arch_atomic_fetch_inc 524#endif 525 526#ifndef arch_atomic_fetch_inc_acquire 527static __always_inline int 528arch_atomic_fetch_inc_acquire(atomic_t *v) 529{ 530 return arch_atomic_fetch_add_acquire(1, v); 531} 532#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire 533#endif 534 535#ifndef arch_atomic_fetch_inc_release 536static __always_inline int 537arch_atomic_fetch_inc_release(atomic_t *v) 538{ 539 return arch_atomic_fetch_add_release(1, v); 540} 541#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release 542#endif 543 544#ifndef arch_atomic_fetch_inc_relaxed 545static __always_inline int 546arch_atomic_fetch_inc_relaxed(atomic_t *v) 547{ 548 return arch_atomic_fetch_add_relaxed(1, v); 549} 550#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed 551#endif 552 553#else /* arch_atomic_fetch_inc_relaxed */ 554 555#ifndef arch_atomic_fetch_inc_acquire 556static __always_inline int 557arch_atomic_fetch_inc_acquire(atomic_t *v) 558{ 559 int ret = arch_atomic_fetch_inc_relaxed(v); 560 __atomic_acquire_fence(); 561 return ret; 562} 563#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire 564#endif 565 566#ifndef arch_atomic_fetch_inc_release 567static __always_inline int 568arch_atomic_fetch_inc_release(atomic_t *v) 569{ 570 __atomic_release_fence(); 571 return arch_atomic_fetch_inc_relaxed(v); 572} 573#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release 574#endif 575 576#ifndef arch_atomic_fetch_inc 577static __always_inline int 578arch_atomic_fetch_inc(atomic_t *v) 579{ 580 int ret; 581 __atomic_pre_full_fence(); 582 ret = arch_atomic_fetch_inc_relaxed(v); 583 __atomic_post_full_fence(); 584 return ret; 585} 586#define arch_atomic_fetch_inc arch_atomic_fetch_inc 587#endif 588 589#endif /* arch_atomic_fetch_inc_relaxed */ 590 591#ifndef arch_atomic_dec 592static __always_inline void 593arch_atomic_dec(atomic_t *v) 594{ 595 arch_atomic_sub(1, v); 596} 597#define arch_atomic_dec arch_atomic_dec 598#endif 599 600#ifndef arch_atomic_dec_return_relaxed 601#ifdef arch_atomic_dec_return 602#define arch_atomic_dec_return_acquire arch_atomic_dec_return 603#define arch_atomic_dec_return_release arch_atomic_dec_return 604#define arch_atomic_dec_return_relaxed arch_atomic_dec_return 605#endif /* arch_atomic_dec_return */ 606 607#ifndef arch_atomic_dec_return 608static __always_inline int 609arch_atomic_dec_return(atomic_t *v) 610{ 611 return arch_atomic_sub_return(1, v); 612} 613#define arch_atomic_dec_return arch_atomic_dec_return 614#endif 615 616#ifndef arch_atomic_dec_return_acquire 617static __always_inline int 618arch_atomic_dec_return_acquire(atomic_t *v) 619{ 620 return arch_atomic_sub_return_acquire(1, v); 621} 622#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire 623#endif 624 625#ifndef arch_atomic_dec_return_release 626static __always_inline int 627arch_atomic_dec_return_release(atomic_t *v) 628{ 629 return arch_atomic_sub_return_release(1, v); 630} 631#define arch_atomic_dec_return_release arch_atomic_dec_return_release 632#endif 633 634#ifndef arch_atomic_dec_return_relaxed 635static __always_inline int 636arch_atomic_dec_return_relaxed(atomic_t *v) 637{ 638 return arch_atomic_sub_return_relaxed(1, v); 639} 640#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed 641#endif 642 643#else /* arch_atomic_dec_return_relaxed */ 644 645#ifndef arch_atomic_dec_return_acquire 646static __always_inline int 647arch_atomic_dec_return_acquire(atomic_t *v) 648{ 649 int ret = arch_atomic_dec_return_relaxed(v); 650 __atomic_acquire_fence(); 651 return ret; 652} 653#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire 654#endif 655 656#ifndef arch_atomic_dec_return_release 657static __always_inline int 658arch_atomic_dec_return_release(atomic_t *v) 659{ 660 __atomic_release_fence(); 661 return arch_atomic_dec_return_relaxed(v); 662} 663#define arch_atomic_dec_return_release arch_atomic_dec_return_release 664#endif 665 666#ifndef arch_atomic_dec_return 667static __always_inline int 668arch_atomic_dec_return(atomic_t *v) 669{ 670 int ret; 671 __atomic_pre_full_fence(); 672 ret = arch_atomic_dec_return_relaxed(v); 673 __atomic_post_full_fence(); 674 return ret; 675} 676#define arch_atomic_dec_return arch_atomic_dec_return 677#endif 678 679#endif /* arch_atomic_dec_return_relaxed */ 680 681#ifndef arch_atomic_fetch_dec_relaxed 682#ifdef arch_atomic_fetch_dec 683#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec 684#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec 685#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec 686#endif /* arch_atomic_fetch_dec */ 687 688#ifndef arch_atomic_fetch_dec 689static __always_inline int 690arch_atomic_fetch_dec(atomic_t *v) 691{ 692 return arch_atomic_fetch_sub(1, v); 693} 694#define arch_atomic_fetch_dec arch_atomic_fetch_dec 695#endif 696 697#ifndef arch_atomic_fetch_dec_acquire 698static __always_inline int 699arch_atomic_fetch_dec_acquire(atomic_t *v) 700{ 701 return arch_atomic_fetch_sub_acquire(1, v); 702} 703#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire 704#endif 705 706#ifndef arch_atomic_fetch_dec_release 707static __always_inline int 708arch_atomic_fetch_dec_release(atomic_t *v) 709{ 710 return arch_atomic_fetch_sub_release(1, v); 711} 712#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release 713#endif 714 715#ifndef arch_atomic_fetch_dec_relaxed 716static __always_inline int 717arch_atomic_fetch_dec_relaxed(atomic_t *v) 718{ 719 return arch_atomic_fetch_sub_relaxed(1, v); 720} 721#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed 722#endif 723 724#else /* arch_atomic_fetch_dec_relaxed */ 725 726#ifndef arch_atomic_fetch_dec_acquire 727static __always_inline int 728arch_atomic_fetch_dec_acquire(atomic_t *v) 729{ 730 int ret = arch_atomic_fetch_dec_relaxed(v); 731 __atomic_acquire_fence(); 732 return ret; 733} 734#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire 735#endif 736 737#ifndef arch_atomic_fetch_dec_release 738static __always_inline int 739arch_atomic_fetch_dec_release(atomic_t *v) 740{ 741 __atomic_release_fence(); 742 return arch_atomic_fetch_dec_relaxed(v); 743} 744#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release 745#endif 746 747#ifndef arch_atomic_fetch_dec 748static __always_inline int 749arch_atomic_fetch_dec(atomic_t *v) 750{ 751 int ret; 752 __atomic_pre_full_fence(); 753 ret = arch_atomic_fetch_dec_relaxed(v); 754 __atomic_post_full_fence(); 755 return ret; 756} 757#define arch_atomic_fetch_dec arch_atomic_fetch_dec 758#endif 759 760#endif /* arch_atomic_fetch_dec_relaxed */ 761 762#ifndef arch_atomic_fetch_and_relaxed 763#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and 764#define arch_atomic_fetch_and_release arch_atomic_fetch_and 765#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and 766#else /* arch_atomic_fetch_and_relaxed */ 767 768#ifndef arch_atomic_fetch_and_acquire 769static __always_inline int 770arch_atomic_fetch_and_acquire(int i, atomic_t *v) 771{ 772 int ret = arch_atomic_fetch_and_relaxed(i, v); 773 __atomic_acquire_fence(); 774 return ret; 775} 776#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire 777#endif 778 779#ifndef arch_atomic_fetch_and_release 780static __always_inline int 781arch_atomic_fetch_and_release(int i, atomic_t *v) 782{ 783 __atomic_release_fence(); 784 return arch_atomic_fetch_and_relaxed(i, v); 785} 786#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release 787#endif 788 789#ifndef arch_atomic_fetch_and 790static __always_inline int 791arch_atomic_fetch_and(int i, atomic_t *v) 792{ 793 int ret; 794 __atomic_pre_full_fence(); 795 ret = arch_atomic_fetch_and_relaxed(i, v); 796 __atomic_post_full_fence(); 797 return ret; 798} 799#define arch_atomic_fetch_and arch_atomic_fetch_and 800#endif 801 802#endif /* arch_atomic_fetch_and_relaxed */ 803 804#ifndef arch_atomic_andnot 805static __always_inline void 806arch_atomic_andnot(int i, atomic_t *v) 807{ 808 arch_atomic_and(~i, v); 809} 810#define arch_atomic_andnot arch_atomic_andnot 811#endif 812 813#ifndef arch_atomic_fetch_andnot_relaxed 814#ifdef arch_atomic_fetch_andnot 815#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot 816#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot 817#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot 818#endif /* arch_atomic_fetch_andnot */ 819 820#ifndef arch_atomic_fetch_andnot 821static __always_inline int 822arch_atomic_fetch_andnot(int i, atomic_t *v) 823{ 824 return arch_atomic_fetch_and(~i, v); 825} 826#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot 827#endif 828 829#ifndef arch_atomic_fetch_andnot_acquire 830static __always_inline int 831arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) 832{ 833 return arch_atomic_fetch_and_acquire(~i, v); 834} 835#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire 836#endif 837 838#ifndef arch_atomic_fetch_andnot_release 839static __always_inline int 840arch_atomic_fetch_andnot_release(int i, atomic_t *v) 841{ 842 return arch_atomic_fetch_and_release(~i, v); 843} 844#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release 845#endif 846 847#ifndef arch_atomic_fetch_andnot_relaxed 848static __always_inline int 849arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v) 850{ 851 return arch_atomic_fetch_and_relaxed(~i, v); 852} 853#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed 854#endif 855 856#else /* arch_atomic_fetch_andnot_relaxed */ 857 858#ifndef arch_atomic_fetch_andnot_acquire 859static __always_inline int 860arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) 861{ 862 int ret = arch_atomic_fetch_andnot_relaxed(i, v); 863 __atomic_acquire_fence(); 864 return ret; 865} 866#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire 867#endif 868 869#ifndef arch_atomic_fetch_andnot_release 870static __always_inline int 871arch_atomic_fetch_andnot_release(int i, atomic_t *v) 872{ 873 __atomic_release_fence(); 874 return arch_atomic_fetch_andnot_relaxed(i, v); 875} 876#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release 877#endif 878 879#ifndef arch_atomic_fetch_andnot 880static __always_inline int 881arch_atomic_fetch_andnot(int i, atomic_t *v) 882{ 883 int ret; 884 __atomic_pre_full_fence(); 885 ret = arch_atomic_fetch_andnot_relaxed(i, v); 886 __atomic_post_full_fence(); 887 return ret; 888} 889#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot 890#endif 891 892#endif /* arch_atomic_fetch_andnot_relaxed */ 893 894#ifndef arch_atomic_fetch_or_relaxed 895#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or 896#define arch_atomic_fetch_or_release arch_atomic_fetch_or 897#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or 898#else /* arch_atomic_fetch_or_relaxed */ 899 900#ifndef arch_atomic_fetch_or_acquire 901static __always_inline int 902arch_atomic_fetch_or_acquire(int i, atomic_t *v) 903{ 904 int ret = arch_atomic_fetch_or_relaxed(i, v); 905 __atomic_acquire_fence(); 906 return ret; 907} 908#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire 909#endif 910 911#ifndef arch_atomic_fetch_or_release 912static __always_inline int 913arch_atomic_fetch_or_release(int i, atomic_t *v) 914{ 915 __atomic_release_fence(); 916 return arch_atomic_fetch_or_relaxed(i, v); 917} 918#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release 919#endif 920 921#ifndef arch_atomic_fetch_or 922static __always_inline int 923arch_atomic_fetch_or(int i, atomic_t *v) 924{ 925 int ret; 926 __atomic_pre_full_fence(); 927 ret = arch_atomic_fetch_or_relaxed(i, v); 928 __atomic_post_full_fence(); 929 return ret; 930} 931#define arch_atomic_fetch_or arch_atomic_fetch_or 932#endif 933 934#endif /* arch_atomic_fetch_or_relaxed */ 935 936#ifndef arch_atomic_fetch_xor_relaxed 937#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor 938#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor 939#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor 940#else /* arch_atomic_fetch_xor_relaxed */ 941 942#ifndef arch_atomic_fetch_xor_acquire 943static __always_inline int 944arch_atomic_fetch_xor_acquire(int i, atomic_t *v) 945{ 946 int ret = arch_atomic_fetch_xor_relaxed(i, v); 947 __atomic_acquire_fence(); 948 return ret; 949} 950#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire 951#endif 952 953#ifndef arch_atomic_fetch_xor_release 954static __always_inline int 955arch_atomic_fetch_xor_release(int i, atomic_t *v) 956{ 957 __atomic_release_fence(); 958 return arch_atomic_fetch_xor_relaxed(i, v); 959} 960#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release 961#endif 962 963#ifndef arch_atomic_fetch_xor 964static __always_inline int 965arch_atomic_fetch_xor(int i, atomic_t *v) 966{ 967 int ret; 968 __atomic_pre_full_fence(); 969 ret = arch_atomic_fetch_xor_relaxed(i, v); 970 __atomic_post_full_fence(); 971 return ret; 972} 973#define arch_atomic_fetch_xor arch_atomic_fetch_xor 974#endif 975 976#endif /* arch_atomic_fetch_xor_relaxed */ 977 978#ifndef arch_atomic_xchg_relaxed 979#define arch_atomic_xchg_acquire arch_atomic_xchg 980#define arch_atomic_xchg_release arch_atomic_xchg 981#define arch_atomic_xchg_relaxed arch_atomic_xchg 982#else /* arch_atomic_xchg_relaxed */ 983 984#ifndef arch_atomic_xchg_acquire 985static __always_inline int 986arch_atomic_xchg_acquire(atomic_t *v, int i) 987{ 988 int ret = arch_atomic_xchg_relaxed(v, i); 989 __atomic_acquire_fence(); 990 return ret; 991} 992#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire 993#endif 994 995#ifndef arch_atomic_xchg_release 996static __always_inline int 997arch_atomic_xchg_release(atomic_t *v, int i) 998{ 999 __atomic_release_fence(); 1000 return arch_atomic_xchg_relaxed(v, i); 1001} 1002#define arch_atomic_xchg_release arch_atomic_xchg_release 1003#endif 1004 1005#ifndef arch_atomic_xchg 1006static __always_inline int 1007arch_atomic_xchg(atomic_t *v, int i) 1008{ 1009 int ret; 1010 __atomic_pre_full_fence(); 1011 ret = arch_atomic_xchg_relaxed(v, i); 1012 __atomic_post_full_fence(); 1013 return ret; 1014} 1015#define arch_atomic_xchg arch_atomic_xchg 1016#endif 1017 1018#endif /* arch_atomic_xchg_relaxed */ 1019 1020#ifndef arch_atomic_cmpxchg_relaxed 1021#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg 1022#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg 1023#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg 1024#else /* arch_atomic_cmpxchg_relaxed */ 1025 1026#ifndef arch_atomic_cmpxchg_acquire 1027static __always_inline int 1028arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 1029{ 1030 int ret = arch_atomic_cmpxchg_relaxed(v, old, new); 1031 __atomic_acquire_fence(); 1032 return ret; 1033} 1034#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire 1035#endif 1036 1037#ifndef arch_atomic_cmpxchg_release 1038static __always_inline int 1039arch_atomic_cmpxchg_release(atomic_t *v, int old, int new) 1040{ 1041 __atomic_release_fence(); 1042 return arch_atomic_cmpxchg_relaxed(v, old, new); 1043} 1044#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release 1045#endif 1046 1047#ifndef arch_atomic_cmpxchg 1048static __always_inline int 1049arch_atomic_cmpxchg(atomic_t *v, int old, int new) 1050{ 1051 int ret; 1052 __atomic_pre_full_fence(); 1053 ret = arch_atomic_cmpxchg_relaxed(v, old, new); 1054 __atomic_post_full_fence(); 1055 return ret; 1056} 1057#define arch_atomic_cmpxchg arch_atomic_cmpxchg 1058#endif 1059 1060#endif /* arch_atomic_cmpxchg_relaxed */ 1061 1062#ifndef arch_atomic_try_cmpxchg_relaxed 1063#ifdef arch_atomic_try_cmpxchg 1064#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg 1065#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg 1066#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg 1067#endif /* arch_atomic_try_cmpxchg */ 1068 1069#ifndef arch_atomic_try_cmpxchg 1070static __always_inline bool 1071arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1072{ 1073 int r, o = *old; 1074 r = arch_atomic_cmpxchg(v, o, new); 1075 if (unlikely(r != o)) 1076 *old = r; 1077 return likely(r == o); 1078} 1079#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg 1080#endif 1081 1082#ifndef arch_atomic_try_cmpxchg_acquire 1083static __always_inline bool 1084arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1085{ 1086 int r, o = *old; 1087 r = arch_atomic_cmpxchg_acquire(v, o, new); 1088 if (unlikely(r != o)) 1089 *old = r; 1090 return likely(r == o); 1091} 1092#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire 1093#endif 1094 1095#ifndef arch_atomic_try_cmpxchg_release 1096static __always_inline bool 1097arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1098{ 1099 int r, o = *old; 1100 r = arch_atomic_cmpxchg_release(v, o, new); 1101 if (unlikely(r != o)) 1102 *old = r; 1103 return likely(r == o); 1104} 1105#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release 1106#endif 1107 1108#ifndef arch_atomic_try_cmpxchg_relaxed 1109static __always_inline bool 1110arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 1111{ 1112 int r, o = *old; 1113 r = arch_atomic_cmpxchg_relaxed(v, o, new); 1114 if (unlikely(r != o)) 1115 *old = r; 1116 return likely(r == o); 1117} 1118#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed 1119#endif 1120 1121#else /* arch_atomic_try_cmpxchg_relaxed */ 1122 1123#ifndef arch_atomic_try_cmpxchg_acquire 1124static __always_inline bool 1125arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1126{ 1127 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 1128 __atomic_acquire_fence(); 1129 return ret; 1130} 1131#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire 1132#endif 1133 1134#ifndef arch_atomic_try_cmpxchg_release 1135static __always_inline bool 1136arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1137{ 1138 __atomic_release_fence(); 1139 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 1140} 1141#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release 1142#endif 1143 1144#ifndef arch_atomic_try_cmpxchg 1145static __always_inline bool 1146arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1147{ 1148 bool ret; 1149 __atomic_pre_full_fence(); 1150 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 1151 __atomic_post_full_fence(); 1152 return ret; 1153} 1154#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg 1155#endif 1156 1157#endif /* arch_atomic_try_cmpxchg_relaxed */ 1158 1159#ifndef arch_atomic_sub_and_test 1160/** 1161 * arch_atomic_sub_and_test - subtract value from variable and test result 1162 * @i: integer value to subtract 1163 * @v: pointer of type atomic_t 1164 * 1165 * Atomically subtracts @i from @v and returns 1166 * true if the result is zero, or false for all 1167 * other cases. 1168 */ 1169static __always_inline bool 1170arch_atomic_sub_and_test(int i, atomic_t *v) 1171{ 1172 return arch_atomic_sub_return(i, v) == 0; 1173} 1174#define arch_atomic_sub_and_test arch_atomic_sub_and_test 1175#endif 1176 1177#ifndef arch_atomic_dec_and_test 1178/** 1179 * arch_atomic_dec_and_test - decrement and test 1180 * @v: pointer of type atomic_t 1181 * 1182 * Atomically decrements @v by 1 and 1183 * returns true if the result is 0, or false for all other 1184 * cases. 1185 */ 1186static __always_inline bool 1187arch_atomic_dec_and_test(atomic_t *v) 1188{ 1189 return arch_atomic_dec_return(v) == 0; 1190} 1191#define arch_atomic_dec_and_test arch_atomic_dec_and_test 1192#endif 1193 1194#ifndef arch_atomic_inc_and_test 1195/** 1196 * arch_atomic_inc_and_test - increment and test 1197 * @v: pointer of type atomic_t 1198 * 1199 * Atomically increments @v by 1 1200 * and returns true if the result is zero, or false for all 1201 * other cases. 1202 */ 1203static __always_inline bool 1204arch_atomic_inc_and_test(atomic_t *v) 1205{ 1206 return arch_atomic_inc_return(v) == 0; 1207} 1208#define arch_atomic_inc_and_test arch_atomic_inc_and_test 1209#endif 1210 1211#ifndef arch_atomic_add_negative 1212/** 1213 * arch_atomic_add_negative - add and test if negative 1214 * @i: integer value to add 1215 * @v: pointer of type atomic_t 1216 * 1217 * Atomically adds @i to @v and returns true 1218 * if the result is negative, or false when 1219 * result is greater than or equal to zero. 1220 */ 1221static __always_inline bool 1222arch_atomic_add_negative(int i, atomic_t *v) 1223{ 1224 return arch_atomic_add_return(i, v) < 0; 1225} 1226#define arch_atomic_add_negative arch_atomic_add_negative 1227#endif 1228 1229#ifndef arch_atomic_fetch_add_unless 1230/** 1231 * arch_atomic_fetch_add_unless - add unless the number is already a given value 1232 * @v: pointer of type atomic_t 1233 * @a: the amount to add to v... 1234 * @u: ...unless v is equal to u. 1235 * 1236 * Atomically adds @a to @v, so long as @v was not already @u. 1237 * Returns original value of @v 1238 */ 1239static __always_inline int 1240arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 1241{ 1242 int c = arch_atomic_read(v); 1243 1244 do { 1245 if (unlikely(c == u)) 1246 break; 1247 } while (!arch_atomic_try_cmpxchg(v, &c, c + a)); 1248 1249 return c; 1250} 1251#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 1252#endif 1253 1254#ifndef arch_atomic_add_unless 1255/** 1256 * arch_atomic_add_unless - add unless the number is already a given value 1257 * @v: pointer of type atomic_t 1258 * @a: the amount to add to v... 1259 * @u: ...unless v is equal to u. 1260 * 1261 * Atomically adds @a to @v, if @v was not already @u. 1262 * Returns true if the addition was done. 1263 */ 1264static __always_inline bool 1265arch_atomic_add_unless(atomic_t *v, int a, int u) 1266{ 1267 return arch_atomic_fetch_add_unless(v, a, u) != u; 1268} 1269#define arch_atomic_add_unless arch_atomic_add_unless 1270#endif 1271 1272#ifndef arch_atomic_inc_not_zero 1273/** 1274 * arch_atomic_inc_not_zero - increment unless the number is zero 1275 * @v: pointer of type atomic_t 1276 * 1277 * Atomically increments @v by 1, if @v is non-zero. 1278 * Returns true if the increment was done. 1279 */ 1280static __always_inline bool 1281arch_atomic_inc_not_zero(atomic_t *v) 1282{ 1283 return arch_atomic_add_unless(v, 1, 0); 1284} 1285#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero 1286#endif 1287 1288#ifndef arch_atomic_inc_unless_negative 1289static __always_inline bool 1290arch_atomic_inc_unless_negative(atomic_t *v) 1291{ 1292 int c = arch_atomic_read(v); 1293 1294 do { 1295 if (unlikely(c < 0)) 1296 return false; 1297 } while (!arch_atomic_try_cmpxchg(v, &c, c + 1)); 1298 1299 return true; 1300} 1301#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative 1302#endif 1303 1304#ifndef arch_atomic_dec_unless_positive 1305static __always_inline bool 1306arch_atomic_dec_unless_positive(atomic_t *v) 1307{ 1308 int c = arch_atomic_read(v); 1309 1310 do { 1311 if (unlikely(c > 0)) 1312 return false; 1313 } while (!arch_atomic_try_cmpxchg(v, &c, c - 1)); 1314 1315 return true; 1316} 1317#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive 1318#endif 1319 1320#ifndef arch_atomic_dec_if_positive 1321static __always_inline int 1322arch_atomic_dec_if_positive(atomic_t *v) 1323{ 1324 int dec, c = arch_atomic_read(v); 1325 1326 do { 1327 dec = c - 1; 1328 if (unlikely(dec < 0)) 1329 break; 1330 } while (!arch_atomic_try_cmpxchg(v, &c, dec)); 1331 1332 return dec; 1333} 1334#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive 1335#endif 1336 1337#ifdef CONFIG_GENERIC_ATOMIC64 1338#include <asm-generic/atomic64.h> 1339#endif 1340 1341#ifndef arch_atomic64_read_acquire 1342static __always_inline s64 1343arch_atomic64_read_acquire(const atomic64_t *v) 1344{ 1345 s64 ret; 1346 1347 if (__native_word(atomic64_t)) { 1348 ret = smp_load_acquire(&(v)->counter); 1349 } else { 1350 ret = arch_atomic64_read(v); 1351 __atomic_acquire_fence(); 1352 } 1353 1354 return ret; 1355} 1356#define arch_atomic64_read_acquire arch_atomic64_read_acquire 1357#endif 1358 1359#ifndef arch_atomic64_set_release 1360static __always_inline void 1361arch_atomic64_set_release(atomic64_t *v, s64 i) 1362{ 1363 if (__native_word(atomic64_t)) { 1364 smp_store_release(&(v)->counter, i); 1365 } else { 1366 __atomic_release_fence(); 1367 arch_atomic64_set(v, i); 1368 } 1369} 1370#define arch_atomic64_set_release arch_atomic64_set_release 1371#endif 1372 1373#ifndef arch_atomic64_add_return_relaxed 1374#define arch_atomic64_add_return_acquire arch_atomic64_add_return 1375#define arch_atomic64_add_return_release arch_atomic64_add_return 1376#define arch_atomic64_add_return_relaxed arch_atomic64_add_return 1377#else /* arch_atomic64_add_return_relaxed */ 1378 1379#ifndef arch_atomic64_add_return_acquire 1380static __always_inline s64 1381arch_atomic64_add_return_acquire(s64 i, atomic64_t *v) 1382{ 1383 s64 ret = arch_atomic64_add_return_relaxed(i, v); 1384 __atomic_acquire_fence(); 1385 return ret; 1386} 1387#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire 1388#endif 1389 1390#ifndef arch_atomic64_add_return_release 1391static __always_inline s64 1392arch_atomic64_add_return_release(s64 i, atomic64_t *v) 1393{ 1394 __atomic_release_fence(); 1395 return arch_atomic64_add_return_relaxed(i, v); 1396} 1397#define arch_atomic64_add_return_release arch_atomic64_add_return_release 1398#endif 1399 1400#ifndef arch_atomic64_add_return 1401static __always_inline s64 1402arch_atomic64_add_return(s64 i, atomic64_t *v) 1403{ 1404 s64 ret; 1405 __atomic_pre_full_fence(); 1406 ret = arch_atomic64_add_return_relaxed(i, v); 1407 __atomic_post_full_fence(); 1408 return ret; 1409} 1410#define arch_atomic64_add_return arch_atomic64_add_return 1411#endif 1412 1413#endif /* arch_atomic64_add_return_relaxed */ 1414 1415#ifndef arch_atomic64_fetch_add_relaxed 1416#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add 1417#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add 1418#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add 1419#else /* arch_atomic64_fetch_add_relaxed */ 1420 1421#ifndef arch_atomic64_fetch_add_acquire 1422static __always_inline s64 1423arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 1424{ 1425 s64 ret = arch_atomic64_fetch_add_relaxed(i, v); 1426 __atomic_acquire_fence(); 1427 return ret; 1428} 1429#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire 1430#endif 1431 1432#ifndef arch_atomic64_fetch_add_release 1433static __always_inline s64 1434arch_atomic64_fetch_add_release(s64 i, atomic64_t *v) 1435{ 1436 __atomic_release_fence(); 1437 return arch_atomic64_fetch_add_relaxed(i, v); 1438} 1439#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release 1440#endif 1441 1442#ifndef arch_atomic64_fetch_add 1443static __always_inline s64 1444arch_atomic64_fetch_add(s64 i, atomic64_t *v) 1445{ 1446 s64 ret; 1447 __atomic_pre_full_fence(); 1448 ret = arch_atomic64_fetch_add_relaxed(i, v); 1449 __atomic_post_full_fence(); 1450 return ret; 1451} 1452#define arch_atomic64_fetch_add arch_atomic64_fetch_add 1453#endif 1454 1455#endif /* arch_atomic64_fetch_add_relaxed */ 1456 1457#ifndef arch_atomic64_sub_return_relaxed 1458#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return 1459#define arch_atomic64_sub_return_release arch_atomic64_sub_return 1460#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return 1461#else /* arch_atomic64_sub_return_relaxed */ 1462 1463#ifndef arch_atomic64_sub_return_acquire 1464static __always_inline s64 1465arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v) 1466{ 1467 s64 ret = arch_atomic64_sub_return_relaxed(i, v); 1468 __atomic_acquire_fence(); 1469 return ret; 1470} 1471#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire 1472#endif 1473 1474#ifndef arch_atomic64_sub_return_release 1475static __always_inline s64 1476arch_atomic64_sub_return_release(s64 i, atomic64_t *v) 1477{ 1478 __atomic_release_fence(); 1479 return arch_atomic64_sub_return_relaxed(i, v); 1480} 1481#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release 1482#endif 1483 1484#ifndef arch_atomic64_sub_return 1485static __always_inline s64 1486arch_atomic64_sub_return(s64 i, atomic64_t *v) 1487{ 1488 s64 ret; 1489 __atomic_pre_full_fence(); 1490 ret = arch_atomic64_sub_return_relaxed(i, v); 1491 __atomic_post_full_fence(); 1492 return ret; 1493} 1494#define arch_atomic64_sub_return arch_atomic64_sub_return 1495#endif 1496 1497#endif /* arch_atomic64_sub_return_relaxed */ 1498 1499#ifndef arch_atomic64_fetch_sub_relaxed 1500#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub 1501#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub 1502#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub 1503#else /* arch_atomic64_fetch_sub_relaxed */ 1504 1505#ifndef arch_atomic64_fetch_sub_acquire 1506static __always_inline s64 1507arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 1508{ 1509 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); 1510 __atomic_acquire_fence(); 1511 return ret; 1512} 1513#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire 1514#endif 1515 1516#ifndef arch_atomic64_fetch_sub_release 1517static __always_inline s64 1518arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v) 1519{ 1520 __atomic_release_fence(); 1521 return arch_atomic64_fetch_sub_relaxed(i, v); 1522} 1523#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release 1524#endif 1525 1526#ifndef arch_atomic64_fetch_sub 1527static __always_inline s64 1528arch_atomic64_fetch_sub(s64 i, atomic64_t *v) 1529{ 1530 s64 ret; 1531 __atomic_pre_full_fence(); 1532 ret = arch_atomic64_fetch_sub_relaxed(i, v); 1533 __atomic_post_full_fence(); 1534 return ret; 1535} 1536#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub 1537#endif 1538 1539#endif /* arch_atomic64_fetch_sub_relaxed */ 1540 1541#ifndef arch_atomic64_inc 1542static __always_inline void 1543arch_atomic64_inc(atomic64_t *v) 1544{ 1545 arch_atomic64_add(1, v); 1546} 1547#define arch_atomic64_inc arch_atomic64_inc 1548#endif 1549 1550#ifndef arch_atomic64_inc_return_relaxed 1551#ifdef arch_atomic64_inc_return 1552#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return 1553#define arch_atomic64_inc_return_release arch_atomic64_inc_return 1554#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return 1555#endif /* arch_atomic64_inc_return */ 1556 1557#ifndef arch_atomic64_inc_return 1558static __always_inline s64 1559arch_atomic64_inc_return(atomic64_t *v) 1560{ 1561 return arch_atomic64_add_return(1, v); 1562} 1563#define arch_atomic64_inc_return arch_atomic64_inc_return 1564#endif 1565 1566#ifndef arch_atomic64_inc_return_acquire 1567static __always_inline s64 1568arch_atomic64_inc_return_acquire(atomic64_t *v) 1569{ 1570 return arch_atomic64_add_return_acquire(1, v); 1571} 1572#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire 1573#endif 1574 1575#ifndef arch_atomic64_inc_return_release 1576static __always_inline s64 1577arch_atomic64_inc_return_release(atomic64_t *v) 1578{ 1579 return arch_atomic64_add_return_release(1, v); 1580} 1581#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release 1582#endif 1583 1584#ifndef arch_atomic64_inc_return_relaxed 1585static __always_inline s64 1586arch_atomic64_inc_return_relaxed(atomic64_t *v) 1587{ 1588 return arch_atomic64_add_return_relaxed(1, v); 1589} 1590#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed 1591#endif 1592 1593#else /* arch_atomic64_inc_return_relaxed */ 1594 1595#ifndef arch_atomic64_inc_return_acquire 1596static __always_inline s64 1597arch_atomic64_inc_return_acquire(atomic64_t *v) 1598{ 1599 s64 ret = arch_atomic64_inc_return_relaxed(v); 1600 __atomic_acquire_fence(); 1601 return ret; 1602} 1603#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire 1604#endif 1605 1606#ifndef arch_atomic64_inc_return_release 1607static __always_inline s64 1608arch_atomic64_inc_return_release(atomic64_t *v) 1609{ 1610 __atomic_release_fence(); 1611 return arch_atomic64_inc_return_relaxed(v); 1612} 1613#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release 1614#endif 1615 1616#ifndef arch_atomic64_inc_return 1617static __always_inline s64 1618arch_atomic64_inc_return(atomic64_t *v) 1619{ 1620 s64 ret; 1621 __atomic_pre_full_fence(); 1622 ret = arch_atomic64_inc_return_relaxed(v); 1623 __atomic_post_full_fence(); 1624 return ret; 1625} 1626#define arch_atomic64_inc_return arch_atomic64_inc_return 1627#endif 1628 1629#endif /* arch_atomic64_inc_return_relaxed */ 1630 1631#ifndef arch_atomic64_fetch_inc_relaxed 1632#ifdef arch_atomic64_fetch_inc 1633#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc 1634#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc 1635#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc 1636#endif /* arch_atomic64_fetch_inc */ 1637 1638#ifndef arch_atomic64_fetch_inc 1639static __always_inline s64 1640arch_atomic64_fetch_inc(atomic64_t *v) 1641{ 1642 return arch_atomic64_fetch_add(1, v); 1643} 1644#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc 1645#endif 1646 1647#ifndef arch_atomic64_fetch_inc_acquire 1648static __always_inline s64 1649arch_atomic64_fetch_inc_acquire(atomic64_t *v) 1650{ 1651 return arch_atomic64_fetch_add_acquire(1, v); 1652} 1653#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire 1654#endif 1655 1656#ifndef arch_atomic64_fetch_inc_release 1657static __always_inline s64 1658arch_atomic64_fetch_inc_release(atomic64_t *v) 1659{ 1660 return arch_atomic64_fetch_add_release(1, v); 1661} 1662#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release 1663#endif 1664 1665#ifndef arch_atomic64_fetch_inc_relaxed 1666static __always_inline s64 1667arch_atomic64_fetch_inc_relaxed(atomic64_t *v) 1668{ 1669 return arch_atomic64_fetch_add_relaxed(1, v); 1670} 1671#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed 1672#endif 1673 1674#else /* arch_atomic64_fetch_inc_relaxed */ 1675 1676#ifndef arch_atomic64_fetch_inc_acquire 1677static __always_inline s64 1678arch_atomic64_fetch_inc_acquire(atomic64_t *v) 1679{ 1680 s64 ret = arch_atomic64_fetch_inc_relaxed(v); 1681 __atomic_acquire_fence(); 1682 return ret; 1683} 1684#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire 1685#endif 1686 1687#ifndef arch_atomic64_fetch_inc_release 1688static __always_inline s64 1689arch_atomic64_fetch_inc_release(atomic64_t *v) 1690{ 1691 __atomic_release_fence(); 1692 return arch_atomic64_fetch_inc_relaxed(v); 1693} 1694#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release 1695#endif 1696 1697#ifndef arch_atomic64_fetch_inc 1698static __always_inline s64 1699arch_atomic64_fetch_inc(atomic64_t *v) 1700{ 1701 s64 ret; 1702 __atomic_pre_full_fence(); 1703 ret = arch_atomic64_fetch_inc_relaxed(v); 1704 __atomic_post_full_fence(); 1705 return ret; 1706} 1707#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc 1708#endif 1709 1710#endif /* arch_atomic64_fetch_inc_relaxed */ 1711 1712#ifndef arch_atomic64_dec 1713static __always_inline void 1714arch_atomic64_dec(atomic64_t *v) 1715{ 1716 arch_atomic64_sub(1, v); 1717} 1718#define arch_atomic64_dec arch_atomic64_dec 1719#endif 1720 1721#ifndef arch_atomic64_dec_return_relaxed 1722#ifdef arch_atomic64_dec_return 1723#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return 1724#define arch_atomic64_dec_return_release arch_atomic64_dec_return 1725#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return 1726#endif /* arch_atomic64_dec_return */ 1727 1728#ifndef arch_atomic64_dec_return 1729static __always_inline s64 1730arch_atomic64_dec_return(atomic64_t *v) 1731{ 1732 return arch_atomic64_sub_return(1, v); 1733} 1734#define arch_atomic64_dec_return arch_atomic64_dec_return 1735#endif 1736 1737#ifndef arch_atomic64_dec_return_acquire 1738static __always_inline s64 1739arch_atomic64_dec_return_acquire(atomic64_t *v) 1740{ 1741 return arch_atomic64_sub_return_acquire(1, v); 1742} 1743#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire 1744#endif 1745 1746#ifndef arch_atomic64_dec_return_release 1747static __always_inline s64 1748arch_atomic64_dec_return_release(atomic64_t *v) 1749{ 1750 return arch_atomic64_sub_return_release(1, v); 1751} 1752#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release 1753#endif 1754 1755#ifndef arch_atomic64_dec_return_relaxed 1756static __always_inline s64 1757arch_atomic64_dec_return_relaxed(atomic64_t *v) 1758{ 1759 return arch_atomic64_sub_return_relaxed(1, v); 1760} 1761#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed 1762#endif 1763 1764#else /* arch_atomic64_dec_return_relaxed */ 1765 1766#ifndef arch_atomic64_dec_return_acquire 1767static __always_inline s64 1768arch_atomic64_dec_return_acquire(atomic64_t *v) 1769{ 1770 s64 ret = arch_atomic64_dec_return_relaxed(v); 1771 __atomic_acquire_fence(); 1772 return ret; 1773} 1774#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire 1775#endif 1776 1777#ifndef arch_atomic64_dec_return_release 1778static __always_inline s64 1779arch_atomic64_dec_return_release(atomic64_t *v) 1780{ 1781 __atomic_release_fence(); 1782 return arch_atomic64_dec_return_relaxed(v); 1783} 1784#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release 1785#endif 1786 1787#ifndef arch_atomic64_dec_return 1788static __always_inline s64 1789arch_atomic64_dec_return(atomic64_t *v) 1790{ 1791 s64 ret; 1792 __atomic_pre_full_fence(); 1793 ret = arch_atomic64_dec_return_relaxed(v); 1794 __atomic_post_full_fence(); 1795 return ret; 1796} 1797#define arch_atomic64_dec_return arch_atomic64_dec_return 1798#endif 1799 1800#endif /* arch_atomic64_dec_return_relaxed */ 1801 1802#ifndef arch_atomic64_fetch_dec_relaxed 1803#ifdef arch_atomic64_fetch_dec 1804#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec 1805#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec 1806#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec 1807#endif /* arch_atomic64_fetch_dec */ 1808 1809#ifndef arch_atomic64_fetch_dec 1810static __always_inline s64 1811arch_atomic64_fetch_dec(atomic64_t *v) 1812{ 1813 return arch_atomic64_fetch_sub(1, v); 1814} 1815#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec 1816#endif 1817 1818#ifndef arch_atomic64_fetch_dec_acquire 1819static __always_inline s64 1820arch_atomic64_fetch_dec_acquire(atomic64_t *v) 1821{ 1822 return arch_atomic64_fetch_sub_acquire(1, v); 1823} 1824#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire 1825#endif 1826 1827#ifndef arch_atomic64_fetch_dec_release 1828static __always_inline s64 1829arch_atomic64_fetch_dec_release(atomic64_t *v) 1830{ 1831 return arch_atomic64_fetch_sub_release(1, v); 1832} 1833#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release 1834#endif 1835 1836#ifndef arch_atomic64_fetch_dec_relaxed 1837static __always_inline s64 1838arch_atomic64_fetch_dec_relaxed(atomic64_t *v) 1839{ 1840 return arch_atomic64_fetch_sub_relaxed(1, v); 1841} 1842#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed 1843#endif 1844 1845#else /* arch_atomic64_fetch_dec_relaxed */ 1846 1847#ifndef arch_atomic64_fetch_dec_acquire 1848static __always_inline s64 1849arch_atomic64_fetch_dec_acquire(atomic64_t *v) 1850{ 1851 s64 ret = arch_atomic64_fetch_dec_relaxed(v); 1852 __atomic_acquire_fence(); 1853 return ret; 1854} 1855#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire 1856#endif 1857 1858#ifndef arch_atomic64_fetch_dec_release 1859static __always_inline s64 1860arch_atomic64_fetch_dec_release(atomic64_t *v) 1861{ 1862 __atomic_release_fence(); 1863 return arch_atomic64_fetch_dec_relaxed(v); 1864} 1865#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release 1866#endif 1867 1868#ifndef arch_atomic64_fetch_dec 1869static __always_inline s64 1870arch_atomic64_fetch_dec(atomic64_t *v) 1871{ 1872 s64 ret; 1873 __atomic_pre_full_fence(); 1874 ret = arch_atomic64_fetch_dec_relaxed(v); 1875 __atomic_post_full_fence(); 1876 return ret; 1877} 1878#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec 1879#endif 1880 1881#endif /* arch_atomic64_fetch_dec_relaxed */ 1882 1883#ifndef arch_atomic64_fetch_and_relaxed 1884#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and 1885#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and 1886#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and 1887#else /* arch_atomic64_fetch_and_relaxed */ 1888 1889#ifndef arch_atomic64_fetch_and_acquire 1890static __always_inline s64 1891arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 1892{ 1893 s64 ret = arch_atomic64_fetch_and_relaxed(i, v); 1894 __atomic_acquire_fence(); 1895 return ret; 1896} 1897#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire 1898#endif 1899 1900#ifndef arch_atomic64_fetch_and_release 1901static __always_inline s64 1902arch_atomic64_fetch_and_release(s64 i, atomic64_t *v) 1903{ 1904 __atomic_release_fence(); 1905 return arch_atomic64_fetch_and_relaxed(i, v); 1906} 1907#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release 1908#endif 1909 1910#ifndef arch_atomic64_fetch_and 1911static __always_inline s64 1912arch_atomic64_fetch_and(s64 i, atomic64_t *v) 1913{ 1914 s64 ret; 1915 __atomic_pre_full_fence(); 1916 ret = arch_atomic64_fetch_and_relaxed(i, v); 1917 __atomic_post_full_fence(); 1918 return ret; 1919} 1920#define arch_atomic64_fetch_and arch_atomic64_fetch_and 1921#endif 1922 1923#endif /* arch_atomic64_fetch_and_relaxed */ 1924 1925#ifndef arch_atomic64_andnot 1926static __always_inline void 1927arch_atomic64_andnot(s64 i, atomic64_t *v) 1928{ 1929 arch_atomic64_and(~i, v); 1930} 1931#define arch_atomic64_andnot arch_atomic64_andnot 1932#endif 1933 1934#ifndef arch_atomic64_fetch_andnot_relaxed 1935#ifdef arch_atomic64_fetch_andnot 1936#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot 1937#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot 1938#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot 1939#endif /* arch_atomic64_fetch_andnot */ 1940 1941#ifndef arch_atomic64_fetch_andnot 1942static __always_inline s64 1943arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) 1944{ 1945 return arch_atomic64_fetch_and(~i, v); 1946} 1947#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot 1948#endif 1949 1950#ifndef arch_atomic64_fetch_andnot_acquire 1951static __always_inline s64 1952arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 1953{ 1954 return arch_atomic64_fetch_and_acquire(~i, v); 1955} 1956#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire 1957#endif 1958 1959#ifndef arch_atomic64_fetch_andnot_release 1960static __always_inline s64 1961arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 1962{ 1963 return arch_atomic64_fetch_and_release(~i, v); 1964} 1965#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release 1966#endif 1967 1968#ifndef arch_atomic64_fetch_andnot_relaxed 1969static __always_inline s64 1970arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 1971{ 1972 return arch_atomic64_fetch_and_relaxed(~i, v); 1973} 1974#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 1975#endif 1976 1977#else /* arch_atomic64_fetch_andnot_relaxed */ 1978 1979#ifndef arch_atomic64_fetch_andnot_acquire 1980static __always_inline s64 1981arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 1982{ 1983 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 1984 __atomic_acquire_fence(); 1985 return ret; 1986} 1987#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire 1988#endif 1989 1990#ifndef arch_atomic64_fetch_andnot_release 1991static __always_inline s64 1992arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 1993{ 1994 __atomic_release_fence(); 1995 return arch_atomic64_fetch_andnot_relaxed(i, v); 1996} 1997#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release 1998#endif 1999 2000#ifndef arch_atomic64_fetch_andnot 2001static __always_inline s64 2002arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) 2003{ 2004 s64 ret; 2005 __atomic_pre_full_fence(); 2006 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 2007 __atomic_post_full_fence(); 2008 return ret; 2009} 2010#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot 2011#endif 2012 2013#endif /* arch_atomic64_fetch_andnot_relaxed */ 2014 2015#ifndef arch_atomic64_fetch_or_relaxed 2016#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or 2017#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or 2018#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or 2019#else /* arch_atomic64_fetch_or_relaxed */ 2020 2021#ifndef arch_atomic64_fetch_or_acquire 2022static __always_inline s64 2023arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 2024{ 2025 s64 ret = arch_atomic64_fetch_or_relaxed(i, v); 2026 __atomic_acquire_fence(); 2027 return ret; 2028} 2029#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire 2030#endif 2031 2032#ifndef arch_atomic64_fetch_or_release 2033static __always_inline s64 2034arch_atomic64_fetch_or_release(s64 i, atomic64_t *v) 2035{ 2036 __atomic_release_fence(); 2037 return arch_atomic64_fetch_or_relaxed(i, v); 2038} 2039#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release 2040#endif 2041 2042#ifndef arch_atomic64_fetch_or 2043static __always_inline s64 2044arch_atomic64_fetch_or(s64 i, atomic64_t *v) 2045{ 2046 s64 ret; 2047 __atomic_pre_full_fence(); 2048 ret = arch_atomic64_fetch_or_relaxed(i, v); 2049 __atomic_post_full_fence(); 2050 return ret; 2051} 2052#define arch_atomic64_fetch_or arch_atomic64_fetch_or 2053#endif 2054 2055#endif /* arch_atomic64_fetch_or_relaxed */ 2056 2057#ifndef arch_atomic64_fetch_xor_relaxed 2058#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor 2059#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor 2060#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor 2061#else /* arch_atomic64_fetch_xor_relaxed */ 2062 2063#ifndef arch_atomic64_fetch_xor_acquire 2064static __always_inline s64 2065arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 2066{ 2067 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); 2068 __atomic_acquire_fence(); 2069 return ret; 2070} 2071#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire 2072#endif 2073 2074#ifndef arch_atomic64_fetch_xor_release 2075static __always_inline s64 2076arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v) 2077{ 2078 __atomic_release_fence(); 2079 return arch_atomic64_fetch_xor_relaxed(i, v); 2080} 2081#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release 2082#endif 2083 2084#ifndef arch_atomic64_fetch_xor 2085static __always_inline s64 2086arch_atomic64_fetch_xor(s64 i, atomic64_t *v) 2087{ 2088 s64 ret; 2089 __atomic_pre_full_fence(); 2090 ret = arch_atomic64_fetch_xor_relaxed(i, v); 2091 __atomic_post_full_fence(); 2092 return ret; 2093} 2094#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor 2095#endif 2096 2097#endif /* arch_atomic64_fetch_xor_relaxed */ 2098 2099#ifndef arch_atomic64_xchg_relaxed 2100#define arch_atomic64_xchg_acquire arch_atomic64_xchg 2101#define arch_atomic64_xchg_release arch_atomic64_xchg 2102#define arch_atomic64_xchg_relaxed arch_atomic64_xchg 2103#else /* arch_atomic64_xchg_relaxed */ 2104 2105#ifndef arch_atomic64_xchg_acquire 2106static __always_inline s64 2107arch_atomic64_xchg_acquire(atomic64_t *v, s64 i) 2108{ 2109 s64 ret = arch_atomic64_xchg_relaxed(v, i); 2110 __atomic_acquire_fence(); 2111 return ret; 2112} 2113#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire 2114#endif 2115 2116#ifndef arch_atomic64_xchg_release 2117static __always_inline s64 2118arch_atomic64_xchg_release(atomic64_t *v, s64 i) 2119{ 2120 __atomic_release_fence(); 2121 return arch_atomic64_xchg_relaxed(v, i); 2122} 2123#define arch_atomic64_xchg_release arch_atomic64_xchg_release 2124#endif 2125 2126#ifndef arch_atomic64_xchg 2127static __always_inline s64 2128arch_atomic64_xchg(atomic64_t *v, s64 i) 2129{ 2130 s64 ret; 2131 __atomic_pre_full_fence(); 2132 ret = arch_atomic64_xchg_relaxed(v, i); 2133 __atomic_post_full_fence(); 2134 return ret; 2135} 2136#define arch_atomic64_xchg arch_atomic64_xchg 2137#endif 2138 2139#endif /* arch_atomic64_xchg_relaxed */ 2140 2141#ifndef arch_atomic64_cmpxchg_relaxed 2142#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg 2143#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg 2144#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg 2145#else /* arch_atomic64_cmpxchg_relaxed */ 2146 2147#ifndef arch_atomic64_cmpxchg_acquire 2148static __always_inline s64 2149arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 2150{ 2151 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 2152 __atomic_acquire_fence(); 2153 return ret; 2154} 2155#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire 2156#endif 2157 2158#ifndef arch_atomic64_cmpxchg_release 2159static __always_inline s64 2160arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 2161{ 2162 __atomic_release_fence(); 2163 return arch_atomic64_cmpxchg_relaxed(v, old, new); 2164} 2165#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release 2166#endif 2167 2168#ifndef arch_atomic64_cmpxchg 2169static __always_inline s64 2170arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 2171{ 2172 s64 ret; 2173 __atomic_pre_full_fence(); 2174 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 2175 __atomic_post_full_fence(); 2176 return ret; 2177} 2178#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg 2179#endif 2180 2181#endif /* arch_atomic64_cmpxchg_relaxed */ 2182 2183#ifndef arch_atomic64_try_cmpxchg_relaxed 2184#ifdef arch_atomic64_try_cmpxchg 2185#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg 2186#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg 2187#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg 2188#endif /* arch_atomic64_try_cmpxchg */ 2189 2190#ifndef arch_atomic64_try_cmpxchg 2191static __always_inline bool 2192arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2193{ 2194 s64 r, o = *old; 2195 r = arch_atomic64_cmpxchg(v, o, new); 2196 if (unlikely(r != o)) 2197 *old = r; 2198 return likely(r == o); 2199} 2200#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg 2201#endif 2202 2203#ifndef arch_atomic64_try_cmpxchg_acquire 2204static __always_inline bool 2205arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2206{ 2207 s64 r, o = *old; 2208 r = arch_atomic64_cmpxchg_acquire(v, o, new); 2209 if (unlikely(r != o)) 2210 *old = r; 2211 return likely(r == o); 2212} 2213#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire 2214#endif 2215 2216#ifndef arch_atomic64_try_cmpxchg_release 2217static __always_inline bool 2218arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2219{ 2220 s64 r, o = *old; 2221 r = arch_atomic64_cmpxchg_release(v, o, new); 2222 if (unlikely(r != o)) 2223 *old = r; 2224 return likely(r == o); 2225} 2226#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release 2227#endif 2228 2229#ifndef arch_atomic64_try_cmpxchg_relaxed 2230static __always_inline bool 2231arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 2232{ 2233 s64 r, o = *old; 2234 r = arch_atomic64_cmpxchg_relaxed(v, o, new); 2235 if (unlikely(r != o)) 2236 *old = r; 2237 return likely(r == o); 2238} 2239#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed 2240#endif 2241 2242#else /* arch_atomic64_try_cmpxchg_relaxed */ 2243 2244#ifndef arch_atomic64_try_cmpxchg_acquire 2245static __always_inline bool 2246arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2247{ 2248 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2249 __atomic_acquire_fence(); 2250 return ret; 2251} 2252#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire 2253#endif 2254 2255#ifndef arch_atomic64_try_cmpxchg_release 2256static __always_inline bool 2257arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2258{ 2259 __atomic_release_fence(); 2260 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2261} 2262#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release 2263#endif 2264 2265#ifndef arch_atomic64_try_cmpxchg 2266static __always_inline bool 2267arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2268{ 2269 bool ret; 2270 __atomic_pre_full_fence(); 2271 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2272 __atomic_post_full_fence(); 2273 return ret; 2274} 2275#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg 2276#endif 2277 2278#endif /* arch_atomic64_try_cmpxchg_relaxed */ 2279 2280#ifndef arch_atomic64_sub_and_test 2281/** 2282 * arch_atomic64_sub_and_test - subtract value from variable and test result 2283 * @i: integer value to subtract 2284 * @v: pointer of type atomic64_t 2285 * 2286 * Atomically subtracts @i from @v and returns 2287 * true if the result is zero, or false for all 2288 * other cases. 2289 */ 2290static __always_inline bool 2291arch_atomic64_sub_and_test(s64 i, atomic64_t *v) 2292{ 2293 return arch_atomic64_sub_return(i, v) == 0; 2294} 2295#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test 2296#endif 2297 2298#ifndef arch_atomic64_dec_and_test 2299/** 2300 * arch_atomic64_dec_and_test - decrement and test 2301 * @v: pointer of type atomic64_t 2302 * 2303 * Atomically decrements @v by 1 and 2304 * returns true if the result is 0, or false for all other 2305 * cases. 2306 */ 2307static __always_inline bool 2308arch_atomic64_dec_and_test(atomic64_t *v) 2309{ 2310 return arch_atomic64_dec_return(v) == 0; 2311} 2312#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test 2313#endif 2314 2315#ifndef arch_atomic64_inc_and_test 2316/** 2317 * arch_atomic64_inc_and_test - increment and test 2318 * @v: pointer of type atomic64_t 2319 * 2320 * Atomically increments @v by 1 2321 * and returns true if the result is zero, or false for all 2322 * other cases. 2323 */ 2324static __always_inline bool 2325arch_atomic64_inc_and_test(atomic64_t *v) 2326{ 2327 return arch_atomic64_inc_return(v) == 0; 2328} 2329#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test 2330#endif 2331 2332#ifndef arch_atomic64_add_negative 2333/** 2334 * arch_atomic64_add_negative - add and test if negative 2335 * @i: integer value to add 2336 * @v: pointer of type atomic64_t 2337 * 2338 * Atomically adds @i to @v and returns true 2339 * if the result is negative, or false when 2340 * result is greater than or equal to zero. 2341 */ 2342static __always_inline bool 2343arch_atomic64_add_negative(s64 i, atomic64_t *v) 2344{ 2345 return arch_atomic64_add_return(i, v) < 0; 2346} 2347#define arch_atomic64_add_negative arch_atomic64_add_negative 2348#endif 2349 2350#ifndef arch_atomic64_fetch_add_unless 2351/** 2352 * arch_atomic64_fetch_add_unless - add unless the number is already a given value 2353 * @v: pointer of type atomic64_t 2354 * @a: the amount to add to v... 2355 * @u: ...unless v is equal to u. 2356 * 2357 * Atomically adds @a to @v, so long as @v was not already @u. 2358 * Returns original value of @v 2359 */ 2360static __always_inline s64 2361arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 2362{ 2363 s64 c = arch_atomic64_read(v); 2364 2365 do { 2366 if (unlikely(c == u)) 2367 break; 2368 } while (!arch_atomic64_try_cmpxchg(v, &c, c + a)); 2369 2370 return c; 2371} 2372#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 2373#endif 2374 2375#ifndef arch_atomic64_add_unless 2376/** 2377 * arch_atomic64_add_unless - add unless the number is already a given value 2378 * @v: pointer of type atomic64_t 2379 * @a: the amount to add to v... 2380 * @u: ...unless v is equal to u. 2381 * 2382 * Atomically adds @a to @v, if @v was not already @u. 2383 * Returns true if the addition was done. 2384 */ 2385static __always_inline bool 2386arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 2387{ 2388 return arch_atomic64_fetch_add_unless(v, a, u) != u; 2389} 2390#define arch_atomic64_add_unless arch_atomic64_add_unless 2391#endif 2392 2393#ifndef arch_atomic64_inc_not_zero 2394/** 2395 * arch_atomic64_inc_not_zero - increment unless the number is zero 2396 * @v: pointer of type atomic64_t 2397 * 2398 * Atomically increments @v by 1, if @v is non-zero. 2399 * Returns true if the increment was done. 2400 */ 2401static __always_inline bool 2402arch_atomic64_inc_not_zero(atomic64_t *v) 2403{ 2404 return arch_atomic64_add_unless(v, 1, 0); 2405} 2406#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero 2407#endif 2408 2409#ifndef arch_atomic64_inc_unless_negative 2410static __always_inline bool 2411arch_atomic64_inc_unless_negative(atomic64_t *v) 2412{ 2413 s64 c = arch_atomic64_read(v); 2414 2415 do { 2416 if (unlikely(c < 0)) 2417 return false; 2418 } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1)); 2419 2420 return true; 2421} 2422#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative 2423#endif 2424 2425#ifndef arch_atomic64_dec_unless_positive 2426static __always_inline bool 2427arch_atomic64_dec_unless_positive(atomic64_t *v) 2428{ 2429 s64 c = arch_atomic64_read(v); 2430 2431 do { 2432 if (unlikely(c > 0)) 2433 return false; 2434 } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1)); 2435 2436 return true; 2437} 2438#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive 2439#endif 2440 2441#ifndef arch_atomic64_dec_if_positive 2442static __always_inline s64 2443arch_atomic64_dec_if_positive(atomic64_t *v) 2444{ 2445 s64 dec, c = arch_atomic64_read(v); 2446 2447 do { 2448 dec = c - 1; 2449 if (unlikely(dec < 0)) 2450 break; 2451 } while (!arch_atomic64_try_cmpxchg(v, &c, dec)); 2452 2453 return dec; 2454} 2455#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 2456#endif 2457 2458#endif /* _LINUX_ATOMIC_FALLBACK_H */ 2459// b5e87bdd5ede61470c29f7a7e4de781af3770f09