include/linux/atomic.h

   1 /* Atomic operations usable in machine independent code */
   2 #ifndef _LINUX_ATOMIC_H
   3 #define _LINUX_ATOMIC_H
   4 #include <asm/atomic.h>
   5 #include <asm/barrier.h>
   6
   7 /*
   8  * Relaxed variants of xchg, cmpxchg and some atomic operations.
   9  *
  10  * We support four variants:
  11  *
  12  * - Fully ordered: The default implementation, no suffix required.
  13  * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
  14  * - Release: Provides RELEASE semantics, _release suffix.
  15  * - Relaxed: No ordering guarantees, _relaxed suffix.
  16  *
  17  * For compound atomics performing both a load and a store, ACQUIRE
  18  * semantics apply only to the load and RELEASE semantics only to the
  19  * store portion of the operation. Note that a failed cmpxchg_acquire
  20  * does -not- imply any memory ordering constraints.
  21  *
  22  * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
  23  */
  24
  25 #ifndef atomic_read_acquire
  26 #define  atomic_read_acquire(v)         smp_load_acquire(&(v)->counter)
  27 #endif
  28
  29 #ifndef atomic_set_release
  30 #define  atomic_set_release(v, i)       smp_store_release(&(v)->counter, (i))
  31 #endif
  32
  33 /*
  34  * The idea here is to build acquire/release variants by adding explicit
  35  * barriers on top of the relaxed variant. In the case where the relaxed
  36  * variant is already fully ordered, no additional barriers are needed.
  37  *
  38  * Besides, if an arch has a special barrier for acquire/release, it could
  39  * implement its own __atomic_op_* and use the same framework for building
  40  * variants
  41  */
  42 #ifndef __atomic_op_acquire
  43 #define __atomic_op_acquire(op, args...)                                \
  44 ({                                                                      \
  45         typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
  46         smp_mb__after_atomic();                                         \
  47         __ret;                                                          \
  48 })
  49 #endif
  50
  51 #ifndef __atomic_op_release
  52 #define __atomic_op_release(op, args...)                                \
  53 ({                                                                      \
  54         smp_mb__before_atomic();                                        \
  55         op##_relaxed(args);                                             \
  56 })
  57 #endif
  58
  59 #ifndef __atomic_op_fence
  60 #define __atomic_op_fence(op, args...)                                  \
  61 ({                                                                      \
  62         typeof(op##_relaxed(args)) __ret;                               \
  63         smp_mb__before_atomic();                                        \
  64         __ret = op##_relaxed(args);                                     \
  65         smp_mb__after_atomic();                                         \
  66         __ret;                                                          \
  67 })
  68 #endif
  69
  70 /* atomic_add_return_relaxed */
  71 #ifndef atomic_add_return_relaxed
  72 #define  atomic_add_return_relaxed      atomic_add_return
  73 #define  atomic_add_return_acquire      atomic_add_return
  74 #define  atomic_add_return_release      atomic_add_return
  75
  76 #else /* atomic_add_return_relaxed */
  77
  78 #ifndef atomic_add_return_acquire
  79 #define  atomic_add_return_acquire(...)                                 \
  80         __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
  81 #endif
  82
  83 #ifndef atomic_add_return_release
  84 #define  atomic_add_return_release(...)                                 \
  85         __atomic_op_release(atomic_add_return, __VA_ARGS__)
  86 #endif
  87
  88 #ifndef atomic_add_return
  89 #define  atomic_add_return(...)                                         \
  90         __atomic_op_fence(atomic_add_return, __VA_ARGS__)
  91 #endif
  92 #endif /* atomic_add_return_relaxed */
  93
  94 /* atomic_inc_return_relaxed */
  95 #ifndef atomic_inc_return_relaxed
  96 #define  atomic_inc_return_relaxed      atomic_inc_return
  97 #define  atomic_inc_return_acquire      atomic_inc_return
  98 #define  atomic_inc_return_release      atomic_inc_return
  99
 100 #else /* atomic_inc_return_relaxed */
 101
 102 #ifndef atomic_inc_return_acquire
 103 #define  atomic_inc_return_acquire(...)                                 \
 104         __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
 105 #endif
 106
 107 #ifndef atomic_inc_return_release
 108 #define  atomic_inc_return_release(...)                                 \
 109         __atomic_op_release(atomic_inc_return, __VA_ARGS__)
 110 #endif
 111
 112 #ifndef atomic_inc_return
 113 #define  atomic_inc_return(...)                                         \
 114         __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
 115 #endif
 116 #endif /* atomic_inc_return_relaxed */
 117
 118 /* atomic_sub_return_relaxed */
 119 #ifndef atomic_sub_return_relaxed
 120 #define  atomic_sub_return_relaxed      atomic_sub_return
 121 #define  atomic_sub_return_acquire      atomic_sub_return
 122 #define  atomic_sub_return_release      atomic_sub_return
 123
 124 #else /* atomic_sub_return_relaxed */
 125
 126 #ifndef atomic_sub_return_acquire
 127 #define  atomic_sub_return_acquire(...)                                 \
 128         __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
 129 #endif
 130
 131 #ifndef atomic_sub_return_release
 132 #define  atomic_sub_return_release(...)                                 \
 133         __atomic_op_release(atomic_sub_return, __VA_ARGS__)
 134 #endif
 135
 136 #ifndef atomic_sub_return
 137 #define  atomic_sub_return(...)                                         \
 138         __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
 139 #endif
 140 #endif /* atomic_sub_return_relaxed */
 141
 142 /* atomic_dec_return_relaxed */
 143 #ifndef atomic_dec_return_relaxed
 144 #define  atomic_dec_return_relaxed      atomic_dec_return
 145 #define  atomic_dec_return_acquire      atomic_dec_return
 146 #define  atomic_dec_return_release      atomic_dec_return
 147
 148 #else /* atomic_dec_return_relaxed */
 149
 150 #ifndef atomic_dec_return_acquire
 151 #define  atomic_dec_return_acquire(...)                                 \
 152         __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
 153 #endif
 154
 155 #ifndef atomic_dec_return_release
 156 #define  atomic_dec_return_release(...)                                 \
 157         __atomic_op_release(atomic_dec_return, __VA_ARGS__)
 158 #endif
 159
 160 #ifndef atomic_dec_return
 161 #define  atomic_dec_return(...)                                         \
 162         __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
 163 #endif
 164 #endif /* atomic_dec_return_relaxed */
 165
 166
 167 /* atomic_fetch_add_relaxed */
 168 #ifndef atomic_fetch_add_relaxed
 169 #define atomic_fetch_add_relaxed        atomic_fetch_add
 170 #define atomic_fetch_add_acquire        atomic_fetch_add
 171 #define atomic_fetch_add_release        atomic_fetch_add
 172
 173 #else /* atomic_fetch_add_relaxed */
 174
 175 #ifndef atomic_fetch_add_acquire
 176 #define atomic_fetch_add_acquire(...)                                   \
 177         __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
 178 #endif
 179
 180 #ifndef atomic_fetch_add_release
 181 #define atomic_fetch_add_release(...)                                   \
 182         __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
 183 #endif
 184
 185 #ifndef atomic_fetch_add
 186 #define atomic_fetch_add(...)                                           \
 187         __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
 188 #endif
 189 #endif /* atomic_fetch_add_relaxed */
 190
 191 /* atomic_fetch_inc_relaxed */
 192 #ifndef atomic_fetch_inc_relaxed
 193
 194 #ifndef atomic_fetch_inc
 195 #define atomic_fetch_inc(v)             atomic_fetch_add(1, (v))
 196 #define atomic_fetch_inc_relaxed(v)     atomic_fetch_add_relaxed(1, (v))
 197 #define atomic_fetch_inc_acquire(v)     atomic_fetch_add_acquire(1, (v))
 198 #define atomic_fetch_inc_release(v)     atomic_fetch_add_release(1, (v))
 199 #else /* atomic_fetch_inc */
 200 #define atomic_fetch_inc_relaxed        atomic_fetch_inc
 201 #define atomic_fetch_inc_acquire        atomic_fetch_inc
 202 #define atomic_fetch_inc_release        atomic_fetch_inc
 203 #endif /* atomic_fetch_inc */
 204
 205 #else /* atomic_fetch_inc_relaxed */
 206
 207 #ifndef atomic_fetch_inc_acquire
 208 #define atomic_fetch_inc_acquire(...)                                   \
 209         __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
 210 #endif
 211
 212 #ifndef atomic_fetch_inc_release
 213 #define atomic_fetch_inc_release(...)                                   \
 214         __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
 215 #endif
 216
 217 #ifndef atomic_fetch_inc
 218 #define atomic_fetch_inc(...)                                           \
 219         __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
 220 #endif
 221 #endif /* atomic_fetch_inc_relaxed */
 222
 223 /* atomic_fetch_sub_relaxed */
 224 #ifndef atomic_fetch_sub_relaxed
 225 #define atomic_fetch_sub_relaxed        atomic_fetch_sub
 226 #define atomic_fetch_sub_acquire        atomic_fetch_sub
 227 #define atomic_fetch_sub_release        atomic_fetch_sub
 228
 229 #else /* atomic_fetch_sub_relaxed */
 230
 231 #ifndef atomic_fetch_sub_acquire
 232 #define atomic_fetch_sub_acquire(...)                                   \
 233         __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
 234 #endif
 235
 236 #ifndef atomic_fetch_sub_release
 237 #define atomic_fetch_sub_release(...)                                   \
 238         __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
 239 #endif
 240
 241 #ifndef atomic_fetch_sub
 242 #define atomic_fetch_sub(...)                                           \
 243         __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
 244 #endif
 245 #endif /* atomic_fetch_sub_relaxed */
 246
 247 /* atomic_fetch_dec_relaxed */
 248 #ifndef atomic_fetch_dec_relaxed
 249
 250 #ifndef atomic_fetch_dec
 251 #define atomic_fetch_dec(v)             atomic_fetch_sub(1, (v))
 252 #define atomic_fetch_dec_relaxed(v)     atomic_fetch_sub_relaxed(1, (v))
 253 #define atomic_fetch_dec_acquire(v)     atomic_fetch_sub_acquire(1, (v))
 254 #define atomic_fetch_dec_release(v)     atomic_fetch_sub_release(1, (v))
 255 #else /* atomic_fetch_dec */
 256 #define atomic_fetch_dec_relaxed        atomic_fetch_dec
 257 #define atomic_fetch_dec_acquire        atomic_fetch_dec
 258 #define atomic_fetch_dec_release        atomic_fetch_dec
 259 #endif /* atomic_fetch_dec */
 260
 261 #else /* atomic_fetch_dec_relaxed */
 262
 263 #ifndef atomic_fetch_dec_acquire
 264 #define atomic_fetch_dec_acquire(...)                                   \
 265         __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
 266 #endif
 267
 268 #ifndef atomic_fetch_dec_release
 269 #define atomic_fetch_dec_release(...)                                   \
 270         __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
 271 #endif
 272
 273 #ifndef atomic_fetch_dec
 274 #define atomic_fetch_dec(...)                                           \
 275         __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
 276 #endif
 277 #endif /* atomic_fetch_dec_relaxed */
 278
 279 /* atomic_fetch_or_relaxed */
 280 #ifndef atomic_fetch_or_relaxed
 281 #define atomic_fetch_or_relaxed atomic_fetch_or
 282 #define atomic_fetch_or_acquire atomic_fetch_or
 283 #define atomic_fetch_or_release atomic_fetch_or
 284
 285 #else /* atomic_fetch_or_relaxed */
 286
 287 #ifndef atomic_fetch_or_acquire
 288 #define atomic_fetch_or_acquire(...)                                    \
 289         __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
 290 #endif
 291
 292 #ifndef atomic_fetch_or_release
 293 #define atomic_fetch_or_release(...)                                    \
 294         __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
 295 #endif
 296
 297 #ifndef atomic_fetch_or
 298 #define atomic_fetch_or(...)                                            \
 299         __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
 300 #endif
 301 #endif /* atomic_fetch_or_relaxed */
 302
 303 /* atomic_fetch_and_relaxed */
 304 #ifndef atomic_fetch_and_relaxed
 305 #define atomic_fetch_and_relaxed        atomic_fetch_and
 306 #define atomic_fetch_and_acquire        atomic_fetch_and
 307 #define atomic_fetch_and_release        atomic_fetch_and
 308
 309 #else /* atomic_fetch_and_relaxed */
 310
 311 #ifndef atomic_fetch_and_acquire
 312 #define atomic_fetch_and_acquire(...)                                   \
 313         __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
 314 #endif
 315
 316 #ifndef atomic_fetch_and_release
 317 #define atomic_fetch_and_release(...)                                   \
 318         __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
 319 #endif
 320
 321 #ifndef atomic_fetch_and
 322 #define atomic_fetch_and(...)                                           \
 323         __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
 324 #endif
 325 #endif /* atomic_fetch_and_relaxed */
 326
 327 #ifdef atomic_andnot
 328 /* atomic_fetch_andnot_relaxed */
 329 #ifndef atomic_fetch_andnot_relaxed
 330 #define atomic_fetch_andnot_relaxed     atomic_fetch_andnot
 331 #define atomic_fetch_andnot_acquire     atomic_fetch_andnot
 332 #define atomic_fetch_andnot_release     atomic_fetch_andnot
 333
 334 #else /* atomic_fetch_andnot_relaxed */
 335
 336 #ifndef atomic_fetch_andnot_acquire
 337 #define atomic_fetch_andnot_acquire(...)                                        \
 338         __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
 339 #endif
 340
 341 #ifndef atomic_fetch_andnot_release
 342 #define atomic_fetch_andnot_release(...)                                        \
 343         __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
 344 #endif
 345
 346 #ifndef atomic_fetch_andnot
 347 #define atomic_fetch_andnot(...)                                                \
 348         __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
 349 #endif
 350 #endif /* atomic_fetch_andnot_relaxed */
 351 #endif /* atomic_andnot */
 352
 353 /* atomic_fetch_xor_relaxed */
 354 #ifndef atomic_fetch_xor_relaxed
 355 #define atomic_fetch_xor_relaxed        atomic_fetch_xor
 356 #define atomic_fetch_xor_acquire        atomic_fetch_xor
 357 #define atomic_fetch_xor_release        atomic_fetch_xor
 358
 359 #else /* atomic_fetch_xor_relaxed */
 360
 361 #ifndef atomic_fetch_xor_acquire
 362 #define atomic_fetch_xor_acquire(...)                                   \
 363         __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
 364 #endif
 365
 366 #ifndef atomic_fetch_xor_release
 367 #define atomic_fetch_xor_release(...)                                   \
 368         __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
 369 #endif
 370
 371 #ifndef atomic_fetch_xor
 372 #define atomic_fetch_xor(...)                                           \
 373         __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
 374 #endif
 375 #endif /* atomic_fetch_xor_relaxed */
 376
 377
 378 /* atomic_xchg_relaxed */
 379 #ifndef atomic_xchg_relaxed
 380 #define  atomic_xchg_relaxed            atomic_xchg
 381 #define  atomic_xchg_acquire            atomic_xchg
 382 #define  atomic_xchg_release            atomic_xchg
 383
 384 #else /* atomic_xchg_relaxed */
 385
 386 #ifndef atomic_xchg_acquire
 387 #define  atomic_xchg_acquire(...)                                       \
 388         __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
 389 #endif
 390
 391 #ifndef atomic_xchg_release
 392 #define  atomic_xchg_release(...)                                       \
 393         __atomic_op_release(atomic_xchg, __VA_ARGS__)
 394 #endif
 395
 396 #ifndef atomic_xchg
 397 #define  atomic_xchg(...)                                               \
 398         __atomic_op_fence(atomic_xchg, __VA_ARGS__)
 399 #endif
 400 #endif /* atomic_xchg_relaxed */
 401
 402 /* atomic_cmpxchg_relaxed */
 403 #ifndef atomic_cmpxchg_relaxed
 404 #define  atomic_cmpxchg_relaxed         atomic_cmpxchg
 405 #define  atomic_cmpxchg_acquire         atomic_cmpxchg
 406 #define  atomic_cmpxchg_release         atomic_cmpxchg
 407
 408 #else /* atomic_cmpxchg_relaxed */
 409
 410 #ifndef atomic_cmpxchg_acquire
 411 #define  atomic_cmpxchg_acquire(...)                                    \
 412         __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
 413 #endif
 414
 415 #ifndef atomic_cmpxchg_release
 416 #define  atomic_cmpxchg_release(...)                                    \
 417         __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
 418 #endif
 419
 420 #ifndef atomic_cmpxchg
 421 #define  atomic_cmpxchg(...)                                            \
 422         __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
 423 #endif
 424 #endif /* atomic_cmpxchg_relaxed */
 425
 426 #ifndef atomic_try_cmpxchg
 427
 428 #define __atomic_try_cmpxchg(type, _p, _po, _n)                         \
 429 ({                                                                      \
 430         typeof(_po) __po = (_po);                                       \
 431         typeof(*(_po)) __r, __o = *__po;                                \
 432         __r = atomic_cmpxchg##type((_p), __o, (_n));                    \
 433         if (unlikely(__r != __o))                                       \
 434                 *__po = __r;                                            \
 435         likely(__r == __o);                                             \
 436 })
 437
 438 #define atomic_try_cmpxchg(_p, _po, _n)         __atomic_try_cmpxchg(, _p, _po, _n)
 439 #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n)
 440 #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n)
 441 #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n)
 442
 443 #else /* atomic_try_cmpxchg */
 444 #define atomic_try_cmpxchg_relaxed      atomic_try_cmpxchg
 445 #define atomic_try_cmpxchg_acquire      atomic_try_cmpxchg
 446 #define atomic_try_cmpxchg_release      atomic_try_cmpxchg
 447 #endif /* atomic_try_cmpxchg */
 448
 449 /* cmpxchg_relaxed */
 450 #ifndef cmpxchg_relaxed
 451 #define  cmpxchg_relaxed                cmpxchg
 452 #define  cmpxchg_acquire                cmpxchg
 453 #define  cmpxchg_release                cmpxchg
 454
 455 #else /* cmpxchg_relaxed */
 456
 457 #ifndef cmpxchg_acquire
 458 #define  cmpxchg_acquire(...)                                           \
 459         __atomic_op_acquire(cmpxchg, __VA_ARGS__)
 460 #endif
 461
 462 #ifndef cmpxchg_release
 463 #define  cmpxchg_release(...)                                           \
 464         __atomic_op_release(cmpxchg, __VA_ARGS__)
 465 #endif
 466
 467 #ifndef cmpxchg
 468 #define  cmpxchg(...)                                                   \
 469         __atomic_op_fence(cmpxchg, __VA_ARGS__)
 470 #endif
 471 #endif /* cmpxchg_relaxed */
 472
 473 /* cmpxchg64_relaxed */
 474 #ifndef cmpxchg64_relaxed
 475 #define  cmpxchg64_relaxed              cmpxchg64
 476 #define  cmpxchg64_acquire              cmpxchg64
 477 #define  cmpxchg64_release              cmpxchg64
 478
 479 #else /* cmpxchg64_relaxed */
 480
 481 #ifndef cmpxchg64_acquire
 482 #define  cmpxchg64_acquire(...)                                         \
 483         __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
 484 #endif
 485
 486 #ifndef cmpxchg64_release
 487 #define  cmpxchg64_release(...)                                         \
 488         __atomic_op_release(cmpxchg64, __VA_ARGS__)
 489 #endif
 490
 491 #ifndef cmpxchg64
 492 #define  cmpxchg64(...)                                                 \
 493         __atomic_op_fence(cmpxchg64, __VA_ARGS__)
 494 #endif
 495 #endif /* cmpxchg64_relaxed */
 496
 497 /* xchg_relaxed */
 498 #ifndef xchg_relaxed
 499 #define  xchg_relaxed                   xchg
 500 #define  xchg_acquire                   xchg
 501 #define  xchg_release                   xchg
 502
 503 #else /* xchg_relaxed */
 504
 505 #ifndef xchg_acquire
 506 #define  xchg_acquire(...)              __atomic_op_acquire(xchg, __VA_ARGS__)
 507 #endif
 508
 509 #ifndef xchg_release
 510 #define  xchg_release(...)              __atomic_op_release(xchg, __VA_ARGS__)
 511 #endif
 512
 513 #ifndef xchg
 514 #define  xchg(...)                      __atomic_op_fence(xchg, __VA_ARGS__)
 515 #endif
 516 #endif /* xchg_relaxed */
 517
 518 /**
 519  * atomic_add_unless - add unless the number is already a given value
 520  * @v: pointer of type atomic_t
 521  * @a: the amount to add to v...
 522  * @u: ...unless v is equal to u.
 523  *
 524  * Atomically adds @a to @v, so long as @v was not already @u.
 525  * Returns non-zero if @v was not @u, and zero otherwise.
 526  */
 527 static inline int atomic_add_unless(atomic_t *v, int a, int u)
 528 {
 529         return __atomic_add_unless(v, a, u) != u;
 530 }
 531
 532 /**
 533  * atomic_inc_not_zero - increment unless the number is zero
 534  * @v: pointer of type atomic_t
 535  *
 536  * Atomically increments @v by 1, so long as @v is non-zero.
 537  * Returns non-zero if @v was non-zero, and zero otherwise.
 538  */
 539 #ifndef atomic_inc_not_zero
 540 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 541 #endif
 542
 543 #ifndef atomic_andnot
 544 static inline void atomic_andnot(int i, atomic_t *v)
 545 {
 546         atomic_and(~i, v);
 547 }
 548
 549 static inline int atomic_fetch_andnot(int i, atomic_t *v)
 550 {
 551         return atomic_fetch_and(~i, v);
 552 }
 553
 554 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 555 {
 556         return atomic_fetch_and_relaxed(~i, v);
 557 }
 558
 559 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v)
 560 {
 561         return atomic_fetch_and_acquire(~i, v);
 562 }
 563
 564 static inline int atomic_fetch_andnot_release(int i, atomic_t *v)
 565 {
 566         return atomic_fetch_and_release(~i, v);
 567 }
 568 #endif
 569
 570 /**
 571  * atomic_inc_not_zero_hint - increment if not null
 572  * @v: pointer of type atomic_t
 573  * @hint: probable value of the atomic before the increment
 574  *
 575  * This version of atomic_inc_not_zero() gives a hint of probable
 576  * value of the atomic. This helps processor to not read the memory
 577  * before doing the atomic read/modify/write cycle, lowering
 578  * number of bus transactions on some arches.
 579  *
 580  * Returns: 0 if increment was not done, 1 otherwise.
 581  */
 582 #ifndef atomic_inc_not_zero_hint
 583 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
 584 {
 585         int val, c = hint;
 586
 587         /* sanity test, should be removed by compiler if hint is a constant */
 588         if (!hint)
 589                 return atomic_inc_not_zero(v);
 590
 591         do {
 592                 val = atomic_cmpxchg(v, c, c + 1);
 593                 if (val == c)
 594                         return 1;
 595                 c = val;
 596         } while (c);
 597
 598         return 0;
 599 }
 600 #endif
 601
 602 #ifndef atomic_inc_unless_negative
 603 static inline int atomic_inc_unless_negative(atomic_t *p)
 604 {
 605         int v, v1;
 606         for (v = 0; v >= 0; v = v1) {
 607                 v1 = atomic_cmpxchg(p, v, v + 1);
 608                 if (likely(v1 == v))
 609                         return 1;
 610         }
 611         return 0;
 612 }
 613 #endif
 614
 615 #ifndef atomic_dec_unless_positive
 616 static inline int atomic_dec_unless_positive(atomic_t *p)
 617 {
 618         int v, v1;
 619         for (v = 0; v <= 0; v = v1) {
 620                 v1 = atomic_cmpxchg(p, v, v - 1);
 621                 if (likely(v1 == v))
 622                         return 1;
 623         }
 624         return 0;
 625 }
 626 #endif
 627
 628 /*
 629  * atomic_dec_if_positive - decrement by 1 if old value positive
 630  * @v: pointer of type atomic_t
 631  *
 632  * The function returns the old value of *v minus 1, even if
 633  * the atomic variable, v, was not decremented.
 634  */
 635 #ifndef atomic_dec_if_positive
 636 static inline int atomic_dec_if_positive(atomic_t *v)
 637 {
 638         int c, old, dec;
 639         c = atomic_read(v);
 640         for (;;) {
 641                 dec = c - 1;
 642                 if (unlikely(dec < 0))
 643                         break;
 644                 old = atomic_cmpxchg((v), c, dec);
 645                 if (likely(old == c))
 646                         break;
 647                 c = old;
 648         }
 649         return dec;
 650 }
 651 #endif
 652
 653 #ifdef CONFIG_GENERIC_ATOMIC64
 654 #include <asm-generic/atomic64.h>
 655 #endif
 656
 657 #ifndef atomic64_read_acquire
 658 #define  atomic64_read_acquire(v)       smp_load_acquire(&(v)->counter)
 659 #endif
 660
 661 #ifndef atomic64_set_release
 662 #define  atomic64_set_release(v, i)     smp_store_release(&(v)->counter, (i))
 663 #endif
 664
 665 /* atomic64_add_return_relaxed */
 666 #ifndef atomic64_add_return_relaxed
 667 #define  atomic64_add_return_relaxed    atomic64_add_return
 668 #define  atomic64_add_return_acquire    atomic64_add_return
 669 #define  atomic64_add_return_release    atomic64_add_return
 670
 671 #else /* atomic64_add_return_relaxed */
 672
 673 #ifndef atomic64_add_return_acquire
 674 #define  atomic64_add_return_acquire(...)                               \
 675         __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
 676 #endif
 677
 678 #ifndef atomic64_add_return_release
 679 #define  atomic64_add_return_release(...)                               \
 680         __atomic_op_release(atomic64_add_return, __VA_ARGS__)
 681 #endif
 682
 683 #ifndef atomic64_add_return
 684 #define  atomic64_add_return(...)                                       \
 685         __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
 686 #endif
 687 #endif /* atomic64_add_return_relaxed */
 688
 689 /* atomic64_inc_return_relaxed */
 690 #ifndef atomic64_inc_return_relaxed
 691 #define  atomic64_inc_return_relaxed    atomic64_inc_return
 692 #define  atomic64_inc_return_acquire    atomic64_inc_return
 693 #define  atomic64_inc_return_release    atomic64_inc_return
 694
 695 #else /* atomic64_inc_return_relaxed */
 696
 697 #ifndef atomic64_inc_return_acquire
 698 #define  atomic64_inc_return_acquire(...)                               \
 699         __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
 700 #endif
 701
 702 #ifndef atomic64_inc_return_release
 703 #define  atomic64_inc_return_release(...)                               \
 704         __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
 705 #endif
 706
 707 #ifndef atomic64_inc_return
 708 #define  atomic64_inc_return(...)                                       \
 709         __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
 710 #endif
 711 #endif /* atomic64_inc_return_relaxed */
 712
 713
 714 /* atomic64_sub_return_relaxed */
 715 #ifndef atomic64_sub_return_relaxed
 716 #define  atomic64_sub_return_relaxed    atomic64_sub_return
 717 #define  atomic64_sub_return_acquire    atomic64_sub_return
 718 #define  atomic64_sub_return_release    atomic64_sub_return
 719
 720 #else /* atomic64_sub_return_relaxed */
 721
 722 #ifndef atomic64_sub_return_acquire
 723 #define  atomic64_sub_return_acquire(...)                               \
 724         __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
 725 #endif
 726
 727 #ifndef atomic64_sub_return_release
 728 #define  atomic64_sub_return_release(...)                               \
 729         __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
 730 #endif
 731
 732 #ifndef atomic64_sub_return
 733 #define  atomic64_sub_return(...)                                       \
 734         __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
 735 #endif
 736 #endif /* atomic64_sub_return_relaxed */
 737
 738 /* atomic64_dec_return_relaxed */
 739 #ifndef atomic64_dec_return_relaxed
 740 #define  atomic64_dec_return_relaxed    atomic64_dec_return
 741 #define  atomic64_dec_return_acquire    atomic64_dec_return
 742 #define  atomic64_dec_return_release    atomic64_dec_return
 743
 744 #else /* atomic64_dec_return_relaxed */
 745
 746 #ifndef atomic64_dec_return_acquire
 747 #define  atomic64_dec_return_acquire(...)                               \
 748         __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
 749 #endif
 750
 751 #ifndef atomic64_dec_return_release
 752 #define  atomic64_dec_return_release(...)                               \
 753         __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
 754 #endif
 755
 756 #ifndef atomic64_dec_return
 757 #define  atomic64_dec_return(...)                                       \
 758         __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
 759 #endif
 760 #endif /* atomic64_dec_return_relaxed */
 761
 762
 763 /* atomic64_fetch_add_relaxed */
 764 #ifndef atomic64_fetch_add_relaxed
 765 #define atomic64_fetch_add_relaxed      atomic64_fetch_add
 766 #define atomic64_fetch_add_acquire      atomic64_fetch_add
 767 #define atomic64_fetch_add_release      atomic64_fetch_add
 768
 769 #else /* atomic64_fetch_add_relaxed */
 770
 771 #ifndef atomic64_fetch_add_acquire
 772 #define atomic64_fetch_add_acquire(...)                                 \
 773         __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
 774 #endif
 775
 776 #ifndef atomic64_fetch_add_release
 777 #define atomic64_fetch_add_release(...)                                 \
 778         __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
 779 #endif
 780
 781 #ifndef atomic64_fetch_add
 782 #define atomic64_fetch_add(...)                                         \
 783         __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
 784 #endif
 785 #endif /* atomic64_fetch_add_relaxed */
 786
 787 /* atomic64_fetch_inc_relaxed */
 788 #ifndef atomic64_fetch_inc_relaxed
 789
 790 #ifndef atomic64_fetch_inc
 791 #define atomic64_fetch_inc(v)           atomic64_fetch_add(1, (v))
 792 #define atomic64_fetch_inc_relaxed(v)   atomic64_fetch_add_relaxed(1, (v))
 793 #define atomic64_fetch_inc_acquire(v)   atomic64_fetch_add_acquire(1, (v))
 794 #define atomic64_fetch_inc_release(v)   atomic64_fetch_add_release(1, (v))
 795 #else /* atomic64_fetch_inc */
 796 #define atomic64_fetch_inc_relaxed      atomic64_fetch_inc
 797 #define atomic64_fetch_inc_acquire      atomic64_fetch_inc
 798 #define atomic64_fetch_inc_release      atomic64_fetch_inc
 799 #endif /* atomic64_fetch_inc */
 800
 801 #else /* atomic64_fetch_inc_relaxed */
 802
 803 #ifndef atomic64_fetch_inc_acquire
 804 #define atomic64_fetch_inc_acquire(...)                                 \
 805         __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
 806 #endif
 807
 808 #ifndef atomic64_fetch_inc_release
 809 #define atomic64_fetch_inc_release(...)                                 \
 810         __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
 811 #endif
 812
 813 #ifndef atomic64_fetch_inc
 814 #define atomic64_fetch_inc(...)                                         \
 815         __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
 816 #endif
 817 #endif /* atomic64_fetch_inc_relaxed */
 818
 819 /* atomic64_fetch_sub_relaxed */
 820 #ifndef atomic64_fetch_sub_relaxed
 821 #define atomic64_fetch_sub_relaxed      atomic64_fetch_sub
 822 #define atomic64_fetch_sub_acquire      atomic64_fetch_sub
 823 #define atomic64_fetch_sub_release      atomic64_fetch_sub
 824
 825 #else /* atomic64_fetch_sub_relaxed */
 826
 827 #ifndef atomic64_fetch_sub_acquire
 828 #define atomic64_fetch_sub_acquire(...)                                 \
 829         __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
 830 #endif
 831
 832 #ifndef atomic64_fetch_sub_release
 833 #define atomic64_fetch_sub_release(...)                                 \
 834         __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
 835 #endif
 836
 837 #ifndef atomic64_fetch_sub
 838 #define atomic64_fetch_sub(...)                                         \
 839         __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
 840 #endif
 841 #endif /* atomic64_fetch_sub_relaxed */
 842
 843 /* atomic64_fetch_dec_relaxed */
 844 #ifndef atomic64_fetch_dec_relaxed
 845
 846 #ifndef atomic64_fetch_dec
 847 #define atomic64_fetch_dec(v)           atomic64_fetch_sub(1, (v))
 848 #define atomic64_fetch_dec_relaxed(v)   atomic64_fetch_sub_relaxed(1, (v))
 849 #define atomic64_fetch_dec_acquire(v)   atomic64_fetch_sub_acquire(1, (v))
 850 #define atomic64_fetch_dec_release(v)   atomic64_fetch_sub_release(1, (v))
 851 #else /* atomic64_fetch_dec */
 852 #define atomic64_fetch_dec_relaxed      atomic64_fetch_dec
 853 #define atomic64_fetch_dec_acquire      atomic64_fetch_dec
 854 #define atomic64_fetch_dec_release      atomic64_fetch_dec
 855 #endif /* atomic64_fetch_dec */
 856
 857 #else /* atomic64_fetch_dec_relaxed */
 858
 859 #ifndef atomic64_fetch_dec_acquire
 860 #define atomic64_fetch_dec_acquire(...)                                 \
 861         __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
 862 #endif
 863
 864 #ifndef atomic64_fetch_dec_release
 865 #define atomic64_fetch_dec_release(...)                                 \
 866         __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
 867 #endif
 868
 869 #ifndef atomic64_fetch_dec
 870 #define atomic64_fetch_dec(...)                                         \
 871         __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
 872 #endif
 873 #endif /* atomic64_fetch_dec_relaxed */
 874
 875 /* atomic64_fetch_or_relaxed */
 876 #ifndef atomic64_fetch_or_relaxed
 877 #define atomic64_fetch_or_relaxed       atomic64_fetch_or
 878 #define atomic64_fetch_or_acquire       atomic64_fetch_or
 879 #define atomic64_fetch_or_release       atomic64_fetch_or
 880
 881 #else /* atomic64_fetch_or_relaxed */
 882
 883 #ifndef atomic64_fetch_or_acquire
 884 #define atomic64_fetch_or_acquire(...)                                  \
 885         __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
 886 #endif
 887
 888 #ifndef atomic64_fetch_or_release
 889 #define atomic64_fetch_or_release(...)                                  \
 890         __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
 891 #endif
 892
 893 #ifndef atomic64_fetch_or
 894 #define atomic64_fetch_or(...)                                          \
 895         __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
 896 #endif
 897 #endif /* atomic64_fetch_or_relaxed */
 898
 899 /* atomic64_fetch_and_relaxed */
 900 #ifndef atomic64_fetch_and_relaxed
 901 #define atomic64_fetch_and_relaxed      atomic64_fetch_and
 902 #define atomic64_fetch_and_acquire      atomic64_fetch_and
 903 #define atomic64_fetch_and_release      atomic64_fetch_and
 904
 905 #else /* atomic64_fetch_and_relaxed */
 906
 907 #ifndef atomic64_fetch_and_acquire
 908 #define atomic64_fetch_and_acquire(...)                                 \
 909         __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
 910 #endif
 911
 912 #ifndef atomic64_fetch_and_release
 913 #define atomic64_fetch_and_release(...)                                 \
 914         __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
 915 #endif
 916
 917 #ifndef atomic64_fetch_and
 918 #define atomic64_fetch_and(...)                                         \
 919         __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
 920 #endif
 921 #endif /* atomic64_fetch_and_relaxed */
 922
 923 #ifdef atomic64_andnot
 924 /* atomic64_fetch_andnot_relaxed */
 925 #ifndef atomic64_fetch_andnot_relaxed
 926 #define atomic64_fetch_andnot_relaxed   atomic64_fetch_andnot
 927 #define atomic64_fetch_andnot_acquire   atomic64_fetch_andnot
 928 #define atomic64_fetch_andnot_release   atomic64_fetch_andnot
 929
 930 #else /* atomic64_fetch_andnot_relaxed */
 931
 932 #ifndef atomic64_fetch_andnot_acquire
 933 #define atomic64_fetch_andnot_acquire(...)                                      \
 934         __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
 935 #endif
 936
 937 #ifndef atomic64_fetch_andnot_release
 938 #define atomic64_fetch_andnot_release(...)                                      \
 939         __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
 940 #endif
 941
 942 #ifndef atomic64_fetch_andnot
 943 #define atomic64_fetch_andnot(...)                                              \
 944         __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
 945 #endif
 946 #endif /* atomic64_fetch_andnot_relaxed */
 947 #endif /* atomic64_andnot */
 948
 949 /* atomic64_fetch_xor_relaxed */
 950 #ifndef atomic64_fetch_xor_relaxed
 951 #define atomic64_fetch_xor_relaxed      atomic64_fetch_xor
 952 #define atomic64_fetch_xor_acquire      atomic64_fetch_xor
 953 #define atomic64_fetch_xor_release      atomic64_fetch_xor
 954
 955 #else /* atomic64_fetch_xor_relaxed */
 956
 957 #ifndef atomic64_fetch_xor_acquire
 958 #define atomic64_fetch_xor_acquire(...)                                 \
 959         __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
 960 #endif
 961
 962 #ifndef atomic64_fetch_xor_release
 963 #define atomic64_fetch_xor_release(...)                                 \
 964         __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
 965 #endif
 966
 967 #ifndef atomic64_fetch_xor
 968 #define atomic64_fetch_xor(...)                                         \
 969         __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
 970 #endif
 971 #endif /* atomic64_fetch_xor_relaxed */
 972
 973
 974 /* atomic64_xchg_relaxed */
 975 #ifndef atomic64_xchg_relaxed
 976 #define  atomic64_xchg_relaxed          atomic64_xchg
 977 #define  atomic64_xchg_acquire          atomic64_xchg
 978 #define  atomic64_xchg_release          atomic64_xchg
 979
 980 #else /* atomic64_xchg_relaxed */
 981
 982 #ifndef atomic64_xchg_acquire
 983 #define  atomic64_xchg_acquire(...)                                     \
 984         __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
 985 #endif
 986
 987 #ifndef atomic64_xchg_release
 988 #define  atomic64_xchg_release(...)                                     \
 989         __atomic_op_release(atomic64_xchg, __VA_ARGS__)
 990 #endif
 991
 992 #ifndef atomic64_xchg
 993 #define  atomic64_xchg(...)                                             \
 994         __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
 995 #endif
 996 #endif /* atomic64_xchg_relaxed */
 997
 998 /* atomic64_cmpxchg_relaxed */
 999 #ifndef atomic64_cmpxchg_relaxed
1000 #define  atomic64_cmpxchg_relaxed       atomic64_cmpxchg
1001 #define  atomic64_cmpxchg_acquire       atomic64_cmpxchg
1002 #define  atomic64_cmpxchg_release       atomic64_cmpxchg
1003
1004 #else /* atomic64_cmpxchg_relaxed */
1005
1006 #ifndef atomic64_cmpxchg_acquire
1007 #define  atomic64_cmpxchg_acquire(...)                                  \
1008         __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
1009 #endif
1010
1011 #ifndef atomic64_cmpxchg_release
1012 #define  atomic64_cmpxchg_release(...)                                  \
1013         __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
1014 #endif
1015
1016 #ifndef atomic64_cmpxchg
1017 #define  atomic64_cmpxchg(...)                                          \
1018         __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
1019 #endif
1020 #endif /* atomic64_cmpxchg_relaxed */
1021
1022 #ifndef atomic64_try_cmpxchg
1023
1024 #define __atomic64_try_cmpxchg(type, _p, _po, _n)                       \
1025 ({                                                                      \
1026         typeof(_po) __po = (_po);                                       \
1027         typeof(*(_po)) __r, __o = *__po;                                \
1028         __r = atomic64_cmpxchg##type((_p), __o, (_n));                  \
1029         if (unlikely(__r != __o))                                       \
1030                 *__po = __r;                                            \
1031         likely(__r == __o);                                             \
1032 })
1033
1034 #define atomic64_try_cmpxchg(_p, _po, _n)               __atomic64_try_cmpxchg(, _p, _po, _n)
1035 #define atomic64_try_cmpxchg_relaxed(_p, _po, _n)       __atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
1036 #define atomic64_try_cmpxchg_acquire(_p, _po, _n)       __atomic64_try_cmpxchg(_acquire, _p, _po, _n)
1037 #define atomic64_try_cmpxchg_release(_p, _po, _n)       __atomic64_try_cmpxchg(_release, _p, _po, _n)
1038
1039 #else /* atomic64_try_cmpxchg */
1040 #define atomic64_try_cmpxchg_relaxed    atomic64_try_cmpxchg
1041 #define atomic64_try_cmpxchg_acquire    atomic64_try_cmpxchg
1042 #define atomic64_try_cmpxchg_release    atomic64_try_cmpxchg
1043 #endif /* atomic64_try_cmpxchg */
1044
1045 #ifndef atomic64_andnot
1046 static inline void atomic64_andnot(long long i, atomic64_t *v)
1047 {
1048         atomic64_and(~i, v);
1049 }
1050
1051 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v)
1052 {
1053         return atomic64_fetch_and(~i, v);
1054 }
1055
1056 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v)
1057 {
1058         return atomic64_fetch_and_relaxed(~i, v);
1059 }
1060
1061 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v)
1062 {
1063         return atomic64_fetch_and_acquire(~i, v);
1064 }
1065
1066 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v)
1067 {
1068         return atomic64_fetch_and_release(~i, v);
1069 }
1070 #endif
1071
1072 #include <asm-generic/atomic-long.h>
1073
1074 #endif /* _LINUX_ATOMIC_H */