arch/mips/include/asm/atomic.h

   1 /*
   2  * Atomic operations that C can't guarantee us.  Useful for
   3  * resource counting etc..
   4  *
   5  * But use these as seldom as possible since they are much more slower
   6  * than regular operations.
   7  *
   8  * This file is subject to the terms and conditions of the GNU General Public
   9  * License.  See the file "COPYING" in the main directory of this archive
  10  * for more details.
  11  *
  12  * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle
  13  */
  14 #ifndef _ASM_ATOMIC_H
  15 #define _ASM_ATOMIC_H
  16
  17 #include <linux/irqflags.h>
  18 #include <linux/types.h>
  19 #include <asm/barrier.h>
  20 #include <asm/compiler.h>
  21 #include <asm/cpu-features.h>
  22 #include <asm/cmpxchg.h>
  23 #include <asm/war.h>
  24
  25 #define ATOMIC_INIT(i)    { (i) }
  26
  27 /*
  28  * atomic_read - read atomic variable
  29  * @v: pointer of type atomic_t
  30  *
  31  * Atomically reads the value of @v.
  32  */
  33 #define atomic_read(v)          ACCESS_ONCE((v)->counter)
  34
  35 /*
  36  * atomic_set - set atomic variable
  37  * @v: pointer of type atomic_t
  38  * @i: required value
  39  *
  40  * Atomically sets the value of @v to @i.
  41  */
  42 #define atomic_set(v, i)                ((v)->counter = (i))
  43
  44 #define ATOMIC_OP(op, c_op, asm_op)                                           \
  45 static __inline__ void atomic_##op(int i, atomic_t * v)                       \
  46 {                                                                             \
  47         if (kernel_uses_llsc && R10000_LLSC_WAR) {                            \
  48                 int temp;                                                     \
  49                                                                               \
  50                 __asm__ __volatile__(                                         \
  51                 "       .set    arch=r4000                              \n"   \
  52                 "1:     ll      %0, %1          # atomic_" #op "        \n"   \
  53                 "       " #asm_op " %0, %2                              \n"   \
  54                 "       sc      %0, %1                                  \n"   \
  55                 "       beqzl   %0, 1b                                  \n"   \
  56                 "       .set    mips0                                   \n"   \
  57                 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter)          \
  58                 : "Ir" (i));                                                  \
  59         } else if (kernel_uses_llsc) {                                        \
  60                 int temp;                                                     \
  61                                                                               \
  62                 do {                                                          \
  63                         __asm__ __volatile__(                                 \
  64                         "       .set    "MIPS_ISA_LEVEL"                \n"   \
  65                         "       ll      %0, %1          # atomic_" #op "\n"   \
  66                         "       " #asm_op " %0, %2                      \n"   \
  67                         "       sc      %0, %1                          \n"   \
  68                         "       .set    mips0                           \n"   \
  69                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter)      \
  70                         : "Ir" (i));                                          \
  71                 } while (unlikely(!temp));                                    \
  72         } else {                                                              \
  73                 unsigned long flags;                                          \
  74                                                                               \
  75                 raw_local_irq_save(flags);                                    \
  76                 v->counter c_op i;                                            \
  77                 raw_local_irq_restore(flags);                                 \
  78         }                                                                     \
  79 }
  80
  81 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                                    \
  82 static __inline__ int atomic_##op##_return(int i, atomic_t * v)               \
  83 {                                                                             \
  84         int result;                                                           \
  85                                                                               \
  86         smp_mb__before_llsc();                                                \
  87                                                                               \
  88         if (kernel_uses_llsc && R10000_LLSC_WAR) {                            \
  89                 int temp;                                                     \
  90                                                                               \
  91                 __asm__ __volatile__(                                         \
  92                 "       .set    arch=r4000                              \n"   \
  93                 "1:     ll      %1, %2          # atomic_" #op "_return \n"   \
  94                 "       " #asm_op " %0, %1, %3                          \n"   \
  95                 "       sc      %0, %2                                  \n"   \
  96                 "       beqzl   %0, 1b                                  \n"   \
  97                 "       " #asm_op " %0, %1, %3                          \n"   \
  98                 "       .set    mips0                                   \n"   \
  99                 : "=&r" (result), "=&r" (temp),                               \
 100                   "+" GCC_OFF_SMALL_ASM() (v->counter)                        \
 101                 : "Ir" (i));                                                  \
 102         } else if (kernel_uses_llsc) {                                        \
 103                 int temp;                                                     \
 104                                                                               \
 105                 do {                                                          \
 106                         __asm__ __volatile__(                                 \
 107                         "       .set    "MIPS_ISA_LEVEL"                \n"   \
 108                         "       ll      %1, %2  # atomic_" #op "_return \n"   \
 109                         "       " #asm_op " %0, %1, %3                  \n"   \
 110                         "       sc      %0, %2                          \n"   \
 111                         "       .set    mips0                           \n"   \
 112                         : "=&r" (result), "=&r" (temp),                       \
 113                           "+" GCC_OFF_SMALL_ASM() (v->counter)                \
 114                         : "Ir" (i));                                          \
 115                 } while (unlikely(!result));                                  \
 116                                                                               \
 117                 result = temp; result c_op i;                                 \
 118         } else {                                                              \
 119                 unsigned long flags;                                          \
 120                                                                               \
 121                 raw_local_irq_save(flags);                                    \
 122                 result = v->counter;                                          \
 123                 result c_op i;                                                \
 124                 v->counter = result;                                          \
 125                 raw_local_irq_restore(flags);                                 \
 126         }                                                                     \
 127                                                                               \
 128         smp_llsc_mb();                                                        \
 129                                                                               \
 130         return result;                                                        \
 131 }
 132
 133 #define ATOMIC_OPS(op, c_op, asm_op)                                          \
 134         ATOMIC_OP(op, c_op, asm_op)                                           \
 135         ATOMIC_OP_RETURN(op, c_op, asm_op)
 136
 137 ATOMIC_OPS(add, +=, addu)
 138 ATOMIC_OPS(sub, -=, subu)
 139
 140 ATOMIC_OP(and, &=, and)
 141 ATOMIC_OP(or, |=, or)
 142 ATOMIC_OP(xor, ^=, xor)
 143
 144 #undef ATOMIC_OPS
 145 #undef ATOMIC_OP_RETURN
 146 #undef ATOMIC_OP
 147
 148 /*
 149  * atomic_sub_if_positive - conditionally subtract integer from atomic variable
 150  * @i: integer value to subtract
 151  * @v: pointer of type atomic_t
 152  *
 153  * Atomically test @v and subtract @i if @v is greater or equal than @i.
 154  * The function returns the old value of @v minus @i.
 155  */
 156 static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
 157 {
 158         int result;
 159
 160         smp_mb__before_llsc();
 161
 162         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 163                 int temp;
 164
 165                 __asm__ __volatile__(
 166                 "       .set    arch=r4000                              \n"
 167                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
 168                 "       subu    %0, %1, %3                              \n"
 169                 "       bltz    %0, 1f                                  \n"
 170                 "       sc      %0, %2                                  \n"
 171                 "       .set    noreorder                               \n"
 172                 "       beqzl   %0, 1b                                  \n"
 173                 "        subu   %0, %1, %3                              \n"
 174                 "       .set    reorder                                 \n"
 175                 "1:                                                     \n"
 176                 "       .set    mips0                                   \n"
 177                 : "=&r" (result), "=&r" (temp),
 178                   "+" GCC_OFF_SMALL_ASM() (v->counter)
 179                 : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter)
 180                 : "memory");
 181         } else if (kernel_uses_llsc) {
 182                 int temp;
 183
 184                 __asm__ __volatile__(
 185                 "       .set    "MIPS_ISA_LEVEL"                        \n"
 186                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
 187                 "       subu    %0, %1, %3                              \n"
 188                 "       bltz    %0, 1f                                  \n"
 189                 "       sc      %0, %2                                  \n"
 190                 "       .set    noreorder                               \n"
 191                 "       beqz    %0, 1b                                  \n"
 192                 "        subu   %0, %1, %3                              \n"
 193                 "       .set    reorder                                 \n"
 194                 "1:                                                     \n"
 195                 "       .set    mips0                                   \n"
 196                 : "=&r" (result), "=&r" (temp),
 197                   "+" GCC_OFF_SMALL_ASM() (v->counter)
 198                 : "Ir" (i));
 199         } else {
 200                 unsigned long flags;
 201
 202                 raw_local_irq_save(flags);
 203                 result = v->counter;
 204                 result -= i;
 205                 if (result >= 0)
 206                         v->counter = result;
 207                 raw_local_irq_restore(flags);
 208         }
 209
 210         smp_llsc_mb();
 211
 212         return result;
 213 }
 214
 215 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 216 #define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
 217
 218 /**
 219  * __atomic_add_unless - add unless the number is a given value
 220  * @v: pointer of type atomic_t
 221  * @a: the amount to add to v...
 222  * @u: ...unless v is equal to u.
 223  *
 224  * Atomically adds @a to @v, so long as it was not @u.
 225  * Returns the old value of @v.
 226  */
 227 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
 228 {
 229         int c, old;
 230         c = atomic_read(v);
 231         for (;;) {
 232                 if (unlikely(c == (u)))
 233                         break;
 234                 old = atomic_cmpxchg((v), c, c + (a));
 235                 if (likely(old == c))
 236                         break;
 237                 c = old;
 238         }
 239         return c;
 240 }
 241
 242 #define atomic_dec_return(v) atomic_sub_return(1, (v))
 243 #define atomic_inc_return(v) atomic_add_return(1, (v))
 244
 245 /*
 246  * atomic_sub_and_test - subtract value from variable and test result
 247  * @i: integer value to subtract
 248  * @v: pointer of type atomic_t
 249  *
 250  * Atomically subtracts @i from @v and returns
 251  * true if the result is zero, or false for all
 252  * other cases.
 253  */
 254 #define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
 255
 256 /*
 257  * atomic_inc_and_test - increment and test
 258  * @v: pointer of type atomic_t
 259  *
 260  * Atomically increments @v by 1
 261  * and returns true if the result is zero, or false for all
 262  * other cases.
 263  */
 264 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
 265
 266 /*
 267  * atomic_dec_and_test - decrement by 1 and test
 268  * @v: pointer of type atomic_t
 269  *
 270  * Atomically decrements @v by 1 and
 271  * returns true if the result is 0, or false for all other
 272  * cases.
 273  */
 274 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
 275
 276 /*
 277  * atomic_dec_if_positive - decrement by 1 if old value positive
 278  * @v: pointer of type atomic_t
 279  */
 280 #define atomic_dec_if_positive(v)       atomic_sub_if_positive(1, v)
 281
 282 /*
 283  * atomic_inc - increment atomic variable
 284  * @v: pointer of type atomic_t
 285  *
 286  * Atomically increments @v by 1.
 287  */
 288 #define atomic_inc(v) atomic_add(1, (v))
 289
 290 /*
 291  * atomic_dec - decrement and test
 292  * @v: pointer of type atomic_t
 293  *
 294  * Atomically decrements @v by 1.
 295  */
 296 #define atomic_dec(v) atomic_sub(1, (v))
 297
 298 /*
 299  * atomic_add_negative - add and test if negative
 300  * @v: pointer of type atomic_t
 301  * @i: integer value to add
 302  *
 303  * Atomically adds @i to @v and returns true
 304  * if the result is negative, or false when
 305  * result is greater than or equal to zero.
 306  */
 307 #define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0)
 308
 309 #ifdef CONFIG_64BIT
 310
 311 #define ATOMIC64_INIT(i)    { (i) }
 312
 313 /*
 314  * atomic64_read - read atomic variable
 315  * @v: pointer of type atomic64_t
 316  *
 317  */
 318 #define atomic64_read(v)        ACCESS_ONCE((v)->counter)
 319
 320 /*
 321  * atomic64_set - set atomic variable
 322  * @v: pointer of type atomic64_t
 323  * @i: required value
 324  */
 325 #define atomic64_set(v, i)      ((v)->counter = (i))
 326
 327 #define ATOMIC64_OP(op, c_op, asm_op)                                         \
 328 static __inline__ void atomic64_##op(long i, atomic64_t * v)                  \
 329 {                                                                             \
 330         if (kernel_uses_llsc && R10000_LLSC_WAR) {                            \
 331                 long temp;                                                    \
 332                                                                               \
 333                 __asm__ __volatile__(                                         \
 334                 "       .set    arch=r4000                              \n"   \
 335                 "1:     lld     %0, %1          # atomic64_" #op "      \n"   \
 336                 "       " #asm_op " %0, %2                              \n"   \
 337                 "       scd     %0, %1                                  \n"   \
 338                 "       beqzl   %0, 1b                                  \n"   \
 339                 "       .set    mips0                                   \n"   \
 340                 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter)          \
 341                 : "Ir" (i));                                                  \
 342         } else if (kernel_uses_llsc) {                                        \
 343                 long temp;                                                    \
 344                                                                               \
 345                 do {                                                          \
 346                         __asm__ __volatile__(                                 \
 347                         "       .set    "MIPS_ISA_LEVEL"                \n"   \
 348                         "       lld     %0, %1          # atomic64_" #op "\n" \
 349                         "       " #asm_op " %0, %2                      \n"   \
 350                         "       scd     %0, %1                          \n"   \
 351                         "       .set    mips0                           \n"   \
 352                         : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter)      \
 353                         : "Ir" (i));                                          \
 354                 } while (unlikely(!temp));                                    \
 355         } else {                                                              \
 356                 unsigned long flags;                                          \
 357                                                                               \
 358                 raw_local_irq_save(flags);                                    \
 359                 v->counter c_op i;                                            \
 360                 raw_local_irq_restore(flags);                                 \
 361         }                                                                     \
 362 }
 363
 364 #define ATOMIC64_OP_RETURN(op, c_op, asm_op)                                  \
 365 static __inline__ long atomic64_##op##_return(long i, atomic64_t * v)         \
 366 {                                                                             \
 367         long result;                                                          \
 368                                                                               \
 369         smp_mb__before_llsc();                                                \
 370                                                                               \
 371         if (kernel_uses_llsc && R10000_LLSC_WAR) {                            \
 372                 long temp;                                                    \
 373                                                                               \
 374                 __asm__ __volatile__(                                         \
 375                 "       .set    arch=r4000                              \n"   \
 376                 "1:     lld     %1, %2          # atomic64_" #op "_return\n"  \
 377                 "       " #asm_op " %0, %1, %3                          \n"   \
 378                 "       scd     %0, %2                                  \n"   \
 379                 "       beqzl   %0, 1b                                  \n"   \
 380                 "       " #asm_op " %0, %1, %3                          \n"   \
 381                 "       .set    mips0                                   \n"   \
 382                 : "=&r" (result), "=&r" (temp),                               \
 383                   "+" GCC_OFF_SMALL_ASM() (v->counter)                        \
 384                 : "Ir" (i));                                                  \
 385         } else if (kernel_uses_llsc) {                                        \
 386                 long temp;                                                    \
 387                                                                               \
 388                 do {                                                          \
 389                         __asm__ __volatile__(                                 \
 390                         "       .set    "MIPS_ISA_LEVEL"                \n"   \
 391                         "       lld     %1, %2  # atomic64_" #op "_return\n"  \
 392                         "       " #asm_op " %0, %1, %3                  \n"   \
 393                         "       scd     %0, %2                          \n"   \
 394                         "       .set    mips0                           \n"   \
 395                         : "=&r" (result), "=&r" (temp),                       \
 396                           "=" GCC_OFF_SMALL_ASM() (v->counter)                \
 397                         : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter)          \
 398                         : "memory");                                          \
 399                 } while (unlikely(!result));                                  \
 400                                                                               \
 401                 result = temp; result c_op i;                                 \
 402         } else {                                                              \
 403                 unsigned long flags;                                          \
 404                                                                               \
 405                 raw_local_irq_save(flags);                                    \
 406                 result = v->counter;                                          \
 407                 result c_op i;                                                \
 408                 v->counter = result;                                          \
 409                 raw_local_irq_restore(flags);                                 \
 410         }                                                                     \
 411                                                                               \
 412         smp_llsc_mb();                                                        \
 413                                                                               \
 414         return result;                                                        \
 415 }
 416
 417 #define ATOMIC64_OPS(op, c_op, asm_op)                                        \
 418         ATOMIC64_OP(op, c_op, asm_op)                                         \
 419         ATOMIC64_OP_RETURN(op, c_op, asm_op)
 420
 421 ATOMIC64_OPS(add, +=, daddu)
 422 ATOMIC64_OPS(sub, -=, dsubu)
 423 ATOMIC64_OP(and, &=, and)
 424 ATOMIC64_OP(or, |=, or)
 425 ATOMIC64_OP(xor, ^=, xor)
 426
 427 #undef ATOMIC64_OPS
 428 #undef ATOMIC64_OP_RETURN
 429 #undef ATOMIC64_OP
 430
 431 /*
 432  * atomic64_sub_if_positive - conditionally subtract integer from atomic
 433  *                            variable
 434  * @i: integer value to subtract
 435  * @v: pointer of type atomic64_t
 436  *
 437  * Atomically test @v and subtract @i if @v is greater or equal than @i.
 438  * The function returns the old value of @v minus @i.
 439  */
 440 static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
 441 {
 442         long result;
 443
 444         smp_mb__before_llsc();
 445
 446         if (kernel_uses_llsc && R10000_LLSC_WAR) {
 447                 long temp;
 448
 449                 __asm__ __volatile__(
 450                 "       .set    arch=r4000                              \n"
 451                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
 452                 "       dsubu   %0, %1, %3                              \n"
 453                 "       bltz    %0, 1f                                  \n"
 454                 "       scd     %0, %2                                  \n"
 455                 "       .set    noreorder                               \n"
 456                 "       beqzl   %0, 1b                                  \n"
 457                 "        dsubu  %0, %1, %3                              \n"
 458                 "       .set    reorder                                 \n"
 459                 "1:                                                     \n"
 460                 "       .set    mips0                                   \n"
 461                 : "=&r" (result), "=&r" (temp),
 462                   "=" GCC_OFF_SMALL_ASM() (v->counter)
 463                 : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter)
 464                 : "memory");
 465         } else if (kernel_uses_llsc) {
 466                 long temp;
 467
 468                 __asm__ __volatile__(
 469                 "       .set    "MIPS_ISA_LEVEL"                        \n"
 470                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
 471                 "       dsubu   %0, %1, %3                              \n"
 472                 "       bltz    %0, 1f                                  \n"
 473                 "       scd     %0, %2                                  \n"
 474                 "       .set    noreorder                               \n"
 475                 "       beqz    %0, 1b                                  \n"
 476                 "        dsubu  %0, %1, %3                              \n"
 477                 "       .set    reorder                                 \n"
 478                 "1:                                                     \n"
 479                 "       .set    mips0                                   \n"
 480                 : "=&r" (result), "=&r" (temp),
 481                   "+" GCC_OFF_SMALL_ASM() (v->counter)
 482                 : "Ir" (i));
 483         } else {
 484                 unsigned long flags;
 485
 486                 raw_local_irq_save(flags);
 487                 result = v->counter;
 488                 result -= i;
 489                 if (result >= 0)
 490                         v->counter = result;
 491                 raw_local_irq_restore(flags);
 492         }
 493
 494         smp_llsc_mb();
 495
 496         return result;
 497 }
 498
 499 #define atomic64_cmpxchg(v, o, n) \
 500         ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
 501 #define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
 502
 503 /**
 504  * atomic64_add_unless - add unless the number is a given value
 505  * @v: pointer of type atomic64_t
 506  * @a: the amount to add to v...
 507  * @u: ...unless v is equal to u.
 508  *
 509  * Atomically adds @a to @v, so long as it was not @u.
 510  * Returns the old value of @v.
 511  */
 512 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
 513 {
 514         long c, old;
 515         c = atomic64_read(v);
 516         for (;;) {
 517                 if (unlikely(c == (u)))
 518                         break;
 519                 old = atomic64_cmpxchg((v), c, c + (a));
 520                 if (likely(old == c))
 521                         break;
 522                 c = old;
 523         }
 524         return c != (u);
 525 }
 526
 527 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
 528
 529 #define atomic64_dec_return(v) atomic64_sub_return(1, (v))
 530 #define atomic64_inc_return(v) atomic64_add_return(1, (v))
 531
 532 /*
 533  * atomic64_sub_and_test - subtract value from variable and test result
 534  * @i: integer value to subtract
 535  * @v: pointer of type atomic64_t
 536  *
 537  * Atomically subtracts @i from @v and returns
 538  * true if the result is zero, or false for all
 539  * other cases.
 540  */
 541 #define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
 542
 543 /*
 544  * atomic64_inc_and_test - increment and test
 545  * @v: pointer of type atomic64_t
 546  *
 547  * Atomically increments @v by 1
 548  * and returns true if the result is zero, or false for all
 549  * other cases.
 550  */
 551 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
 552
 553 /*
 554  * atomic64_dec_and_test - decrement by 1 and test
 555  * @v: pointer of type atomic64_t
 556  *
 557  * Atomically decrements @v by 1 and
 558  * returns true if the result is 0, or false for all other
 559  * cases.
 560  */
 561 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
 562
 563 /*
 564  * atomic64_dec_if_positive - decrement by 1 if old value positive
 565  * @v: pointer of type atomic64_t
 566  */
 567 #define atomic64_dec_if_positive(v)     atomic64_sub_if_positive(1, v)
 568
 569 /*
 570  * atomic64_inc - increment atomic variable
 571  * @v: pointer of type atomic64_t
 572  *
 573  * Atomically increments @v by 1.
 574  */
 575 #define atomic64_inc(v) atomic64_add(1, (v))
 576
 577 /*
 578  * atomic64_dec - decrement and test
 579  * @v: pointer of type atomic64_t
 580  *
 581  * Atomically decrements @v by 1.
 582  */
 583 #define atomic64_dec(v) atomic64_sub(1, (v))
 584
 585 /*
 586  * atomic64_add_negative - add and test if negative
 587  * @v: pointer of type atomic64_t
 588  * @i: integer value to add
 589  *
 590  * Atomically adds @i to @v and returns true
 591  * if the result is negative, or false when
 592  * result is greater than or equal to zero.
 593  */
 594 #define atomic64_add_negative(i, v) (atomic64_add_return(i, (v)) < 0)
 595
 596 #endif /* CONFIG_64BIT */
 597
 598 #endif /* _ASM_ATOMIC_H */