arch/s390/include/asm/atomic.h

   1 /*
   2  * Copyright IBM Corp. 1999, 2009
   3  * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
   4  *            Denis Joseph Barrow,
   5  *            Arnd Bergmann <arndb@de.ibm.com>,
   6  *
   7  * Atomic operations that C can't guarantee us.
   8  * Useful for resource counting etc.
   9  * s390 uses 'Compare And Swap' for atomicity in SMP environment.
  10  *
  11  */
  12
  13 #ifndef __ARCH_S390_ATOMIC__
  14 #define __ARCH_S390_ATOMIC__
  15
  16 #include <linux/compiler.h>
  17 #include <linux/types.h>
  18 #include <asm/barrier.h>
  19 #include <asm/cmpxchg.h>
  20
  21 #define ATOMIC_INIT(i)  { (i) }
  22
  23 #define __ATOMIC_NO_BARRIER     "\n"
  24
  25 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
  26
  27 #define __ATOMIC_OR     "lao"
  28 #define __ATOMIC_AND    "lan"
  29 #define __ATOMIC_ADD    "laa"
  30 #define __ATOMIC_XOR    "lax"
  31 #define __ATOMIC_BARRIER "bcr   14,0\n"
  32
  33 #define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)                \
  34 ({                                                                      \
  35         int old_val;                                                    \
  36                                                                         \
  37         typecheck(atomic_t *, ptr);                                     \
  38         asm volatile(                                                   \
  39                 op_string "     %0,%2,%1\n"                             \
  40                 __barrier                                               \
  41                 : "=d" (old_val), "+Q" ((ptr)->counter)                 \
  42                 : "d" (op_val)                                          \
  43                 : "cc", "memory");                                      \
  44         old_val;                                                        \
  45 })
  46
  47 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
  48
  49 #define __ATOMIC_OR     "or"
  50 #define __ATOMIC_AND    "nr"
  51 #define __ATOMIC_ADD    "ar"
  52 #define __ATOMIC_XOR    "xr"
  53 #define __ATOMIC_BARRIER "\n"
  54
  55 #define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)                \
  56 ({                                                                      \
  57         int old_val, new_val;                                           \
  58                                                                         \
  59         typecheck(atomic_t *, ptr);                                     \
  60         asm volatile(                                                   \
  61                 "       l       %0,%2\n"                                \
  62                 "0:     lr      %1,%0\n"                                \
  63                 op_string "     %1,%3\n"                                \
  64                 "       cs      %0,%1,%2\n"                             \
  65                 "       jl      0b"                                     \
  66                 : "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
  67                 : "d" (op_val)                                          \
  68                 : "cc", "memory");                                      \
  69         old_val;                                                        \
  70 })
  71
  72 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
  73
  74 static inline int atomic_read(const atomic_t *v)
  75 {
  76         int c;
  77
  78         asm volatile(
  79                 "       l       %0,%1\n"
  80                 : "=d" (c) : "Q" (v->counter));
  81         return c;
  82 }
  83
  84 static inline void atomic_set(atomic_t *v, int i)
  85 {
  86         asm volatile(
  87                 "       st      %1,%0\n"
  88                 : "=Q" (v->counter) : "d" (i));
  89 }
  90
  91 static inline int atomic_add_return(int i, atomic_t *v)
  92 {
  93         return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i;
  94 }
  95
  96 static inline int atomic_fetch_add(int i, atomic_t *v)
  97 {
  98         return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER);
  99 }
 100
 101 static inline void atomic_add(int i, atomic_t *v)
 102 {
 103 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 104         if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
 105                 asm volatile(
 106                         "asi    %0,%1\n"
 107                         : "+Q" (v->counter)
 108                         : "i" (i)
 109                         : "cc", "memory");
 110                 return;
 111         }
 112 #endif
 113         __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER);
 114 }
 115
 116 #define atomic_add_negative(_i, _v)     (atomic_add_return(_i, _v) < 0)
 117 #define atomic_inc(_v)                  atomic_add(1, _v)
 118 #define atomic_inc_return(_v)           atomic_add_return(1, _v)
 119 #define atomic_inc_and_test(_v)         (atomic_add_return(1, _v) == 0)
 120 #define atomic_sub(_i, _v)              atomic_add(-(int)(_i), _v)
 121 #define atomic_sub_return(_i, _v)       atomic_add_return(-(int)(_i), _v)
 122 #define atomic_fetch_sub(_i, _v)        atomic_fetch_add(-(int)(_i), _v)
 123 #define atomic_sub_and_test(_i, _v)     (atomic_sub_return(_i, _v) == 0)
 124 #define atomic_dec(_v)                  atomic_sub(1, _v)
 125 #define atomic_dec_return(_v)           atomic_sub_return(1, _v)
 126 #define atomic_dec_and_test(_v)         (atomic_sub_return(1, _v) == 0)
 127
 128 #define ATOMIC_OPS(op, OP)                                              \
 129 static inline void atomic_##op(int i, atomic_t *v)                      \
 130 {                                                                       \
 131         __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_NO_BARRIER);        \
 132 }                                                                       \
 133 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 134 {                                                                       \
 135         return __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_BARRIER);    \
 136 }
 137
 138 ATOMIC_OPS(and, AND)
 139 ATOMIC_OPS(or, OR)
 140 ATOMIC_OPS(xor, XOR)
 141
 142 #undef ATOMIC_OPS
 143
 144 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
 145
 146 static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
 147 {
 148         asm volatile(
 149                 "       cs      %0,%2,%1"
 150                 : "+d" (old), "+Q" (v->counter)
 151                 : "d" (new)
 152                 : "cc", "memory");
 153         return old;
 154 }
 155
 156 static inline int __atomic_add_unless(atomic_t *v, int a, int u)
 157 {
 158         int c, old;
 159         c = atomic_read(v);
 160         for (;;) {
 161                 if (unlikely(c == u))
 162                         break;
 163                 old = atomic_cmpxchg(v, c, c + a);
 164                 if (likely(old == c))
 165                         break;
 166                 c = old;
 167         }
 168         return c;
 169 }
 170
 171
 172 #undef __ATOMIC_LOOP
 173
 174 #define ATOMIC64_INIT(i)  { (i) }
 175
 176 #define __ATOMIC64_NO_BARRIER   "\n"
 177
 178 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 179
 180 #define __ATOMIC64_OR   "laog"
 181 #define __ATOMIC64_AND  "lang"
 182 #define __ATOMIC64_ADD  "laag"
 183 #define __ATOMIC64_XOR  "laxg"
 184 #define __ATOMIC64_BARRIER "bcr 14,0\n"
 185
 186 #define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)              \
 187 ({                                                                      \
 188         long long old_val;                                              \
 189                                                                         \
 190         typecheck(atomic64_t *, ptr);                                   \
 191         asm volatile(                                                   \
 192                 op_string "     %0,%2,%1\n"                             \
 193                 __barrier                                               \
 194                 : "=d" (old_val), "+Q" ((ptr)->counter)                 \
 195                 : "d" (op_val)                                          \
 196                 : "cc", "memory");                                      \
 197         old_val;                                                        \
 198 })
 199
 200 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 201
 202 #define __ATOMIC64_OR   "ogr"
 203 #define __ATOMIC64_AND  "ngr"
 204 #define __ATOMIC64_ADD  "agr"
 205 #define __ATOMIC64_XOR  "xgr"
 206 #define __ATOMIC64_BARRIER "\n"
 207
 208 #define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)              \
 209 ({                                                                      \
 210         long long old_val, new_val;                                     \
 211                                                                         \
 212         typecheck(atomic64_t *, ptr);                                   \
 213         asm volatile(                                                   \
 214                 "       lg      %0,%2\n"                                \
 215                 "0:     lgr     %1,%0\n"                                \
 216                 op_string "     %1,%3\n"                                \
 217                 "       csg     %0,%1,%2\n"                             \
 218                 "       jl      0b"                                     \
 219                 : "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
 220                 : "d" (op_val)                                          \
 221                 : "cc", "memory");                                      \
 222         old_val;                                                        \
 223 })
 224
 225 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 226
 227 static inline long long atomic64_read(const atomic64_t *v)
 228 {
 229         long long c;
 230
 231         asm volatile(
 232                 "       lg      %0,%1\n"
 233                 : "=d" (c) : "Q" (v->counter));
 234         return c;
 235 }
 236
 237 static inline void atomic64_set(atomic64_t *v, long long i)
 238 {
 239         asm volatile(
 240                 "       stg     %1,%0\n"
 241                 : "=Q" (v->counter) : "d" (i));
 242 }
 243
 244 static inline long long atomic64_add_return(long long i, atomic64_t *v)
 245 {
 246         return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i;
 247 }
 248
 249 static inline long long atomic64_fetch_add(long long i, atomic64_t *v)
 250 {
 251         return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER);
 252 }
 253
 254 static inline void atomic64_add(long long i, atomic64_t *v)
 255 {
 256 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 257         if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
 258                 asm volatile(
 259                         "agsi   %0,%1\n"
 260                         : "+Q" (v->counter)
 261                         : "i" (i)
 262                         : "cc", "memory");
 263                 return;
 264         }
 265 #endif
 266         __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER);
 267 }
 268
 269 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
 270
 271 static inline long long atomic64_cmpxchg(atomic64_t *v,
 272                                              long long old, long long new)
 273 {
 274         asm volatile(
 275                 "       csg     %0,%2,%1"
 276                 : "+d" (old), "+Q" (v->counter)
 277                 : "d" (new)
 278                 : "cc", "memory");
 279         return old;
 280 }
 281
 282 #define ATOMIC64_OPS(op, OP)                                            \
 283 static inline void atomic64_##op(long i, atomic64_t *v)                 \
 284 {                                                                       \
 285         __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_NO_BARRIER);  \
 286 }                                                                       \
 287 static inline long atomic64_fetch_##op(long i, atomic64_t *v)           \
 288 {                                                                       \
 289         return __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_BARRIER); \
 290 }
 291
 292 ATOMIC64_OPS(and, AND)
 293 ATOMIC64_OPS(or, OR)
 294 ATOMIC64_OPS(xor, XOR)
 295
 296 #undef ATOMIC64_OPS
 297 #undef __ATOMIC64_LOOP
 298
 299 static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u)
 300 {
 301         long long c, old;
 302
 303         c = atomic64_read(v);
 304         for (;;) {
 305                 if (unlikely(c == u))
 306                         break;
 307                 old = atomic64_cmpxchg(v, c, c + i);
 308                 if (likely(old == c))
 309                         break;
 310                 c = old;
 311         }
 312         return c != u;
 313 }
 314
 315 static inline long long atomic64_dec_if_positive(atomic64_t *v)
 316 {
 317         long long c, old, dec;
 318
 319         c = atomic64_read(v);
 320         for (;;) {
 321                 dec = c - 1;
 322                 if (unlikely(dec < 0))
 323                         break;
 324                 old = atomic64_cmpxchg((v), c, dec);
 325                 if (likely(old == c))
 326                         break;
 327                 c = old;
 328         }
 329         return dec;
 330 }
 331
 332 #define atomic64_add_negative(_i, _v)   (atomic64_add_return(_i, _v) < 0)
 333 #define atomic64_inc(_v)                atomic64_add(1, _v)
 334 #define atomic64_inc_return(_v)         atomic64_add_return(1, _v)
 335 #define atomic64_inc_and_test(_v)       (atomic64_add_return(1, _v) == 0)
 336 #define atomic64_sub_return(_i, _v)     atomic64_add_return(-(long long)(_i), _v)
 337 #define atomic64_fetch_sub(_i, _v)      atomic64_fetch_add(-(long long)(_i), _v)
 338 #define atomic64_sub(_i, _v)            atomic64_add(-(long long)(_i), _v)
 339 #define atomic64_sub_and_test(_i, _v)   (atomic64_sub_return(_i, _v) == 0)
 340 #define atomic64_dec(_v)                atomic64_sub(1, _v)
 341 #define atomic64_dec_return(_v)         atomic64_sub_return(1, _v)
 342 #define atomic64_dec_and_test(_v)       (atomic64_sub_return(1, _v) == 0)
 343 #define atomic64_inc_not_zero(v)        atomic64_add_unless((v), 1, 0)
 344
 345 #endif /* __ARCH_S390_ATOMIC__  */