arch/arm/include/asm/cmpxchg.h

   1 #ifndef __ASM_ARM_CMPXCHG_H
   2 #define __ASM_ARM_CMPXCHG_H
   3
   4 #include <linux/irqflags.h>
   5 #include <linux/prefetch.h>
   6 #include <asm/barrier.h>
   7
   8 #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
   9 /*
  10  * On the StrongARM, "swp" is terminally broken since it bypasses the
  11  * cache totally.  This means that the cache becomes inconsistent, and,
  12  * since we use normal loads/stores as well, this is really bad.
  13  * Typically, this causes oopsen in filp_close, but could have other,
  14  * more disastrous effects.  There are two work-arounds:
  15  *  1. Disable interrupts and emulate the atomic swap
  16  *  2. Clean the cache, perform atomic swap, flush the cache
  17  *
  18  * We choose (1) since its the "easiest" to achieve here and is not
  19  * dependent on the processor type.
  20  *
  21  * NOTE that this solution won't work on an SMP system, so explcitly
  22  * forbid it here.
  23  */
  24 #define swp_is_buggy
  25 #endif
  26
  27 static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
  28 {
  29         extern void __bad_xchg(volatile void *, int);
  30         unsigned long ret;
  31 #ifdef swp_is_buggy
  32         unsigned long flags;
  33 #endif
  34 #if __LINUX_ARM_ARCH__ >= 6
  35         unsigned int tmp;
  36 #endif
  37
  38         prefetchw((const void *)ptr);
  39
  40         switch (size) {
  41 #if __LINUX_ARM_ARCH__ >= 6
  42         case 1:
  43                 asm volatile("@ __xchg1\n"
  44                 "1:     ldrexb  %0, [%3]\n"
  45                 "       strexb  %1, %2, [%3]\n"
  46                 "       teq     %1, #0\n"
  47                 "       bne     1b"
  48                         : "=&r" (ret), "=&r" (tmp)
  49                         : "r" (x), "r" (ptr)
  50                         : "memory", "cc");
  51                 break;
  52         case 4:
  53                 asm volatile("@ __xchg4\n"
  54                 "1:     ldrex   %0, [%3]\n"
  55                 "       strex   %1, %2, [%3]\n"
  56                 "       teq     %1, #0\n"
  57                 "       bne     1b"
  58                         : "=&r" (ret), "=&r" (tmp)
  59                         : "r" (x), "r" (ptr)
  60                         : "memory", "cc");
  61                 break;
  62 #elif defined(swp_is_buggy)
  63 #ifdef CONFIG_SMP
  64 #error SMP is not supported on this platform
  65 #endif
  66         case 1:
  67                 raw_local_irq_save(flags);
  68                 ret = *(volatile unsigned char *)ptr;
  69                 *(volatile unsigned char *)ptr = x;
  70                 raw_local_irq_restore(flags);
  71                 break;
  72
  73         case 4:
  74                 raw_local_irq_save(flags);
  75                 ret = *(volatile unsigned long *)ptr;
  76                 *(volatile unsigned long *)ptr = x;
  77                 raw_local_irq_restore(flags);
  78                 break;
  79 #else
  80         case 1:
  81                 asm volatile("@ __xchg1\n"
  82                 "       swpb    %0, %1, [%2]"
  83                         : "=&r" (ret)
  84                         : "r" (x), "r" (ptr)
  85                         : "memory", "cc");
  86                 break;
  87         case 4:
  88                 asm volatile("@ __xchg4\n"
  89                 "       swp     %0, %1, [%2]"
  90                         : "=&r" (ret)
  91                         : "r" (x), "r" (ptr)
  92                         : "memory", "cc");
  93                 break;
  94 #endif
  95         default:
  96                 /* Cause a link-time error, the xchg() size is not supported */
  97                 __bad_xchg(ptr, size), ret = 0;
  98                 break;
  99         }
 100
 101         return ret;
 102 }
 103
 104 #define xchg_relaxed(ptr, x) ({                                         \
 105         (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr),           \
 106                                    sizeof(*(ptr)));                     \
 107 })
 108
 109 #include <asm-generic/cmpxchg-local.h>
 110
 111 #if __LINUX_ARM_ARCH__ < 6
 112 /* min ARCH < ARMv6 */
 113
 114 #ifdef CONFIG_SMP
 115 #error "SMP is not supported on this platform"
 116 #endif
 117
 118 #define xchg xchg_relaxed
 119
 120 /*
 121  * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
 122  * them available.
 123  */
 124 #define cmpxchg_local(ptr, o, n) ({                                     \
 125         (__typeof(*ptr))__cmpxchg_local_generic((ptr),                  \
 126                                                 (unsigned long)(o),     \
 127                                                 (unsigned long)(n),     \
 128                                                 sizeof(*(ptr)));        \
 129 })
 130
 131 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
 132
 133 #include <asm-generic/cmpxchg.h>
 134
 135 #else   /* min ARCH >= ARMv6 */
 136
 137 extern void __bad_cmpxchg(volatile void *ptr, int size);
 138
 139 /*
 140  * cmpxchg only support 32-bits operands on ARMv6.
 141  */
 142
 143 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 144                                       unsigned long new, int size)
 145 {
 146         unsigned long oldval, res;
 147
 148         prefetchw((const void *)ptr);
 149
 150         switch (size) {
 151 #ifndef CONFIG_CPU_V6   /* min ARCH >= ARMv6K */
 152         case 1:
 153                 do {
 154                         asm volatile("@ __cmpxchg1\n"
 155                         "       ldrexb  %1, [%2]\n"
 156                         "       mov     %0, #0\n"
 157                         "       teq     %1, %3\n"
 158                         "       strexbeq %0, %4, [%2]\n"
 159                                 : "=&r" (res), "=&r" (oldval)
 160                                 : "r" (ptr), "Ir" (old), "r" (new)
 161                                 : "memory", "cc");
 162                 } while (res);
 163                 break;
 164         case 2:
 165                 do {
 166                         asm volatile("@ __cmpxchg1\n"
 167                         "       ldrexh  %1, [%2]\n"
 168                         "       mov     %0, #0\n"
 169                         "       teq     %1, %3\n"
 170                         "       strexheq %0, %4, [%2]\n"
 171                                 : "=&r" (res), "=&r" (oldval)
 172                                 : "r" (ptr), "Ir" (old), "r" (new)
 173                                 : "memory", "cc");
 174                 } while (res);
 175                 break;
 176 #endif
 177         case 4:
 178                 do {
 179                         asm volatile("@ __cmpxchg4\n"
 180                         "       ldrex   %1, [%2]\n"
 181                         "       mov     %0, #0\n"
 182                         "       teq     %1, %3\n"
 183                         "       strexeq %0, %4, [%2]\n"
 184                                 : "=&r" (res), "=&r" (oldval)
 185                                 : "r" (ptr), "Ir" (old), "r" (new)
 186                                 : "memory", "cc");
 187                 } while (res);
 188                 break;
 189         default:
 190                 __bad_cmpxchg(ptr, size);
 191                 oldval = 0;
 192         }
 193
 194         return oldval;
 195 }
 196
 197 #define cmpxchg_relaxed(ptr,o,n) ({                                     \
 198         (__typeof__(*(ptr)))__cmpxchg((ptr),                            \
 199                                       (unsigned long)(o),               \
 200                                       (unsigned long)(n),               \
 201                                       sizeof(*(ptr)));                  \
 202 })
 203
 204 static inline unsigned long __cmpxchg_local(volatile void *ptr,
 205                                             unsigned long old,
 206                                             unsigned long new, int size)
 207 {
 208         unsigned long ret;
 209
 210         switch (size) {
 211 #ifdef CONFIG_CPU_V6    /* min ARCH == ARMv6 */
 212         case 1:
 213         case 2:
 214                 ret = __cmpxchg_local_generic(ptr, old, new, size);
 215                 break;
 216 #endif
 217         default:
 218                 ret = __cmpxchg(ptr, old, new, size);
 219         }
 220
 221         return ret;
 222 }
 223
 224 #define cmpxchg_local(ptr, o, n) ({                                     \
 225         (__typeof(*ptr))__cmpxchg_local((ptr),                          \
 226                                         (unsigned long)(o),             \
 227                                         (unsigned long)(n),             \
 228                                         sizeof(*(ptr)));                \
 229 })
 230
 231 static inline unsigned long long __cmpxchg64(unsigned long long *ptr,
 232                                              unsigned long long old,
 233                                              unsigned long long new)
 234 {
 235         unsigned long long oldval;
 236         unsigned long res;
 237
 238         prefetchw(ptr);
 239
 240         __asm__ __volatile__(
 241 "1:     ldrexd          %1, %H1, [%3]\n"
 242 "       teq             %1, %4\n"
 243 "       teqeq           %H1, %H4\n"
 244 "       bne             2f\n"
 245 "       strexd          %0, %5, %H5, [%3]\n"
 246 "       teq             %0, #0\n"
 247 "       bne             1b\n"
 248 "2:"
 249         : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr)
 250         : "r" (ptr), "r" (old), "r" (new)
 251         : "cc");
 252
 253         return oldval;
 254 }
 255
 256 #define cmpxchg64_relaxed(ptr, o, n) ({                                 \
 257         (__typeof__(*(ptr)))__cmpxchg64((ptr),                          \
 258                                         (unsigned long long)(o),        \
 259                                         (unsigned long long)(n));       \
 260 })
 261
 262 #define cmpxchg64_local(ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n))
 263
 264 #endif  /* __LINUX_ARM_ARCH__ >= 6 */
 265
 266 #endif /* __ASM_ARM_CMPXCHG_H */