arch/arm/kernel/perf_event_v7.c

   1 /*
   2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
   3  *
   4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
   5  * 2010 (c) MontaVista Software, LLC.
   6  *
   7  * Copied from ARMv6 code, with the low level code inspired
   8  *  by the ARMv7 Oprofile code.
   9  *
  10  * Cortex-A8 has up to 4 configurable performance counters and
  11  *  a single cycle counter.
  12  * Cortex-A9 has up to 31 configurable performance counters and
  13  *  a single cycle counter.
  14  *
  15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
  16  *  counter and all 4 performance counters together can be reset separately.
  17  */
  18
  19 #ifdef CONFIG_CPU_V7
  20
  21 #include <asm/cp15.h>
  22 #include <asm/cputype.h>
  23 #include <asm/irq_regs.h>
  24 #include <asm/vfp.h>
  25 #include "../vfp/vfpinstr.h"
  26
  27 #include <linux/of.h>
  28 #include <linux/perf/arm_pmu.h>
  29 #include <linux/platform_device.h>
  30
  31 /*
  32  * Common ARMv7 event types
  33  *
  34  * Note: An implementation may not be able to count all of these events
  35  * but the encodings are considered to be `reserved' in the case that
  36  * they are not available.
  37  */
  38 enum armv7_perf_types {
  39         ARMV7_PERFCTR_PMNC_SW_INCR                      = 0x00,
  40         ARMV7_PERFCTR_L1_ICACHE_REFILL                  = 0x01,
  41         ARMV7_PERFCTR_ITLB_REFILL                       = 0x02,
  42         ARMV7_PERFCTR_L1_DCACHE_REFILL                  = 0x03,
  43         ARMV7_PERFCTR_L1_DCACHE_ACCESS                  = 0x04,
  44         ARMV7_PERFCTR_DTLB_REFILL                       = 0x05,
  45         ARMV7_PERFCTR_MEM_READ                          = 0x06,
  46         ARMV7_PERFCTR_MEM_WRITE                         = 0x07,
  47         ARMV7_PERFCTR_INSTR_EXECUTED                    = 0x08,
  48         ARMV7_PERFCTR_EXC_TAKEN                         = 0x09,
  49         ARMV7_PERFCTR_EXC_EXECUTED                      = 0x0A,
  50         ARMV7_PERFCTR_CID_WRITE                         = 0x0B,
  51
  52         /*
  53          * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  54          * It counts:
  55          *  - all (taken) branch instructions,
  56          *  - instructions that explicitly write the PC,
  57          *  - exception generating instructions.
  58          */
  59         ARMV7_PERFCTR_PC_WRITE                          = 0x0C,
  60         ARMV7_PERFCTR_PC_IMM_BRANCH                     = 0x0D,
  61         ARMV7_PERFCTR_PC_PROC_RETURN                    = 0x0E,
  62         ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS              = 0x0F,
  63         ARMV7_PERFCTR_PC_BRANCH_MIS_PRED                = 0x10,
  64         ARMV7_PERFCTR_CLOCK_CYCLES                      = 0x11,
  65         ARMV7_PERFCTR_PC_BRANCH_PRED                    = 0x12,
  66
  67         /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
  68         ARMV7_PERFCTR_MEM_ACCESS                        = 0x13,
  69         ARMV7_PERFCTR_L1_ICACHE_ACCESS                  = 0x14,
  70         ARMV7_PERFCTR_L1_DCACHE_WB                      = 0x15,
  71         ARMV7_PERFCTR_L2_CACHE_ACCESS                   = 0x16,
  72         ARMV7_PERFCTR_L2_CACHE_REFILL                   = 0x17,
  73         ARMV7_PERFCTR_L2_CACHE_WB                       = 0x18,
  74         ARMV7_PERFCTR_BUS_ACCESS                        = 0x19,
  75         ARMV7_PERFCTR_MEM_ERROR                         = 0x1A,
  76         ARMV7_PERFCTR_INSTR_SPEC                        = 0x1B,
  77         ARMV7_PERFCTR_TTBR_WRITE                        = 0x1C,
  78         ARMV7_PERFCTR_BUS_CYCLES                        = 0x1D,
  79
  80         ARMV7_PERFCTR_CPU_CYCLES                        = 0xFF
  81 };
  82
  83 /* ARMv7 Cortex-A8 specific event types */
  84 enum armv7_a8_perf_types {
  85         ARMV7_A8_PERFCTR_L2_CACHE_ACCESS                = 0x43,
  86         ARMV7_A8_PERFCTR_L2_CACHE_REFILL                = 0x44,
  87         ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS               = 0x50,
  88         ARMV7_A8_PERFCTR_STALL_ISIDE                    = 0x56,
  89 };
  90
  91 /* ARMv7 Cortex-A9 specific event types */
  92 enum armv7_a9_perf_types {
  93         ARMV7_A9_PERFCTR_INSTR_CORE_RENAME              = 0x68,
  94         ARMV7_A9_PERFCTR_STALL_ICACHE                   = 0x60,
  95         ARMV7_A9_PERFCTR_STALL_DISPATCH                 = 0x66,
  96 };
  97
  98 /* ARMv7 Cortex-A5 specific event types */
  99 enum armv7_a5_perf_types {
 100         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL              = 0xc2,
 101         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP         = 0xc3,
 102 };
 103
 104 /* ARMv7 Cortex-A15 specific event types */
 105 enum armv7_a15_perf_types {
 106         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ         = 0x40,
 107         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE        = 0x41,
 108         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ         = 0x42,
 109         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE        = 0x43,
 110
 111         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ           = 0x4C,
 112         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE          = 0x4D,
 113
 114         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ          = 0x50,
 115         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE         = 0x51,
 116         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ          = 0x52,
 117         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE         = 0x53,
 118
 119         ARMV7_A15_PERFCTR_PC_WRITE_SPEC                 = 0x76,
 120 };
 121
 122 /* ARMv7 Cortex-A12 specific event types */
 123 enum armv7_a12_perf_types {
 124         ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ         = 0x40,
 125         ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE        = 0x41,
 126
 127         ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ          = 0x50,
 128         ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE         = 0x51,
 129
 130         ARMV7_A12_PERFCTR_PC_WRITE_SPEC                 = 0x76,
 131
 132         ARMV7_A12_PERFCTR_PF_TLB_REFILL                 = 0xe7,
 133 };
 134
 135 /* ARMv7 Krait specific event types */
 136 enum krait_perf_types {
 137         KRAIT_PMRESR0_GROUP0                            = 0xcc,
 138         KRAIT_PMRESR1_GROUP0                            = 0xd0,
 139         KRAIT_PMRESR2_GROUP0                            = 0xd4,
 140         KRAIT_VPMRESR0_GROUP0                           = 0xd8,
 141
 142         KRAIT_PERFCTR_L1_ICACHE_ACCESS                  = 0x10011,
 143         KRAIT_PERFCTR_L1_ICACHE_MISS                    = 0x10010,
 144
 145         KRAIT_PERFCTR_L1_ITLB_ACCESS                    = 0x12222,
 146         KRAIT_PERFCTR_L1_DTLB_ACCESS                    = 0x12210,
 147 };
 148
 149 /* ARMv7 Scorpion specific event types */
 150 enum scorpion_perf_types {
 151         SCORPION_LPM0_GROUP0                            = 0x4c,
 152         SCORPION_LPM1_GROUP0                            = 0x50,
 153         SCORPION_LPM2_GROUP0                            = 0x54,
 154         SCORPION_L2LPM_GROUP0                           = 0x58,
 155         SCORPION_VLPM_GROUP0                            = 0x5c,
 156
 157         SCORPION_ICACHE_ACCESS                          = 0x10053,
 158         SCORPION_ICACHE_MISS                            = 0x10052,
 159
 160         SCORPION_DTLB_ACCESS                            = 0x12013,
 161         SCORPION_DTLB_MISS                              = 0x12012,
 162
 163         SCORPION_ITLB_MISS                              = 0x12021,
 164 };
 165
 166 /*
 167  * Cortex-A8 HW events mapping
 168  *
 169  * The hardware events that we support. We do support cache operations but
 170  * we have harvard caches and no way to combine instruction and data
 171  * accesses/misses in hardware.
 172  */
 173 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
 174         PERF_MAP_ALL_UNSUPPORTED,
 175         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 176         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 177         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 178         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 179         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 180         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 181         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
 182 };
 183
 184 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 185                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 186                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 187         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 188
 189         /*
 190          * The performance counters don't differentiate between read and write
 191          * accesses/misses so this isn't strictly correct, but it's the best we
 192          * can do. Writes and reads get combined.
 193          */
 194         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 195         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 196         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 197         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 198
 199         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
 200         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 201
 202         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 203         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 204         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 205         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 206
 207         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 208         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 209
 210         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 211         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 212
 213         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 214         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 215         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 216         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 217 };
 218
 219 /*
 220  * Cortex-A9 HW events mapping
 221  */
 222 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
 223         PERF_MAP_ALL_UNSUPPORTED,
 224         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 225         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
 226         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 227         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 228         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 229         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 230         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
 231         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV7_A9_PERFCTR_STALL_DISPATCH,
 232 };
 233
 234 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 235                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 236                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 237         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 238
 239         /*
 240          * The performance counters don't differentiate between read and write
 241          * accesses/misses so this isn't strictly correct, but it's the best we
 242          * can do. Writes and reads get combined.
 243          */
 244         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 245         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 246         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 247         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 248
 249         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 250
 251         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 252         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 253
 254         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 255         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 256
 257         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 258         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 259         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 260         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 261 };
 262
 263 /*
 264  * Cortex-A5 HW events mapping
 265  */
 266 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
 267         PERF_MAP_ALL_UNSUPPORTED,
 268         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 269         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 270         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 271         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 272         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 273         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 274 };
 275
 276 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 277                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 278                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 279         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 280
 281         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 282         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 283         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 284         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 285         [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 286         [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 287
 288         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 289         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 290         /*
 291          * The prefetch counters don't differentiate between the I side and the
 292          * D side.
 293          */
 294         [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 295         [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 296
 297         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 298         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 299
 300         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 301         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 302
 303         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 304         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 305         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 306         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 307 };
 308
 309 /*
 310  * Cortex-A15 HW events mapping
 311  */
 312 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
 313         PERF_MAP_ALL_UNSUPPORTED,
 314         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 315         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 316         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 317         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 318         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
 319         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 320         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 321 };
 322
 323 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 324                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 325                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 326         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 327
 328         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
 329         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
 330         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 331         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
 332
 333         /*
 334          * Not all performance counters differentiate between read and write
 335          * accesses/misses so we're not always strictly correct, but it's the
 336          * best we can do. Writes and reads get combined in these cases.
 337          */
 338         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 339         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 340
 341         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
 342         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
 343         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
 344         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
 345
 346         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
 347         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
 348
 349         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 350         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 351
 352         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 353         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 354         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 355         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 356 };
 357
 358 /*
 359  * Cortex-A7 HW events mapping
 360  */
 361 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
 362         PERF_MAP_ALL_UNSUPPORTED,
 363         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 364         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 365         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 366         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 367         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 368         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 369         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 370 };
 371
 372 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 373                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 374                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 375         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 376
 377         /*
 378          * The performance counters don't differentiate between read and write
 379          * accesses/misses so this isn't strictly correct, but it's the best we
 380          * can do. Writes and reads get combined.
 381          */
 382         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 383         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 384         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 385         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 386
 387         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 388         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 389
 390         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 391         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
 392         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 393         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
 394
 395         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 396         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 397
 398         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 399         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 400
 401         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 402         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 403         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 404         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 405 };
 406
 407 /*
 408  * Cortex-A12 HW events mapping
 409  */
 410 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
 411         PERF_MAP_ALL_UNSUPPORTED,
 412         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 413         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 414         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 415         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 416         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
 417         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 418         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 419 };
 420
 421 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 422                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 423                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 424         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 425
 426         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
 427         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 428         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 429         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 430
 431         /*
 432          * Not all performance counters differentiate between read and write
 433          * accesses/misses so we're not always strictly correct, but it's the
 434          * best we can do. Writes and reads get combined in these cases.
 435          */
 436         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 437         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 438
 439         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
 440         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
 441         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
 442         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
 443
 444         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 445         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 446         [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)]       = ARMV7_A12_PERFCTR_PF_TLB_REFILL,
 447
 448         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 449         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 450
 451         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 452         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 453         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 454         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 455 };
 456
 457 /*
 458  * Krait HW events mapping
 459  */
 460 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
 461         PERF_MAP_ALL_UNSUPPORTED,
 462         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 463         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 464         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 465         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 466         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 467 };
 468
 469 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
 470         PERF_MAP_ALL_UNSUPPORTED,
 471         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 472         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 473         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 474         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 475 };
 476
 477 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 478                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 479                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 480         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 481
 482         /*
 483          * The performance counters don't differentiate between read and write
 484          * accesses/misses so this isn't strictly correct, but it's the best we
 485          * can do. Writes and reads get combined.
 486          */
 487         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 488         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 489         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 490         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 491
 492         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = KRAIT_PERFCTR_L1_ICACHE_ACCESS,
 493         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = KRAIT_PERFCTR_L1_ICACHE_MISS,
 494
 495         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
 496         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_DTLB_ACCESS,
 497
 498         [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
 499         [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_ITLB_ACCESS,
 500
 501         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 502         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 503         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 504         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 505 };
 506
 507 /*
 508  * Scorpion HW events mapping
 509  */
 510 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
 511         PERF_MAP_ALL_UNSUPPORTED,
 512         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 513         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 514         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 515         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 516         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 517 };
 518
 519 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 520                                             [PERF_COUNT_HW_CACHE_OP_MAX]
 521                                             [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 522         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 523         /*
 524          * The performance counters don't differentiate between read and write
 525          * accesses/misses so this isn't strictly correct, but it's the best we
 526          * can do. Writes and reads get combined.
 527          */
 528         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 529         [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 530         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 531         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 532         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
 533         [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
 534         /*
 535          * Only ITLB misses and DTLB refills are supported.  If users want the
 536          * DTLB refills misses a raw counter must be used.
 537          */
 538         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 539         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 540         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 541         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 542         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 543         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 544         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 545         [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 546         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 547         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 548 };
 549
 550 /*
 551  * Perf Events' indices
 552  */
 553 #define ARMV7_IDX_CYCLE_COUNTER 0
 554 #define ARMV7_IDX_COUNTER0      1
 555 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
 556         (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 557
 558 #define ARMV7_MAX_COUNTERS      32
 559 #define ARMV7_COUNTER_MASK      (ARMV7_MAX_COUNTERS - 1)
 560
 561 /*
 562  * ARMv7 low level PMNC access
 563  */
 564
 565 /*
 566  * Perf Event to low level counters mapping
 567  */
 568 #define ARMV7_IDX_TO_COUNTER(x) \
 569         (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
 570
 571 /*
 572  * Per-CPU PMNC: config reg
 573  */
 574 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
 575 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
 576 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
 577 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
 578 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
 579 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
 580 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
 581 #define ARMV7_PMNC_N_MASK       0x1f
 582 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
 583
 584 /*
 585  * FLAG: counters overflow flag status reg
 586  */
 587 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
 588 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
 589
 590 /*
 591  * PMXEVTYPER: Event selection reg
 592  */
 593 #define ARMV7_EVTYPE_MASK       0xc80000ff      /* Mask for writable bits */
 594 #define ARMV7_EVTYPE_EVENT      0xff            /* Mask for EVENT bits */
 595
 596 /*
 597  * Event filters for PMUv2
 598  */
 599 #define ARMV7_EXCLUDE_PL1       (1 << 31)
 600 #define ARMV7_EXCLUDE_USER      (1 << 30)
 601 #define ARMV7_INCLUDE_HYP       (1 << 27)
 602
 603 static inline u32 armv7_pmnc_read(void)
 604 {
 605         u32 val;
 606         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
 607         return val;
 608 }
 609
 610 static inline void armv7_pmnc_write(u32 val)
 611 {
 612         val &= ARMV7_PMNC_MASK;
 613         isb();
 614         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
 615 }
 616
 617 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
 618 {
 619         return pmnc & ARMV7_OVERFLOWED_MASK;
 620 }
 621
 622 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
 623 {
 624         return idx >= ARMV7_IDX_CYCLE_COUNTER &&
 625                 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
 626 }
 627
 628 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
 629 {
 630         return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
 631 }
 632
 633 static inline void armv7_pmnc_select_counter(int idx)
 634 {
 635         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 636         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
 637         isb();
 638 }
 639
 640 static inline u32 armv7pmu_read_counter(struct perf_event *event)
 641 {
 642         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 643         struct hw_perf_event *hwc = &event->hw;
 644         int idx = hwc->idx;
 645         u32 value = 0;
 646
 647         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 648                 pr_err("CPU%u reading wrong counter %d\n",
 649                         smp_processor_id(), idx);
 650         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 651                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
 652         } else {
 653                 armv7_pmnc_select_counter(idx);
 654                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
 655         }
 656
 657         return value;
 658 }
 659
 660 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
 661 {
 662         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 663         struct hw_perf_event *hwc = &event->hw;
 664         int idx = hwc->idx;
 665
 666         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 667                 pr_err("CPU%u writing wrong counter %d\n",
 668                         smp_processor_id(), idx);
 669         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 670                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
 671         } else {
 672                 armv7_pmnc_select_counter(idx);
 673                 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
 674         }
 675 }
 676
 677 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
 678 {
 679         armv7_pmnc_select_counter(idx);
 680         val &= ARMV7_EVTYPE_MASK;
 681         asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
 682 }
 683
 684 static inline void armv7_pmnc_enable_counter(int idx)
 685 {
 686         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 687         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
 688 }
 689
 690 static inline void armv7_pmnc_disable_counter(int idx)
 691 {
 692         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 693         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
 694 }
 695
 696 static inline void armv7_pmnc_enable_intens(int idx)
 697 {
 698         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 699         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
 700 }
 701
 702 static inline void armv7_pmnc_disable_intens(int idx)
 703 {
 704         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 705         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
 706         isb();
 707         /* Clear the overflow flag in case an interrupt is pending. */
 708         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
 709         isb();
 710 }
 711
 712 static inline u32 armv7_pmnc_getreset_flags(void)
 713 {
 714         u32 val;
 715
 716         /* Read */
 717         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 718
 719         /* Write to clear flags */
 720         val &= ARMV7_FLAG_MASK;
 721         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
 722
 723         return val;
 724 }
 725
 726 #ifdef DEBUG
 727 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
 728 {
 729         u32 val;
 730         unsigned int cnt;
 731
 732         pr_info("PMNC registers dump:\n");
 733
 734         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
 735         pr_info("PMNC  =0x%08x\n", val);
 736
 737         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
 738         pr_info("CNTENS=0x%08x\n", val);
 739
 740         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
 741         pr_info("INTENS=0x%08x\n", val);
 742
 743         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 744         pr_info("FLAGS =0x%08x\n", val);
 745
 746         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
 747         pr_info("SELECT=0x%08x\n", val);
 748
 749         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
 750         pr_info("CCNT  =0x%08x\n", val);
 751
 752         for (cnt = ARMV7_IDX_COUNTER0;
 753                         cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
 754                 armv7_pmnc_select_counter(cnt);
 755                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
 756                 pr_info("CNT[%d] count =0x%08x\n",
 757                         ARMV7_IDX_TO_COUNTER(cnt), val);
 758                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
 759                 pr_info("CNT[%d] evtsel=0x%08x\n",
 760                         ARMV7_IDX_TO_COUNTER(cnt), val);
 761         }
 762 }
 763 #endif
 764
 765 static void armv7pmu_enable_event(struct perf_event *event)
 766 {
 767         unsigned long flags;
 768         struct hw_perf_event *hwc = &event->hw;
 769         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 770         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 771         int idx = hwc->idx;
 772
 773         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 774                 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
 775                         smp_processor_id(), idx);
 776                 return;
 777         }
 778
 779         /*
 780          * Enable counter and interrupt, and set the counter to count
 781          * the event that we're interested in.
 782          */
 783         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 784
 785         /*
 786          * Disable counter
 787          */
 788         armv7_pmnc_disable_counter(idx);
 789
 790         /*
 791          * Set event (if destined for PMNx counters)
 792          * We only need to set the event for the cycle counter if we
 793          * have the ability to perform event filtering.
 794          */
 795         if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
 796                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
 797
 798         /*
 799          * Enable interrupt for this counter
 800          */
 801         armv7_pmnc_enable_intens(idx);
 802
 803         /*
 804          * Enable counter
 805          */
 806         armv7_pmnc_enable_counter(idx);
 807
 808         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 809 }
 810
 811 static void armv7pmu_disable_event(struct perf_event *event)
 812 {
 813         unsigned long flags;
 814         struct hw_perf_event *hwc = &event->hw;
 815         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 816         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 817         int idx = hwc->idx;
 818
 819         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 820                 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
 821                         smp_processor_id(), idx);
 822                 return;
 823         }
 824
 825         /*
 826          * Disable counter and interrupt
 827          */
 828         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 829
 830         /*
 831          * Disable counter
 832          */
 833         armv7_pmnc_disable_counter(idx);
 834
 835         /*
 836          * Disable interrupt for this counter
 837          */
 838         armv7_pmnc_disable_intens(idx);
 839
 840         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 841 }
 842
 843 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
 844 {
 845         u32 pmnc;
 846         struct perf_sample_data data;
 847         struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 848         struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 849         struct pt_regs *regs;
 850         int idx;
 851
 852         /*
 853          * Get and reset the IRQ flags
 854          */
 855         pmnc = armv7_pmnc_getreset_flags();
 856
 857         /*
 858          * Did an overflow occur?
 859          */
 860         if (!armv7_pmnc_has_overflowed(pmnc))
 861                 return IRQ_NONE;
 862
 863         /*
 864          * Handle the counter(s) overflow(s)
 865          */
 866         regs = get_irq_regs();
 867
 868         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 869                 struct perf_event *event = cpuc->events[idx];
 870                 struct hw_perf_event *hwc;
 871
 872                 /* Ignore if we don't have an event. */
 873                 if (!event)
 874                         continue;
 875
 876                 /*
 877                  * We have a single interrupt for all counters. Check that
 878                  * each counter has overflowed before we process it.
 879                  */
 880                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
 881                         continue;
 882
 883                 hwc = &event->hw;
 884                 armpmu_event_update(event);
 885                 perf_sample_data_init(&data, 0, hwc->last_period);
 886                 if (!armpmu_event_set_period(event))
 887                         continue;
 888
 889                 if (perf_event_overflow(event, &data, regs))
 890                         cpu_pmu->disable(event);
 891         }
 892
 893         /*
 894          * Handle the pending perf events.
 895          *
 896          * Note: this call *must* be run with interrupts disabled. For
 897          * platforms that can have the PMU interrupts raised as an NMI, this
 898          * will not work.
 899          */
 900         irq_work_run();
 901
 902         return IRQ_HANDLED;
 903 }
 904
 905 static void armv7pmu_start(struct arm_pmu *cpu_pmu)
 906 {
 907         unsigned long flags;
 908         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 909
 910         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 911         /* Enable all counters */
 912         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
 913         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 914 }
 915
 916 static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
 917 {
 918         unsigned long flags;
 919         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 920
 921         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 922         /* Disable all counters */
 923         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
 924         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 925 }
 926
 927 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
 928                                   struct perf_event *event)
 929 {
 930         int idx;
 931         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 932         struct hw_perf_event *hwc = &event->hw;
 933         unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
 934
 935         /* Always place a cycle counter into the cycle counter. */
 936         if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
 937                 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
 938                         return -EAGAIN;
 939
 940                 return ARMV7_IDX_CYCLE_COUNTER;
 941         }
 942
 943         /*
 944          * For anything other than a cycle counter, try and use
 945          * the events counters
 946          */
 947         for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
 948                 if (!test_and_set_bit(idx, cpuc->used_mask))
 949                         return idx;
 950         }
 951
 952         /* The counters are all in use. */
 953         return -EAGAIN;
 954 }
 955
 956 /*
 957  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
 958  */
 959 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
 960                                      struct perf_event_attr *attr)
 961 {
 962         unsigned long config_base = 0;
 963
 964         if (attr->exclude_idle)
 965                 return -EPERM;
 966         if (attr->exclude_user)
 967                 config_base |= ARMV7_EXCLUDE_USER;
 968         if (attr->exclude_kernel)
 969                 config_base |= ARMV7_EXCLUDE_PL1;
 970         if (!attr->exclude_hv)
 971                 config_base |= ARMV7_INCLUDE_HYP;
 972
 973         /*
 974          * Install the filter into config_base as this is used to
 975          * construct the event type.
 976          */
 977         event->config_base = config_base;
 978
 979         return 0;
 980 }
 981
 982 static void armv7pmu_reset(void *info)
 983 {
 984         struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
 985         u32 idx, nb_cnt = cpu_pmu->num_events;
 986
 987         /* The counter and interrupt enable registers are unknown at reset. */
 988         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
 989                 armv7_pmnc_disable_counter(idx);
 990                 armv7_pmnc_disable_intens(idx);
 991         }
 992
 993         /* Initialize & Reset PMNC: C and P bits */
 994         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
 995 }
 996
 997 static int armv7_a8_map_event(struct perf_event *event)
 998 {
 999         return armpmu_map_event(event, &armv7_a8_perf_map,
1000                                 &armv7_a8_perf_cache_map, 0xFF);
1001 }
1002
1003 static int armv7_a9_map_event(struct perf_event *event)
1004 {
1005         return armpmu_map_event(event, &armv7_a9_perf_map,
1006                                 &armv7_a9_perf_cache_map, 0xFF);
1007 }
1008
1009 static int armv7_a5_map_event(struct perf_event *event)
1010 {
1011         return armpmu_map_event(event, &armv7_a5_perf_map,
1012                                 &armv7_a5_perf_cache_map, 0xFF);
1013 }
1014
1015 static int armv7_a15_map_event(struct perf_event *event)
1016 {
1017         return armpmu_map_event(event, &armv7_a15_perf_map,
1018                                 &armv7_a15_perf_cache_map, 0xFF);
1019 }
1020
1021 static int armv7_a7_map_event(struct perf_event *event)
1022 {
1023         return armpmu_map_event(event, &armv7_a7_perf_map,
1024                                 &armv7_a7_perf_cache_map, 0xFF);
1025 }
1026
1027 static int armv7_a12_map_event(struct perf_event *event)
1028 {
1029         return armpmu_map_event(event, &armv7_a12_perf_map,
1030                                 &armv7_a12_perf_cache_map, 0xFF);
1031 }
1032
1033 static int krait_map_event(struct perf_event *event)
1034 {
1035         return armpmu_map_event(event, &krait_perf_map,
1036                                 &krait_perf_cache_map, 0xFFFFF);
1037 }
1038
1039 static int krait_map_event_no_branch(struct perf_event *event)
1040 {
1041         return armpmu_map_event(event, &krait_perf_map_no_branch,
1042                                 &krait_perf_cache_map, 0xFFFFF);
1043 }
1044
1045 static int scorpion_map_event(struct perf_event *event)
1046 {
1047         return armpmu_map_event(event, &scorpion_perf_map,
1048                                 &scorpion_perf_cache_map, 0xFFFFF);
1049 }
1050
1051 static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1052 {
1053         cpu_pmu->handle_irq     = armv7pmu_handle_irq;
1054         cpu_pmu->enable         = armv7pmu_enable_event;
1055         cpu_pmu->disable        = armv7pmu_disable_event;
1056         cpu_pmu->read_counter   = armv7pmu_read_counter;
1057         cpu_pmu->write_counter  = armv7pmu_write_counter;
1058         cpu_pmu->get_event_idx  = armv7pmu_get_event_idx;
1059         cpu_pmu->start          = armv7pmu_start;
1060         cpu_pmu->stop           = armv7pmu_stop;
1061         cpu_pmu->reset          = armv7pmu_reset;
1062         cpu_pmu->max_period     = (1LLU << 32) - 1;
1063 };
1064
1065 static void armv7_read_num_pmnc_events(void *info)
1066 {
1067         int *nb_cnt = info;
1068
1069         /* Read the nb of CNTx counters supported from PMNC */
1070         *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1071
1072         /* Add the CPU cycles counter */
1073         *nb_cnt += 1;
1074 }
1075
1076 static int armv7_probe_num_events(struct arm_pmu *arm_pmu)
1077 {
1078         return smp_call_function_any(&arm_pmu->supported_cpus,
1079                                      armv7_read_num_pmnc_events,
1080                                      &arm_pmu->num_events, 1);
1081 }
1082
1083 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1084 {
1085         armv7pmu_init(cpu_pmu);
1086         cpu_pmu->name           = "armv7_cortex_a8";
1087         cpu_pmu->map_event      = armv7_a8_map_event;
1088         return armv7_probe_num_events(cpu_pmu);
1089 }
1090
1091 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1092 {
1093         armv7pmu_init(cpu_pmu);
1094         cpu_pmu->name           = "armv7_cortex_a9";
1095         cpu_pmu->map_event      = armv7_a9_map_event;
1096         return armv7_probe_num_events(cpu_pmu);
1097 }
1098
1099 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1100 {
1101         armv7pmu_init(cpu_pmu);
1102         cpu_pmu->name           = "armv7_cortex_a5";
1103         cpu_pmu->map_event      = armv7_a5_map_event;
1104         return armv7_probe_num_events(cpu_pmu);
1105 }
1106
1107 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1108 {
1109         armv7pmu_init(cpu_pmu);
1110         cpu_pmu->name           = "armv7_cortex_a15";
1111         cpu_pmu->map_event      = armv7_a15_map_event;
1112         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1113         return armv7_probe_num_events(cpu_pmu);
1114 }
1115
1116 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1117 {
1118         armv7pmu_init(cpu_pmu);
1119         cpu_pmu->name           = "armv7_cortex_a7";
1120         cpu_pmu->map_event      = armv7_a7_map_event;
1121         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1122         return armv7_probe_num_events(cpu_pmu);
1123 }
1124
1125 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1126 {
1127         armv7pmu_init(cpu_pmu);
1128         cpu_pmu->name           = "armv7_cortex_a12";
1129         cpu_pmu->map_event      = armv7_a12_map_event;
1130         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1131         return armv7_probe_num_events(cpu_pmu);
1132 }
1133
1134 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1135 {
1136         int ret = armv7_a12_pmu_init(cpu_pmu);
1137         cpu_pmu->name = "armv7_cortex_a17";
1138         return ret;
1139 }
1140
1141 /*
1142  * Krait Performance Monitor Region Event Selection Register (PMRESRn)
1143  *
1144  *            31   30     24     16     8      0
1145  *            +--------------------------------+
1146  *  PMRESR0   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1147  *            +--------------------------------+
1148  *  PMRESR1   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1149  *            +--------------------------------+
1150  *  PMRESR2   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1151  *            +--------------------------------+
1152  *  VPMRESR0  | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1153  *            +--------------------------------+
1154  *              EN | G=3  | G=2  | G=1  | G=0
1155  *
1156  *  Event Encoding:
1157  *
1158  *      hwc->config_base = 0xNRCCG
1159  *
1160  *      N  = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
1161  *      R  = region register
1162  *      CC = class of events the group G is choosing from
1163  *      G  = group or particular event
1164  *
1165  *  Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
1166  *
1167  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1168  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1169  *  events (interrupts for example). An event code is broken down into
1170  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1171  *  example).
1172  */
1173
1174 #define KRAIT_EVENT             (1 << 16)
1175 #define VENUM_EVENT             (2 << 16)
1176 #define KRAIT_EVENT_MASK        (KRAIT_EVENT | VENUM_EVENT)
1177 #define PMRESRn_EN              BIT(31)
1178
1179 #define EVENT_REGION(event)     (((event) >> 12) & 0xf)         /* R */
1180 #define EVENT_GROUP(event)      ((event) & 0xf)                 /* G */
1181 #define EVENT_CODE(event)       (((event) >> 4) & 0xff)         /* CC */
1182 #define EVENT_VENUM(event)      (!!(event & VENUM_EVENT))       /* N=2 */
1183 #define EVENT_CPU(event)        (!!(event & KRAIT_EVENT))       /* N=1 */
1184
1185 static u32 krait_read_pmresrn(int n)
1186 {
1187         u32 val;
1188
1189         switch (n) {
1190         case 0:
1191                 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1192                 break;
1193         case 1:
1194                 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1195                 break;
1196         case 2:
1197                 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1198                 break;
1199         default:
1200                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1201         }
1202
1203         return val;
1204 }
1205
1206 static void krait_write_pmresrn(int n, u32 val)
1207 {
1208         switch (n) {
1209         case 0:
1210                 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1211                 break;
1212         case 1:
1213                 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1214                 break;
1215         case 2:
1216                 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1217                 break;
1218         default:
1219                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1220         }
1221 }
1222
1223 static u32 venum_read_pmresr(void)
1224 {
1225         u32 val;
1226         asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1227         return val;
1228 }
1229
1230 static void venum_write_pmresr(u32 val)
1231 {
1232         asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1233 }
1234
1235 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1236 {
1237         u32 venum_new_val;
1238         u32 fp_new_val;
1239
1240         BUG_ON(preemptible());
1241         /* CPACR Enable CP10 and CP11 access */
1242         *venum_orig_val = get_copro_access();
1243         venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1244         set_copro_access(venum_new_val);
1245
1246         /* Enable FPEXC */
1247         *fp_orig_val = fmrx(FPEXC);
1248         fp_new_val = *fp_orig_val | FPEXC_EN;
1249         fmxr(FPEXC, fp_new_val);
1250 }
1251
1252 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1253 {
1254         BUG_ON(preemptible());
1255         /* Restore FPEXC */
1256         fmxr(FPEXC, fp_orig_val);
1257         isb();
1258         /* Restore CPACR */
1259         set_copro_access(venum_orig_val);
1260 }
1261
1262 static u32 krait_get_pmresrn_event(unsigned int region)
1263 {
1264         static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1265                                              KRAIT_PMRESR1_GROUP0,
1266                                              KRAIT_PMRESR2_GROUP0 };
1267         return pmresrn_table[region];
1268 }
1269
1270 static void krait_evt_setup(int idx, u32 config_base)
1271 {
1272         u32 val;
1273         u32 mask;
1274         u32 vval, fval;
1275         unsigned int region = EVENT_REGION(config_base);
1276         unsigned int group = EVENT_GROUP(config_base);
1277         unsigned int code = EVENT_CODE(config_base);
1278         unsigned int group_shift;
1279         bool venum_event = EVENT_VENUM(config_base);
1280
1281         group_shift = group * 8;
1282         mask = 0xff << group_shift;
1283
1284         /* Configure evtsel for the region and group */
1285         if (venum_event)
1286                 val = KRAIT_VPMRESR0_GROUP0;
1287         else
1288                 val = krait_get_pmresrn_event(region);
1289         val += group;
1290         /* Mix in mode-exclusion bits */
1291         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1292         armv7_pmnc_write_evtsel(idx, val);
1293
1294         if (venum_event) {
1295                 venum_pre_pmresr(&vval, &fval);
1296                 val = venum_read_pmresr();
1297                 val &= ~mask;
1298                 val |= code << group_shift;
1299                 val |= PMRESRn_EN;
1300                 venum_write_pmresr(val);
1301                 venum_post_pmresr(vval, fval);
1302         } else {
1303                 val = krait_read_pmresrn(region);
1304                 val &= ~mask;
1305                 val |= code << group_shift;
1306                 val |= PMRESRn_EN;
1307                 krait_write_pmresrn(region, val);
1308         }
1309 }
1310
1311 static u32 clear_pmresrn_group(u32 val, int group)
1312 {
1313         u32 mask;
1314         int group_shift;
1315
1316         group_shift = group * 8;
1317         mask = 0xff << group_shift;
1318         val &= ~mask;
1319
1320         /* Don't clear enable bit if entire region isn't disabled */
1321         if (val & ~PMRESRn_EN)
1322                 return val |= PMRESRn_EN;
1323
1324         return 0;
1325 }
1326
1327 static void krait_clearpmu(u32 config_base)
1328 {
1329         u32 val;
1330         u32 vval, fval;
1331         unsigned int region = EVENT_REGION(config_base);
1332         unsigned int group = EVENT_GROUP(config_base);
1333         bool venum_event = EVENT_VENUM(config_base);
1334
1335         if (venum_event) {
1336                 venum_pre_pmresr(&vval, &fval);
1337                 val = venum_read_pmresr();
1338                 val = clear_pmresrn_group(val, group);
1339                 venum_write_pmresr(val);
1340                 venum_post_pmresr(vval, fval);
1341         } else {
1342                 val = krait_read_pmresrn(region);
1343                 val = clear_pmresrn_group(val, group);
1344                 krait_write_pmresrn(region, val);
1345         }
1346 }
1347
1348 static void krait_pmu_disable_event(struct perf_event *event)
1349 {
1350         unsigned long flags;
1351         struct hw_perf_event *hwc = &event->hw;
1352         int idx = hwc->idx;
1353         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1354         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1355
1356         /* Disable counter and interrupt */
1357         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1358
1359         /* Disable counter */
1360         armv7_pmnc_disable_counter(idx);
1361
1362         /*
1363          * Clear pmresr code (if destined for PMNx counters)
1364          */
1365         if (hwc->config_base & KRAIT_EVENT_MASK)
1366                 krait_clearpmu(hwc->config_base);
1367
1368         /* Disable interrupt for this counter */
1369         armv7_pmnc_disable_intens(idx);
1370
1371         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1372 }
1373
1374 static void krait_pmu_enable_event(struct perf_event *event)
1375 {
1376         unsigned long flags;
1377         struct hw_perf_event *hwc = &event->hw;
1378         int idx = hwc->idx;
1379         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1380         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1381
1382         /*
1383          * Enable counter and interrupt, and set the counter to count
1384          * the event that we're interested in.
1385          */
1386         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1387
1388         /* Disable counter */
1389         armv7_pmnc_disable_counter(idx);
1390
1391         /*
1392          * Set event (if destined for PMNx counters)
1393          * We set the event for the cycle counter because we
1394          * have the ability to perform event filtering.
1395          */
1396         if (hwc->config_base & KRAIT_EVENT_MASK)
1397                 krait_evt_setup(idx, hwc->config_base);
1398         else
1399                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1400
1401         /* Enable interrupt for this counter */
1402         armv7_pmnc_enable_intens(idx);
1403
1404         /* Enable counter */
1405         armv7_pmnc_enable_counter(idx);
1406
1407         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1408 }
1409
1410 static void krait_pmu_reset(void *info)
1411 {
1412         u32 vval, fval;
1413         struct arm_pmu *cpu_pmu = info;
1414         u32 idx, nb_cnt = cpu_pmu->num_events;
1415
1416         armv7pmu_reset(info);
1417
1418         /* Clear all pmresrs */
1419         krait_write_pmresrn(0, 0);
1420         krait_write_pmresrn(1, 0);
1421         krait_write_pmresrn(2, 0);
1422
1423         venum_pre_pmresr(&vval, &fval);
1424         venum_write_pmresr(0);
1425         venum_post_pmresr(vval, fval);
1426
1427         /* Reset PMxEVNCTCR to sane default */
1428         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1429                 armv7_pmnc_select_counter(idx);
1430                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1431         }
1432
1433 }
1434
1435 static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1436                               unsigned int group)
1437 {
1438         int bit;
1439         struct hw_perf_event *hwc = &event->hw;
1440         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1441
1442         if (hwc->config_base & VENUM_EVENT)
1443                 bit = KRAIT_VPMRESR0_GROUP0;
1444         else
1445                 bit = krait_get_pmresrn_event(region);
1446         bit -= krait_get_pmresrn_event(0);
1447         bit += group;
1448         /*
1449          * Lower bits are reserved for use by the counters (see
1450          * armv7pmu_get_event_idx() for more info)
1451          */
1452         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1453
1454         return bit;
1455 }
1456
1457 /*
1458  * We check for column exclusion constraints here.
1459  * Two events cant use the same group within a pmresr register.
1460  */
1461 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1462                                    struct perf_event *event)
1463 {
1464         int idx;
1465         int bit = -1;
1466         struct hw_perf_event *hwc = &event->hw;
1467         unsigned int region = EVENT_REGION(hwc->config_base);
1468         unsigned int code = EVENT_CODE(hwc->config_base);
1469         unsigned int group = EVENT_GROUP(hwc->config_base);
1470         bool venum_event = EVENT_VENUM(hwc->config_base);
1471         bool krait_event = EVENT_CPU(hwc->config_base);
1472
1473         if (venum_event || krait_event) {
1474                 /* Ignore invalid events */
1475                 if (group > 3 || region > 2)
1476                         return -EINVAL;
1477                 if (venum_event && (code & 0xe0))
1478                         return -EINVAL;
1479
1480                 bit = krait_event_to_bit(event, region, group);
1481                 if (test_and_set_bit(bit, cpuc->used_mask))
1482                         return -EAGAIN;
1483         }
1484
1485         idx = armv7pmu_get_event_idx(cpuc, event);
1486         if (idx < 0 && bit >= 0)
1487                 clear_bit(bit, cpuc->used_mask);
1488
1489         return idx;
1490 }
1491
1492 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1493                                       struct perf_event *event)
1494 {
1495         int bit;
1496         struct hw_perf_event *hwc = &event->hw;
1497         unsigned int region = EVENT_REGION(hwc->config_base);
1498         unsigned int group = EVENT_GROUP(hwc->config_base);
1499         bool venum_event = EVENT_VENUM(hwc->config_base);
1500         bool krait_event = EVENT_CPU(hwc->config_base);
1501
1502         if (venum_event || krait_event) {
1503                 bit = krait_event_to_bit(event, region, group);
1504                 clear_bit(bit, cpuc->used_mask);
1505         }
1506 }
1507
1508 static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1509 {
1510         armv7pmu_init(cpu_pmu);
1511         cpu_pmu->name           = "armv7_krait";
1512         /* Some early versions of Krait don't support PC write events */
1513         if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1514                                   "qcom,no-pc-write"))
1515                 cpu_pmu->map_event = krait_map_event_no_branch;
1516         else
1517                 cpu_pmu->map_event = krait_map_event;
1518         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1519         cpu_pmu->reset          = krait_pmu_reset;
1520         cpu_pmu->enable         = krait_pmu_enable_event;
1521         cpu_pmu->disable        = krait_pmu_disable_event;
1522         cpu_pmu->get_event_idx  = krait_pmu_get_event_idx;
1523         cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1524         return armv7_probe_num_events(cpu_pmu);
1525 }
1526
1527 /*
1528  * Scorpion Local Performance Monitor Register (LPMn)
1529  *
1530  *            31   30     24     16     8      0
1531  *            +--------------------------------+
1532  *  LPM0      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1533  *            +--------------------------------+
1534  *  LPM1      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1535  *            +--------------------------------+
1536  *  LPM2      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1537  *            +--------------------------------+
1538  *  L2LPM     | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 3
1539  *            +--------------------------------+
1540  *  VLPM      | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1541  *            +--------------------------------+
1542  *              EN | G=3  | G=2  | G=1  | G=0
1543  *
1544  *
1545  *  Event Encoding:
1546  *
1547  *      hwc->config_base = 0xNRCCG
1548  *
1549  *      N  = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1550  *      R  = region register
1551  *      CC = class of events the group G is choosing from
1552  *      G  = group or particular event
1553  *
1554  *  Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1555  *
1556  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1557  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1558  *  events (interrupts for example). An event code is broken down into
1559  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1560  *  example).
1561  */
1562
1563 static u32 scorpion_read_pmresrn(int n)
1564 {
1565         u32 val;
1566
1567         switch (n) {
1568         case 0:
1569                 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1570                 break;
1571         case 1:
1572                 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1573                 break;
1574         case 2:
1575                 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1576                 break;
1577         case 3:
1578                 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1579                 break;
1580         default:
1581                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1582         }
1583
1584         return val;
1585 }
1586
1587 static void scorpion_write_pmresrn(int n, u32 val)
1588 {
1589         switch (n) {
1590         case 0:
1591                 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1592                 break;
1593         case 1:
1594                 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1595                 break;
1596         case 2:
1597                 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1598                 break;
1599         case 3:
1600                 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1601                 break;
1602         default:
1603                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1604         }
1605 }
1606
1607 static u32 scorpion_get_pmresrn_event(unsigned int region)
1608 {
1609         static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1610                                              SCORPION_LPM1_GROUP0,
1611                                              SCORPION_LPM2_GROUP0,
1612                                              SCORPION_L2LPM_GROUP0 };
1613         return pmresrn_table[region];
1614 }
1615
1616 static void scorpion_evt_setup(int idx, u32 config_base)
1617 {
1618         u32 val;
1619         u32 mask;
1620         u32 vval, fval;
1621         unsigned int region = EVENT_REGION(config_base);
1622         unsigned int group = EVENT_GROUP(config_base);
1623         unsigned int code = EVENT_CODE(config_base);
1624         unsigned int group_shift;
1625         bool venum_event = EVENT_VENUM(config_base);
1626
1627         group_shift = group * 8;
1628         mask = 0xff << group_shift;
1629
1630         /* Configure evtsel for the region and group */
1631         if (venum_event)
1632                 val = SCORPION_VLPM_GROUP0;
1633         else
1634                 val = scorpion_get_pmresrn_event(region);
1635         val += group;
1636         /* Mix in mode-exclusion bits */
1637         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1638         armv7_pmnc_write_evtsel(idx, val);
1639
1640         asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1641
1642         if (venum_event) {
1643                 venum_pre_pmresr(&vval, &fval);
1644                 val = venum_read_pmresr();
1645                 val &= ~mask;
1646                 val |= code << group_shift;
1647                 val |= PMRESRn_EN;
1648                 venum_write_pmresr(val);
1649                 venum_post_pmresr(vval, fval);
1650         } else {
1651                 val = scorpion_read_pmresrn(region);
1652                 val &= ~mask;
1653                 val |= code << group_shift;
1654                 val |= PMRESRn_EN;
1655                 scorpion_write_pmresrn(region, val);
1656         }
1657 }
1658
1659 static void scorpion_clearpmu(u32 config_base)
1660 {
1661         u32 val;
1662         u32 vval, fval;
1663         unsigned int region = EVENT_REGION(config_base);
1664         unsigned int group = EVENT_GROUP(config_base);
1665         bool venum_event = EVENT_VENUM(config_base);
1666
1667         if (venum_event) {
1668                 venum_pre_pmresr(&vval, &fval);
1669                 val = venum_read_pmresr();
1670                 val = clear_pmresrn_group(val, group);
1671                 venum_write_pmresr(val);
1672                 venum_post_pmresr(vval, fval);
1673         } else {
1674                 val = scorpion_read_pmresrn(region);
1675                 val = clear_pmresrn_group(val, group);
1676                 scorpion_write_pmresrn(region, val);
1677         }
1678 }
1679
1680 static void scorpion_pmu_disable_event(struct perf_event *event)
1681 {
1682         unsigned long flags;
1683         struct hw_perf_event *hwc = &event->hw;
1684         int idx = hwc->idx;
1685         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1686         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1687
1688         /* Disable counter and interrupt */
1689         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1690
1691         /* Disable counter */
1692         armv7_pmnc_disable_counter(idx);
1693
1694         /*
1695          * Clear pmresr code (if destined for PMNx counters)
1696          */
1697         if (hwc->config_base & KRAIT_EVENT_MASK)
1698                 scorpion_clearpmu(hwc->config_base);
1699
1700         /* Disable interrupt for this counter */
1701         armv7_pmnc_disable_intens(idx);
1702
1703         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1704 }
1705
1706 static void scorpion_pmu_enable_event(struct perf_event *event)
1707 {
1708         unsigned long flags;
1709         struct hw_perf_event *hwc = &event->hw;
1710         int idx = hwc->idx;
1711         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1712         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1713
1714         /*
1715          * Enable counter and interrupt, and set the counter to count
1716          * the event that we're interested in.
1717          */
1718         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1719
1720         /* Disable counter */
1721         armv7_pmnc_disable_counter(idx);
1722
1723         /*
1724          * Set event (if destined for PMNx counters)
1725          * We don't set the event for the cycle counter because we
1726          * don't have the ability to perform event filtering.
1727          */
1728         if (hwc->config_base & KRAIT_EVENT_MASK)
1729                 scorpion_evt_setup(idx, hwc->config_base);
1730         else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1731                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1732
1733         /* Enable interrupt for this counter */
1734         armv7_pmnc_enable_intens(idx);
1735
1736         /* Enable counter */
1737         armv7_pmnc_enable_counter(idx);
1738
1739         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1740 }
1741
1742 static void scorpion_pmu_reset(void *info)
1743 {
1744         u32 vval, fval;
1745         struct arm_pmu *cpu_pmu = info;
1746         u32 idx, nb_cnt = cpu_pmu->num_events;
1747
1748         armv7pmu_reset(info);
1749
1750         /* Clear all pmresrs */
1751         scorpion_write_pmresrn(0, 0);
1752         scorpion_write_pmresrn(1, 0);
1753         scorpion_write_pmresrn(2, 0);
1754         scorpion_write_pmresrn(3, 0);
1755
1756         venum_pre_pmresr(&vval, &fval);
1757         venum_write_pmresr(0);
1758         venum_post_pmresr(vval, fval);
1759
1760         /* Reset PMxEVNCTCR to sane default */
1761         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1762                 armv7_pmnc_select_counter(idx);
1763                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1764         }
1765 }
1766
1767 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1768                               unsigned int group)
1769 {
1770         int bit;
1771         struct hw_perf_event *hwc = &event->hw;
1772         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1773
1774         if (hwc->config_base & VENUM_EVENT)
1775                 bit = SCORPION_VLPM_GROUP0;
1776         else
1777                 bit = scorpion_get_pmresrn_event(region);
1778         bit -= scorpion_get_pmresrn_event(0);
1779         bit += group;
1780         /*
1781          * Lower bits are reserved for use by the counters (see
1782          * armv7pmu_get_event_idx() for more info)
1783          */
1784         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1785
1786         return bit;
1787 }
1788
1789 /*
1790  * We check for column exclusion constraints here.
1791  * Two events cant use the same group within a pmresr register.
1792  */
1793 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1794                                    struct perf_event *event)
1795 {
1796         int idx;
1797         int bit = -1;
1798         struct hw_perf_event *hwc = &event->hw;
1799         unsigned int region = EVENT_REGION(hwc->config_base);
1800         unsigned int group = EVENT_GROUP(hwc->config_base);
1801         bool venum_event = EVENT_VENUM(hwc->config_base);
1802         bool scorpion_event = EVENT_CPU(hwc->config_base);
1803
1804         if (venum_event || scorpion_event) {
1805                 /* Ignore invalid events */
1806                 if (group > 3 || region > 3)
1807                         return -EINVAL;
1808
1809                 bit = scorpion_event_to_bit(event, region, group);
1810                 if (test_and_set_bit(bit, cpuc->used_mask))
1811                         return -EAGAIN;
1812         }
1813
1814         idx = armv7pmu_get_event_idx(cpuc, event);
1815         if (idx < 0 && bit >= 0)
1816                 clear_bit(bit, cpuc->used_mask);
1817
1818         return idx;
1819 }
1820
1821 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1822                                       struct perf_event *event)
1823 {
1824         int bit;
1825         struct hw_perf_event *hwc = &event->hw;
1826         unsigned int region = EVENT_REGION(hwc->config_base);
1827         unsigned int group = EVENT_GROUP(hwc->config_base);
1828         bool venum_event = EVENT_VENUM(hwc->config_base);
1829         bool scorpion_event = EVENT_CPU(hwc->config_base);
1830
1831         if (venum_event || scorpion_event) {
1832                 bit = scorpion_event_to_bit(event, region, group);
1833                 clear_bit(bit, cpuc->used_mask);
1834         }
1835 }
1836
1837 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1838 {
1839         armv7pmu_init(cpu_pmu);
1840         cpu_pmu->name           = "armv7_scorpion";
1841         cpu_pmu->map_event      = scorpion_map_event;
1842         cpu_pmu->reset          = scorpion_pmu_reset;
1843         cpu_pmu->enable         = scorpion_pmu_enable_event;
1844         cpu_pmu->disable        = scorpion_pmu_disable_event;
1845         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
1846         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1847         return armv7_probe_num_events(cpu_pmu);
1848 }
1849
1850 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1851 {
1852         armv7pmu_init(cpu_pmu);
1853         cpu_pmu->name           = "armv7_scorpion_mp";
1854         cpu_pmu->map_event      = scorpion_map_event;
1855         cpu_pmu->reset          = scorpion_pmu_reset;
1856         cpu_pmu->enable         = scorpion_pmu_enable_event;
1857         cpu_pmu->disable        = scorpion_pmu_disable_event;
1858         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
1859         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1860         return armv7_probe_num_events(cpu_pmu);
1861 }
1862
1863 static const struct of_device_id armv7_pmu_of_device_ids[] = {
1864         {.compatible = "arm,cortex-a17-pmu",    .data = armv7_a17_pmu_init},
1865         {.compatible = "arm,cortex-a15-pmu",    .data = armv7_a15_pmu_init},
1866         {.compatible = "arm,cortex-a12-pmu",    .data = armv7_a12_pmu_init},
1867         {.compatible = "arm,cortex-a9-pmu",     .data = armv7_a9_pmu_init},
1868         {.compatible = "arm,cortex-a8-pmu",     .data = armv7_a8_pmu_init},
1869         {.compatible = "arm,cortex-a7-pmu",     .data = armv7_a7_pmu_init},
1870         {.compatible = "arm,cortex-a5-pmu",     .data = armv7_a5_pmu_init},
1871         {.compatible = "qcom,krait-pmu",        .data = krait_pmu_init},
1872         {.compatible = "qcom,scorpion-pmu",     .data = scorpion_pmu_init},
1873         {.compatible = "qcom,scorpion-mp-pmu",  .data = scorpion_mp_pmu_init},
1874         {},
1875 };
1876
1877 static const struct pmu_probe_info armv7_pmu_probe_table[] = {
1878         ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),
1879         ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),
1880         { /* sentinel value */ }
1881 };
1882
1883
1884 static int armv7_pmu_device_probe(struct platform_device *pdev)
1885 {
1886         return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids,
1887                                     armv7_pmu_probe_table);
1888 }
1889
1890 static struct platform_driver armv7_pmu_driver = {
1891         .driver         = {
1892                 .name   = "armv7-pmu",
1893                 .of_match_table = armv7_pmu_of_device_ids,
1894         },
1895         .probe          = armv7_pmu_device_probe,
1896 };
1897
1898 static int __init register_armv7_pmu_driver(void)
1899 {
1900         return platform_driver_register(&armv7_pmu_driver);
1901 }
1902 device_initcall(register_armv7_pmu_driver);
1903 #endif  /* CONFIG_CPU_V7 */