arch/arm64/kernel/perf_event.c

   1 /*
   2  * PMU support
   3  *
   4  * Copyright (C) 2012 ARM Limited
   5  * Author: Will Deacon <will.deacon@arm.com>
   6  *
   7  * This code is based heavily on the ARMv7 perf event code.
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20  */
  21 #define pr_fmt(fmt) "hw perfevents: " fmt
  22
  23 #include <linux/bitmap.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/irq.h>
  26 #include <linux/kernel.h>
  27 #include <linux/export.h>
  28 #include <linux/of_device.h>
  29 #include <linux/perf_event.h>
  30 #include <linux/platform_device.h>
  31 #include <linux/slab.h>
  32 #include <linux/spinlock.h>
  33 #include <linux/uaccess.h>
  34
  35 #include <asm/cputype.h>
  36 #include <asm/irq.h>
  37 #include <asm/irq_regs.h>
  38 #include <asm/pmu.h>
  39
  40 /*
  41  * ARMv8 supports a maximum of 32 events.
  42  * The cycle counter is included in this total.
  43  */
  44 #define ARMPMU_MAX_HWEVENTS             32
  45
  46 static DEFINE_PER_CPU(struct perf_event * [ARMPMU_MAX_HWEVENTS], hw_events);
  47 static DEFINE_PER_CPU(unsigned long [BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)], used_mask);
  48 static DEFINE_PER_CPU(struct pmu_hw_events, cpu_hw_events);
  49
  50 #define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))
  51
  52 /* Set at runtime when we know what CPU type we are. */
  53 static struct arm_pmu *cpu_pmu;
  54
  55 int
  56 armpmu_get_max_events(void)
  57 {
  58         int max_events = 0;
  59
  60         if (cpu_pmu != NULL)
  61                 max_events = cpu_pmu->num_events;
  62
  63         return max_events;
  64 }
  65 EXPORT_SYMBOL_GPL(armpmu_get_max_events);
  66
  67 int perf_num_counters(void)
  68 {
  69         return armpmu_get_max_events();
  70 }
  71 EXPORT_SYMBOL_GPL(perf_num_counters);
  72
  73 #define HW_OP_UNSUPPORTED               0xFFFF
  74
  75 #define C(_x) \
  76         PERF_COUNT_HW_CACHE_##_x
  77
  78 #define CACHE_OP_UNSUPPORTED            0xFFFF
  79
  80 #define PERF_MAP_ALL_UNSUPPORTED                                        \
  81         [0 ... PERF_COUNT_HW_MAX - 1] = HW_OP_UNSUPPORTED
  82
  83 #define PERF_CACHE_MAP_ALL_UNSUPPORTED                                  \
  84 [0 ... C(MAX) - 1] = {                                                  \
  85         [0 ... C(OP_MAX) - 1] = {                                       \
  86                 [0 ... C(RESULT_MAX) - 1] = CACHE_OP_UNSUPPORTED,       \
  87         },                                                              \
  88 }
  89
  90 static int
  91 armpmu_map_cache_event(const unsigned (*cache_map)
  92                                       [PERF_COUNT_HW_CACHE_MAX]
  93                                       [PERF_COUNT_HW_CACHE_OP_MAX]
  94                                       [PERF_COUNT_HW_CACHE_RESULT_MAX],
  95                        u64 config)
  96 {
  97         unsigned int cache_type, cache_op, cache_result, ret;
  98
  99         cache_type = (config >>  0) & 0xff;
 100         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 101                 return -EINVAL;
 102
 103         cache_op = (config >>  8) & 0xff;
 104         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 105                 return -EINVAL;
 106
 107         cache_result = (config >> 16) & 0xff;
 108         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 109                 return -EINVAL;
 110
 111         ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
 112
 113         if (ret == CACHE_OP_UNSUPPORTED)
 114                 return -ENOENT;
 115
 116         return ret;
 117 }
 118
 119 static int
 120 armpmu_map_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config)
 121 {
 122         int mapping;
 123
 124         if (config >= PERF_COUNT_HW_MAX)
 125                 return -EINVAL;
 126
 127         mapping = (*event_map)[config];
 128         return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
 129 }
 130
 131 static int
 132 armpmu_map_raw_event(u32 raw_event_mask, u64 config)
 133 {
 134         return (int)(config & raw_event_mask);
 135 }
 136
 137 static int map_cpu_event(struct perf_event *event,
 138                          const unsigned (*event_map)[PERF_COUNT_HW_MAX],
 139                          const unsigned (*cache_map)
 140                                         [PERF_COUNT_HW_CACHE_MAX]
 141                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 142                                         [PERF_COUNT_HW_CACHE_RESULT_MAX],
 143                          u32 raw_event_mask)
 144 {
 145         u64 config = event->attr.config;
 146
 147         switch (event->attr.type) {
 148         case PERF_TYPE_HARDWARE:
 149                 return armpmu_map_event(event_map, config);
 150         case PERF_TYPE_HW_CACHE:
 151                 return armpmu_map_cache_event(cache_map, config);
 152         case PERF_TYPE_RAW:
 153                 return armpmu_map_raw_event(raw_event_mask, config);
 154         }
 155
 156         return -ENOENT;
 157 }
 158
 159 int
 160 armpmu_event_set_period(struct perf_event *event,
 161                         struct hw_perf_event *hwc,
 162                         int idx)
 163 {
 164         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 165         s64 left = local64_read(&hwc->period_left);
 166         s64 period = hwc->sample_period;
 167         int ret = 0;
 168
 169         if (unlikely(left <= -period)) {
 170                 left = period;
 171                 local64_set(&hwc->period_left, left);
 172                 hwc->last_period = period;
 173                 ret = 1;
 174         }
 175
 176         if (unlikely(left <= 0)) {
 177                 left += period;
 178                 local64_set(&hwc->period_left, left);
 179                 hwc->last_period = period;
 180                 ret = 1;
 181         }
 182
 183         /*
 184          * Limit the maximum period to prevent the counter value
 185          * from overtaking the one we are about to program. In
 186          * effect we are reducing max_period to account for
 187          * interrupt latency (and we are being very conservative).
 188          */
 189         if (left > (armpmu->max_period >> 1))
 190                 left = armpmu->max_period >> 1;
 191
 192         local64_set(&hwc->prev_count, (u64)-left);
 193
 194         armpmu->write_counter(idx, (u64)(-left) & 0xffffffff);
 195
 196         perf_event_update_userpage(event);
 197
 198         return ret;
 199 }
 200
 201 u64
 202 armpmu_event_update(struct perf_event *event,
 203                     struct hw_perf_event *hwc,
 204                     int idx)
 205 {
 206         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 207         u64 delta, prev_raw_count, new_raw_count;
 208
 209 again:
 210         prev_raw_count = local64_read(&hwc->prev_count);
 211         new_raw_count = armpmu->read_counter(idx);
 212
 213         if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 214                              new_raw_count) != prev_raw_count)
 215                 goto again;
 216
 217         delta = (new_raw_count - prev_raw_count) & armpmu->max_period;
 218
 219         local64_add(delta, &event->count);
 220         local64_sub(delta, &hwc->period_left);
 221
 222         return new_raw_count;
 223 }
 224
 225 static void
 226 armpmu_read(struct perf_event *event)
 227 {
 228         struct hw_perf_event *hwc = &event->hw;
 229
 230         /* Don't read disabled counters! */
 231         if (hwc->idx < 0)
 232                 return;
 233
 234         armpmu_event_update(event, hwc, hwc->idx);
 235 }
 236
 237 static void
 238 armpmu_stop(struct perf_event *event, int flags)
 239 {
 240         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 241         struct hw_perf_event *hwc = &event->hw;
 242
 243         /*
 244          * ARM pmu always has to update the counter, so ignore
 245          * PERF_EF_UPDATE, see comments in armpmu_start().
 246          */
 247         if (!(hwc->state & PERF_HES_STOPPED)) {
 248                 armpmu->disable(hwc, hwc->idx);
 249                 barrier(); /* why? */
 250                 armpmu_event_update(event, hwc, hwc->idx);
 251                 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 252         }
 253 }
 254
 255 static void
 256 armpmu_start(struct perf_event *event, int flags)
 257 {
 258         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 259         struct hw_perf_event *hwc = &event->hw;
 260
 261         /*
 262          * ARM pmu always has to reprogram the period, so ignore
 263          * PERF_EF_RELOAD, see the comment below.
 264          */
 265         if (flags & PERF_EF_RELOAD)
 266                 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 267
 268         hwc->state = 0;
 269         /*
 270          * Set the period again. Some counters can't be stopped, so when we
 271          * were stopped we simply disabled the IRQ source and the counter
 272          * may have been left counting. If we don't do this step then we may
 273          * get an interrupt too soon or *way* too late if the overflow has
 274          * happened since disabling.
 275          */
 276         armpmu_event_set_period(event, hwc, hwc->idx);
 277         armpmu->enable(hwc, hwc->idx);
 278 }
 279
 280 static void
 281 armpmu_del(struct perf_event *event, int flags)
 282 {
 283         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 284         struct pmu_hw_events *hw_events = armpmu->get_hw_events();
 285         struct hw_perf_event *hwc = &event->hw;
 286         int idx = hwc->idx;
 287
 288         WARN_ON(idx < 0);
 289
 290         armpmu_stop(event, PERF_EF_UPDATE);
 291         hw_events->events[idx] = NULL;
 292         clear_bit(idx, hw_events->used_mask);
 293
 294         perf_event_update_userpage(event);
 295 }
 296
 297 static int
 298 armpmu_add(struct perf_event *event, int flags)
 299 {
 300         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 301         struct pmu_hw_events *hw_events = armpmu->get_hw_events();
 302         struct hw_perf_event *hwc = &event->hw;
 303         int idx;
 304         int err = 0;
 305
 306         perf_pmu_disable(event->pmu);
 307
 308         /* If we don't have a space for the counter then finish early. */
 309         idx = armpmu->get_event_idx(hw_events, hwc);
 310         if (idx < 0) {
 311                 err = idx;
 312                 goto out;
 313         }
 314
 315         /*
 316          * If there is an event in the counter we are going to use then make
 317          * sure it is disabled.
 318          */
 319         event->hw.idx = idx;
 320         armpmu->disable(hwc, idx);
 321         hw_events->events[idx] = event;
 322
 323         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 324         if (flags & PERF_EF_START)
 325                 armpmu_start(event, PERF_EF_RELOAD);
 326
 327         /* Propagate our changes to the userspace mapping. */
 328         perf_event_update_userpage(event);
 329
 330 out:
 331         perf_pmu_enable(event->pmu);
 332         return err;
 333 }
 334
 335 static int
 336 validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events,
 337                                 struct perf_event *event)
 338 {
 339         struct arm_pmu *armpmu;
 340         struct hw_perf_event fake_event = event->hw;
 341         struct pmu *leader_pmu = event->group_leader->pmu;
 342
 343         if (is_software_event(event))
 344                 return 1;
 345
 346         /*
 347          * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
 348          * core perf code won't check that the pmu->ctx == leader->ctx
 349          * until after pmu->event_init(event).
 350          */
 351         if (event->pmu != pmu)
 352                 return 0;
 353
 354         if (event->pmu != leader_pmu || event->state < PERF_EVENT_STATE_OFF)
 355                 return 1;
 356
 357         if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
 358                 return 1;
 359
 360         armpmu = to_arm_pmu(event->pmu);
 361         return armpmu->get_event_idx(hw_events, &fake_event) >= 0;
 362 }
 363
 364 static int
 365 validate_group(struct perf_event *event)
 366 {
 367         struct perf_event *sibling, *leader = event->group_leader;
 368         struct pmu_hw_events fake_pmu;
 369         DECLARE_BITMAP(fake_used_mask, ARMPMU_MAX_HWEVENTS);
 370
 371         /*
 372          * Initialise the fake PMU. We only need to populate the
 373          * used_mask for the purposes of validation.
 374          */
 375         memset(fake_used_mask, 0, sizeof(fake_used_mask));
 376         fake_pmu.used_mask = fake_used_mask;
 377
 378         if (!validate_event(event->pmu, &fake_pmu, leader))
 379                 return -EINVAL;
 380
 381         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
 382                 if (!validate_event(event->pmu, &fake_pmu, sibling))
 383                         return -EINVAL;
 384         }
 385
 386         if (!validate_event(event->pmu, &fake_pmu, event))
 387                 return -EINVAL;
 388
 389         return 0;
 390 }
 391
 392 static void
 393 armpmu_disable_percpu_irq(void *data)
 394 {
 395         unsigned int irq = *(unsigned int *)data;
 396         disable_percpu_irq(irq);
 397 }
 398
 399 static void
 400 armpmu_release_hardware(struct arm_pmu *armpmu)
 401 {
 402         int irq;
 403         unsigned int i, irqs;
 404         struct platform_device *pmu_device = armpmu->plat_device;
 405
 406         irqs = min(pmu_device->num_resources, num_possible_cpus());
 407         if (!irqs)
 408                 return;
 409
 410         irq = platform_get_irq(pmu_device, 0);
 411         if (irq <= 0)
 412                 return;
 413
 414         if (irq_is_percpu(irq)) {
 415                 on_each_cpu(armpmu_disable_percpu_irq, &irq, 1);
 416                 free_percpu_irq(irq, &cpu_hw_events);
 417         } else {
 418                 for (i = 0; i < irqs; ++i) {
 419                         int cpu = i;
 420
 421                         if (armpmu->irq_affinity)
 422                                 cpu = armpmu->irq_affinity[i];
 423
 424                         if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs))
 425                                 continue;
 426                         irq = platform_get_irq(pmu_device, i);
 427                         if (irq > 0)
 428                                 free_irq(irq, armpmu);
 429                 }
 430         }
 431 }
 432
 433 static void
 434 armpmu_enable_percpu_irq(void *data)
 435 {
 436         unsigned int irq = *(unsigned int *)data;
 437         enable_percpu_irq(irq, IRQ_TYPE_NONE);
 438 }
 439
 440 static int
 441 armpmu_reserve_hardware(struct arm_pmu *armpmu)
 442 {
 443         int err, irq;
 444         unsigned int i, irqs;
 445         struct platform_device *pmu_device = armpmu->plat_device;
 446
 447         if (!pmu_device)
 448                 return -ENODEV;
 449
 450         irqs = min(pmu_device->num_resources, num_possible_cpus());
 451         if (!irqs) {
 452                 pr_err("no irqs for PMUs defined\n");
 453                 return -ENODEV;
 454         }
 455
 456         irq = platform_get_irq(pmu_device, 0);
 457         if (irq <= 0) {
 458                 pr_err("failed to get valid irq for PMU device\n");
 459                 return -ENODEV;
 460         }
 461
 462         if (irq_is_percpu(irq)) {
 463                 err = request_percpu_irq(irq, armpmu->handle_irq,
 464                                 "arm-pmu", &cpu_hw_events);
 465
 466                 if (err) {
 467                         pr_err("unable to request percpu IRQ%d for ARM PMU counters\n",
 468                                         irq);
 469                         armpmu_release_hardware(armpmu);
 470                         return err;
 471                 }
 472
 473                 on_each_cpu(armpmu_enable_percpu_irq, &irq, 1);
 474         } else {
 475                 for (i = 0; i < irqs; ++i) {
 476                         int cpu = i;
 477
 478                         err = 0;
 479                         irq = platform_get_irq(pmu_device, i);
 480                         if (irq <= 0)
 481                                 continue;
 482
 483                         if (armpmu->irq_affinity)
 484                                 cpu = armpmu->irq_affinity[i];
 485
 486                         /*
 487                          * If we have a single PMU interrupt that we can't shift,
 488                          * assume that we're running on a uniprocessor machine and
 489                          * continue. Otherwise, continue without this interrupt.
 490                          */
 491                         if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) {
 492                                 pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n",
 493                                                 irq, cpu);
 494                                 continue;
 495                         }
 496
 497                         err = request_irq(irq, armpmu->handle_irq,
 498                                         IRQF_NOBALANCING | IRQF_NO_THREAD,
 499                                         "arm-pmu", armpmu);
 500                         if (err) {
 501                                 pr_err("unable to request IRQ%d for ARM PMU counters\n",
 502                                                 irq);
 503                                 armpmu_release_hardware(armpmu);
 504                                 return err;
 505                         }
 506
 507                         cpumask_set_cpu(cpu, &armpmu->active_irqs);
 508                 }
 509         }
 510
 511         return 0;
 512 }
 513
 514 static void
 515 hw_perf_event_destroy(struct perf_event *event)
 516 {
 517         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 518         atomic_t *active_events  = &armpmu->active_events;
 519         struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex;
 520
 521         if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) {
 522                 armpmu_release_hardware(armpmu);
 523                 mutex_unlock(pmu_reserve_mutex);
 524         }
 525 }
 526
 527 static int
 528 event_requires_mode_exclusion(struct perf_event_attr *attr)
 529 {
 530         return attr->exclude_idle || attr->exclude_user ||
 531                attr->exclude_kernel || attr->exclude_hv;
 532 }
 533
 534 static int
 535 __hw_perf_event_init(struct perf_event *event)
 536 {
 537         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 538         struct hw_perf_event *hwc = &event->hw;
 539         int mapping, err;
 540
 541         mapping = armpmu->map_event(event);
 542
 543         if (mapping < 0) {
 544                 pr_debug("event %x:%llx not supported\n", event->attr.type,
 545                          event->attr.config);
 546                 return mapping;
 547         }
 548
 549         /*
 550          * We don't assign an index until we actually place the event onto
 551          * hardware. Use -1 to signify that we haven't decided where to put it
 552          * yet. For SMP systems, each core has it's own PMU so we can't do any
 553          * clever allocation or constraints checking at this point.
 554          */
 555         hwc->idx                = -1;
 556         hwc->config_base        = 0;
 557         hwc->config             = 0;
 558         hwc->event_base         = 0;
 559
 560         /*
 561          * Check whether we need to exclude the counter from certain modes.
 562          */
 563         if ((!armpmu->set_event_filter ||
 564              armpmu->set_event_filter(hwc, &event->attr)) &&
 565              event_requires_mode_exclusion(&event->attr)) {
 566                 pr_debug("ARM performance counters do not support mode exclusion\n");
 567                 return -EPERM;
 568         }
 569
 570         /*
 571          * Store the event encoding into the config_base field.
 572          */
 573         hwc->config_base            |= (unsigned long)mapping;
 574
 575         if (!hwc->sample_period) {
 576                 /*
 577                  * For non-sampling runs, limit the sample_period to half
 578                  * of the counter width. That way, the new counter value
 579                  * is far less likely to overtake the previous one unless
 580                  * you have some serious IRQ latency issues.
 581                  */
 582                 hwc->sample_period  = armpmu->max_period >> 1;
 583                 hwc->last_period    = hwc->sample_period;
 584                 local64_set(&hwc->period_left, hwc->sample_period);
 585         }
 586
 587         err = 0;
 588         if (event->group_leader != event) {
 589                 err = validate_group(event);
 590                 if (err)
 591                         return -EINVAL;
 592         }
 593
 594         return err;
 595 }
 596
 597 static int armpmu_event_init(struct perf_event *event)
 598 {
 599         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 600         int err = 0;
 601         atomic_t *active_events = &armpmu->active_events;
 602
 603         if (armpmu->map_event(event) == -ENOENT)
 604                 return -ENOENT;
 605
 606         event->destroy = hw_perf_event_destroy;
 607
 608         if (!atomic_inc_not_zero(active_events)) {
 609                 mutex_lock(&armpmu->reserve_mutex);
 610                 if (atomic_read(active_events) == 0)
 611                         err = armpmu_reserve_hardware(armpmu);
 612
 613                 if (!err)
 614                         atomic_inc(active_events);
 615                 mutex_unlock(&armpmu->reserve_mutex);
 616         }
 617
 618         if (err)
 619                 return err;
 620
 621         err = __hw_perf_event_init(event);
 622         if (err)
 623                 hw_perf_event_destroy(event);
 624
 625         return err;
 626 }
 627
 628 static void armpmu_enable(struct pmu *pmu)
 629 {
 630         struct arm_pmu *armpmu = to_arm_pmu(pmu);
 631         struct pmu_hw_events *hw_events = armpmu->get_hw_events();
 632         int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events);
 633
 634         if (enabled)
 635                 armpmu->start();
 636 }
 637
 638 static void armpmu_disable(struct pmu *pmu)
 639 {
 640         struct arm_pmu *armpmu = to_arm_pmu(pmu);
 641         armpmu->stop();
 642 }
 643
 644 static void __init armpmu_init(struct arm_pmu *armpmu)
 645 {
 646         atomic_set(&armpmu->active_events, 0);
 647         mutex_init(&armpmu->reserve_mutex);
 648
 649         armpmu->pmu = (struct pmu) {
 650                 .pmu_enable     = armpmu_enable,
 651                 .pmu_disable    = armpmu_disable,
 652                 .event_init     = armpmu_event_init,
 653                 .add            = armpmu_add,
 654                 .del            = armpmu_del,
 655                 .start          = armpmu_start,
 656                 .stop           = armpmu_stop,
 657                 .read           = armpmu_read,
 658         };
 659 }
 660
 661 int __init armpmu_register(struct arm_pmu *armpmu, char *name, int type)
 662 {
 663         armpmu_init(armpmu);
 664         return perf_pmu_register(&armpmu->pmu, name, type);
 665 }
 666
 667 /*
 668  * ARMv8 PMUv3 Performance Events handling code.
 669  * Common event types.
 670  */
 671 enum armv8_pmuv3_perf_types {
 672         /* Required events. */
 673         ARMV8_PMUV3_PERFCTR_PMNC_SW_INCR                        = 0x00,
 674         ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL                    = 0x03,
 675         ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS                    = 0x04,
 676         ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED                  = 0x10,
 677         ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES                        = 0x11,
 678         ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED                      = 0x12,
 679
 680         /* At least one of the following is required. */
 681         ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED                      = 0x08,
 682         ARMV8_PMUV3_PERFCTR_OP_SPEC                             = 0x1B,
 683
 684         /* Common architectural events. */
 685         ARMV8_PMUV3_PERFCTR_MEM_READ                            = 0x06,
 686         ARMV8_PMUV3_PERFCTR_MEM_WRITE                           = 0x07,
 687         ARMV8_PMUV3_PERFCTR_EXC_TAKEN                           = 0x09,
 688         ARMV8_PMUV3_PERFCTR_EXC_EXECUTED                        = 0x0A,
 689         ARMV8_PMUV3_PERFCTR_CID_WRITE                           = 0x0B,
 690         ARMV8_PMUV3_PERFCTR_PC_WRITE                            = 0x0C,
 691         ARMV8_PMUV3_PERFCTR_PC_IMM_BRANCH                       = 0x0D,
 692         ARMV8_PMUV3_PERFCTR_PC_PROC_RETURN                      = 0x0E,
 693         ARMV8_PMUV3_PERFCTR_MEM_UNALIGNED_ACCESS                = 0x0F,
 694         ARMV8_PMUV3_PERFCTR_TTBR_WRITE                          = 0x1C,
 695
 696         /* Common microarchitectural events. */
 697         ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL                    = 0x01,
 698         ARMV8_PMUV3_PERFCTR_ITLB_REFILL                         = 0x02,
 699         ARMV8_PMUV3_PERFCTR_DTLB_REFILL                         = 0x05,
 700         ARMV8_PMUV3_PERFCTR_MEM_ACCESS                          = 0x13,
 701         ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS                    = 0x14,
 702         ARMV8_PMUV3_PERFCTR_L1_DCACHE_WB                        = 0x15,
 703         ARMV8_PMUV3_PERFCTR_L2_CACHE_ACCESS                     = 0x16,
 704         ARMV8_PMUV3_PERFCTR_L2_CACHE_REFILL                     = 0x17,
 705         ARMV8_PMUV3_PERFCTR_L2_CACHE_WB                         = 0x18,
 706         ARMV8_PMUV3_PERFCTR_BUS_ACCESS                          = 0x19,
 707         ARMV8_PMUV3_PERFCTR_MEM_ERROR                           = 0x1A,
 708         ARMV8_PMUV3_PERFCTR_BUS_CYCLES                          = 0x1D,
 709 };
 710
 711 /* PMUv3 HW events mapping. */
 712 static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
 713         PERF_MAP_ALL_UNSUPPORTED,
 714         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
 715         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
 716         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 717         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 718         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 719 };
 720
 721 static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 722                                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 723                                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 724         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 725
 726         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 727         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 728         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 729         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 730
 731         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 732         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 733         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 734         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 735 };
 736
 737 /*
 738  * Perf Events' indices
 739  */
 740 #define ARMV8_IDX_CYCLE_COUNTER 0
 741 #define ARMV8_IDX_COUNTER0      1
 742 #define ARMV8_IDX_COUNTER_LAST  (ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 743
 744 #define ARMV8_MAX_COUNTERS      32
 745 #define ARMV8_COUNTER_MASK      (ARMV8_MAX_COUNTERS - 1)
 746
 747 /*
 748  * ARMv8 low level PMU access
 749  */
 750
 751 /*
 752  * Perf Event to low level counters mapping
 753  */
 754 #define ARMV8_IDX_TO_COUNTER(x) \
 755         (((x) - ARMV8_IDX_COUNTER0) & ARMV8_COUNTER_MASK)
 756
 757 /*
 758  * Per-CPU PMCR: config reg
 759  */
 760 #define ARMV8_PMCR_E            (1 << 0) /* Enable all counters */
 761 #define ARMV8_PMCR_P            (1 << 1) /* Reset all counters */
 762 #define ARMV8_PMCR_C            (1 << 2) /* Cycle counter reset */
 763 #define ARMV8_PMCR_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
 764 #define ARMV8_PMCR_X            (1 << 4) /* Export to ETM */
 765 #define ARMV8_PMCR_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
 766 #define ARMV8_PMCR_N_SHIFT      11       /* Number of counters supported */
 767 #define ARMV8_PMCR_N_MASK       0x1f
 768 #define ARMV8_PMCR_MASK         0x3f     /* Mask for writable bits */
 769
 770 /*
 771  * PMOVSR: counters overflow flag status reg
 772  */
 773 #define ARMV8_OVSR_MASK         0xffffffff      /* Mask for writable bits */
 774 #define ARMV8_OVERFLOWED_MASK   ARMV8_OVSR_MASK
 775
 776 /*
 777  * PMXEVTYPER: Event selection reg
 778  */
 779 #define ARMV8_EVTYPE_MASK       0xc80003ff      /* Mask for writable bits */
 780 #define ARMV8_EVTYPE_EVENT      0x3ff           /* Mask for EVENT bits */
 781
 782 /*
 783  * Event filters for PMUv3
 784  */
 785 #define ARMV8_EXCLUDE_EL1       (1 << 31)
 786 #define ARMV8_EXCLUDE_EL0       (1 << 30)
 787 #define ARMV8_INCLUDE_EL2       (1 << 27)
 788
 789 static inline u32 armv8pmu_pmcr_read(void)
 790 {
 791         u32 val;
 792         asm volatile("mrs %0, pmcr_el0" : "=r" (val));
 793         return val;
 794 }
 795
 796 static inline void armv8pmu_pmcr_write(u32 val)
 797 {
 798         val &= ARMV8_PMCR_MASK;
 799         isb();
 800         asm volatile("msr pmcr_el0, %0" :: "r" (val));
 801 }
 802
 803 static inline int armv8pmu_has_overflowed(u32 pmovsr)
 804 {
 805         return pmovsr & ARMV8_OVERFLOWED_MASK;
 806 }
 807
 808 static inline int armv8pmu_counter_valid(int idx)
 809 {
 810         return idx >= ARMV8_IDX_CYCLE_COUNTER && idx <= ARMV8_IDX_COUNTER_LAST;
 811 }
 812
 813 static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx)
 814 {
 815         int ret = 0;
 816         u32 counter;
 817
 818         if (!armv8pmu_counter_valid(idx)) {
 819                 pr_err("CPU%u checking wrong counter %d overflow status\n",
 820                         smp_processor_id(), idx);
 821         } else {
 822                 counter = ARMV8_IDX_TO_COUNTER(idx);
 823                 ret = pmnc & BIT(counter);
 824         }
 825
 826         return ret;
 827 }
 828
 829 static inline int armv8pmu_select_counter(int idx)
 830 {
 831         u32 counter;
 832
 833         if (!armv8pmu_counter_valid(idx)) {
 834                 pr_err("CPU%u selecting wrong PMNC counter %d\n",
 835                         smp_processor_id(), idx);
 836                 return -EINVAL;
 837         }
 838
 839         counter = ARMV8_IDX_TO_COUNTER(idx);
 840         asm volatile("msr pmselr_el0, %0" :: "r" (counter));
 841         isb();
 842
 843         return idx;
 844 }
 845
 846 static inline u32 armv8pmu_read_counter(int idx)
 847 {
 848         u32 value = 0;
 849
 850         if (!armv8pmu_counter_valid(idx))
 851                 pr_err("CPU%u reading wrong counter %d\n",
 852                         smp_processor_id(), idx);
 853         else if (idx == ARMV8_IDX_CYCLE_COUNTER)
 854                 asm volatile("mrs %0, pmccntr_el0" : "=r" (value));
 855         else if (armv8pmu_select_counter(idx) == idx)
 856                 asm volatile("mrs %0, pmxevcntr_el0" : "=r" (value));
 857
 858         return value;
 859 }
 860
 861 static inline void armv8pmu_write_counter(int idx, u32 value)
 862 {
 863         if (!armv8pmu_counter_valid(idx))
 864                 pr_err("CPU%u writing wrong counter %d\n",
 865                         smp_processor_id(), idx);
 866         else if (idx == ARMV8_IDX_CYCLE_COUNTER)
 867                 asm volatile("msr pmccntr_el0, %0" :: "r" (value));
 868         else if (armv8pmu_select_counter(idx) == idx)
 869                 asm volatile("msr pmxevcntr_el0, %0" :: "r" (value));
 870 }
 871
 872 static inline void armv8pmu_write_evtype(int idx, u32 val)
 873 {
 874         if (armv8pmu_select_counter(idx) == idx) {
 875                 val &= ARMV8_EVTYPE_MASK;
 876                 asm volatile("msr pmxevtyper_el0, %0" :: "r" (val));
 877         }
 878 }
 879
 880 static inline int armv8pmu_enable_counter(int idx)
 881 {
 882         u32 counter;
 883
 884         if (!armv8pmu_counter_valid(idx)) {
 885                 pr_err("CPU%u enabling wrong PMNC counter %d\n",
 886                         smp_processor_id(), idx);
 887                 return -EINVAL;
 888         }
 889
 890         counter = ARMV8_IDX_TO_COUNTER(idx);
 891         asm volatile("msr pmcntenset_el0, %0" :: "r" (BIT(counter)));
 892         return idx;
 893 }
 894
 895 static inline int armv8pmu_disable_counter(int idx)
 896 {
 897         u32 counter;
 898
 899         if (!armv8pmu_counter_valid(idx)) {
 900                 pr_err("CPU%u disabling wrong PMNC counter %d\n",
 901                         smp_processor_id(), idx);
 902                 return -EINVAL;
 903         }
 904
 905         counter = ARMV8_IDX_TO_COUNTER(idx);
 906         asm volatile("msr pmcntenclr_el0, %0" :: "r" (BIT(counter)));
 907         return idx;
 908 }
 909
 910 static inline int armv8pmu_enable_intens(int idx)
 911 {
 912         u32 counter;
 913
 914         if (!armv8pmu_counter_valid(idx)) {
 915                 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
 916                         smp_processor_id(), idx);
 917                 return -EINVAL;
 918         }
 919
 920         counter = ARMV8_IDX_TO_COUNTER(idx);
 921         asm volatile("msr pmintenset_el1, %0" :: "r" (BIT(counter)));
 922         return idx;
 923 }
 924
 925 static inline int armv8pmu_disable_intens(int idx)
 926 {
 927         u32 counter;
 928
 929         if (!armv8pmu_counter_valid(idx)) {
 930                 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
 931                         smp_processor_id(), idx);
 932                 return -EINVAL;
 933         }
 934
 935         counter = ARMV8_IDX_TO_COUNTER(idx);
 936         asm volatile("msr pmintenclr_el1, %0" :: "r" (BIT(counter)));
 937         isb();
 938         /* Clear the overflow flag in case an interrupt is pending. */
 939         asm volatile("msr pmovsclr_el0, %0" :: "r" (BIT(counter)));
 940         isb();
 941         return idx;
 942 }
 943
 944 static inline u32 armv8pmu_getreset_flags(void)
 945 {
 946         u32 value;
 947
 948         /* Read */
 949         asm volatile("mrs %0, pmovsclr_el0" : "=r" (value));
 950
 951         /* Write to clear flags */
 952         value &= ARMV8_OVSR_MASK;
 953         asm volatile("msr pmovsclr_el0, %0" :: "r" (value));
 954
 955         return value;
 956 }
 957
 958 static void armv8pmu_enable_event(struct hw_perf_event *hwc, int idx)
 959 {
 960         unsigned long flags;
 961         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 962
 963         /*
 964          * Enable counter and interrupt, and set the counter to count
 965          * the event that we're interested in.
 966          */
 967         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 968
 969         /*
 970          * Disable counter
 971          */
 972         armv8pmu_disable_counter(idx);
 973
 974         /*
 975          * Set event (if destined for PMNx counters).
 976          */
 977         armv8pmu_write_evtype(idx, hwc->config_base);
 978
 979         /*
 980          * Enable interrupt for this counter
 981          */
 982         armv8pmu_enable_intens(idx);
 983
 984         /*
 985          * Enable counter
 986          */
 987         armv8pmu_enable_counter(idx);
 988
 989         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 990 }
 991
 992 static void armv8pmu_disable_event(struct hw_perf_event *hwc, int idx)
 993 {
 994         unsigned long flags;
 995         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 996
 997         /*
 998          * Disable counter and interrupt
 999          */
1000         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1001
1002         /*
1003          * Disable counter
1004          */
1005         armv8pmu_disable_counter(idx);
1006
1007         /*
1008          * Disable interrupt for this counter
1009          */
1010         armv8pmu_disable_intens(idx);
1011
1012         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1013 }
1014
1015 static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
1016 {
1017         u32 pmovsr;
1018         struct perf_sample_data data;
1019         struct pmu_hw_events *cpuc;
1020         struct pt_regs *regs;
1021         int idx;
1022
1023         /*
1024          * Get and reset the IRQ flags
1025          */
1026         pmovsr = armv8pmu_getreset_flags();
1027
1028         /*
1029          * Did an overflow occur?
1030          */
1031         if (!armv8pmu_has_overflowed(pmovsr))
1032                 return IRQ_NONE;
1033
1034         /*
1035          * Handle the counter(s) overflow(s)
1036          */
1037         regs = get_irq_regs();
1038
1039         cpuc = this_cpu_ptr(&cpu_hw_events);
1040         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
1041                 struct perf_event *event = cpuc->events[idx];
1042                 struct hw_perf_event *hwc;
1043
1044                 /* Ignore if we don't have an event. */
1045                 if (!event)
1046                         continue;
1047
1048                 /*
1049                  * We have a single interrupt for all counters. Check that
1050                  * each counter has overflowed before we process it.
1051                  */
1052                 if (!armv8pmu_counter_has_overflowed(pmovsr, idx))
1053                         continue;
1054
1055                 hwc = &event->hw;
1056                 armpmu_event_update(event, hwc, idx);
1057                 perf_sample_data_init(&data, 0, hwc->last_period);
1058                 if (!armpmu_event_set_period(event, hwc, idx))
1059                         continue;
1060
1061                 if (perf_event_overflow(event, &data, regs))
1062                         cpu_pmu->disable(hwc, idx);
1063         }
1064
1065         /*
1066          * Handle the pending perf events.
1067          *
1068          * Note: this call *must* be run with interrupts disabled. For
1069          * platforms that can have the PMU interrupts raised as an NMI, this
1070          * will not work.
1071          */
1072         irq_work_run();
1073
1074         return IRQ_HANDLED;
1075 }
1076
1077 static void armv8pmu_start(void)
1078 {
1079         unsigned long flags;
1080         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1081
1082         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1083         /* Enable all counters */
1084         armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMCR_E);
1085         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1086 }
1087
1088 static void armv8pmu_stop(void)
1089 {
1090         unsigned long flags;
1091         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1092
1093         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1094         /* Disable all counters */
1095         armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMCR_E);
1096         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1097 }
1098
1099 static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
1100                                   struct hw_perf_event *event)
1101 {
1102         int idx;
1103         unsigned long evtype = event->config_base & ARMV8_EVTYPE_EVENT;
1104
1105         /* Always place a cycle counter into the cycle counter. */
1106         if (evtype == ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES) {
1107                 if (test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))
1108                         return -EAGAIN;
1109
1110                 return ARMV8_IDX_CYCLE_COUNTER;
1111         }
1112
1113         /*
1114          * For anything other than a cycle counter, try and use
1115          * the events counters
1116          */
1117         for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1118                 if (!test_and_set_bit(idx, cpuc->used_mask))
1119                         return idx;
1120         }
1121
1122         /* The counters are all in use. */
1123         return -EAGAIN;
1124 }
1125
1126 /*
1127  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1128  */
1129 static int armv8pmu_set_event_filter(struct hw_perf_event *event,
1130                                      struct perf_event_attr *attr)
1131 {
1132         unsigned long config_base = 0;
1133
1134         if (attr->exclude_idle)
1135                 return -EPERM;
1136         if (attr->exclude_user)
1137                 config_base |= ARMV8_EXCLUDE_EL0;
1138         if (attr->exclude_kernel)
1139                 config_base |= ARMV8_EXCLUDE_EL1;
1140         if (!attr->exclude_hv)
1141                 config_base |= ARMV8_INCLUDE_EL2;
1142
1143         /*
1144          * Install the filter into config_base as this is used to
1145          * construct the event type.
1146          */
1147         event->config_base = config_base;
1148
1149         return 0;
1150 }
1151
1152 static void armv8pmu_reset(void *info)
1153 {
1154         u32 idx, nb_cnt = cpu_pmu->num_events;
1155
1156         /* The counter and interrupt enable registers are unknown at reset. */
1157         for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx)
1158                 armv8pmu_disable_event(NULL, idx);
1159
1160         /* Initialize & Reset PMNC: C and P bits. */
1161         armv8pmu_pmcr_write(ARMV8_PMCR_P | ARMV8_PMCR_C);
1162
1163         /* Disable access from userspace. */
1164         asm volatile("msr pmuserenr_el0, %0" :: "r" (0));
1165 }
1166
1167 static int armv8_pmuv3_map_event(struct perf_event *event)
1168 {
1169         return map_cpu_event(event, &armv8_pmuv3_perf_map,
1170                                 &armv8_pmuv3_perf_cache_map,
1171                                 ARMV8_EVTYPE_EVENT);
1172 }
1173
1174 static struct arm_pmu armv8pmu = {
1175         .handle_irq             = armv8pmu_handle_irq,
1176         .enable                 = armv8pmu_enable_event,
1177         .disable                = armv8pmu_disable_event,
1178         .read_counter           = armv8pmu_read_counter,
1179         .write_counter          = armv8pmu_write_counter,
1180         .get_event_idx          = armv8pmu_get_event_idx,
1181         .start                  = armv8pmu_start,
1182         .stop                   = armv8pmu_stop,
1183         .reset                  = armv8pmu_reset,
1184         .max_period             = (1LLU << 32) - 1,
1185 };
1186
1187 static u32 __init armv8pmu_read_num_pmnc_events(void)
1188 {
1189         u32 nb_cnt;
1190
1191         /* Read the nb of CNTx counters supported from PMNC */
1192         nb_cnt = (armv8pmu_pmcr_read() >> ARMV8_PMCR_N_SHIFT) & ARMV8_PMCR_N_MASK;
1193
1194         /* Add the CPU cycles counter and return */
1195         return nb_cnt + 1;
1196 }
1197
1198 static struct arm_pmu *__init armv8_pmuv3_pmu_init(void)
1199 {
1200         armv8pmu.name                   = "arm/armv8-pmuv3";
1201         armv8pmu.map_event              = armv8_pmuv3_map_event;
1202         armv8pmu.num_events             = armv8pmu_read_num_pmnc_events();
1203         armv8pmu.set_event_filter       = armv8pmu_set_event_filter;
1204         return &armv8pmu;
1205 }
1206
1207 /*
1208  * Ensure the PMU has sane values out of reset.
1209  * This requires SMP to be available, so exists as a separate initcall.
1210  */
1211 static int __init
1212 cpu_pmu_reset(void)
1213 {
1214         if (cpu_pmu && cpu_pmu->reset)
1215                 return on_each_cpu(cpu_pmu->reset, NULL, 1);
1216         return 0;
1217 }
1218 arch_initcall(cpu_pmu_reset);
1219
1220 /*
1221  * PMU platform driver and devicetree bindings.
1222  */
1223 static const struct of_device_id armpmu_of_device_ids[] = {
1224         {.compatible = "arm,armv8-pmuv3"},
1225         {},
1226 };
1227
1228 static int armpmu_device_probe(struct platform_device *pdev)
1229 {
1230         int i, irq, *irqs;
1231
1232         if (!cpu_pmu)
1233                 return -ENODEV;
1234
1235         /* Don't bother with PPIs; they're already affine */
1236         irq = platform_get_irq(pdev, 0);
1237         if (irq >= 0 && irq_is_percpu(irq))
1238                 goto out;
1239
1240         irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL);
1241         if (!irqs)
1242                 return -ENOMEM;
1243
1244         for (i = 0; i < pdev->num_resources; ++i) {
1245                 struct device_node *dn;
1246                 int cpu;
1247
1248                 dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity",
1249                                       i);
1250                 if (!dn) {
1251                         pr_warn("Failed to parse %s/interrupt-affinity[%d]\n",
1252                                 of_node_full_name(pdev->dev.of_node), i);
1253                         break;
1254                 }
1255
1256                 for_each_possible_cpu(cpu)
1257                         if (dn == of_cpu_device_node_get(cpu))
1258                                 break;
1259
1260                 if (cpu >= nr_cpu_ids) {
1261                         pr_warn("Failed to find logical CPU for %s\n",
1262                                 dn->name);
1263                         of_node_put(dn);
1264                         break;
1265                 }
1266                 of_node_put(dn);
1267
1268                 irqs[i] = cpu;
1269         }
1270
1271         if (i == pdev->num_resources)
1272                 cpu_pmu->irq_affinity = irqs;
1273         else
1274                 kfree(irqs);
1275
1276 out:
1277         cpu_pmu->plat_device = pdev;
1278         return 0;
1279 }
1280
1281 static struct platform_driver armpmu_driver = {
1282         .driver         = {
1283                 .name   = "arm-pmu",
1284                 .of_match_table = armpmu_of_device_ids,
1285         },
1286         .probe          = armpmu_device_probe,
1287 };
1288
1289 static int __init register_pmu_driver(void)
1290 {
1291         return platform_driver_register(&armpmu_driver);
1292 }
1293 device_initcall(register_pmu_driver);
1294
1295 static struct pmu_hw_events *armpmu_get_cpu_events(void)
1296 {
1297         return this_cpu_ptr(&cpu_hw_events);
1298 }
1299
1300 static void __init cpu_pmu_init(struct arm_pmu *armpmu)
1301 {
1302         int cpu;
1303         for_each_possible_cpu(cpu) {
1304                 struct pmu_hw_events *events = &per_cpu(cpu_hw_events, cpu);
1305                 events->events = per_cpu(hw_events, cpu);
1306                 events->used_mask = per_cpu(used_mask, cpu);
1307                 raw_spin_lock_init(&events->pmu_lock);
1308         }
1309         armpmu->get_hw_events = armpmu_get_cpu_events;
1310 }
1311
1312 static int __init init_hw_perf_events(void)
1313 {
1314         u64 dfr = read_cpuid(ID_AA64DFR0_EL1);
1315
1316         switch ((dfr >> 8) & 0xf) {
1317         case 0x1:       /* PMUv3 */
1318                 cpu_pmu = armv8_pmuv3_pmu_init();
1319                 break;
1320         }
1321
1322         if (cpu_pmu) {
1323                 pr_info("enabled with %s PMU driver, %d counters available\n",
1324                         cpu_pmu->name, cpu_pmu->num_events);
1325                 cpu_pmu_init(cpu_pmu);
1326                 armpmu_register(cpu_pmu, "cpu", PERF_TYPE_RAW);
1327         } else {
1328                 pr_info("no hardware support available\n");
1329         }
1330
1331         return 0;
1332 }
1333 early_initcall(init_hw_perf_events);
1334