drivers/perf/qcom_l2_pmu.c

   1 /* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved.
   2  *
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License version 2 and
   5  * only version 2 as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12 #include <linux/acpi.h>
  13 #include <linux/bitops.h>
  14 #include <linux/bug.h>
  15 #include <linux/cpuhotplug.h>
  16 #include <linux/cpumask.h>
  17 #include <linux/device.h>
  18 #include <linux/errno.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/irq.h>
  21 #include <linux/kernel.h>
  22 #include <linux/list.h>
  23 #include <linux/percpu.h>
  24 #include <linux/perf_event.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/smp.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/sysfs.h>
  29 #include <linux/types.h>
  30
  31 #include <asm/barrier.h>
  32 #include <asm/local64.h>
  33 #include <asm/sysreg.h>
  34
  35 #define MAX_L2_CTRS             9
  36
  37 #define L2PMCR_NUM_EV_SHIFT     11
  38 #define L2PMCR_NUM_EV_MASK      0x1F
  39
  40 #define L2PMCR                  0x400
  41 #define L2PMCNTENCLR            0x403
  42 #define L2PMCNTENSET            0x404
  43 #define L2PMINTENCLR            0x405
  44 #define L2PMINTENSET            0x406
  45 #define L2PMOVSCLR              0x407
  46 #define L2PMOVSSET              0x408
  47 #define L2PMCCNTCR              0x409
  48 #define L2PMCCNTR               0x40A
  49 #define L2PMCCNTSR              0x40C
  50 #define L2PMRESR                0x410
  51 #define IA_L2PMXEVCNTCR_BASE    0x420
  52 #define IA_L2PMXEVCNTR_BASE     0x421
  53 #define IA_L2PMXEVFILTER_BASE   0x423
  54 #define IA_L2PMXEVTYPER_BASE    0x424
  55
  56 #define IA_L2_REG_OFFSET        0x10
  57
  58 #define L2PMXEVFILTER_SUFILTER_ALL      0x000E0000
  59 #define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004
  60 #define L2PMXEVFILTER_ORGFILTER_ALL     0x00000003
  61
  62 #define L2EVTYPER_REG_SHIFT     3
  63
  64 #define L2PMRESR_GROUP_BITS     8
  65 #define L2PMRESR_GROUP_MASK     GENMASK(7, 0)
  66
  67 #define L2CYCLE_CTR_BIT         31
  68 #define L2CYCLE_CTR_RAW_CODE    0xFE
  69
  70 #define L2PMCR_RESET_ALL        0x6
  71 #define L2PMCR_COUNTERS_ENABLE  0x1
  72 #define L2PMCR_COUNTERS_DISABLE 0x0
  73
  74 #define L2PMRESR_EN             BIT_ULL(63)
  75
  76 #define L2_EVT_MASK             0x00000FFF
  77 #define L2_EVT_CODE_MASK        0x00000FF0
  78 #define L2_EVT_GRP_MASK         0x0000000F
  79 #define L2_EVT_CODE_SHIFT       4
  80 #define L2_EVT_GRP_SHIFT        0
  81
  82 #define L2_EVT_CODE(event)   (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT)
  83 #define L2_EVT_GROUP(event)  (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT)
  84
  85 #define L2_EVT_GROUP_MAX        7
  86
  87 #define L2_COUNTER_RELOAD       BIT_ULL(31)
  88 #define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63)
  89
  90 #define L2CPUSRSELR_EL1         sys_reg(3, 3, 15, 0, 6)
  91 #define L2CPUSRDR_EL1           sys_reg(3, 3, 15, 0, 7)
  92
  93 #define reg_idx(reg, i)         (((i) * IA_L2_REG_OFFSET) + reg##_BASE)
  94
  95 static DEFINE_RAW_SPINLOCK(l2_access_lock);
  96
  97 /**
  98  * set_l2_indirect_reg: write value to an L2 register
  99  * @reg: Address of L2 register.
 100  * @value: Value to be written to register.
 101  *
 102  * Use architecturally required barriers for ordering between system register
 103  * accesses
 104  */
 105 static void set_l2_indirect_reg(u64 reg, u64 val)
 106 {
 107         unsigned long flags;
 108
 109         raw_spin_lock_irqsave(&l2_access_lock, flags);
 110         write_sysreg_s(reg, L2CPUSRSELR_EL1);
 111         isb();
 112         write_sysreg_s(val, L2CPUSRDR_EL1);
 113         isb();
 114         raw_spin_unlock_irqrestore(&l2_access_lock, flags);
 115 }
 116
 117 /**
 118  * get_l2_indirect_reg: read an L2 register value
 119  * @reg: Address of L2 register.
 120  *
 121  * Use architecturally required barriers for ordering between system register
 122  * accesses
 123  */
 124 static u64 get_l2_indirect_reg(u64 reg)
 125 {
 126         u64 val;
 127         unsigned long flags;
 128
 129         raw_spin_lock_irqsave(&l2_access_lock, flags);
 130         write_sysreg_s(reg, L2CPUSRSELR_EL1);
 131         isb();
 132         val = read_sysreg_s(L2CPUSRDR_EL1);
 133         raw_spin_unlock_irqrestore(&l2_access_lock, flags);
 134
 135         return val;
 136 }
 137
 138 struct cluster_pmu;
 139
 140 /*
 141  * Aggregate PMU. Implements the core pmu functions and manages
 142  * the hardware PMUs.
 143  */
 144 struct l2cache_pmu {
 145         struct hlist_node node;
 146         u32 num_pmus;
 147         struct pmu pmu;
 148         int num_counters;
 149         cpumask_t cpumask;
 150         struct platform_device *pdev;
 151         struct cluster_pmu * __percpu *pmu_cluster;
 152         struct list_head clusters;
 153 };
 154
 155 /*
 156  * The cache is made up of one or more clusters, each cluster has its own PMU.
 157  * Each cluster is associated with one or more CPUs.
 158  * This structure represents one of the hardware PMUs.
 159  *
 160  * Events can be envisioned as a 2-dimensional array. Each column represents
 161  * a group of events. There are 8 groups. Only one entry from each
 162  * group can be in use at a time.
 163  *
 164  * Events are specified as 0xCCG, where CC is 2 hex digits specifying
 165  * the code (array row) and G specifies the group (column).
 166  *
 167  * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE
 168  * which is outside the above scheme.
 169  */
 170 struct cluster_pmu {
 171         struct list_head next;
 172         struct perf_event *events[MAX_L2_CTRS];
 173         struct l2cache_pmu *l2cache_pmu;
 174         DECLARE_BITMAP(used_counters, MAX_L2_CTRS);
 175         DECLARE_BITMAP(used_groups, L2_EVT_GROUP_MAX + 1);
 176         int irq;
 177         int cluster_id;
 178         /* The CPU that is used for collecting events on this cluster */
 179         int on_cpu;
 180         /* All the CPUs associated with this cluster */
 181         cpumask_t cluster_cpus;
 182         spinlock_t pmu_lock;
 183 };
 184
 185 #define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu))
 186
 187 static u32 l2_cycle_ctr_idx;
 188 static u32 l2_counter_present_mask;
 189
 190 static inline u32 idx_to_reg_bit(u32 idx)
 191 {
 192         if (idx == l2_cycle_ctr_idx)
 193                 return BIT(L2CYCLE_CTR_BIT);
 194
 195         return BIT(idx);
 196 }
 197
 198 static inline struct cluster_pmu *get_cluster_pmu(
 199         struct l2cache_pmu *l2cache_pmu, int cpu)
 200 {
 201         return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu);
 202 }
 203
 204 static void cluster_pmu_reset(void)
 205 {
 206         /* Reset all counters */
 207         set_l2_indirect_reg(L2PMCR, L2PMCR_RESET_ALL);
 208         set_l2_indirect_reg(L2PMCNTENCLR, l2_counter_present_mask);
 209         set_l2_indirect_reg(L2PMINTENCLR, l2_counter_present_mask);
 210         set_l2_indirect_reg(L2PMOVSCLR, l2_counter_present_mask);
 211 }
 212
 213 static inline void cluster_pmu_enable(void)
 214 {
 215         set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_ENABLE);
 216 }
 217
 218 static inline void cluster_pmu_disable(void)
 219 {
 220         set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_DISABLE);
 221 }
 222
 223 static inline void cluster_pmu_counter_set_value(u32 idx, u64 value)
 224 {
 225         if (idx == l2_cycle_ctr_idx)
 226                 set_l2_indirect_reg(L2PMCCNTR, value);
 227         else
 228                 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx), value);
 229 }
 230
 231 static inline u64 cluster_pmu_counter_get_value(u32 idx)
 232 {
 233         u64 value;
 234
 235         if (idx == l2_cycle_ctr_idx)
 236                 value = get_l2_indirect_reg(L2PMCCNTR);
 237         else
 238                 value = get_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx));
 239
 240         return value;
 241 }
 242
 243 static inline void cluster_pmu_counter_enable(u32 idx)
 244 {
 245         set_l2_indirect_reg(L2PMCNTENSET, idx_to_reg_bit(idx));
 246 }
 247
 248 static inline void cluster_pmu_counter_disable(u32 idx)
 249 {
 250         set_l2_indirect_reg(L2PMCNTENCLR, idx_to_reg_bit(idx));
 251 }
 252
 253 static inline void cluster_pmu_counter_enable_interrupt(u32 idx)
 254 {
 255         set_l2_indirect_reg(L2PMINTENSET, idx_to_reg_bit(idx));
 256 }
 257
 258 static inline void cluster_pmu_counter_disable_interrupt(u32 idx)
 259 {
 260         set_l2_indirect_reg(L2PMINTENCLR, idx_to_reg_bit(idx));
 261 }
 262
 263 static inline void cluster_pmu_set_evccntcr(u32 val)
 264 {
 265         set_l2_indirect_reg(L2PMCCNTCR, val);
 266 }
 267
 268 static inline void cluster_pmu_set_evcntcr(u32 ctr, u32 val)
 269 {
 270         set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTCR, ctr), val);
 271 }
 272
 273 static inline void cluster_pmu_set_evtyper(u32 ctr, u32 val)
 274 {
 275         set_l2_indirect_reg(reg_idx(IA_L2PMXEVTYPER, ctr), val);
 276 }
 277
 278 static void cluster_pmu_set_resr(struct cluster_pmu *cluster,
 279                                u32 event_group, u32 event_cc)
 280 {
 281         u64 field;
 282         u64 resr_val;
 283         u32 shift;
 284         unsigned long flags;
 285
 286         shift = L2PMRESR_GROUP_BITS * event_group;
 287         field = ((u64)(event_cc & L2PMRESR_GROUP_MASK) << shift);
 288
 289         spin_lock_irqsave(&cluster->pmu_lock, flags);
 290
 291         resr_val = get_l2_indirect_reg(L2PMRESR);
 292         resr_val &= ~(L2PMRESR_GROUP_MASK << shift);
 293         resr_val |= field;
 294         resr_val |= L2PMRESR_EN;
 295         set_l2_indirect_reg(L2PMRESR, resr_val);
 296
 297         spin_unlock_irqrestore(&cluster->pmu_lock, flags);
 298 }
 299
 300 /*
 301  * Hardware allows filtering of events based on the originating
 302  * CPU. Turn this off by setting filter bits to allow events from
 303  * all CPUS, subunits and ID independent events in this cluster.
 304  */
 305 static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr)
 306 {
 307         u32 val =  L2PMXEVFILTER_SUFILTER_ALL |
 308                    L2PMXEVFILTER_ORGFILTER_IDINDEP |
 309                    L2PMXEVFILTER_ORGFILTER_ALL;
 310
 311         set_l2_indirect_reg(reg_idx(IA_L2PMXEVFILTER, ctr), val);
 312 }
 313
 314 static inline u32 cluster_pmu_getreset_ovsr(void)
 315 {
 316         u32 result = get_l2_indirect_reg(L2PMOVSSET);
 317
 318         set_l2_indirect_reg(L2PMOVSCLR, result);
 319         return result;
 320 }
 321
 322 static inline bool cluster_pmu_has_overflowed(u32 ovsr)
 323 {
 324         return !!(ovsr & l2_counter_present_mask);
 325 }
 326
 327 static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr, u32 idx)
 328 {
 329         return !!(ovsr & idx_to_reg_bit(idx));
 330 }
 331
 332 static void l2_cache_event_update(struct perf_event *event)
 333 {
 334         struct hw_perf_event *hwc = &event->hw;
 335         u64 delta, prev, now;
 336         u32 idx = hwc->idx;
 337
 338         do {
 339                 prev = local64_read(&hwc->prev_count);
 340                 now = cluster_pmu_counter_get_value(idx);
 341         } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
 342
 343         /*
 344          * The cycle counter is 64-bit, but all other counters are
 345          * 32-bit, and we must handle 32-bit overflow explicitly.
 346          */
 347         delta = now - prev;
 348         if (idx != l2_cycle_ctr_idx)
 349                 delta &= 0xffffffff;
 350
 351         local64_add(delta, &event->count);
 352 }
 353
 354 static void l2_cache_cluster_set_period(struct cluster_pmu *cluster,
 355                                        struct hw_perf_event *hwc)
 356 {
 357         u32 idx = hwc->idx;
 358         u64 new;
 359
 360         /*
 361          * We limit the max period to half the max counter value so
 362          * that even in the case of extreme interrupt latency the
 363          * counter will (hopefully) not wrap past its initial value.
 364          */
 365         if (idx == l2_cycle_ctr_idx)
 366                 new = L2_CYCLE_COUNTER_RELOAD;
 367         else
 368                 new = L2_COUNTER_RELOAD;
 369
 370         local64_set(&hwc->prev_count, new);
 371         cluster_pmu_counter_set_value(idx, new);
 372 }
 373
 374 static int l2_cache_get_event_idx(struct cluster_pmu *cluster,
 375                                    struct perf_event *event)
 376 {
 377         struct hw_perf_event *hwc = &event->hw;
 378         int idx;
 379         int num_ctrs = cluster->l2cache_pmu->num_counters - 1;
 380         unsigned int group;
 381
 382         if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
 383                 if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters))
 384                         return -EAGAIN;
 385
 386                 return l2_cycle_ctr_idx;
 387         }
 388
 389         idx = find_first_zero_bit(cluster->used_counters, num_ctrs);
 390         if (idx == num_ctrs)
 391                 /* The counters are all in use. */
 392                 return -EAGAIN;
 393
 394         /*
 395          * Check for column exclusion: event column already in use by another
 396          * event. This is for events which are not in the same group.
 397          * Conflicting events in the same group are detected in event_init.
 398          */
 399         group = L2_EVT_GROUP(hwc->config_base);
 400         if (test_bit(group, cluster->used_groups))
 401                 return -EAGAIN;
 402
 403         set_bit(idx, cluster->used_counters);
 404         set_bit(group, cluster->used_groups);
 405
 406         return idx;
 407 }
 408
 409 static void l2_cache_clear_event_idx(struct cluster_pmu *cluster,
 410                                       struct perf_event *event)
 411 {
 412         struct hw_perf_event *hwc = &event->hw;
 413         int idx = hwc->idx;
 414
 415         clear_bit(idx, cluster->used_counters);
 416         if (hwc->config_base != L2CYCLE_CTR_RAW_CODE)
 417                 clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups);
 418 }
 419
 420 static irqreturn_t l2_cache_handle_irq(int irq_num, void *data)
 421 {
 422         struct cluster_pmu *cluster = data;
 423         int num_counters = cluster->l2cache_pmu->num_counters;
 424         u32 ovsr;
 425         int idx;
 426
 427         ovsr = cluster_pmu_getreset_ovsr();
 428         if (!cluster_pmu_has_overflowed(ovsr))
 429                 return IRQ_NONE;
 430
 431         for_each_set_bit(idx, cluster->used_counters, num_counters) {
 432                 struct perf_event *event = cluster->events[idx];
 433                 struct hw_perf_event *hwc;
 434
 435                 if (WARN_ON_ONCE(!event))
 436                         continue;
 437
 438                 if (!cluster_pmu_counter_has_overflowed(ovsr, idx))
 439                         continue;
 440
 441                 l2_cache_event_update(event);
 442                 hwc = &event->hw;
 443
 444                 l2_cache_cluster_set_period(cluster, hwc);
 445         }
 446
 447         return IRQ_HANDLED;
 448 }
 449
 450 /*
 451  * Implementation of abstract pmu functionality required by
 452  * the core perf events code.
 453  */
 454
 455 static void l2_cache_pmu_enable(struct pmu *pmu)
 456 {
 457         /*
 458          * Although there is only one PMU (per socket) controlling multiple
 459          * physical PMUs (per cluster), because we do not support per-task mode
 460          * each event is associated with a CPU. Each event has pmu_enable
 461          * called on its CPU, so here it is only necessary to enable the
 462          * counters for the current CPU.
 463          */
 464
 465         cluster_pmu_enable();
 466 }
 467
 468 static void l2_cache_pmu_disable(struct pmu *pmu)
 469 {
 470         cluster_pmu_disable();
 471 }
 472
 473 static int l2_cache_event_init(struct perf_event *event)
 474 {
 475         struct hw_perf_event *hwc = &event->hw;
 476         struct cluster_pmu *cluster;
 477         struct perf_event *sibling;
 478         struct l2cache_pmu *l2cache_pmu;
 479
 480         if (event->attr.type != event->pmu->type)
 481                 return -ENOENT;
 482
 483         l2cache_pmu = to_l2cache_pmu(event->pmu);
 484
 485         if (hwc->sample_period) {
 486                 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 487                                     "Sampling not supported\n");
 488                 return -EOPNOTSUPP;
 489         }
 490
 491         if (event->cpu < 0) {
 492                 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 493                                     "Per-task mode not supported\n");
 494                 return -EOPNOTSUPP;
 495         }
 496
 497         /* We cannot filter accurately so we just don't allow it. */
 498         if (event->attr.exclude_user || event->attr.exclude_kernel ||
 499             event->attr.exclude_hv || event->attr.exclude_idle) {
 500                 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 501                                     "Can't exclude execution levels\n");
 502                 return -EOPNOTSUPP;
 503         }
 504
 505         if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) ||
 506              ((event->attr.config & ~L2_EVT_MASK) != 0)) &&
 507             (event->attr.config != L2CYCLE_CTR_RAW_CODE)) {
 508                 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 509                                     "Invalid config %llx\n",
 510                                     event->attr.config);
 511                 return -EINVAL;
 512         }
 513
 514         /* Don't allow groups with mixed PMUs, except for s/w events */
 515         if (event->group_leader->pmu != event->pmu &&
 516             !is_software_event(event->group_leader)) {
 517                 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 518                          "Can't create mixed PMU group\n");
 519                 return -EINVAL;
 520         }
 521
 522         list_for_each_entry(sibling, &event->group_leader->sibling_list,
 523                             group_entry)
 524                 if (sibling->pmu != event->pmu &&
 525                     !is_software_event(sibling)) {
 526                         dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 527                                  "Can't create mixed PMU group\n");
 528                         return -EINVAL;
 529                 }
 530
 531         cluster = get_cluster_pmu(l2cache_pmu, event->cpu);
 532         if (!cluster) {
 533                 /* CPU has not been initialised */
 534                 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 535                         "CPU%d not associated with L2 cluster\n", event->cpu);
 536                 return -EINVAL;
 537         }
 538
 539         /* Ensure all events in a group are on the same cpu */
 540         if ((event->group_leader != event) &&
 541             (cluster->on_cpu != event->group_leader->cpu)) {
 542                 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 543                          "Can't create group on CPUs %d and %d",
 544                          event->cpu, event->group_leader->cpu);
 545                 return -EINVAL;
 546         }
 547
 548         if ((event != event->group_leader) &&
 549             !is_software_event(event->group_leader) &&
 550             (L2_EVT_GROUP(event->group_leader->attr.config) ==
 551              L2_EVT_GROUP(event->attr.config))) {
 552                 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 553                          "Column exclusion: conflicting events %llx %llx\n",
 554                        event->group_leader->attr.config,
 555                        event->attr.config);
 556                 return -EINVAL;
 557         }
 558
 559         list_for_each_entry(sibling, &event->group_leader->sibling_list,
 560                             group_entry) {
 561                 if ((sibling != event) &&
 562                     !is_software_event(sibling) &&
 563                     (L2_EVT_GROUP(sibling->attr.config) ==
 564                      L2_EVT_GROUP(event->attr.config))) {
 565                         dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
 566                              "Column exclusion: conflicting events %llx %llx\n",
 567                                             sibling->attr.config,
 568                                             event->attr.config);
 569                         return -EINVAL;
 570                 }
 571         }
 572
 573         hwc->idx = -1;
 574         hwc->config_base = event->attr.config;
 575
 576         /*
 577          * Ensure all events are on the same cpu so all events are in the
 578          * same cpu context, to avoid races on pmu_enable etc.
 579          */
 580         event->cpu = cluster->on_cpu;
 581
 582         return 0;
 583 }
 584
 585 static void l2_cache_event_start(struct perf_event *event, int flags)
 586 {
 587         struct cluster_pmu *cluster;
 588         struct hw_perf_event *hwc = &event->hw;
 589         int idx = hwc->idx;
 590         u32 config;
 591         u32 event_cc, event_group;
 592
 593         hwc->state = 0;
 594
 595         cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
 596
 597         l2_cache_cluster_set_period(cluster, hwc);
 598
 599         if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
 600                 cluster_pmu_set_evccntcr(0);
 601         } else {
 602                 config = hwc->config_base;
 603                 event_cc    = L2_EVT_CODE(config);
 604                 event_group = L2_EVT_GROUP(config);
 605
 606                 cluster_pmu_set_evcntcr(idx, 0);
 607                 cluster_pmu_set_evtyper(idx, event_group);
 608                 cluster_pmu_set_resr(cluster, event_group, event_cc);
 609                 cluster_pmu_set_evfilter_sys_mode(idx);
 610         }
 611
 612         cluster_pmu_counter_enable_interrupt(idx);
 613         cluster_pmu_counter_enable(idx);
 614 }
 615
 616 static void l2_cache_event_stop(struct perf_event *event, int flags)
 617 {
 618         struct hw_perf_event *hwc = &event->hw;
 619         int idx = hwc->idx;
 620
 621         if (hwc->state & PERF_HES_STOPPED)
 622                 return;
 623
 624         cluster_pmu_counter_disable_interrupt(idx);
 625         cluster_pmu_counter_disable(idx);
 626
 627         if (flags & PERF_EF_UPDATE)
 628                 l2_cache_event_update(event);
 629         hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 630 }
 631
 632 static int l2_cache_event_add(struct perf_event *event, int flags)
 633 {
 634         struct hw_perf_event *hwc = &event->hw;
 635         int idx;
 636         int err = 0;
 637         struct cluster_pmu *cluster;
 638
 639         cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
 640
 641         idx = l2_cache_get_event_idx(cluster, event);
 642         if (idx < 0)
 643                 return idx;
 644
 645         hwc->idx = idx;
 646         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 647         cluster->events[idx] = event;
 648         local64_set(&hwc->prev_count, 0);
 649
 650         if (flags & PERF_EF_START)
 651                 l2_cache_event_start(event, flags);
 652
 653         /* Propagate changes to the userspace mapping. */
 654         perf_event_update_userpage(event);
 655
 656         return err;
 657 }
 658
 659 static void l2_cache_event_del(struct perf_event *event, int flags)
 660 {
 661         struct hw_perf_event *hwc = &event->hw;
 662         struct cluster_pmu *cluster;
 663         int idx = hwc->idx;
 664
 665         cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
 666
 667         l2_cache_event_stop(event, flags | PERF_EF_UPDATE);
 668         cluster->events[idx] = NULL;
 669         l2_cache_clear_event_idx(cluster, event);
 670
 671         perf_event_update_userpage(event);
 672 }
 673
 674 static void l2_cache_event_read(struct perf_event *event)
 675 {
 676         l2_cache_event_update(event);
 677 }
 678
 679 static ssize_t l2_cache_pmu_cpumask_show(struct device *dev,
 680                                          struct device_attribute *attr,
 681                                          char *buf)
 682 {
 683         struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev));
 684
 685         return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask);
 686 }
 687
 688 static struct device_attribute l2_cache_pmu_cpumask_attr =
 689                 __ATTR(cpumask, S_IRUGO, l2_cache_pmu_cpumask_show, NULL);
 690
 691 static struct attribute *l2_cache_pmu_cpumask_attrs[] = {
 692         &l2_cache_pmu_cpumask_attr.attr,
 693         NULL,
 694 };
 695
 696 static struct attribute_group l2_cache_pmu_cpumask_group = {
 697         .attrs = l2_cache_pmu_cpumask_attrs,
 698 };
 699
 700 /* CCG format for perf RAW codes. */
 701 PMU_FORMAT_ATTR(l2_code,   "config:4-11");
 702 PMU_FORMAT_ATTR(l2_group,  "config:0-3");
 703 static struct attribute *l2_cache_pmu_formats[] = {
 704         &format_attr_l2_code.attr,
 705         &format_attr_l2_group.attr,
 706         NULL,
 707 };
 708
 709 static struct attribute_group l2_cache_pmu_format_group = {
 710         .name = "format",
 711         .attrs = l2_cache_pmu_formats,
 712 };
 713
 714 static const struct attribute_group *l2_cache_pmu_attr_grps[] = {
 715         &l2_cache_pmu_format_group,
 716         &l2_cache_pmu_cpumask_group,
 717         NULL,
 718 };
 719
 720 /*
 721  * Generic device handlers
 722  */
 723
 724 static const struct acpi_device_id l2_cache_pmu_acpi_match[] = {
 725         { "QCOM8130", },
 726         { }
 727 };
 728
 729 static int get_num_counters(void)
 730 {
 731         int val;
 732
 733         val = get_l2_indirect_reg(L2PMCR);
 734
 735         /*
 736          * Read number of counters from L2PMCR and add 1
 737          * for the cycle counter.
 738          */
 739         return ((val >> L2PMCR_NUM_EV_SHIFT) & L2PMCR_NUM_EV_MASK) + 1;
 740 }
 741
 742 static struct cluster_pmu *l2_cache_associate_cpu_with_cluster(
 743         struct l2cache_pmu *l2cache_pmu, int cpu)
 744 {
 745         u64 mpidr;
 746         int cpu_cluster_id;
 747         struct cluster_pmu *cluster = NULL;
 748
 749         /*
 750          * This assumes that the cluster_id is in MPIDR[aff1] for
 751          * single-threaded cores, and MPIDR[aff2] for multi-threaded
 752          * cores. This logic will have to be updated if this changes.
 753          */
 754         mpidr = read_cpuid_mpidr();
 755         if (mpidr & MPIDR_MT_BITMASK)
 756                 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
 757         else
 758                 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
 759
 760         list_for_each_entry(cluster, &l2cache_pmu->clusters, next) {
 761                 if (cluster->cluster_id != cpu_cluster_id)
 762                         continue;
 763
 764                 dev_info(&l2cache_pmu->pdev->dev,
 765                          "CPU%d associated with cluster %d\n", cpu,
 766                          cluster->cluster_id);
 767                 cpumask_set_cpu(cpu, &cluster->cluster_cpus);
 768                 *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster;
 769                 break;
 770         }
 771
 772         return cluster;
 773 }
 774
 775 static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
 776 {
 777         struct cluster_pmu *cluster;
 778         struct l2cache_pmu *l2cache_pmu;
 779
 780         l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
 781         cluster = get_cluster_pmu(l2cache_pmu, cpu);
 782         if (!cluster) {
 783                 /* First time this CPU has come online */
 784                 cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu);
 785                 if (!cluster) {
 786                         /* Only if broken firmware doesn't list every cluster */
 787                         WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu);
 788                         return 0;
 789                 }
 790         }
 791
 792         /* If another CPU is managing this cluster, we're done */
 793         if (cluster->on_cpu != -1)
 794                 return 0;
 795
 796         /*
 797          * All CPUs on this cluster were down, use this one.
 798          * Reset to put it into sane state.
 799          */
 800         cluster->on_cpu = cpu;
 801         cpumask_set_cpu(cpu, &l2cache_pmu->cpumask);
 802         cluster_pmu_reset();
 803
 804         WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu)));
 805         enable_irq(cluster->irq);
 806
 807         return 0;
 808 }
 809
 810 static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 811 {
 812         struct cluster_pmu *cluster;
 813         struct l2cache_pmu *l2cache_pmu;
 814         cpumask_t cluster_online_cpus;
 815         unsigned int target;
 816
 817         l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
 818         cluster = get_cluster_pmu(l2cache_pmu, cpu);
 819         if (!cluster)
 820                 return 0;
 821
 822         /* If this CPU is not managing the cluster, we're done */
 823         if (cluster->on_cpu != cpu)
 824                 return 0;
 825
 826         /* Give up ownership of cluster */
 827         cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask);
 828         cluster->on_cpu = -1;
 829
 830         /* Any other CPU for this cluster which is still online */
 831         cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus,
 832                     cpu_online_mask);
 833         target = cpumask_any_but(&cluster_online_cpus, cpu);
 834         if (target >= nr_cpu_ids) {
 835                 disable_irq(cluster->irq);
 836                 return 0;
 837         }
 838
 839         perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target);
 840         cluster->on_cpu = target;
 841         cpumask_set_cpu(target, &l2cache_pmu->cpumask);
 842         WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target)));
 843
 844         return 0;
 845 }
 846
 847 static int l2_cache_pmu_probe_cluster(struct device *dev, void *data)
 848 {
 849         struct platform_device *pdev = to_platform_device(dev->parent);
 850         struct platform_device *sdev = to_platform_device(dev);
 851         struct l2cache_pmu *l2cache_pmu = data;
 852         struct cluster_pmu *cluster;
 853         struct acpi_device *device;
 854         unsigned long fw_cluster_id;
 855         int err;
 856         int irq;
 857
 858         if (acpi_bus_get_device(ACPI_HANDLE(dev), &device))
 859                 return -ENODEV;
 860
 861         if (kstrtoul(device->pnp.unique_id, 10, &fw_cluster_id) < 0) {
 862                 dev_err(&pdev->dev, "unable to read ACPI uid\n");
 863                 return -ENODEV;
 864         }
 865
 866         cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL);
 867         if (!cluster)
 868                 return -ENOMEM;
 869
 870         INIT_LIST_HEAD(&cluster->next);
 871         list_add(&cluster->next, &l2cache_pmu->clusters);
 872         cluster->cluster_id = fw_cluster_id;
 873
 874         irq = platform_get_irq(sdev, 0);
 875         if (irq < 0) {
 876                 dev_err(&pdev->dev,
 877                         "Failed to get valid irq for cluster %ld\n",
 878                         fw_cluster_id);
 879                 return irq;
 880         }
 881         irq_set_status_flags(irq, IRQ_NOAUTOEN);
 882         cluster->irq = irq;
 883
 884         cluster->l2cache_pmu = l2cache_pmu;
 885         cluster->on_cpu = -1;
 886
 887         err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq,
 888                                IRQF_NOBALANCING | IRQF_NO_THREAD,
 889                                "l2-cache-pmu", cluster);
 890         if (err) {
 891                 dev_err(&pdev->dev,
 892                         "Unable to request IRQ%d for L2 PMU counters\n", irq);
 893                 return err;
 894         }
 895
 896         dev_info(&pdev->dev,
 897                 "Registered L2 cache PMU cluster %ld\n", fw_cluster_id);
 898
 899         spin_lock_init(&cluster->pmu_lock);
 900
 901         l2cache_pmu->num_pmus++;
 902
 903         return 0;
 904 }
 905
 906 static int l2_cache_pmu_probe(struct platform_device *pdev)
 907 {
 908         int err;
 909         struct l2cache_pmu *l2cache_pmu;
 910
 911         l2cache_pmu =
 912                 devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL);
 913         if (!l2cache_pmu)
 914                 return -ENOMEM;
 915
 916         INIT_LIST_HEAD(&l2cache_pmu->clusters);
 917
 918         platform_set_drvdata(pdev, l2cache_pmu);
 919         l2cache_pmu->pmu = (struct pmu) {
 920                 /* suffix is instance id for future use with multiple sockets */
 921                 .name           = "l2cache_0",
 922                 .task_ctx_nr    = perf_invalid_context,
 923                 .pmu_enable     = l2_cache_pmu_enable,
 924                 .pmu_disable    = l2_cache_pmu_disable,
 925                 .event_init     = l2_cache_event_init,
 926                 .add            = l2_cache_event_add,
 927                 .del            = l2_cache_event_del,
 928                 .start          = l2_cache_event_start,
 929                 .stop           = l2_cache_event_stop,
 930                 .read           = l2_cache_event_read,
 931                 .attr_groups    = l2_cache_pmu_attr_grps,
 932         };
 933
 934         l2cache_pmu->num_counters = get_num_counters();
 935         l2cache_pmu->pdev = pdev;
 936         l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev,
 937                                                      struct cluster_pmu *);
 938         if (!l2cache_pmu->pmu_cluster)
 939                 return -ENOMEM;
 940
 941         l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1;
 942         l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) |
 943                 BIT(L2CYCLE_CTR_BIT);
 944
 945         cpumask_clear(&l2cache_pmu->cpumask);
 946
 947         /* Read cluster info and initialize each cluster */
 948         err = device_for_each_child(&pdev->dev, l2cache_pmu,
 949                                     l2_cache_pmu_probe_cluster);
 950         if (err)
 951                 return err;
 952
 953         if (l2cache_pmu->num_pmus == 0) {
 954                 dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n");
 955                 return -ENODEV;
 956         }
 957
 958         err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
 959                                        &l2cache_pmu->node);
 960         if (err) {
 961                 dev_err(&pdev->dev, "Error %d registering hotplug", err);
 962                 return err;
 963         }
 964
 965         err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1);
 966         if (err) {
 967                 dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err);
 968                 goto out_unregister;
 969         }
 970
 971         dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n",
 972                  l2cache_pmu->num_pmus);
 973
 974         return err;
 975
 976 out_unregister:
 977         cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
 978                                     &l2cache_pmu->node);
 979         return err;
 980 }
 981
 982 static int l2_cache_pmu_remove(struct platform_device *pdev)
 983 {
 984         struct l2cache_pmu *l2cache_pmu =
 985                 to_l2cache_pmu(platform_get_drvdata(pdev));
 986
 987         perf_pmu_unregister(&l2cache_pmu->pmu);
 988         cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
 989                                     &l2cache_pmu->node);
 990         return 0;
 991 }
 992
 993 static struct platform_driver l2_cache_pmu_driver = {
 994         .driver = {
 995                 .name = "qcom-l2cache-pmu",
 996                 .acpi_match_table = ACPI_PTR(l2_cache_pmu_acpi_match),
 997         },
 998         .probe = l2_cache_pmu_probe,
 999         .remove = l2_cache_pmu_remove,
1000 };
1001
1002 static int __init register_l2_cache_pmu_driver(void)
1003 {
1004         int err;
1005
1006         err = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
1007                                       "AP_PERF_ARM_QCOM_L2_ONLINE",
1008                                       l2cache_pmu_online_cpu,
1009                                       l2cache_pmu_offline_cpu);
1010         if (err)
1011                 return err;
1012
1013         return platform_driver_register(&l2_cache_pmu_driver);
1014 }
1015 device_initcall(register_l2_cache_pmu_driver);