arch/arm/mach-imx/mmdc.c

   1 /*
   2  * Copyright 2011,2016 Freescale Semiconductor, Inc.
   3  * Copyright 2011 Linaro Ltd.
   4  *
   5  * The code contained herein is licensed under the GNU General Public
   6  * License. You may obtain a copy of the GNU General Public License
   7  * Version 2 or later at the following locations:
   8  *
   9  * http://www.opensource.org/licenses/gpl-license.html
  10  * http://www.gnu.org/copyleft/gpl.html
  11  */
  12
  13 #include <linux/hrtimer.h>
  14 #include <linux/init.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/io.h>
  17 #include <linux/module.h>
  18 #include <linux/of.h>
  19 #include <linux/of_address.h>
  20 #include <linux/of_device.h>
  21 #include <linux/perf_event.h>
  22 #include <linux/slab.h>
  23
  24 #include "common.h"
  25
  26 #define MMDC_MAPSR              0x404
  27 #define BP_MMDC_MAPSR_PSD       0
  28 #define BP_MMDC_MAPSR_PSS       4
  29
  30 #define MMDC_MDMISC             0x18
  31 #define BM_MMDC_MDMISC_DDR_TYPE 0x18
  32 #define BP_MMDC_MDMISC_DDR_TYPE 0x3
  33
  34 #define TOTAL_CYCLES            0x0
  35 #define BUSY_CYCLES             0x1
  36 #define READ_ACCESSES           0x2
  37 #define WRITE_ACCESSES          0x3
  38 #define READ_BYTES              0x4
  39 #define WRITE_BYTES             0x5
  40
  41 /* Enables, resets, freezes, overflow profiling*/
  42 #define DBG_DIS                 0x0
  43 #define DBG_EN                  0x1
  44 #define DBG_RST                 0x2
  45 #define PRF_FRZ                 0x4
  46 #define CYC_OVF                 0x8
  47 #define PROFILE_SEL             0x10
  48
  49 #define MMDC_MADPCR0    0x410
  50 #define MMDC_MADPSR0    0x418
  51 #define MMDC_MADPSR1    0x41C
  52 #define MMDC_MADPSR2    0x420
  53 #define MMDC_MADPSR3    0x424
  54 #define MMDC_MADPSR4    0x428
  55 #define MMDC_MADPSR5    0x42C
  56
  57 #define MMDC_NUM_COUNTERS       6
  58
  59 #define MMDC_FLAG_PROFILE_SEL   0x1
  60
  61 #define to_mmdc_pmu(p) container_of(p, struct mmdc_pmu, pmu)
  62
  63 static enum cpuhp_state cpuhp_mmdc_state;
  64 static int ddr_type;
  65
  66 struct fsl_mmdc_devtype_data {
  67         unsigned int flags;
  68 };
  69
  70 static const struct fsl_mmdc_devtype_data imx6q_data = {
  71 };
  72
  73 static const struct fsl_mmdc_devtype_data imx6qp_data = {
  74         .flags = MMDC_FLAG_PROFILE_SEL,
  75 };
  76
  77 static const struct of_device_id imx_mmdc_dt_ids[] = {
  78         { .compatible = "fsl,imx6q-mmdc", .data = (void *)&imx6q_data},
  79         { .compatible = "fsl,imx6qp-mmdc", .data = (void *)&imx6qp_data},
  80         { /* sentinel */ }
  81 };
  82
  83 #ifdef CONFIG_PERF_EVENTS
  84
  85 static DEFINE_IDA(mmdc_ida);
  86
  87 PMU_EVENT_ATTR_STRING(total-cycles, mmdc_pmu_total_cycles, "event=0x00")
  88 PMU_EVENT_ATTR_STRING(busy-cycles, mmdc_pmu_busy_cycles, "event=0x01")
  89 PMU_EVENT_ATTR_STRING(read-accesses, mmdc_pmu_read_accesses, "event=0x02")
  90 PMU_EVENT_ATTR_STRING(write-accesses, mmdc_pmu_write_accesses, "config=0x03")
  91 PMU_EVENT_ATTR_STRING(read-bytes, mmdc_pmu_read_bytes, "event=0x04")
  92 PMU_EVENT_ATTR_STRING(read-bytes.unit, mmdc_pmu_read_bytes_unit, "MB");
  93 PMU_EVENT_ATTR_STRING(read-bytes.scale, mmdc_pmu_read_bytes_scale, "0.000001");
  94 PMU_EVENT_ATTR_STRING(write-bytes, mmdc_pmu_write_bytes, "event=0x05")
  95 PMU_EVENT_ATTR_STRING(write-bytes.unit, mmdc_pmu_write_bytes_unit, "MB");
  96 PMU_EVENT_ATTR_STRING(write-bytes.scale, mmdc_pmu_write_bytes_scale, "0.000001");
  97
  98 struct mmdc_pmu {
  99         struct pmu pmu;
 100         void __iomem *mmdc_base;
 101         cpumask_t cpu;
 102         struct hrtimer hrtimer;
 103         unsigned int active_events;
 104         struct device *dev;
 105         struct perf_event *mmdc_events[MMDC_NUM_COUNTERS];
 106         struct hlist_node node;
 107         struct fsl_mmdc_devtype_data *devtype_data;
 108 };
 109
 110 /*
 111  * Polling period is set to one second, overflow of total-cycles (the fastest
 112  * increasing counter) takes ten seconds so one second is safe
 113  */
 114 static unsigned int mmdc_pmu_poll_period_us = 1000000;
 115
 116 module_param_named(pmu_pmu_poll_period_us, mmdc_pmu_poll_period_us, uint,
 117                 S_IRUGO | S_IWUSR);
 118
 119 static ktime_t mmdc_pmu_timer_period(void)
 120 {
 121         return ns_to_ktime((u64)mmdc_pmu_poll_period_us * 1000);
 122 }
 123
 124 static ssize_t mmdc_pmu_cpumask_show(struct device *dev,
 125                 struct device_attribute *attr, char *buf)
 126 {
 127         struct mmdc_pmu *pmu_mmdc = dev_get_drvdata(dev);
 128
 129         return cpumap_print_to_pagebuf(true, buf, &pmu_mmdc->cpu);
 130 }
 131
 132 static struct device_attribute mmdc_pmu_cpumask_attr =
 133         __ATTR(cpumask, S_IRUGO, mmdc_pmu_cpumask_show, NULL);
 134
 135 static struct attribute *mmdc_pmu_cpumask_attrs[] = {
 136         &mmdc_pmu_cpumask_attr.attr,
 137         NULL,
 138 };
 139
 140 static struct attribute_group mmdc_pmu_cpumask_attr_group = {
 141         .attrs = mmdc_pmu_cpumask_attrs,
 142 };
 143
 144 static struct attribute *mmdc_pmu_events_attrs[] = {
 145         &mmdc_pmu_total_cycles.attr.attr,
 146         &mmdc_pmu_busy_cycles.attr.attr,
 147         &mmdc_pmu_read_accesses.attr.attr,
 148         &mmdc_pmu_write_accesses.attr.attr,
 149         &mmdc_pmu_read_bytes.attr.attr,
 150         &mmdc_pmu_read_bytes_unit.attr.attr,
 151         &mmdc_pmu_read_bytes_scale.attr.attr,
 152         &mmdc_pmu_write_bytes.attr.attr,
 153         &mmdc_pmu_write_bytes_unit.attr.attr,
 154         &mmdc_pmu_write_bytes_scale.attr.attr,
 155         NULL,
 156 };
 157
 158 static struct attribute_group mmdc_pmu_events_attr_group = {
 159         .name = "events",
 160         .attrs = mmdc_pmu_events_attrs,
 161 };
 162
 163 PMU_FORMAT_ATTR(event, "config:0-63");
 164 static struct attribute *mmdc_pmu_format_attrs[] = {
 165         &format_attr_event.attr,
 166         NULL,
 167 };
 168
 169 static struct attribute_group mmdc_pmu_format_attr_group = {
 170         .name = "format",
 171         .attrs = mmdc_pmu_format_attrs,
 172 };
 173
 174 static const struct attribute_group *attr_groups[] = {
 175         &mmdc_pmu_events_attr_group,
 176         &mmdc_pmu_format_attr_group,
 177         &mmdc_pmu_cpumask_attr_group,
 178         NULL,
 179 };
 180
 181 static u32 mmdc_pmu_read_counter(struct mmdc_pmu *pmu_mmdc, int cfg)
 182 {
 183         void __iomem *mmdc_base, *reg;
 184
 185         mmdc_base = pmu_mmdc->mmdc_base;
 186
 187         switch (cfg) {
 188         case TOTAL_CYCLES:
 189                 reg = mmdc_base + MMDC_MADPSR0;
 190                 break;
 191         case BUSY_CYCLES:
 192                 reg = mmdc_base + MMDC_MADPSR1;
 193                 break;
 194         case READ_ACCESSES:
 195                 reg = mmdc_base + MMDC_MADPSR2;
 196                 break;
 197         case WRITE_ACCESSES:
 198                 reg = mmdc_base + MMDC_MADPSR3;
 199                 break;
 200         case READ_BYTES:
 201                 reg = mmdc_base + MMDC_MADPSR4;
 202                 break;
 203         case WRITE_BYTES:
 204                 reg = mmdc_base + MMDC_MADPSR5;
 205                 break;
 206         default:
 207                 return WARN_ONCE(1,
 208                         "invalid configuration %d for mmdc counter", cfg);
 209         }
 210         return readl(reg);
 211 }
 212
 213 static int mmdc_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 214 {
 215         struct mmdc_pmu *pmu_mmdc = hlist_entry_safe(node, struct mmdc_pmu, node);
 216         int target;
 217
 218         if (!cpumask_test_and_clear_cpu(cpu, &pmu_mmdc->cpu))
 219                 return 0;
 220
 221         target = cpumask_any_but(cpu_online_mask, cpu);
 222         if (target >= nr_cpu_ids)
 223                 return 0;
 224
 225         perf_pmu_migrate_context(&pmu_mmdc->pmu, cpu, target);
 226         cpumask_set_cpu(target, &pmu_mmdc->cpu);
 227
 228         return 0;
 229 }
 230
 231 static bool mmdc_pmu_group_event_is_valid(struct perf_event *event,
 232                                           struct pmu *pmu,
 233                                           unsigned long *used_counters)
 234 {
 235         int cfg = event->attr.config;
 236
 237         if (is_software_event(event))
 238                 return true;
 239
 240         if (event->pmu != pmu)
 241                 return false;
 242
 243         return !test_and_set_bit(cfg, used_counters);
 244 }
 245
 246 /*
 247  * Each event has a single fixed-purpose counter, so we can only have a
 248  * single active event for each at any point in time. Here we just check
 249  * for duplicates, and rely on mmdc_pmu_event_init to verify that the HW
 250  * event numbers are valid.
 251  */
 252 static bool mmdc_pmu_group_is_valid(struct perf_event *event)
 253 {
 254         struct pmu *pmu = event->pmu;
 255         struct perf_event *leader = event->group_leader;
 256         struct perf_event *sibling;
 257         unsigned long counter_mask = 0;
 258
 259         set_bit(leader->attr.config, &counter_mask);
 260
 261         if (event != leader) {
 262                 if (!mmdc_pmu_group_event_is_valid(event, pmu, &counter_mask))
 263                         return false;
 264         }
 265
 266         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
 267                 if (!mmdc_pmu_group_event_is_valid(sibling, pmu, &counter_mask))
 268                         return false;
 269         }
 270
 271         return true;
 272 }
 273
 274 static int mmdc_pmu_event_init(struct perf_event *event)
 275 {
 276         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 277         int cfg = event->attr.config;
 278
 279         if (event->attr.type != event->pmu->type)
 280                 return -ENOENT;
 281
 282         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 283                 return -EOPNOTSUPP;
 284
 285         if (event->cpu < 0) {
 286                 dev_warn(pmu_mmdc->dev, "Can't provide per-task data!\n");
 287                 return -EOPNOTSUPP;
 288         }
 289
 290         if (event->attr.exclude_user            ||
 291                         event->attr.exclude_kernel      ||
 292                         event->attr.exclude_hv          ||
 293                         event->attr.exclude_idle        ||
 294                         event->attr.exclude_host        ||
 295                         event->attr.exclude_guest       ||
 296                         event->attr.sample_period)
 297                 return -EINVAL;
 298
 299         if (cfg < 0 || cfg >= MMDC_NUM_COUNTERS)
 300                 return -EINVAL;
 301
 302         if (!mmdc_pmu_group_is_valid(event))
 303                 return -EINVAL;
 304
 305         event->cpu = cpumask_first(&pmu_mmdc->cpu);
 306         return 0;
 307 }
 308
 309 static void mmdc_pmu_event_update(struct perf_event *event)
 310 {
 311         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 312         struct hw_perf_event *hwc = &event->hw;
 313         u64 delta, prev_raw_count, new_raw_count;
 314
 315         do {
 316                 prev_raw_count = local64_read(&hwc->prev_count);
 317                 new_raw_count = mmdc_pmu_read_counter(pmu_mmdc,
 318                                                       event->attr.config);
 319         } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 320                 new_raw_count) != prev_raw_count);
 321
 322         delta = (new_raw_count - prev_raw_count) & 0xFFFFFFFF;
 323
 324         local64_add(delta, &event->count);
 325 }
 326
 327 static void mmdc_pmu_event_start(struct perf_event *event, int flags)
 328 {
 329         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 330         struct hw_perf_event *hwc = &event->hw;
 331         void __iomem *mmdc_base, *reg;
 332         u32 val;
 333
 334         mmdc_base = pmu_mmdc->mmdc_base;
 335         reg = mmdc_base + MMDC_MADPCR0;
 336
 337         /*
 338          * hrtimer is required because mmdc does not provide an interrupt so
 339          * polling is necessary
 340          */
 341         hrtimer_start(&pmu_mmdc->hrtimer, mmdc_pmu_timer_period(),
 342                         HRTIMER_MODE_REL_PINNED);
 343
 344         local64_set(&hwc->prev_count, 0);
 345
 346         writel(DBG_RST, reg);
 347
 348         val = DBG_EN;
 349         if (pmu_mmdc->devtype_data->flags & MMDC_FLAG_PROFILE_SEL)
 350                 val |= PROFILE_SEL;
 351
 352         writel(val, reg);
 353 }
 354
 355 static int mmdc_pmu_event_add(struct perf_event *event, int flags)
 356 {
 357         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 358         struct hw_perf_event *hwc = &event->hw;
 359
 360         int cfg = event->attr.config;
 361
 362         if (flags & PERF_EF_START)
 363                 mmdc_pmu_event_start(event, flags);
 364
 365         if (pmu_mmdc->mmdc_events[cfg] != NULL)
 366                 return -EAGAIN;
 367
 368         pmu_mmdc->mmdc_events[cfg] = event;
 369         pmu_mmdc->active_events++;
 370
 371         local64_set(&hwc->prev_count, mmdc_pmu_read_counter(pmu_mmdc, cfg));
 372
 373         return 0;
 374 }
 375
 376 static void mmdc_pmu_event_stop(struct perf_event *event, int flags)
 377 {
 378         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 379         void __iomem *mmdc_base, *reg;
 380
 381         mmdc_base = pmu_mmdc->mmdc_base;
 382         reg = mmdc_base + MMDC_MADPCR0;
 383
 384         writel(PRF_FRZ, reg);
 385         mmdc_pmu_event_update(event);
 386 }
 387
 388 static void mmdc_pmu_event_del(struct perf_event *event, int flags)
 389 {
 390         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 391         int cfg = event->attr.config;
 392
 393         pmu_mmdc->mmdc_events[cfg] = NULL;
 394         pmu_mmdc->active_events--;
 395
 396         if (pmu_mmdc->active_events == 0)
 397                 hrtimer_cancel(&pmu_mmdc->hrtimer);
 398
 399         mmdc_pmu_event_stop(event, PERF_EF_UPDATE);
 400 }
 401
 402 static void mmdc_pmu_overflow_handler(struct mmdc_pmu *pmu_mmdc)
 403 {
 404         int i;
 405
 406         for (i = 0; i < MMDC_NUM_COUNTERS; i++) {
 407                 struct perf_event *event = pmu_mmdc->mmdc_events[i];
 408
 409                 if (event)
 410                         mmdc_pmu_event_update(event);
 411         }
 412 }
 413
 414 static enum hrtimer_restart mmdc_pmu_timer_handler(struct hrtimer *hrtimer)
 415 {
 416         struct mmdc_pmu *pmu_mmdc = container_of(hrtimer, struct mmdc_pmu,
 417                         hrtimer);
 418
 419         mmdc_pmu_overflow_handler(pmu_mmdc);
 420         hrtimer_forward_now(hrtimer, mmdc_pmu_timer_period());
 421
 422         return HRTIMER_RESTART;
 423 }
 424
 425 static int mmdc_pmu_init(struct mmdc_pmu *pmu_mmdc,
 426                 void __iomem *mmdc_base, struct device *dev)
 427 {
 428         int mmdc_num;
 429
 430         *pmu_mmdc = (struct mmdc_pmu) {
 431                 .pmu = (struct pmu) {
 432                         .task_ctx_nr    = perf_invalid_context,
 433                         .attr_groups    = attr_groups,
 434                         .event_init     = mmdc_pmu_event_init,
 435                         .add            = mmdc_pmu_event_add,
 436                         .del            = mmdc_pmu_event_del,
 437                         .start          = mmdc_pmu_event_start,
 438                         .stop           = mmdc_pmu_event_stop,
 439                         .read           = mmdc_pmu_event_update,
 440                 },
 441                 .mmdc_base = mmdc_base,
 442                 .dev = dev,
 443                 .active_events = 0,
 444         };
 445
 446         mmdc_num = ida_simple_get(&mmdc_ida, 0, 0, GFP_KERNEL);
 447
 448         return mmdc_num;
 449 }
 450
 451 static int imx_mmdc_remove(struct platform_device *pdev)
 452 {
 453         struct mmdc_pmu *pmu_mmdc = platform_get_drvdata(pdev);
 454
 455         cpuhp_state_remove_instance_nocalls(cpuhp_mmdc_state, &pmu_mmdc->node);
 456         perf_pmu_unregister(&pmu_mmdc->pmu);
 457         kfree(pmu_mmdc);
 458         return 0;
 459 }
 460
 461 static int imx_mmdc_perf_init(struct platform_device *pdev, void __iomem *mmdc_base)
 462 {
 463         struct mmdc_pmu *pmu_mmdc;
 464         char *name;
 465         int mmdc_num;
 466         int ret;
 467         const struct of_device_id *of_id =
 468                 of_match_device(imx_mmdc_dt_ids, &pdev->dev);
 469
 470         pmu_mmdc = kzalloc(sizeof(*pmu_mmdc), GFP_KERNEL);
 471         if (!pmu_mmdc) {
 472                 pr_err("failed to allocate PMU device!\n");
 473                 return -ENOMEM;
 474         }
 475
 476         /* The first instance registers the hotplug state */
 477         if (!cpuhp_mmdc_state) {
 478                 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 479                                               "perf/arm/mmdc:online", NULL,
 480                                               mmdc_pmu_offline_cpu);
 481                 if (ret < 0) {
 482                         pr_err("cpuhp_setup_state_multi failed\n");
 483                         goto pmu_free;
 484                 }
 485                 cpuhp_mmdc_state = ret;
 486         }
 487
 488         mmdc_num = mmdc_pmu_init(pmu_mmdc, mmdc_base, &pdev->dev);
 489         if (mmdc_num == 0)
 490                 name = "mmdc";
 491         else
 492                 name = devm_kasprintf(&pdev->dev,
 493                                 GFP_KERNEL, "mmdc%d", mmdc_num);
 494
 495         pmu_mmdc->devtype_data = (struct fsl_mmdc_devtype_data *)of_id->data;
 496
 497         hrtimer_init(&pmu_mmdc->hrtimer, CLOCK_MONOTONIC,
 498                         HRTIMER_MODE_REL);
 499         pmu_mmdc->hrtimer.function = mmdc_pmu_timer_handler;
 500
 501         cpumask_set_cpu(raw_smp_processor_id(), &pmu_mmdc->cpu);
 502
 503         /* Register the pmu instance for cpu hotplug */
 504         cpuhp_state_add_instance_nocalls(cpuhp_mmdc_state, &pmu_mmdc->node);
 505
 506         ret = perf_pmu_register(&(pmu_mmdc->pmu), name, -1);
 507         if (ret)
 508                 goto pmu_register_err;
 509
 510         platform_set_drvdata(pdev, pmu_mmdc);
 511         return 0;
 512
 513 pmu_register_err:
 514         pr_warn("MMDC Perf PMU failed (%d), disabled\n", ret);
 515         cpuhp_state_remove_instance_nocalls(cpuhp_mmdc_state, &pmu_mmdc->node);
 516         hrtimer_cancel(&pmu_mmdc->hrtimer);
 517 pmu_free:
 518         kfree(pmu_mmdc);
 519         return ret;
 520 }
 521
 522 #else
 523 #define imx_mmdc_remove NULL
 524 #define imx_mmdc_perf_init(pdev, mmdc_base) 0
 525 #endif
 526
 527 static int imx_mmdc_probe(struct platform_device *pdev)
 528 {
 529         struct device_node *np = pdev->dev.of_node;
 530         void __iomem *mmdc_base, *reg;
 531         u32 val;
 532         int timeout = 0x400;
 533
 534         mmdc_base = of_iomap(np, 0);
 535         WARN_ON(!mmdc_base);
 536
 537         reg = mmdc_base + MMDC_MDMISC;
 538         /* Get ddr type */
 539         val = readl_relaxed(reg);
 540         ddr_type = (val & BM_MMDC_MDMISC_DDR_TYPE) >>
 541                  BP_MMDC_MDMISC_DDR_TYPE;
 542
 543         reg = mmdc_base + MMDC_MAPSR;
 544
 545         /* Enable automatic power saving */
 546         val = readl_relaxed(reg);
 547         val &= ~(1 << BP_MMDC_MAPSR_PSD);
 548         writel_relaxed(val, reg);
 549
 550         /* Ensure it's successfully enabled */
 551         while (!(readl_relaxed(reg) & 1 << BP_MMDC_MAPSR_PSS) && --timeout)
 552                 cpu_relax();
 553
 554         if (unlikely(!timeout)) {
 555                 pr_warn("%s: failed to enable automatic power saving\n",
 556                         __func__);
 557                 return -EBUSY;
 558         }
 559
 560         return imx_mmdc_perf_init(pdev, mmdc_base);
 561 }
 562
 563 int imx_mmdc_get_ddr_type(void)
 564 {
 565         return ddr_type;
 566 }
 567
 568 static struct platform_driver imx_mmdc_driver = {
 569         .driver         = {
 570                 .name   = "imx-mmdc",
 571                 .of_match_table = imx_mmdc_dt_ids,
 572         },
 573         .probe          = imx_mmdc_probe,
 574         .remove         = imx_mmdc_remove,
 575 };
 576
 577 static int __init imx_mmdc_init(void)
 578 {
 579         return platform_driver_register(&imx_mmdc_driver);
 580 }
 581 postcore_initcall(imx_mmdc_init);