drivers/edac/thunderx_edac.c

   1 /*
   2  * Cavium ThunderX memory controller kernel module
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
   9  *
  10  */
  11
  12 #include <linux/module.h>
  13 #include <linux/pci.h>
  14 #include <linux/edac.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/string.h>
  17 #include <linux/stop_machine.h>
  18 #include <linux/delay.h>
  19 #include <linux/sizes.h>
  20 #include <linux/atomic.h>
  21 #include <linux/bitfield.h>
  22 #include <linux/circ_buf.h>
  23
  24 #include <asm/page.h>
  25
  26 #include "edac_module.h"
  27
  28 #define phys_to_pfn(phys)       (PFN_DOWN(phys))
  29
  30 #define THUNDERX_NODE           GENMASK(45, 44)
  31
  32 enum {
  33         ERR_CORRECTED   = 1,
  34         ERR_UNCORRECTED = 2,
  35         ERR_UNKNOWN     = 3,
  36 };
  37
  38 #define MAX_SYNDROME_REGS 4
  39
  40 struct error_syndrome {
  41         u64 reg[MAX_SYNDROME_REGS];
  42 };
  43
  44 struct error_descr {
  45         int     type;
  46         u64     mask;
  47         char    *descr;
  48 };
  49
  50 static void decode_register(char *str, size_t size,
  51                            const struct error_descr *descr,
  52                            const uint64_t reg)
  53 {
  54         int ret = 0;
  55
  56         while (descr->type && descr->mask && descr->descr) {
  57                 if (reg & descr->mask) {
  58                         ret = snprintf(str, size, "\n\t%s, %s",
  59                                        descr->type == ERR_CORRECTED ?
  60                                          "Corrected" : "Uncorrected",
  61                                        descr->descr);
  62                         str += ret;
  63                         size -= ret;
  64                 }
  65                 descr++;
  66         }
  67 }
  68
  69 static unsigned long get_bits(unsigned long data, int pos, int width)
  70 {
  71         return (data >> pos) & ((1 << width) - 1);
  72 }
  73
  74 #define L2C_CTL                 0x87E080800000
  75 #define L2C_CTL_DISIDXALIAS     BIT(0)
  76
  77 #define PCI_DEVICE_ID_THUNDER_LMC 0xa022
  78
  79 #define LMC_FADR                0x20
  80 #define LMC_FADR_FDIMM(x)       ((x >> 37) & 0x1)
  81 #define LMC_FADR_FBUNK(x)       ((x >> 36) & 0x1)
  82 #define LMC_FADR_FBANK(x)       ((x >> 32) & 0xf)
  83 #define LMC_FADR_FROW(x)        ((x >> 14) & 0xffff)
  84 #define LMC_FADR_FCOL(x)        ((x >> 0) & 0x1fff)
  85
  86 #define LMC_NXM_FADR            0x28
  87 #define LMC_ECC_SYND            0x38
  88
  89 #define LMC_ECC_PARITY_TEST     0x108
  90
  91 #define LMC_INT_W1S             0x150
  92
  93 #define LMC_INT_ENA_W1C         0x158
  94 #define LMC_INT_ENA_W1S         0x160
  95
  96 #define LMC_CONFIG              0x188
  97
  98 #define LMC_CONFIG_BG2          BIT(62)
  99 #define LMC_CONFIG_RANK_ENA     BIT(42)
 100 #define LMC_CONFIG_PBANK_LSB(x) (((x) >> 5) & 0xF)
 101 #define LMC_CONFIG_ROW_LSB(x)   (((x) >> 2) & 0x7)
 102
 103 #define LMC_CONTROL             0x190
 104 #define LMC_CONTROL_XOR_BANK    BIT(16)
 105
 106 #define LMC_INT                 0x1F0
 107
 108 #define LMC_INT_DDR_ERR         BIT(11)
 109 #define LMC_INT_DED_ERR         (0xFUL << 5)
 110 #define LMC_INT_SEC_ERR         (0xFUL << 1)
 111 #define LMC_INT_NXM_WR_MASK     BIT(0)
 112
 113 #define LMC_DDR_PLL_CTL         0x258
 114 #define LMC_DDR_PLL_CTL_DDR4    BIT(29)
 115
 116 #define LMC_FADR_SCRAMBLED      0x330
 117
 118 #define LMC_INT_UE              (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \
 119                                  LMC_INT_NXM_WR_MASK)
 120
 121 #define LMC_INT_CE              (LMC_INT_SEC_ERR)
 122
 123 static const struct error_descr lmc_errors[] = {
 124         {
 125                 .type  = ERR_CORRECTED,
 126                 .mask  = LMC_INT_SEC_ERR,
 127                 .descr = "Single-bit ECC error",
 128         },
 129         {
 130                 .type  = ERR_UNCORRECTED,
 131                 .mask  = LMC_INT_DDR_ERR,
 132                 .descr = "DDR chip error",
 133         },
 134         {
 135                 .type  = ERR_UNCORRECTED,
 136                 .mask  = LMC_INT_DED_ERR,
 137                 .descr = "Double-bit ECC error",
 138         },
 139         {
 140                 .type = ERR_UNCORRECTED,
 141                 .mask = LMC_INT_NXM_WR_MASK,
 142                 .descr = "Non-existent memory write",
 143         },
 144         {0, 0, NULL},
 145 };
 146
 147 #define LMC_INT_EN_DDR_ERROR_ALERT_ENA  BIT(5)
 148 #define LMC_INT_EN_DLCRAM_DED_ERR       BIT(4)
 149 #define LMC_INT_EN_DLCRAM_SEC_ERR       BIT(3)
 150 #define LMC_INT_INTR_DED_ENA            BIT(2)
 151 #define LMC_INT_INTR_SEC_ENA            BIT(1)
 152 #define LMC_INT_INTR_NXM_WR_ENA         BIT(0)
 153
 154 #define LMC_INT_ENA_ALL                 GENMASK(5, 0)
 155
 156 #define LMC_DDR_PLL_CTL         0x258
 157 #define LMC_DDR_PLL_CTL_DDR4    BIT(29)
 158
 159 #define LMC_CONTROL             0x190
 160 #define LMC_CONTROL_RDIMM       BIT(0)
 161
 162 #define LMC_SCRAM_FADR          0x330
 163
 164 #define LMC_CHAR_MASK0          0x228
 165 #define LMC_CHAR_MASK2          0x238
 166
 167 #define RING_ENTRIES    8
 168
 169 struct debugfs_entry {
 170         const char *name;
 171         umode_t mode;
 172         const struct file_operations fops;
 173 };
 174
 175 struct lmc_err_ctx {
 176         u64 reg_int;
 177         u64 reg_fadr;
 178         u64 reg_nxm_fadr;
 179         u64 reg_scram_fadr;
 180         u64 reg_ecc_synd;
 181 };
 182
 183 struct thunderx_lmc {
 184         void __iomem *regs;
 185         struct pci_dev *pdev;
 186         struct msix_entry msix_ent;
 187
 188         atomic_t ecc_int;
 189
 190         u64 mask0;
 191         u64 mask2;
 192         u64 parity_test;
 193         u64 node;
 194
 195         int xbits;
 196         int bank_width;
 197         int pbank_lsb;
 198         int dimm_lsb;
 199         int rank_lsb;
 200         int bank_lsb;
 201         int row_lsb;
 202         int col_hi_lsb;
 203
 204         int xor_bank;
 205         int l2c_alias;
 206
 207         struct page *mem;
 208
 209         struct lmc_err_ctx err_ctx[RING_ENTRIES];
 210         unsigned long ring_head;
 211         unsigned long ring_tail;
 212 };
 213
 214 #define ring_pos(pos, size) ((pos) & (size - 1))
 215
 216 #define DEBUGFS_STRUCT(_name, _mode, _write, _read)                         \
 217 static struct debugfs_entry debugfs_##_name = {                             \
 218         .name = __stringify(_name),                                         \
 219         .mode = VERIFY_OCTAL_PERMISSIONS(_mode),                            \
 220         .fops = {                                                           \
 221                 .open = simple_open,                                        \
 222                 .write = _write,                                            \
 223                 .read  = _read,                                             \
 224                 .llseek = generic_file_llseek,                              \
 225         },                                                                  \
 226 }
 227
 228 #define DEBUGFS_FIELD_ATTR(_type, _field)                                   \
 229 static ssize_t thunderx_##_type##_##_field##_read(struct file *file,        \
 230                                             char __user *data,              \
 231                                             size_t count, loff_t *ppos)     \
 232 {                                                                           \
 233         struct thunderx_##_type *pdata = file->private_data;                \
 234         char buf[20];                                                       \
 235                                                                             \
 236         snprintf(buf, count, "0x%016llx", pdata->_field);                   \
 237         return simple_read_from_buffer(data, count, ppos,                   \
 238                                        buf, sizeof(buf));                   \
 239 }                                                                           \
 240                                                                             \
 241 static ssize_t thunderx_##_type##_##_field##_write(struct file *file,       \
 242                                              const char __user *data,       \
 243                                              size_t count, loff_t *ppos)    \
 244 {                                                                           \
 245         struct thunderx_##_type *pdata = file->private_data;                \
 246         int res;                                                            \
 247                                                                             \
 248         res = kstrtoull_from_user(data, count, 0, &pdata->_field);          \
 249                                                                             \
 250         return res ? res : count;                                           \
 251 }                                                                           \
 252                                                                             \
 253 DEBUGFS_STRUCT(_field, 0600,                                                \
 254                    thunderx_##_type##_##_field##_write,                     \
 255                    thunderx_##_type##_##_field##_read)                      \
 256
 257 #define DEBUGFS_REG_ATTR(_type, _name, _reg)                                \
 258 static ssize_t thunderx_##_type##_##_name##_read(struct file *file,         \
 259                                            char __user *data,               \
 260                                            size_t count, loff_t *ppos)      \
 261 {                                                                           \
 262         struct thunderx_##_type *pdata = file->private_data;                \
 263         char buf[20];                                                       \
 264                                                                             \
 265         sprintf(buf, "0x%016llx", readq(pdata->regs + _reg));               \
 266         return simple_read_from_buffer(data, count, ppos,                   \
 267                                        buf, sizeof(buf));                   \
 268 }                                                                           \
 269                                                                             \
 270 static ssize_t thunderx_##_type##_##_name##_write(struct file *file,        \
 271                                             const char __user *data,        \
 272                                             size_t count, loff_t *ppos)     \
 273 {                                                                           \
 274         struct thunderx_##_type *pdata = file->private_data;                \
 275         u64 val;                                                            \
 276         int res;                                                            \
 277                                                                             \
 278         res = kstrtoull_from_user(data, count, 0, &val);                    \
 279                                                                             \
 280         if (!res) {                                                         \
 281                 writeq(val, pdata->regs + _reg);                            \
 282                 res = count;                                                \
 283         }                                                                   \
 284                                                                             \
 285         return res;                                                         \
 286 }                                                                           \
 287                                                                             \
 288 DEBUGFS_STRUCT(_name, 0600,                                                 \
 289                thunderx_##_type##_##_name##_write,                          \
 290                thunderx_##_type##_##_name##_read)
 291
 292 #define LMC_DEBUGFS_ENT(_field) DEBUGFS_FIELD_ATTR(lmc, _field)
 293
 294 /*
 295  * To get an ECC error injected, the following steps are needed:
 296  * - Setup the ECC injection by writing the appropriate parameters:
 297  *      echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0
 298  *      echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2
 299  *      echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test
 300  * - Do the actual injection:
 301  *      echo 1 > /sys/kernel/debug/<device number>/inject_ecc
 302  */
 303 static ssize_t thunderx_lmc_inject_int_write(struct file *file,
 304                                              const char __user *data,
 305                                              size_t count, loff_t *ppos)
 306 {
 307         struct thunderx_lmc *lmc = file->private_data;
 308         u64 val;
 309         int res;
 310
 311         res = kstrtoull_from_user(data, count, 0, &val);
 312
 313         if (!res) {
 314                 /* Trigger the interrupt */
 315                 writeq(val, lmc->regs + LMC_INT_W1S);
 316                 res = count;
 317         }
 318
 319         return res;
 320 }
 321
 322 static ssize_t thunderx_lmc_int_read(struct file *file,
 323                                      char __user *data,
 324                                      size_t count, loff_t *ppos)
 325 {
 326         struct thunderx_lmc *lmc = file->private_data;
 327         char buf[20];
 328         u64 lmc_int = readq(lmc->regs + LMC_INT);
 329
 330         snprintf(buf, sizeof(buf), "0x%016llx", lmc_int);
 331         return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf));
 332 }
 333
 334 #define TEST_PATTERN 0xa5
 335
 336 static int inject_ecc_fn(void *arg)
 337 {
 338         struct thunderx_lmc *lmc = arg;
 339         uintptr_t addr, phys;
 340         unsigned int cline_size = cache_line_size();
 341         const unsigned int lines = PAGE_SIZE / cline_size;
 342         unsigned int i, cl_idx;
 343
 344         addr = (uintptr_t)page_address(lmc->mem);
 345         phys = (uintptr_t)page_to_phys(lmc->mem);
 346
 347         cl_idx = (phys & 0x7f) >> 4;
 348         lmc->parity_test &= ~(7ULL << 8);
 349         lmc->parity_test |= (cl_idx << 8);
 350
 351         writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);
 352         writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);
 353         writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);
 354
 355         readq(lmc->regs + LMC_CHAR_MASK0);
 356         readq(lmc->regs + LMC_CHAR_MASK2);
 357         readq(lmc->regs + LMC_ECC_PARITY_TEST);
 358
 359         for (i = 0; i < lines; i++) {
 360                 memset((void *)addr, TEST_PATTERN, cline_size);
 361                 barrier();
 362
 363                 /*
 364                  * Flush L1 cachelines to the PoC (L2).
 365                  * This will cause cacheline eviction to the L2.
 366                  */
 367                 asm volatile("dc civac, %0\n"
 368                              "dsb sy\n"
 369                              : : "r"(addr + i * cline_size));
 370         }
 371
 372         for (i = 0; i < lines; i++) {
 373                 /*
 374                  * Flush L2 cachelines to the DRAM.
 375                  * This will cause cacheline eviction to the DRAM
 376                  * and ECC corruption according to the masks set.
 377                  */
 378                 __asm__ volatile("sys #0,c11,C1,#2, %0\n"
 379                                  : : "r"(phys + i * cline_size));
 380         }
 381
 382         for (i = 0; i < lines; i++) {
 383                 /*
 384                  * Invalidate L2 cachelines.
 385                  * The subsequent load will cause cacheline fetch
 386                  * from the DRAM and an error interrupt
 387                  */
 388                 __asm__ volatile("sys #0,c11,C1,#1, %0"
 389                                  : : "r"(phys + i * cline_size));
 390         }
 391
 392         for (i = 0; i < lines; i++) {
 393                 /*
 394                  * Invalidate L1 cachelines.
 395                  * The subsequent load will cause cacheline fetch
 396                  * from the L2 and/or DRAM
 397                  */
 398                 asm volatile("dc ivac, %0\n"
 399                              "dsb sy\n"
 400                              : : "r"(addr + i * cline_size));
 401         }
 402
 403         return 0;
 404 }
 405
 406 static ssize_t thunderx_lmc_inject_ecc_write(struct file *file,
 407                                              const char __user *data,
 408                                              size_t count, loff_t *ppos)
 409 {
 410         struct thunderx_lmc *lmc = file->private_data;
 411
 412         unsigned int cline_size = cache_line_size();
 413
 414         u8 tmp[cline_size];
 415         void __iomem *addr;
 416         unsigned int offs, timeout = 100000;
 417
 418         atomic_set(&lmc->ecc_int, 0);
 419
 420         lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);
 421
 422         if (!lmc->mem)
 423                 return -ENOMEM;
 424
 425         addr = page_address(lmc->mem);
 426
 427         while (!atomic_read(&lmc->ecc_int) && timeout--) {
 428                 stop_machine(inject_ecc_fn, lmc, NULL);
 429
 430                 for (offs = 0; offs < PAGE_SIZE; offs += sizeof(tmp)) {
 431                         /*
 432                          * Do a load from the previously rigged location
 433                          * This should generate an error interrupt.
 434                          */
 435                         memcpy(tmp, addr + offs, cline_size);
 436                         asm volatile("dsb ld\n");
 437                 }
 438         }
 439
 440         __free_pages(lmc->mem, 0);
 441
 442         return count;
 443 }
 444
 445 LMC_DEBUGFS_ENT(mask0);
 446 LMC_DEBUGFS_ENT(mask2);
 447 LMC_DEBUGFS_ENT(parity_test);
 448
 449 DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL);
 450 DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
 451 DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
 452
 453 struct debugfs_entry *lmc_dfs_ents[] = {
 454         &debugfs_mask0,
 455         &debugfs_mask2,
 456         &debugfs_parity_test,
 457         &debugfs_inject_ecc,
 458         &debugfs_inject_int,
 459         &debugfs_int_w1c,
 460 };
 461
 462 static int thunderx_create_debugfs_nodes(struct dentry *parent,
 463                                           struct debugfs_entry *attrs[],
 464                                           void *data,
 465                                           size_t num)
 466 {
 467         int i;
 468         struct dentry *ent;
 469
 470         if (!IS_ENABLED(CONFIG_EDAC_DEBUG))
 471                 return 0;
 472
 473         if (!parent)
 474                 return -ENOENT;
 475
 476         for (i = 0; i < num; i++) {
 477                 ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,
 478                                                parent, data, &attrs[i]->fops);
 479
 480                 if (!ent)
 481                         break;
 482         }
 483
 484         return i;
 485 }
 486
 487 static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)
 488 {
 489         phys_addr_t addr = 0;
 490         int bank, xbits;
 491
 492         addr |= lmc->node << 40;
 493         addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;
 494         addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;
 495         addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;
 496         addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;
 497
 498         bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;
 499
 500         if (lmc->xor_bank)
 501                 bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);
 502
 503         addr |= bank << lmc->bank_lsb;
 504
 505         xbits = PCI_FUNC(lmc->pdev->devfn);
 506
 507         if (lmc->l2c_alias)
 508                 xbits ^= get_bits(addr, 20, lmc->xbits) ^
 509                          get_bits(addr, 12, lmc->xbits);
 510
 511         addr |= xbits << 7;
 512
 513         return addr;
 514 }
 515
 516 static unsigned int thunderx_get_num_lmcs(unsigned int node)
 517 {
 518         unsigned int number = 0;
 519         struct pci_dev *pdev = NULL;
 520
 521         do {
 522                 pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
 523                                       PCI_DEVICE_ID_THUNDER_LMC,
 524                                       pdev);
 525                 if (pdev) {
 526 #ifdef CONFIG_NUMA
 527                         if (pdev->dev.numa_node == node)
 528                                 number++;
 529 #else
 530                         number++;
 531 #endif
 532                 }
 533         } while (pdev);
 534
 535         return number;
 536 }
 537
 538 #define LMC_MESSAGE_SIZE        120
 539 #define LMC_OTHER_SIZE          (50 * ARRAY_SIZE(lmc_errors))
 540
 541 static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id)
 542 {
 543         struct mem_ctl_info *mci = dev_id;
 544         struct thunderx_lmc *lmc = mci->pvt_info;
 545
 546         unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));
 547         struct lmc_err_ctx *ctx = &lmc->err_ctx[head];
 548
 549         writeq(0, lmc->regs + LMC_CHAR_MASK0);
 550         writeq(0, lmc->regs + LMC_CHAR_MASK2);
 551         writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);
 552
 553         ctx->reg_int = readq(lmc->regs + LMC_INT);
 554         ctx->reg_fadr = readq(lmc->regs + LMC_FADR);
 555         ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);
 556         ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);
 557         ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);
 558
 559         lmc->ring_head++;
 560
 561         atomic_set(&lmc->ecc_int, 1);
 562
 563         /* Clear the interrupt */
 564         writeq(ctx->reg_int, lmc->regs + LMC_INT);
 565
 566         return IRQ_WAKE_THREAD;
 567 }
 568
 569 static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id)
 570 {
 571         struct mem_ctl_info *mci = dev_id;
 572         struct thunderx_lmc *lmc = mci->pvt_info;
 573         phys_addr_t phys_addr;
 574
 575         unsigned long tail;
 576         struct lmc_err_ctx *ctx;
 577
 578         irqreturn_t ret = IRQ_NONE;
 579
 580         char *msg;
 581         char *other;
 582
 583         msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL);
 584         other =  kmalloc(LMC_OTHER_SIZE, GFP_KERNEL);
 585
 586         if (!msg || !other)
 587                 goto err_free;
 588
 589         while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,
 590                 ARRAY_SIZE(lmc->err_ctx))) {
 591                 tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));
 592
 593                 ctx = &lmc->err_ctx[tail];
 594
 595                 dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",
 596                         ctx->reg_int);
 597                 dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",
 598                         ctx->reg_fadr);
 599                 dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",
 600                         ctx->reg_nxm_fadr);
 601                 dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",
 602                         ctx->reg_scram_fadr);
 603                 dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",
 604                         ctx->reg_ecc_synd);
 605
 606                 snprintf(msg, LMC_MESSAGE_SIZE,
 607                          "DIMM %lld rank %lld bank %lld row %lld col %lld",
 608                          LMC_FADR_FDIMM(ctx->reg_scram_fadr),
 609                          LMC_FADR_FBUNK(ctx->reg_scram_fadr),
 610                          LMC_FADR_FBANK(ctx->reg_scram_fadr),
 611                          LMC_FADR_FROW(ctx->reg_scram_fadr),
 612                          LMC_FADR_FCOL(ctx->reg_scram_fadr));
 613
 614                 decode_register(other, LMC_OTHER_SIZE, lmc_errors,
 615                                 ctx->reg_int);
 616
 617                 phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);
 618
 619                 if (ctx->reg_int & LMC_INT_UE)
 620                         edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
 621                                              phys_to_pfn(phys_addr),
 622                                              offset_in_page(phys_addr),
 623                                              0, -1, -1, -1, msg, other);
 624                 else if (ctx->reg_int & LMC_INT_CE)
 625                         edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
 626                                              phys_to_pfn(phys_addr),
 627                                              offset_in_page(phys_addr),
 628                                              0, -1, -1, -1, msg, other);
 629
 630                 lmc->ring_tail++;
 631         }
 632
 633         ret = IRQ_HANDLED;
 634
 635 err_free:
 636         kfree(msg);
 637         kfree(other);
 638
 639         return ret;
 640 }
 641
 642 #ifdef CONFIG_PM
 643 static int thunderx_lmc_suspend(struct pci_dev *pdev, pm_message_t state)
 644 {
 645         pci_save_state(pdev);
 646         pci_disable_device(pdev);
 647
 648         pci_set_power_state(pdev, pci_choose_state(pdev, state));
 649
 650         return 0;
 651 }
 652
 653 static int thunderx_lmc_resume(struct pci_dev *pdev)
 654 {
 655         pci_set_power_state(pdev, PCI_D0);
 656         pci_enable_wake(pdev, PCI_D0, 0);
 657         pci_restore_state(pdev);
 658
 659         return 0;
 660 }
 661 #endif
 662
 663 static const struct pci_device_id thunderx_lmc_pci_tbl[] = {
 664         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) },
 665         { 0, },
 666 };
 667
 668 static inline int pci_dev_to_mc_idx(struct pci_dev *pdev)
 669 {
 670         int node = dev_to_node(&pdev->dev);
 671         int ret = PCI_FUNC(pdev->devfn);
 672
 673         ret += max(node, 0) << 3;
 674
 675         return ret;
 676 }
 677
 678 static int thunderx_lmc_probe(struct pci_dev *pdev,
 679                                 const struct pci_device_id *id)
 680 {
 681         struct thunderx_lmc *lmc;
 682         struct edac_mc_layer layer;
 683         struct mem_ctl_info *mci;
 684         u64 lmc_control, lmc_ddr_pll_ctl, lmc_config;
 685         int ret;
 686         u64 lmc_int;
 687         void *l2c_ioaddr;
 688
 689         layer.type = EDAC_MC_LAYER_SLOT;
 690         layer.size = 2;
 691         layer.is_virt_csrow = false;
 692
 693         ret = pcim_enable_device(pdev);
 694         if (ret) {
 695                 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
 696                 return ret;
 697         }
 698
 699         ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc");
 700         if (ret) {
 701                 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
 702                 return ret;
 703         }
 704
 705         mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer,
 706                             sizeof(struct thunderx_lmc));
 707         if (!mci)
 708                 return -ENOMEM;
 709
 710         mci->pdev = &pdev->dev;
 711         lmc = mci->pvt_info;
 712
 713         pci_set_drvdata(pdev, mci);
 714
 715         lmc->regs = pcim_iomap_table(pdev)[0];
 716
 717         lmc_control = readq(lmc->regs + LMC_CONTROL);
 718         lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);
 719         lmc_config = readq(lmc->regs + LMC_CONFIG);
 720
 721         if (lmc_control & LMC_CONTROL_RDIMM) {
 722                 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
 723                                            lmc_ddr_pll_ctl) ?
 724                                 MEM_RDDR4 : MEM_RDDR3;
 725         } else {
 726                 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
 727                                            lmc_ddr_pll_ctl) ?
 728                                 MEM_DDR4 : MEM_DDR3;
 729         }
 730
 731         mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 732         mci->edac_cap = EDAC_FLAG_SECDED;
 733
 734         mci->mod_name = "thunderx-lmc";
 735         mci->mod_ver = "1";
 736         mci->ctl_name = "thunderx-lmc";
 737         mci->dev_name = dev_name(&pdev->dev);
 738         mci->scrub_mode = SCRUB_NONE;
 739
 740         lmc->pdev = pdev;
 741         lmc->msix_ent.entry = 0;
 742
 743         lmc->ring_head = 0;
 744         lmc->ring_tail = 0;
 745
 746         ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);
 747         if (ret) {
 748                 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
 749                 goto err_free;
 750         }
 751
 752         ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,
 753                                         thunderx_lmc_err_isr,
 754                                         thunderx_lmc_threaded_isr, 0,
 755                                         "[EDAC] ThunderX LMC", mci);
 756         if (ret) {
 757                 dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret);
 758                 goto err_free;
 759         }
 760
 761         lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));
 762
 763         lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;
 764         lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&
 765                            FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3;
 766
 767         lmc->pbank_lsb = (lmc_config >> 5) & 0xf;
 768         lmc->dimm_lsb  = 28 + lmc->pbank_lsb + lmc->xbits;
 769         lmc->rank_lsb = lmc->dimm_lsb;
 770         lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;
 771         lmc->bank_lsb = 7 + lmc->xbits;
 772         lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;
 773
 774         lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;
 775
 776         lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
 777
 778         l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node),
 779                              PAGE_SIZE);
 780
 781         if (!l2c_ioaddr) {
 782                 dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
 783                 goto err_free;
 784         }
 785
 786         lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);
 787
 788         iounmap(l2c_ioaddr);
 789
 790         ret = edac_mc_add_mc(mci);
 791         if (ret) {
 792                 dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);
 793                 goto err_free;
 794         }
 795
 796         lmc_int = readq(lmc->regs + LMC_INT);
 797         writeq(lmc_int, lmc->regs + LMC_INT);
 798
 799         writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);
 800
 801         if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
 802                 ret = thunderx_create_debugfs_nodes(mci->debugfs,
 803                                                     lmc_dfs_ents,
 804                                                     lmc,
 805                                                     ARRAY_SIZE(lmc_dfs_ents));
 806
 807                 if (ret != ARRAY_SIZE(lmc_dfs_ents)) {
 808                         dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
 809                                  ret, ret >= 0 ? " created" : "");
 810                 }
 811         }
 812
 813         return 0;
 814
 815 err_free:
 816         pci_set_drvdata(pdev, NULL);
 817         edac_mc_free(mci);
 818
 819         return ret;
 820 }
 821
 822 static void thunderx_lmc_remove(struct pci_dev *pdev)
 823 {
 824         struct mem_ctl_info *mci = pci_get_drvdata(pdev);
 825         struct thunderx_lmc *lmc = mci->pvt_info;
 826
 827         writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);
 828
 829         edac_mc_del_mc(&pdev->dev);
 830         edac_mc_free(mci);
 831 }
 832
 833 MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl);
 834
 835 static struct pci_driver thunderx_lmc_driver = {
 836         .name     = "thunderx_lmc_edac",
 837         .probe    = thunderx_lmc_probe,
 838         .remove   = thunderx_lmc_remove,
 839 #ifdef CONFIG_PM
 840         .suspend  = thunderx_lmc_suspend,
 841         .resume   = thunderx_lmc_resume,
 842 #endif
 843         .id_table = thunderx_lmc_pci_tbl,
 844 };
 845
 846 /*---------------------- OCX driver ---------------------------------*/
 847
 848 #define PCI_DEVICE_ID_THUNDER_OCX 0xa013
 849
 850 #define OCX_LINK_INTS           3
 851 #define OCX_INTS                (OCX_LINK_INTS + 1)
 852 #define OCX_RX_LANES            24
 853 #define OCX_RX_LANE_STATS       15
 854
 855 #define OCX_COM_INT             0x100
 856 #define OCX_COM_INT_W1S         0x108
 857 #define OCX_COM_INT_ENA_W1S     0x110
 858 #define OCX_COM_INT_ENA_W1C     0x118
 859
 860 #define OCX_COM_IO_BADID                BIT(54)
 861 #define OCX_COM_MEM_BADID               BIT(53)
 862 #define OCX_COM_COPR_BADID              BIT(52)
 863 #define OCX_COM_WIN_REQ_BADID           BIT(51)
 864 #define OCX_COM_WIN_REQ_TOUT            BIT(50)
 865 #define OCX_COM_RX_LANE                 GENMASK(23, 0)
 866
 867 #define OCX_COM_INT_CE                  (OCX_COM_IO_BADID      | \
 868                                          OCX_COM_MEM_BADID     | \
 869                                          OCX_COM_COPR_BADID    | \
 870                                          OCX_COM_WIN_REQ_BADID | \
 871                                          OCX_COM_WIN_REQ_TOUT)
 872
 873 static const struct error_descr ocx_com_errors[] = {
 874         {
 875                 .type  = ERR_CORRECTED,
 876                 .mask  = OCX_COM_IO_BADID,
 877                 .descr = "Invalid IO transaction node ID",
 878         },
 879         {
 880                 .type  = ERR_CORRECTED,
 881                 .mask  = OCX_COM_MEM_BADID,
 882                 .descr = "Invalid memory transaction node ID",
 883         },
 884         {
 885                 .type  = ERR_CORRECTED,
 886                 .mask  = OCX_COM_COPR_BADID,
 887                 .descr = "Invalid coprocessor transaction node ID",
 888         },
 889         {
 890                 .type  = ERR_CORRECTED,
 891                 .mask  = OCX_COM_WIN_REQ_BADID,
 892                 .descr = "Invalid SLI transaction node ID",
 893         },
 894         {
 895                 .type  = ERR_CORRECTED,
 896                 .mask  = OCX_COM_WIN_REQ_TOUT,
 897                 .descr = "Window/core request timeout",
 898         },
 899         {0, 0, NULL},
 900 };
 901
 902 #define OCX_COM_LINKX_INT(x)            (0x120 + (x) * 8)
 903 #define OCX_COM_LINKX_INT_W1S(x)        (0x140 + (x) * 8)
 904 #define OCX_COM_LINKX_INT_ENA_W1S(x)    (0x160 + (x) * 8)
 905 #define OCX_COM_LINKX_INT_ENA_W1C(x)    (0x180 + (x) * 8)
 906
 907 #define OCX_COM_LINK_BAD_WORD                   BIT(13)
 908 #define OCX_COM_LINK_ALIGN_FAIL                 BIT(12)
 909 #define OCX_COM_LINK_ALIGN_DONE                 BIT(11)
 910 #define OCX_COM_LINK_UP                         BIT(10)
 911 #define OCX_COM_LINK_STOP                       BIT(9)
 912 #define OCX_COM_LINK_BLK_ERR                    BIT(8)
 913 #define OCX_COM_LINK_REINIT                     BIT(7)
 914 #define OCX_COM_LINK_LNK_DATA                   BIT(6)
 915 #define OCX_COM_LINK_RXFIFO_DBE                 BIT(5)
 916 #define OCX_COM_LINK_RXFIFO_SBE                 BIT(4)
 917 #define OCX_COM_LINK_TXFIFO_DBE                 BIT(3)
 918 #define OCX_COM_LINK_TXFIFO_SBE                 BIT(2)
 919 #define OCX_COM_LINK_REPLAY_DBE                 BIT(1)
 920 #define OCX_COM_LINK_REPLAY_SBE                 BIT(0)
 921
 922 static const struct error_descr ocx_com_link_errors[] = {
 923         {
 924                 .type  = ERR_CORRECTED,
 925                 .mask  = OCX_COM_LINK_REPLAY_SBE,
 926                 .descr = "Replay buffer single-bit error",
 927         },
 928         {
 929                 .type  = ERR_CORRECTED,
 930                 .mask  = OCX_COM_LINK_TXFIFO_SBE,
 931                 .descr = "TX FIFO single-bit error",
 932         },
 933         {
 934                 .type  = ERR_CORRECTED,
 935                 .mask  = OCX_COM_LINK_RXFIFO_SBE,
 936                 .descr = "RX FIFO single-bit error",
 937         },
 938         {
 939                 .type  = ERR_CORRECTED,
 940                 .mask  = OCX_COM_LINK_BLK_ERR,
 941                 .descr = "Block code error",
 942         },
 943         {
 944                 .type  = ERR_CORRECTED,
 945                 .mask  = OCX_COM_LINK_ALIGN_FAIL,
 946                 .descr = "Link alignment failure",
 947         },
 948         {
 949                 .type  = ERR_CORRECTED,
 950                 .mask  = OCX_COM_LINK_BAD_WORD,
 951                 .descr = "Bad code word",
 952         },
 953         {
 954                 .type  = ERR_UNCORRECTED,
 955                 .mask  = OCX_COM_LINK_REPLAY_DBE,
 956                 .descr = "Replay buffer double-bit error",
 957         },
 958         {
 959                 .type  = ERR_UNCORRECTED,
 960                 .mask  = OCX_COM_LINK_TXFIFO_DBE,
 961                 .descr = "TX FIFO double-bit error",
 962         },
 963         {
 964                 .type  = ERR_UNCORRECTED,
 965                 .mask  = OCX_COM_LINK_RXFIFO_DBE,
 966                 .descr = "RX FIFO double-bit error",
 967         },
 968         {
 969                 .type  = ERR_UNCORRECTED,
 970                 .mask  = OCX_COM_LINK_STOP,
 971                 .descr = "Link stopped",
 972         },
 973         {0, 0, NULL},
 974 };
 975
 976 #define OCX_COM_LINK_INT_UE       (OCX_COM_LINK_REPLAY_DBE | \
 977                                    OCX_COM_LINK_TXFIFO_DBE | \
 978                                    OCX_COM_LINK_RXFIFO_DBE | \
 979                                    OCX_COM_LINK_STOP)
 980
 981 #define OCX_COM_LINK_INT_CE       (OCX_COM_LINK_REPLAY_SBE | \
 982                                    OCX_COM_LINK_TXFIFO_SBE | \
 983                                    OCX_COM_LINK_RXFIFO_SBE | \
 984                                    OCX_COM_LINK_BLK_ERR    | \
 985                                    OCX_COM_LINK_ALIGN_FAIL | \
 986                                    OCX_COM_LINK_BAD_WORD)
 987
 988 #define OCX_LNE_INT(x)                  (0x8018 + (x) * 0x100)
 989 #define OCX_LNE_INT_EN(x)               (0x8020 + (x) * 0x100)
 990 #define OCX_LNE_BAD_CNT(x)              (0x8028 + (x) * 0x100)
 991 #define OCX_LNE_CFG(x)                  (0x8000 + (x) * 0x100)
 992 #define OCX_LNE_STAT(x, y)              (0x8040 + (x) * 0x100 + (y) * 8)
 993
 994 #define OCX_LNE_CFG_RX_BDRY_LOCK_DIS            BIT(8)
 995 #define OCX_LNE_CFG_RX_STAT_WRAP_DIS            BIT(2)
 996 #define OCX_LNE_CFG_RX_STAT_RDCLR               BIT(1)
 997 #define OCX_LNE_CFG_RX_STAT_ENA                 BIT(0)
 998
 999
1000 #define OCX_LANE_BAD_64B67B                     BIT(8)
1001 #define OCX_LANE_DSKEW_FIFO_OVFL                BIT(5)
1002 #define OCX_LANE_SCRM_SYNC_LOSS                 BIT(4)
1003 #define OCX_LANE_UKWN_CNTL_WORD                 BIT(3)
1004 #define OCX_LANE_CRC32_ERR                      BIT(2)
1005 #define OCX_LANE_BDRY_SYNC_LOSS                 BIT(1)
1006 #define OCX_LANE_SERDES_LOCK_LOSS               BIT(0)
1007
1008 #define OCX_COM_LANE_INT_UE       (0)
1009 #define OCX_COM_LANE_INT_CE       (OCX_LANE_SERDES_LOCK_LOSS | \
1010                                    OCX_LANE_BDRY_SYNC_LOSS   | \
1011                                    OCX_LANE_CRC32_ERR        | \
1012                                    OCX_LANE_UKWN_CNTL_WORD   | \
1013                                    OCX_LANE_SCRM_SYNC_LOSS   | \
1014                                    OCX_LANE_DSKEW_FIFO_OVFL  | \
1015                                    OCX_LANE_BAD_64B67B)
1016
1017 static const struct error_descr ocx_lane_errors[] = {
1018         {
1019                 .type  = ERR_CORRECTED,
1020                 .mask  = OCX_LANE_SERDES_LOCK_LOSS,
1021                 .descr = "RX SerDes lock lost",
1022         },
1023         {
1024                 .type  = ERR_CORRECTED,
1025                 .mask  = OCX_LANE_BDRY_SYNC_LOSS,
1026                 .descr = "RX word boundary lost",
1027         },
1028         {
1029                 .type  = ERR_CORRECTED,
1030                 .mask  = OCX_LANE_CRC32_ERR,
1031                 .descr = "CRC32 error",
1032         },
1033         {
1034                 .type  = ERR_CORRECTED,
1035                 .mask  = OCX_LANE_UKWN_CNTL_WORD,
1036                 .descr = "Unknown control word",
1037         },
1038         {
1039                 .type  = ERR_CORRECTED,
1040                 .mask  = OCX_LANE_SCRM_SYNC_LOSS,
1041                 .descr = "Scrambler synchronization lost",
1042         },
1043         {
1044                 .type  = ERR_CORRECTED,
1045                 .mask  = OCX_LANE_DSKEW_FIFO_OVFL,
1046                 .descr = "RX deskew FIFO overflow",
1047         },
1048         {
1049                 .type  = ERR_CORRECTED,
1050                 .mask  = OCX_LANE_BAD_64B67B,
1051                 .descr = "Bad 64B/67B codeword",
1052         },
1053         {0, 0, NULL},
1054 };
1055
1056 #define OCX_LNE_INT_ENA_ALL             (GENMASK(9, 8) | GENMASK(6, 0))
1057 #define OCX_COM_INT_ENA_ALL             (GENMASK(54, 50) | GENMASK(23, 0))
1058 #define OCX_COM_LINKX_INT_ENA_ALL       (GENMASK(13, 12) | \
1059                                          GENMASK(9, 7) | GENMASK(5, 0))
1060
1061 #define OCX_TLKX_ECC_CTL(x)             (0x10018 + (x) * 0x2000)
1062 #define OCX_RLKX_ECC_CTL(x)             (0x18018 + (x) * 0x2000)
1063
1064 struct ocx_com_err_ctx {
1065         u64 reg_com_int;
1066         u64 reg_lane_int[OCX_RX_LANES];
1067         u64 reg_lane_stat11[OCX_RX_LANES];
1068 };
1069
1070 struct ocx_link_err_ctx {
1071         u64 reg_com_link_int;
1072         int link;
1073 };
1074
1075 struct thunderx_ocx {
1076         void __iomem *regs;
1077         int com_link;
1078         struct pci_dev *pdev;
1079         struct edac_device_ctl_info *edac_dev;
1080
1081         struct dentry *debugfs;
1082         struct msix_entry msix_ent[OCX_INTS];
1083
1084         struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES];
1085         struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES];
1086
1087         unsigned long com_ring_head;
1088         unsigned long com_ring_tail;
1089
1090         unsigned long link_ring_head;
1091         unsigned long link_ring_tail;
1092 };
1093
1094 #define OCX_MESSAGE_SIZE        SZ_1K
1095 #define OCX_OTHER_SIZE          (50 * ARRAY_SIZE(ocx_com_link_errors))
1096
1097 /* This handler is threaded */
1098 static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id)
1099 {
1100         struct msix_entry *msix = irq_id;
1101         struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1102                                                 msix_ent[msix->entry]);
1103
1104         int lane;
1105         unsigned long head = ring_pos(ocx->com_ring_head,
1106                                       ARRAY_SIZE(ocx->com_err_ctx));
1107         struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];
1108
1109         ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);
1110
1111         for (lane = 0; lane < OCX_RX_LANES; lane++) {
1112                 ctx->reg_lane_int[lane] =
1113                         readq(ocx->regs + OCX_LNE_INT(lane));
1114                 ctx->reg_lane_stat11[lane] =
1115                         readq(ocx->regs + OCX_LNE_STAT(lane, 11));
1116
1117                 writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));
1118         }
1119
1120         writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);
1121
1122         ocx->com_ring_head++;
1123
1124         return IRQ_WAKE_THREAD;
1125 }
1126
1127 static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id)
1128 {
1129         struct msix_entry *msix = irq_id;
1130         struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1131                                                 msix_ent[msix->entry]);
1132
1133         irqreturn_t ret = IRQ_NONE;
1134
1135         unsigned long tail;
1136         struct ocx_com_err_ctx *ctx;
1137         int lane;
1138         char *msg;
1139         char *other;
1140
1141         msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1142         other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1143
1144         if (!msg || !other)
1145                 goto err_free;
1146
1147         while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,
1148                         ARRAY_SIZE(ocx->com_err_ctx))) {
1149                 tail = ring_pos(ocx->com_ring_tail,
1150                                 ARRAY_SIZE(ocx->com_err_ctx));
1151                 ctx = &ocx->com_err_ctx[tail];
1152
1153                 snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx",
1154                         ocx->edac_dev->ctl_name, ctx->reg_com_int);
1155
1156                 decode_register(other, OCX_OTHER_SIZE,
1157                                 ocx_com_errors, ctx->reg_com_int);
1158
1159                 strncat(msg, other, OCX_MESSAGE_SIZE);
1160
1161                 for (lane = 0; lane < OCX_RX_LANES; lane++)
1162                         if (ctx->reg_com_int & BIT(lane)) {
1163                                 snprintf(other, OCX_OTHER_SIZE,
1164                                          "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx",
1165                                          lane, ctx->reg_lane_int[lane],
1166                                          lane, ctx->reg_lane_stat11[lane]);
1167
1168                                 strncat(msg, other, OCX_MESSAGE_SIZE);
1169
1170                                 decode_register(other, OCX_OTHER_SIZE,
1171                                                 ocx_lane_errors,
1172                                                 ctx->reg_lane_int[lane]);
1173                                 strncat(msg, other, OCX_MESSAGE_SIZE);
1174                         }
1175
1176                 if (ctx->reg_com_int & OCX_COM_INT_CE)
1177                         edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1178
1179                 ocx->com_ring_tail++;
1180         }
1181
1182         ret = IRQ_HANDLED;
1183
1184 err_free:
1185         kfree(other);
1186         kfree(msg);
1187
1188         return ret;
1189 }
1190
1191 static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id)
1192 {
1193         struct msix_entry *msix = irq_id;
1194         struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1195                                                 msix_ent[msix->entry]);
1196         unsigned long head = ring_pos(ocx->link_ring_head,
1197                                       ARRAY_SIZE(ocx->link_err_ctx));
1198         struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];
1199
1200         ctx->link = msix->entry;
1201         ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1202
1203         writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1204
1205         ocx->link_ring_head++;
1206
1207         return IRQ_WAKE_THREAD;
1208 }
1209
1210 static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id)
1211 {
1212         struct msix_entry *msix = irq_id;
1213         struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1214                                                 msix_ent[msix->entry]);
1215         irqreturn_t ret = IRQ_NONE;
1216         unsigned long tail;
1217         struct ocx_link_err_ctx *ctx;
1218
1219         char *msg;
1220         char *other;
1221
1222         msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1223         other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1224
1225         if (!msg || !other)
1226                 goto err_free;
1227
1228         while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,
1229                         ARRAY_SIZE(ocx->link_err_ctx))) {
1230                 tail = ring_pos(ocx->link_ring_head,
1231                                 ARRAY_SIZE(ocx->link_err_ctx));
1232
1233                 ctx = &ocx->link_err_ctx[tail];
1234
1235                 snprintf(msg, OCX_MESSAGE_SIZE,
1236                          "%s: OCX_COM_LINK_INT[%d]: %016llx",
1237                          ocx->edac_dev->ctl_name,
1238                          ctx->link, ctx->reg_com_link_int);
1239
1240                 decode_register(other, OCX_OTHER_SIZE,
1241                                 ocx_com_link_errors, ctx->reg_com_link_int);
1242
1243                 strncat(msg, other, OCX_MESSAGE_SIZE);
1244
1245                 if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)
1246                         edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
1247                 else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)
1248                         edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1249
1250                 ocx->link_ring_tail++;
1251         }
1252
1253         ret = IRQ_HANDLED;
1254 err_free:
1255         kfree(other);
1256         kfree(msg);
1257
1258         return ret;
1259 }
1260
1261 #define OCX_DEBUGFS_ATTR(_name, _reg)   DEBUGFS_REG_ATTR(ocx, _name, _reg)
1262
1263 OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0));
1264 OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1));
1265 OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2));
1266
1267 OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0));
1268 OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1));
1269 OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2));
1270
1271 OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0));
1272 OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1));
1273 OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2));
1274
1275 OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0));
1276 OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1));
1277 OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2));
1278 OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3));
1279 OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4));
1280 OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5));
1281 OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6));
1282 OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7));
1283
1284 OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8));
1285 OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9));
1286 OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10));
1287 OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11));
1288 OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12));
1289 OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13));
1290 OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14));
1291 OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15));
1292
1293 OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16));
1294 OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17));
1295 OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18));
1296 OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19));
1297 OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20));
1298 OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21));
1299 OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22));
1300 OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23));
1301
1302 OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S);
1303
1304 struct debugfs_entry *ocx_dfs_ents[] = {
1305         &debugfs_tlk0_ecc_ctl,
1306         &debugfs_tlk1_ecc_ctl,
1307         &debugfs_tlk2_ecc_ctl,
1308
1309         &debugfs_rlk0_ecc_ctl,
1310         &debugfs_rlk1_ecc_ctl,
1311         &debugfs_rlk2_ecc_ctl,
1312
1313         &debugfs_com_link0_int,
1314         &debugfs_com_link1_int,
1315         &debugfs_com_link2_int,
1316
1317         &debugfs_lne00_badcnt,
1318         &debugfs_lne01_badcnt,
1319         &debugfs_lne02_badcnt,
1320         &debugfs_lne03_badcnt,
1321         &debugfs_lne04_badcnt,
1322         &debugfs_lne05_badcnt,
1323         &debugfs_lne06_badcnt,
1324         &debugfs_lne07_badcnt,
1325         &debugfs_lne08_badcnt,
1326         &debugfs_lne09_badcnt,
1327         &debugfs_lne10_badcnt,
1328         &debugfs_lne11_badcnt,
1329         &debugfs_lne12_badcnt,
1330         &debugfs_lne13_badcnt,
1331         &debugfs_lne14_badcnt,
1332         &debugfs_lne15_badcnt,
1333         &debugfs_lne16_badcnt,
1334         &debugfs_lne17_badcnt,
1335         &debugfs_lne18_badcnt,
1336         &debugfs_lne19_badcnt,
1337         &debugfs_lne20_badcnt,
1338         &debugfs_lne21_badcnt,
1339         &debugfs_lne22_badcnt,
1340         &debugfs_lne23_badcnt,
1341
1342         &debugfs_com_int,
1343 };
1344
1345 static const struct pci_device_id thunderx_ocx_pci_tbl[] = {
1346         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) },
1347         { 0, },
1348 };
1349
1350 static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx)
1351 {
1352         int lane, stat, cfg;
1353
1354         for (lane = 0; lane < OCX_RX_LANES; lane++) {
1355                 cfg = readq(ocx->regs + OCX_LNE_CFG(lane));
1356                 cfg |= OCX_LNE_CFG_RX_STAT_RDCLR;
1357                 cfg &= ~OCX_LNE_CFG_RX_STAT_ENA;
1358                 writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));
1359
1360                 for (stat = 0; stat < OCX_RX_LANE_STATS; stat++)
1361                         readq(ocx->regs + OCX_LNE_STAT(lane, stat));
1362         }
1363 }
1364
1365 static int thunderx_ocx_probe(struct pci_dev *pdev,
1366                               const struct pci_device_id *id)
1367 {
1368         struct thunderx_ocx *ocx;
1369         struct edac_device_ctl_info *edac_dev;
1370         char name[32];
1371         int idx;
1372         int i;
1373         int ret;
1374         u64 reg;
1375
1376         ret = pcim_enable_device(pdev);
1377         if (ret) {
1378                 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1379                 return ret;
1380         }
1381
1382         ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx");
1383         if (ret) {
1384                 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1385                 return ret;
1386         }
1387
1388         idx = edac_device_alloc_index();
1389         snprintf(name, sizeof(name), "OCX%d", idx);
1390         edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx),
1391                                               name, 1, "CCPI", 1,
1392                                               0, NULL, 0, idx);
1393         if (!edac_dev) {
1394                 dev_err(&pdev->dev, "Cannot allocate EDAC device: %d\n", ret);
1395                 return -ENOMEM;
1396         }
1397         ocx = edac_dev->pvt_info;
1398         ocx->edac_dev = edac_dev;
1399         ocx->com_ring_head = 0;
1400         ocx->com_ring_tail = 0;
1401         ocx->link_ring_head = 0;
1402         ocx->link_ring_tail = 0;
1403
1404         ocx->regs = pcim_iomap_table(pdev)[0];
1405         if (!ocx->regs) {
1406                 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1407                 ret = -ENODEV;
1408                 goto err_free;
1409         }
1410
1411         ocx->pdev = pdev;
1412
1413         for (i = 0; i < OCX_INTS; i++) {
1414                 ocx->msix_ent[i].entry = i;
1415                 ocx->msix_ent[i].vector = 0;
1416         }
1417
1418         ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);
1419         if (ret) {
1420                 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
1421                 goto err_free;
1422         }
1423
1424         for (i = 0; i < OCX_INTS; i++) {
1425                 ret = devm_request_threaded_irq(&pdev->dev,
1426                                                 ocx->msix_ent[i].vector,
1427                                                 (i == 3) ?
1428                                                  thunderx_ocx_com_isr :
1429                                                  thunderx_ocx_lnk_isr,
1430                                                 (i == 3) ?
1431                                                  thunderx_ocx_com_threaded_isr :
1432                                                  thunderx_ocx_lnk_threaded_isr,
1433                                                 0, "[EDAC] ThunderX OCX",
1434                                                 &ocx->msix_ent[i]);
1435                 if (ret)
1436                         goto err_free;
1437         }
1438
1439         edac_dev->dev = &pdev->dev;
1440         edac_dev->dev_name = dev_name(&pdev->dev);
1441         edac_dev->mod_name = "thunderx-ocx";
1442         edac_dev->ctl_name = "thunderx-ocx";
1443
1444         ret = edac_device_add_device(edac_dev);
1445         if (ret) {
1446                 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
1447                 goto err_free;
1448         }
1449
1450         if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
1451                 ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
1452
1453                 ret = thunderx_create_debugfs_nodes(ocx->debugfs,
1454                                                     ocx_dfs_ents,
1455                                                     ocx,
1456                                                     ARRAY_SIZE(ocx_dfs_ents));
1457                 if (ret != ARRAY_SIZE(ocx_dfs_ents)) {
1458                         dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
1459                                  ret, ret >= 0 ? " created" : "");
1460                 }
1461         }
1462
1463         pci_set_drvdata(pdev, edac_dev);
1464
1465         thunderx_ocx_clearstats(ocx);
1466
1467         for (i = 0; i < OCX_RX_LANES; i++) {
1468                 writeq(OCX_LNE_INT_ENA_ALL,
1469                        ocx->regs + OCX_LNE_INT_EN(i));
1470
1471                 reg = readq(ocx->regs + OCX_LNE_INT(i));
1472                 writeq(reg, ocx->regs + OCX_LNE_INT(i));
1473
1474         }
1475
1476         for (i = 0; i < OCX_LINK_INTS; i++) {
1477                 reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));
1478                 writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));
1479
1480                 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1481                        ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));
1482         }
1483
1484         reg = readq(ocx->regs + OCX_COM_INT);
1485         writeq(reg, ocx->regs + OCX_COM_INT);
1486
1487         writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);
1488
1489         return 0;
1490 err_free:
1491         edac_device_free_ctl_info(edac_dev);
1492
1493         return ret;
1494 }
1495
1496 static void thunderx_ocx_remove(struct pci_dev *pdev)
1497 {
1498         struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
1499         struct thunderx_ocx *ocx = edac_dev->pvt_info;
1500         int i;
1501
1502         writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);
1503
1504         for (i = 0; i < OCX_INTS; i++) {
1505                 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1506                        ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));
1507         }
1508
1509         edac_debugfs_remove_recursive(ocx->debugfs);
1510
1511         edac_device_del_device(&pdev->dev);
1512         edac_device_free_ctl_info(edac_dev);
1513 }
1514
1515 MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl);
1516
1517 static struct pci_driver thunderx_ocx_driver = {
1518         .name     = "thunderx_ocx_edac",
1519         .probe    = thunderx_ocx_probe,
1520         .remove   = thunderx_ocx_remove,
1521         .id_table = thunderx_ocx_pci_tbl,
1522 };
1523
1524 /*---------------------- L2C driver ---------------------------------*/
1525
1526 #define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e
1527 #define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f
1528 #define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030
1529
1530 #define L2C_TAD_INT_W1C         0x40000
1531 #define L2C_TAD_INT_W1S         0x40008
1532
1533 #define L2C_TAD_INT_ENA_W1C     0x40020
1534 #define L2C_TAD_INT_ENA_W1S     0x40028
1535
1536
1537 #define L2C_TAD_INT_L2DDBE       BIT(1)
1538 #define L2C_TAD_INT_SBFSBE       BIT(2)
1539 #define L2C_TAD_INT_SBFDBE       BIT(3)
1540 #define L2C_TAD_INT_FBFSBE       BIT(4)
1541 #define L2C_TAD_INT_FBFDBE       BIT(5)
1542 #define L2C_TAD_INT_TAGDBE       BIT(9)
1543 #define L2C_TAD_INT_RDDISLMC     BIT(15)
1544 #define L2C_TAD_INT_WRDISLMC     BIT(16)
1545 #define L2C_TAD_INT_LFBTO        BIT(17)
1546 #define L2C_TAD_INT_GSYNCTO      BIT(18)
1547 #define L2C_TAD_INT_RTGSBE       BIT(32)
1548 #define L2C_TAD_INT_RTGDBE       BIT(33)
1549 #define L2C_TAD_INT_RDDISOCI     BIT(34)
1550 #define L2C_TAD_INT_WRDISOCI     BIT(35)
1551
1552 #define L2C_TAD_INT_ECC         (L2C_TAD_INT_L2DDBE | \
1553                                  L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \
1554                                  L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE)
1555
1556 #define L2C_TAD_INT_CE          (L2C_TAD_INT_SBFSBE | \
1557                                  L2C_TAD_INT_FBFSBE)
1558
1559 #define L2C_TAD_INT_UE          (L2C_TAD_INT_L2DDBE | \
1560                                  L2C_TAD_INT_SBFDBE | \
1561                                  L2C_TAD_INT_FBFDBE | \
1562                                  L2C_TAD_INT_TAGDBE | \
1563                                  L2C_TAD_INT_RTGDBE | \
1564                                  L2C_TAD_INT_WRDISOCI | \
1565                                  L2C_TAD_INT_RDDISOCI | \
1566                                  L2C_TAD_INT_WRDISLMC | \
1567                                  L2C_TAD_INT_RDDISLMC | \
1568                                  L2C_TAD_INT_LFBTO    | \
1569                                  L2C_TAD_INT_GSYNCTO)
1570
1571 static const struct error_descr l2_tad_errors[] = {
1572         {
1573                 .type  = ERR_CORRECTED,
1574                 .mask  = L2C_TAD_INT_SBFSBE,
1575                 .descr = "SBF single-bit error",
1576         },
1577         {
1578                 .type  = ERR_CORRECTED,
1579                 .mask  = L2C_TAD_INT_FBFSBE,
1580                 .descr = "FBF single-bit error",
1581         },
1582         {
1583                 .type  = ERR_UNCORRECTED,
1584                 .mask  = L2C_TAD_INT_L2DDBE,
1585                 .descr = "L2D double-bit error",
1586         },
1587         {
1588                 .type  = ERR_UNCORRECTED,
1589                 .mask  = L2C_TAD_INT_SBFDBE,
1590                 .descr = "SBF double-bit error",
1591         },
1592         {
1593                 .type  = ERR_UNCORRECTED,
1594                 .mask  = L2C_TAD_INT_FBFDBE,
1595                 .descr = "FBF double-bit error",
1596         },
1597         {
1598                 .type  = ERR_UNCORRECTED,
1599                 .mask  = L2C_TAD_INT_TAGDBE,
1600                 .descr = "TAG double-bit error",
1601         },
1602         {
1603                 .type  = ERR_UNCORRECTED,
1604                 .mask  = L2C_TAD_INT_RTGDBE,
1605                 .descr = "RTG double-bit error",
1606         },
1607         {
1608                 .type  = ERR_UNCORRECTED,
1609                 .mask  = L2C_TAD_INT_WRDISOCI,
1610                 .descr = "Write to a disabled CCPI",
1611         },
1612         {
1613                 .type  = ERR_UNCORRECTED,
1614                 .mask  = L2C_TAD_INT_RDDISOCI,
1615                 .descr = "Read from a disabled CCPI",
1616         },
1617         {
1618                 .type  = ERR_UNCORRECTED,
1619                 .mask  = L2C_TAD_INT_WRDISLMC,
1620                 .descr = "Write to a disabled LMC",
1621         },
1622         {
1623                 .type  = ERR_UNCORRECTED,
1624                 .mask  = L2C_TAD_INT_RDDISLMC,
1625                 .descr = "Read from a disabled LMC",
1626         },
1627         {
1628                 .type  = ERR_UNCORRECTED,
1629                 .mask  = L2C_TAD_INT_LFBTO,
1630                 .descr = "LFB entry timeout",
1631         },
1632         {
1633                 .type  = ERR_UNCORRECTED,
1634                 .mask  = L2C_TAD_INT_GSYNCTO,
1635                 .descr = "Global sync CCPI timeout",
1636         },
1637         {0, 0, NULL},
1638 };
1639
1640 #define L2C_TAD_INT_TAG         (L2C_TAD_INT_TAGDBE)
1641
1642 #define L2C_TAD_INT_RTG         (L2C_TAD_INT_RTGDBE)
1643
1644 #define L2C_TAD_INT_DISLMC      (L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC)
1645
1646 #define L2C_TAD_INT_DISOCI      (L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI)
1647
1648 #define L2C_TAD_INT_ENA_ALL     (L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \
1649                                  L2C_TAD_INT_RTG | \
1650                                  L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \
1651                                  L2C_TAD_INT_LFBTO)
1652
1653 #define L2C_TAD_TIMETWO         0x50000
1654 #define L2C_TAD_TIMEOUT         0x50100
1655 #define L2C_TAD_ERR             0x60000
1656 #define L2C_TAD_TQD_ERR         0x60100
1657 #define L2C_TAD_TTG_ERR         0x60200
1658
1659
1660 #define L2C_CBC_INT_W1C         0x60000
1661
1662 #define L2C_CBC_INT_RSDSBE       BIT(0)
1663 #define L2C_CBC_INT_RSDDBE       BIT(1)
1664
1665 #define L2C_CBC_INT_RSD          (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE)
1666
1667 #define L2C_CBC_INT_MIBSBE       BIT(4)
1668 #define L2C_CBC_INT_MIBDBE       BIT(5)
1669
1670 #define L2C_CBC_INT_MIB          (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE)
1671
1672 #define L2C_CBC_INT_IORDDISOCI   BIT(6)
1673 #define L2C_CBC_INT_IOWRDISOCI   BIT(7)
1674
1675 #define L2C_CBC_INT_IODISOCI     (L2C_CBC_INT_IORDDISOCI | \
1676                                   L2C_CBC_INT_IOWRDISOCI)
1677
1678 #define L2C_CBC_INT_CE           (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE)
1679 #define L2C_CBC_INT_UE           (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE)
1680
1681
1682 static const struct error_descr l2_cbc_errors[] = {
1683         {
1684                 .type  = ERR_CORRECTED,
1685                 .mask  = L2C_CBC_INT_RSDSBE,
1686                 .descr = "RSD single-bit error",
1687         },
1688         {
1689                 .type  = ERR_CORRECTED,
1690                 .mask  = L2C_CBC_INT_MIBSBE,
1691                 .descr = "MIB single-bit error",
1692         },
1693         {
1694                 .type  = ERR_UNCORRECTED,
1695                 .mask  = L2C_CBC_INT_RSDDBE,
1696                 .descr = "RSD double-bit error",
1697         },
1698         {
1699                 .type  = ERR_UNCORRECTED,
1700                 .mask  = L2C_CBC_INT_MIBDBE,
1701                 .descr = "MIB double-bit error",
1702         },
1703         {
1704                 .type  = ERR_UNCORRECTED,
1705                 .mask  = L2C_CBC_INT_IORDDISOCI,
1706                 .descr = "Read from a disabled CCPI",
1707         },
1708         {
1709                 .type  = ERR_UNCORRECTED,
1710                 .mask  = L2C_CBC_INT_IOWRDISOCI,
1711                 .descr = "Write to a disabled CCPI",
1712         },
1713         {0, 0, NULL},
1714 };
1715
1716 #define L2C_CBC_INT_W1S         0x60008
1717 #define L2C_CBC_INT_ENA_W1C     0x60020
1718
1719 #define L2C_CBC_INT_ENA_ALL      (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \
1720                                   L2C_CBC_INT_IODISOCI)
1721
1722 #define L2C_CBC_INT_ENA_W1S     0x60028
1723
1724 #define L2C_CBC_IODISOCIERR     0x80008
1725 #define L2C_CBC_IOCERR          0x80010
1726 #define L2C_CBC_RSDERR          0x80018
1727 #define L2C_CBC_MIBERR          0x80020
1728
1729
1730 #define L2C_MCI_INT_W1C         0x0
1731
1732 #define L2C_MCI_INT_VBFSBE       BIT(0)
1733 #define L2C_MCI_INT_VBFDBE       BIT(1)
1734
1735 static const struct error_descr l2_mci_errors[] = {
1736         {
1737                 .type  = ERR_CORRECTED,
1738                 .mask  = L2C_MCI_INT_VBFSBE,
1739                 .descr = "VBF single-bit error",
1740         },
1741         {
1742                 .type  = ERR_UNCORRECTED,
1743                 .mask  = L2C_MCI_INT_VBFDBE,
1744                 .descr = "VBF double-bit error",
1745         },
1746         {0, 0, NULL},
1747 };
1748
1749 #define L2C_MCI_INT_W1S         0x8
1750 #define L2C_MCI_INT_ENA_W1C     0x20
1751
1752 #define L2C_MCI_INT_ENA_ALL      (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE)
1753
1754 #define L2C_MCI_INT_ENA_W1S     0x28
1755
1756 #define L2C_MCI_ERR             0x10000
1757
1758 #define L2C_MESSAGE_SIZE        SZ_1K
1759 #define L2C_OTHER_SIZE          (50 * ARRAY_SIZE(l2_tad_errors))
1760
1761 struct l2c_err_ctx {
1762         char *reg_ext_name;
1763         u64  reg_int;
1764         u64  reg_ext;
1765 };
1766
1767 struct thunderx_l2c {
1768         void __iomem *regs;
1769         struct pci_dev *pdev;
1770         struct edac_device_ctl_info *edac_dev;
1771
1772         struct dentry *debugfs;
1773
1774         int index;
1775
1776         struct msix_entry msix_ent;
1777
1778         struct l2c_err_ctx err_ctx[RING_ENTRIES];
1779         unsigned long ring_head;
1780         unsigned long ring_tail;
1781 };
1782
1783 static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id)
1784 {
1785         struct msix_entry *msix = irq_id;
1786         struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,
1787                                                 msix_ent);
1788
1789         unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));
1790         struct l2c_err_ctx *ctx = &tad->err_ctx[head];
1791
1792         ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);
1793
1794         if (ctx->reg_int & L2C_TAD_INT_ECC) {
1795                 ctx->reg_ext_name = "TQD_ERR";
1796                 ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);
1797         } else if (ctx->reg_int & L2C_TAD_INT_TAG) {
1798                 ctx->reg_ext_name = "TTG_ERR";
1799                 ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);
1800         } else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {
1801                 ctx->reg_ext_name = "TIMEOUT";
1802                 ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);
1803         } else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {
1804                 ctx->reg_ext_name = "ERR";
1805                 ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
1806         }
1807
1808         writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);
1809
1810         tad->ring_head++;
1811
1812         return IRQ_WAKE_THREAD;
1813 }
1814
1815 static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id)
1816 {
1817         struct msix_entry *msix = irq_id;
1818         struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c,
1819                                                 msix_ent);
1820
1821         unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));
1822         struct l2c_err_ctx *ctx = &cbc->err_ctx[head];
1823
1824         ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);
1825
1826         if (ctx->reg_int & L2C_CBC_INT_RSD) {
1827                 ctx->reg_ext_name = "RSDERR";
1828                 ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);
1829         } else if (ctx->reg_int & L2C_CBC_INT_MIB) {
1830                 ctx->reg_ext_name = "MIBERR";
1831                 ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);
1832         } else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {
1833                 ctx->reg_ext_name = "IODISOCIERR";
1834                 ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);
1835         }
1836
1837         writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);
1838
1839         cbc->ring_head++;
1840
1841         return IRQ_WAKE_THREAD;
1842 }
1843
1844 static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id)
1845 {
1846         struct msix_entry *msix = irq_id;
1847         struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c,
1848                                                 msix_ent);
1849
1850         unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));
1851         struct l2c_err_ctx *ctx = &mci->err_ctx[head];
1852
1853         ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);
1854         ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);
1855
1856         writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);
1857
1858         ctx->reg_ext_name = "ERR";
1859
1860         mci->ring_head++;
1861
1862         return IRQ_WAKE_THREAD;
1863 }
1864
1865 static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id)
1866 {
1867         struct msix_entry *msix = irq_id;
1868         struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c,
1869                                                 msix_ent);
1870
1871         unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));
1872         struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];
1873         irqreturn_t ret = IRQ_NONE;
1874
1875         u64 mask_ue, mask_ce;
1876         const struct error_descr *l2_errors;
1877         char *reg_int_name;
1878
1879         char *msg;
1880         char *other;
1881
1882         msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1883         other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1884
1885         if (!msg || !other)
1886                 goto err_free;
1887
1888         switch (l2c->pdev->device) {
1889         case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1890                 reg_int_name = "L2C_TAD_INT";
1891                 mask_ue = L2C_TAD_INT_UE;
1892                 mask_ce = L2C_TAD_INT_CE;
1893                 l2_errors = l2_tad_errors;
1894                 break;
1895         case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1896                 reg_int_name = "L2C_CBC_INT";
1897                 mask_ue = L2C_CBC_INT_UE;
1898                 mask_ce = L2C_CBC_INT_CE;
1899                 l2_errors = l2_cbc_errors;
1900                 break;
1901         case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1902                 reg_int_name = "L2C_MCI_INT";
1903                 mask_ue = L2C_MCI_INT_VBFDBE;
1904                 mask_ce = L2C_MCI_INT_VBFSBE;
1905                 l2_errors = l2_mci_errors;
1906                 break;
1907         default:
1908                 dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",
1909                         l2c->pdev->device);
1910                 return IRQ_NONE;
1911         }
1912
1913         while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,
1914                         ARRAY_SIZE(l2c->err_ctx))) {
1915                 snprintf(msg, L2C_MESSAGE_SIZE,
1916                          "%s: %s: %016llx, %s: %016llx",
1917                          l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,
1918                          ctx->reg_ext_name, ctx->reg_ext);
1919
1920                 decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);
1921
1922                 strncat(msg, other, L2C_MESSAGE_SIZE);
1923
1924                 if (ctx->reg_int & mask_ue)
1925                         edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);
1926                 else if (ctx->reg_int & mask_ce)
1927                         edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);
1928
1929                 l2c->ring_tail++;
1930         }
1931
1932         return IRQ_HANDLED;
1933
1934 err_free:
1935         kfree(other);
1936         kfree(msg);
1937
1938         return ret;
1939 }
1940
1941 #define L2C_DEBUGFS_ATTR(_name, _reg)   DEBUGFS_REG_ATTR(l2c, _name, _reg)
1942
1943 L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S);
1944
1945 struct debugfs_entry *l2c_tad_dfs_ents[] = {
1946         &debugfs_tad_int,
1947 };
1948
1949 L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S);
1950
1951 struct debugfs_entry *l2c_cbc_dfs_ents[] = {
1952         &debugfs_cbc_int,
1953 };
1954
1955 L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S);
1956
1957 struct debugfs_entry *l2c_mci_dfs_ents[] = {
1958         &debugfs_mci_int,
1959 };
1960
1961 static const struct pci_device_id thunderx_l2c_pci_tbl[] = {
1962         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), },
1963         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), },
1964         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), },
1965         { 0, },
1966 };
1967
1968 static int thunderx_l2c_probe(struct pci_dev *pdev,
1969                               const struct pci_device_id *id)
1970 {
1971         struct thunderx_l2c *l2c;
1972         struct edac_device_ctl_info *edac_dev;
1973         struct debugfs_entry **l2c_devattr;
1974         size_t dfs_entries;
1975         irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL;
1976         char name[32];
1977         const char *fmt;
1978         u64 reg_en_offs, reg_en_mask;
1979         int idx;
1980         int ret;
1981
1982         ret = pcim_enable_device(pdev);
1983         if (ret) {
1984                 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1985                 return ret;
1986         }
1987
1988         ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c");
1989         if (ret) {
1990                 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1991                 return ret;
1992         }
1993
1994         switch (pdev->device) {
1995         case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1996                 thunderx_l2c_isr = thunderx_l2c_tad_isr;
1997                 l2c_devattr = l2c_tad_dfs_ents;
1998                 dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents);
1999                 fmt = "L2C-TAD%d";
2000                 reg_en_offs = L2C_TAD_INT_ENA_W1S;
2001                 reg_en_mask = L2C_TAD_INT_ENA_ALL;
2002                 break;
2003         case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2004                 thunderx_l2c_isr = thunderx_l2c_cbc_isr;
2005                 l2c_devattr = l2c_cbc_dfs_ents;
2006                 dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents);
2007                 fmt = "L2C-CBC%d";
2008                 reg_en_offs = L2C_CBC_INT_ENA_W1S;
2009                 reg_en_mask = L2C_CBC_INT_ENA_ALL;
2010                 break;
2011         case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2012                 thunderx_l2c_isr = thunderx_l2c_mci_isr;
2013                 l2c_devattr = l2c_mci_dfs_ents;
2014                 dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents);
2015                 fmt = "L2C-MCI%d";
2016                 reg_en_offs = L2C_MCI_INT_ENA_W1S;
2017                 reg_en_mask = L2C_MCI_INT_ENA_ALL;
2018                 break;
2019         default:
2020                 //Should never ever get here
2021                 dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",
2022                         pdev->device);
2023                 return -EINVAL;
2024         }
2025
2026         idx = edac_device_alloc_index();
2027         snprintf(name, sizeof(name), fmt, idx);
2028
2029         edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c),
2030                                               name, 1, "L2C", 1, 0,
2031                                               NULL, 0, idx);
2032         if (!edac_dev) {
2033                 dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
2034                 return -ENOMEM;
2035         }
2036
2037         l2c = edac_dev->pvt_info;
2038         l2c->edac_dev = edac_dev;
2039
2040         l2c->regs = pcim_iomap_table(pdev)[0];
2041         if (!l2c->regs) {
2042                 dev_err(&pdev->dev, "Cannot map PCI resources\n");
2043                 ret = -ENODEV;
2044                 goto err_free;
2045         }
2046
2047         l2c->pdev = pdev;
2048
2049         l2c->ring_head = 0;
2050         l2c->ring_tail = 0;
2051
2052         l2c->msix_ent.entry = 0;
2053         l2c->msix_ent.vector = 0;
2054
2055         ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);
2056         if (ret) {
2057                 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
2058                 goto err_free;
2059         }
2060
2061         ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,
2062                                         thunderx_l2c_isr,
2063                                         thunderx_l2c_threaded_isr,
2064                                         0, "[EDAC] ThunderX L2C",
2065                                         &l2c->msix_ent);
2066         if (ret)
2067                 goto err_free;
2068
2069         edac_dev->dev = &pdev->dev;
2070         edac_dev->dev_name = dev_name(&pdev->dev);
2071         edac_dev->mod_name = "thunderx-l2c";
2072         edac_dev->ctl_name = "thunderx-l2c";
2073
2074         ret = edac_device_add_device(edac_dev);
2075         if (ret) {
2076                 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
2077                 goto err_free;
2078         }
2079
2080         if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
2081                 l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
2082
2083                 ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,
2084                                               l2c, dfs_entries);
2085
2086                 if (ret != dfs_entries) {
2087                         dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
2088                                  ret, ret >= 0 ? " created" : "");
2089                 }
2090         }
2091
2092         pci_set_drvdata(pdev, edac_dev);
2093
2094         writeq(reg_en_mask, l2c->regs + reg_en_offs);
2095
2096         return 0;
2097
2098 err_free:
2099         edac_device_free_ctl_info(edac_dev);
2100
2101         return ret;
2102 }
2103
2104 static void thunderx_l2c_remove(struct pci_dev *pdev)
2105 {
2106         struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
2107         struct thunderx_l2c *l2c = edac_dev->pvt_info;
2108
2109         switch (pdev->device) {
2110         case PCI_DEVICE_ID_THUNDER_L2C_TAD:
2111                 writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);
2112                 break;
2113         case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2114                 writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);
2115                 break;
2116         case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2117                 writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);
2118                 break;
2119         }
2120
2121         edac_debugfs_remove_recursive(l2c->debugfs);
2122
2123         edac_device_del_device(&pdev->dev);
2124         edac_device_free_ctl_info(edac_dev);
2125 }
2126
2127 MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl);
2128
2129 static struct pci_driver thunderx_l2c_driver = {
2130         .name     = "thunderx_l2c_edac",
2131         .probe    = thunderx_l2c_probe,
2132         .remove   = thunderx_l2c_remove,
2133         .id_table = thunderx_l2c_pci_tbl,
2134 };
2135
2136 static int __init thunderx_edac_init(void)
2137 {
2138         int rc = 0;
2139
2140         rc = pci_register_driver(&thunderx_lmc_driver);
2141         if (rc)
2142                 return rc;
2143
2144         rc = pci_register_driver(&thunderx_ocx_driver);
2145         if (rc)
2146                 goto err_lmc;
2147
2148         rc = pci_register_driver(&thunderx_l2c_driver);
2149         if (rc)
2150                 goto err_ocx;
2151
2152         return rc;
2153 err_ocx:
2154         pci_unregister_driver(&thunderx_ocx_driver);
2155 err_lmc:
2156         pci_unregister_driver(&thunderx_lmc_driver);
2157
2158         return rc;
2159 }
2160
2161 static void __exit thunderx_edac_exit(void)
2162 {
2163         pci_unregister_driver(&thunderx_l2c_driver);
2164         pci_unregister_driver(&thunderx_ocx_driver);
2165         pci_unregister_driver(&thunderx_lmc_driver);
2166
2167 }
2168
2169 module_init(thunderx_edac_init);
2170 module_exit(thunderx_edac_exit);
2171
2172 MODULE_LICENSE("GPL v2");
2173 MODULE_AUTHOR("Cavium, Inc.");
2174 MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX");