drivers/irqchip/irq-gic-v3-its.c

   1 /*
   2  * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/bitmap.h>
  19 #include <linux/cpu.h>
  20 #include <linux/delay.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/log2.h>
  23 #include <linux/mm.h>
  24 #include <linux/msi.h>
  25 #include <linux/of.h>
  26 #include <linux/of_address.h>
  27 #include <linux/of_irq.h>
  28 #include <linux/of_pci.h>
  29 #include <linux/of_platform.h>
  30 #include <linux/percpu.h>
  31 #include <linux/slab.h>
  32
  33 #include <linux/irqchip.h>
  34 #include <linux/irqchip/arm-gic-v3.h>
  35
  36 #include <asm/cacheflush.h>
  37 #include <asm/cputype.h>
  38 #include <asm/exception.h>
  39
  40 #include "irq-gic-common.h"
  41
  42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING           (1ULL << 0)
  43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375       (1ULL << 1)
  44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144       (1ULL << 2)
  45
  46 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING     (1 << 0)
  47
  48 /*
  49  * Collection structure - just an ID, and a redistributor address to
  50  * ping. We use one per CPU as a bag of interrupts assigned to this
  51  * CPU.
  52  */
  53 struct its_collection {
  54         u64                     target_address;
  55         u16                     col_id;
  56 };
  57
  58 /*
  59  * The ITS_BASER structure - contains memory information, cached
  60  * value of BASER register configuration and ITS page size.
  61  */
  62 struct its_baser {
  63         void            *base;
  64         u64             val;
  65         u32             order;
  66         u32             psz;
  67 };
  68
  69 /*
  70  * The ITS structure - contains most of the infrastructure, with the
  71  * top-level MSI domain, the command queue, the collections, and the
  72  * list of devices writing to it.
  73  */
  74 struct its_node {
  75         raw_spinlock_t          lock;
  76         struct list_head        entry;
  77         void __iomem            *base;
  78         unsigned long           phys_base;
  79         struct its_cmd_block    *cmd_base;
  80         struct its_cmd_block    *cmd_write;
  81         struct its_baser        tables[GITS_BASER_NR_REGS];
  82         struct its_collection   *collections;
  83         struct list_head        its_device_list;
  84         u64                     flags;
  85         u32                     ite_size;
  86         u32                     device_ids;
  87         int                     numa_node;
  88 };
  89
  90 #define ITS_ITT_ALIGN           SZ_256
  91
  92 /* Convert page order to size in bytes */
  93 #define PAGE_ORDER_TO_SIZE(o)   (PAGE_SIZE << (o))
  94
  95 struct event_lpi_map {
  96         unsigned long           *lpi_map;
  97         u16                     *col_map;
  98         irq_hw_number_t         lpi_base;
  99         int                     nr_lpis;
 100 };
 101
 102 /*
 103  * The ITS view of a device - belongs to an ITS, a collection, owns an
 104  * interrupt translation table, and a list of interrupts.
 105  */
 106 struct its_device {
 107         struct list_head        entry;
 108         struct its_node         *its;
 109         struct event_lpi_map    event_map;
 110         void                    *itt;
 111         u32                     nr_ites;
 112         u32                     device_id;
 113 };
 114
 115 static LIST_HEAD(its_nodes);
 116 static DEFINE_SPINLOCK(its_lock);
 117 static struct rdists *gic_rdists;
 118
 119 #define gic_data_rdist()                (raw_cpu_ptr(gic_rdists->rdist))
 120 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 121
 122 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
 123                                                u32 event)
 124 {
 125         struct its_node *its = its_dev->its;
 126
 127         return its->collections + its_dev->event_map.col_map[event];
 128 }
 129
 130 /*
 131  * ITS command descriptors - parameters to be encoded in a command
 132  * block.
 133  */
 134 struct its_cmd_desc {
 135         union {
 136                 struct {
 137                         struct its_device *dev;
 138                         u32 event_id;
 139                 } its_inv_cmd;
 140
 141                 struct {
 142                         struct its_device *dev;
 143                         u32 event_id;
 144                 } its_int_cmd;
 145
 146                 struct {
 147                         struct its_device *dev;
 148                         int valid;
 149                 } its_mapd_cmd;
 150
 151                 struct {
 152                         struct its_collection *col;
 153                         int valid;
 154                 } its_mapc_cmd;
 155
 156                 struct {
 157                         struct its_device *dev;
 158                         u32 phys_id;
 159                         u32 event_id;
 160                 } its_mapvi_cmd;
 161
 162                 struct {
 163                         struct its_device *dev;
 164                         struct its_collection *col;
 165                         u32 event_id;
 166                 } its_movi_cmd;
 167
 168                 struct {
 169                         struct its_device *dev;
 170                         u32 event_id;
 171                 } its_discard_cmd;
 172
 173                 struct {
 174                         struct its_collection *col;
 175                 } its_invall_cmd;
 176         };
 177 };
 178
 179 /*
 180  * The ITS command block, which is what the ITS actually parses.
 181  */
 182 struct its_cmd_block {
 183         u64     raw_cmd[4];
 184 };
 185
 186 #define ITS_CMD_QUEUE_SZ                SZ_64K
 187 #define ITS_CMD_QUEUE_NR_ENTRIES        (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
 188
 189 typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *,
 190                                                     struct its_cmd_desc *);
 191
 192 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
 193 {
 194         cmd->raw_cmd[0] &= ~0xffUL;
 195         cmd->raw_cmd[0] |= cmd_nr;
 196 }
 197
 198 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
 199 {
 200         cmd->raw_cmd[0] &= BIT_ULL(32) - 1;
 201         cmd->raw_cmd[0] |= ((u64)devid) << 32;
 202 }
 203
 204 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
 205 {
 206         cmd->raw_cmd[1] &= ~0xffffffffUL;
 207         cmd->raw_cmd[1] |= id;
 208 }
 209
 210 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
 211 {
 212         cmd->raw_cmd[1] &= 0xffffffffUL;
 213         cmd->raw_cmd[1] |= ((u64)phys_id) << 32;
 214 }
 215
 216 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
 217 {
 218         cmd->raw_cmd[1] &= ~0x1fUL;
 219         cmd->raw_cmd[1] |= size & 0x1f;
 220 }
 221
 222 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
 223 {
 224         cmd->raw_cmd[2] &= ~0xffffffffffffUL;
 225         cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL;
 226 }
 227
 228 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
 229 {
 230         cmd->raw_cmd[2] &= ~(1UL << 63);
 231         cmd->raw_cmd[2] |= ((u64)!!valid) << 63;
 232 }
 233
 234 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
 235 {
 236         cmd->raw_cmd[2] &= ~(0xffffffffUL << 16);
 237         cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16));
 238 }
 239
 240 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
 241 {
 242         cmd->raw_cmd[2] &= ~0xffffUL;
 243         cmd->raw_cmd[2] |= col;
 244 }
 245
 246 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
 247 {
 248         /* Let's fixup BE commands */
 249         cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]);
 250         cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]);
 251         cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]);
 252         cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);
 253 }
 254
 255 static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd,
 256                                                  struct its_cmd_desc *desc)
 257 {
 258         unsigned long itt_addr;
 259         u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
 260
 261         itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
 262         itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
 263
 264         its_encode_cmd(cmd, GITS_CMD_MAPD);
 265         its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
 266         its_encode_size(cmd, size - 1);
 267         its_encode_itt(cmd, itt_addr);
 268         its_encode_valid(cmd, desc->its_mapd_cmd.valid);
 269
 270         its_fixup_cmd(cmd);
 271
 272         return NULL;
 273 }
 274
 275 static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd,
 276                                                  struct its_cmd_desc *desc)
 277 {
 278         its_encode_cmd(cmd, GITS_CMD_MAPC);
 279         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 280         its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
 281         its_encode_valid(cmd, desc->its_mapc_cmd.valid);
 282
 283         its_fixup_cmd(cmd);
 284
 285         return desc->its_mapc_cmd.col;
 286 }
 287
 288 static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd,
 289                                                   struct its_cmd_desc *desc)
 290 {
 291         struct its_collection *col;
 292
 293         col = dev_event_to_col(desc->its_mapvi_cmd.dev,
 294                                desc->its_mapvi_cmd.event_id);
 295
 296         its_encode_cmd(cmd, GITS_CMD_MAPVI);
 297         its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id);
 298         its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id);
 299         its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id);
 300         its_encode_collection(cmd, col->col_id);
 301
 302         its_fixup_cmd(cmd);
 303
 304         return col;
 305 }
 306
 307 static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd,
 308                                                  struct its_cmd_desc *desc)
 309 {
 310         struct its_collection *col;
 311
 312         col = dev_event_to_col(desc->its_movi_cmd.dev,
 313                                desc->its_movi_cmd.event_id);
 314
 315         its_encode_cmd(cmd, GITS_CMD_MOVI);
 316         its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
 317         its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
 318         its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
 319
 320         its_fixup_cmd(cmd);
 321
 322         return col;
 323 }
 324
 325 static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd,
 326                                                     struct its_cmd_desc *desc)
 327 {
 328         struct its_collection *col;
 329
 330         col = dev_event_to_col(desc->its_discard_cmd.dev,
 331                                desc->its_discard_cmd.event_id);
 332
 333         its_encode_cmd(cmd, GITS_CMD_DISCARD);
 334         its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
 335         its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
 336
 337         its_fixup_cmd(cmd);
 338
 339         return col;
 340 }
 341
 342 static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd,
 343                                                 struct its_cmd_desc *desc)
 344 {
 345         struct its_collection *col;
 346
 347         col = dev_event_to_col(desc->its_inv_cmd.dev,
 348                                desc->its_inv_cmd.event_id);
 349
 350         its_encode_cmd(cmd, GITS_CMD_INV);
 351         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 352         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 353
 354         its_fixup_cmd(cmd);
 355
 356         return col;
 357 }
 358
 359 static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd,
 360                                                    struct its_cmd_desc *desc)
 361 {
 362         its_encode_cmd(cmd, GITS_CMD_INVALL);
 363         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 364
 365         its_fixup_cmd(cmd);
 366
 367         return NULL;
 368 }
 369
 370 static u64 its_cmd_ptr_to_offset(struct its_node *its,
 371                                  struct its_cmd_block *ptr)
 372 {
 373         return (ptr - its->cmd_base) * sizeof(*ptr);
 374 }
 375
 376 static int its_queue_full(struct its_node *its)
 377 {
 378         int widx;
 379         int ridx;
 380
 381         widx = its->cmd_write - its->cmd_base;
 382         ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
 383
 384         /* This is incredibly unlikely to happen, unless the ITS locks up. */
 385         if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
 386                 return 1;
 387
 388         return 0;
 389 }
 390
 391 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
 392 {
 393         struct its_cmd_block *cmd;
 394         u32 count = 1000000;    /* 1s! */
 395
 396         while (its_queue_full(its)) {
 397                 count--;
 398                 if (!count) {
 399                         pr_err_ratelimited("ITS queue not draining\n");
 400                         return NULL;
 401                 }
 402                 cpu_relax();
 403                 udelay(1);
 404         }
 405
 406         cmd = its->cmd_write++;
 407
 408         /* Handle queue wrapping */
 409         if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
 410                 its->cmd_write = its->cmd_base;
 411
 412         return cmd;
 413 }
 414
 415 static struct its_cmd_block *its_post_commands(struct its_node *its)
 416 {
 417         u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
 418
 419         writel_relaxed(wr, its->base + GITS_CWRITER);
 420
 421         return its->cmd_write;
 422 }
 423
 424 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
 425 {
 426         /*
 427          * Make sure the commands written to memory are observable by
 428          * the ITS.
 429          */
 430         if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
 431                 __flush_dcache_area(cmd, sizeof(*cmd));
 432         else
 433                 dsb(ishst);
 434 }
 435
 436 static void its_wait_for_range_completion(struct its_node *its,
 437                                           struct its_cmd_block *from,
 438                                           struct its_cmd_block *to)
 439 {
 440         u64 rd_idx, from_idx, to_idx;
 441         u32 count = 1000000;    /* 1s! */
 442
 443         from_idx = its_cmd_ptr_to_offset(its, from);
 444         to_idx = its_cmd_ptr_to_offset(its, to);
 445
 446         while (1) {
 447                 rd_idx = readl_relaxed(its->base + GITS_CREADR);
 448                 if (rd_idx >= to_idx || rd_idx < from_idx)
 449                         break;
 450
 451                 count--;
 452                 if (!count) {
 453                         pr_err_ratelimited("ITS queue timeout\n");
 454                         return;
 455                 }
 456                 cpu_relax();
 457                 udelay(1);
 458         }
 459 }
 460
 461 static void its_send_single_command(struct its_node *its,
 462                                     its_cmd_builder_t builder,
 463                                     struct its_cmd_desc *desc)
 464 {
 465         struct its_cmd_block *cmd, *sync_cmd, *next_cmd;
 466         struct its_collection *sync_col;
 467         unsigned long flags;
 468
 469         raw_spin_lock_irqsave(&its->lock, flags);
 470
 471         cmd = its_allocate_entry(its);
 472         if (!cmd) {             /* We're soooooo screewed... */
 473                 pr_err_ratelimited("ITS can't allocate, dropping command\n");
 474                 raw_spin_unlock_irqrestore(&its->lock, flags);
 475                 return;
 476         }
 477         sync_col = builder(cmd, desc);
 478         its_flush_cmd(its, cmd);
 479
 480         if (sync_col) {
 481                 sync_cmd = its_allocate_entry(its);
 482                 if (!sync_cmd) {
 483                         pr_err_ratelimited("ITS can't SYNC, skipping\n");
 484                         goto post;
 485                 }
 486                 its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
 487                 its_encode_target(sync_cmd, sync_col->target_address);
 488                 its_fixup_cmd(sync_cmd);
 489                 its_flush_cmd(its, sync_cmd);
 490         }
 491
 492 post:
 493         next_cmd = its_post_commands(its);
 494         raw_spin_unlock_irqrestore(&its->lock, flags);
 495
 496         its_wait_for_range_completion(its, cmd, next_cmd);
 497 }
 498
 499 static void its_send_inv(struct its_device *dev, u32 event_id)
 500 {
 501         struct its_cmd_desc desc;
 502
 503         desc.its_inv_cmd.dev = dev;
 504         desc.its_inv_cmd.event_id = event_id;
 505
 506         its_send_single_command(dev->its, its_build_inv_cmd, &desc);
 507 }
 508
 509 static void its_send_mapd(struct its_device *dev, int valid)
 510 {
 511         struct its_cmd_desc desc;
 512
 513         desc.its_mapd_cmd.dev = dev;
 514         desc.its_mapd_cmd.valid = !!valid;
 515
 516         its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
 517 }
 518
 519 static void its_send_mapc(struct its_node *its, struct its_collection *col,
 520                           int valid)
 521 {
 522         struct its_cmd_desc desc;
 523
 524         desc.its_mapc_cmd.col = col;
 525         desc.its_mapc_cmd.valid = !!valid;
 526
 527         its_send_single_command(its, its_build_mapc_cmd, &desc);
 528 }
 529
 530 static void its_send_mapvi(struct its_device *dev, u32 irq_id, u32 id)
 531 {
 532         struct its_cmd_desc desc;
 533
 534         desc.its_mapvi_cmd.dev = dev;
 535         desc.its_mapvi_cmd.phys_id = irq_id;
 536         desc.its_mapvi_cmd.event_id = id;
 537
 538         its_send_single_command(dev->its, its_build_mapvi_cmd, &desc);
 539 }
 540
 541 static void its_send_movi(struct its_device *dev,
 542                           struct its_collection *col, u32 id)
 543 {
 544         struct its_cmd_desc desc;
 545
 546         desc.its_movi_cmd.dev = dev;
 547         desc.its_movi_cmd.col = col;
 548         desc.its_movi_cmd.event_id = id;
 549
 550         its_send_single_command(dev->its, its_build_movi_cmd, &desc);
 551 }
 552
 553 static void its_send_discard(struct its_device *dev, u32 id)
 554 {
 555         struct its_cmd_desc desc;
 556
 557         desc.its_discard_cmd.dev = dev;
 558         desc.its_discard_cmd.event_id = id;
 559
 560         its_send_single_command(dev->its, its_build_discard_cmd, &desc);
 561 }
 562
 563 static void its_send_invall(struct its_node *its, struct its_collection *col)
 564 {
 565         struct its_cmd_desc desc;
 566
 567         desc.its_invall_cmd.col = col;
 568
 569         its_send_single_command(its, its_build_invall_cmd, &desc);
 570 }
 571
 572 /*
 573  * irqchip functions - assumes MSI, mostly.
 574  */
 575
 576 static inline u32 its_get_event_id(struct irq_data *d)
 577 {
 578         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 579         return d->hwirq - its_dev->event_map.lpi_base;
 580 }
 581
 582 static void lpi_set_config(struct irq_data *d, bool enable)
 583 {
 584         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 585         irq_hw_number_t hwirq = d->hwirq;
 586         u32 id = its_get_event_id(d);
 587         u8 *cfg = page_address(gic_rdists->prop_page) + hwirq - 8192;
 588
 589         if (enable)
 590                 *cfg |= LPI_PROP_ENABLED;
 591         else
 592                 *cfg &= ~LPI_PROP_ENABLED;
 593
 594         /*
 595          * Make the above write visible to the redistributors.
 596          * And yes, we're flushing exactly: One. Single. Byte.
 597          * Humpf...
 598          */
 599         if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
 600                 __flush_dcache_area(cfg, sizeof(*cfg));
 601         else
 602                 dsb(ishst);
 603         its_send_inv(its_dev, id);
 604 }
 605
 606 static void its_mask_irq(struct irq_data *d)
 607 {
 608         lpi_set_config(d, false);
 609 }
 610
 611 static void its_unmask_irq(struct irq_data *d)
 612 {
 613         lpi_set_config(d, true);
 614 }
 615
 616 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 617                             bool force)
 618 {
 619         unsigned int cpu;
 620         const struct cpumask *cpu_mask = cpu_online_mask;
 621         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 622         struct its_collection *target_col;
 623         u32 id = its_get_event_id(d);
 624
 625        /* lpi cannot be routed to a redistributor that is on a foreign node */
 626         if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
 627                 if (its_dev->its->numa_node >= 0) {
 628                         cpu_mask = cpumask_of_node(its_dev->its->numa_node);
 629                         if (!cpumask_intersects(mask_val, cpu_mask))
 630                                 return -EINVAL;
 631                 }
 632         }
 633
 634         cpu = cpumask_any_and(mask_val, cpu_mask);
 635
 636         if (cpu >= nr_cpu_ids)
 637                 return -EINVAL;
 638
 639         target_col = &its_dev->its->collections[cpu];
 640         its_send_movi(its_dev, target_col, id);
 641         its_dev->event_map.col_map[id] = cpu;
 642
 643         return IRQ_SET_MASK_OK_DONE;
 644 }
 645
 646 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
 647 {
 648         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 649         struct its_node *its;
 650         u64 addr;
 651
 652         its = its_dev->its;
 653         addr = its->phys_base + GITS_TRANSLATER;
 654
 655         msg->address_lo         = addr & ((1UL << 32) - 1);
 656         msg->address_hi         = addr >> 32;
 657         msg->data               = its_get_event_id(d);
 658 }
 659
 660 static struct irq_chip its_irq_chip = {
 661         .name                   = "ITS",
 662         .irq_mask               = its_mask_irq,
 663         .irq_unmask             = its_unmask_irq,
 664         .irq_eoi                = irq_chip_eoi_parent,
 665         .irq_set_affinity       = its_set_affinity,
 666         .irq_compose_msi_msg    = its_irq_compose_msi_msg,
 667 };
 668
 669 /*
 670  * How we allocate LPIs:
 671  *
 672  * The GIC has id_bits bits for interrupt identifiers. From there, we
 673  * must subtract 8192 which are reserved for SGIs/PPIs/SPIs. Then, as
 674  * we allocate LPIs by chunks of 32, we can shift the whole thing by 5
 675  * bits to the right.
 676  *
 677  * This gives us (((1UL << id_bits) - 8192) >> 5) possible allocations.
 678  */
 679 #define IRQS_PER_CHUNK_SHIFT    5
 680 #define IRQS_PER_CHUNK          (1 << IRQS_PER_CHUNK_SHIFT)
 681
 682 static unsigned long *lpi_bitmap;
 683 static u32 lpi_chunks;
 684 static DEFINE_SPINLOCK(lpi_lock);
 685
 686 static int its_lpi_to_chunk(int lpi)
 687 {
 688         return (lpi - 8192) >> IRQS_PER_CHUNK_SHIFT;
 689 }
 690
 691 static int its_chunk_to_lpi(int chunk)
 692 {
 693         return (chunk << IRQS_PER_CHUNK_SHIFT) + 8192;
 694 }
 695
 696 static int __init its_lpi_init(u32 id_bits)
 697 {
 698         lpi_chunks = its_lpi_to_chunk(1UL << id_bits);
 699
 700         lpi_bitmap = kzalloc(BITS_TO_LONGS(lpi_chunks) * sizeof(long),
 701                              GFP_KERNEL);
 702         if (!lpi_bitmap) {
 703                 lpi_chunks = 0;
 704                 return -ENOMEM;
 705         }
 706
 707         pr_info("ITS: Allocated %d chunks for LPIs\n", (int)lpi_chunks);
 708         return 0;
 709 }
 710
 711 static unsigned long *its_lpi_alloc_chunks(int nr_irqs, int *base, int *nr_ids)
 712 {
 713         unsigned long *bitmap = NULL;
 714         int chunk_id;
 715         int nr_chunks;
 716         int i;
 717
 718         nr_chunks = DIV_ROUND_UP(nr_irqs, IRQS_PER_CHUNK);
 719
 720         spin_lock(&lpi_lock);
 721
 722         do {
 723                 chunk_id = bitmap_find_next_zero_area(lpi_bitmap, lpi_chunks,
 724                                                       0, nr_chunks, 0);
 725                 if (chunk_id < lpi_chunks)
 726                         break;
 727
 728                 nr_chunks--;
 729         } while (nr_chunks > 0);
 730
 731         if (!nr_chunks)
 732                 goto out;
 733
 734         bitmap = kzalloc(BITS_TO_LONGS(nr_chunks * IRQS_PER_CHUNK) * sizeof (long),
 735                          GFP_ATOMIC);
 736         if (!bitmap)
 737                 goto out;
 738
 739         for (i = 0; i < nr_chunks; i++)
 740                 set_bit(chunk_id + i, lpi_bitmap);
 741
 742         *base = its_chunk_to_lpi(chunk_id);
 743         *nr_ids = nr_chunks * IRQS_PER_CHUNK;
 744
 745 out:
 746         spin_unlock(&lpi_lock);
 747
 748         if (!bitmap)
 749                 *base = *nr_ids = 0;
 750
 751         return bitmap;
 752 }
 753
 754 static void its_lpi_free(struct event_lpi_map *map)
 755 {
 756         int base = map->lpi_base;
 757         int nr_ids = map->nr_lpis;
 758         int lpi;
 759
 760         spin_lock(&lpi_lock);
 761
 762         for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) {
 763                 int chunk = its_lpi_to_chunk(lpi);
 764                 BUG_ON(chunk > lpi_chunks);
 765                 if (test_bit(chunk, lpi_bitmap)) {
 766                         clear_bit(chunk, lpi_bitmap);
 767                 } else {
 768                         pr_err("Bad LPI chunk %d\n", chunk);
 769                 }
 770         }
 771
 772         spin_unlock(&lpi_lock);
 773
 774         kfree(map->lpi_map);
 775         kfree(map->col_map);
 776 }
 777
 778 /*
 779  * We allocate 64kB for PROPBASE. That gives us at most 64K LPIs to
 780  * deal with (one configuration byte per interrupt). PENDBASE has to
 781  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
 782  */
 783 #define LPI_PROPBASE_SZ         SZ_64K
 784 #define LPI_PENDBASE_SZ         (LPI_PROPBASE_SZ / 8 + SZ_1K)
 785
 786 /*
 787  * This is how many bits of ID we need, including the useless ones.
 788  */
 789 #define LPI_NRBITS              ilog2(LPI_PROPBASE_SZ + SZ_8K)
 790
 791 #define LPI_PROP_DEFAULT_PRIO   0xa0
 792
 793 static int __init its_alloc_lpi_tables(void)
 794 {
 795         phys_addr_t paddr;
 796
 797         gic_rdists->prop_page = alloc_pages(GFP_NOWAIT,
 798                                            get_order(LPI_PROPBASE_SZ));
 799         if (!gic_rdists->prop_page) {
 800                 pr_err("Failed to allocate PROPBASE\n");
 801                 return -ENOMEM;
 802         }
 803
 804         paddr = page_to_phys(gic_rdists->prop_page);
 805         pr_info("GIC: using LPI property table @%pa\n", &paddr);
 806
 807         /* Priority 0xa0, Group-1, disabled */
 808         memset(page_address(gic_rdists->prop_page),
 809                LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1,
 810                LPI_PROPBASE_SZ);
 811
 812         /* Make sure the GIC will observe the written configuration */
 813         __flush_dcache_area(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ);
 814
 815         return 0;
 816 }
 817
 818 static const char *its_base_type_string[] = {
 819         [GITS_BASER_TYPE_DEVICE]        = "Devices",
 820         [GITS_BASER_TYPE_VCPU]          = "Virtual CPUs",
 821         [GITS_BASER_TYPE_CPU]           = "Physical CPUs",
 822         [GITS_BASER_TYPE_COLLECTION]    = "Interrupt Collections",
 823         [GITS_BASER_TYPE_RESERVED5]     = "Reserved (5)",
 824         [GITS_BASER_TYPE_RESERVED6]     = "Reserved (6)",
 825         [GITS_BASER_TYPE_RESERVED7]     = "Reserved (7)",
 826 };
 827
 828 static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
 829 {
 830         u32 idx = baser - its->tables;
 831
 832         return readq_relaxed(its->base + GITS_BASER + (idx << 3));
 833 }
 834
 835 static void its_write_baser(struct its_node *its, struct its_baser *baser,
 836                             u64 val)
 837 {
 838         u32 idx = baser - its->tables;
 839
 840         writeq_relaxed(val, its->base + GITS_BASER + (idx << 3));
 841         baser->val = its_read_baser(its, baser);
 842 }
 843
 844 static int its_setup_baser(struct its_node *its, struct its_baser *baser,
 845                            u64 cache, u64 shr, u32 psz, u32 order,
 846                            bool indirect)
 847 {
 848         u64 val = its_read_baser(its, baser);
 849         u64 esz = GITS_BASER_ENTRY_SIZE(val);
 850         u64 type = GITS_BASER_TYPE(val);
 851         u32 alloc_pages;
 852         void *base;
 853         u64 tmp;
 854
 855 retry_alloc_baser:
 856         alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
 857         if (alloc_pages > GITS_BASER_PAGES_MAX) {
 858                 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
 859                         &its->phys_base, its_base_type_string[type],
 860                         alloc_pages, GITS_BASER_PAGES_MAX);
 861                 alloc_pages = GITS_BASER_PAGES_MAX;
 862                 order = get_order(GITS_BASER_PAGES_MAX * psz);
 863         }
 864
 865         base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
 866         if (!base)
 867                 return -ENOMEM;
 868
 869 retry_baser:
 870         val = (virt_to_phys(base)                                |
 871                 (type << GITS_BASER_TYPE_SHIFT)                  |
 872                 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)       |
 873                 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)    |
 874                 cache                                            |
 875                 shr                                              |
 876                 GITS_BASER_VALID);
 877
 878         val |=  indirect ? GITS_BASER_INDIRECT : 0x0;
 879
 880         switch (psz) {
 881         case SZ_4K:
 882                 val |= GITS_BASER_PAGE_SIZE_4K;
 883                 break;
 884         case SZ_16K:
 885                 val |= GITS_BASER_PAGE_SIZE_16K;
 886                 break;
 887         case SZ_64K:
 888                 val |= GITS_BASER_PAGE_SIZE_64K;
 889                 break;
 890         }
 891
 892         its_write_baser(its, baser, val);
 893         tmp = baser->val;
 894
 895         if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
 896                 /*
 897                  * Shareability didn't stick. Just use
 898                  * whatever the read reported, which is likely
 899                  * to be the only thing this redistributor
 900                  * supports. If that's zero, make it
 901                  * non-cacheable as well.
 902                  */
 903                 shr = tmp & GITS_BASER_SHAREABILITY_MASK;
 904                 if (!shr) {
 905                         cache = GITS_BASER_nC;
 906                         __flush_dcache_area(base, PAGE_ORDER_TO_SIZE(order));
 907                 }
 908                 goto retry_baser;
 909         }
 910
 911         if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) {
 912                 /*
 913                  * Page size didn't stick. Let's try a smaller
 914                  * size and retry. If we reach 4K, then
 915                  * something is horribly wrong...
 916                  */
 917                 free_pages((unsigned long)base, order);
 918                 baser->base = NULL;
 919
 920                 switch (psz) {
 921                 case SZ_16K:
 922                         psz = SZ_4K;
 923                         goto retry_alloc_baser;
 924                 case SZ_64K:
 925                         psz = SZ_16K;
 926                         goto retry_alloc_baser;
 927                 }
 928         }
 929
 930         if (val != tmp) {
 931                 pr_err("ITS@%pa: %s doesn't stick: %lx %lx\n",
 932                        &its->phys_base, its_base_type_string[type],
 933                        (unsigned long) val, (unsigned long) tmp);
 934                 free_pages((unsigned long)base, order);
 935                 return -ENXIO;
 936         }
 937
 938         baser->order = order;
 939         baser->base = base;
 940         baser->psz = psz;
 941         tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
 942
 943         pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
 944                 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / tmp),
 945                 its_base_type_string[type],
 946                 (unsigned long)virt_to_phys(base),
 947                 indirect ? "indirect" : "flat", (int)esz,
 948                 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
 949
 950         return 0;
 951 }
 952
 953 static bool its_parse_baser_device(struct its_node *its, struct its_baser *baser,
 954                                    u32 psz, u32 *order)
 955 {
 956         u64 esz = GITS_BASER_ENTRY_SIZE(its_read_baser(its, baser));
 957         u64 val = GITS_BASER_InnerShareable | GITS_BASER_WaWb;
 958         u32 ids = its->device_ids;
 959         u32 new_order = *order;
 960         bool indirect = false;
 961
 962         /* No need to enable Indirection if memory requirement < (psz*2)bytes */
 963         if ((esz << ids) > (psz * 2)) {
 964                 /*
 965                  * Find out whether hw supports a single or two-level table by
 966                  * table by reading bit at offset '62' after writing '1' to it.
 967                  */
 968                 its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
 969                 indirect = !!(baser->val & GITS_BASER_INDIRECT);
 970
 971                 if (indirect) {
 972                         /*
 973                          * The size of the lvl2 table is equal to ITS page size
 974                          * which is 'psz'. For computing lvl1 table size,
 975                          * subtract ID bits that sparse lvl2 table from 'ids'
 976                          * which is reported by ITS hardware times lvl1 table
 977                          * entry size.
 978                          */
 979                         ids -= ilog2(psz / esz);
 980                         esz = GITS_LVL1_ENTRY_SIZE;
 981                 }
 982         }
 983
 984         /*
 985          * Allocate as many entries as required to fit the
 986          * range of device IDs that the ITS can grok... The ID
 987          * space being incredibly sparse, this results in a
 988          * massive waste of memory if two-level device table
 989          * feature is not supported by hardware.
 990          */
 991         new_order = max_t(u32, get_order(esz << ids), new_order);
 992         if (new_order >= MAX_ORDER) {
 993                 new_order = MAX_ORDER - 1;
 994                 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / esz);
 995                 pr_warn("ITS@%pa: Device Table too large, reduce ids %u->%u\n",
 996                         &its->phys_base, its->device_ids, ids);
 997         }
 998
 999         *order = new_order;
1000
1001         return indirect;
1002 }
1003
1004 static void its_free_tables(struct its_node *its)
1005 {
1006         int i;
1007
1008         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
1009                 if (its->tables[i].base) {
1010                         free_pages((unsigned long)its->tables[i].base,
1011                                    its->tables[i].order);
1012                         its->tables[i].base = NULL;
1013                 }
1014         }
1015 }
1016
1017 static int its_alloc_tables(struct its_node *its)
1018 {
1019         u64 typer = readq_relaxed(its->base + GITS_TYPER);
1020         u32 ids = GITS_TYPER_DEVBITS(typer);
1021         u64 shr = GITS_BASER_InnerShareable;
1022         u64 cache = GITS_BASER_WaWb;
1023         u32 psz = SZ_64K;
1024         int err, i;
1025
1026         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) {
1027                 /*
1028                 * erratum 22375: only alloc 8MB table size
1029                 * erratum 24313: ignore memory access type
1030                 */
1031                 cache   = GITS_BASER_nCnB;
1032                 ids     = 0x14;                 /* 20 bits, 8MB */
1033         }
1034
1035         its->device_ids = ids;
1036
1037         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
1038                 struct its_baser *baser = its->tables + i;
1039                 u64 val = its_read_baser(its, baser);
1040                 u64 type = GITS_BASER_TYPE(val);
1041                 u32 order = get_order(psz);
1042                 bool indirect = false;
1043
1044                 if (type == GITS_BASER_TYPE_NONE)
1045                         continue;
1046
1047                 if (type == GITS_BASER_TYPE_DEVICE)
1048                         indirect = its_parse_baser_device(its, baser, psz, &order);
1049
1050                 err = its_setup_baser(its, baser, cache, shr, psz, order, indirect);
1051                 if (err < 0) {
1052                         its_free_tables(its);
1053                         return err;
1054                 }
1055
1056                 /* Update settings which will be used for next BASERn */
1057                 psz = baser->psz;
1058                 cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
1059                 shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
1060         }
1061
1062         return 0;
1063 }
1064
1065 static int its_alloc_collections(struct its_node *its)
1066 {
1067         its->collections = kzalloc(nr_cpu_ids * sizeof(*its->collections),
1068                                    GFP_KERNEL);
1069         if (!its->collections)
1070                 return -ENOMEM;
1071
1072         return 0;
1073 }
1074
1075 static void its_cpu_init_lpis(void)
1076 {
1077         void __iomem *rbase = gic_data_rdist_rd_base();
1078         struct page *pend_page;
1079         u64 val, tmp;
1080
1081         /* If we didn't allocate the pending table yet, do it now */
1082         pend_page = gic_data_rdist()->pend_page;
1083         if (!pend_page) {
1084                 phys_addr_t paddr;
1085                 /*
1086                  * The pending pages have to be at least 64kB aligned,
1087                  * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
1088                  */
1089                 pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO,
1090                                         get_order(max(LPI_PENDBASE_SZ, SZ_64K)));
1091                 if (!pend_page) {
1092                         pr_err("Failed to allocate PENDBASE for CPU%d\n",
1093                                smp_processor_id());
1094                         return;
1095                 }
1096
1097                 /* Make sure the GIC will observe the zero-ed page */
1098                 __flush_dcache_area(page_address(pend_page), LPI_PENDBASE_SZ);
1099
1100                 paddr = page_to_phys(pend_page);
1101                 pr_info("CPU%d: using LPI pending table @%pa\n",
1102                         smp_processor_id(), &paddr);
1103                 gic_data_rdist()->pend_page = pend_page;
1104         }
1105
1106         /* Disable LPIs */
1107         val = readl_relaxed(rbase + GICR_CTLR);
1108         val &= ~GICR_CTLR_ENABLE_LPIS;
1109         writel_relaxed(val, rbase + GICR_CTLR);
1110
1111         /*
1112          * Make sure any change to the table is observable by the GIC.
1113          */
1114         dsb(sy);
1115
1116         /* set PROPBASE */
1117         val = (page_to_phys(gic_rdists->prop_page) |
1118                GICR_PROPBASER_InnerShareable |
1119                GICR_PROPBASER_WaWb |
1120                ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
1121
1122         writeq_relaxed(val, rbase + GICR_PROPBASER);
1123         tmp = readq_relaxed(rbase + GICR_PROPBASER);
1124
1125         if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
1126                 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
1127                         /*
1128                          * The HW reports non-shareable, we must
1129                          * remove the cacheability attributes as
1130                          * well.
1131                          */
1132                         val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
1133                                  GICR_PROPBASER_CACHEABILITY_MASK);
1134                         val |= GICR_PROPBASER_nC;
1135                         writeq_relaxed(val, rbase + GICR_PROPBASER);
1136                 }
1137                 pr_info_once("GIC: using cache flushing for LPI property table\n");
1138                 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
1139         }
1140
1141         /* set PENDBASE */
1142         val = (page_to_phys(pend_page) |
1143                GICR_PENDBASER_InnerShareable |
1144                GICR_PENDBASER_WaWb);
1145
1146         writeq_relaxed(val, rbase + GICR_PENDBASER);
1147         tmp = readq_relaxed(rbase + GICR_PENDBASER);
1148
1149         if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
1150                 /*
1151                  * The HW reports non-shareable, we must remove the
1152                  * cacheability attributes as well.
1153                  */
1154                 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
1155                          GICR_PENDBASER_CACHEABILITY_MASK);
1156                 val |= GICR_PENDBASER_nC;
1157                 writeq_relaxed(val, rbase + GICR_PENDBASER);
1158         }
1159
1160         /* Enable LPIs */
1161         val = readl_relaxed(rbase + GICR_CTLR);
1162         val |= GICR_CTLR_ENABLE_LPIS;
1163         writel_relaxed(val, rbase + GICR_CTLR);
1164
1165         /* Make sure the GIC has seen the above */
1166         dsb(sy);
1167 }
1168
1169 static void its_cpu_init_collection(void)
1170 {
1171         struct its_node *its;
1172         int cpu;
1173
1174         spin_lock(&its_lock);
1175         cpu = smp_processor_id();
1176
1177         list_for_each_entry(its, &its_nodes, entry) {
1178                 u64 target;
1179
1180                 /* avoid cross node collections and its mapping */
1181                 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
1182                         struct device_node *cpu_node;
1183
1184                         cpu_node = of_get_cpu_node(cpu, NULL);
1185                         if (its->numa_node != NUMA_NO_NODE &&
1186                                 its->numa_node != of_node_to_nid(cpu_node))
1187                                 continue;
1188                 }
1189
1190                 /*
1191                  * We now have to bind each collection to its target
1192                  * redistributor.
1193                  */
1194                 if (readq_relaxed(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
1195                         /*
1196                          * This ITS wants the physical address of the
1197                          * redistributor.
1198                          */
1199                         target = gic_data_rdist()->phys_base;
1200                 } else {
1201                         /*
1202                          * This ITS wants a linear CPU number.
1203                          */
1204                         target = readq_relaxed(gic_data_rdist_rd_base() + GICR_TYPER);
1205                         target = GICR_TYPER_CPU_NUMBER(target) << 16;
1206                 }
1207
1208                 /* Perform collection mapping */
1209                 its->collections[cpu].target_address = target;
1210                 its->collections[cpu].col_id = cpu;
1211
1212                 its_send_mapc(its, &its->collections[cpu], 1);
1213                 its_send_invall(its, &its->collections[cpu]);
1214         }
1215
1216         spin_unlock(&its_lock);
1217 }
1218
1219 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
1220 {
1221         struct its_device *its_dev = NULL, *tmp;
1222         unsigned long flags;
1223
1224         raw_spin_lock_irqsave(&its->lock, flags);
1225
1226         list_for_each_entry(tmp, &its->its_device_list, entry) {
1227                 if (tmp->device_id == dev_id) {
1228                         its_dev = tmp;
1229                         break;
1230                 }
1231         }
1232
1233         raw_spin_unlock_irqrestore(&its->lock, flags);
1234
1235         return its_dev;
1236 }
1237
1238 static struct its_baser *its_get_baser(struct its_node *its, u32 type)
1239 {
1240         int i;
1241
1242         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
1243                 if (GITS_BASER_TYPE(its->tables[i].val) == type)
1244                         return &its->tables[i];
1245         }
1246
1247         return NULL;
1248 }
1249
1250 static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
1251 {
1252         struct its_baser *baser;
1253         struct page *page;
1254         u32 esz, idx;
1255         __le64 *table;
1256
1257         baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
1258
1259         /* Don't allow device id that exceeds ITS hardware limit */
1260         if (!baser)
1261                 return (ilog2(dev_id) < its->device_ids);
1262
1263         /* Don't allow device id that exceeds single, flat table limit */
1264         esz = GITS_BASER_ENTRY_SIZE(baser->val);
1265         if (!(baser->val & GITS_BASER_INDIRECT))
1266                 return (dev_id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
1267
1268         /* Compute 1st level table index & check if that exceeds table limit */
1269         idx = dev_id >> ilog2(baser->psz / esz);
1270         if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
1271                 return false;
1272
1273         table = baser->base;
1274
1275         /* Allocate memory for 2nd level table */
1276         if (!table[idx]) {
1277                 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(baser->psz));
1278                 if (!page)
1279                         return false;
1280
1281                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
1282                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
1283                         __flush_dcache_area(page_address(page), baser->psz);
1284
1285                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
1286
1287                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
1288                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
1289                         __flush_dcache_area(table + idx, GITS_LVL1_ENTRY_SIZE);
1290
1291                 /* Ensure updated table contents are visible to ITS hardware */
1292                 dsb(sy);
1293         }
1294
1295         return true;
1296 }
1297
1298 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
1299                                             int nvecs)
1300 {
1301         struct its_device *dev;
1302         unsigned long *lpi_map;
1303         unsigned long flags;
1304         u16 *col_map = NULL;
1305         void *itt;
1306         int lpi_base;
1307         int nr_lpis;
1308         int nr_ites;
1309         int sz;
1310
1311         if (!its_alloc_device_table(its, dev_id))
1312                 return NULL;
1313
1314         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1315         /*
1316          * At least one bit of EventID is being used, hence a minimum
1317          * of two entries. No, the architecture doesn't let you
1318          * express an ITT with a single entry.
1319          */
1320         nr_ites = max(2UL, roundup_pow_of_two(nvecs));
1321         sz = nr_ites * its->ite_size;
1322         sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
1323         itt = kzalloc(sz, GFP_KERNEL);
1324         lpi_map = its_lpi_alloc_chunks(nvecs, &lpi_base, &nr_lpis);
1325         if (lpi_map)
1326                 col_map = kzalloc(sizeof(*col_map) * nr_lpis, GFP_KERNEL);
1327
1328         if (!dev || !itt || !lpi_map || !col_map) {
1329                 kfree(dev);
1330                 kfree(itt);
1331                 kfree(lpi_map);
1332                 kfree(col_map);
1333                 return NULL;
1334         }
1335
1336         __flush_dcache_area(itt, sz);
1337
1338         dev->its = its;
1339         dev->itt = itt;
1340         dev->nr_ites = nr_ites;
1341         dev->event_map.lpi_map = lpi_map;
1342         dev->event_map.col_map = col_map;
1343         dev->event_map.lpi_base = lpi_base;
1344         dev->event_map.nr_lpis = nr_lpis;
1345         dev->device_id = dev_id;
1346         INIT_LIST_HEAD(&dev->entry);
1347
1348         raw_spin_lock_irqsave(&its->lock, flags);
1349         list_add(&dev->entry, &its->its_device_list);
1350         raw_spin_unlock_irqrestore(&its->lock, flags);
1351
1352         /* Map device to its ITT */
1353         its_send_mapd(dev, 1);
1354
1355         return dev;
1356 }
1357
1358 static void its_free_device(struct its_device *its_dev)
1359 {
1360         unsigned long flags;
1361
1362         raw_spin_lock_irqsave(&its_dev->its->lock, flags);
1363         list_del(&its_dev->entry);
1364         raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
1365         kfree(its_dev->itt);
1366         kfree(its_dev);
1367 }
1368
1369 static int its_alloc_device_irq(struct its_device *dev, irq_hw_number_t *hwirq)
1370 {
1371         int idx;
1372
1373         idx = find_first_zero_bit(dev->event_map.lpi_map,
1374                                   dev->event_map.nr_lpis);
1375         if (idx == dev->event_map.nr_lpis)
1376                 return -ENOSPC;
1377
1378         *hwirq = dev->event_map.lpi_base + idx;
1379         set_bit(idx, dev->event_map.lpi_map);
1380
1381         return 0;
1382 }
1383
1384 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
1385                            int nvec, msi_alloc_info_t *info)
1386 {
1387         struct its_node *its;
1388         struct its_device *its_dev;
1389         struct msi_domain_info *msi_info;
1390         u32 dev_id;
1391
1392         /*
1393          * We ignore "dev" entierely, and rely on the dev_id that has
1394          * been passed via the scratchpad. This limits this domain's
1395          * usefulness to upper layers that definitely know that they
1396          * are built on top of the ITS.
1397          */
1398         dev_id = info->scratchpad[0].ul;
1399
1400         msi_info = msi_get_domain_info(domain);
1401         its = msi_info->data;
1402
1403         its_dev = its_find_device(its, dev_id);
1404         if (its_dev) {
1405                 /*
1406                  * We already have seen this ID, probably through
1407                  * another alias (PCI bridge of some sort). No need to
1408                  * create the device.
1409                  */
1410                 pr_debug("Reusing ITT for devID %x\n", dev_id);
1411                 goto out;
1412         }
1413
1414         its_dev = its_create_device(its, dev_id, nvec);
1415         if (!its_dev)
1416                 return -ENOMEM;
1417
1418         pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
1419 out:
1420         info->scratchpad[0].ptr = its_dev;
1421         return 0;
1422 }
1423
1424 static struct msi_domain_ops its_msi_domain_ops = {
1425         .msi_prepare    = its_msi_prepare,
1426 };
1427
1428 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
1429                                     unsigned int virq,
1430                                     irq_hw_number_t hwirq)
1431 {
1432         struct irq_fwspec fwspec;
1433
1434         if (irq_domain_get_of_node(domain->parent)) {
1435                 fwspec.fwnode = domain->parent->fwnode;
1436                 fwspec.param_count = 3;
1437                 fwspec.param[0] = GIC_IRQ_TYPE_LPI;
1438                 fwspec.param[1] = hwirq;
1439                 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
1440         } else {
1441                 return -EINVAL;
1442         }
1443
1444         return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
1445 }
1446
1447 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1448                                 unsigned int nr_irqs, void *args)
1449 {
1450         msi_alloc_info_t *info = args;
1451         struct its_device *its_dev = info->scratchpad[0].ptr;
1452         irq_hw_number_t hwirq;
1453         int err;
1454         int i;
1455
1456         for (i = 0; i < nr_irqs; i++) {
1457                 err = its_alloc_device_irq(its_dev, &hwirq);
1458                 if (err)
1459                         return err;
1460
1461                 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq);
1462                 if (err)
1463                         return err;
1464
1465                 irq_domain_set_hwirq_and_chip(domain, virq + i,
1466                                               hwirq, &its_irq_chip, its_dev);
1467                 pr_debug("ID:%d pID:%d vID:%d\n",
1468                          (int)(hwirq - its_dev->event_map.lpi_base),
1469                          (int) hwirq, virq + i);
1470         }
1471
1472         return 0;
1473 }
1474
1475 static void its_irq_domain_activate(struct irq_domain *domain,
1476                                     struct irq_data *d)
1477 {
1478         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1479         u32 event = its_get_event_id(d);
1480         const struct cpumask *cpu_mask = cpu_online_mask;
1481
1482         /* get the cpu_mask of local node */
1483         if (its_dev->its->numa_node >= 0)
1484                 cpu_mask = cpumask_of_node(its_dev->its->numa_node);
1485
1486         /* Bind the LPI to the first possible CPU */
1487         its_dev->event_map.col_map[event] = cpumask_first(cpu_mask);
1488
1489         /* Map the GIC IRQ and event to the device */
1490         its_send_mapvi(its_dev, d->hwirq, event);
1491 }
1492
1493 static void its_irq_domain_deactivate(struct irq_domain *domain,
1494                                       struct irq_data *d)
1495 {
1496         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1497         u32 event = its_get_event_id(d);
1498
1499         /* Stop the delivery of interrupts */
1500         its_send_discard(its_dev, event);
1501 }
1502
1503 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1504                                 unsigned int nr_irqs)
1505 {
1506         struct irq_data *d = irq_domain_get_irq_data(domain, virq);
1507         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1508         int i;
1509
1510         for (i = 0; i < nr_irqs; i++) {
1511                 struct irq_data *data = irq_domain_get_irq_data(domain,
1512                                                                 virq + i);
1513                 u32 event = its_get_event_id(data);
1514
1515                 /* Mark interrupt index as unused */
1516                 clear_bit(event, its_dev->event_map.lpi_map);
1517
1518                 /* Nuke the entry in the domain */
1519                 irq_domain_reset_irq_data(data);
1520         }
1521
1522         /* If all interrupts have been freed, start mopping the floor */
1523         if (bitmap_empty(its_dev->event_map.lpi_map,
1524                          its_dev->event_map.nr_lpis)) {
1525                 its_lpi_free(&its_dev->event_map);
1526
1527                 /* Unmap device/itt */
1528                 its_send_mapd(its_dev, 0);
1529                 its_free_device(its_dev);
1530         }
1531
1532         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
1533 }
1534
1535 static const struct irq_domain_ops its_domain_ops = {
1536         .alloc                  = its_irq_domain_alloc,
1537         .free                   = its_irq_domain_free,
1538         .activate               = its_irq_domain_activate,
1539         .deactivate             = its_irq_domain_deactivate,
1540 };
1541
1542 static int its_force_quiescent(void __iomem *base)
1543 {
1544         u32 count = 1000000;    /* 1s */
1545         u32 val;
1546
1547         val = readl_relaxed(base + GITS_CTLR);
1548         if (val & GITS_CTLR_QUIESCENT)
1549                 return 0;
1550
1551         /* Disable the generation of all interrupts to this ITS */
1552         val &= ~GITS_CTLR_ENABLE;
1553         writel_relaxed(val, base + GITS_CTLR);
1554
1555         /* Poll GITS_CTLR and wait until ITS becomes quiescent */
1556         while (1) {
1557                 val = readl_relaxed(base + GITS_CTLR);
1558                 if (val & GITS_CTLR_QUIESCENT)
1559                         return 0;
1560
1561                 count--;
1562                 if (!count)
1563                         return -EBUSY;
1564
1565                 cpu_relax();
1566                 udelay(1);
1567         }
1568 }
1569
1570 static void __maybe_unused its_enable_quirk_cavium_22375(void *data)
1571 {
1572         struct its_node *its = data;
1573
1574         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
1575 }
1576
1577 static void __maybe_unused its_enable_quirk_cavium_23144(void *data)
1578 {
1579         struct its_node *its = data;
1580
1581         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
1582 }
1583
1584 static const struct gic_quirk its_quirks[] = {
1585 #ifdef CONFIG_CAVIUM_ERRATUM_22375
1586         {
1587                 .desc   = "ITS: Cavium errata 22375, 24313",
1588                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
1589                 .mask   = 0xffff0fff,
1590                 .init   = its_enable_quirk_cavium_22375,
1591         },
1592 #endif
1593 #ifdef CONFIG_CAVIUM_ERRATUM_23144
1594         {
1595                 .desc   = "ITS: Cavium erratum 23144",
1596                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
1597                 .mask   = 0xffff0fff,
1598                 .init   = its_enable_quirk_cavium_23144,
1599         },
1600 #endif
1601         {
1602         }
1603 };
1604
1605 static void its_enable_quirks(struct its_node *its)
1606 {
1607         u32 iidr = readl_relaxed(its->base + GITS_IIDR);
1608
1609         gic_enable_quirks(iidr, its_quirks, its);
1610 }
1611
1612 static int __init its_probe(struct device_node *node,
1613                             struct irq_domain *parent)
1614 {
1615         struct resource res;
1616         struct its_node *its;
1617         void __iomem *its_base;
1618         struct irq_domain *inner_domain;
1619         u32 val;
1620         u64 baser, tmp;
1621         int err;
1622
1623         err = of_address_to_resource(node, 0, &res);
1624         if (err) {
1625                 pr_warn("%s: no regs?\n", node->full_name);
1626                 return -ENXIO;
1627         }
1628
1629         its_base = ioremap(res.start, resource_size(&res));
1630         if (!its_base) {
1631                 pr_warn("%s: unable to map registers\n", node->full_name);
1632                 return -ENOMEM;
1633         }
1634
1635         val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
1636         if (val != 0x30 && val != 0x40) {
1637                 pr_warn("%s: no ITS detected, giving up\n", node->full_name);
1638                 err = -ENODEV;
1639                 goto out_unmap;
1640         }
1641
1642         err = its_force_quiescent(its_base);
1643         if (err) {
1644                 pr_warn("%s: failed to quiesce, giving up\n",
1645                         node->full_name);
1646                 goto out_unmap;
1647         }
1648
1649         pr_info("ITS: %s\n", node->full_name);
1650
1651         its = kzalloc(sizeof(*its), GFP_KERNEL);
1652         if (!its) {
1653                 err = -ENOMEM;
1654                 goto out_unmap;
1655         }
1656
1657         raw_spin_lock_init(&its->lock);
1658         INIT_LIST_HEAD(&its->entry);
1659         INIT_LIST_HEAD(&its->its_device_list);
1660         its->base = its_base;
1661         its->phys_base = res.start;
1662         its->ite_size = ((readl_relaxed(its_base + GITS_TYPER) >> 4) & 0xf) + 1;
1663         its->numa_node = of_node_to_nid(node);
1664
1665         its->cmd_base = kzalloc(ITS_CMD_QUEUE_SZ, GFP_KERNEL);
1666         if (!its->cmd_base) {
1667                 err = -ENOMEM;
1668                 goto out_free_its;
1669         }
1670         its->cmd_write = its->cmd_base;
1671
1672         its_enable_quirks(its);
1673
1674         err = its_alloc_tables(its);
1675         if (err)
1676                 goto out_free_cmd;
1677
1678         err = its_alloc_collections(its);
1679         if (err)
1680                 goto out_free_tables;
1681
1682         baser = (virt_to_phys(its->cmd_base)    |
1683                  GITS_CBASER_WaWb               |
1684                  GITS_CBASER_InnerShareable     |
1685                  (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
1686                  GITS_CBASER_VALID);
1687
1688         writeq_relaxed(baser, its->base + GITS_CBASER);
1689         tmp = readq_relaxed(its->base + GITS_CBASER);
1690
1691         if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
1692                 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
1693                         /*
1694                          * The HW reports non-shareable, we must
1695                          * remove the cacheability attributes as
1696                          * well.
1697                          */
1698                         baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
1699                                    GITS_CBASER_CACHEABILITY_MASK);
1700                         baser |= GITS_CBASER_nC;
1701                         writeq_relaxed(baser, its->base + GITS_CBASER);
1702                 }
1703                 pr_info("ITS: using cache flushing for cmd queue\n");
1704                 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
1705         }
1706
1707         writeq_relaxed(0, its->base + GITS_CWRITER);
1708         writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR);
1709
1710         if (of_property_read_bool(node, "msi-controller")) {
1711                 struct msi_domain_info *info;
1712
1713                 info = kzalloc(sizeof(*info), GFP_KERNEL);
1714                 if (!info) {
1715                         err = -ENOMEM;
1716                         goto out_free_tables;
1717                 }
1718
1719                 inner_domain = irq_domain_add_tree(node, &its_domain_ops, its);
1720                 if (!inner_domain) {
1721                         err = -ENOMEM;
1722                         kfree(info);
1723                         goto out_free_tables;
1724                 }
1725
1726                 inner_domain->parent = parent;
1727                 inner_domain->bus_token = DOMAIN_BUS_NEXUS;
1728                 info->ops = &its_msi_domain_ops;
1729                 info->data = its;
1730                 inner_domain->host_data = info;
1731         }
1732
1733         spin_lock(&its_lock);
1734         list_add(&its->entry, &its_nodes);
1735         spin_unlock(&its_lock);
1736
1737         return 0;
1738
1739 out_free_tables:
1740         its_free_tables(its);
1741 out_free_cmd:
1742         kfree(its->cmd_base);
1743 out_free_its:
1744         kfree(its);
1745 out_unmap:
1746         iounmap(its_base);
1747         pr_err("ITS: failed probing %s (%d)\n", node->full_name, err);
1748         return err;
1749 }
1750
1751 static bool gic_rdists_supports_plpis(void)
1752 {
1753         return !!(readl_relaxed(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
1754 }
1755
1756 int its_cpu_init(void)
1757 {
1758         if (!list_empty(&its_nodes)) {
1759                 if (!gic_rdists_supports_plpis()) {
1760                         pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
1761                         return -ENXIO;
1762                 }
1763                 its_cpu_init_lpis();
1764                 its_cpu_init_collection();
1765         }
1766
1767         return 0;
1768 }
1769
1770 static struct of_device_id its_device_id[] = {
1771         {       .compatible     = "arm,gic-v3-its",     },
1772         {},
1773 };
1774
1775 int __init its_init(struct device_node *node, struct rdists *rdists,
1776              struct irq_domain *parent_domain)
1777 {
1778         struct device_node *np;
1779
1780         for (np = of_find_matching_node(node, its_device_id); np;
1781              np = of_find_matching_node(np, its_device_id)) {
1782                 its_probe(np, parent_domain);
1783         }
1784
1785         if (list_empty(&its_nodes)) {
1786                 pr_warn("ITS: No ITS available, not enabling LPIs\n");
1787                 return -ENXIO;
1788         }
1789
1790         gic_rdists = rdists;
1791         its_alloc_lpi_tables();
1792         its_lpi_init(rdists->id_bits);
1793
1794         return 0;
1795 }