drivers/irqchip/irq-mips-gic.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org)
   7  * Copyright (C) 2012 MIPS Technologies, Inc.  All rights reserved.
   8  */
   9 #include <linux/bitmap.h>
  10 #include <linux/clocksource.h>
  11 #include <linux/init.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/irq.h>
  14 #include <linux/irqchip.h>
  15 #include <linux/irqchip/mips-gic.h>
  16 #include <linux/of_address.h>
  17 #include <linux/sched.h>
  18 #include <linux/smp.h>
  19
  20 #include <asm/mips-cm.h>
  21 #include <asm/setup.h>
  22 #include <asm/traps.h>
  23
  24 #include <dt-bindings/interrupt-controller/mips-gic.h>
  25
  26 unsigned int gic_present;
  27
  28 struct gic_pcpu_mask {
  29         DECLARE_BITMAP(pcpu_mask, GIC_MAX_INTRS);
  30 };
  31
  32 struct gic_irq_spec {
  33         enum {
  34                 GIC_DEVICE,
  35                 GIC_IPI
  36         } type;
  37
  38         union {
  39                 struct cpumask *ipimask;
  40                 unsigned int hwirq;
  41         };
  42 };
  43
  44 static unsigned long __gic_base_addr;
  45
  46 static void __iomem *gic_base;
  47 static struct gic_pcpu_mask pcpu_masks[NR_CPUS];
  48 static DEFINE_SPINLOCK(gic_lock);
  49 static struct irq_domain *gic_irq_domain;
  50 static struct irq_domain *gic_dev_domain;
  51 static struct irq_domain *gic_ipi_domain;
  52 static int gic_shared_intrs;
  53 static int gic_vpes;
  54 static unsigned int gic_cpu_pin;
  55 static unsigned int timer_cpu_pin;
  56 static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller;
  57 DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS);
  58
  59 static void __gic_irq_dispatch(void);
  60
  61 static inline u32 gic_read32(unsigned int reg)
  62 {
  63         return __raw_readl(gic_base + reg);
  64 }
  65
  66 static inline u64 gic_read64(unsigned int reg)
  67 {
  68         return __raw_readq(gic_base + reg);
  69 }
  70
  71 static inline unsigned long gic_read(unsigned int reg)
  72 {
  73         if (!mips_cm_is64)
  74                 return gic_read32(reg);
  75         else
  76                 return gic_read64(reg);
  77 }
  78
  79 static inline void gic_write32(unsigned int reg, u32 val)
  80 {
  81         return __raw_writel(val, gic_base + reg);
  82 }
  83
  84 static inline void gic_write64(unsigned int reg, u64 val)
  85 {
  86         return __raw_writeq(val, gic_base + reg);
  87 }
  88
  89 static inline void gic_write(unsigned int reg, unsigned long val)
  90 {
  91         if (!mips_cm_is64)
  92                 return gic_write32(reg, (u32)val);
  93         else
  94                 return gic_write64(reg, (u64)val);
  95 }
  96
  97 static inline void gic_update_bits(unsigned int reg, unsigned long mask,
  98                                    unsigned long val)
  99 {
 100         unsigned long regval;
 101
 102         regval = gic_read(reg);
 103         regval &= ~mask;
 104         regval |= val;
 105         gic_write(reg, regval);
 106 }
 107
 108 static inline void gic_reset_mask(unsigned int intr)
 109 {
 110         gic_write(GIC_REG(SHARED, GIC_SH_RMASK) + GIC_INTR_OFS(intr),
 111                   1ul << GIC_INTR_BIT(intr));
 112 }
 113
 114 static inline void gic_set_mask(unsigned int intr)
 115 {
 116         gic_write(GIC_REG(SHARED, GIC_SH_SMASK) + GIC_INTR_OFS(intr),
 117                   1ul << GIC_INTR_BIT(intr));
 118 }
 119
 120 static inline void gic_set_polarity(unsigned int intr, unsigned int pol)
 121 {
 122         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_POLARITY) +
 123                         GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
 124                         (unsigned long)pol << GIC_INTR_BIT(intr));
 125 }
 126
 127 static inline void gic_set_trigger(unsigned int intr, unsigned int trig)
 128 {
 129         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_TRIGGER) +
 130                         GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
 131                         (unsigned long)trig << GIC_INTR_BIT(intr));
 132 }
 133
 134 static inline void gic_set_dual_edge(unsigned int intr, unsigned int dual)
 135 {
 136         gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_DUAL) + GIC_INTR_OFS(intr),
 137                         1ul << GIC_INTR_BIT(intr),
 138                         (unsigned long)dual << GIC_INTR_BIT(intr));
 139 }
 140
 141 static inline void gic_map_to_pin(unsigned int intr, unsigned int pin)
 142 {
 143         gic_write32(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_PIN_BASE) +
 144                     GIC_SH_MAP_TO_PIN(intr), GIC_MAP_TO_PIN_MSK | pin);
 145 }
 146
 147 static inline void gic_map_to_vpe(unsigned int intr, unsigned int vpe)
 148 {
 149         gic_write(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_VPE_BASE) +
 150                   GIC_SH_MAP_TO_VPE_REG_OFF(intr, vpe),
 151                   GIC_SH_MAP_TO_VPE_REG_BIT(vpe));
 152 }
 153
 154 #ifdef CONFIG_CLKSRC_MIPS_GIC
 155 cycle_t gic_read_count(void)
 156 {
 157         unsigned int hi, hi2, lo;
 158
 159         if (mips_cm_is64)
 160                 return (cycle_t)gic_read(GIC_REG(SHARED, GIC_SH_COUNTER));
 161
 162         do {
 163                 hi = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
 164                 lo = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_31_00));
 165                 hi2 = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
 166         } while (hi2 != hi);
 167
 168         return (((cycle_t) hi) << 32) + lo;
 169 }
 170
 171 unsigned int gic_get_count_width(void)
 172 {
 173         unsigned int bits, config;
 174
 175         config = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 176         bits = 32 + 4 * ((config & GIC_SH_CONFIG_COUNTBITS_MSK) >>
 177                          GIC_SH_CONFIG_COUNTBITS_SHF);
 178
 179         return bits;
 180 }
 181
 182 void gic_write_compare(cycle_t cnt)
 183 {
 184         if (mips_cm_is64) {
 185                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE), cnt);
 186         } else {
 187                 gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI),
 188                                         (int)(cnt >> 32));
 189                 gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO),
 190                                         (int)(cnt & 0xffffffff));
 191         }
 192 }
 193
 194 void gic_write_cpu_compare(cycle_t cnt, int cpu)
 195 {
 196         unsigned long flags;
 197
 198         local_irq_save(flags);
 199
 200         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), mips_cm_vp_id(cpu));
 201
 202         if (mips_cm_is64) {
 203                 gic_write(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE), cnt);
 204         } else {
 205                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_HI),
 206                                         (int)(cnt >> 32));
 207                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_LO),
 208                                         (int)(cnt & 0xffffffff));
 209         }
 210
 211         local_irq_restore(flags);
 212 }
 213
 214 cycle_t gic_read_compare(void)
 215 {
 216         unsigned int hi, lo;
 217
 218         if (mips_cm_is64)
 219                 return (cycle_t)gic_read(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE));
 220
 221         hi = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI));
 222         lo = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO));
 223
 224         return (((cycle_t) hi) << 32) + lo;
 225 }
 226
 227 void gic_start_count(void)
 228 {
 229         u32 gicconfig;
 230
 231         /* Start the counter */
 232         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 233         gicconfig &= ~(1 << GIC_SH_CONFIG_COUNTSTOP_SHF);
 234         gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
 235 }
 236
 237 void gic_stop_count(void)
 238 {
 239         u32 gicconfig;
 240
 241         /* Stop the counter */
 242         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 243         gicconfig |= 1 << GIC_SH_CONFIG_COUNTSTOP_SHF;
 244         gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
 245 }
 246
 247 #endif
 248
 249 unsigned gic_read_local_vp_id(void)
 250 {
 251         unsigned long ident;
 252
 253         ident = gic_read(GIC_REG(VPE_LOCAL, GIC_VP_IDENT));
 254         return ident & GIC_VP_IDENT_VCNUM_MSK;
 255 }
 256
 257 static bool gic_local_irq_is_routable(int intr)
 258 {
 259         u32 vpe_ctl;
 260
 261         /* All local interrupts are routable in EIC mode. */
 262         if (cpu_has_veic)
 263                 return true;
 264
 265         vpe_ctl = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_CTL));
 266         switch (intr) {
 267         case GIC_LOCAL_INT_TIMER:
 268                 return vpe_ctl & GIC_VPE_CTL_TIMER_RTBL_MSK;
 269         case GIC_LOCAL_INT_PERFCTR:
 270                 return vpe_ctl & GIC_VPE_CTL_PERFCNT_RTBL_MSK;
 271         case GIC_LOCAL_INT_FDC:
 272                 return vpe_ctl & GIC_VPE_CTL_FDC_RTBL_MSK;
 273         case GIC_LOCAL_INT_SWINT0:
 274         case GIC_LOCAL_INT_SWINT1:
 275                 return vpe_ctl & GIC_VPE_CTL_SWINT_RTBL_MSK;
 276         default:
 277                 return true;
 278         }
 279 }
 280
 281 static void gic_bind_eic_interrupt(int irq, int set)
 282 {
 283         /* Convert irq vector # to hw int # */
 284         irq -= GIC_PIN_TO_VEC_OFFSET;
 285
 286         /* Set irq to use shadow set */
 287         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_EIC_SHADOW_SET_BASE) +
 288                   GIC_VPE_EIC_SS(irq), set);
 289 }
 290
 291 static void gic_send_ipi(struct irq_data *d, unsigned int cpu)
 292 {
 293         irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d));
 294
 295         gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_SET(hwirq));
 296 }
 297
 298 int gic_get_c0_compare_int(void)
 299 {
 300         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER))
 301                 return MIPS_CPU_IRQ_BASE + cp0_compare_irq;
 302         return irq_create_mapping(gic_irq_domain,
 303                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_TIMER));
 304 }
 305
 306 int gic_get_c0_perfcount_int(void)
 307 {
 308         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_PERFCTR)) {
 309                 /* Is the performance counter shared with the timer? */
 310                 if (cp0_perfcount_irq < 0)
 311                         return -1;
 312                 return MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
 313         }
 314         return irq_create_mapping(gic_irq_domain,
 315                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_PERFCTR));
 316 }
 317
 318 int gic_get_c0_fdc_int(void)
 319 {
 320         if (!gic_local_irq_is_routable(GIC_LOCAL_INT_FDC)) {
 321                 /* Is the FDC IRQ even present? */
 322                 if (cp0_fdc_irq < 0)
 323                         return -1;
 324                 return MIPS_CPU_IRQ_BASE + cp0_fdc_irq;
 325         }
 326
 327         return irq_create_mapping(gic_irq_domain,
 328                                   GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC));
 329 }
 330
 331 int gic_get_usm_range(struct resource *gic_usm_res)
 332 {
 333         if (!gic_present)
 334                 return -1;
 335
 336         gic_usm_res->start = __gic_base_addr + USM_VISIBLE_SECTION_OFS;
 337         gic_usm_res->end = gic_usm_res->start + (USM_VISIBLE_SECTION_SIZE - 1);
 338
 339         return 0;
 340 }
 341
 342 static void gic_handle_shared_int(bool chained)
 343 {
 344         unsigned int i, intr, virq, gic_reg_step = mips_cm_is64 ? 8 : 4;
 345         unsigned long *pcpu_mask;
 346         unsigned long pending_reg, intrmask_reg;
 347         DECLARE_BITMAP(pending, GIC_MAX_INTRS);
 348         DECLARE_BITMAP(intrmask, GIC_MAX_INTRS);
 349
 350         /* Get per-cpu bitmaps */
 351         pcpu_mask = pcpu_masks[smp_processor_id()].pcpu_mask;
 352
 353         pending_reg = GIC_REG(SHARED, GIC_SH_PEND);
 354         intrmask_reg = GIC_REG(SHARED, GIC_SH_MASK);
 355
 356         for (i = 0; i < BITS_TO_LONGS(gic_shared_intrs); i++) {
 357                 pending[i] = gic_read(pending_reg);
 358                 intrmask[i] = gic_read(intrmask_reg);
 359                 pending_reg += gic_reg_step;
 360                 intrmask_reg += gic_reg_step;
 361
 362                 if (!config_enabled(CONFIG_64BIT) || mips_cm_is64)
 363                         continue;
 364
 365                 pending[i] |= (u64)gic_read(pending_reg) << 32;
 366                 intrmask[i] |= (u64)gic_read(intrmask_reg) << 32;
 367                 pending_reg += gic_reg_step;
 368                 intrmask_reg += gic_reg_step;
 369         }
 370
 371         bitmap_and(pending, pending, intrmask, gic_shared_intrs);
 372         bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs);
 373
 374         intr = find_first_bit(pending, gic_shared_intrs);
 375         while (intr != gic_shared_intrs) {
 376                 virq = irq_linear_revmap(gic_irq_domain,
 377                                          GIC_SHARED_TO_HWIRQ(intr));
 378                 if (chained)
 379                         generic_handle_irq(virq);
 380                 else
 381                         do_IRQ(virq);
 382
 383                 /* go to next pending bit */
 384                 bitmap_clear(pending, intr, 1);
 385                 intr = find_first_bit(pending, gic_shared_intrs);
 386         }
 387 }
 388
 389 static void gic_mask_irq(struct irq_data *d)
 390 {
 391         gic_reset_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
 392 }
 393
 394 static void gic_unmask_irq(struct irq_data *d)
 395 {
 396         gic_set_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
 397 }
 398
 399 static void gic_ack_irq(struct irq_data *d)
 400 {
 401         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 402
 403         gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_CLR(irq));
 404 }
 405
 406 static int gic_set_type(struct irq_data *d, unsigned int type)
 407 {
 408         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 409         unsigned long flags;
 410         bool is_edge;
 411
 412         spin_lock_irqsave(&gic_lock, flags);
 413         switch (type & IRQ_TYPE_SENSE_MASK) {
 414         case IRQ_TYPE_EDGE_FALLING:
 415                 gic_set_polarity(irq, GIC_POL_NEG);
 416                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 417                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 418                 is_edge = true;
 419                 break;
 420         case IRQ_TYPE_EDGE_RISING:
 421                 gic_set_polarity(irq, GIC_POL_POS);
 422                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 423                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 424                 is_edge = true;
 425                 break;
 426         case IRQ_TYPE_EDGE_BOTH:
 427                 /* polarity is irrelevant in this case */
 428                 gic_set_trigger(irq, GIC_TRIG_EDGE);
 429                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_ENABLE);
 430                 is_edge = true;
 431                 break;
 432         case IRQ_TYPE_LEVEL_LOW:
 433                 gic_set_polarity(irq, GIC_POL_NEG);
 434                 gic_set_trigger(irq, GIC_TRIG_LEVEL);
 435                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 436                 is_edge = false;
 437                 break;
 438         case IRQ_TYPE_LEVEL_HIGH:
 439         default:
 440                 gic_set_polarity(irq, GIC_POL_POS);
 441                 gic_set_trigger(irq, GIC_TRIG_LEVEL);
 442                 gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
 443                 is_edge = false;
 444                 break;
 445         }
 446
 447         if (is_edge)
 448                 irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller,
 449                                                  handle_edge_irq, NULL);
 450         else
 451                 irq_set_chip_handler_name_locked(d, &gic_level_irq_controller,
 452                                                  handle_level_irq, NULL);
 453         spin_unlock_irqrestore(&gic_lock, flags);
 454
 455         return 0;
 456 }
 457
 458 #ifdef CONFIG_SMP
 459 static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
 460                             bool force)
 461 {
 462         unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 463         cpumask_t       tmp = CPU_MASK_NONE;
 464         unsigned long   flags;
 465         int             i;
 466
 467         cpumask_and(&tmp, cpumask, cpu_online_mask);
 468         if (cpumask_empty(&tmp))
 469                 return -EINVAL;
 470
 471         /* Assumption : cpumask refers to a single CPU */
 472         spin_lock_irqsave(&gic_lock, flags);
 473
 474         /* Re-route this IRQ */
 475         gic_map_to_vpe(irq, mips_cm_vp_id(cpumask_first(&tmp)));
 476
 477         /* Update the pcpu_masks */
 478         for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
 479                 clear_bit(irq, pcpu_masks[i].pcpu_mask);
 480         set_bit(irq, pcpu_masks[cpumask_first(&tmp)].pcpu_mask);
 481
 482         cpumask_copy(irq_data_get_affinity_mask(d), cpumask);
 483         spin_unlock_irqrestore(&gic_lock, flags);
 484
 485         return IRQ_SET_MASK_OK_NOCOPY;
 486 }
 487 #endif
 488
 489 static struct irq_chip gic_level_irq_controller = {
 490         .name                   =       "MIPS GIC",
 491         .irq_mask               =       gic_mask_irq,
 492         .irq_unmask             =       gic_unmask_irq,
 493         .irq_set_type           =       gic_set_type,
 494 #ifdef CONFIG_SMP
 495         .irq_set_affinity       =       gic_set_affinity,
 496 #endif
 497 };
 498
 499 static struct irq_chip gic_edge_irq_controller = {
 500         .name                   =       "MIPS GIC",
 501         .irq_ack                =       gic_ack_irq,
 502         .irq_mask               =       gic_mask_irq,
 503         .irq_unmask             =       gic_unmask_irq,
 504         .irq_set_type           =       gic_set_type,
 505 #ifdef CONFIG_SMP
 506         .irq_set_affinity       =       gic_set_affinity,
 507 #endif
 508         .ipi_send_single        =       gic_send_ipi,
 509 };
 510
 511 static void gic_handle_local_int(bool chained)
 512 {
 513         unsigned long pending, masked;
 514         unsigned int intr, virq;
 515
 516         pending = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_PEND));
 517         masked = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_MASK));
 518
 519         bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS);
 520
 521         intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
 522         while (intr != GIC_NUM_LOCAL_INTRS) {
 523                 virq = irq_linear_revmap(gic_irq_domain,
 524                                          GIC_LOCAL_TO_HWIRQ(intr));
 525                 if (chained)
 526                         generic_handle_irq(virq);
 527                 else
 528                         do_IRQ(virq);
 529
 530                 /* go to next pending bit */
 531                 bitmap_clear(&pending, intr, 1);
 532                 intr = find_first_bit(&pending, GIC_NUM_LOCAL_INTRS);
 533         }
 534 }
 535
 536 static void gic_mask_local_irq(struct irq_data *d)
 537 {
 538         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 539
 540         gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_RMASK), 1 << intr);
 541 }
 542
 543 static void gic_unmask_local_irq(struct irq_data *d)
 544 {
 545         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 546
 547         gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_SMASK), 1 << intr);
 548 }
 549
 550 static struct irq_chip gic_local_irq_controller = {
 551         .name                   =       "MIPS GIC Local",
 552         .irq_mask               =       gic_mask_local_irq,
 553         .irq_unmask             =       gic_unmask_local_irq,
 554 };
 555
 556 static void gic_mask_local_irq_all_vpes(struct irq_data *d)
 557 {
 558         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 559         int i;
 560         unsigned long flags;
 561
 562         spin_lock_irqsave(&gic_lock, flags);
 563         for (i = 0; i < gic_vpes; i++) {
 564                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 565                           mips_cm_vp_id(i));
 566                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << intr);
 567         }
 568         spin_unlock_irqrestore(&gic_lock, flags);
 569 }
 570
 571 static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
 572 {
 573         int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 574         int i;
 575         unsigned long flags;
 576
 577         spin_lock_irqsave(&gic_lock, flags);
 578         for (i = 0; i < gic_vpes; i++) {
 579                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 580                           mips_cm_vp_id(i));
 581                 gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SMASK), 1 << intr);
 582         }
 583         spin_unlock_irqrestore(&gic_lock, flags);
 584 }
 585
 586 static struct irq_chip gic_all_vpes_local_irq_controller = {
 587         .name                   =       "MIPS GIC Local",
 588         .irq_mask               =       gic_mask_local_irq_all_vpes,
 589         .irq_unmask             =       gic_unmask_local_irq_all_vpes,
 590 };
 591
 592 static void __gic_irq_dispatch(void)
 593 {
 594         gic_handle_local_int(false);
 595         gic_handle_shared_int(false);
 596 }
 597
 598 static void gic_irq_dispatch(struct irq_desc *desc)
 599 {
 600         gic_handle_local_int(true);
 601         gic_handle_shared_int(true);
 602 }
 603
 604 static void __init gic_basic_init(void)
 605 {
 606         unsigned int i;
 607
 608         board_bind_eic_interrupt = &gic_bind_eic_interrupt;
 609
 610         /* Setup defaults */
 611         for (i = 0; i < gic_shared_intrs; i++) {
 612                 gic_set_polarity(i, GIC_POL_POS);
 613                 gic_set_trigger(i, GIC_TRIG_LEVEL);
 614                 gic_reset_mask(i);
 615         }
 616
 617         for (i = 0; i < gic_vpes; i++) {
 618                 unsigned int j;
 619
 620                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 621                           mips_cm_vp_id(i));
 622                 for (j = 0; j < GIC_NUM_LOCAL_INTRS; j++) {
 623                         if (!gic_local_irq_is_routable(j))
 624                                 continue;
 625                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << j);
 626                 }
 627         }
 628 }
 629
 630 static int gic_local_irq_domain_map(struct irq_domain *d, unsigned int virq,
 631                                     irq_hw_number_t hw)
 632 {
 633         int intr = GIC_HWIRQ_TO_LOCAL(hw);
 634         int ret = 0;
 635         int i;
 636         unsigned long flags;
 637
 638         if (!gic_local_irq_is_routable(intr))
 639                 return -EPERM;
 640
 641         /*
 642          * HACK: These are all really percpu interrupts, but the rest
 643          * of the MIPS kernel code does not use the percpu IRQ API for
 644          * the CP0 timer and performance counter interrupts.
 645          */
 646         switch (intr) {
 647         case GIC_LOCAL_INT_TIMER:
 648         case GIC_LOCAL_INT_PERFCTR:
 649         case GIC_LOCAL_INT_FDC:
 650                 irq_set_chip_and_handler(virq,
 651                                          &gic_all_vpes_local_irq_controller,
 652                                          handle_percpu_irq);
 653                 break;
 654         default:
 655                 irq_set_chip_and_handler(virq,
 656                                          &gic_local_irq_controller,
 657                                          handle_percpu_devid_irq);
 658                 irq_set_percpu_devid(virq);
 659                 break;
 660         }
 661
 662         spin_lock_irqsave(&gic_lock, flags);
 663         for (i = 0; i < gic_vpes; i++) {
 664                 u32 val = GIC_MAP_TO_PIN_MSK | gic_cpu_pin;
 665
 666                 gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
 667                           mips_cm_vp_id(i));
 668
 669                 switch (intr) {
 670                 case GIC_LOCAL_INT_WD:
 671                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_WD_MAP), val);
 672                         break;
 673                 case GIC_LOCAL_INT_COMPARE:
 674                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_MAP),
 675                                     val);
 676                         break;
 677                 case GIC_LOCAL_INT_TIMER:
 678                         /* CONFIG_MIPS_CMP workaround (see __gic_init) */
 679                         val = GIC_MAP_TO_PIN_MSK | timer_cpu_pin;
 680                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_TIMER_MAP),
 681                                     val);
 682                         break;
 683                 case GIC_LOCAL_INT_PERFCTR:
 684                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_PERFCTR_MAP),
 685                                     val);
 686                         break;
 687                 case GIC_LOCAL_INT_SWINT0:
 688                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT0_MAP),
 689                                     val);
 690                         break;
 691                 case GIC_LOCAL_INT_SWINT1:
 692                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT1_MAP),
 693                                     val);
 694                         break;
 695                 case GIC_LOCAL_INT_FDC:
 696                         gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_FDC_MAP), val);
 697                         break;
 698                 default:
 699                         pr_err("Invalid local IRQ %d\n", intr);
 700                         ret = -EINVAL;
 701                         break;
 702                 }
 703         }
 704         spin_unlock_irqrestore(&gic_lock, flags);
 705
 706         return ret;
 707 }
 708
 709 static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
 710                                      irq_hw_number_t hw, unsigned int vpe)
 711 {
 712         int intr = GIC_HWIRQ_TO_SHARED(hw);
 713         unsigned long flags;
 714         int i;
 715
 716         irq_set_chip_and_handler(virq, &gic_level_irq_controller,
 717                                  handle_level_irq);
 718
 719         spin_lock_irqsave(&gic_lock, flags);
 720         gic_map_to_pin(intr, gic_cpu_pin);
 721         gic_map_to_vpe(intr, mips_cm_vp_id(vpe));
 722         for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
 723                 clear_bit(intr, pcpu_masks[i].pcpu_mask);
 724         set_bit(intr, pcpu_masks[vpe].pcpu_mask);
 725         spin_unlock_irqrestore(&gic_lock, flags);
 726
 727         return 0;
 728 }
 729
 730 static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
 731                               irq_hw_number_t hw)
 732 {
 733         if (GIC_HWIRQ_TO_LOCAL(hw) < GIC_NUM_LOCAL_INTRS)
 734                 return gic_local_irq_domain_map(d, virq, hw);
 735         return gic_shared_irq_domain_map(d, virq, hw, 0);
 736 }
 737
 738 static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq,
 739                                 unsigned int nr_irqs, void *arg)
 740 {
 741         struct gic_irq_spec *spec = arg;
 742         irq_hw_number_t hwirq, base_hwirq;
 743         int cpu, ret, i;
 744
 745         if (spec->type == GIC_DEVICE) {
 746                 /* verify that it doesn't conflict with an IPI irq */
 747                 if (test_bit(spec->hwirq, ipi_resrv))
 748                         return -EBUSY;
 749
 750                 hwirq = GIC_SHARED_TO_HWIRQ(spec->hwirq);
 751
 752                 return irq_domain_set_hwirq_and_chip(d, virq, hwirq,
 753                                                      &gic_level_irq_controller,
 754                                                      NULL);
 755         } else {
 756                 base_hwirq = find_first_bit(ipi_resrv, gic_shared_intrs);
 757                 if (base_hwirq == gic_shared_intrs) {
 758                         return -ENOMEM;
 759                 }
 760
 761                 /* check that we have enough space */
 762                 for (i = base_hwirq; i < nr_irqs; i++) {
 763                         if (!test_bit(i, ipi_resrv))
 764                                 return -EBUSY;
 765                 }
 766                 bitmap_clear(ipi_resrv, base_hwirq, nr_irqs);
 767
 768                 /* map the hwirq for each cpu consecutively */
 769                 i = 0;
 770                 for_each_cpu(cpu, spec->ipimask) {
 771                         hwirq = GIC_SHARED_TO_HWIRQ(base_hwirq + i);
 772
 773                         ret = irq_domain_set_hwirq_and_chip(d, virq + i, hwirq,
 774                                                             &gic_edge_irq_controller,
 775                                                             NULL);
 776                         if (ret)
 777                                 goto error;
 778
 779                         ret = gic_shared_irq_domain_map(d, virq + i, hwirq, cpu);
 780                         if (ret)
 781                                 goto error;
 782
 783                         i++;
 784                 }
 785
 786                 /*
 787                  * tell the parent about the base hwirq we allocated so it can
 788                  * set its own domain data
 789                  */
 790                 spec->hwirq = base_hwirq;
 791         }
 792
 793         return 0;
 794 error:
 795         bitmap_set(ipi_resrv, base_hwirq, nr_irqs);
 796         return ret;
 797 }
 798
 799 void gic_irq_domain_free(struct irq_domain *d, unsigned int virq,
 800                          unsigned int nr_irqs)
 801 {
 802         irq_hw_number_t base_hwirq;
 803         struct irq_data *data;
 804
 805         data = irq_get_irq_data(virq);
 806         if (!data)
 807                 return;
 808
 809         base_hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(data));
 810         bitmap_set(ipi_resrv, base_hwirq, nr_irqs);
 811 }
 812
 813 int gic_irq_domain_match(struct irq_domain *d, struct device_node *node,
 814                          enum irq_domain_bus_token bus_token)
 815 {
 816         /* this domain should'nt be accessed directly */
 817         return 0;
 818 }
 819
 820 static const struct irq_domain_ops gic_irq_domain_ops = {
 821         .map = gic_irq_domain_map,
 822         .alloc = gic_irq_domain_alloc,
 823         .free = gic_irq_domain_free,
 824         .match = gic_irq_domain_match,
 825 };
 826
 827 static int gic_dev_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
 828                                 const u32 *intspec, unsigned int intsize,
 829                                 irq_hw_number_t *out_hwirq,
 830                                 unsigned int *out_type)
 831 {
 832         if (intsize != 3)
 833                 return -EINVAL;
 834
 835         if (intspec[0] == GIC_SHARED)
 836                 *out_hwirq = GIC_SHARED_TO_HWIRQ(intspec[1]);
 837         else if (intspec[0] == GIC_LOCAL)
 838                 *out_hwirq = GIC_LOCAL_TO_HWIRQ(intspec[1]);
 839         else
 840                 return -EINVAL;
 841         *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
 842
 843         return 0;
 844 }
 845
 846 static int gic_dev_domain_alloc(struct irq_domain *d, unsigned int virq,
 847                                 unsigned int nr_irqs, void *arg)
 848 {
 849         struct irq_fwspec *fwspec = arg;
 850         struct gic_irq_spec spec = {
 851                 .type = GIC_DEVICE,
 852                 .hwirq = fwspec->param[1],
 853         };
 854         int i, ret;
 855         bool is_shared = fwspec->param[0] == GIC_SHARED;
 856
 857         if (is_shared) {
 858                 ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
 859                 if (ret)
 860                         return ret;
 861         }
 862
 863         for (i = 0; i < nr_irqs; i++) {
 864                 irq_hw_number_t hwirq;
 865
 866                 if (is_shared)
 867                         hwirq = GIC_SHARED_TO_HWIRQ(spec.hwirq + i);
 868                 else
 869                         hwirq = GIC_LOCAL_TO_HWIRQ(spec.hwirq + i);
 870
 871                 ret = irq_domain_set_hwirq_and_chip(d, virq + i,
 872                                                     hwirq,
 873                                                     &gic_level_irq_controller,
 874                                                     NULL);
 875                 if (ret)
 876                         goto error;
 877         }
 878
 879         return 0;
 880
 881 error:
 882         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 883         return ret;
 884 }
 885
 886 void gic_dev_domain_free(struct irq_domain *d, unsigned int virq,
 887                          unsigned int nr_irqs)
 888 {
 889         /* no real allocation is done for dev irqs, so no need to free anything */
 890         return;
 891 }
 892
 893 static struct irq_domain_ops gic_dev_domain_ops = {
 894         .xlate = gic_dev_domain_xlate,
 895         .alloc = gic_dev_domain_alloc,
 896         .free = gic_dev_domain_free,
 897 };
 898
 899 static int gic_ipi_domain_xlate(struct irq_domain *d, struct device_node *ctrlr,
 900                                 const u32 *intspec, unsigned int intsize,
 901                                 irq_hw_number_t *out_hwirq,
 902                                 unsigned int *out_type)
 903 {
 904         /*
 905          * There's nothing to translate here. hwirq is dynamically allocated and
 906          * the irq type is always edge triggered.
 907          * */
 908         *out_hwirq = 0;
 909         *out_type = IRQ_TYPE_EDGE_RISING;
 910
 911         return 0;
 912 }
 913
 914 static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
 915                                 unsigned int nr_irqs, void *arg)
 916 {
 917         struct cpumask *ipimask = arg;
 918         struct gic_irq_spec spec = {
 919                 .type = GIC_IPI,
 920                 .ipimask = ipimask
 921         };
 922         int ret, i;
 923
 924         ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, &spec);
 925         if (ret)
 926                 return ret;
 927
 928         /* the parent should have set spec.hwirq to the base_hwirq it allocated */
 929         for (i = 0; i < nr_irqs; i++) {
 930                 ret = irq_domain_set_hwirq_and_chip(d, virq + i,
 931                                                     GIC_SHARED_TO_HWIRQ(spec.hwirq + i),
 932                                                     &gic_edge_irq_controller,
 933                                                     NULL);
 934                 if (ret)
 935                         goto error;
 936
 937                 ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING);
 938                 if (ret)
 939                         goto error;
 940         }
 941
 942         return 0;
 943 error:
 944         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 945         return ret;
 946 }
 947
 948 void gic_ipi_domain_free(struct irq_domain *d, unsigned int virq,
 949                          unsigned int nr_irqs)
 950 {
 951         irq_domain_free_irqs_parent(d, virq, nr_irqs);
 952 }
 953
 954 int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node,
 955                          enum irq_domain_bus_token bus_token)
 956 {
 957         bool is_ipi;
 958
 959         switch (bus_token) {
 960         case DOMAIN_BUS_IPI:
 961                 is_ipi = d->bus_token == bus_token;
 962                 return (!node || to_of_node(d->fwnode) == node) && is_ipi;
 963                 break;
 964         default:
 965                 return 0;
 966         }
 967 }
 968
 969 static struct irq_domain_ops gic_ipi_domain_ops = {
 970         .xlate = gic_ipi_domain_xlate,
 971         .alloc = gic_ipi_domain_alloc,
 972         .free = gic_ipi_domain_free,
 973         .match = gic_ipi_domain_match,
 974 };
 975
 976 static void __init __gic_init(unsigned long gic_base_addr,
 977                               unsigned long gic_addrspace_size,
 978                               unsigned int cpu_vec, unsigned int irqbase,
 979                               struct device_node *node)
 980 {
 981         unsigned int gicconfig, cpu;
 982         unsigned int v[2];
 983
 984         __gic_base_addr = gic_base_addr;
 985
 986         gic_base = ioremap_nocache(gic_base_addr, gic_addrspace_size);
 987
 988         gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
 989         gic_shared_intrs = (gicconfig & GIC_SH_CONFIG_NUMINTRS_MSK) >>
 990                    GIC_SH_CONFIG_NUMINTRS_SHF;
 991         gic_shared_intrs = ((gic_shared_intrs + 1) * 8);
 992
 993         gic_vpes = (gicconfig & GIC_SH_CONFIG_NUMVPES_MSK) >>
 994                   GIC_SH_CONFIG_NUMVPES_SHF;
 995         gic_vpes = gic_vpes + 1;
 996
 997         if (cpu_has_veic) {
 998                 /* Set EIC mode for all VPEs */
 999                 for_each_present_cpu(cpu) {
1000                         gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
1001                                   mips_cm_vp_id(cpu));
1002                         gic_write(GIC_REG(VPE_OTHER, GIC_VPE_CTL),
1003                                   GIC_VPE_CTL_EIC_MODE_MSK);
1004                 }
1005
1006                 /* Always use vector 1 in EIC mode */
1007                 gic_cpu_pin = 0;
1008                 timer_cpu_pin = gic_cpu_pin;
1009                 set_vi_handler(gic_cpu_pin + GIC_PIN_TO_VEC_OFFSET,
1010                                __gic_irq_dispatch);
1011         } else {
1012                 gic_cpu_pin = cpu_vec - GIC_CPU_PIN_OFFSET;
1013                 irq_set_chained_handler(MIPS_CPU_IRQ_BASE + cpu_vec,
1014                                         gic_irq_dispatch);
1015                 /*
1016                  * With the CMP implementation of SMP (deprecated), other CPUs
1017                  * are started by the bootloader and put into a timer based
1018                  * waiting poll loop. We must not re-route those CPU's local
1019                  * timer interrupts as the wait instruction will never finish,
1020                  * so just handle whatever CPU interrupt it is routed to by
1021                  * default.
1022                  *
1023                  * This workaround should be removed when CMP support is
1024                  * dropped.
1025                  */
1026                 if (IS_ENABLED(CONFIG_MIPS_CMP) &&
1027                     gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) {
1028                         timer_cpu_pin = gic_read32(GIC_REG(VPE_LOCAL,
1029                                                          GIC_VPE_TIMER_MAP)) &
1030                                         GIC_MAP_MSK;
1031                         irq_set_chained_handler(MIPS_CPU_IRQ_BASE +
1032                                                 GIC_CPU_PIN_OFFSET +
1033                                                 timer_cpu_pin,
1034                                                 gic_irq_dispatch);
1035                 } else {
1036                         timer_cpu_pin = gic_cpu_pin;
1037                 }
1038         }
1039
1040         gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS +
1041                                                gic_shared_intrs, irqbase,
1042                                                &gic_irq_domain_ops, NULL);
1043         if (!gic_irq_domain)
1044                 panic("Failed to add GIC IRQ domain");
1045         gic_irq_domain->name = "mips-gic-irq";
1046
1047         gic_dev_domain = irq_domain_add_hierarchy(gic_irq_domain, 0,
1048                                                   GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
1049                                                   node, &gic_dev_domain_ops, NULL);
1050         if (!gic_dev_domain)
1051                 panic("Failed to add GIC DEV domain");
1052         gic_dev_domain->name = "mips-gic-dev";
1053
1054         gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain,
1055                                                   IRQ_DOMAIN_FLAG_IPI_PER_CPU,
1056                                                   GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
1057                                                   node, &gic_ipi_domain_ops, NULL);
1058         if (!gic_ipi_domain)
1059                 panic("Failed to add GIC IPI domain");
1060
1061         gic_ipi_domain->name = "mips-gic-ipi";
1062         gic_ipi_domain->bus_token = DOMAIN_BUS_IPI;
1063
1064         if (node &&
1065             !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) {
1066                 bitmap_set(ipi_resrv, v[0], v[1]);
1067         } else {
1068                 /* Make the last 2 * gic_vpes available for IPIs */
1069                 bitmap_set(ipi_resrv,
1070                            gic_shared_intrs - 2 * gic_vpes,
1071                            2 * gic_vpes);
1072         }
1073
1074         gic_basic_init();
1075 }
1076
1077 void __init gic_init(unsigned long gic_base_addr,
1078                      unsigned long gic_addrspace_size,
1079                      unsigned int cpu_vec, unsigned int irqbase)
1080 {
1081         __gic_init(gic_base_addr, gic_addrspace_size, cpu_vec, irqbase, NULL);
1082 }
1083
1084 static int __init gic_of_init(struct device_node *node,
1085                               struct device_node *parent)
1086 {
1087         struct resource res;
1088         unsigned int cpu_vec, i = 0, reserved = 0;
1089         phys_addr_t gic_base;
1090         size_t gic_len;
1091
1092         /* Find the first available CPU vector. */
1093         while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors",
1094                                            i++, &cpu_vec))
1095                 reserved |= BIT(cpu_vec);
1096         for (cpu_vec = 2; cpu_vec < 8; cpu_vec++) {
1097                 if (!(reserved & BIT(cpu_vec)))
1098                         break;
1099         }
1100         if (cpu_vec == 8) {
1101                 pr_err("No CPU vectors available for GIC\n");
1102                 return -ENODEV;
1103         }
1104
1105         if (of_address_to_resource(node, 0, &res)) {
1106                 /*
1107                  * Probe the CM for the GIC base address if not specified
1108                  * in the device-tree.
1109                  */
1110                 if (mips_cm_present()) {
1111                         gic_base = read_gcr_gic_base() &
1112                                 ~CM_GCR_GIC_BASE_GICEN_MSK;
1113                         gic_len = 0x20000;
1114                 } else {
1115                         pr_err("Failed to get GIC memory range\n");
1116                         return -ENODEV;
1117                 }
1118         } else {
1119                 gic_base = res.start;
1120                 gic_len = resource_size(&res);
1121         }
1122
1123         if (mips_cm_present())
1124                 write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN_MSK);
1125         gic_present = true;
1126
1127         __gic_init(gic_base, gic_len, cpu_vec, 0, node);
1128
1129         return 0;
1130 }
1131 IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init);