arch/mips/cavium-octeon/octeon-irq.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2004-2014 Cavium, Inc.
   7  */
   8
   9 #include <linux/of_address.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/irqdomain.h>
  12 #include <linux/bitops.h>
  13 #include <linux/of_irq.h>
  14 #include <linux/percpu.h>
  15 #include <linux/slab.h>
  16 #include <linux/irq.h>
  17 #include <linux/smp.h>
  18 #include <linux/of.h>
  19
  20 #include <asm/octeon/octeon.h>
  21 #include <asm/octeon/cvmx-ciu2-defs.h>
  22
  23 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror);
  24 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror);
  25 static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock);
  26
  27 struct octeon_irq_ciu_domain_data {
  28         int num_sum;  /* number of sum registers (2 or 3). */
  29 };
  30
  31 static __read_mostly u8 octeon_irq_ciu_to_irq[8][64];
  32
  33 struct octeon_ciu_chip_data {
  34         union {
  35                 struct {                /* only used for ciu3 */
  36                         u64 ciu3_addr;
  37                         unsigned int intsn;
  38                 };
  39                 struct {                /* only used for ciu/ciu2 */
  40                         u8 line;
  41                         u8 bit;
  42                         u8 gpio_line;
  43                 };
  44         };
  45         int current_cpu;        /* Next CPU expected to take this irq */
  46 };
  47
  48 struct octeon_core_chip_data {
  49         struct mutex core_irq_mutex;
  50         bool current_en;
  51         bool desired_en;
  52         u8 bit;
  53 };
  54
  55 #define MIPS_CORE_IRQ_LINES 8
  56
  57 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES];
  58
  59 static int octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line,
  60                                       struct irq_chip *chip,
  61                                       irq_flow_handler_t handler)
  62 {
  63         struct octeon_ciu_chip_data *cd;
  64
  65         cd = kzalloc(sizeof(*cd), GFP_KERNEL);
  66         if (!cd)
  67                 return -ENOMEM;
  68
  69         irq_set_chip_and_handler(irq, chip, handler);
  70
  71         cd->line = line;
  72         cd->bit = bit;
  73         cd->gpio_line = gpio_line;
  74
  75         irq_set_chip_data(irq, cd);
  76         octeon_irq_ciu_to_irq[line][bit] = irq;
  77         return 0;
  78 }
  79
  80 static void octeon_irq_free_cd(struct irq_domain *d, unsigned int irq)
  81 {
  82         struct irq_data *data = irq_get_irq_data(irq);
  83         struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data);
  84
  85         irq_set_chip_data(irq, NULL);
  86         kfree(cd);
  87 }
  88
  89 static int octeon_irq_force_ciu_mapping(struct irq_domain *domain,
  90                                         int irq, int line, int bit)
  91 {
  92         return irq_domain_associate(domain, irq, line << 6 | bit);
  93 }
  94
  95 static int octeon_coreid_for_cpu(int cpu)
  96 {
  97 #ifdef CONFIG_SMP
  98         return cpu_logical_map(cpu);
  99 #else
 100         return cvmx_get_core_num();
 101 #endif
 102 }
 103
 104 static int octeon_cpu_for_coreid(int coreid)
 105 {
 106 #ifdef CONFIG_SMP
 107         return cpu_number_map(coreid);
 108 #else
 109         return smp_processor_id();
 110 #endif
 111 }
 112
 113 static void octeon_irq_core_ack(struct irq_data *data)
 114 {
 115         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 116         unsigned int bit = cd->bit;
 117
 118         /*
 119          * We don't need to disable IRQs to make these atomic since
 120          * they are already disabled earlier in the low level
 121          * interrupt code.
 122          */
 123         clear_c0_status(0x100 << bit);
 124         /* The two user interrupts must be cleared manually. */
 125         if (bit < 2)
 126                 clear_c0_cause(0x100 << bit);
 127 }
 128
 129 static void octeon_irq_core_eoi(struct irq_data *data)
 130 {
 131         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 132
 133         /*
 134          * We don't need to disable IRQs to make these atomic since
 135          * they are already disabled earlier in the low level
 136          * interrupt code.
 137          */
 138         set_c0_status(0x100 << cd->bit);
 139 }
 140
 141 static void octeon_irq_core_set_enable_local(void *arg)
 142 {
 143         struct irq_data *data = arg;
 144         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 145         unsigned int mask = 0x100 << cd->bit;
 146
 147         /*
 148          * Interrupts are already disabled, so these are atomic.
 149          */
 150         if (cd->desired_en)
 151                 set_c0_status(mask);
 152         else
 153                 clear_c0_status(mask);
 154
 155 }
 156
 157 static void octeon_irq_core_disable(struct irq_data *data)
 158 {
 159         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 160         cd->desired_en = false;
 161 }
 162
 163 static void octeon_irq_core_enable(struct irq_data *data)
 164 {
 165         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 166         cd->desired_en = true;
 167 }
 168
 169 static void octeon_irq_core_bus_lock(struct irq_data *data)
 170 {
 171         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 172
 173         mutex_lock(&cd->core_irq_mutex);
 174 }
 175
 176 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data)
 177 {
 178         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 179
 180         if (cd->desired_en != cd->current_en) {
 181                 on_each_cpu(octeon_irq_core_set_enable_local, data, 1);
 182
 183                 cd->current_en = cd->desired_en;
 184         }
 185
 186         mutex_unlock(&cd->core_irq_mutex);
 187 }
 188
 189 static struct irq_chip octeon_irq_chip_core = {
 190         .name = "Core",
 191         .irq_enable = octeon_irq_core_enable,
 192         .irq_disable = octeon_irq_core_disable,
 193         .irq_ack = octeon_irq_core_ack,
 194         .irq_eoi = octeon_irq_core_eoi,
 195         .irq_bus_lock = octeon_irq_core_bus_lock,
 196         .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock,
 197
 198         .irq_cpu_online = octeon_irq_core_eoi,
 199         .irq_cpu_offline = octeon_irq_core_ack,
 200         .flags = IRQCHIP_ONOFFLINE_ENABLED,
 201 };
 202
 203 static void __init octeon_irq_init_core(void)
 204 {
 205         int i;
 206         int irq;
 207         struct octeon_core_chip_data *cd;
 208
 209         for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) {
 210                 cd = &octeon_irq_core_chip_data[i];
 211                 cd->current_en = false;
 212                 cd->desired_en = false;
 213                 cd->bit = i;
 214                 mutex_init(&cd->core_irq_mutex);
 215
 216                 irq = OCTEON_IRQ_SW0 + i;
 217                 irq_set_chip_data(irq, cd);
 218                 irq_set_chip_and_handler(irq, &octeon_irq_chip_core,
 219                                          handle_percpu_irq);
 220         }
 221 }
 222
 223 static int next_cpu_for_irq(struct irq_data *data)
 224 {
 225
 226 #ifdef CONFIG_SMP
 227         int cpu;
 228         struct cpumask *mask = irq_data_get_affinity_mask(data);
 229         int weight = cpumask_weight(mask);
 230         struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data);
 231
 232         if (weight > 1) {
 233                 cpu = cd->current_cpu;
 234                 for (;;) {
 235                         cpu = cpumask_next(cpu, mask);
 236                         if (cpu >= nr_cpu_ids) {
 237                                 cpu = -1;
 238                                 continue;
 239                         } else if (cpumask_test_cpu(cpu, cpu_online_mask)) {
 240                                 break;
 241                         }
 242                 }
 243         } else if (weight == 1) {
 244                 cpu = cpumask_first(mask);
 245         } else {
 246                 cpu = smp_processor_id();
 247         }
 248         cd->current_cpu = cpu;
 249         return cpu;
 250 #else
 251         return smp_processor_id();
 252 #endif
 253 }
 254
 255 static void octeon_irq_ciu_enable(struct irq_data *data)
 256 {
 257         int cpu = next_cpu_for_irq(data);
 258         int coreid = octeon_coreid_for_cpu(cpu);
 259         unsigned long *pen;
 260         unsigned long flags;
 261         struct octeon_ciu_chip_data *cd;
 262         raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 263
 264         cd = irq_data_get_irq_chip_data(data);
 265
 266         raw_spin_lock_irqsave(lock, flags);
 267         if (cd->line == 0) {
 268                 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 269                 __set_bit(cd->bit, pen);
 270                 /*
 271                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 272                  * enabling the irq.
 273                  */
 274                 wmb();
 275                 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 276         } else {
 277                 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 278                 __set_bit(cd->bit, pen);
 279                 /*
 280                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 281                  * enabling the irq.
 282                  */
 283                 wmb();
 284                 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 285         }
 286         raw_spin_unlock_irqrestore(lock, flags);
 287 }
 288
 289 static void octeon_irq_ciu_enable_local(struct irq_data *data)
 290 {
 291         unsigned long *pen;
 292         unsigned long flags;
 293         struct octeon_ciu_chip_data *cd;
 294         raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock);
 295
 296         cd = irq_data_get_irq_chip_data(data);
 297
 298         raw_spin_lock_irqsave(lock, flags);
 299         if (cd->line == 0) {
 300                 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror);
 301                 __set_bit(cd->bit, pen);
 302                 /*
 303                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 304                  * enabling the irq.
 305                  */
 306                 wmb();
 307                 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
 308         } else {
 309                 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror);
 310                 __set_bit(cd->bit, pen);
 311                 /*
 312                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 313                  * enabling the irq.
 314                  */
 315                 wmb();
 316                 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
 317         }
 318         raw_spin_unlock_irqrestore(lock, flags);
 319 }
 320
 321 static void octeon_irq_ciu_disable_local(struct irq_data *data)
 322 {
 323         unsigned long *pen;
 324         unsigned long flags;
 325         struct octeon_ciu_chip_data *cd;
 326         raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock);
 327
 328         cd = irq_data_get_irq_chip_data(data);
 329
 330         raw_spin_lock_irqsave(lock, flags);
 331         if (cd->line == 0) {
 332                 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror);
 333                 __clear_bit(cd->bit, pen);
 334                 /*
 335                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 336                  * enabling the irq.
 337                  */
 338                 wmb();
 339                 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
 340         } else {
 341                 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror);
 342                 __clear_bit(cd->bit, pen);
 343                 /*
 344                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 345                  * enabling the irq.
 346                  */
 347                 wmb();
 348                 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
 349         }
 350         raw_spin_unlock_irqrestore(lock, flags);
 351 }
 352
 353 static void octeon_irq_ciu_disable_all(struct irq_data *data)
 354 {
 355         unsigned long flags;
 356         unsigned long *pen;
 357         int cpu;
 358         struct octeon_ciu_chip_data *cd;
 359         raw_spinlock_t *lock;
 360
 361         cd = irq_data_get_irq_chip_data(data);
 362
 363         for_each_online_cpu(cpu) {
 364                 int coreid = octeon_coreid_for_cpu(cpu);
 365                 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 366                 if (cd->line == 0)
 367                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 368                 else
 369                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 370
 371                 raw_spin_lock_irqsave(lock, flags);
 372                 __clear_bit(cd->bit, pen);
 373                 /*
 374                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 375                  * enabling the irq.
 376                  */
 377                 wmb();
 378                 if (cd->line == 0)
 379                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 380                 else
 381                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 382                 raw_spin_unlock_irqrestore(lock, flags);
 383         }
 384 }
 385
 386 static void octeon_irq_ciu_enable_all(struct irq_data *data)
 387 {
 388         unsigned long flags;
 389         unsigned long *pen;
 390         int cpu;
 391         struct octeon_ciu_chip_data *cd;
 392         raw_spinlock_t *lock;
 393
 394         cd = irq_data_get_irq_chip_data(data);
 395
 396         for_each_online_cpu(cpu) {
 397                 int coreid = octeon_coreid_for_cpu(cpu);
 398                 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 399                 if (cd->line == 0)
 400                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 401                 else
 402                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 403
 404                 raw_spin_lock_irqsave(lock, flags);
 405                 __set_bit(cd->bit, pen);
 406                 /*
 407                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 408                  * enabling the irq.
 409                  */
 410                 wmb();
 411                 if (cd->line == 0)
 412                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 413                 else
 414                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 415                 raw_spin_unlock_irqrestore(lock, flags);
 416         }
 417 }
 418
 419 /*
 420  * Enable the irq on the next core in the affinity set for chips that
 421  * have the EN*_W1{S,C} registers.
 422  */
 423 static void octeon_irq_ciu_enable_v2(struct irq_data *data)
 424 {
 425         u64 mask;
 426         int cpu = next_cpu_for_irq(data);
 427         struct octeon_ciu_chip_data *cd;
 428
 429         cd = irq_data_get_irq_chip_data(data);
 430         mask = 1ull << (cd->bit);
 431
 432         /*
 433          * Called under the desc lock, so these should never get out
 434          * of sync.
 435          */
 436         if (cd->line == 0) {
 437                 int index = octeon_coreid_for_cpu(cpu) * 2;
 438                 set_bit(cd->bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
 439                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 440         } else {
 441                 int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 442                 set_bit(cd->bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 443                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 444         }
 445 }
 446
 447 /*
 448  * Enable the irq in the sum2 registers.
 449  */
 450 static void octeon_irq_ciu_enable_sum2(struct irq_data *data)
 451 {
 452         u64 mask;
 453         int cpu = next_cpu_for_irq(data);
 454         int index = octeon_coreid_for_cpu(cpu);
 455         struct octeon_ciu_chip_data *cd;
 456
 457         cd = irq_data_get_irq_chip_data(data);
 458         mask = 1ull << (cd->bit);
 459
 460         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask);
 461 }
 462
 463 /*
 464  * Disable the irq in the sum2 registers.
 465  */
 466 static void octeon_irq_ciu_disable_local_sum2(struct irq_data *data)
 467 {
 468         u64 mask;
 469         int cpu = next_cpu_for_irq(data);
 470         int index = octeon_coreid_for_cpu(cpu);
 471         struct octeon_ciu_chip_data *cd;
 472
 473         cd = irq_data_get_irq_chip_data(data);
 474         mask = 1ull << (cd->bit);
 475
 476         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask);
 477 }
 478
 479 static void octeon_irq_ciu_ack_sum2(struct irq_data *data)
 480 {
 481         u64 mask;
 482         int cpu = next_cpu_for_irq(data);
 483         int index = octeon_coreid_for_cpu(cpu);
 484         struct octeon_ciu_chip_data *cd;
 485
 486         cd = irq_data_get_irq_chip_data(data);
 487         mask = 1ull << (cd->bit);
 488
 489         cvmx_write_csr(CVMX_CIU_SUM2_PPX_IP4(index), mask);
 490 }
 491
 492 static void octeon_irq_ciu_disable_all_sum2(struct irq_data *data)
 493 {
 494         int cpu;
 495         struct octeon_ciu_chip_data *cd;
 496         u64 mask;
 497
 498         cd = irq_data_get_irq_chip_data(data);
 499         mask = 1ull << (cd->bit);
 500
 501         for_each_online_cpu(cpu) {
 502                 int coreid = octeon_coreid_for_cpu(cpu);
 503
 504                 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(coreid), mask);
 505         }
 506 }
 507
 508 /*
 509  * Enable the irq on the current CPU for chips that
 510  * have the EN*_W1{S,C} registers.
 511  */
 512 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data)
 513 {
 514         u64 mask;
 515         struct octeon_ciu_chip_data *cd;
 516
 517         cd = irq_data_get_irq_chip_data(data);
 518         mask = 1ull << (cd->bit);
 519
 520         if (cd->line == 0) {
 521                 int index = cvmx_get_core_num() * 2;
 522                 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror));
 523                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 524         } else {
 525                 int index = cvmx_get_core_num() * 2 + 1;
 526                 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror));
 527                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 528         }
 529 }
 530
 531 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data)
 532 {
 533         u64 mask;
 534         struct octeon_ciu_chip_data *cd;
 535
 536         cd = irq_data_get_irq_chip_data(data);
 537         mask = 1ull << (cd->bit);
 538
 539         if (cd->line == 0) {
 540                 int index = cvmx_get_core_num() * 2;
 541                 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror));
 542                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 543         } else {
 544                 int index = cvmx_get_core_num() * 2 + 1;
 545                 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror));
 546                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 547         }
 548 }
 549
 550 /*
 551  * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq.
 552  */
 553 static void octeon_irq_ciu_ack(struct irq_data *data)
 554 {
 555         u64 mask;
 556         struct octeon_ciu_chip_data *cd;
 557
 558         cd = irq_data_get_irq_chip_data(data);
 559         mask = 1ull << (cd->bit);
 560
 561         if (cd->line == 0) {
 562                 int index = cvmx_get_core_num() * 2;
 563                 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask);
 564         } else {
 565                 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask);
 566         }
 567 }
 568
 569 /*
 570  * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
 571  * registers.
 572  */
 573 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data)
 574 {
 575         int cpu;
 576         u64 mask;
 577         struct octeon_ciu_chip_data *cd;
 578
 579         cd = irq_data_get_irq_chip_data(data);
 580         mask = 1ull << (cd->bit);
 581
 582         if (cd->line == 0) {
 583                 for_each_online_cpu(cpu) {
 584                         int index = octeon_coreid_for_cpu(cpu) * 2;
 585                         clear_bit(cd->bit,
 586                                 &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
 587                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 588                 }
 589         } else {
 590                 for_each_online_cpu(cpu) {
 591                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 592                         clear_bit(cd->bit,
 593                                 &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 594                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 595                 }
 596         }
 597 }
 598
 599 /*
 600  * Enable the irq on the all cores for chips that have the EN*_W1{S,C}
 601  * registers.
 602  */
 603 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data)
 604 {
 605         int cpu;
 606         u64 mask;
 607         struct octeon_ciu_chip_data *cd;
 608
 609         cd = irq_data_get_irq_chip_data(data);
 610         mask = 1ull << (cd->bit);
 611
 612         if (cd->line == 0) {
 613                 for_each_online_cpu(cpu) {
 614                         int index = octeon_coreid_for_cpu(cpu) * 2;
 615                         set_bit(cd->bit,
 616                                 &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
 617                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 618                 }
 619         } else {
 620                 for_each_online_cpu(cpu) {
 621                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 622                         set_bit(cd->bit,
 623                                 &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 624                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 625                 }
 626         }
 627 }
 628
 629 static void octeon_irq_gpio_setup(struct irq_data *data)
 630 {
 631         union cvmx_gpio_bit_cfgx cfg;
 632         struct octeon_ciu_chip_data *cd;
 633         u32 t = irqd_get_trigger_type(data);
 634
 635         cd = irq_data_get_irq_chip_data(data);
 636
 637         cfg.u64 = 0;
 638         cfg.s.int_en = 1;
 639         cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0;
 640         cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) != 0;
 641
 642         /* 140 nS glitch filter*/
 643         cfg.s.fil_cnt = 7;
 644         cfg.s.fil_sel = 3;
 645
 646         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), cfg.u64);
 647 }
 648
 649 static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data)
 650 {
 651         octeon_irq_gpio_setup(data);
 652         octeon_irq_ciu_enable_v2(data);
 653 }
 654
 655 static void octeon_irq_ciu_enable_gpio(struct irq_data *data)
 656 {
 657         octeon_irq_gpio_setup(data);
 658         octeon_irq_ciu_enable(data);
 659 }
 660
 661 static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t)
 662 {
 663         irqd_set_trigger_type(data, t);
 664         octeon_irq_gpio_setup(data);
 665
 666         if (irqd_get_trigger_type(data) & IRQ_TYPE_EDGE_BOTH)
 667                 irq_set_handler_locked(data, handle_edge_irq);
 668         else
 669                 irq_set_handler_locked(data, handle_level_irq);
 670
 671         return IRQ_SET_MASK_OK;
 672 }
 673
 674 static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data)
 675 {
 676         struct octeon_ciu_chip_data *cd;
 677
 678         cd = irq_data_get_irq_chip_data(data);
 679         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0);
 680
 681         octeon_irq_ciu_disable_all_v2(data);
 682 }
 683
 684 static void octeon_irq_ciu_disable_gpio(struct irq_data *data)
 685 {
 686         struct octeon_ciu_chip_data *cd;
 687
 688         cd = irq_data_get_irq_chip_data(data);
 689         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0);
 690
 691         octeon_irq_ciu_disable_all(data);
 692 }
 693
 694 static void octeon_irq_ciu_gpio_ack(struct irq_data *data)
 695 {
 696         struct octeon_ciu_chip_data *cd;
 697         u64 mask;
 698
 699         cd = irq_data_get_irq_chip_data(data);
 700         mask = 1ull << (cd->gpio_line);
 701
 702         cvmx_write_csr(CVMX_GPIO_INT_CLR, mask);
 703 }
 704
 705 #ifdef CONFIG_SMP
 706
 707 static void octeon_irq_cpu_offline_ciu(struct irq_data *data)
 708 {
 709         int cpu = smp_processor_id();
 710         cpumask_t new_affinity;
 711         struct cpumask *mask = irq_data_get_affinity_mask(data);
 712
 713         if (!cpumask_test_cpu(cpu, mask))
 714                 return;
 715
 716         if (cpumask_weight(mask) > 1) {
 717                 /*
 718                  * It has multi CPU affinity, just remove this CPU
 719                  * from the affinity set.
 720                  */
 721                 cpumask_copy(&new_affinity, mask);
 722                 cpumask_clear_cpu(cpu, &new_affinity);
 723         } else {
 724                 /* Otherwise, put it on lowest numbered online CPU. */
 725                 cpumask_clear(&new_affinity);
 726                 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity);
 727         }
 728         irq_set_affinity_locked(data, &new_affinity, false);
 729 }
 730
 731 static int octeon_irq_ciu_set_affinity(struct irq_data *data,
 732                                        const struct cpumask *dest, bool force)
 733 {
 734         int cpu;
 735         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
 736         unsigned long flags;
 737         struct octeon_ciu_chip_data *cd;
 738         unsigned long *pen;
 739         raw_spinlock_t *lock;
 740
 741         cd = irq_data_get_irq_chip_data(data);
 742
 743         /*
 744          * For non-v2 CIU, we will allow only single CPU affinity.
 745          * This removes the need to do locking in the .ack/.eoi
 746          * functions.
 747          */
 748         if (cpumask_weight(dest) != 1)
 749                 return -EINVAL;
 750
 751         if (!enable_one)
 752                 return 0;
 753
 754
 755         for_each_online_cpu(cpu) {
 756                 int coreid = octeon_coreid_for_cpu(cpu);
 757
 758                 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 759                 raw_spin_lock_irqsave(lock, flags);
 760
 761                 if (cd->line == 0)
 762                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 763                 else
 764                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 765
 766                 if (cpumask_test_cpu(cpu, dest) && enable_one) {
 767                         enable_one = 0;
 768                         __set_bit(cd->bit, pen);
 769                 } else {
 770                         __clear_bit(cd->bit, pen);
 771                 }
 772                 /*
 773                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 774                  * enabling the irq.
 775                  */
 776                 wmb();
 777
 778                 if (cd->line == 0)
 779                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 780                 else
 781                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 782
 783                 raw_spin_unlock_irqrestore(lock, flags);
 784         }
 785         return 0;
 786 }
 787
 788 /*
 789  * Set affinity for the irq for chips that have the EN*_W1{S,C}
 790  * registers.
 791  */
 792 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data,
 793                                           const struct cpumask *dest,
 794                                           bool force)
 795 {
 796         int cpu;
 797         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
 798         u64 mask;
 799         struct octeon_ciu_chip_data *cd;
 800
 801         if (!enable_one)
 802                 return 0;
 803
 804         cd = irq_data_get_irq_chip_data(data);
 805         mask = 1ull << cd->bit;
 806
 807         if (cd->line == 0) {
 808                 for_each_online_cpu(cpu) {
 809                         unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 810                         int index = octeon_coreid_for_cpu(cpu) * 2;
 811                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
 812                                 enable_one = false;
 813                                 set_bit(cd->bit, pen);
 814                                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 815                         } else {
 816                                 clear_bit(cd->bit, pen);
 817                                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 818                         }
 819                 }
 820         } else {
 821                 for_each_online_cpu(cpu) {
 822                         unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 823                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 824                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
 825                                 enable_one = false;
 826                                 set_bit(cd->bit, pen);
 827                                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 828                         } else {
 829                                 clear_bit(cd->bit, pen);
 830                                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 831                         }
 832                 }
 833         }
 834         return 0;
 835 }
 836
 837 static int octeon_irq_ciu_set_affinity_sum2(struct irq_data *data,
 838                                             const struct cpumask *dest,
 839                                             bool force)
 840 {
 841         int cpu;
 842         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
 843         u64 mask;
 844         struct octeon_ciu_chip_data *cd;
 845
 846         if (!enable_one)
 847                 return 0;
 848
 849         cd = irq_data_get_irq_chip_data(data);
 850         mask = 1ull << cd->bit;
 851
 852         for_each_online_cpu(cpu) {
 853                 int index = octeon_coreid_for_cpu(cpu);
 854
 855                 if (cpumask_test_cpu(cpu, dest) && enable_one) {
 856                         enable_one = false;
 857                         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask);
 858                 } else {
 859                         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask);
 860                 }
 861         }
 862         return 0;
 863 }
 864 #endif
 865
 866 /*
 867  * Newer octeon chips have support for lockless CIU operation.
 868  */
 869 static struct irq_chip octeon_irq_chip_ciu_v2 = {
 870         .name = "CIU",
 871         .irq_enable = octeon_irq_ciu_enable_v2,
 872         .irq_disable = octeon_irq_ciu_disable_all_v2,
 873         .irq_mask = octeon_irq_ciu_disable_local_v2,
 874         .irq_unmask = octeon_irq_ciu_enable_v2,
 875 #ifdef CONFIG_SMP
 876         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
 877         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 878 #endif
 879 };
 880
 881 static struct irq_chip octeon_irq_chip_ciu_v2_edge = {
 882         .name = "CIU",
 883         .irq_enable = octeon_irq_ciu_enable_v2,
 884         .irq_disable = octeon_irq_ciu_disable_all_v2,
 885         .irq_ack = octeon_irq_ciu_ack,
 886         .irq_mask = octeon_irq_ciu_disable_local_v2,
 887         .irq_unmask = octeon_irq_ciu_enable_v2,
 888 #ifdef CONFIG_SMP
 889         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
 890         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 891 #endif
 892 };
 893
 894 /*
 895  * Newer octeon chips have support for lockless CIU operation.
 896  */
 897 static struct irq_chip octeon_irq_chip_ciu_sum2 = {
 898         .name = "CIU",
 899         .irq_enable = octeon_irq_ciu_enable_sum2,
 900         .irq_disable = octeon_irq_ciu_disable_all_sum2,
 901         .irq_mask = octeon_irq_ciu_disable_local_sum2,
 902         .irq_unmask = octeon_irq_ciu_enable_sum2,
 903 #ifdef CONFIG_SMP
 904         .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2,
 905         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 906 #endif
 907 };
 908
 909 static struct irq_chip octeon_irq_chip_ciu_sum2_edge = {
 910         .name = "CIU",
 911         .irq_enable = octeon_irq_ciu_enable_sum2,
 912         .irq_disable = octeon_irq_ciu_disable_all_sum2,
 913         .irq_ack = octeon_irq_ciu_ack_sum2,
 914         .irq_mask = octeon_irq_ciu_disable_local_sum2,
 915         .irq_unmask = octeon_irq_ciu_enable_sum2,
 916 #ifdef CONFIG_SMP
 917         .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2,
 918         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 919 #endif
 920 };
 921
 922 static struct irq_chip octeon_irq_chip_ciu = {
 923         .name = "CIU",
 924         .irq_enable = octeon_irq_ciu_enable,
 925         .irq_disable = octeon_irq_ciu_disable_all,
 926         .irq_mask = octeon_irq_ciu_disable_local,
 927         .irq_unmask = octeon_irq_ciu_enable,
 928 #ifdef CONFIG_SMP
 929         .irq_set_affinity = octeon_irq_ciu_set_affinity,
 930         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 931 #endif
 932 };
 933
 934 static struct irq_chip octeon_irq_chip_ciu_edge = {
 935         .name = "CIU",
 936         .irq_enable = octeon_irq_ciu_enable,
 937         .irq_disable = octeon_irq_ciu_disable_all,
 938         .irq_ack = octeon_irq_ciu_ack,
 939         .irq_mask = octeon_irq_ciu_disable_local,
 940         .irq_unmask = octeon_irq_ciu_enable,
 941 #ifdef CONFIG_SMP
 942         .irq_set_affinity = octeon_irq_ciu_set_affinity,
 943         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 944 #endif
 945 };
 946
 947 /* The mbox versions don't do any affinity or round-robin. */
 948 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = {
 949         .name = "CIU-M",
 950         .irq_enable = octeon_irq_ciu_enable_all_v2,
 951         .irq_disable = octeon_irq_ciu_disable_all_v2,
 952         .irq_ack = octeon_irq_ciu_disable_local_v2,
 953         .irq_eoi = octeon_irq_ciu_enable_local_v2,
 954
 955         .irq_cpu_online = octeon_irq_ciu_enable_local_v2,
 956         .irq_cpu_offline = octeon_irq_ciu_disable_local_v2,
 957         .flags = IRQCHIP_ONOFFLINE_ENABLED,
 958 };
 959
 960 static struct irq_chip octeon_irq_chip_ciu_mbox = {
 961         .name = "CIU-M",
 962         .irq_enable = octeon_irq_ciu_enable_all,
 963         .irq_disable = octeon_irq_ciu_disable_all,
 964         .irq_ack = octeon_irq_ciu_disable_local,
 965         .irq_eoi = octeon_irq_ciu_enable_local,
 966
 967         .irq_cpu_online = octeon_irq_ciu_enable_local,
 968         .irq_cpu_offline = octeon_irq_ciu_disable_local,
 969         .flags = IRQCHIP_ONOFFLINE_ENABLED,
 970 };
 971
 972 static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = {
 973         .name = "CIU-GPIO",
 974         .irq_enable = octeon_irq_ciu_enable_gpio_v2,
 975         .irq_disable = octeon_irq_ciu_disable_gpio_v2,
 976         .irq_ack = octeon_irq_ciu_gpio_ack,
 977         .irq_mask = octeon_irq_ciu_disable_local_v2,
 978         .irq_unmask = octeon_irq_ciu_enable_v2,
 979         .irq_set_type = octeon_irq_ciu_gpio_set_type,
 980 #ifdef CONFIG_SMP
 981         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
 982         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 983 #endif
 984         .flags = IRQCHIP_SET_TYPE_MASKED,
 985 };
 986
 987 static struct irq_chip octeon_irq_chip_ciu_gpio = {
 988         .name = "CIU-GPIO",
 989         .irq_enable = octeon_irq_ciu_enable_gpio,
 990         .irq_disable = octeon_irq_ciu_disable_gpio,
 991         .irq_mask = octeon_irq_ciu_disable_local,
 992         .irq_unmask = octeon_irq_ciu_enable,
 993         .irq_ack = octeon_irq_ciu_gpio_ack,
 994         .irq_set_type = octeon_irq_ciu_gpio_set_type,
 995 #ifdef CONFIG_SMP
 996         .irq_set_affinity = octeon_irq_ciu_set_affinity,
 997         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 998 #endif
 999         .flags = IRQCHIP_SET_TYPE_MASKED,
1000 };
1001
1002 /*
1003  * Watchdog interrupts are special.  They are associated with a single
1004  * core, so we hardwire the affinity to that core.
1005  */
1006 static void octeon_irq_ciu_wd_enable(struct irq_data *data)
1007 {
1008         unsigned long flags;
1009         unsigned long *pen;
1010         int coreid = data->irq - OCTEON_IRQ_WDOG0;      /* Bit 0-63 of EN1 */
1011         int cpu = octeon_cpu_for_coreid(coreid);
1012         raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
1013
1014         raw_spin_lock_irqsave(lock, flags);
1015         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
1016         __set_bit(coreid, pen);
1017         /*
1018          * Must be visible to octeon_irq_ip{2,3}_ciu() before enabling
1019          * the irq.
1020          */
1021         wmb();
1022         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
1023         raw_spin_unlock_irqrestore(lock, flags);
1024 }
1025
1026 /*
1027  * Watchdog interrupts are special.  They are associated with a single
1028  * core, so we hardwire the affinity to that core.
1029  */
1030 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data)
1031 {
1032         int coreid = data->irq - OCTEON_IRQ_WDOG0;
1033         int cpu = octeon_cpu_for_coreid(coreid);
1034
1035         set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
1036         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid);
1037 }
1038
1039
1040 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = {
1041         .name = "CIU-W",
1042         .irq_enable = octeon_irq_ciu1_wd_enable_v2,
1043         .irq_disable = octeon_irq_ciu_disable_all_v2,
1044         .irq_mask = octeon_irq_ciu_disable_local_v2,
1045         .irq_unmask = octeon_irq_ciu_enable_local_v2,
1046 };
1047
1048 static struct irq_chip octeon_irq_chip_ciu_wd = {
1049         .name = "CIU-W",
1050         .irq_enable = octeon_irq_ciu_wd_enable,
1051         .irq_disable = octeon_irq_ciu_disable_all,
1052         .irq_mask = octeon_irq_ciu_disable_local,
1053         .irq_unmask = octeon_irq_ciu_enable_local,
1054 };
1055
1056 static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit)
1057 {
1058         bool edge = false;
1059
1060         if (line == 0)
1061                 switch (bit) {
1062                 case 48 ... 49: /* GMX DRP */
1063                 case 50: /* IPD_DRP */
1064                 case 52 ... 55: /* Timers */
1065                 case 58: /* MPI */
1066                         edge = true;
1067                         break;
1068                 default:
1069                         break;
1070                 }
1071         else /* line == 1 */
1072                 switch (bit) {
1073                 case 47: /* PTP */
1074                         edge = true;
1075                         break;
1076                 default:
1077                         break;
1078                 }
1079         return edge;
1080 }
1081
1082 struct octeon_irq_gpio_domain_data {
1083         unsigned int base_hwirq;
1084 };
1085
1086 static int octeon_irq_gpio_xlat(struct irq_domain *d,
1087                                 struct device_node *node,
1088                                 const u32 *intspec,
1089                                 unsigned int intsize,
1090                                 unsigned long *out_hwirq,
1091                                 unsigned int *out_type)
1092 {
1093         unsigned int type;
1094         unsigned int pin;
1095         unsigned int trigger;
1096
1097         if (irq_domain_get_of_node(d) != node)
1098                 return -EINVAL;
1099
1100         if (intsize < 2)
1101                 return -EINVAL;
1102
1103         pin = intspec[0];
1104         if (pin >= 16)
1105                 return -EINVAL;
1106
1107         trigger = intspec[1];
1108
1109         switch (trigger) {
1110         case 1:
1111                 type = IRQ_TYPE_EDGE_RISING;
1112                 break;
1113         case 2:
1114                 type = IRQ_TYPE_EDGE_FALLING;
1115                 break;
1116         case 4:
1117                 type = IRQ_TYPE_LEVEL_HIGH;
1118                 break;
1119         case 8:
1120                 type = IRQ_TYPE_LEVEL_LOW;
1121                 break;
1122         default:
1123                 pr_err("Error: (%s) Invalid irq trigger specification: %x\n",
1124                        node->name,
1125                        trigger);
1126                 type = IRQ_TYPE_LEVEL_LOW;
1127                 break;
1128         }
1129         *out_type = type;
1130         *out_hwirq = pin;
1131
1132         return 0;
1133 }
1134
1135 static int octeon_irq_ciu_xlat(struct irq_domain *d,
1136                                struct device_node *node,
1137                                const u32 *intspec,
1138                                unsigned int intsize,
1139                                unsigned long *out_hwirq,
1140                                unsigned int *out_type)
1141 {
1142         unsigned int ciu, bit;
1143         struct octeon_irq_ciu_domain_data *dd = d->host_data;
1144
1145         ciu = intspec[0];
1146         bit = intspec[1];
1147
1148         if (ciu >= dd->num_sum || bit > 63)
1149                 return -EINVAL;
1150
1151         *out_hwirq = (ciu << 6) | bit;
1152         *out_type = 0;
1153
1154         return 0;
1155 }
1156
1157 static struct irq_chip *octeon_irq_ciu_chip;
1158 static struct irq_chip *octeon_irq_ciu_chip_edge;
1159 static struct irq_chip *octeon_irq_gpio_chip;
1160
1161 static bool octeon_irq_virq_in_range(unsigned int virq)
1162 {
1163         /* We cannot let it overflow the mapping array. */
1164         if (virq < (1ul << 8 * sizeof(octeon_irq_ciu_to_irq[0][0])))
1165                 return true;
1166
1167         WARN_ONCE(true, "virq out of range %u.\n", virq);
1168         return false;
1169 }
1170
1171 static int octeon_irq_ciu_map(struct irq_domain *d,
1172                               unsigned int virq, irq_hw_number_t hw)
1173 {
1174         int rv;
1175         unsigned int line = hw >> 6;
1176         unsigned int bit = hw & 63;
1177         struct octeon_irq_ciu_domain_data *dd = d->host_data;
1178
1179         if (!octeon_irq_virq_in_range(virq))
1180                 return -EINVAL;
1181
1182         /* Don't map irq if it is reserved for GPIO. */
1183         if (line == 0 && bit >= 16 && bit <32)
1184                 return 0;
1185
1186         if (line >= dd->num_sum || octeon_irq_ciu_to_irq[line][bit] != 0)
1187                 return -EINVAL;
1188
1189         if (line == 2) {
1190                 if (octeon_irq_ciu_is_edge(line, bit))
1191                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1192                                 &octeon_irq_chip_ciu_sum2_edge,
1193                                 handle_edge_irq);
1194                 else
1195                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1196                                 &octeon_irq_chip_ciu_sum2,
1197                                 handle_level_irq);
1198         } else {
1199                 if (octeon_irq_ciu_is_edge(line, bit))
1200                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1201                                 octeon_irq_ciu_chip_edge,
1202                                 handle_edge_irq);
1203                 else
1204                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1205                                 octeon_irq_ciu_chip,
1206                                 handle_level_irq);
1207         }
1208         return rv;
1209 }
1210
1211 static int octeon_irq_gpio_map(struct irq_domain *d,
1212                                unsigned int virq, irq_hw_number_t hw)
1213 {
1214         struct octeon_irq_gpio_domain_data *gpiod = d->host_data;
1215         unsigned int line, bit;
1216         int r;
1217
1218         if (!octeon_irq_virq_in_range(virq))
1219                 return -EINVAL;
1220
1221         line = (hw + gpiod->base_hwirq) >> 6;
1222         bit = (hw + gpiod->base_hwirq) & 63;
1223         if (line > ARRAY_SIZE(octeon_irq_ciu_to_irq) ||
1224                 octeon_irq_ciu_to_irq[line][bit] != 0)
1225                 return -EINVAL;
1226
1227         /*
1228          * Default to handle_level_irq. If the DT contains a different
1229          * trigger type, it will call the irq_set_type callback and
1230          * the handler gets updated.
1231          */
1232         r = octeon_irq_set_ciu_mapping(virq, line, bit, hw,
1233                                        octeon_irq_gpio_chip, handle_level_irq);
1234         return r;
1235 }
1236
1237 static struct irq_domain_ops octeon_irq_domain_ciu_ops = {
1238         .map = octeon_irq_ciu_map,
1239         .unmap = octeon_irq_free_cd,
1240         .xlate = octeon_irq_ciu_xlat,
1241 };
1242
1243 static struct irq_domain_ops octeon_irq_domain_gpio_ops = {
1244         .map = octeon_irq_gpio_map,
1245         .unmap = octeon_irq_free_cd,
1246         .xlate = octeon_irq_gpio_xlat,
1247 };
1248
1249 static void octeon_irq_ip2_ciu(void)
1250 {
1251         const unsigned long core_id = cvmx_get_core_num();
1252         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
1253
1254         ciu_sum &= __this_cpu_read(octeon_irq_ciu0_en_mirror);
1255         if (likely(ciu_sum)) {
1256                 int bit = fls64(ciu_sum) - 1;
1257                 int irq = octeon_irq_ciu_to_irq[0][bit];
1258                 if (likely(irq))
1259                         do_IRQ(irq);
1260                 else
1261                         spurious_interrupt();
1262         } else {
1263                 spurious_interrupt();
1264         }
1265 }
1266
1267 static void octeon_irq_ip3_ciu(void)
1268 {
1269         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
1270
1271         ciu_sum &= __this_cpu_read(octeon_irq_ciu1_en_mirror);
1272         if (likely(ciu_sum)) {
1273                 int bit = fls64(ciu_sum) - 1;
1274                 int irq = octeon_irq_ciu_to_irq[1][bit];
1275                 if (likely(irq))
1276                         do_IRQ(irq);
1277                 else
1278                         spurious_interrupt();
1279         } else {
1280                 spurious_interrupt();
1281         }
1282 }
1283
1284 static void octeon_irq_ip4_ciu(void)
1285 {
1286         int coreid = cvmx_get_core_num();
1287         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_SUM2_PPX_IP4(coreid));
1288         u64 ciu_en = cvmx_read_csr(CVMX_CIU_EN2_PPX_IP4(coreid));
1289
1290         ciu_sum &= ciu_en;
1291         if (likely(ciu_sum)) {
1292                 int bit = fls64(ciu_sum) - 1;
1293                 int irq = octeon_irq_ciu_to_irq[2][bit];
1294
1295                 if (likely(irq))
1296                         do_IRQ(irq);
1297                 else
1298                         spurious_interrupt();
1299         } else {
1300                 spurious_interrupt();
1301         }
1302 }
1303
1304 static bool octeon_irq_use_ip4;
1305
1306 static void octeon_irq_local_enable_ip4(void *arg)
1307 {
1308         set_c0_status(STATUSF_IP4);
1309 }
1310
1311 static void octeon_irq_ip4_mask(void)
1312 {
1313         clear_c0_status(STATUSF_IP4);
1314         spurious_interrupt();
1315 }
1316
1317 static void (*octeon_irq_ip2)(void);
1318 static void (*octeon_irq_ip3)(void);
1319 static void (*octeon_irq_ip4)(void);
1320
1321 void (*octeon_irq_setup_secondary)(void);
1322
1323 void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h)
1324 {
1325         octeon_irq_ip4 = h;
1326         octeon_irq_use_ip4 = true;
1327         on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1);
1328 }
1329
1330 static void octeon_irq_percpu_enable(void)
1331 {
1332         irq_cpu_online();
1333 }
1334
1335 static void octeon_irq_init_ciu_percpu(void)
1336 {
1337         int coreid = cvmx_get_core_num();
1338
1339
1340         __this_cpu_write(octeon_irq_ciu0_en_mirror, 0);
1341         __this_cpu_write(octeon_irq_ciu1_en_mirror, 0);
1342         wmb();
1343         raw_spin_lock_init(this_cpu_ptr(&octeon_irq_ciu_spinlock));
1344         /*
1345          * Disable All CIU Interrupts. The ones we need will be
1346          * enabled later.  Read the SUM register so we know the write
1347          * completed.
1348          */
1349         cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0);
1350         cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0);
1351         cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0);
1352         cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0);
1353         cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2)));
1354 }
1355
1356 static void octeon_irq_init_ciu2_percpu(void)
1357 {
1358         u64 regx, ipx;
1359         int coreid = cvmx_get_core_num();
1360         u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid);
1361
1362         /*
1363          * Disable All CIU2 Interrupts. The ones we need will be
1364          * enabled later.  Read the SUM register so we know the write
1365          * completed.
1366          *
1367          * There are 9 registers and 3 IPX levels with strides 0x1000
1368          * and 0x200 respectivly.  Use loops to clear them.
1369          */
1370         for (regx = 0; regx <= 0x8000; regx += 0x1000) {
1371                 for (ipx = 0; ipx <= 0x400; ipx += 0x200)
1372                         cvmx_write_csr(base + regx + ipx, 0);
1373         }
1374
1375         cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid));
1376 }
1377
1378 static void octeon_irq_setup_secondary_ciu(void)
1379 {
1380         octeon_irq_init_ciu_percpu();
1381         octeon_irq_percpu_enable();
1382
1383         /* Enable the CIU lines */
1384         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1385         if (octeon_irq_use_ip4)
1386                 set_c0_status(STATUSF_IP4);
1387         else
1388                 clear_c0_status(STATUSF_IP4);
1389 }
1390
1391 static void octeon_irq_setup_secondary_ciu2(void)
1392 {
1393         octeon_irq_init_ciu2_percpu();
1394         octeon_irq_percpu_enable();
1395
1396         /* Enable the CIU lines */
1397         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1398         if (octeon_irq_use_ip4)
1399                 set_c0_status(STATUSF_IP4);
1400         else
1401                 clear_c0_status(STATUSF_IP4);
1402 }
1403
1404 static int __init octeon_irq_init_ciu(
1405         struct device_node *ciu_node, struct device_node *parent)
1406 {
1407         unsigned int i, r;
1408         struct irq_chip *chip;
1409         struct irq_chip *chip_edge;
1410         struct irq_chip *chip_mbox;
1411         struct irq_chip *chip_wd;
1412         struct irq_domain *ciu_domain = NULL;
1413         struct octeon_irq_ciu_domain_data *dd;
1414
1415         dd = kzalloc(sizeof(*dd), GFP_KERNEL);
1416         if (!dd)
1417                 return -ENOMEM;
1418
1419         octeon_irq_init_ciu_percpu();
1420         octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu;
1421
1422         octeon_irq_ip2 = octeon_irq_ip2_ciu;
1423         octeon_irq_ip3 = octeon_irq_ip3_ciu;
1424         if ((OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3())
1425                 && !OCTEON_IS_MODEL(OCTEON_CN63XX)) {
1426                 octeon_irq_ip4 =  octeon_irq_ip4_ciu;
1427                 dd->num_sum = 3;
1428                 octeon_irq_use_ip4 = true;
1429         } else {
1430                 octeon_irq_ip4 = octeon_irq_ip4_mask;
1431                 dd->num_sum = 2;
1432                 octeon_irq_use_ip4 = false;
1433         }
1434         if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) ||
1435             OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) ||
1436             OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) ||
1437             OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) {
1438                 chip = &octeon_irq_chip_ciu_v2;
1439                 chip_edge = &octeon_irq_chip_ciu_v2_edge;
1440                 chip_mbox = &octeon_irq_chip_ciu_mbox_v2;
1441                 chip_wd = &octeon_irq_chip_ciu_wd_v2;
1442                 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2;
1443         } else {
1444                 chip = &octeon_irq_chip_ciu;
1445                 chip_edge = &octeon_irq_chip_ciu_edge;
1446                 chip_mbox = &octeon_irq_chip_ciu_mbox;
1447                 chip_wd = &octeon_irq_chip_ciu_wd;
1448                 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio;
1449         }
1450         octeon_irq_ciu_chip = chip;
1451         octeon_irq_ciu_chip_edge = chip_edge;
1452
1453         /* Mips internal */
1454         octeon_irq_init_core();
1455
1456         ciu_domain = irq_domain_add_tree(
1457                 ciu_node, &octeon_irq_domain_ciu_ops, dd);
1458         irq_set_default_host(ciu_domain);
1459
1460         /* CIU_0 */
1461         for (i = 0; i < 16; i++) {
1462                 r = octeon_irq_force_ciu_mapping(
1463                         ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0);
1464                 if (r)
1465                         goto err;
1466         }
1467
1468         r = octeon_irq_set_ciu_mapping(
1469                 OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq);
1470         if (r)
1471                 goto err;
1472         r = octeon_irq_set_ciu_mapping(
1473                 OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq);
1474         if (r)
1475                 goto err;
1476
1477         for (i = 0; i < 4; i++) {
1478                 r = octeon_irq_force_ciu_mapping(
1479                         ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36);
1480                 if (r)
1481                         goto err;
1482         }
1483         for (i = 0; i < 4; i++) {
1484                 r = octeon_irq_force_ciu_mapping(
1485                         ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40);
1486                 if (r)
1487                         goto err;
1488         }
1489
1490         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI, 0, 45);
1491         if (r)
1492                 goto err;
1493
1494         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46);
1495         if (r)
1496                 goto err;
1497
1498         for (i = 0; i < 4; i++) {
1499                 r = octeon_irq_force_ciu_mapping(
1500                         ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52);
1501                 if (r)
1502                         goto err;
1503         }
1504
1505         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 0, 56);
1506         if (r)
1507                 goto err;
1508
1509         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI2, 0, 59);
1510         if (r)
1511                 goto err;
1512
1513         /* CIU_1 */
1514         for (i = 0; i < 16; i++) {
1515                 r = octeon_irq_set_ciu_mapping(
1516                         i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd,
1517                         handle_level_irq);
1518                 if (r)
1519                         goto err;
1520         }
1521
1522         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB1, 1, 17);
1523         if (r)
1524                 goto err;
1525
1526         /* Enable the CIU lines */
1527         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1528         if (octeon_irq_use_ip4)
1529                 set_c0_status(STATUSF_IP4);
1530         else
1531                 clear_c0_status(STATUSF_IP4);
1532
1533         return 0;
1534 err:
1535         return r;
1536 }
1537
1538 static int __init octeon_irq_init_gpio(
1539         struct device_node *gpio_node, struct device_node *parent)
1540 {
1541         struct octeon_irq_gpio_domain_data *gpiod;
1542         u32 interrupt_cells;
1543         unsigned int base_hwirq;
1544         int r;
1545
1546         r = of_property_read_u32(parent, "#interrupt-cells", &interrupt_cells);
1547         if (r)
1548                 return r;
1549
1550         if (interrupt_cells == 1) {
1551                 u32 v;
1552
1553                 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v);
1554                 if (r) {
1555                         pr_warn("No \"interrupts\" property.\n");
1556                         return r;
1557                 }
1558                 base_hwirq = v;
1559         } else if (interrupt_cells == 2) {
1560                 u32 v0, v1;
1561
1562                 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v0);
1563                 if (r) {
1564                         pr_warn("No \"interrupts\" property.\n");
1565                         return r;
1566                 }
1567                 r = of_property_read_u32_index(gpio_node, "interrupts", 1, &v1);
1568                 if (r) {
1569                         pr_warn("No \"interrupts\" property.\n");
1570                         return r;
1571                 }
1572                 base_hwirq = (v0 << 6) | v1;
1573         } else {
1574                 pr_warn("Bad \"#interrupt-cells\" property: %u\n",
1575                         interrupt_cells);
1576                 return -EINVAL;
1577         }
1578
1579         gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL);
1580         if (gpiod) {
1581                 /* gpio domain host_data is the base hwirq number. */
1582                 gpiod->base_hwirq = base_hwirq;
1583                 irq_domain_add_linear(
1584                         gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod);
1585         } else {
1586                 pr_warn("Cannot allocate memory for GPIO irq_domain.\n");
1587                 return -ENOMEM;
1588         }
1589
1590         return 0;
1591 }
1592 /*
1593  * Watchdog interrupts are special.  They are associated with a single
1594  * core, so we hardwire the affinity to that core.
1595  */
1596 static void octeon_irq_ciu2_wd_enable(struct irq_data *data)
1597 {
1598         u64 mask;
1599         u64 en_addr;
1600         int coreid = data->irq - OCTEON_IRQ_WDOG0;
1601         struct octeon_ciu_chip_data *cd;
1602
1603         cd = irq_data_get_irq_chip_data(data);
1604         mask = 1ull << (cd->bit);
1605
1606         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) +
1607                 (0x1000ull * cd->line);
1608         cvmx_write_csr(en_addr, mask);
1609
1610 }
1611
1612 static void octeon_irq_ciu2_enable(struct irq_data *data)
1613 {
1614         u64 mask;
1615         u64 en_addr;
1616         int cpu = next_cpu_for_irq(data);
1617         int coreid = octeon_coreid_for_cpu(cpu);
1618         struct octeon_ciu_chip_data *cd;
1619
1620         cd = irq_data_get_irq_chip_data(data);
1621         mask = 1ull << (cd->bit);
1622
1623         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) +
1624                 (0x1000ull * cd->line);
1625         cvmx_write_csr(en_addr, mask);
1626 }
1627
1628 static void octeon_irq_ciu2_enable_local(struct irq_data *data)
1629 {
1630         u64 mask;
1631         u64 en_addr;
1632         int coreid = cvmx_get_core_num();
1633         struct octeon_ciu_chip_data *cd;
1634
1635         cd = irq_data_get_irq_chip_data(data);
1636         mask = 1ull << (cd->bit);
1637
1638         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) +
1639                 (0x1000ull * cd->line);
1640         cvmx_write_csr(en_addr, mask);
1641
1642 }
1643
1644 static void octeon_irq_ciu2_disable_local(struct irq_data *data)
1645 {
1646         u64 mask;
1647         u64 en_addr;
1648         int coreid = cvmx_get_core_num();
1649         struct octeon_ciu_chip_data *cd;
1650
1651         cd = irq_data_get_irq_chip_data(data);
1652         mask = 1ull << (cd->bit);
1653
1654         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) +
1655                 (0x1000ull * cd->line);
1656         cvmx_write_csr(en_addr, mask);
1657
1658 }
1659
1660 static void octeon_irq_ciu2_ack(struct irq_data *data)
1661 {
1662         u64 mask;
1663         u64 en_addr;
1664         int coreid = cvmx_get_core_num();
1665         struct octeon_ciu_chip_data *cd;
1666
1667         cd = irq_data_get_irq_chip_data(data);
1668         mask = 1ull << (cd->bit);
1669
1670         en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd->line);
1671         cvmx_write_csr(en_addr, mask);
1672
1673 }
1674
1675 static void octeon_irq_ciu2_disable_all(struct irq_data *data)
1676 {
1677         int cpu;
1678         u64 mask;
1679         struct octeon_ciu_chip_data *cd;
1680
1681         cd = irq_data_get_irq_chip_data(data);
1682         mask = 1ull << (cd->bit);
1683
1684         for_each_online_cpu(cpu) {
1685                 u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(
1686                         octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd->line);
1687                 cvmx_write_csr(en_addr, mask);
1688         }
1689 }
1690
1691 static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data)
1692 {
1693         int cpu;
1694         u64 mask;
1695
1696         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1697
1698         for_each_online_cpu(cpu) {
1699                 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(
1700                         octeon_coreid_for_cpu(cpu));
1701                 cvmx_write_csr(en_addr, mask);
1702         }
1703 }
1704
1705 static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data)
1706 {
1707         int cpu;
1708         u64 mask;
1709
1710         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1711
1712         for_each_online_cpu(cpu) {
1713                 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(
1714                         octeon_coreid_for_cpu(cpu));
1715                 cvmx_write_csr(en_addr, mask);
1716         }
1717 }
1718
1719 static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data)
1720 {
1721         u64 mask;
1722         u64 en_addr;
1723         int coreid = cvmx_get_core_num();
1724
1725         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1726         en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid);
1727         cvmx_write_csr(en_addr, mask);
1728 }
1729
1730 static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data)
1731 {
1732         u64 mask;
1733         u64 en_addr;
1734         int coreid = cvmx_get_core_num();
1735
1736         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1737         en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid);
1738         cvmx_write_csr(en_addr, mask);
1739 }
1740
1741 #ifdef CONFIG_SMP
1742 static int octeon_irq_ciu2_set_affinity(struct irq_data *data,
1743                                         const struct cpumask *dest, bool force)
1744 {
1745         int cpu;
1746         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
1747         u64 mask;
1748         struct octeon_ciu_chip_data *cd;
1749
1750         if (!enable_one)
1751                 return 0;
1752
1753         cd = irq_data_get_irq_chip_data(data);
1754         mask = 1ull << cd->bit;
1755
1756         for_each_online_cpu(cpu) {
1757                 u64 en_addr;
1758                 if (cpumask_test_cpu(cpu, dest) && enable_one) {
1759                         enable_one = false;
1760                         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(
1761                                 octeon_coreid_for_cpu(cpu)) +
1762                                 (0x1000ull * cd->line);
1763                 } else {
1764                         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(
1765                                 octeon_coreid_for_cpu(cpu)) +
1766                                 (0x1000ull * cd->line);
1767                 }
1768                 cvmx_write_csr(en_addr, mask);
1769         }
1770
1771         return 0;
1772 }
1773 #endif
1774
1775 static void octeon_irq_ciu2_enable_gpio(struct irq_data *data)
1776 {
1777         octeon_irq_gpio_setup(data);
1778         octeon_irq_ciu2_enable(data);
1779 }
1780
1781 static void octeon_irq_ciu2_disable_gpio(struct irq_data *data)
1782 {
1783         struct octeon_ciu_chip_data *cd;
1784
1785         cd = irq_data_get_irq_chip_data(data);
1786
1787         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0);
1788
1789         octeon_irq_ciu2_disable_all(data);
1790 }
1791
1792 static struct irq_chip octeon_irq_chip_ciu2 = {
1793         .name = "CIU2-E",
1794         .irq_enable = octeon_irq_ciu2_enable,
1795         .irq_disable = octeon_irq_ciu2_disable_all,
1796         .irq_mask = octeon_irq_ciu2_disable_local,
1797         .irq_unmask = octeon_irq_ciu2_enable,
1798 #ifdef CONFIG_SMP
1799         .irq_set_affinity = octeon_irq_ciu2_set_affinity,
1800         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1801 #endif
1802 };
1803
1804 static struct irq_chip octeon_irq_chip_ciu2_edge = {
1805         .name = "CIU2-E",
1806         .irq_enable = octeon_irq_ciu2_enable,
1807         .irq_disable = octeon_irq_ciu2_disable_all,
1808         .irq_ack = octeon_irq_ciu2_ack,
1809         .irq_mask = octeon_irq_ciu2_disable_local,
1810         .irq_unmask = octeon_irq_ciu2_enable,
1811 #ifdef CONFIG_SMP
1812         .irq_set_affinity = octeon_irq_ciu2_set_affinity,
1813         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1814 #endif
1815 };
1816
1817 static struct irq_chip octeon_irq_chip_ciu2_mbox = {
1818         .name = "CIU2-M",
1819         .irq_enable = octeon_irq_ciu2_mbox_enable_all,
1820         .irq_disable = octeon_irq_ciu2_mbox_disable_all,
1821         .irq_ack = octeon_irq_ciu2_mbox_disable_local,
1822         .irq_eoi = octeon_irq_ciu2_mbox_enable_local,
1823
1824         .irq_cpu_online = octeon_irq_ciu2_mbox_enable_local,
1825         .irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local,
1826         .flags = IRQCHIP_ONOFFLINE_ENABLED,
1827 };
1828
1829 static struct irq_chip octeon_irq_chip_ciu2_wd = {
1830         .name = "CIU2-W",
1831         .irq_enable = octeon_irq_ciu2_wd_enable,
1832         .irq_disable = octeon_irq_ciu2_disable_all,
1833         .irq_mask = octeon_irq_ciu2_disable_local,
1834         .irq_unmask = octeon_irq_ciu2_enable_local,
1835 };
1836
1837 static struct irq_chip octeon_irq_chip_ciu2_gpio = {
1838         .name = "CIU-GPIO",
1839         .irq_enable = octeon_irq_ciu2_enable_gpio,
1840         .irq_disable = octeon_irq_ciu2_disable_gpio,
1841         .irq_ack = octeon_irq_ciu_gpio_ack,
1842         .irq_mask = octeon_irq_ciu2_disable_local,
1843         .irq_unmask = octeon_irq_ciu2_enable,
1844         .irq_set_type = octeon_irq_ciu_gpio_set_type,
1845 #ifdef CONFIG_SMP
1846         .irq_set_affinity = octeon_irq_ciu2_set_affinity,
1847         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1848 #endif
1849         .flags = IRQCHIP_SET_TYPE_MASKED,
1850 };
1851
1852 static int octeon_irq_ciu2_xlat(struct irq_domain *d,
1853                                 struct device_node *node,
1854                                 const u32 *intspec,
1855                                 unsigned int intsize,
1856                                 unsigned long *out_hwirq,
1857                                 unsigned int *out_type)
1858 {
1859         unsigned int ciu, bit;
1860
1861         ciu = intspec[0];
1862         bit = intspec[1];
1863
1864         *out_hwirq = (ciu << 6) | bit;
1865         *out_type = 0;
1866
1867         return 0;
1868 }
1869
1870 static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit)
1871 {
1872         bool edge = false;
1873
1874         if (line == 3) /* MIO */
1875                 switch (bit) {
1876                 case 2:  /* IPD_DRP */
1877                 case 8 ... 11: /* Timers */
1878                 case 48: /* PTP */
1879                         edge = true;
1880                         break;
1881                 default:
1882                         break;
1883                 }
1884         else if (line == 6) /* PKT */
1885                 switch (bit) {
1886                 case 52 ... 53: /* ILK_DRP */
1887                 case 8 ... 12:  /* GMX_DRP */
1888                         edge = true;
1889                         break;
1890                 default:
1891                         break;
1892                 }
1893         return edge;
1894 }
1895
1896 static int octeon_irq_ciu2_map(struct irq_domain *d,
1897                                unsigned int virq, irq_hw_number_t hw)
1898 {
1899         unsigned int line = hw >> 6;
1900         unsigned int bit = hw & 63;
1901
1902         if (!octeon_irq_virq_in_range(virq))
1903                 return -EINVAL;
1904
1905         /*
1906          * Don't map irq if it is reserved for GPIO.
1907          * (Line 7 are the GPIO lines.)
1908          */
1909         if (line == 7)
1910                 return 0;
1911
1912         if (line > 7 || octeon_irq_ciu_to_irq[line][bit] != 0)
1913                 return -EINVAL;
1914
1915         if (octeon_irq_ciu2_is_edge(line, bit))
1916                 octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1917                                            &octeon_irq_chip_ciu2_edge,
1918                                            handle_edge_irq);
1919         else
1920                 octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1921                                            &octeon_irq_chip_ciu2,
1922                                            handle_level_irq);
1923
1924         return 0;
1925 }
1926
1927 static struct irq_domain_ops octeon_irq_domain_ciu2_ops = {
1928         .map = octeon_irq_ciu2_map,
1929         .unmap = octeon_irq_free_cd,
1930         .xlate = octeon_irq_ciu2_xlat,
1931 };
1932
1933 static void octeon_irq_ciu2(void)
1934 {
1935         int line;
1936         int bit;
1937         int irq;
1938         u64 src_reg, src, sum;
1939         const unsigned long core_id = cvmx_get_core_num();
1940
1941         sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful;
1942
1943         if (unlikely(!sum))
1944                 goto spurious;
1945
1946         line = fls64(sum) - 1;
1947         src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line);
1948         src = cvmx_read_csr(src_reg);
1949
1950         if (unlikely(!src))
1951                 goto spurious;
1952
1953         bit = fls64(src) - 1;
1954         irq = octeon_irq_ciu_to_irq[line][bit];
1955         if (unlikely(!irq))
1956                 goto spurious;
1957
1958         do_IRQ(irq);
1959         goto out;
1960
1961 spurious:
1962         spurious_interrupt();
1963 out:
1964         /* CN68XX pass 1.x has an errata that accessing the ACK registers
1965                 can stop interrupts from propagating */
1966         if (OCTEON_IS_MODEL(OCTEON_CN68XX))
1967                 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);
1968         else
1969                 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id));
1970         return;
1971 }
1972
1973 static void octeon_irq_ciu2_mbox(void)
1974 {
1975         int line;
1976
1977         const unsigned long core_id = cvmx_get_core_num();
1978         u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60;
1979
1980         if (unlikely(!sum))
1981                 goto spurious;
1982
1983         line = fls64(sum) - 1;
1984
1985         do_IRQ(OCTEON_IRQ_MBOX0 + line);
1986         goto out;
1987
1988 spurious:
1989         spurious_interrupt();
1990 out:
1991         /* CN68XX pass 1.x has an errata that accessing the ACK registers
1992                 can stop interrupts from propagating */
1993         if (OCTEON_IS_MODEL(OCTEON_CN68XX))
1994                 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);
1995         else
1996                 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id));
1997         return;
1998 }
1999
2000 static int __init octeon_irq_init_ciu2(
2001         struct device_node *ciu_node, struct device_node *parent)
2002 {
2003         unsigned int i, r;
2004         struct irq_domain *ciu_domain = NULL;
2005
2006         octeon_irq_init_ciu2_percpu();
2007         octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2;
2008
2009         octeon_irq_gpio_chip = &octeon_irq_chip_ciu2_gpio;
2010         octeon_irq_ip2 = octeon_irq_ciu2;
2011         octeon_irq_ip3 = octeon_irq_ciu2_mbox;
2012         octeon_irq_ip4 = octeon_irq_ip4_mask;
2013
2014         /* Mips internal */
2015         octeon_irq_init_core();
2016
2017         ciu_domain = irq_domain_add_tree(
2018                 ciu_node, &octeon_irq_domain_ciu2_ops, NULL);
2019         irq_set_default_host(ciu_domain);
2020
2021         /* CUI2 */
2022         for (i = 0; i < 64; i++) {
2023                 r = octeon_irq_force_ciu_mapping(
2024                         ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i);
2025                 if (r)
2026                         goto err;
2027         }
2028
2029         for (i = 0; i < 32; i++) {
2030                 r = octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0,
2031                         &octeon_irq_chip_ciu2_wd, handle_level_irq);
2032                 if (r)
2033                         goto err;
2034         }
2035
2036         for (i = 0; i < 4; i++) {
2037                 r = octeon_irq_force_ciu_mapping(
2038                         ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8);
2039                 if (r)
2040                         goto err;
2041         }
2042
2043         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 3, 44);
2044         if (r)
2045                 goto err;
2046
2047         for (i = 0; i < 4; i++) {
2048                 r = octeon_irq_force_ciu_mapping(
2049                         ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i);
2050                 if (r)
2051                         goto err;
2052         }
2053
2054         for (i = 0; i < 4; i++) {
2055                 r = octeon_irq_force_ciu_mapping(
2056                         ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8);
2057                 if (r)
2058                         goto err;
2059         }
2060
2061         irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2062         irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2063         irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2064         irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2065
2066         /* Enable the CIU lines */
2067         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
2068         clear_c0_status(STATUSF_IP4);
2069         return 0;
2070 err:
2071         return r;
2072 }
2073
2074 struct octeon_irq_cib_host_data {
2075         raw_spinlock_t lock;
2076         u64 raw_reg;
2077         u64 en_reg;
2078         int max_bits;
2079 };
2080
2081 struct octeon_irq_cib_chip_data {
2082         struct octeon_irq_cib_host_data *host_data;
2083         int bit;
2084 };
2085
2086 static void octeon_irq_cib_enable(struct irq_data *data)
2087 {
2088         unsigned long flags;
2089         u64 en;
2090         struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data);
2091         struct octeon_irq_cib_host_data *host_data = cd->host_data;
2092
2093         raw_spin_lock_irqsave(&host_data->lock, flags);
2094         en = cvmx_read_csr(host_data->en_reg);
2095         en |= 1ull << cd->bit;
2096         cvmx_write_csr(host_data->en_reg, en);
2097         raw_spin_unlock_irqrestore(&host_data->lock, flags);
2098 }
2099
2100 static void octeon_irq_cib_disable(struct irq_data *data)
2101 {
2102         unsigned long flags;
2103         u64 en;
2104         struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data);
2105         struct octeon_irq_cib_host_data *host_data = cd->host_data;
2106
2107         raw_spin_lock_irqsave(&host_data->lock, flags);
2108         en = cvmx_read_csr(host_data->en_reg);
2109         en &= ~(1ull << cd->bit);
2110         cvmx_write_csr(host_data->en_reg, en);
2111         raw_spin_unlock_irqrestore(&host_data->lock, flags);
2112 }
2113
2114 static int octeon_irq_cib_set_type(struct irq_data *data, unsigned int t)
2115 {
2116         irqd_set_trigger_type(data, t);
2117         return IRQ_SET_MASK_OK;
2118 }
2119
2120 static struct irq_chip octeon_irq_chip_cib = {
2121         .name = "CIB",
2122         .irq_enable = octeon_irq_cib_enable,
2123         .irq_disable = octeon_irq_cib_disable,
2124         .irq_mask = octeon_irq_cib_disable,
2125         .irq_unmask = octeon_irq_cib_enable,
2126         .irq_set_type = octeon_irq_cib_set_type,
2127 };
2128
2129 static int octeon_irq_cib_xlat(struct irq_domain *d,
2130                                    struct device_node *node,
2131                                    const u32 *intspec,
2132                                    unsigned int intsize,
2133                                    unsigned long *out_hwirq,
2134                                    unsigned int *out_type)
2135 {
2136         unsigned int type = 0;
2137
2138         if (intsize == 2)
2139                 type = intspec[1];
2140
2141         switch (type) {
2142         case 0: /* unofficial value, but we might as well let it work. */
2143         case 4: /* official value for level triggering. */
2144                 *out_type = IRQ_TYPE_LEVEL_HIGH;
2145                 break;
2146         case 1: /* official value for edge triggering. */
2147                 *out_type = IRQ_TYPE_EDGE_RISING;
2148                 break;
2149         default: /* Nothing else is acceptable. */
2150                 return -EINVAL;
2151         }
2152
2153         *out_hwirq = intspec[0];
2154
2155         return 0;
2156 }
2157
2158 static int octeon_irq_cib_map(struct irq_domain *d,
2159                               unsigned int virq, irq_hw_number_t hw)
2160 {
2161         struct octeon_irq_cib_host_data *host_data = d->host_data;
2162         struct octeon_irq_cib_chip_data *cd;
2163
2164         if (hw >= host_data->max_bits) {
2165                 pr_err("ERROR: %s mapping %u is to big!\n",
2166                        irq_domain_get_of_node(d)->name, (unsigned)hw);
2167                 return -EINVAL;
2168         }
2169
2170         cd = kzalloc(sizeof(*cd), GFP_KERNEL);
2171         cd->host_data = host_data;
2172         cd->bit = hw;
2173
2174         irq_set_chip_and_handler(virq, &octeon_irq_chip_cib,
2175                                  handle_simple_irq);
2176         irq_set_chip_data(virq, cd);
2177         return 0;
2178 }
2179
2180 static struct irq_domain_ops octeon_irq_domain_cib_ops = {
2181         .map = octeon_irq_cib_map,
2182         .unmap = octeon_irq_free_cd,
2183         .xlate = octeon_irq_cib_xlat,
2184 };
2185
2186 /* Chain to real handler. */
2187 static irqreturn_t octeon_irq_cib_handler(int my_irq, void *data)
2188 {
2189         u64 en;
2190         u64 raw;
2191         u64 bits;
2192         int i;
2193         int irq;
2194         struct irq_domain *cib_domain = data;
2195         struct octeon_irq_cib_host_data *host_data = cib_domain->host_data;
2196
2197         en = cvmx_read_csr(host_data->en_reg);
2198         raw = cvmx_read_csr(host_data->raw_reg);
2199
2200         bits = en & raw;
2201
2202         for (i = 0; i < host_data->max_bits; i++) {
2203                 if ((bits & 1ull << i) == 0)
2204                         continue;
2205                 irq = irq_find_mapping(cib_domain, i);
2206                 if (!irq) {
2207                         unsigned long flags;
2208
2209                         pr_err("ERROR: CIB bit %d@%llx IRQ unhandled, disabling\n",
2210                                 i, host_data->raw_reg);
2211                         raw_spin_lock_irqsave(&host_data->lock, flags);
2212                         en = cvmx_read_csr(host_data->en_reg);
2213                         en &= ~(1ull << i);
2214                         cvmx_write_csr(host_data->en_reg, en);
2215                         cvmx_write_csr(host_data->raw_reg, 1ull << i);
2216                         raw_spin_unlock_irqrestore(&host_data->lock, flags);
2217                 } else {
2218                         struct irq_desc *desc = irq_to_desc(irq);
2219                         struct irq_data *irq_data = irq_desc_get_irq_data(desc);
2220                         /* If edge, acknowledge the bit we will be sending. */
2221                         if (irqd_get_trigger_type(irq_data) &
2222                                 IRQ_TYPE_EDGE_BOTH)
2223                                 cvmx_write_csr(host_data->raw_reg, 1ull << i);
2224                         generic_handle_irq_desc(desc);
2225                 }
2226         }
2227
2228         return IRQ_HANDLED;
2229 }
2230
2231 static int __init octeon_irq_init_cib(struct device_node *ciu_node,
2232                                       struct device_node *parent)
2233 {
2234         const __be32 *addr;
2235         u32 val;
2236         struct octeon_irq_cib_host_data *host_data;
2237         int parent_irq;
2238         int r;
2239         struct irq_domain *cib_domain;
2240
2241         parent_irq = irq_of_parse_and_map(ciu_node, 0);
2242         if (!parent_irq) {
2243                 pr_err("ERROR: Couldn't acquire parent_irq for %s\n.",
2244                         ciu_node->name);
2245                 return -EINVAL;
2246         }
2247
2248         host_data = kzalloc(sizeof(*host_data), GFP_KERNEL);
2249         raw_spin_lock_init(&host_data->lock);
2250
2251         addr = of_get_address(ciu_node, 0, NULL, NULL);
2252         if (!addr) {
2253                 pr_err("ERROR: Couldn't acquire reg(0) %s\n.", ciu_node->name);
2254                 return -EINVAL;
2255         }
2256         host_data->raw_reg = (u64)phys_to_virt(
2257                 of_translate_address(ciu_node, addr));
2258
2259         addr = of_get_address(ciu_node, 1, NULL, NULL);
2260         if (!addr) {
2261                 pr_err("ERROR: Couldn't acquire reg(1) %s\n.", ciu_node->name);
2262                 return -EINVAL;
2263         }
2264         host_data->en_reg = (u64)phys_to_virt(
2265                 of_translate_address(ciu_node, addr));
2266
2267         r = of_property_read_u32(ciu_node, "cavium,max-bits", &val);
2268         if (r) {
2269                 pr_err("ERROR: Couldn't read cavium,max-bits from %s\n.",
2270                         ciu_node->name);
2271                 return r;
2272         }
2273         host_data->max_bits = val;
2274
2275         cib_domain = irq_domain_add_linear(ciu_node, host_data->max_bits,
2276                                            &octeon_irq_domain_cib_ops,
2277                                            host_data);
2278         if (!cib_domain) {
2279                 pr_err("ERROR: Couldn't irq_domain_add_linear()\n.");
2280                 return -ENOMEM;
2281         }
2282
2283         cvmx_write_csr(host_data->en_reg, 0); /* disable all IRQs */
2284         cvmx_write_csr(host_data->raw_reg, ~0); /* ack any outstanding */
2285
2286         r = request_irq(parent_irq, octeon_irq_cib_handler,
2287                         IRQF_NO_THREAD, "cib", cib_domain);
2288         if (r) {
2289                 pr_err("request_irq cib failed %d\n", r);
2290                 return r;
2291         }
2292         pr_info("CIB interrupt controller probed: %llx %d\n",
2293                 host_data->raw_reg, host_data->max_bits);
2294         return 0;
2295 }
2296
2297 static struct of_device_id ciu_types[] __initdata = {
2298         {.compatible = "cavium,octeon-3860-ciu", .data = octeon_irq_init_ciu},
2299         {.compatible = "cavium,octeon-3860-gpio", .data = octeon_irq_init_gpio},
2300         {.compatible = "cavium,octeon-6880-ciu2", .data = octeon_irq_init_ciu2},
2301         {.compatible = "cavium,octeon-7130-cib", .data = octeon_irq_init_cib},
2302         {}
2303 };
2304
2305 void __init arch_init_irq(void)
2306 {
2307 #ifdef CONFIG_SMP
2308         /* Set the default affinity to the boot cpu. */
2309         cpumask_clear(irq_default_affinity);
2310         cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
2311 #endif
2312         of_irq_init(ciu_types);
2313 }
2314
2315 asmlinkage void plat_irq_dispatch(void)
2316 {
2317         unsigned long cop0_cause;
2318         unsigned long cop0_status;
2319
2320         while (1) {
2321                 cop0_cause = read_c0_cause();
2322                 cop0_status = read_c0_status();
2323                 cop0_cause &= cop0_status;
2324                 cop0_cause &= ST0_IM;
2325
2326                 if (cop0_cause & STATUSF_IP2)
2327                         octeon_irq_ip2();
2328                 else if (cop0_cause & STATUSF_IP3)
2329                         octeon_irq_ip3();
2330                 else if (cop0_cause & STATUSF_IP4)
2331                         octeon_irq_ip4();
2332                 else if (cop0_cause)
2333                         do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE);
2334                 else
2335                         break;
2336         }
2337 }
2338
2339 #ifdef CONFIG_HOTPLUG_CPU
2340
2341 void octeon_fixup_irqs(void)
2342 {
2343         irq_cpu_offline();
2344 }
2345
2346 #endif /* CONFIG_HOTPLUG_CPU */