2 * Copyright (c) 2003-2012 Broadcom Corporation
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the Broadcom
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in
19 * the documentation and/or other materials provided with the
22 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
24 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
29 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
30 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
31 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
32 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 #include <linux/types.h>
36 #include <linux/pci.h>
37 #include <linux/kernel.h>
38 #include <linux/init.h>
39 #include <linux/msi.h>
41 #include <linux/irq.h>
42 #include <linux/irqdesc.h>
43 #include <linux/console.h>
47 #include <asm/netlogic/interrupt.h>
48 #include <asm/netlogic/haldefs.h>
49 #include <asm/netlogic/common.h>
50 #include <asm/netlogic/mips-extns.h>
52 #include <asm/netlogic/xlp-hal/iomap.h>
53 #include <asm/netlogic/xlp-hal/xlp.h>
54 #include <asm/netlogic/xlp-hal/pic.h>
55 #include <asm/netlogic/xlp-hal/pcibus.h>
56 #include <asm/netlogic/xlp-hal/bridge.h>
58 #define XLP_MSIVEC_PER_LINK 32
59 #define XLP_MSIXVEC_TOTAL (cpu_is_xlp9xx() ? 128 : 32)
60 #define XLP_MSIXVEC_PER_LINK (cpu_is_xlp9xx() ? 32 : 8)
62 /* 128 MSI irqs per node, mapped starting at NLM_MSI_VEC_BASE */
63 static inline int nlm_link_msiirq(int link, int msivec)
65 return NLM_MSI_VEC_BASE + link * XLP_MSIVEC_PER_LINK + msivec;
68 /* get the link MSI vector from irq number */
69 static inline int nlm_irq_msivec(int irq)
71 return (irq - NLM_MSI_VEC_BASE) % XLP_MSIVEC_PER_LINK;
74 /* get the link from the irq number */
75 static inline int nlm_irq_msilink(int irq)
77 int total_msivec = XLP_MSIVEC_PER_LINK * PCIE_NLINKS;
79 return ((irq - NLM_MSI_VEC_BASE) % total_msivec) /
84 * For XLP 8xx/4xx/3xx/2xx, only 32 MSI-X vectors are possible because
85 * there are only 32 PIC interrupts for MSI. We split them statically
86 * and use 8 MSI-X vectors per link - this keeps the allocation and
88 * On XLP 9xx, there are 32 vectors per link, and the interrupts are
89 * not routed thru PIC, so we can use all 128 MSI-X vectors.
91 static inline int nlm_link_msixirq(int link, int bit)
93 return NLM_MSIX_VEC_BASE + link * XLP_MSIXVEC_PER_LINK + bit;
96 /* get the link MSI vector from irq number */
97 static inline int nlm_irq_msixvec(int irq)
99 return (irq - NLM_MSIX_VEC_BASE) % XLP_MSIXVEC_TOTAL;
102 /* get the link from MSIX vec */
103 static inline int nlm_irq_msixlink(int msixvec)
105 return msixvec / XLP_MSIXVEC_PER_LINK;
109 * Per link MSI and MSI-X information, set as IRQ handler data for
110 * MSI and MSI-X interrupts.
112 struct xlp_msi_data {
113 struct nlm_soc_info *node;
115 uint32_t msi_enabled_mask;
116 uint32_t msi_alloc_mask;
117 uint32_t msix_alloc_mask;
122 * MSI Chip definitions
124 * On XLP, there is a PIC interrupt associated with each PCIe link on the
125 * chip (which appears as a PCI bridge to us). This gives us 32 MSI irqa
126 * per link and 128 overall.
128 * When a device connected to the link raises a MSI interrupt, we get a
129 * link interrupt and we then have to look at PCIE_MSI_STATUS register at
130 * the bridge to map it to the IRQ
132 static void xlp_msi_enable(struct irq_data *d)
134 struct xlp_msi_data *md = irq_data_get_irq_handler_data(d);
138 vec = nlm_irq_msivec(d->irq);
139 spin_lock_irqsave(&md->msi_lock, flags);
140 md->msi_enabled_mask |= 1u << vec;
142 nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN,
143 md->msi_enabled_mask);
145 nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
146 spin_unlock_irqrestore(&md->msi_lock, flags);
149 static void xlp_msi_disable(struct irq_data *d)
151 struct xlp_msi_data *md = irq_data_get_irq_handler_data(d);
155 vec = nlm_irq_msivec(d->irq);
156 spin_lock_irqsave(&md->msi_lock, flags);
157 md->msi_enabled_mask &= ~(1u << vec);
159 nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN,
160 md->msi_enabled_mask);
162 nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
163 spin_unlock_irqrestore(&md->msi_lock, flags);
166 static void xlp_msi_mask_ack(struct irq_data *d)
168 struct xlp_msi_data *md = irq_data_get_irq_handler_data(d);
171 link = nlm_irq_msilink(d->irq);
172 vec = nlm_irq_msivec(d->irq);
175 /* Ack MSI on bridge */
177 nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_STATUS, 1u << vec);
179 nlm_write_reg(md->lnkbase, PCIE_MSI_STATUS, 1u << vec);
181 /* Ack at eirr and PIC */
182 ack_c0_eirr(PIC_PCIE_LINK_MSI_IRQ(link));
184 nlm_pic_ack(md->node->picbase,
185 PIC_9XX_IRT_PCIE_LINK_INDEX(link));
187 nlm_pic_ack(md->node->picbase, PIC_IRT_PCIE_LINK_INDEX(link));
190 static struct irq_chip xlp_msi_chip = {
192 .irq_enable = xlp_msi_enable,
193 .irq_disable = xlp_msi_disable,
194 .irq_mask_ack = xlp_msi_mask_ack,
195 .irq_unmask = xlp_msi_enable,
199 * XLP8XX/4XX/3XX/2XX:
200 * The MSI-X interrupt handling is different from MSI, there are 32 MSI-X
201 * interrupts generated by the PIC and each of these correspond to a MSI-X
202 * vector (0-31) that can be assigned.
204 * We divide the MSI-X vectors to 8 per link and do a per-link allocation
207 * 32 MSI-X vectors are available per link, and the interrupts are not routed
208 * thru the PIC. PIC ack not needed.
210 * Enable and disable done using standard MSI functions.
212 static void xlp_msix_mask_ack(struct irq_data *d)
214 struct xlp_msi_data *md;
216 uint32_t status_reg, bit;
218 msixvec = nlm_irq_msixvec(d->irq);
219 link = nlm_irq_msixlink(msixvec);
221 md = irq_data_get_irq_handler_data(d);
223 /* Ack MSI on bridge */
224 if (cpu_is_xlp9xx()) {
225 status_reg = PCIE_9XX_MSIX_STATUSX(link);
226 bit = msixvec % XLP_MSIXVEC_PER_LINK;
228 status_reg = PCIE_MSIX_STATUS;
231 nlm_write_reg(md->lnkbase, status_reg, 1u << bit);
233 /* Ack at eirr and PIC */
234 ack_c0_eirr(PIC_PCIE_MSIX_IRQ(link));
235 if (!cpu_is_xlp9xx())
236 nlm_pic_ack(md->node->picbase,
237 PIC_IRT_PCIE_MSIX_INDEX(msixvec));
240 static struct irq_chip xlp_msix_chip = {
242 .irq_enable = unmask_msi_irq,
243 .irq_disable = mask_msi_irq,
244 .irq_mask_ack = xlp_msix_mask_ack,
245 .irq_unmask = unmask_msi_irq,
248 void arch_teardown_msi_irq(unsigned int irq)
253 * Setup a PCIe link for MSI. By default, the links are in
254 * legacy interrupt mode. We will switch them to MSI mode
255 * at the first MSI request.
257 static void xlp_config_link_msi(uint64_t lnkbase, int lirq, uint64_t msiaddr)
261 if (cpu_is_xlp9xx()) {
262 val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0);
263 if ((val & 0x200) == 0) {
264 val |= 0x200; /* MSI Interrupt enable */
265 nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val);
268 val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
269 if ((val & 0x200) == 0) {
271 nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
275 val = nlm_read_reg(lnkbase, 0x1); /* CMD */
276 if ((val & 0x0400) == 0) {
278 nlm_write_reg(lnkbase, 0x1, val);
281 /* Update IRQ in the PCI irq reg */
282 val = nlm_read_pci_reg(lnkbase, 0xf);
284 val |= (1 << 8) | lirq;
285 nlm_write_pci_reg(lnkbase, 0xf, val);
288 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRH, msiaddr >> 32);
289 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRL, msiaddr & 0xffffffff);
291 /* MSI cap for bridge */
292 val = nlm_read_reg(lnkbase, PCIE_BRIDGE_MSI_CAP);
293 if ((val & (1 << 16)) == 0) {
294 val |= 0xb << 16; /* mmc32, msi enable */
295 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_CAP, val);
300 * Allocate a MSI vector on a link
302 static int xlp_setup_msi(uint64_t lnkbase, int node, int link,
303 struct msi_desc *desc)
305 struct xlp_msi_data *md;
308 int msivec, irt, lirq, xirq, ret;
311 /* Get MSI data for the link */
312 lirq = PIC_PCIE_LINK_MSI_IRQ(link);
313 xirq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
314 md = irq_get_handler_data(xirq);
315 msiaddr = MSI_LINK_ADDR(node, link);
317 spin_lock_irqsave(&md->msi_lock, flags);
318 if (md->msi_alloc_mask == 0) {
319 xlp_config_link_msi(lnkbase, lirq, msiaddr);
320 /* switch the link IRQ to MSI range */
322 irt = PIC_9XX_IRT_PCIE_LINK_INDEX(link);
324 irt = PIC_IRT_PCIE_LINK_INDEX(link);
325 nlm_setup_pic_irq(node, lirq, lirq, irt);
326 nlm_pic_init_irt(nlm_get_node(node)->picbase, irt, lirq,
327 node * nlm_threads_per_node(), 1 /*en */);
330 /* allocate a MSI vec, and tell the bridge about it */
331 msivec = fls(md->msi_alloc_mask);
332 if (msivec == XLP_MSIVEC_PER_LINK) {
333 spin_unlock_irqrestore(&md->msi_lock, flags);
336 md->msi_alloc_mask |= (1u << msivec);
337 spin_unlock_irqrestore(&md->msi_lock, flags);
339 msg.address_hi = msiaddr >> 32;
340 msg.address_lo = msiaddr & 0xffffffff;
341 msg.data = 0xc00 | msivec;
343 xirq = xirq + msivec; /* msi mapped to global irq space */
344 ret = irq_set_msi_desc(xirq, desc);
348 write_msi_msg(xirq, &msg);
353 * Switch a link to MSI-X mode
355 static void xlp_config_link_msix(uint64_t lnkbase, int lirq, uint64_t msixaddr)
359 val = nlm_read_reg(lnkbase, 0x2C);
360 if ((val & 0x80000000U) == 0) {
362 nlm_write_reg(lnkbase, 0x2C, val);
365 if (cpu_is_xlp9xx()) {
366 val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0);
367 if ((val & 0x200) == 0) {
368 val |= 0x200; /* MSI Interrupt enable */
369 nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val);
372 val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
373 if ((val & 0x200) == 0) {
374 val |= 0x200; /* MSI Interrupt enable */
375 nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
379 val = nlm_read_reg(lnkbase, 0x1); /* CMD */
380 if ((val & 0x0400) == 0) {
382 nlm_write_reg(lnkbase, 0x1, val);
385 /* Update IRQ in the PCI irq reg */
386 val = nlm_read_pci_reg(lnkbase, 0xf);
388 val |= (1 << 8) | lirq;
389 nlm_write_pci_reg(lnkbase, 0xf, val);
391 if (cpu_is_xlp9xx()) {
392 /* MSI-X addresses */
393 nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_BASE,
395 nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_LIMIT,
396 (msixaddr + MSI_ADDR_SZ) >> 8);
398 /* MSI-X addresses */
399 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_BASE,
401 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_LIMIT,
402 (msixaddr + MSI_ADDR_SZ) >> 8);
407 * Allocate a MSI-X vector
409 static int xlp_setup_msix(uint64_t lnkbase, int node, int link,
410 struct msi_desc *desc)
412 struct xlp_msi_data *md;
415 int t, msixvec, lirq, xirq, ret;
418 /* Get MSI data for the link */
419 lirq = PIC_PCIE_MSIX_IRQ(link);
420 xirq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
421 md = irq_get_handler_data(xirq);
422 msixaddr = MSIX_LINK_ADDR(node, link);
424 spin_lock_irqsave(&md->msi_lock, flags);
425 /* switch the PCIe link to MSI-X mode at the first alloc */
426 if (md->msix_alloc_mask == 0)
427 xlp_config_link_msix(lnkbase, lirq, msixaddr);
429 /* allocate a MSI-X vec, and tell the bridge about it */
430 t = fls(md->msix_alloc_mask);
431 if (t == XLP_MSIXVEC_PER_LINK) {
432 spin_unlock_irqrestore(&md->msi_lock, flags);
435 md->msix_alloc_mask |= (1u << t);
436 spin_unlock_irqrestore(&md->msi_lock, flags);
439 msixvec = nlm_irq_msixvec(xirq);
441 msg.address_hi = msixaddr >> 32;
442 msg.address_lo = msixaddr & 0xffffffff;
443 msg.data = 0xc00 | msixvec;
445 ret = irq_set_msi_desc(xirq, desc);
451 write_msi_msg(xirq, &msg);
455 int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
457 struct pci_dev *lnkdev;
459 int node, link, slot;
461 lnkdev = xlp_get_pcie_link(dev);
462 if (lnkdev == NULL) {
463 dev_err(&dev->dev, "Could not find bridge\n");
466 slot = PCI_SLOT(lnkdev->devfn);
467 link = PCI_FUNC(lnkdev->devfn);
469 lnkbase = nlm_get_pcie_base(node, link);
471 if (desc->msi_attrib.is_msix)
472 return xlp_setup_msix(lnkbase, node, link, desc);
474 return xlp_setup_msi(lnkbase, node, link, desc);
477 void __init xlp_init_node_msi_irqs(int node, int link)
479 struct nlm_soc_info *nodep;
480 struct xlp_msi_data *md;
481 int irq, i, irt, msixvec, val;
483 pr_info("[%d %d] Init node PCI IRT\n", node, link);
484 nodep = nlm_get_node(node);
486 /* Alloc an MSI block for the link */
487 md = kzalloc(sizeof(*md), GFP_KERNEL);
488 spin_lock_init(&md->msi_lock);
489 md->msi_enabled_mask = 0;
490 md->msi_alloc_mask = 0;
491 md->msix_alloc_mask = 0;
493 md->lnkbase = nlm_get_pcie_base(node, link);
495 /* extended space for MSI interrupts */
496 irq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
497 for (i = irq; i < irq + XLP_MSIVEC_PER_LINK; i++) {
498 irq_set_chip_and_handler(i, &xlp_msi_chip, handle_level_irq);
499 irq_set_handler_data(i, md);
502 for (i = 0; i < XLP_MSIXVEC_PER_LINK ; i++) {
503 if (cpu_is_xlp9xx()) {
504 val = ((node * nlm_threads_per_node()) << 7 |
505 PIC_PCIE_MSIX_IRQ(link) << 1 | 0 << 0);
506 nlm_write_pcie_reg(md->lnkbase, PCIE_9XX_MSIX_VECX(i +
507 (link * XLP_MSIXVEC_PER_LINK)), val);
509 /* Initialize MSI-X irts to generate one interrupt
512 msixvec = link * XLP_MSIXVEC_PER_LINK + i;
513 irt = PIC_IRT_PCIE_MSIX_INDEX(msixvec);
514 nlm_pic_init_irt(nodep->picbase, irt,
515 PIC_PCIE_MSIX_IRQ(link),
516 node * nlm_threads_per_node(), 1);
519 /* Initialize MSI-X extended irq space for the link */
520 irq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, i));
521 irq_set_chip_and_handler(irq, &xlp_msix_chip, handle_level_irq);
522 irq_set_handler_data(irq, md);
526 void nlm_dispatch_msi(int node, int lirq)
528 struct xlp_msi_data *md;
529 int link, i, irqbase;
532 link = lirq - PIC_PCIE_LINK_MSI_IRQ_BASE;
533 irqbase = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
534 md = irq_get_handler_data(irqbase);
536 status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSI_STATUS) &
537 md->msi_enabled_mask;
539 status = nlm_read_reg(md->lnkbase, PCIE_MSI_STATUS) &
540 md->msi_enabled_mask;
544 status &= status - 1;
548 void nlm_dispatch_msix(int node, int lirq)
550 struct xlp_msi_data *md;
551 int link, i, irqbase;
554 link = lirq - PIC_PCIE_MSIX_IRQ_BASE;
555 irqbase = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
556 md = irq_get_handler_data(irqbase);
558 status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSIX_STATUSX(link));
560 status = nlm_read_reg(md->lnkbase, PCIE_MSIX_STATUS);
562 /* narrow it down to the MSI-x vectors for our link */
563 if (!cpu_is_xlp9xx())
564 status = (status >> (link * XLP_MSIXVEC_PER_LINK)) &
565 ((1 << XLP_MSIXVEC_PER_LINK) - 1);
570 status &= status - 1;