2 * PCIe host controller driver for Tegra SoCs
4 * Copyright (c) 2010, CompuLab, Ltd.
5 * Author: Mike Rapoport <mike@compulab.co.il>
7 * Based on NVIDIA PCIe driver
8 * Copyright (c) 2008-2009, NVIDIA Corporation.
10 * Bits taken from arch/arm/mach-dove/pcie.c
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
22 * You should have received a copy of the GNU General Public License along
23 * with this program; if not, write to the Free Software Foundation, Inc.,
24 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
27 #include <linux/clk.h>
28 #include <linux/debugfs.h>
29 #include <linux/delay.h>
30 #include <linux/export.h>
31 #include <linux/interrupt.h>
32 #include <linux/irq.h>
33 #include <linux/irqdomain.h>
34 #include <linux/kernel.h>
35 #include <linux/module.h>
36 #include <linux/msi.h>
37 #include <linux/of_address.h>
38 #include <linux/of_pci.h>
39 #include <linux/of_platform.h>
40 #include <linux/pci.h>
41 #include <linux/phy/phy.h>
42 #include <linux/platform_device.h>
43 #include <linux/reset.h>
44 #include <linux/sizes.h>
45 #include <linux/slab.h>
46 #include <linux/vmalloc.h>
47 #include <linux/regulator/consumer.h>
49 #include <soc/tegra/cpuidle.h>
50 #include <soc/tegra/pmc.h>
52 #include <asm/mach/irq.h>
53 #include <asm/mach/map.h>
54 #include <asm/mach/pci.h>
56 #define INT_PCI_MSI_NR (8 * 32)
58 /* register definitions */
60 #define AFI_AXI_BAR0_SZ 0x00
61 #define AFI_AXI_BAR1_SZ 0x04
62 #define AFI_AXI_BAR2_SZ 0x08
63 #define AFI_AXI_BAR3_SZ 0x0c
64 #define AFI_AXI_BAR4_SZ 0x10
65 #define AFI_AXI_BAR5_SZ 0x14
67 #define AFI_AXI_BAR0_START 0x18
68 #define AFI_AXI_BAR1_START 0x1c
69 #define AFI_AXI_BAR2_START 0x20
70 #define AFI_AXI_BAR3_START 0x24
71 #define AFI_AXI_BAR4_START 0x28
72 #define AFI_AXI_BAR5_START 0x2c
74 #define AFI_FPCI_BAR0 0x30
75 #define AFI_FPCI_BAR1 0x34
76 #define AFI_FPCI_BAR2 0x38
77 #define AFI_FPCI_BAR3 0x3c
78 #define AFI_FPCI_BAR4 0x40
79 #define AFI_FPCI_BAR5 0x44
81 #define AFI_CACHE_BAR0_SZ 0x48
82 #define AFI_CACHE_BAR0_ST 0x4c
83 #define AFI_CACHE_BAR1_SZ 0x50
84 #define AFI_CACHE_BAR1_ST 0x54
86 #define AFI_MSI_BAR_SZ 0x60
87 #define AFI_MSI_FPCI_BAR_ST 0x64
88 #define AFI_MSI_AXI_BAR_ST 0x68
90 #define AFI_MSI_VEC0 0x6c
91 #define AFI_MSI_VEC1 0x70
92 #define AFI_MSI_VEC2 0x74
93 #define AFI_MSI_VEC3 0x78
94 #define AFI_MSI_VEC4 0x7c
95 #define AFI_MSI_VEC5 0x80
96 #define AFI_MSI_VEC6 0x84
97 #define AFI_MSI_VEC7 0x88
99 #define AFI_MSI_EN_VEC0 0x8c
100 #define AFI_MSI_EN_VEC1 0x90
101 #define AFI_MSI_EN_VEC2 0x94
102 #define AFI_MSI_EN_VEC3 0x98
103 #define AFI_MSI_EN_VEC4 0x9c
104 #define AFI_MSI_EN_VEC5 0xa0
105 #define AFI_MSI_EN_VEC6 0xa4
106 #define AFI_MSI_EN_VEC7 0xa8
108 #define AFI_CONFIGURATION 0xac
109 #define AFI_CONFIGURATION_EN_FPCI (1 << 0)
111 #define AFI_FPCI_ERROR_MASKS 0xb0
113 #define AFI_INTR_MASK 0xb4
114 #define AFI_INTR_MASK_INT_MASK (1 << 0)
115 #define AFI_INTR_MASK_MSI_MASK (1 << 8)
117 #define AFI_INTR_CODE 0xb8
118 #define AFI_INTR_CODE_MASK 0xf
119 #define AFI_INTR_INI_SLAVE_ERROR 1
120 #define AFI_INTR_INI_DECODE_ERROR 2
121 #define AFI_INTR_TARGET_ABORT 3
122 #define AFI_INTR_MASTER_ABORT 4
123 #define AFI_INTR_INVALID_WRITE 5
124 #define AFI_INTR_LEGACY 6
125 #define AFI_INTR_FPCI_DECODE_ERROR 7
126 #define AFI_INTR_AXI_DECODE_ERROR 8
127 #define AFI_INTR_FPCI_TIMEOUT 9
128 #define AFI_INTR_PE_PRSNT_SENSE 10
129 #define AFI_INTR_PE_CLKREQ_SENSE 11
130 #define AFI_INTR_CLKCLAMP_SENSE 12
131 #define AFI_INTR_RDY4PD_SENSE 13
132 #define AFI_INTR_P2P_ERROR 14
134 #define AFI_INTR_SIGNATURE 0xbc
135 #define AFI_UPPER_FPCI_ADDRESS 0xc0
136 #define AFI_SM_INTR_ENABLE 0xc4
137 #define AFI_SM_INTR_INTA_ASSERT (1 << 0)
138 #define AFI_SM_INTR_INTB_ASSERT (1 << 1)
139 #define AFI_SM_INTR_INTC_ASSERT (1 << 2)
140 #define AFI_SM_INTR_INTD_ASSERT (1 << 3)
141 #define AFI_SM_INTR_INTA_DEASSERT (1 << 4)
142 #define AFI_SM_INTR_INTB_DEASSERT (1 << 5)
143 #define AFI_SM_INTR_INTC_DEASSERT (1 << 6)
144 #define AFI_SM_INTR_INTD_DEASSERT (1 << 7)
146 #define AFI_AFI_INTR_ENABLE 0xc8
147 #define AFI_INTR_EN_INI_SLVERR (1 << 0)
148 #define AFI_INTR_EN_INI_DECERR (1 << 1)
149 #define AFI_INTR_EN_TGT_SLVERR (1 << 2)
150 #define AFI_INTR_EN_TGT_DECERR (1 << 3)
151 #define AFI_INTR_EN_TGT_WRERR (1 << 4)
152 #define AFI_INTR_EN_DFPCI_DECERR (1 << 5)
153 #define AFI_INTR_EN_AXI_DECERR (1 << 6)
154 #define AFI_INTR_EN_FPCI_TIMEOUT (1 << 7)
155 #define AFI_INTR_EN_PRSNT_SENSE (1 << 8)
157 #define AFI_PCIE_CONFIG 0x0f8
158 #define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1))
159 #define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0xe
160 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20)
161 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE (0x0 << 20)
162 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420 (0x0 << 20)
163 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1 (0x0 << 20)
164 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL (0x1 << 20)
165 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222 (0x1 << 20)
166 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1 (0x1 << 20)
167 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20)
169 #define AFI_FUSE 0x104
170 #define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2)
172 #define AFI_PEX0_CTRL 0x110
173 #define AFI_PEX1_CTRL 0x118
174 #define AFI_PEX2_CTRL 0x128
175 #define AFI_PEX_CTRL_RST (1 << 0)
176 #define AFI_PEX_CTRL_CLKREQ_EN (1 << 1)
177 #define AFI_PEX_CTRL_REFCLK_EN (1 << 3)
178 #define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4)
180 #define AFI_PLLE_CONTROL 0x160
181 #define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9)
182 #define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1)
184 #define AFI_PEXBIAS_CTRL_0 0x168
186 #define RP_VEND_XP 0x00000F00
187 #define RP_VEND_XP_DL_UP (1 << 30)
189 #define RP_PRIV_MISC 0x00000FE0
190 #define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xE << 0)
191 #define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xF << 0)
193 #define RP_LINK_CONTROL_STATUS 0x00000090
194 #define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000
195 #define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000
197 #define PADS_CTL_SEL 0x0000009C
199 #define PADS_CTL 0x000000A0
200 #define PADS_CTL_IDDQ_1L (1 << 0)
201 #define PADS_CTL_TX_DATA_EN_1L (1 << 6)
202 #define PADS_CTL_RX_DATA_EN_1L (1 << 10)
204 #define PADS_PLL_CTL_TEGRA20 0x000000B8
205 #define PADS_PLL_CTL_TEGRA30 0x000000B4
206 #define PADS_PLL_CTL_RST_B4SM (1 << 1)
207 #define PADS_PLL_CTL_LOCKDET (1 << 8)
208 #define PADS_PLL_CTL_REFCLK_MASK (0x3 << 16)
209 #define PADS_PLL_CTL_REFCLK_INTERNAL_CML (0 << 16)
210 #define PADS_PLL_CTL_REFCLK_INTERNAL_CMOS (1 << 16)
211 #define PADS_PLL_CTL_REFCLK_EXTERNAL (2 << 16)
212 #define PADS_PLL_CTL_TXCLKREF_MASK (0x1 << 20)
213 #define PADS_PLL_CTL_TXCLKREF_DIV10 (0 << 20)
214 #define PADS_PLL_CTL_TXCLKREF_DIV5 (1 << 20)
215 #define PADS_PLL_CTL_TXCLKREF_BUF_EN (1 << 22)
217 #define PADS_REFCLK_CFG0 0x000000C8
218 #define PADS_REFCLK_CFG1 0x000000CC
219 #define PADS_REFCLK_BIAS 0x000000D0
222 * Fields in PADS_REFCLK_CFG*. Those registers form an array of 16-bit
223 * entries, one entry per PCIe port. These field definitions and desired
224 * values aren't in the TRM, but do come from NVIDIA.
226 #define PADS_REFCLK_CFG_TERM_SHIFT 2 /* 6:2 */
227 #define PADS_REFCLK_CFG_E_TERM_SHIFT 7
228 #define PADS_REFCLK_CFG_PREDI_SHIFT 8 /* 11:8 */
229 #define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */
231 /* Default value provided by HW engineering is 0xfa5c */
232 #define PADS_REFCLK_CFG_VALUE \
234 (0x17 << PADS_REFCLK_CFG_TERM_SHIFT) | \
235 (0 << PADS_REFCLK_CFG_E_TERM_SHIFT) | \
236 (0xa << PADS_REFCLK_CFG_PREDI_SHIFT) | \
237 (0xf << PADS_REFCLK_CFG_DRVI_SHIFT) \
241 struct msi_controller chip;
242 DECLARE_BITMAP(used, INT_PCI_MSI_NR);
243 struct irq_domain *domain;
249 /* used to differentiate between Tegra SoC generations */
250 struct tegra_pcie_soc_data {
251 unsigned int num_ports;
252 unsigned int msi_base_shift;
255 bool has_pex_clkreq_en;
256 bool has_pex_bias_ctrl;
257 bool has_intr_prsnt_sense;
262 static inline struct tegra_msi *to_tegra_msi(struct msi_controller *chip)
264 return container_of(chip, struct tegra_msi, chip);
274 struct list_head buses;
280 struct resource prefetch;
281 struct resource busn;
293 struct reset_control *pex_rst;
294 struct reset_control *afi_rst;
295 struct reset_control *pcie_xrst;
300 struct tegra_msi msi;
302 struct list_head ports;
305 struct regulator_bulk_data *supplies;
306 unsigned int num_supplies;
308 const struct tegra_pcie_soc_data *soc_data;
309 struct dentry *debugfs;
312 struct tegra_pcie_port {
313 struct tegra_pcie *pcie;
314 struct device_node *np;
315 struct list_head list;
316 struct resource regs;
324 struct tegra_pcie_bus {
325 struct vm_struct *area;
326 struct list_head list;
330 static inline struct tegra_pcie *sys_to_pcie(struct pci_sys_data *sys)
332 return sys->private_data;
335 static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
336 unsigned long offset)
338 writel(value, pcie->afi + offset);
341 static inline u32 afi_readl(struct tegra_pcie *pcie, unsigned long offset)
343 return readl(pcie->afi + offset);
346 static inline void pads_writel(struct tegra_pcie *pcie, u32 value,
347 unsigned long offset)
349 writel(value, pcie->pads + offset);
352 static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
354 return readl(pcie->pads + offset);
358 * The configuration space mapping on Tegra is somewhat similar to the ECAM
359 * defined by PCIe. However it deviates a bit in how the 4 bits for extended
360 * register accesses are mapped:
362 * [27:24] extended register number
364 * [15:11] device number
365 * [10: 8] function number
366 * [ 7: 0] register number
368 * Mapping the whole extended configuration space would require 256 MiB of
369 * virtual address space, only a small part of which will actually be used.
370 * To work around this, a 1 MiB of virtual addresses are allocated per bus
371 * when the bus is first accessed. When the physical range is mapped, the
372 * the bus number bits are hidden so that the extended register number bits
373 * appear as bits [19:16]. Therefore the virtual mapping looks like this:
375 * [19:16] extended register number
376 * [15:11] device number
377 * [10: 8] function number
378 * [ 7: 0] register number
380 * This is achieved by stitching together 16 chunks of 64 KiB of physical
381 * address space via the MMU.
383 static unsigned long tegra_pcie_conf_offset(unsigned int devfn, int where)
385 return ((where & 0xf00) << 8) | (PCI_SLOT(devfn) << 11) |
386 (PCI_FUNC(devfn) << 8) | (where & 0xfc);
389 static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
392 pgprot_t prot = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
393 L_PTE_XN | L_PTE_MT_DEV_SHARED | L_PTE_SHARED);
394 phys_addr_t cs = pcie->cs->start;
395 struct tegra_pcie_bus *bus;
399 bus = kzalloc(sizeof(*bus), GFP_KERNEL);
401 return ERR_PTR(-ENOMEM);
403 INIT_LIST_HEAD(&bus->list);
406 /* allocate 1 MiB of virtual addresses */
407 bus->area = get_vm_area(SZ_1M, VM_IOREMAP);
413 /* map each of the 16 chunks of 64 KiB each */
414 for (i = 0; i < 16; i++) {
415 unsigned long virt = (unsigned long)bus->area->addr +
417 phys_addr_t phys = cs + i * SZ_16M + busnr * SZ_64K;
419 err = ioremap_page_range(virt, virt + SZ_64K, phys, prot);
421 dev_err(pcie->dev, "ioremap_page_range() failed: %d\n",
430 vunmap(bus->area->addr);
436 static int tegra_pcie_add_bus(struct pci_bus *bus)
438 struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
439 struct tegra_pcie_bus *b;
441 b = tegra_pcie_bus_alloc(pcie, bus->number);
445 list_add_tail(&b->list, &pcie->buses);
450 static void tegra_pcie_remove_bus(struct pci_bus *child)
452 struct tegra_pcie *pcie = sys_to_pcie(child->sysdata);
453 struct tegra_pcie_bus *bus, *tmp;
455 list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
456 if (bus->nr == child->number) {
457 vunmap(bus->area->addr);
458 list_del(&bus->list);
465 static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
469 struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
470 void __iomem *addr = NULL;
472 if (bus->number == 0) {
473 unsigned int slot = PCI_SLOT(devfn);
474 struct tegra_pcie_port *port;
476 list_for_each_entry(port, &pcie->ports, list) {
477 if (port->index + 1 == slot) {
478 addr = port->base + (where & ~3);
483 struct tegra_pcie_bus *b;
485 list_for_each_entry(b, &pcie->buses, list)
486 if (b->nr == bus->number)
487 addr = (void __iomem *)b->area->addr;
491 "failed to map cfg. space for bus %u\n",
496 addr += tegra_pcie_conf_offset(devfn, where);
502 static struct pci_ops tegra_pcie_ops = {
503 .add_bus = tegra_pcie_add_bus,
504 .remove_bus = tegra_pcie_remove_bus,
505 .map_bus = tegra_pcie_map_bus,
506 .read = pci_generic_config_read32,
507 .write = pci_generic_config_write32,
510 static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port)
512 unsigned long ret = 0;
514 switch (port->index) {
531 static void tegra_pcie_port_reset(struct tegra_pcie_port *port)
533 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
536 /* pulse reset signal */
537 value = afi_readl(port->pcie, ctrl);
538 value &= ~AFI_PEX_CTRL_RST;
539 afi_writel(port->pcie, value, ctrl);
541 usleep_range(1000, 2000);
543 value = afi_readl(port->pcie, ctrl);
544 value |= AFI_PEX_CTRL_RST;
545 afi_writel(port->pcie, value, ctrl);
548 static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
550 const struct tegra_pcie_soc_data *soc = port->pcie->soc_data;
551 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
554 /* enable reference clock */
555 value = afi_readl(port->pcie, ctrl);
556 value |= AFI_PEX_CTRL_REFCLK_EN;
558 if (soc->has_pex_clkreq_en)
559 value |= AFI_PEX_CTRL_CLKREQ_EN;
561 value |= AFI_PEX_CTRL_OVERRIDE_EN;
563 afi_writel(port->pcie, value, ctrl);
565 tegra_pcie_port_reset(port);
568 static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
570 const struct tegra_pcie_soc_data *soc = port->pcie->soc_data;
571 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
574 /* assert port reset */
575 value = afi_readl(port->pcie, ctrl);
576 value &= ~AFI_PEX_CTRL_RST;
577 afi_writel(port->pcie, value, ctrl);
579 /* disable reference clock */
580 value = afi_readl(port->pcie, ctrl);
582 if (soc->has_pex_clkreq_en)
583 value &= ~AFI_PEX_CTRL_CLKREQ_EN;
585 value &= ~AFI_PEX_CTRL_REFCLK_EN;
586 afi_writel(port->pcie, value, ctrl);
589 static void tegra_pcie_port_free(struct tegra_pcie_port *port)
591 struct tegra_pcie *pcie = port->pcie;
593 devm_iounmap(pcie->dev, port->base);
594 devm_release_mem_region(pcie->dev, port->regs.start,
595 resource_size(&port->regs));
596 list_del(&port->list);
597 devm_kfree(pcie->dev, port);
600 /* Tegra PCIE root complex wrongly reports device class */
601 static void tegra_pcie_fixup_class(struct pci_dev *dev)
603 dev->class = PCI_CLASS_BRIDGE_PCI << 8;
605 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_fixup_class);
606 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class);
607 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class);
608 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class);
610 /* Tegra PCIE requires relaxed ordering */
611 static void tegra_pcie_relax_enable(struct pci_dev *dev)
613 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
615 DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, tegra_pcie_relax_enable);
617 static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)
619 struct tegra_pcie *pcie = sys_to_pcie(sys);
622 sys->mem_offset = pcie->offset.mem;
623 sys->io_offset = pcie->offset.io;
625 err = devm_request_resource(pcie->dev, &iomem_resource, &pcie->io);
629 pci_ioremap_io(pcie->pio.start, pcie->io.start);
631 pci_add_resource_offset(&sys->resources, &pcie->pio, sys->io_offset);
632 pci_add_resource_offset(&sys->resources, &pcie->mem, sys->mem_offset);
633 pci_add_resource_offset(&sys->resources, &pcie->prefetch,
635 pci_add_resource(&sys->resources, &pcie->busn);
637 err = devm_request_pci_bus_resources(pcie->dev, &sys->resources);
644 static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
646 struct tegra_pcie *pcie = sys_to_pcie(pdev->bus->sysdata);
649 tegra_cpuidle_pcie_irqs_in_use();
651 irq = of_irq_parse_and_map_pci(pdev, slot, pin);
658 static irqreturn_t tegra_pcie_isr(int irq, void *arg)
660 const char *err_msg[] = {
668 "Response decoding error",
669 "AXI response decoding error",
670 "Transaction timeout",
671 "Slot present pin change",
672 "Slot clock request change",
673 "TMS clock ramp change",
674 "TMS ready for power down",
677 struct tegra_pcie *pcie = arg;
680 code = afi_readl(pcie, AFI_INTR_CODE) & AFI_INTR_CODE_MASK;
681 signature = afi_readl(pcie, AFI_INTR_SIGNATURE);
682 afi_writel(pcie, 0, AFI_INTR_CODE);
684 if (code == AFI_INTR_LEGACY)
687 if (code >= ARRAY_SIZE(err_msg))
691 * do not pollute kernel log with master abort reports since they
692 * happen a lot during enumeration
694 if (code == AFI_INTR_MASTER_ABORT)
695 dev_dbg(pcie->dev, "%s, signature: %08x\n", err_msg[code],
698 dev_err(pcie->dev, "%s, signature: %08x\n", err_msg[code],
701 if (code == AFI_INTR_TARGET_ABORT || code == AFI_INTR_MASTER_ABORT ||
702 code == AFI_INTR_FPCI_DECODE_ERROR) {
703 u32 fpci = afi_readl(pcie, AFI_UPPER_FPCI_ADDRESS) & 0xff;
704 u64 address = (u64)fpci << 32 | (signature & 0xfffffffc);
706 if (code == AFI_INTR_MASTER_ABORT)
707 dev_dbg(pcie->dev, " FPCI address: %10llx\n", address);
709 dev_err(pcie->dev, " FPCI address: %10llx\n", address);
716 * FPCI map is as follows:
717 * - 0xfdfc000000: I/O space
718 * - 0xfdfe000000: type 0 configuration space
719 * - 0xfdff000000: type 1 configuration space
720 * - 0xfe00000000: type 0 extended configuration space
721 * - 0xfe10000000: type 1 extended configuration space
723 static void tegra_pcie_setup_translations(struct tegra_pcie *pcie)
725 u32 fpci_bar, size, axi_address;
727 /* Bar 0: type 1 extended configuration space */
728 fpci_bar = 0xfe100000;
729 size = resource_size(pcie->cs);
730 axi_address = pcie->cs->start;
731 afi_writel(pcie, axi_address, AFI_AXI_BAR0_START);
732 afi_writel(pcie, size >> 12, AFI_AXI_BAR0_SZ);
733 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR0);
735 /* Bar 1: downstream IO bar */
736 fpci_bar = 0xfdfc0000;
737 size = resource_size(&pcie->io);
738 axi_address = pcie->io.start;
739 afi_writel(pcie, axi_address, AFI_AXI_BAR1_START);
740 afi_writel(pcie, size >> 12, AFI_AXI_BAR1_SZ);
741 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR1);
743 /* Bar 2: prefetchable memory BAR */
744 fpci_bar = (((pcie->prefetch.start >> 12) & 0x0fffffff) << 4) | 0x1;
745 size = resource_size(&pcie->prefetch);
746 axi_address = pcie->prefetch.start;
747 afi_writel(pcie, axi_address, AFI_AXI_BAR2_START);
748 afi_writel(pcie, size >> 12, AFI_AXI_BAR2_SZ);
749 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR2);
751 /* Bar 3: non prefetchable memory BAR */
752 fpci_bar = (((pcie->mem.start >> 12) & 0x0fffffff) << 4) | 0x1;
753 size = resource_size(&pcie->mem);
754 axi_address = pcie->mem.start;
755 afi_writel(pcie, axi_address, AFI_AXI_BAR3_START);
756 afi_writel(pcie, size >> 12, AFI_AXI_BAR3_SZ);
757 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR3);
759 /* NULL out the remaining BARs as they are not used */
760 afi_writel(pcie, 0, AFI_AXI_BAR4_START);
761 afi_writel(pcie, 0, AFI_AXI_BAR4_SZ);
762 afi_writel(pcie, 0, AFI_FPCI_BAR4);
764 afi_writel(pcie, 0, AFI_AXI_BAR5_START);
765 afi_writel(pcie, 0, AFI_AXI_BAR5_SZ);
766 afi_writel(pcie, 0, AFI_FPCI_BAR5);
768 /* map all upstream transactions as uncached */
769 afi_writel(pcie, 0, AFI_CACHE_BAR0_ST);
770 afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ);
771 afi_writel(pcie, 0, AFI_CACHE_BAR1_ST);
772 afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ);
774 /* MSI translations are setup only when needed */
775 afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST);
776 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
777 afi_writel(pcie, 0, AFI_MSI_AXI_BAR_ST);
778 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
781 static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout)
783 const struct tegra_pcie_soc_data *soc = pcie->soc_data;
786 timeout = jiffies + msecs_to_jiffies(timeout);
788 while (time_before(jiffies, timeout)) {
789 value = pads_readl(pcie, soc->pads_pll_ctl);
790 if (value & PADS_PLL_CTL_LOCKDET)
797 static int tegra_pcie_phy_enable(struct tegra_pcie *pcie)
799 const struct tegra_pcie_soc_data *soc = pcie->soc_data;
803 /* initialize internal PHY, enable up to 16 PCIE lanes */
804 pads_writel(pcie, 0x0, PADS_CTL_SEL);
806 /* override IDDQ to 1 on all 4 lanes */
807 value = pads_readl(pcie, PADS_CTL);
808 value |= PADS_CTL_IDDQ_1L;
809 pads_writel(pcie, value, PADS_CTL);
812 * Set up PHY PLL inputs select PLLE output as refclock,
813 * set TX ref sel to div10 (not div5).
815 value = pads_readl(pcie, soc->pads_pll_ctl);
816 value &= ~(PADS_PLL_CTL_REFCLK_MASK | PADS_PLL_CTL_TXCLKREF_MASK);
817 value |= PADS_PLL_CTL_REFCLK_INTERNAL_CML | soc->tx_ref_sel;
818 pads_writel(pcie, value, soc->pads_pll_ctl);
821 value = pads_readl(pcie, soc->pads_pll_ctl);
822 value &= ~PADS_PLL_CTL_RST_B4SM;
823 pads_writel(pcie, value, soc->pads_pll_ctl);
825 usleep_range(20, 100);
827 /* take PLL out of reset */
828 value = pads_readl(pcie, soc->pads_pll_ctl);
829 value |= PADS_PLL_CTL_RST_B4SM;
830 pads_writel(pcie, value, soc->pads_pll_ctl);
832 /* Configure the reference clock driver */
833 value = PADS_REFCLK_CFG_VALUE | (PADS_REFCLK_CFG_VALUE << 16);
834 pads_writel(pcie, value, PADS_REFCLK_CFG0);
835 if (soc->num_ports > 2)
836 pads_writel(pcie, PADS_REFCLK_CFG_VALUE, PADS_REFCLK_CFG1);
838 /* wait for the PLL to lock */
839 err = tegra_pcie_pll_wait(pcie, 500);
841 dev_err(pcie->dev, "PLL failed to lock: %d\n", err);
845 /* turn off IDDQ override */
846 value = pads_readl(pcie, PADS_CTL);
847 value &= ~PADS_CTL_IDDQ_1L;
848 pads_writel(pcie, value, PADS_CTL);
850 /* enable TX/RX data */
851 value = pads_readl(pcie, PADS_CTL);
852 value |= PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L;
853 pads_writel(pcie, value, PADS_CTL);
858 static int tegra_pcie_phy_disable(struct tegra_pcie *pcie)
860 const struct tegra_pcie_soc_data *soc = pcie->soc_data;
863 /* disable TX/RX data */
864 value = pads_readl(pcie, PADS_CTL);
865 value &= ~(PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L);
866 pads_writel(pcie, value, PADS_CTL);
869 value = pads_readl(pcie, PADS_CTL);
870 value |= PADS_CTL_IDDQ_1L;
871 pads_writel(pcie, PADS_CTL, value);
874 value = pads_readl(pcie, soc->pads_pll_ctl);
875 value &= ~PADS_PLL_CTL_RST_B4SM;
876 pads_writel(pcie, value, soc->pads_pll_ctl);
878 usleep_range(20, 100);
883 static int tegra_pcie_port_phy_power_on(struct tegra_pcie_port *port)
885 struct device *dev = port->pcie->dev;
889 for (i = 0; i < port->lanes; i++) {
890 err = phy_power_on(port->phys[i]);
892 dev_err(dev, "failed to power on PHY#%u: %d\n", i,
901 static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port)
903 struct device *dev = port->pcie->dev;
907 for (i = 0; i < port->lanes; i++) {
908 err = phy_power_off(port->phys[i]);
910 dev_err(dev, "failed to power off PHY#%u: %d\n", i,
919 static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie)
921 struct tegra_pcie_port *port;
924 if (pcie->legacy_phy) {
926 err = phy_power_on(pcie->phy);
928 err = tegra_pcie_phy_enable(pcie);
931 dev_err(pcie->dev, "failed to power on PHY: %d\n", err);
936 list_for_each_entry(port, &pcie->ports, list) {
937 err = tegra_pcie_port_phy_power_on(port);
940 "failed to power on PCIe port %u PHY: %d\n",
949 static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie)
951 struct tegra_pcie_port *port;
954 if (pcie->legacy_phy) {
956 err = phy_power_off(pcie->phy);
958 err = tegra_pcie_phy_disable(pcie);
961 dev_err(pcie->dev, "failed to power off PHY: %d\n",
967 list_for_each_entry(port, &pcie->ports, list) {
968 err = tegra_pcie_port_phy_power_off(port);
971 "failed to power off PCIe port %u PHY: %d\n",
980 static int tegra_pcie_enable_controller(struct tegra_pcie *pcie)
982 const struct tegra_pcie_soc_data *soc = pcie->soc_data;
983 struct tegra_pcie_port *port;
987 /* enable PLL power down */
989 value = afi_readl(pcie, AFI_PLLE_CONTROL);
990 value &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL;
991 value |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN;
992 afi_writel(pcie, value, AFI_PLLE_CONTROL);
995 /* power down PCIe slot clock bias pad */
996 if (soc->has_pex_bias_ctrl)
997 afi_writel(pcie, 0, AFI_PEXBIAS_CTRL_0);
999 /* configure mode and disable all ports */
1000 value = afi_readl(pcie, AFI_PCIE_CONFIG);
1001 value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK;
1002 value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config;
1004 list_for_each_entry(port, &pcie->ports, list)
1005 value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
1007 afi_writel(pcie, value, AFI_PCIE_CONFIG);
1009 if (soc->has_gen2) {
1010 value = afi_readl(pcie, AFI_FUSE);
1011 value &= ~AFI_FUSE_PCIE_T0_GEN2_DIS;
1012 afi_writel(pcie, value, AFI_FUSE);
1014 value = afi_readl(pcie, AFI_FUSE);
1015 value |= AFI_FUSE_PCIE_T0_GEN2_DIS;
1016 afi_writel(pcie, value, AFI_FUSE);
1019 err = tegra_pcie_phy_power_on(pcie);
1021 dev_err(pcie->dev, "failed to power on PHY(s): %d\n", err);
1025 /* take the PCIe interface module out of reset */
1026 reset_control_deassert(pcie->pcie_xrst);
1028 /* finally enable PCIe */
1029 value = afi_readl(pcie, AFI_CONFIGURATION);
1030 value |= AFI_CONFIGURATION_EN_FPCI;
1031 afi_writel(pcie, value, AFI_CONFIGURATION);
1033 value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR |
1034 AFI_INTR_EN_TGT_SLVERR | AFI_INTR_EN_TGT_DECERR |
1035 AFI_INTR_EN_TGT_WRERR | AFI_INTR_EN_DFPCI_DECERR;
1037 if (soc->has_intr_prsnt_sense)
1038 value |= AFI_INTR_EN_PRSNT_SENSE;
1040 afi_writel(pcie, value, AFI_AFI_INTR_ENABLE);
1041 afi_writel(pcie, 0xffffffff, AFI_SM_INTR_ENABLE);
1043 /* don't enable MSI for now, only when needed */
1044 afi_writel(pcie, AFI_INTR_MASK_INT_MASK, AFI_INTR_MASK);
1046 /* disable all exceptions */
1047 afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS);
1052 static void tegra_pcie_power_off(struct tegra_pcie *pcie)
1056 /* TODO: disable and unprepare clocks? */
1058 err = tegra_pcie_phy_power_off(pcie);
1060 dev_err(pcie->dev, "failed to power off PHY(s): %d\n", err);
1062 reset_control_assert(pcie->pcie_xrst);
1063 reset_control_assert(pcie->afi_rst);
1064 reset_control_assert(pcie->pex_rst);
1066 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1068 err = regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1070 dev_warn(pcie->dev, "failed to disable regulators: %d\n", err);
1073 static int tegra_pcie_power_on(struct tegra_pcie *pcie)
1075 const struct tegra_pcie_soc_data *soc = pcie->soc_data;
1078 reset_control_assert(pcie->pcie_xrst);
1079 reset_control_assert(pcie->afi_rst);
1080 reset_control_assert(pcie->pex_rst);
1082 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1084 /* enable regulators */
1085 err = regulator_bulk_enable(pcie->num_supplies, pcie->supplies);
1087 dev_err(pcie->dev, "failed to enable regulators: %d\n", err);
1089 err = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_PCIE,
1093 dev_err(pcie->dev, "powerup sequence failed: %d\n", err);
1097 reset_control_deassert(pcie->afi_rst);
1099 err = clk_prepare_enable(pcie->afi_clk);
1101 dev_err(pcie->dev, "failed to enable AFI clock: %d\n", err);
1105 if (soc->has_cml_clk) {
1106 err = clk_prepare_enable(pcie->cml_clk);
1108 dev_err(pcie->dev, "failed to enable CML clock: %d\n",
1114 err = clk_prepare_enable(pcie->pll_e);
1116 dev_err(pcie->dev, "failed to enable PLLE clock: %d\n", err);
1123 static int tegra_pcie_clocks_get(struct tegra_pcie *pcie)
1125 const struct tegra_pcie_soc_data *soc = pcie->soc_data;
1127 pcie->pex_clk = devm_clk_get(pcie->dev, "pex");
1128 if (IS_ERR(pcie->pex_clk))
1129 return PTR_ERR(pcie->pex_clk);
1131 pcie->afi_clk = devm_clk_get(pcie->dev, "afi");
1132 if (IS_ERR(pcie->afi_clk))
1133 return PTR_ERR(pcie->afi_clk);
1135 pcie->pll_e = devm_clk_get(pcie->dev, "pll_e");
1136 if (IS_ERR(pcie->pll_e))
1137 return PTR_ERR(pcie->pll_e);
1139 if (soc->has_cml_clk) {
1140 pcie->cml_clk = devm_clk_get(pcie->dev, "cml");
1141 if (IS_ERR(pcie->cml_clk))
1142 return PTR_ERR(pcie->cml_clk);
1148 static int tegra_pcie_resets_get(struct tegra_pcie *pcie)
1150 pcie->pex_rst = devm_reset_control_get(pcie->dev, "pex");
1151 if (IS_ERR(pcie->pex_rst))
1152 return PTR_ERR(pcie->pex_rst);
1154 pcie->afi_rst = devm_reset_control_get(pcie->dev, "afi");
1155 if (IS_ERR(pcie->afi_rst))
1156 return PTR_ERR(pcie->afi_rst);
1158 pcie->pcie_xrst = devm_reset_control_get(pcie->dev, "pcie_x");
1159 if (IS_ERR(pcie->pcie_xrst))
1160 return PTR_ERR(pcie->pcie_xrst);
1165 static int tegra_pcie_phys_get_legacy(struct tegra_pcie *pcie)
1169 pcie->phy = devm_phy_optional_get(pcie->dev, "pcie");
1170 if (IS_ERR(pcie->phy)) {
1171 err = PTR_ERR(pcie->phy);
1172 dev_err(pcie->dev, "failed to get PHY: %d\n", err);
1176 err = phy_init(pcie->phy);
1178 dev_err(pcie->dev, "failed to initialize PHY: %d\n", err);
1182 pcie->legacy_phy = true;
1187 static struct phy *devm_of_phy_optional_get_index(struct device *dev,
1188 struct device_node *np,
1189 const char *consumer,
1195 name = kasprintf(GFP_KERNEL, "%s-%u", consumer, index);
1197 return ERR_PTR(-ENOMEM);
1199 phy = devm_of_phy_get(dev, np, name);
1202 if (IS_ERR(phy) && PTR_ERR(phy) == -ENODEV)
1208 static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port)
1210 struct device *dev = port->pcie->dev;
1215 port->phys = devm_kcalloc(dev, sizeof(phy), port->lanes, GFP_KERNEL);
1219 for (i = 0; i < port->lanes; i++) {
1220 phy = devm_of_phy_optional_get_index(dev, port->np, "pcie", i);
1222 dev_err(dev, "failed to get PHY#%u: %ld\n", i,
1224 return PTR_ERR(phy);
1227 err = phy_init(phy);
1229 dev_err(dev, "failed to initialize PHY#%u: %d\n", i,
1234 port->phys[i] = phy;
1240 static int tegra_pcie_phys_get(struct tegra_pcie *pcie)
1242 const struct tegra_pcie_soc_data *soc = pcie->soc_data;
1243 struct device_node *np = pcie->dev->of_node;
1244 struct tegra_pcie_port *port;
1247 if (!soc->has_gen2 || of_find_property(np, "phys", NULL) != NULL)
1248 return tegra_pcie_phys_get_legacy(pcie);
1250 list_for_each_entry(port, &pcie->ports, list) {
1251 err = tegra_pcie_port_get_phys(port);
1259 static int tegra_pcie_get_resources(struct tegra_pcie *pcie)
1261 struct platform_device *pdev = to_platform_device(pcie->dev);
1262 struct resource *pads, *afi, *res;
1265 err = tegra_pcie_clocks_get(pcie);
1267 dev_err(&pdev->dev, "failed to get clocks: %d\n", err);
1271 err = tegra_pcie_resets_get(pcie);
1273 dev_err(&pdev->dev, "failed to get resets: %d\n", err);
1277 err = tegra_pcie_phys_get(pcie);
1279 dev_err(&pdev->dev, "failed to get PHYs: %d\n", err);
1283 err = tegra_pcie_power_on(pcie);
1285 dev_err(&pdev->dev, "failed to power up: %d\n", err);
1289 pads = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pads");
1290 pcie->pads = devm_ioremap_resource(&pdev->dev, pads);
1291 if (IS_ERR(pcie->pads)) {
1292 err = PTR_ERR(pcie->pads);
1296 afi = platform_get_resource_byname(pdev, IORESOURCE_MEM, "afi");
1297 pcie->afi = devm_ioremap_resource(&pdev->dev, afi);
1298 if (IS_ERR(pcie->afi)) {
1299 err = PTR_ERR(pcie->afi);
1303 /* request configuration space, but remap later, on demand */
1304 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs");
1306 err = -EADDRNOTAVAIL;
1310 pcie->cs = devm_request_mem_region(pcie->dev, res->start,
1311 resource_size(res), res->name);
1313 err = -EADDRNOTAVAIL;
1317 /* request interrupt */
1318 err = platform_get_irq_byname(pdev, "intr");
1320 dev_err(&pdev->dev, "failed to get IRQ: %d\n", err);
1326 err = request_irq(pcie->irq, tegra_pcie_isr, IRQF_SHARED, "PCIE", pcie);
1328 dev_err(&pdev->dev, "failed to register IRQ: %d\n", err);
1335 tegra_pcie_power_off(pcie);
1339 static int tegra_pcie_put_resources(struct tegra_pcie *pcie)
1344 free_irq(pcie->irq, pcie);
1346 tegra_pcie_power_off(pcie);
1348 err = phy_exit(pcie->phy);
1350 dev_err(pcie->dev, "failed to teardown PHY: %d\n", err);
1355 static int tegra_msi_alloc(struct tegra_msi *chip)
1359 mutex_lock(&chip->lock);
1361 msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR);
1362 if (msi < INT_PCI_MSI_NR)
1363 set_bit(msi, chip->used);
1367 mutex_unlock(&chip->lock);
1372 static void tegra_msi_free(struct tegra_msi *chip, unsigned long irq)
1374 struct device *dev = chip->chip.dev;
1376 mutex_lock(&chip->lock);
1378 if (!test_bit(irq, chip->used))
1379 dev_err(dev, "trying to free unused MSI#%lu\n", irq);
1381 clear_bit(irq, chip->used);
1383 mutex_unlock(&chip->lock);
1386 static irqreturn_t tegra_pcie_msi_irq(int irq, void *data)
1388 struct tegra_pcie *pcie = data;
1389 struct tegra_msi *msi = &pcie->msi;
1390 unsigned int i, processed = 0;
1392 for (i = 0; i < 8; i++) {
1393 unsigned long reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4);
1396 unsigned int offset = find_first_bit(®, 32);
1397 unsigned int index = i * 32 + offset;
1400 /* clear the interrupt */
1401 afi_writel(pcie, 1 << offset, AFI_MSI_VEC0 + i * 4);
1403 irq = irq_find_mapping(msi->domain, index);
1405 if (test_bit(index, msi->used))
1406 generic_handle_irq(irq);
1408 dev_info(pcie->dev, "unhandled MSI\n");
1411 * that's weird who triggered this?
1414 dev_info(pcie->dev, "unexpected MSI\n");
1417 /* see if there's any more pending in this vector */
1418 reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4);
1424 return processed > 0 ? IRQ_HANDLED : IRQ_NONE;
1427 static int tegra_msi_setup_irq(struct msi_controller *chip,
1428 struct pci_dev *pdev, struct msi_desc *desc)
1430 struct tegra_msi *msi = to_tegra_msi(chip);
1435 hwirq = tegra_msi_alloc(msi);
1439 irq = irq_create_mapping(msi->domain, hwirq);
1441 tegra_msi_free(msi, hwirq);
1445 irq_set_msi_desc(irq, desc);
1447 msg.address_lo = virt_to_phys((void *)msi->pages);
1448 /* 32 bit address only */
1452 pci_write_msi_msg(irq, &msg);
1457 static void tegra_msi_teardown_irq(struct msi_controller *chip,
1460 struct tegra_msi *msi = to_tegra_msi(chip);
1461 struct irq_data *d = irq_get_irq_data(irq);
1462 irq_hw_number_t hwirq = irqd_to_hwirq(d);
1464 irq_dispose_mapping(irq);
1465 tegra_msi_free(msi, hwirq);
1468 static struct irq_chip tegra_msi_irq_chip = {
1469 .name = "Tegra PCIe MSI",
1470 .irq_enable = pci_msi_unmask_irq,
1471 .irq_disable = pci_msi_mask_irq,
1472 .irq_mask = pci_msi_mask_irq,
1473 .irq_unmask = pci_msi_unmask_irq,
1476 static int tegra_msi_map(struct irq_domain *domain, unsigned int irq,
1477 irq_hw_number_t hwirq)
1479 irq_set_chip_and_handler(irq, &tegra_msi_irq_chip, handle_simple_irq);
1480 irq_set_chip_data(irq, domain->host_data);
1482 tegra_cpuidle_pcie_irqs_in_use();
1487 static const struct irq_domain_ops msi_domain_ops = {
1488 .map = tegra_msi_map,
1491 static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
1493 struct platform_device *pdev = to_platform_device(pcie->dev);
1494 const struct tegra_pcie_soc_data *soc = pcie->soc_data;
1495 struct tegra_msi *msi = &pcie->msi;
1500 mutex_init(&msi->lock);
1502 msi->chip.dev = pcie->dev;
1503 msi->chip.setup_irq = tegra_msi_setup_irq;
1504 msi->chip.teardown_irq = tegra_msi_teardown_irq;
1506 msi->domain = irq_domain_add_linear(pcie->dev->of_node, INT_PCI_MSI_NR,
1507 &msi_domain_ops, &msi->chip);
1509 dev_err(&pdev->dev, "failed to create IRQ domain\n");
1513 err = platform_get_irq_byname(pdev, "msi");
1515 dev_err(&pdev->dev, "failed to get IRQ: %d\n", err);
1521 err = request_irq(msi->irq, tegra_pcie_msi_irq, IRQF_NO_THREAD,
1522 tegra_msi_irq_chip.name, pcie);
1524 dev_err(&pdev->dev, "failed to request IRQ: %d\n", err);
1528 /* setup AFI/FPCI range */
1529 msi->pages = __get_free_pages(GFP_KERNEL, 0);
1530 base = virt_to_phys((void *)msi->pages);
1532 afi_writel(pcie, base >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
1533 afi_writel(pcie, base, AFI_MSI_AXI_BAR_ST);
1534 /* this register is in 4K increments */
1535 afi_writel(pcie, 1, AFI_MSI_BAR_SZ);
1537 /* enable all MSI vectors */
1538 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC0);
1539 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC1);
1540 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC2);
1541 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC3);
1542 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC4);
1543 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC5);
1544 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC6);
1545 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC7);
1547 /* and unmask the MSI interrupt */
1548 reg = afi_readl(pcie, AFI_INTR_MASK);
1549 reg |= AFI_INTR_MASK_MSI_MASK;
1550 afi_writel(pcie, reg, AFI_INTR_MASK);
1555 irq_domain_remove(msi->domain);
1559 static int tegra_pcie_disable_msi(struct tegra_pcie *pcie)
1561 struct tegra_msi *msi = &pcie->msi;
1562 unsigned int i, irq;
1565 /* mask the MSI interrupt */
1566 value = afi_readl(pcie, AFI_INTR_MASK);
1567 value &= ~AFI_INTR_MASK_MSI_MASK;
1568 afi_writel(pcie, value, AFI_INTR_MASK);
1570 /* disable all MSI vectors */
1571 afi_writel(pcie, 0, AFI_MSI_EN_VEC0);
1572 afi_writel(pcie, 0, AFI_MSI_EN_VEC1);
1573 afi_writel(pcie, 0, AFI_MSI_EN_VEC2);
1574 afi_writel(pcie, 0, AFI_MSI_EN_VEC3);
1575 afi_writel(pcie, 0, AFI_MSI_EN_VEC4);
1576 afi_writel(pcie, 0, AFI_MSI_EN_VEC5);
1577 afi_writel(pcie, 0, AFI_MSI_EN_VEC6);
1578 afi_writel(pcie, 0, AFI_MSI_EN_VEC7);
1580 free_pages(msi->pages, 0);
1583 free_irq(msi->irq, pcie);
1585 for (i = 0; i < INT_PCI_MSI_NR; i++) {
1586 irq = irq_find_mapping(msi->domain, i);
1588 irq_dispose_mapping(irq);
1591 irq_domain_remove(msi->domain);
1596 static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
1599 struct device_node *np = pcie->dev->of_node;
1601 if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
1604 dev_info(pcie->dev, "4x1, 1x1 configuration\n");
1605 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1;
1609 dev_info(pcie->dev, "2x1, 1x1 configuration\n");
1610 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1;
1613 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
1616 dev_info(pcie->dev, "4x1, 2x1 configuration\n");
1617 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420;
1621 dev_info(pcie->dev, "2x3 configuration\n");
1622 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222;
1626 dev_info(pcie->dev, "4x1, 1x2 configuration\n");
1627 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411;
1630 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
1633 dev_info(pcie->dev, "single-mode configuration\n");
1634 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE;
1638 dev_info(pcie->dev, "dual-mode configuration\n");
1639 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL;
1648 * Check whether a given set of supplies is available in a device tree node.
1649 * This is used to check whether the new or the legacy device tree bindings
1652 static bool of_regulator_bulk_available(struct device_node *np,
1653 struct regulator_bulk_data *supplies,
1654 unsigned int num_supplies)
1659 for (i = 0; i < num_supplies; i++) {
1660 snprintf(property, 32, "%s-supply", supplies[i].supply);
1662 if (of_find_property(np, property, NULL) == NULL)
1670 * Old versions of the device tree binding for this device used a set of power
1671 * supplies that didn't match the hardware inputs. This happened to work for a
1672 * number of cases but is not future proof. However to preserve backwards-
1673 * compatibility with old device trees, this function will try to use the old
1676 static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie)
1678 struct device_node *np = pcie->dev->of_node;
1680 if (of_device_is_compatible(np, "nvidia,tegra30-pcie"))
1681 pcie->num_supplies = 3;
1682 else if (of_device_is_compatible(np, "nvidia,tegra20-pcie"))
1683 pcie->num_supplies = 2;
1685 if (pcie->num_supplies == 0) {
1686 dev_err(pcie->dev, "device %s not supported in legacy mode\n",
1691 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
1692 sizeof(*pcie->supplies),
1694 if (!pcie->supplies)
1697 pcie->supplies[0].supply = "pex-clk";
1698 pcie->supplies[1].supply = "vdd";
1700 if (pcie->num_supplies > 2)
1701 pcie->supplies[2].supply = "avdd";
1703 return devm_regulator_bulk_get(pcie->dev, pcie->num_supplies,
1708 * Obtains the list of regulators required for a particular generation of the
1711 * This would've been nice to do simply by providing static tables for use
1712 * with the regulator_bulk_*() API, but unfortunately Tegra30 is a bit quirky
1713 * in that it has two pairs or AVDD_PEX and VDD_PEX supplies (PEXA and PEXB)
1714 * and either seems to be optional depending on which ports are being used.
1716 static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
1718 struct device_node *np = pcie->dev->of_node;
1721 if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
1722 pcie->num_supplies = 7;
1724 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
1725 sizeof(*pcie->supplies),
1727 if (!pcie->supplies)
1730 pcie->supplies[i++].supply = "avddio-pex";
1731 pcie->supplies[i++].supply = "dvddio-pex";
1732 pcie->supplies[i++].supply = "avdd-pex-pll";
1733 pcie->supplies[i++].supply = "hvdd-pex";
1734 pcie->supplies[i++].supply = "hvdd-pex-pll-e";
1735 pcie->supplies[i++].supply = "vddio-pex-ctl";
1736 pcie->supplies[i++].supply = "avdd-pll-erefe";
1737 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
1738 bool need_pexa = false, need_pexb = false;
1740 /* VDD_PEXA and AVDD_PEXA supply lanes 0 to 3 */
1741 if (lane_mask & 0x0f)
1744 /* VDD_PEXB and AVDD_PEXB supply lanes 4 to 5 */
1745 if (lane_mask & 0x30)
1748 pcie->num_supplies = 4 + (need_pexa ? 2 : 0) +
1749 (need_pexb ? 2 : 0);
1751 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
1752 sizeof(*pcie->supplies),
1754 if (!pcie->supplies)
1757 pcie->supplies[i++].supply = "avdd-pex-pll";
1758 pcie->supplies[i++].supply = "hvdd-pex";
1759 pcie->supplies[i++].supply = "vddio-pex-ctl";
1760 pcie->supplies[i++].supply = "avdd-plle";
1763 pcie->supplies[i++].supply = "avdd-pexa";
1764 pcie->supplies[i++].supply = "vdd-pexa";
1768 pcie->supplies[i++].supply = "avdd-pexb";
1769 pcie->supplies[i++].supply = "vdd-pexb";
1771 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
1772 pcie->num_supplies = 5;
1774 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
1775 sizeof(*pcie->supplies),
1777 if (!pcie->supplies)
1780 pcie->supplies[0].supply = "avdd-pex";
1781 pcie->supplies[1].supply = "vdd-pex";
1782 pcie->supplies[2].supply = "avdd-pex-pll";
1783 pcie->supplies[3].supply = "avdd-plle";
1784 pcie->supplies[4].supply = "vddio-pex-clk";
1787 if (of_regulator_bulk_available(pcie->dev->of_node, pcie->supplies,
1788 pcie->num_supplies))
1789 return devm_regulator_bulk_get(pcie->dev, pcie->num_supplies,
1793 * If not all regulators are available for this new scheme, assume
1794 * that the device tree complies with an older version of the device
1797 dev_info(pcie->dev, "using legacy DT binding for power supplies\n");
1799 devm_kfree(pcie->dev, pcie->supplies);
1800 pcie->num_supplies = 0;
1802 return tegra_pcie_get_legacy_regulators(pcie);
1805 static int tegra_pcie_parse_dt(struct tegra_pcie *pcie)
1807 const struct tegra_pcie_soc_data *soc = pcie->soc_data;
1808 struct device_node *np = pcie->dev->of_node, *port;
1809 struct of_pci_range_parser parser;
1810 struct of_pci_range range;
1811 u32 lanes = 0, mask = 0;
1812 unsigned int lane = 0;
1813 struct resource res;
1816 if (of_pci_range_parser_init(&parser, np)) {
1817 dev_err(pcie->dev, "missing \"ranges\" property\n");
1821 for_each_of_pci_range(&parser, &range) {
1822 err = of_pci_range_to_resource(&range, np, &res);
1826 switch (res.flags & IORESOURCE_TYPE_BITS) {
1828 /* Track the bus -> CPU I/O mapping offset. */
1829 pcie->offset.io = res.start - range.pci_addr;
1831 memcpy(&pcie->pio, &res, sizeof(res));
1832 pcie->pio.name = np->full_name;
1835 * The Tegra PCIe host bridge uses this to program the
1836 * mapping of the I/O space to the physical address,
1837 * so we override the .start and .end fields here that
1838 * of_pci_range_to_resource() converted to I/O space.
1839 * We also set the IORESOURCE_MEM type to clarify that
1840 * the resource is in the physical memory space.
1842 pcie->io.start = range.cpu_addr;
1843 pcie->io.end = range.cpu_addr + range.size - 1;
1844 pcie->io.flags = IORESOURCE_MEM;
1845 pcie->io.name = "I/O";
1847 memcpy(&res, &pcie->io, sizeof(res));
1850 case IORESOURCE_MEM:
1852 * Track the bus -> CPU memory mapping offset. This
1853 * assumes that the prefetchable and non-prefetchable
1854 * regions will be the last of type IORESOURCE_MEM in
1855 * the ranges property.
1857 pcie->offset.mem = res.start - range.pci_addr;
1859 if (res.flags & IORESOURCE_PREFETCH) {
1860 memcpy(&pcie->prefetch, &res, sizeof(res));
1861 pcie->prefetch.name = "prefetchable";
1863 memcpy(&pcie->mem, &res, sizeof(res));
1864 pcie->mem.name = "non-prefetchable";
1870 err = of_pci_parse_bus_range(np, &pcie->busn);
1872 dev_err(pcie->dev, "failed to parse ranges property: %d\n",
1874 pcie->busn.name = np->name;
1875 pcie->busn.start = 0;
1876 pcie->busn.end = 0xff;
1877 pcie->busn.flags = IORESOURCE_BUS;
1880 /* parse root ports */
1881 for_each_child_of_node(np, port) {
1882 struct tegra_pcie_port *rp;
1886 err = of_pci_get_devfn(port);
1888 dev_err(pcie->dev, "failed to parse address: %d\n",
1893 index = PCI_SLOT(err);
1895 if (index < 1 || index > soc->num_ports) {
1896 dev_err(pcie->dev, "invalid port number: %d\n", index);
1902 err = of_property_read_u32(port, "nvidia,num-lanes", &value);
1904 dev_err(pcie->dev, "failed to parse # of lanes: %d\n",
1910 dev_err(pcie->dev, "invalid # of lanes: %u\n", value);
1914 lanes |= value << (index << 3);
1916 if (!of_device_is_available(port)) {
1921 mask |= ((1 << value) - 1) << lane;
1924 rp = devm_kzalloc(pcie->dev, sizeof(*rp), GFP_KERNEL);
1928 err = of_address_to_resource(port, 0, &rp->regs);
1930 dev_err(pcie->dev, "failed to parse address: %d\n",
1935 INIT_LIST_HEAD(&rp->list);
1941 rp->base = devm_ioremap_resource(pcie->dev, &rp->regs);
1942 if (IS_ERR(rp->base))
1943 return PTR_ERR(rp->base);
1945 list_add_tail(&rp->list, &pcie->ports);
1948 err = tegra_pcie_get_xbar_config(pcie, lanes, &pcie->xbar_config);
1950 dev_err(pcie->dev, "invalid lane configuration\n");
1954 err = tegra_pcie_get_regulators(pcie, mask);
1962 * FIXME: If there are no PCIe cards attached, then calling this function
1963 * can result in the increase of the bootup time as there are big timeout
1966 #define TEGRA_PCIE_LINKUP_TIMEOUT 200 /* up to 1.2 seconds */
1967 static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
1969 unsigned int retries = 3;
1970 unsigned long value;
1972 /* override presence detection */
1973 value = readl(port->base + RP_PRIV_MISC);
1974 value &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT;
1975 value |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT;
1976 writel(value, port->base + RP_PRIV_MISC);
1979 unsigned int timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
1982 value = readl(port->base + RP_VEND_XP);
1984 if (value & RP_VEND_XP_DL_UP)
1987 usleep_range(1000, 2000);
1988 } while (--timeout);
1991 dev_err(port->pcie->dev, "link %u down, retrying\n",
1996 timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
1999 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2001 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2004 usleep_range(1000, 2000);
2005 } while (--timeout);
2008 tegra_pcie_port_reset(port);
2009 } while (--retries);
2014 static int tegra_pcie_enable(struct tegra_pcie *pcie)
2016 struct tegra_pcie_port *port, *tmp;
2019 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2020 dev_info(pcie->dev, "probing port %u, using %u lanes\n",
2021 port->index, port->lanes);
2023 tegra_pcie_port_enable(port);
2025 if (tegra_pcie_port_check_link(port))
2028 dev_info(pcie->dev, "link %u down, ignoring\n", port->index);
2030 tegra_pcie_port_disable(port);
2031 tegra_pcie_port_free(port);
2034 memset(&hw, 0, sizeof(hw));
2036 #ifdef CONFIG_PCI_MSI
2037 hw.msi_ctrl = &pcie->msi.chip;
2040 hw.nr_controllers = 1;
2041 hw.private_data = (void **)&pcie;
2042 hw.setup = tegra_pcie_setup;
2043 hw.map_irq = tegra_pcie_map_irq;
2044 hw.ops = &tegra_pcie_ops;
2046 pci_common_init_dev(pcie->dev, &hw);
2051 static const struct tegra_pcie_soc_data tegra20_pcie_data = {
2053 .msi_base_shift = 0,
2054 .pads_pll_ctl = PADS_PLL_CTL_TEGRA20,
2055 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10,
2056 .has_pex_clkreq_en = false,
2057 .has_pex_bias_ctrl = false,
2058 .has_intr_prsnt_sense = false,
2059 .has_cml_clk = false,
2063 static const struct tegra_pcie_soc_data tegra30_pcie_data = {
2065 .msi_base_shift = 8,
2066 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2067 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2068 .has_pex_clkreq_en = true,
2069 .has_pex_bias_ctrl = true,
2070 .has_intr_prsnt_sense = true,
2071 .has_cml_clk = true,
2075 static const struct tegra_pcie_soc_data tegra124_pcie_data = {
2077 .msi_base_shift = 8,
2078 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2079 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2080 .has_pex_clkreq_en = true,
2081 .has_pex_bias_ctrl = true,
2082 .has_intr_prsnt_sense = true,
2083 .has_cml_clk = true,
2087 static const struct of_device_id tegra_pcie_of_match[] = {
2088 { .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie_data },
2089 { .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie_data },
2090 { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie_data },
2093 MODULE_DEVICE_TABLE(of, tegra_pcie_of_match);
2095 static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos)
2097 struct tegra_pcie *pcie = s->private;
2099 if (list_empty(&pcie->ports))
2102 seq_printf(s, "Index Status\n");
2104 return seq_list_start(&pcie->ports, *pos);
2107 static void *tegra_pcie_ports_seq_next(struct seq_file *s, void *v, loff_t *pos)
2109 struct tegra_pcie *pcie = s->private;
2111 return seq_list_next(v, &pcie->ports, pos);
2114 static void tegra_pcie_ports_seq_stop(struct seq_file *s, void *v)
2118 static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v)
2120 bool up = false, active = false;
2121 struct tegra_pcie_port *port;
2124 port = list_entry(v, struct tegra_pcie_port, list);
2126 value = readl(port->base + RP_VEND_XP);
2128 if (value & RP_VEND_XP_DL_UP)
2131 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2133 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2136 seq_printf(s, "%2u ", port->index);
2139 seq_printf(s, "up");
2143 seq_printf(s, ", ");
2145 seq_printf(s, "active");
2148 seq_printf(s, "\n");
2152 static const struct seq_operations tegra_pcie_ports_seq_ops = {
2153 .start = tegra_pcie_ports_seq_start,
2154 .next = tegra_pcie_ports_seq_next,
2155 .stop = tegra_pcie_ports_seq_stop,
2156 .show = tegra_pcie_ports_seq_show,
2159 static int tegra_pcie_ports_open(struct inode *inode, struct file *file)
2161 struct tegra_pcie *pcie = inode->i_private;
2165 err = seq_open(file, &tegra_pcie_ports_seq_ops);
2169 s = file->private_data;
2175 static const struct file_operations tegra_pcie_ports_ops = {
2176 .owner = THIS_MODULE,
2177 .open = tegra_pcie_ports_open,
2179 .llseek = seq_lseek,
2180 .release = seq_release,
2183 static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
2185 struct dentry *file;
2187 pcie->debugfs = debugfs_create_dir("pcie", NULL);
2191 file = debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs,
2192 pcie, &tegra_pcie_ports_ops);
2199 debugfs_remove_recursive(pcie->debugfs);
2200 pcie->debugfs = NULL;
2204 static int tegra_pcie_probe(struct platform_device *pdev)
2206 const struct of_device_id *match;
2207 struct tegra_pcie *pcie;
2210 match = of_match_device(tegra_pcie_of_match, &pdev->dev);
2214 pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL);
2218 INIT_LIST_HEAD(&pcie->buses);
2219 INIT_LIST_HEAD(&pcie->ports);
2220 pcie->soc_data = match->data;
2221 pcie->dev = &pdev->dev;
2223 err = tegra_pcie_parse_dt(pcie);
2227 pcibios_min_mem = 0;
2229 err = tegra_pcie_get_resources(pcie);
2231 dev_err(&pdev->dev, "failed to request resources: %d\n", err);
2235 err = tegra_pcie_enable_controller(pcie);
2239 /* setup the AFI address translations */
2240 tegra_pcie_setup_translations(pcie);
2242 if (IS_ENABLED(CONFIG_PCI_MSI)) {
2243 err = tegra_pcie_enable_msi(pcie);
2246 "failed to enable MSI support: %d\n",
2252 err = tegra_pcie_enable(pcie);
2254 dev_err(&pdev->dev, "failed to enable PCIe ports: %d\n", err);
2258 if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2259 err = tegra_pcie_debugfs_init(pcie);
2261 dev_err(&pdev->dev, "failed to setup debugfs: %d\n",
2265 platform_set_drvdata(pdev, pcie);
2269 if (IS_ENABLED(CONFIG_PCI_MSI))
2270 tegra_pcie_disable_msi(pcie);
2272 tegra_pcie_put_resources(pcie);
2276 static struct platform_driver tegra_pcie_driver = {
2278 .name = "tegra-pcie",
2279 .of_match_table = tegra_pcie_of_match,
2280 .suppress_bind_attrs = true,
2282 .probe = tegra_pcie_probe,
2284 module_platform_driver(tegra_pcie_driver);
2286 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
2287 MODULE_DESCRIPTION("NVIDIA Tegra PCIe driver");
2288 MODULE_LICENSE("GPL v2");