Loading...
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PCIe host controller driver for Tegra SoCs
4 *
5 * Copyright (c) 2010, CompuLab, Ltd.
6 * Author: Mike Rapoport <mike@compulab.co.il>
7 *
8 * Based on NVIDIA PCIe driver
9 * Copyright (c) 2008-2009, NVIDIA Corporation.
10 *
11 * Bits taken from arch/arm/mach-dove/pcie.c
12 *
13 * Author: Thierry Reding <treding@nvidia.com>
14 */
15
16#include <linux/clk.h>
17#include <linux/debugfs.h>
18#include <linux/delay.h>
19#include <linux/export.h>
20#include <linux/gpio/consumer.h>
21#include <linux/interrupt.h>
22#include <linux/iopoll.h>
23#include <linux/irq.h>
24#include <linux/irqchip/chained_irq.h>
25#include <linux/irqdomain.h>
26#include <linux/kernel.h>
27#include <linux/init.h>
28#include <linux/module.h>
29#include <linux/msi.h>
30#include <linux/of_address.h>
31#include <linux/of_pci.h>
32#include <linux/of_platform.h>
33#include <linux/pci.h>
34#include <linux/phy/phy.h>
35#include <linux/pinctrl/consumer.h>
36#include <linux/platform_device.h>
37#include <linux/reset.h>
38#include <linux/sizes.h>
39#include <linux/slab.h>
40#include <linux/vmalloc.h>
41#include <linux/regulator/consumer.h>
42
43#include <soc/tegra/cpuidle.h>
44#include <soc/tegra/pmc.h>
45
46#include "../pci.h"
47
48#define INT_PCI_MSI_NR (8 * 32)
49
50/* register definitions */
51
52#define AFI_AXI_BAR0_SZ 0x00
53#define AFI_AXI_BAR1_SZ 0x04
54#define AFI_AXI_BAR2_SZ 0x08
55#define AFI_AXI_BAR3_SZ 0x0c
56#define AFI_AXI_BAR4_SZ 0x10
57#define AFI_AXI_BAR5_SZ 0x14
58
59#define AFI_AXI_BAR0_START 0x18
60#define AFI_AXI_BAR1_START 0x1c
61#define AFI_AXI_BAR2_START 0x20
62#define AFI_AXI_BAR3_START 0x24
63#define AFI_AXI_BAR4_START 0x28
64#define AFI_AXI_BAR5_START 0x2c
65
66#define AFI_FPCI_BAR0 0x30
67#define AFI_FPCI_BAR1 0x34
68#define AFI_FPCI_BAR2 0x38
69#define AFI_FPCI_BAR3 0x3c
70#define AFI_FPCI_BAR4 0x40
71#define AFI_FPCI_BAR5 0x44
72
73#define AFI_CACHE_BAR0_SZ 0x48
74#define AFI_CACHE_BAR0_ST 0x4c
75#define AFI_CACHE_BAR1_SZ 0x50
76#define AFI_CACHE_BAR1_ST 0x54
77
78#define AFI_MSI_BAR_SZ 0x60
79#define AFI_MSI_FPCI_BAR_ST 0x64
80#define AFI_MSI_AXI_BAR_ST 0x68
81
82#define AFI_MSI_VEC(x) (0x6c + ((x) * 4))
83#define AFI_MSI_EN_VEC(x) (0x8c + ((x) * 4))
84
85#define AFI_CONFIGURATION 0xac
86#define AFI_CONFIGURATION_EN_FPCI (1 << 0)
87#define AFI_CONFIGURATION_CLKEN_OVERRIDE (1 << 31)
88
89#define AFI_FPCI_ERROR_MASKS 0xb0
90
91#define AFI_INTR_MASK 0xb4
92#define AFI_INTR_MASK_INT_MASK (1 << 0)
93#define AFI_INTR_MASK_MSI_MASK (1 << 8)
94
95#define AFI_INTR_CODE 0xb8
96#define AFI_INTR_CODE_MASK 0xf
97#define AFI_INTR_INI_SLAVE_ERROR 1
98#define AFI_INTR_INI_DECODE_ERROR 2
99#define AFI_INTR_TARGET_ABORT 3
100#define AFI_INTR_MASTER_ABORT 4
101#define AFI_INTR_INVALID_WRITE 5
102#define AFI_INTR_LEGACY 6
103#define AFI_INTR_FPCI_DECODE_ERROR 7
104#define AFI_INTR_AXI_DECODE_ERROR 8
105#define AFI_INTR_FPCI_TIMEOUT 9
106#define AFI_INTR_PE_PRSNT_SENSE 10
107#define AFI_INTR_PE_CLKREQ_SENSE 11
108#define AFI_INTR_CLKCLAMP_SENSE 12
109#define AFI_INTR_RDY4PD_SENSE 13
110#define AFI_INTR_P2P_ERROR 14
111
112#define AFI_INTR_SIGNATURE 0xbc
113#define AFI_UPPER_FPCI_ADDRESS 0xc0
114#define AFI_SM_INTR_ENABLE 0xc4
115#define AFI_SM_INTR_INTA_ASSERT (1 << 0)
116#define AFI_SM_INTR_INTB_ASSERT (1 << 1)
117#define AFI_SM_INTR_INTC_ASSERT (1 << 2)
118#define AFI_SM_INTR_INTD_ASSERT (1 << 3)
119#define AFI_SM_INTR_INTA_DEASSERT (1 << 4)
120#define AFI_SM_INTR_INTB_DEASSERT (1 << 5)
121#define AFI_SM_INTR_INTC_DEASSERT (1 << 6)
122#define AFI_SM_INTR_INTD_DEASSERT (1 << 7)
123
124#define AFI_AFI_INTR_ENABLE 0xc8
125#define AFI_INTR_EN_INI_SLVERR (1 << 0)
126#define AFI_INTR_EN_INI_DECERR (1 << 1)
127#define AFI_INTR_EN_TGT_SLVERR (1 << 2)
128#define AFI_INTR_EN_TGT_DECERR (1 << 3)
129#define AFI_INTR_EN_TGT_WRERR (1 << 4)
130#define AFI_INTR_EN_DFPCI_DECERR (1 << 5)
131#define AFI_INTR_EN_AXI_DECERR (1 << 6)
132#define AFI_INTR_EN_FPCI_TIMEOUT (1 << 7)
133#define AFI_INTR_EN_PRSNT_SENSE (1 << 8)
134
135#define AFI_PCIE_PME 0xf0
136
137#define AFI_PCIE_CONFIG 0x0f8
138#define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1))
139#define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0xe
140#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20)
141#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE (0x0 << 20)
142#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420 (0x0 << 20)
143#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1 (0x0 << 20)
144#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401 (0x0 << 20)
145#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL (0x1 << 20)
146#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222 (0x1 << 20)
147#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1 (0x1 << 20)
148#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211 (0x1 << 20)
149#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20)
150#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111 (0x2 << 20)
151#define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(x) (1 << ((x) + 29))
152#define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL (0x7 << 29)
153
154#define AFI_FUSE 0x104
155#define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2)
156
157#define AFI_PEX0_CTRL 0x110
158#define AFI_PEX1_CTRL 0x118
159#define AFI_PEX_CTRL_RST (1 << 0)
160#define AFI_PEX_CTRL_CLKREQ_EN (1 << 1)
161#define AFI_PEX_CTRL_REFCLK_EN (1 << 3)
162#define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4)
163
164#define AFI_PLLE_CONTROL 0x160
165#define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9)
166#define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1)
167
168#define AFI_PEXBIAS_CTRL_0 0x168
169
170#define RP_ECTL_2_R1 0x00000e84
171#define RP_ECTL_2_R1_RX_CTLE_1C_MASK 0xffff
172
173#define RP_ECTL_4_R1 0x00000e8c
174#define RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK (0xffff << 16)
175#define RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT 16
176
177#define RP_ECTL_5_R1 0x00000e90
178#define RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK 0xffffffff
179
180#define RP_ECTL_6_R1 0x00000e94
181#define RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK 0xffffffff
182
183#define RP_ECTL_2_R2 0x00000ea4
184#define RP_ECTL_2_R2_RX_CTLE_1C_MASK 0xffff
185
186#define RP_ECTL_4_R2 0x00000eac
187#define RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK (0xffff << 16)
188#define RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT 16
189
190#define RP_ECTL_5_R2 0x00000eb0
191#define RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK 0xffffffff
192
193#define RP_ECTL_6_R2 0x00000eb4
194#define RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK 0xffffffff
195
196#define RP_VEND_XP 0x00000f00
197#define RP_VEND_XP_DL_UP (1 << 30)
198#define RP_VEND_XP_OPPORTUNISTIC_ACK (1 << 27)
199#define RP_VEND_XP_OPPORTUNISTIC_UPDATEFC (1 << 28)
200#define RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK (0xff << 18)
201
202#define RP_VEND_CTL0 0x00000f44
203#define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK (0xf << 12)
204#define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH (0x9 << 12)
205
206#define RP_VEND_CTL1 0x00000f48
207#define RP_VEND_CTL1_ERPT (1 << 13)
208
209#define RP_VEND_XP_BIST 0x00000f4c
210#define RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE (1 << 28)
211
212#define RP_VEND_CTL2 0x00000fa8
213#define RP_VEND_CTL2_PCA_ENABLE (1 << 7)
214
215#define RP_PRIV_MISC 0x00000fe0
216#define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0)
217#define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0)
218#define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK (0x7f << 16)
219#define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD (0xf << 16)
220#define RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE (1 << 23)
221#define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK (0x7f << 24)
222#define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD (0xf << 24)
223#define RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE (1 << 31)
224
225#define RP_LINK_CONTROL_STATUS 0x00000090
226#define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000
227#define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000
228
229#define RP_LINK_CONTROL_STATUS_2 0x000000b0
230
231#define PADS_CTL_SEL 0x0000009c
232
233#define PADS_CTL 0x000000a0
234#define PADS_CTL_IDDQ_1L (1 << 0)
235#define PADS_CTL_TX_DATA_EN_1L (1 << 6)
236#define PADS_CTL_RX_DATA_EN_1L (1 << 10)
237
238#define PADS_PLL_CTL_TEGRA20 0x000000b8
239#define PADS_PLL_CTL_TEGRA30 0x000000b4
240#define PADS_PLL_CTL_RST_B4SM (1 << 1)
241#define PADS_PLL_CTL_LOCKDET (1 << 8)
242#define PADS_PLL_CTL_REFCLK_MASK (0x3 << 16)
243#define PADS_PLL_CTL_REFCLK_INTERNAL_CML (0 << 16)
244#define PADS_PLL_CTL_REFCLK_INTERNAL_CMOS (1 << 16)
245#define PADS_PLL_CTL_REFCLK_EXTERNAL (2 << 16)
246#define PADS_PLL_CTL_TXCLKREF_MASK (0x1 << 20)
247#define PADS_PLL_CTL_TXCLKREF_DIV10 (0 << 20)
248#define PADS_PLL_CTL_TXCLKREF_DIV5 (1 << 20)
249#define PADS_PLL_CTL_TXCLKREF_BUF_EN (1 << 22)
250
251#define PADS_REFCLK_CFG0 0x000000c8
252#define PADS_REFCLK_CFG1 0x000000cc
253#define PADS_REFCLK_BIAS 0x000000d0
254
255/*
256 * Fields in PADS_REFCLK_CFG*. Those registers form an array of 16-bit
257 * entries, one entry per PCIe port. These field definitions and desired
258 * values aren't in the TRM, but do come from NVIDIA.
259 */
260#define PADS_REFCLK_CFG_TERM_SHIFT 2 /* 6:2 */
261#define PADS_REFCLK_CFG_E_TERM_SHIFT 7
262#define PADS_REFCLK_CFG_PREDI_SHIFT 8 /* 11:8 */
263#define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */
264
265#define PME_ACK_TIMEOUT 10000
266#define LINK_RETRAIN_TIMEOUT 100000 /* in usec */
267
268struct tegra_msi {
269 DECLARE_BITMAP(used, INT_PCI_MSI_NR);
270 struct irq_domain *domain;
271 struct mutex map_lock;
272 spinlock_t mask_lock;
273 void *virt;
274 dma_addr_t phys;
275 int irq;
276};
277
278/* used to differentiate between Tegra SoC generations */
279struct tegra_pcie_port_soc {
280 struct {
281 u8 turnoff_bit;
282 u8 ack_bit;
283 } pme;
284};
285
286struct tegra_pcie_soc {
287 unsigned int num_ports;
288 const struct tegra_pcie_port_soc *ports;
289 unsigned int msi_base_shift;
290 unsigned long afi_pex2_ctrl;
291 u32 pads_pll_ctl;
292 u32 tx_ref_sel;
293 u32 pads_refclk_cfg0;
294 u32 pads_refclk_cfg1;
295 u32 update_fc_threshold;
296 bool has_pex_clkreq_en;
297 bool has_pex_bias_ctrl;
298 bool has_intr_prsnt_sense;
299 bool has_cml_clk;
300 bool has_gen2;
301 bool force_pca_enable;
302 bool program_uphy;
303 bool update_clamp_threshold;
304 bool program_deskew_time;
305 bool update_fc_timer;
306 bool has_cache_bars;
307 struct {
308 struct {
309 u32 rp_ectl_2_r1;
310 u32 rp_ectl_4_r1;
311 u32 rp_ectl_5_r1;
312 u32 rp_ectl_6_r1;
313 u32 rp_ectl_2_r2;
314 u32 rp_ectl_4_r2;
315 u32 rp_ectl_5_r2;
316 u32 rp_ectl_6_r2;
317 } regs;
318 bool enable;
319 } ectl;
320};
321
322struct tegra_pcie {
323 struct device *dev;
324
325 void __iomem *pads;
326 void __iomem *afi;
327 void __iomem *cfg;
328 int irq;
329
330 struct resource cs;
331
332 struct clk *pex_clk;
333 struct clk *afi_clk;
334 struct clk *pll_e;
335 struct clk *cml_clk;
336
337 struct reset_control *pex_rst;
338 struct reset_control *afi_rst;
339 struct reset_control *pcie_xrst;
340
341 bool legacy_phy;
342 struct phy *phy;
343
344 struct tegra_msi msi;
345
346 struct list_head ports;
347 u32 xbar_config;
348
349 struct regulator_bulk_data *supplies;
350 unsigned int num_supplies;
351
352 const struct tegra_pcie_soc *soc;
353 struct dentry *debugfs;
354};
355
356static inline struct tegra_pcie *msi_to_pcie(struct tegra_msi *msi)
357{
358 return container_of(msi, struct tegra_pcie, msi);
359}
360
361struct tegra_pcie_port {
362 struct tegra_pcie *pcie;
363 struct device_node *np;
364 struct list_head list;
365 struct resource regs;
366 void __iomem *base;
367 unsigned int index;
368 unsigned int lanes;
369
370 struct phy **phys;
371
372 struct gpio_desc *reset_gpio;
373};
374
375static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
376 unsigned long offset)
377{
378 writel(value, pcie->afi + offset);
379}
380
381static inline u32 afi_readl(struct tegra_pcie *pcie, unsigned long offset)
382{
383 return readl(pcie->afi + offset);
384}
385
386static inline void pads_writel(struct tegra_pcie *pcie, u32 value,
387 unsigned long offset)
388{
389 writel(value, pcie->pads + offset);
390}
391
392static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
393{
394 return readl(pcie->pads + offset);
395}
396
397/*
398 * The configuration space mapping on Tegra is somewhat similar to the ECAM
399 * defined by PCIe. However it deviates a bit in how the 4 bits for extended
400 * register accesses are mapped:
401 *
402 * [27:24] extended register number
403 * [23:16] bus number
404 * [15:11] device number
405 * [10: 8] function number
406 * [ 7: 0] register number
407 *
408 * Mapping the whole extended configuration space would require 256 MiB of
409 * virtual address space, only a small part of which will actually be used.
410 *
411 * To work around this, a 4 KiB region is used to generate the required
412 * configuration transaction with relevant B:D:F and register offset values.
413 * This is achieved by dynamically programming base address and size of
414 * AFI_AXI_BAR used for end point config space mapping to make sure that the
415 * address (access to which generates correct config transaction) falls in
416 * this 4 KiB region.
417 */
418static unsigned int tegra_pcie_conf_offset(u8 bus, unsigned int devfn,
419 unsigned int where)
420{
421 return ((where & 0xf00) << 16) | (bus << 16) | (PCI_SLOT(devfn) << 11) |
422 (PCI_FUNC(devfn) << 8) | (where & 0xff);
423}
424
425static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
426 unsigned int devfn,
427 int where)
428{
429 struct tegra_pcie *pcie = bus->sysdata;
430 void __iomem *addr = NULL;
431
432 if (bus->number == 0) {
433 unsigned int slot = PCI_SLOT(devfn);
434 struct tegra_pcie_port *port;
435
436 list_for_each_entry(port, &pcie->ports, list) {
437 if (port->index + 1 == slot) {
438 addr = port->base + (where & ~3);
439 break;
440 }
441 }
442 } else {
443 unsigned int offset;
444 u32 base;
445
446 offset = tegra_pcie_conf_offset(bus->number, devfn, where);
447
448 /* move 4 KiB window to offset within the FPCI region */
449 base = 0xfe100000 + ((offset & ~(SZ_4K - 1)) >> 8);
450 afi_writel(pcie, base, AFI_FPCI_BAR0);
451
452 /* move to correct offset within the 4 KiB page */
453 addr = pcie->cfg + (offset & (SZ_4K - 1));
454 }
455
456 return addr;
457}
458
459static int tegra_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
460 int where, int size, u32 *value)
461{
462 if (bus->number == 0)
463 return pci_generic_config_read32(bus, devfn, where, size,
464 value);
465
466 return pci_generic_config_read(bus, devfn, where, size, value);
467}
468
469static int tegra_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
470 int where, int size, u32 value)
471{
472 if (bus->number == 0)
473 return pci_generic_config_write32(bus, devfn, where, size,
474 value);
475
476 return pci_generic_config_write(bus, devfn, where, size, value);
477}
478
479static struct pci_ops tegra_pcie_ops = {
480 .map_bus = tegra_pcie_map_bus,
481 .read = tegra_pcie_config_read,
482 .write = tegra_pcie_config_write,
483};
484
485static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port)
486{
487 const struct tegra_pcie_soc *soc = port->pcie->soc;
488 unsigned long ret = 0;
489
490 switch (port->index) {
491 case 0:
492 ret = AFI_PEX0_CTRL;
493 break;
494
495 case 1:
496 ret = AFI_PEX1_CTRL;
497 break;
498
499 case 2:
500 ret = soc->afi_pex2_ctrl;
501 break;
502 }
503
504 return ret;
505}
506
507static void tegra_pcie_port_reset(struct tegra_pcie_port *port)
508{
509 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
510 unsigned long value;
511
512 /* pulse reset signal */
513 if (port->reset_gpio) {
514 gpiod_set_value(port->reset_gpio, 1);
515 } else {
516 value = afi_readl(port->pcie, ctrl);
517 value &= ~AFI_PEX_CTRL_RST;
518 afi_writel(port->pcie, value, ctrl);
519 }
520
521 usleep_range(1000, 2000);
522
523 if (port->reset_gpio) {
524 gpiod_set_value(port->reset_gpio, 0);
525 } else {
526 value = afi_readl(port->pcie, ctrl);
527 value |= AFI_PEX_CTRL_RST;
528 afi_writel(port->pcie, value, ctrl);
529 }
530}
531
532static void tegra_pcie_enable_rp_features(struct tegra_pcie_port *port)
533{
534 const struct tegra_pcie_soc *soc = port->pcie->soc;
535 u32 value;
536
537 /* Enable AER capability */
538 value = readl(port->base + RP_VEND_CTL1);
539 value |= RP_VEND_CTL1_ERPT;
540 writel(value, port->base + RP_VEND_CTL1);
541
542 /* Optimal settings to enhance bandwidth */
543 value = readl(port->base + RP_VEND_XP);
544 value |= RP_VEND_XP_OPPORTUNISTIC_ACK;
545 value |= RP_VEND_XP_OPPORTUNISTIC_UPDATEFC;
546 writel(value, port->base + RP_VEND_XP);
547
548 /*
549 * LTSSM will wait for DLLP to finish before entering L1 or L2,
550 * to avoid truncation of PM messages which results in receiver errors
551 */
552 value = readl(port->base + RP_VEND_XP_BIST);
553 value |= RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE;
554 writel(value, port->base + RP_VEND_XP_BIST);
555
556 value = readl(port->base + RP_PRIV_MISC);
557 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE;
558 value |= RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE;
559
560 if (soc->update_clamp_threshold) {
561 value &= ~(RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK |
562 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK);
563 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD |
564 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD;
565 }
566
567 writel(value, port->base + RP_PRIV_MISC);
568}
569
570static void tegra_pcie_program_ectl_settings(struct tegra_pcie_port *port)
571{
572 const struct tegra_pcie_soc *soc = port->pcie->soc;
573 u32 value;
574
575 value = readl(port->base + RP_ECTL_2_R1);
576 value &= ~RP_ECTL_2_R1_RX_CTLE_1C_MASK;
577 value |= soc->ectl.regs.rp_ectl_2_r1;
578 writel(value, port->base + RP_ECTL_2_R1);
579
580 value = readl(port->base + RP_ECTL_4_R1);
581 value &= ~RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK;
582 value |= soc->ectl.regs.rp_ectl_4_r1 <<
583 RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT;
584 writel(value, port->base + RP_ECTL_4_R1);
585
586 value = readl(port->base + RP_ECTL_5_R1);
587 value &= ~RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK;
588 value |= soc->ectl.regs.rp_ectl_5_r1;
589 writel(value, port->base + RP_ECTL_5_R1);
590
591 value = readl(port->base + RP_ECTL_6_R1);
592 value &= ~RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK;
593 value |= soc->ectl.regs.rp_ectl_6_r1;
594 writel(value, port->base + RP_ECTL_6_R1);
595
596 value = readl(port->base + RP_ECTL_2_R2);
597 value &= ~RP_ECTL_2_R2_RX_CTLE_1C_MASK;
598 value |= soc->ectl.regs.rp_ectl_2_r2;
599 writel(value, port->base + RP_ECTL_2_R2);
600
601 value = readl(port->base + RP_ECTL_4_R2);
602 value &= ~RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK;
603 value |= soc->ectl.regs.rp_ectl_4_r2 <<
604 RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT;
605 writel(value, port->base + RP_ECTL_4_R2);
606
607 value = readl(port->base + RP_ECTL_5_R2);
608 value &= ~RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK;
609 value |= soc->ectl.regs.rp_ectl_5_r2;
610 writel(value, port->base + RP_ECTL_5_R2);
611
612 value = readl(port->base + RP_ECTL_6_R2);
613 value &= ~RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK;
614 value |= soc->ectl.regs.rp_ectl_6_r2;
615 writel(value, port->base + RP_ECTL_6_R2);
616}
617
618static void tegra_pcie_apply_sw_fixup(struct tegra_pcie_port *port)
619{
620 const struct tegra_pcie_soc *soc = port->pcie->soc;
621 u32 value;
622
623 /*
624 * Sometimes link speed change from Gen2 to Gen1 fails due to
625 * instability in deskew logic on lane-0. Increase the deskew
626 * retry time to resolve this issue.
627 */
628 if (soc->program_deskew_time) {
629 value = readl(port->base + RP_VEND_CTL0);
630 value &= ~RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK;
631 value |= RP_VEND_CTL0_DSK_RST_PULSE_WIDTH;
632 writel(value, port->base + RP_VEND_CTL0);
633 }
634
635 if (soc->update_fc_timer) {
636 value = readl(port->base + RP_VEND_XP);
637 value &= ~RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK;
638 value |= soc->update_fc_threshold;
639 writel(value, port->base + RP_VEND_XP);
640 }
641
642 /*
643 * PCIe link doesn't come up with few legacy PCIe endpoints if
644 * root port advertises both Gen-1 and Gen-2 speeds in Tegra.
645 * Hence, the strategy followed here is to initially advertise
646 * only Gen-1 and after link is up, retrain link to Gen-2 speed
647 */
648 value = readl(port->base + RP_LINK_CONTROL_STATUS_2);
649 value &= ~PCI_EXP_LNKSTA_CLS;
650 value |= PCI_EXP_LNKSTA_CLS_2_5GB;
651 writel(value, port->base + RP_LINK_CONTROL_STATUS_2);
652}
653
654static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
655{
656 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
657 const struct tegra_pcie_soc *soc = port->pcie->soc;
658 unsigned long value;
659
660 /* enable reference clock */
661 value = afi_readl(port->pcie, ctrl);
662 value |= AFI_PEX_CTRL_REFCLK_EN;
663
664 if (soc->has_pex_clkreq_en)
665 value |= AFI_PEX_CTRL_CLKREQ_EN;
666
667 value |= AFI_PEX_CTRL_OVERRIDE_EN;
668
669 afi_writel(port->pcie, value, ctrl);
670
671 tegra_pcie_port_reset(port);
672
673 if (soc->force_pca_enable) {
674 value = readl(port->base + RP_VEND_CTL2);
675 value |= RP_VEND_CTL2_PCA_ENABLE;
676 writel(value, port->base + RP_VEND_CTL2);
677 }
678
679 tegra_pcie_enable_rp_features(port);
680
681 if (soc->ectl.enable)
682 tegra_pcie_program_ectl_settings(port);
683
684 tegra_pcie_apply_sw_fixup(port);
685}
686
687static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
688{
689 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
690 const struct tegra_pcie_soc *soc = port->pcie->soc;
691 unsigned long value;
692
693 /* assert port reset */
694 value = afi_readl(port->pcie, ctrl);
695 value &= ~AFI_PEX_CTRL_RST;
696 afi_writel(port->pcie, value, ctrl);
697
698 /* disable reference clock */
699 value = afi_readl(port->pcie, ctrl);
700
701 if (soc->has_pex_clkreq_en)
702 value &= ~AFI_PEX_CTRL_CLKREQ_EN;
703
704 value &= ~AFI_PEX_CTRL_REFCLK_EN;
705 afi_writel(port->pcie, value, ctrl);
706
707 /* disable PCIe port and set CLKREQ# as GPIO to allow PLLE power down */
708 value = afi_readl(port->pcie, AFI_PCIE_CONFIG);
709 value |= AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
710 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index);
711 afi_writel(port->pcie, value, AFI_PCIE_CONFIG);
712}
713
714static void tegra_pcie_port_free(struct tegra_pcie_port *port)
715{
716 struct tegra_pcie *pcie = port->pcie;
717 struct device *dev = pcie->dev;
718
719 devm_iounmap(dev, port->base);
720 devm_release_mem_region(dev, port->regs.start,
721 resource_size(&port->regs));
722 list_del(&port->list);
723 devm_kfree(dev, port);
724}
725
726/* Tegra PCIE root complex wrongly reports device class */
727static void tegra_pcie_fixup_class(struct pci_dev *dev)
728{
729 dev->class = PCI_CLASS_BRIDGE_PCI_NORMAL;
730}
731DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_fixup_class);
732DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class);
733DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class);
734DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class);
735
736/* Tegra20 and Tegra30 PCIE requires relaxed ordering */
737static void tegra_pcie_relax_enable(struct pci_dev *dev)
738{
739 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
740}
741DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_relax_enable);
742DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_relax_enable);
743DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_relax_enable);
744DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_relax_enable);
745
746static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
747{
748 struct tegra_pcie *pcie = pdev->bus->sysdata;
749 int irq;
750
751 tegra_cpuidle_pcie_irqs_in_use();
752
753 irq = of_irq_parse_and_map_pci(pdev, slot, pin);
754 if (!irq)
755 irq = pcie->irq;
756
757 return irq;
758}
759
760static irqreturn_t tegra_pcie_isr(int irq, void *arg)
761{
762 static const char * const err_msg[] = {
763 "Unknown",
764 "AXI slave error",
765 "AXI decode error",
766 "Target abort",
767 "Master abort",
768 "Invalid write",
769 "Legacy interrupt",
770 "Response decoding error",
771 "AXI response decoding error",
772 "Transaction timeout",
773 "Slot present pin change",
774 "Slot clock request change",
775 "TMS clock ramp change",
776 "TMS ready for power down",
777 "Peer2Peer error",
778 };
779 struct tegra_pcie *pcie = arg;
780 struct device *dev = pcie->dev;
781 u32 code, signature;
782
783 code = afi_readl(pcie, AFI_INTR_CODE) & AFI_INTR_CODE_MASK;
784 signature = afi_readl(pcie, AFI_INTR_SIGNATURE);
785 afi_writel(pcie, 0, AFI_INTR_CODE);
786
787 if (code == AFI_INTR_LEGACY)
788 return IRQ_NONE;
789
790 if (code >= ARRAY_SIZE(err_msg))
791 code = 0;
792
793 /*
794 * do not pollute kernel log with master abort reports since they
795 * happen a lot during enumeration
796 */
797 if (code == AFI_INTR_MASTER_ABORT || code == AFI_INTR_PE_PRSNT_SENSE)
798 dev_dbg(dev, "%s, signature: %08x\n", err_msg[code], signature);
799 else
800 dev_err(dev, "%s, signature: %08x\n", err_msg[code], signature);
801
802 if (code == AFI_INTR_TARGET_ABORT || code == AFI_INTR_MASTER_ABORT ||
803 code == AFI_INTR_FPCI_DECODE_ERROR) {
804 u32 fpci = afi_readl(pcie, AFI_UPPER_FPCI_ADDRESS) & 0xff;
805 u64 address = (u64)fpci << 32 | (signature & 0xfffffffc);
806
807 if (code == AFI_INTR_MASTER_ABORT)
808 dev_dbg(dev, " FPCI address: %10llx\n", address);
809 else
810 dev_err(dev, " FPCI address: %10llx\n", address);
811 }
812
813 return IRQ_HANDLED;
814}
815
816/*
817 * FPCI map is as follows:
818 * - 0xfdfc000000: I/O space
819 * - 0xfdfe000000: type 0 configuration space
820 * - 0xfdff000000: type 1 configuration space
821 * - 0xfe00000000: type 0 extended configuration space
822 * - 0xfe10000000: type 1 extended configuration space
823 */
824static void tegra_pcie_setup_translations(struct tegra_pcie *pcie)
825{
826 u32 size;
827 struct resource_entry *entry;
828 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
829
830 /* Bar 0: type 1 extended configuration space */
831 size = resource_size(&pcie->cs);
832 afi_writel(pcie, pcie->cs.start, AFI_AXI_BAR0_START);
833 afi_writel(pcie, size >> 12, AFI_AXI_BAR0_SZ);
834
835 resource_list_for_each_entry(entry, &bridge->windows) {
836 u32 fpci_bar, axi_address;
837 struct resource *res = entry->res;
838
839 size = resource_size(res);
840
841 switch (resource_type(res)) {
842 case IORESOURCE_IO:
843 /* Bar 1: downstream IO bar */
844 fpci_bar = 0xfdfc0000;
845 axi_address = pci_pio_to_address(res->start);
846 afi_writel(pcie, axi_address, AFI_AXI_BAR1_START);
847 afi_writel(pcie, size >> 12, AFI_AXI_BAR1_SZ);
848 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR1);
849 break;
850 case IORESOURCE_MEM:
851 fpci_bar = (((res->start >> 12) & 0x0fffffff) << 4) | 0x1;
852 axi_address = res->start;
853
854 if (res->flags & IORESOURCE_PREFETCH) {
855 /* Bar 2: prefetchable memory BAR */
856 afi_writel(pcie, axi_address, AFI_AXI_BAR2_START);
857 afi_writel(pcie, size >> 12, AFI_AXI_BAR2_SZ);
858 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR2);
859
860 } else {
861 /* Bar 3: non prefetchable memory BAR */
862 afi_writel(pcie, axi_address, AFI_AXI_BAR3_START);
863 afi_writel(pcie, size >> 12, AFI_AXI_BAR3_SZ);
864 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR3);
865 }
866 break;
867 }
868 }
869
870 /* NULL out the remaining BARs as they are not used */
871 afi_writel(pcie, 0, AFI_AXI_BAR4_START);
872 afi_writel(pcie, 0, AFI_AXI_BAR4_SZ);
873 afi_writel(pcie, 0, AFI_FPCI_BAR4);
874
875 afi_writel(pcie, 0, AFI_AXI_BAR5_START);
876 afi_writel(pcie, 0, AFI_AXI_BAR5_SZ);
877 afi_writel(pcie, 0, AFI_FPCI_BAR5);
878
879 if (pcie->soc->has_cache_bars) {
880 /* map all upstream transactions as uncached */
881 afi_writel(pcie, 0, AFI_CACHE_BAR0_ST);
882 afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ);
883 afi_writel(pcie, 0, AFI_CACHE_BAR1_ST);
884 afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ);
885 }
886
887 /* MSI translations are setup only when needed */
888 afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST);
889 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
890 afi_writel(pcie, 0, AFI_MSI_AXI_BAR_ST);
891 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
892}
893
894static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout)
895{
896 const struct tegra_pcie_soc *soc = pcie->soc;
897 u32 value;
898
899 timeout = jiffies + msecs_to_jiffies(timeout);
900
901 while (time_before(jiffies, timeout)) {
902 value = pads_readl(pcie, soc->pads_pll_ctl);
903 if (value & PADS_PLL_CTL_LOCKDET)
904 return 0;
905 }
906
907 return -ETIMEDOUT;
908}
909
910static int tegra_pcie_phy_enable(struct tegra_pcie *pcie)
911{
912 struct device *dev = pcie->dev;
913 const struct tegra_pcie_soc *soc = pcie->soc;
914 u32 value;
915 int err;
916
917 /* initialize internal PHY, enable up to 16 PCIE lanes */
918 pads_writel(pcie, 0x0, PADS_CTL_SEL);
919
920 /* override IDDQ to 1 on all 4 lanes */
921 value = pads_readl(pcie, PADS_CTL);
922 value |= PADS_CTL_IDDQ_1L;
923 pads_writel(pcie, value, PADS_CTL);
924
925 /*
926 * Set up PHY PLL inputs select PLLE output as refclock,
927 * set TX ref sel to div10 (not div5).
928 */
929 value = pads_readl(pcie, soc->pads_pll_ctl);
930 value &= ~(PADS_PLL_CTL_REFCLK_MASK | PADS_PLL_CTL_TXCLKREF_MASK);
931 value |= PADS_PLL_CTL_REFCLK_INTERNAL_CML | soc->tx_ref_sel;
932 pads_writel(pcie, value, soc->pads_pll_ctl);
933
934 /* reset PLL */
935 value = pads_readl(pcie, soc->pads_pll_ctl);
936 value &= ~PADS_PLL_CTL_RST_B4SM;
937 pads_writel(pcie, value, soc->pads_pll_ctl);
938
939 usleep_range(20, 100);
940
941 /* take PLL out of reset */
942 value = pads_readl(pcie, soc->pads_pll_ctl);
943 value |= PADS_PLL_CTL_RST_B4SM;
944 pads_writel(pcie, value, soc->pads_pll_ctl);
945
946 /* wait for the PLL to lock */
947 err = tegra_pcie_pll_wait(pcie, 500);
948 if (err < 0) {
949 dev_err(dev, "PLL failed to lock: %d\n", err);
950 return err;
951 }
952
953 /* turn off IDDQ override */
954 value = pads_readl(pcie, PADS_CTL);
955 value &= ~PADS_CTL_IDDQ_1L;
956 pads_writel(pcie, value, PADS_CTL);
957
958 /* enable TX/RX data */
959 value = pads_readl(pcie, PADS_CTL);
960 value |= PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L;
961 pads_writel(pcie, value, PADS_CTL);
962
963 return 0;
964}
965
966static int tegra_pcie_phy_disable(struct tegra_pcie *pcie)
967{
968 const struct tegra_pcie_soc *soc = pcie->soc;
969 u32 value;
970
971 /* disable TX/RX data */
972 value = pads_readl(pcie, PADS_CTL);
973 value &= ~(PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L);
974 pads_writel(pcie, value, PADS_CTL);
975
976 /* override IDDQ */
977 value = pads_readl(pcie, PADS_CTL);
978 value |= PADS_CTL_IDDQ_1L;
979 pads_writel(pcie, value, PADS_CTL);
980
981 /* reset PLL */
982 value = pads_readl(pcie, soc->pads_pll_ctl);
983 value &= ~PADS_PLL_CTL_RST_B4SM;
984 pads_writel(pcie, value, soc->pads_pll_ctl);
985
986 usleep_range(20, 100);
987
988 return 0;
989}
990
991static int tegra_pcie_port_phy_power_on(struct tegra_pcie_port *port)
992{
993 struct device *dev = port->pcie->dev;
994 unsigned int i;
995 int err;
996
997 for (i = 0; i < port->lanes; i++) {
998 err = phy_power_on(port->phys[i]);
999 if (err < 0) {
1000 dev_err(dev, "failed to power on PHY#%u: %d\n", i, err);
1001 return err;
1002 }
1003 }
1004
1005 return 0;
1006}
1007
1008static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port)
1009{
1010 struct device *dev = port->pcie->dev;
1011 unsigned int i;
1012 int err;
1013
1014 for (i = 0; i < port->lanes; i++) {
1015 err = phy_power_off(port->phys[i]);
1016 if (err < 0) {
1017 dev_err(dev, "failed to power off PHY#%u: %d\n", i,
1018 err);
1019 return err;
1020 }
1021 }
1022
1023 return 0;
1024}
1025
1026static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie)
1027{
1028 struct device *dev = pcie->dev;
1029 struct tegra_pcie_port *port;
1030 int err;
1031
1032 if (pcie->legacy_phy) {
1033 if (pcie->phy)
1034 err = phy_power_on(pcie->phy);
1035 else
1036 err = tegra_pcie_phy_enable(pcie);
1037
1038 if (err < 0)
1039 dev_err(dev, "failed to power on PHY: %d\n", err);
1040
1041 return err;
1042 }
1043
1044 list_for_each_entry(port, &pcie->ports, list) {
1045 err = tegra_pcie_port_phy_power_on(port);
1046 if (err < 0) {
1047 dev_err(dev,
1048 "failed to power on PCIe port %u PHY: %d\n",
1049 port->index, err);
1050 return err;
1051 }
1052 }
1053
1054 return 0;
1055}
1056
1057static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie)
1058{
1059 struct device *dev = pcie->dev;
1060 struct tegra_pcie_port *port;
1061 int err;
1062
1063 if (pcie->legacy_phy) {
1064 if (pcie->phy)
1065 err = phy_power_off(pcie->phy);
1066 else
1067 err = tegra_pcie_phy_disable(pcie);
1068
1069 if (err < 0)
1070 dev_err(dev, "failed to power off PHY: %d\n", err);
1071
1072 return err;
1073 }
1074
1075 list_for_each_entry(port, &pcie->ports, list) {
1076 err = tegra_pcie_port_phy_power_off(port);
1077 if (err < 0) {
1078 dev_err(dev,
1079 "failed to power off PCIe port %u PHY: %d\n",
1080 port->index, err);
1081 return err;
1082 }
1083 }
1084
1085 return 0;
1086}
1087
1088static void tegra_pcie_enable_controller(struct tegra_pcie *pcie)
1089{
1090 const struct tegra_pcie_soc *soc = pcie->soc;
1091 struct tegra_pcie_port *port;
1092 unsigned long value;
1093
1094 /* enable PLL power down */
1095 if (pcie->phy) {
1096 value = afi_readl(pcie, AFI_PLLE_CONTROL);
1097 value &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL;
1098 value |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN;
1099 afi_writel(pcie, value, AFI_PLLE_CONTROL);
1100 }
1101
1102 /* power down PCIe slot clock bias pad */
1103 if (soc->has_pex_bias_ctrl)
1104 afi_writel(pcie, 0, AFI_PEXBIAS_CTRL_0);
1105
1106 /* configure mode and disable all ports */
1107 value = afi_readl(pcie, AFI_PCIE_CONFIG);
1108 value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK;
1109 value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config;
1110 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL;
1111
1112 list_for_each_entry(port, &pcie->ports, list) {
1113 value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
1114 value &= ~AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index);
1115 }
1116
1117 afi_writel(pcie, value, AFI_PCIE_CONFIG);
1118
1119 if (soc->has_gen2) {
1120 value = afi_readl(pcie, AFI_FUSE);
1121 value &= ~AFI_FUSE_PCIE_T0_GEN2_DIS;
1122 afi_writel(pcie, value, AFI_FUSE);
1123 } else {
1124 value = afi_readl(pcie, AFI_FUSE);
1125 value |= AFI_FUSE_PCIE_T0_GEN2_DIS;
1126 afi_writel(pcie, value, AFI_FUSE);
1127 }
1128
1129 /* Disable AFI dynamic clock gating and enable PCIe */
1130 value = afi_readl(pcie, AFI_CONFIGURATION);
1131 value |= AFI_CONFIGURATION_EN_FPCI;
1132 value |= AFI_CONFIGURATION_CLKEN_OVERRIDE;
1133 afi_writel(pcie, value, AFI_CONFIGURATION);
1134
1135 value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR |
1136 AFI_INTR_EN_TGT_SLVERR | AFI_INTR_EN_TGT_DECERR |
1137 AFI_INTR_EN_TGT_WRERR | AFI_INTR_EN_DFPCI_DECERR;
1138
1139 if (soc->has_intr_prsnt_sense)
1140 value |= AFI_INTR_EN_PRSNT_SENSE;
1141
1142 afi_writel(pcie, value, AFI_AFI_INTR_ENABLE);
1143 afi_writel(pcie, 0xffffffff, AFI_SM_INTR_ENABLE);
1144
1145 /* don't enable MSI for now, only when needed */
1146 afi_writel(pcie, AFI_INTR_MASK_INT_MASK, AFI_INTR_MASK);
1147
1148 /* disable all exceptions */
1149 afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS);
1150}
1151
1152static void tegra_pcie_power_off(struct tegra_pcie *pcie)
1153{
1154 struct device *dev = pcie->dev;
1155 const struct tegra_pcie_soc *soc = pcie->soc;
1156 int err;
1157
1158 reset_control_assert(pcie->afi_rst);
1159
1160 clk_disable_unprepare(pcie->pll_e);
1161 if (soc->has_cml_clk)
1162 clk_disable_unprepare(pcie->cml_clk);
1163 clk_disable_unprepare(pcie->afi_clk);
1164
1165 if (!dev->pm_domain)
1166 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1167
1168 err = regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1169 if (err < 0)
1170 dev_warn(dev, "failed to disable regulators: %d\n", err);
1171}
1172
1173static int tegra_pcie_power_on(struct tegra_pcie *pcie)
1174{
1175 struct device *dev = pcie->dev;
1176 const struct tegra_pcie_soc *soc = pcie->soc;
1177 int err;
1178
1179 reset_control_assert(pcie->pcie_xrst);
1180 reset_control_assert(pcie->afi_rst);
1181 reset_control_assert(pcie->pex_rst);
1182
1183 if (!dev->pm_domain)
1184 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1185
1186 /* enable regulators */
1187 err = regulator_bulk_enable(pcie->num_supplies, pcie->supplies);
1188 if (err < 0)
1189 dev_err(dev, "failed to enable regulators: %d\n", err);
1190
1191 if (!dev->pm_domain) {
1192 err = tegra_powergate_power_on(TEGRA_POWERGATE_PCIE);
1193 if (err) {
1194 dev_err(dev, "failed to power ungate: %d\n", err);
1195 goto regulator_disable;
1196 }
1197 err = tegra_powergate_remove_clamping(TEGRA_POWERGATE_PCIE);
1198 if (err) {
1199 dev_err(dev, "failed to remove clamp: %d\n", err);
1200 goto powergate;
1201 }
1202 }
1203
1204 err = clk_prepare_enable(pcie->afi_clk);
1205 if (err < 0) {
1206 dev_err(dev, "failed to enable AFI clock: %d\n", err);
1207 goto powergate;
1208 }
1209
1210 if (soc->has_cml_clk) {
1211 err = clk_prepare_enable(pcie->cml_clk);
1212 if (err < 0) {
1213 dev_err(dev, "failed to enable CML clock: %d\n", err);
1214 goto disable_afi_clk;
1215 }
1216 }
1217
1218 err = clk_prepare_enable(pcie->pll_e);
1219 if (err < 0) {
1220 dev_err(dev, "failed to enable PLLE clock: %d\n", err);
1221 goto disable_cml_clk;
1222 }
1223
1224 reset_control_deassert(pcie->afi_rst);
1225
1226 return 0;
1227
1228disable_cml_clk:
1229 if (soc->has_cml_clk)
1230 clk_disable_unprepare(pcie->cml_clk);
1231disable_afi_clk:
1232 clk_disable_unprepare(pcie->afi_clk);
1233powergate:
1234 if (!dev->pm_domain)
1235 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1236regulator_disable:
1237 regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1238
1239 return err;
1240}
1241
1242static void tegra_pcie_apply_pad_settings(struct tegra_pcie *pcie)
1243{
1244 const struct tegra_pcie_soc *soc = pcie->soc;
1245
1246 /* Configure the reference clock driver */
1247 pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0);
1248
1249 if (soc->num_ports > 2)
1250 pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1);
1251}
1252
1253static int tegra_pcie_clocks_get(struct tegra_pcie *pcie)
1254{
1255 struct device *dev = pcie->dev;
1256 const struct tegra_pcie_soc *soc = pcie->soc;
1257
1258 pcie->pex_clk = devm_clk_get(dev, "pex");
1259 if (IS_ERR(pcie->pex_clk))
1260 return PTR_ERR(pcie->pex_clk);
1261
1262 pcie->afi_clk = devm_clk_get(dev, "afi");
1263 if (IS_ERR(pcie->afi_clk))
1264 return PTR_ERR(pcie->afi_clk);
1265
1266 pcie->pll_e = devm_clk_get(dev, "pll_e");
1267 if (IS_ERR(pcie->pll_e))
1268 return PTR_ERR(pcie->pll_e);
1269
1270 if (soc->has_cml_clk) {
1271 pcie->cml_clk = devm_clk_get(dev, "cml");
1272 if (IS_ERR(pcie->cml_clk))
1273 return PTR_ERR(pcie->cml_clk);
1274 }
1275
1276 return 0;
1277}
1278
1279static int tegra_pcie_resets_get(struct tegra_pcie *pcie)
1280{
1281 struct device *dev = pcie->dev;
1282
1283 pcie->pex_rst = devm_reset_control_get_exclusive(dev, "pex");
1284 if (IS_ERR(pcie->pex_rst))
1285 return PTR_ERR(pcie->pex_rst);
1286
1287 pcie->afi_rst = devm_reset_control_get_exclusive(dev, "afi");
1288 if (IS_ERR(pcie->afi_rst))
1289 return PTR_ERR(pcie->afi_rst);
1290
1291 pcie->pcie_xrst = devm_reset_control_get_exclusive(dev, "pcie_x");
1292 if (IS_ERR(pcie->pcie_xrst))
1293 return PTR_ERR(pcie->pcie_xrst);
1294
1295 return 0;
1296}
1297
1298static int tegra_pcie_phys_get_legacy(struct tegra_pcie *pcie)
1299{
1300 struct device *dev = pcie->dev;
1301 int err;
1302
1303 pcie->phy = devm_phy_optional_get(dev, "pcie");
1304 if (IS_ERR(pcie->phy)) {
1305 err = PTR_ERR(pcie->phy);
1306 dev_err(dev, "failed to get PHY: %d\n", err);
1307 return err;
1308 }
1309
1310 err = phy_init(pcie->phy);
1311 if (err < 0) {
1312 dev_err(dev, "failed to initialize PHY: %d\n", err);
1313 return err;
1314 }
1315
1316 pcie->legacy_phy = true;
1317
1318 return 0;
1319}
1320
1321static struct phy *devm_of_phy_optional_get_index(struct device *dev,
1322 struct device_node *np,
1323 const char *consumer,
1324 unsigned int index)
1325{
1326 struct phy *phy;
1327 char *name;
1328
1329 name = kasprintf(GFP_KERNEL, "%s-%u", consumer, index);
1330 if (!name)
1331 return ERR_PTR(-ENOMEM);
1332
1333 phy = devm_of_phy_optional_get(dev, np, name);
1334 kfree(name);
1335
1336 return phy;
1337}
1338
1339static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port)
1340{
1341 struct device *dev = port->pcie->dev;
1342 struct phy *phy;
1343 unsigned int i;
1344 int err;
1345
1346 port->phys = devm_kcalloc(dev, sizeof(phy), port->lanes, GFP_KERNEL);
1347 if (!port->phys)
1348 return -ENOMEM;
1349
1350 for (i = 0; i < port->lanes; i++) {
1351 phy = devm_of_phy_optional_get_index(dev, port->np, "pcie", i);
1352 if (IS_ERR(phy)) {
1353 dev_err(dev, "failed to get PHY#%u: %ld\n", i,
1354 PTR_ERR(phy));
1355 return PTR_ERR(phy);
1356 }
1357
1358 err = phy_init(phy);
1359 if (err < 0) {
1360 dev_err(dev, "failed to initialize PHY#%u: %d\n", i,
1361 err);
1362 return err;
1363 }
1364
1365 port->phys[i] = phy;
1366 }
1367
1368 return 0;
1369}
1370
1371static int tegra_pcie_phys_get(struct tegra_pcie *pcie)
1372{
1373 const struct tegra_pcie_soc *soc = pcie->soc;
1374 struct device_node *np = pcie->dev->of_node;
1375 struct tegra_pcie_port *port;
1376 int err;
1377
1378 if (!soc->has_gen2 || of_property_present(np, "phys"))
1379 return tegra_pcie_phys_get_legacy(pcie);
1380
1381 list_for_each_entry(port, &pcie->ports, list) {
1382 err = tegra_pcie_port_get_phys(port);
1383 if (err < 0)
1384 return err;
1385 }
1386
1387 return 0;
1388}
1389
1390static void tegra_pcie_phys_put(struct tegra_pcie *pcie)
1391{
1392 struct tegra_pcie_port *port;
1393 struct device *dev = pcie->dev;
1394 int err, i;
1395
1396 if (pcie->legacy_phy) {
1397 err = phy_exit(pcie->phy);
1398 if (err < 0)
1399 dev_err(dev, "failed to teardown PHY: %d\n", err);
1400 return;
1401 }
1402
1403 list_for_each_entry(port, &pcie->ports, list) {
1404 for (i = 0; i < port->lanes; i++) {
1405 err = phy_exit(port->phys[i]);
1406 if (err < 0)
1407 dev_err(dev, "failed to teardown PHY#%u: %d\n",
1408 i, err);
1409 }
1410 }
1411}
1412
1413static int tegra_pcie_get_resources(struct tegra_pcie *pcie)
1414{
1415 struct device *dev = pcie->dev;
1416 struct platform_device *pdev = to_platform_device(dev);
1417 struct resource *res;
1418 const struct tegra_pcie_soc *soc = pcie->soc;
1419 int err;
1420
1421 err = tegra_pcie_clocks_get(pcie);
1422 if (err) {
1423 dev_err(dev, "failed to get clocks: %d\n", err);
1424 return err;
1425 }
1426
1427 err = tegra_pcie_resets_get(pcie);
1428 if (err) {
1429 dev_err(dev, "failed to get resets: %d\n", err);
1430 return err;
1431 }
1432
1433 if (soc->program_uphy) {
1434 err = tegra_pcie_phys_get(pcie);
1435 if (err < 0) {
1436 dev_err(dev, "failed to get PHYs: %d\n", err);
1437 return err;
1438 }
1439 }
1440
1441 pcie->pads = devm_platform_ioremap_resource_byname(pdev, "pads");
1442 if (IS_ERR(pcie->pads)) {
1443 err = PTR_ERR(pcie->pads);
1444 goto phys_put;
1445 }
1446
1447 pcie->afi = devm_platform_ioremap_resource_byname(pdev, "afi");
1448 if (IS_ERR(pcie->afi)) {
1449 err = PTR_ERR(pcie->afi);
1450 goto phys_put;
1451 }
1452
1453 /* request configuration space, but remap later, on demand */
1454 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs");
1455 if (!res) {
1456 err = -EADDRNOTAVAIL;
1457 goto phys_put;
1458 }
1459
1460 pcie->cs = *res;
1461
1462 /* constrain configuration space to 4 KiB */
1463 resource_set_size(&pcie->cs, SZ_4K);
1464
1465 pcie->cfg = devm_ioremap_resource(dev, &pcie->cs);
1466 if (IS_ERR(pcie->cfg)) {
1467 err = PTR_ERR(pcie->cfg);
1468 goto phys_put;
1469 }
1470
1471 /* request interrupt */
1472 err = platform_get_irq_byname(pdev, "intr");
1473 if (err < 0)
1474 goto phys_put;
1475
1476 pcie->irq = err;
1477
1478 err = request_irq(pcie->irq, tegra_pcie_isr, IRQF_SHARED, "PCIE", pcie);
1479 if (err) {
1480 dev_err(dev, "failed to register IRQ: %d\n", err);
1481 goto phys_put;
1482 }
1483
1484 return 0;
1485
1486phys_put:
1487 if (soc->program_uphy)
1488 tegra_pcie_phys_put(pcie);
1489
1490 return err;
1491}
1492
1493static int tegra_pcie_put_resources(struct tegra_pcie *pcie)
1494{
1495 const struct tegra_pcie_soc *soc = pcie->soc;
1496
1497 if (pcie->irq > 0)
1498 free_irq(pcie->irq, pcie);
1499
1500 if (soc->program_uphy)
1501 tegra_pcie_phys_put(pcie);
1502
1503 return 0;
1504}
1505
1506static void tegra_pcie_pme_turnoff(struct tegra_pcie_port *port)
1507{
1508 struct tegra_pcie *pcie = port->pcie;
1509 const struct tegra_pcie_soc *soc = pcie->soc;
1510 int err;
1511 u32 val;
1512 u8 ack_bit;
1513
1514 val = afi_readl(pcie, AFI_PCIE_PME);
1515 val |= (0x1 << soc->ports[port->index].pme.turnoff_bit);
1516 afi_writel(pcie, val, AFI_PCIE_PME);
1517
1518 ack_bit = soc->ports[port->index].pme.ack_bit;
1519 err = readl_poll_timeout(pcie->afi + AFI_PCIE_PME, val,
1520 val & (0x1 << ack_bit), 1, PME_ACK_TIMEOUT);
1521 if (err)
1522 dev_err(pcie->dev, "PME Ack is not received on port: %d\n",
1523 port->index);
1524
1525 usleep_range(10000, 11000);
1526
1527 val = afi_readl(pcie, AFI_PCIE_PME);
1528 val &= ~(0x1 << soc->ports[port->index].pme.turnoff_bit);
1529 afi_writel(pcie, val, AFI_PCIE_PME);
1530}
1531
1532static void tegra_pcie_msi_irq(struct irq_desc *desc)
1533{
1534 struct tegra_pcie *pcie = irq_desc_get_handler_data(desc);
1535 struct irq_chip *chip = irq_desc_get_chip(desc);
1536 struct tegra_msi *msi = &pcie->msi;
1537 struct device *dev = pcie->dev;
1538 unsigned int i;
1539
1540 chained_irq_enter(chip, desc);
1541
1542 for (i = 0; i < 8; i++) {
1543 unsigned long reg = afi_readl(pcie, AFI_MSI_VEC(i));
1544
1545 while (reg) {
1546 unsigned int offset = find_first_bit(®, 32);
1547 unsigned int index = i * 32 + offset;
1548 int ret;
1549
1550 ret = generic_handle_domain_irq(msi->domain->parent, index);
1551 if (ret) {
1552 /*
1553 * that's weird who triggered this?
1554 * just clear it
1555 */
1556 dev_info(dev, "unexpected MSI\n");
1557 afi_writel(pcie, BIT(index % 32), AFI_MSI_VEC(index));
1558 }
1559
1560 /* see if there's any more pending in this vector */
1561 reg = afi_readl(pcie, AFI_MSI_VEC(i));
1562 }
1563 }
1564
1565 chained_irq_exit(chip, desc);
1566}
1567
1568static void tegra_msi_top_irq_ack(struct irq_data *d)
1569{
1570 irq_chip_ack_parent(d);
1571}
1572
1573static void tegra_msi_top_irq_mask(struct irq_data *d)
1574{
1575 pci_msi_mask_irq(d);
1576 irq_chip_mask_parent(d);
1577}
1578
1579static void tegra_msi_top_irq_unmask(struct irq_data *d)
1580{
1581 pci_msi_unmask_irq(d);
1582 irq_chip_unmask_parent(d);
1583}
1584
1585static struct irq_chip tegra_msi_top_chip = {
1586 .name = "Tegra PCIe MSI",
1587 .irq_ack = tegra_msi_top_irq_ack,
1588 .irq_mask = tegra_msi_top_irq_mask,
1589 .irq_unmask = tegra_msi_top_irq_unmask,
1590};
1591
1592static void tegra_msi_irq_ack(struct irq_data *d)
1593{
1594 struct tegra_msi *msi = irq_data_get_irq_chip_data(d);
1595 struct tegra_pcie *pcie = msi_to_pcie(msi);
1596 unsigned int index = d->hwirq / 32;
1597
1598 /* clear the interrupt */
1599 afi_writel(pcie, BIT(d->hwirq % 32), AFI_MSI_VEC(index));
1600}
1601
1602static void tegra_msi_irq_mask(struct irq_data *d)
1603{
1604 struct tegra_msi *msi = irq_data_get_irq_chip_data(d);
1605 struct tegra_pcie *pcie = msi_to_pcie(msi);
1606 unsigned int index = d->hwirq / 32;
1607 unsigned long flags;
1608 u32 value;
1609
1610 spin_lock_irqsave(&msi->mask_lock, flags);
1611 value = afi_readl(pcie, AFI_MSI_EN_VEC(index));
1612 value &= ~BIT(d->hwirq % 32);
1613 afi_writel(pcie, value, AFI_MSI_EN_VEC(index));
1614 spin_unlock_irqrestore(&msi->mask_lock, flags);
1615}
1616
1617static void tegra_msi_irq_unmask(struct irq_data *d)
1618{
1619 struct tegra_msi *msi = irq_data_get_irq_chip_data(d);
1620 struct tegra_pcie *pcie = msi_to_pcie(msi);
1621 unsigned int index = d->hwirq / 32;
1622 unsigned long flags;
1623 u32 value;
1624
1625 spin_lock_irqsave(&msi->mask_lock, flags);
1626 value = afi_readl(pcie, AFI_MSI_EN_VEC(index));
1627 value |= BIT(d->hwirq % 32);
1628 afi_writel(pcie, value, AFI_MSI_EN_VEC(index));
1629 spin_unlock_irqrestore(&msi->mask_lock, flags);
1630}
1631
1632static void tegra_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1633{
1634 struct tegra_msi *msi = irq_data_get_irq_chip_data(data);
1635
1636 msg->address_lo = lower_32_bits(msi->phys);
1637 msg->address_hi = upper_32_bits(msi->phys);
1638 msg->data = data->hwirq;
1639}
1640
1641static struct irq_chip tegra_msi_bottom_chip = {
1642 .name = "Tegra MSI",
1643 .irq_ack = tegra_msi_irq_ack,
1644 .irq_mask = tegra_msi_irq_mask,
1645 .irq_unmask = tegra_msi_irq_unmask,
1646 .irq_compose_msi_msg = tegra_compose_msi_msg,
1647};
1648
1649static int tegra_msi_domain_alloc(struct irq_domain *domain, unsigned int virq,
1650 unsigned int nr_irqs, void *args)
1651{
1652 struct tegra_msi *msi = domain->host_data;
1653 unsigned int i;
1654 int hwirq;
1655
1656 mutex_lock(&msi->map_lock);
1657
1658 hwirq = bitmap_find_free_region(msi->used, INT_PCI_MSI_NR, order_base_2(nr_irqs));
1659
1660 mutex_unlock(&msi->map_lock);
1661
1662 if (hwirq < 0)
1663 return -ENOSPC;
1664
1665 for (i = 0; i < nr_irqs; i++)
1666 irq_domain_set_info(domain, virq + i, hwirq + i,
1667 &tegra_msi_bottom_chip, domain->host_data,
1668 handle_edge_irq, NULL, NULL);
1669
1670 tegra_cpuidle_pcie_irqs_in_use();
1671
1672 return 0;
1673}
1674
1675static void tegra_msi_domain_free(struct irq_domain *domain, unsigned int virq,
1676 unsigned int nr_irqs)
1677{
1678 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
1679 struct tegra_msi *msi = domain->host_data;
1680
1681 mutex_lock(&msi->map_lock);
1682
1683 bitmap_release_region(msi->used, d->hwirq, order_base_2(nr_irqs));
1684
1685 mutex_unlock(&msi->map_lock);
1686}
1687
1688static const struct irq_domain_ops tegra_msi_domain_ops = {
1689 .alloc = tegra_msi_domain_alloc,
1690 .free = tegra_msi_domain_free,
1691};
1692
1693static struct msi_domain_info tegra_msi_info = {
1694 .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
1695 MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX,
1696 .chip = &tegra_msi_top_chip,
1697};
1698
1699static int tegra_allocate_domains(struct tegra_msi *msi)
1700{
1701 struct tegra_pcie *pcie = msi_to_pcie(msi);
1702 struct fwnode_handle *fwnode = dev_fwnode(pcie->dev);
1703 struct irq_domain *parent;
1704
1705 parent = irq_domain_create_linear(fwnode, INT_PCI_MSI_NR,
1706 &tegra_msi_domain_ops, msi);
1707 if (!parent) {
1708 dev_err(pcie->dev, "failed to create IRQ domain\n");
1709 return -ENOMEM;
1710 }
1711 irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS);
1712
1713 msi->domain = pci_msi_create_irq_domain(fwnode, &tegra_msi_info, parent);
1714 if (!msi->domain) {
1715 dev_err(pcie->dev, "failed to create MSI domain\n");
1716 irq_domain_remove(parent);
1717 return -ENOMEM;
1718 }
1719
1720 return 0;
1721}
1722
1723static void tegra_free_domains(struct tegra_msi *msi)
1724{
1725 struct irq_domain *parent = msi->domain->parent;
1726
1727 irq_domain_remove(msi->domain);
1728 irq_domain_remove(parent);
1729}
1730
1731static int tegra_pcie_msi_setup(struct tegra_pcie *pcie)
1732{
1733 struct platform_device *pdev = to_platform_device(pcie->dev);
1734 struct tegra_msi *msi = &pcie->msi;
1735 struct device *dev = pcie->dev;
1736 int err;
1737
1738 mutex_init(&msi->map_lock);
1739 spin_lock_init(&msi->mask_lock);
1740
1741 if (IS_ENABLED(CONFIG_PCI_MSI)) {
1742 err = tegra_allocate_domains(msi);
1743 if (err)
1744 return err;
1745 }
1746
1747 err = platform_get_irq_byname(pdev, "msi");
1748 if (err < 0)
1749 goto free_irq_domain;
1750
1751 msi->irq = err;
1752
1753 irq_set_chained_handler_and_data(msi->irq, tegra_pcie_msi_irq, pcie);
1754
1755 /* Though the PCIe controller can address >32-bit address space, to
1756 * facilitate endpoints that support only 32-bit MSI target address,
1757 * the mask is set to 32-bit to make sure that MSI target address is
1758 * always a 32-bit address
1759 */
1760 err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
1761 if (err < 0) {
1762 dev_err(dev, "failed to set DMA coherent mask: %d\n", err);
1763 goto free_irq;
1764 }
1765
1766 msi->virt = dma_alloc_attrs(dev, PAGE_SIZE, &msi->phys, GFP_KERNEL,
1767 DMA_ATTR_NO_KERNEL_MAPPING);
1768 if (!msi->virt) {
1769 dev_err(dev, "failed to allocate DMA memory for MSI\n");
1770 err = -ENOMEM;
1771 goto free_irq;
1772 }
1773
1774 return 0;
1775
1776free_irq:
1777 irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
1778free_irq_domain:
1779 if (IS_ENABLED(CONFIG_PCI_MSI))
1780 tegra_free_domains(msi);
1781
1782 return err;
1783}
1784
1785static void tegra_pcie_enable_msi(struct tegra_pcie *pcie)
1786{
1787 const struct tegra_pcie_soc *soc = pcie->soc;
1788 struct tegra_msi *msi = &pcie->msi;
1789 u32 reg, msi_state[INT_PCI_MSI_NR / 32];
1790 int i;
1791
1792 afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
1793 afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST);
1794 /* this register is in 4K increments */
1795 afi_writel(pcie, 1, AFI_MSI_BAR_SZ);
1796
1797 /* Restore the MSI allocation state */
1798 bitmap_to_arr32(msi_state, msi->used, INT_PCI_MSI_NR);
1799 for (i = 0; i < ARRAY_SIZE(msi_state); i++)
1800 afi_writel(pcie, msi_state[i], AFI_MSI_EN_VEC(i));
1801
1802 /* and unmask the MSI interrupt */
1803 reg = afi_readl(pcie, AFI_INTR_MASK);
1804 reg |= AFI_INTR_MASK_MSI_MASK;
1805 afi_writel(pcie, reg, AFI_INTR_MASK);
1806}
1807
1808static void tegra_pcie_msi_teardown(struct tegra_pcie *pcie)
1809{
1810 struct tegra_msi *msi = &pcie->msi;
1811 unsigned int i, irq;
1812
1813 dma_free_attrs(pcie->dev, PAGE_SIZE, msi->virt, msi->phys,
1814 DMA_ATTR_NO_KERNEL_MAPPING);
1815
1816 for (i = 0; i < INT_PCI_MSI_NR; i++) {
1817 irq = irq_find_mapping(msi->domain, i);
1818 if (irq > 0)
1819 irq_domain_free_irqs(irq, 1);
1820 }
1821
1822 irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
1823
1824 if (IS_ENABLED(CONFIG_PCI_MSI))
1825 tegra_free_domains(msi);
1826}
1827
1828static int tegra_pcie_disable_msi(struct tegra_pcie *pcie)
1829{
1830 u32 value;
1831
1832 /* mask the MSI interrupt */
1833 value = afi_readl(pcie, AFI_INTR_MASK);
1834 value &= ~AFI_INTR_MASK_MSI_MASK;
1835 afi_writel(pcie, value, AFI_INTR_MASK);
1836
1837 return 0;
1838}
1839
1840static void tegra_pcie_disable_interrupts(struct tegra_pcie *pcie)
1841{
1842 u32 value;
1843
1844 value = afi_readl(pcie, AFI_INTR_MASK);
1845 value &= ~AFI_INTR_MASK_INT_MASK;
1846 afi_writel(pcie, value, AFI_INTR_MASK);
1847}
1848
1849static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
1850 u32 *xbar)
1851{
1852 struct device *dev = pcie->dev;
1853 struct device_node *np = dev->of_node;
1854
1855 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) {
1856 switch (lanes) {
1857 case 0x010004:
1858 dev_info(dev, "4x1, 1x1 configuration\n");
1859 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401;
1860 return 0;
1861
1862 case 0x010102:
1863 dev_info(dev, "2x1, 1X1, 1x1 configuration\n");
1864 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211;
1865 return 0;
1866
1867 case 0x010101:
1868 dev_info(dev, "1x1, 1x1, 1x1 configuration\n");
1869 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111;
1870 return 0;
1871
1872 default:
1873 dev_info(dev, "wrong configuration updated in DT, "
1874 "switching to default 2x1, 1x1, 1x1 "
1875 "configuration\n");
1876 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211;
1877 return 0;
1878 }
1879 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie") ||
1880 of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
1881 switch (lanes) {
1882 case 0x0000104:
1883 dev_info(dev, "4x1, 1x1 configuration\n");
1884 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1;
1885 return 0;
1886
1887 case 0x0000102:
1888 dev_info(dev, "2x1, 1x1 configuration\n");
1889 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1;
1890 return 0;
1891 }
1892 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
1893 switch (lanes) {
1894 case 0x00000204:
1895 dev_info(dev, "4x1, 2x1 configuration\n");
1896 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420;
1897 return 0;
1898
1899 case 0x00020202:
1900 dev_info(dev, "2x3 configuration\n");
1901 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222;
1902 return 0;
1903
1904 case 0x00010104:
1905 dev_info(dev, "4x1, 1x2 configuration\n");
1906 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411;
1907 return 0;
1908 }
1909 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
1910 switch (lanes) {
1911 case 0x00000004:
1912 dev_info(dev, "single-mode configuration\n");
1913 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE;
1914 return 0;
1915
1916 case 0x00000202:
1917 dev_info(dev, "dual-mode configuration\n");
1918 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL;
1919 return 0;
1920 }
1921 }
1922
1923 return -EINVAL;
1924}
1925
1926/*
1927 * Check whether a given set of supplies is available in a device tree node.
1928 * This is used to check whether the new or the legacy device tree bindings
1929 * should be used.
1930 */
1931static bool of_regulator_bulk_available(struct device_node *np,
1932 struct regulator_bulk_data *supplies,
1933 unsigned int num_supplies)
1934{
1935 char property[32];
1936 unsigned int i;
1937
1938 for (i = 0; i < num_supplies; i++) {
1939 snprintf(property, 32, "%s-supply", supplies[i].supply);
1940
1941 if (!of_property_present(np, property))
1942 return false;
1943 }
1944
1945 return true;
1946}
1947
1948/*
1949 * Old versions of the device tree binding for this device used a set of power
1950 * supplies that didn't match the hardware inputs. This happened to work for a
1951 * number of cases but is not future proof. However to preserve backwards-
1952 * compatibility with old device trees, this function will try to use the old
1953 * set of supplies.
1954 */
1955static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie)
1956{
1957 struct device *dev = pcie->dev;
1958 struct device_node *np = dev->of_node;
1959
1960 if (of_device_is_compatible(np, "nvidia,tegra30-pcie"))
1961 pcie->num_supplies = 3;
1962 else if (of_device_is_compatible(np, "nvidia,tegra20-pcie"))
1963 pcie->num_supplies = 2;
1964
1965 if (pcie->num_supplies == 0) {
1966 dev_err(dev, "device %pOF not supported in legacy mode\n", np);
1967 return -ENODEV;
1968 }
1969
1970 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
1971 sizeof(*pcie->supplies),
1972 GFP_KERNEL);
1973 if (!pcie->supplies)
1974 return -ENOMEM;
1975
1976 pcie->supplies[0].supply = "pex-clk";
1977 pcie->supplies[1].supply = "vdd";
1978
1979 if (pcie->num_supplies > 2)
1980 pcie->supplies[2].supply = "avdd";
1981
1982 return devm_regulator_bulk_get(dev, pcie->num_supplies, pcie->supplies);
1983}
1984
1985/*
1986 * Obtains the list of regulators required for a particular generation of the
1987 * IP block.
1988 *
1989 * This would've been nice to do simply by providing static tables for use
1990 * with the regulator_bulk_*() API, but unfortunately Tegra30 is a bit quirky
1991 * in that it has two pairs or AVDD_PEX and VDD_PEX supplies (PEXA and PEXB)
1992 * and either seems to be optional depending on which ports are being used.
1993 */
1994static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
1995{
1996 struct device *dev = pcie->dev;
1997 struct device_node *np = dev->of_node;
1998 unsigned int i = 0;
1999
2000 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) {
2001 pcie->num_supplies = 4;
2002
2003 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
2004 sizeof(*pcie->supplies),
2005 GFP_KERNEL);
2006 if (!pcie->supplies)
2007 return -ENOMEM;
2008
2009 pcie->supplies[i++].supply = "dvdd-pex";
2010 pcie->supplies[i++].supply = "hvdd-pex-pll";
2011 pcie->supplies[i++].supply = "hvdd-pex";
2012 pcie->supplies[i++].supply = "vddio-pexctl-aud";
2013 } else if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
2014 pcie->num_supplies = 3;
2015
2016 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
2017 sizeof(*pcie->supplies),
2018 GFP_KERNEL);
2019 if (!pcie->supplies)
2020 return -ENOMEM;
2021
2022 pcie->supplies[i++].supply = "hvddio-pex";
2023 pcie->supplies[i++].supply = "dvddio-pex";
2024 pcie->supplies[i++].supply = "vddio-pex-ctl";
2025 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
2026 pcie->num_supplies = 4;
2027
2028 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2029 sizeof(*pcie->supplies),
2030 GFP_KERNEL);
2031 if (!pcie->supplies)
2032 return -ENOMEM;
2033
2034 pcie->supplies[i++].supply = "avddio-pex";
2035 pcie->supplies[i++].supply = "dvddio-pex";
2036 pcie->supplies[i++].supply = "hvdd-pex";
2037 pcie->supplies[i++].supply = "vddio-pex-ctl";
2038 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
2039 bool need_pexa = false, need_pexb = false;
2040
2041 /* VDD_PEXA and AVDD_PEXA supply lanes 0 to 3 */
2042 if (lane_mask & 0x0f)
2043 need_pexa = true;
2044
2045 /* VDD_PEXB and AVDD_PEXB supply lanes 4 to 5 */
2046 if (lane_mask & 0x30)
2047 need_pexb = true;
2048
2049 pcie->num_supplies = 4 + (need_pexa ? 2 : 0) +
2050 (need_pexb ? 2 : 0);
2051
2052 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2053 sizeof(*pcie->supplies),
2054 GFP_KERNEL);
2055 if (!pcie->supplies)
2056 return -ENOMEM;
2057
2058 pcie->supplies[i++].supply = "avdd-pex-pll";
2059 pcie->supplies[i++].supply = "hvdd-pex";
2060 pcie->supplies[i++].supply = "vddio-pex-ctl";
2061 pcie->supplies[i++].supply = "avdd-plle";
2062
2063 if (need_pexa) {
2064 pcie->supplies[i++].supply = "avdd-pexa";
2065 pcie->supplies[i++].supply = "vdd-pexa";
2066 }
2067
2068 if (need_pexb) {
2069 pcie->supplies[i++].supply = "avdd-pexb";
2070 pcie->supplies[i++].supply = "vdd-pexb";
2071 }
2072 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
2073 pcie->num_supplies = 5;
2074
2075 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2076 sizeof(*pcie->supplies),
2077 GFP_KERNEL);
2078 if (!pcie->supplies)
2079 return -ENOMEM;
2080
2081 pcie->supplies[0].supply = "avdd-pex";
2082 pcie->supplies[1].supply = "vdd-pex";
2083 pcie->supplies[2].supply = "avdd-pex-pll";
2084 pcie->supplies[3].supply = "avdd-plle";
2085 pcie->supplies[4].supply = "vddio-pex-clk";
2086 }
2087
2088 if (of_regulator_bulk_available(dev->of_node, pcie->supplies,
2089 pcie->num_supplies))
2090 return devm_regulator_bulk_get(dev, pcie->num_supplies,
2091 pcie->supplies);
2092
2093 /*
2094 * If not all regulators are available for this new scheme, assume
2095 * that the device tree complies with an older version of the device
2096 * tree binding.
2097 */
2098 dev_info(dev, "using legacy DT binding for power supplies\n");
2099
2100 devm_kfree(dev, pcie->supplies);
2101 pcie->num_supplies = 0;
2102
2103 return tegra_pcie_get_legacy_regulators(pcie);
2104}
2105
2106static int tegra_pcie_parse_dt(struct tegra_pcie *pcie)
2107{
2108 struct device *dev = pcie->dev;
2109 struct device_node *np = dev->of_node, *port;
2110 const struct tegra_pcie_soc *soc = pcie->soc;
2111 u32 lanes = 0, mask = 0;
2112 unsigned int lane = 0;
2113 int err;
2114
2115 /* parse root ports */
2116 for_each_child_of_node(np, port) {
2117 struct tegra_pcie_port *rp;
2118 unsigned int index;
2119 u32 value;
2120 char *label;
2121
2122 err = of_pci_get_devfn(port);
2123 if (err < 0) {
2124 dev_err(dev, "failed to parse address: %d\n", err);
2125 goto err_node_put;
2126 }
2127
2128 index = PCI_SLOT(err);
2129
2130 if (index < 1 || index > soc->num_ports) {
2131 dev_err(dev, "invalid port number: %d\n", index);
2132 err = -EINVAL;
2133 goto err_node_put;
2134 }
2135
2136 index--;
2137
2138 err = of_property_read_u32(port, "nvidia,num-lanes", &value);
2139 if (err < 0) {
2140 dev_err(dev, "failed to parse # of lanes: %d\n",
2141 err);
2142 goto err_node_put;
2143 }
2144
2145 if (value > 16) {
2146 dev_err(dev, "invalid # of lanes: %u\n", value);
2147 err = -EINVAL;
2148 goto err_node_put;
2149 }
2150
2151 lanes |= value << (index << 3);
2152
2153 if (!of_device_is_available(port)) {
2154 lane += value;
2155 continue;
2156 }
2157
2158 mask |= ((1 << value) - 1) << lane;
2159 lane += value;
2160
2161 rp = devm_kzalloc(dev, sizeof(*rp), GFP_KERNEL);
2162 if (!rp) {
2163 err = -ENOMEM;
2164 goto err_node_put;
2165 }
2166
2167 err = of_address_to_resource(port, 0, &rp->regs);
2168 if (err < 0) {
2169 dev_err(dev, "failed to parse address: %d\n", err);
2170 goto err_node_put;
2171 }
2172
2173 INIT_LIST_HEAD(&rp->list);
2174 rp->index = index;
2175 rp->lanes = value;
2176 rp->pcie = pcie;
2177 rp->np = port;
2178
2179 rp->base = devm_pci_remap_cfg_resource(dev, &rp->regs);
2180 if (IS_ERR(rp->base)) {
2181 err = PTR_ERR(rp->base);
2182 goto err_node_put;
2183 }
2184
2185 label = devm_kasprintf(dev, GFP_KERNEL, "pex-reset-%u", index);
2186 if (!label) {
2187 err = -ENOMEM;
2188 goto err_node_put;
2189 }
2190
2191 /*
2192 * Returns -ENOENT if reset-gpios property is not populated
2193 * and in this case fall back to using AFI per port register
2194 * to toggle PERST# SFIO line.
2195 */
2196 rp->reset_gpio = devm_fwnode_gpiod_get(dev,
2197 of_fwnode_handle(port),
2198 "reset",
2199 GPIOD_OUT_LOW,
2200 label);
2201 if (IS_ERR(rp->reset_gpio)) {
2202 if (PTR_ERR(rp->reset_gpio) == -ENOENT) {
2203 rp->reset_gpio = NULL;
2204 } else {
2205 dev_err(dev, "failed to get reset GPIO: %ld\n",
2206 PTR_ERR(rp->reset_gpio));
2207 err = PTR_ERR(rp->reset_gpio);
2208 goto err_node_put;
2209 }
2210 }
2211
2212 list_add_tail(&rp->list, &pcie->ports);
2213 }
2214
2215 err = tegra_pcie_get_xbar_config(pcie, lanes, &pcie->xbar_config);
2216 if (err < 0) {
2217 dev_err(dev, "invalid lane configuration\n");
2218 return err;
2219 }
2220
2221 err = tegra_pcie_get_regulators(pcie, mask);
2222 if (err < 0)
2223 return err;
2224
2225 return 0;
2226
2227err_node_put:
2228 of_node_put(port);
2229 return err;
2230}
2231
2232/*
2233 * FIXME: If there are no PCIe cards attached, then calling this function
2234 * can result in the increase of the bootup time as there are big timeout
2235 * loops.
2236 */
2237#define TEGRA_PCIE_LINKUP_TIMEOUT 200 /* up to 1.2 seconds */
2238static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
2239{
2240 struct device *dev = port->pcie->dev;
2241 unsigned int retries = 3;
2242 unsigned long value;
2243
2244 /* override presence detection */
2245 value = readl(port->base + RP_PRIV_MISC);
2246 value &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT;
2247 value |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT;
2248 writel(value, port->base + RP_PRIV_MISC);
2249
2250 do {
2251 unsigned int timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
2252
2253 do {
2254 value = readl(port->base + RP_VEND_XP);
2255
2256 if (value & RP_VEND_XP_DL_UP)
2257 break;
2258
2259 usleep_range(1000, 2000);
2260 } while (--timeout);
2261
2262 if (!timeout) {
2263 dev_dbg(dev, "link %u down, retrying\n", port->index);
2264 goto retry;
2265 }
2266
2267 timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
2268
2269 do {
2270 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2271
2272 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2273 return true;
2274
2275 usleep_range(1000, 2000);
2276 } while (--timeout);
2277
2278retry:
2279 tegra_pcie_port_reset(port);
2280 } while (--retries);
2281
2282 return false;
2283}
2284
2285static void tegra_pcie_change_link_speed(struct tegra_pcie *pcie)
2286{
2287 struct device *dev = pcie->dev;
2288 struct tegra_pcie_port *port;
2289 ktime_t deadline;
2290 u32 value;
2291
2292 list_for_each_entry(port, &pcie->ports, list) {
2293 /*
2294 * "Supported Link Speeds Vector" in "Link Capabilities 2"
2295 * is not supported by Tegra. tegra_pcie_change_link_speed()
2296 * is called only for Tegra chips which support Gen2.
2297 * So there no harm if supported link speed is not verified.
2298 */
2299 value = readl(port->base + RP_LINK_CONTROL_STATUS_2);
2300 value &= ~PCI_EXP_LNKSTA_CLS;
2301 value |= PCI_EXP_LNKSTA_CLS_5_0GB;
2302 writel(value, port->base + RP_LINK_CONTROL_STATUS_2);
2303
2304 /*
2305 * Poll until link comes back from recovery to avoid race
2306 * condition.
2307 */
2308 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT);
2309
2310 while (ktime_before(ktime_get(), deadline)) {
2311 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2312 if ((value & PCI_EXP_LNKSTA_LT) == 0)
2313 break;
2314
2315 usleep_range(2000, 3000);
2316 }
2317
2318 if (value & PCI_EXP_LNKSTA_LT)
2319 dev_warn(dev, "PCIe port %u link is in recovery\n",
2320 port->index);
2321
2322 /* Retrain the link */
2323 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2324 value |= PCI_EXP_LNKCTL_RL;
2325 writel(value, port->base + RP_LINK_CONTROL_STATUS);
2326
2327 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT);
2328
2329 while (ktime_before(ktime_get(), deadline)) {
2330 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2331 if ((value & PCI_EXP_LNKSTA_LT) == 0)
2332 break;
2333
2334 usleep_range(2000, 3000);
2335 }
2336
2337 if (value & PCI_EXP_LNKSTA_LT)
2338 dev_err(dev, "failed to retrain link of port %u\n",
2339 port->index);
2340 }
2341}
2342
2343static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
2344{
2345 struct device *dev = pcie->dev;
2346 struct tegra_pcie_port *port, *tmp;
2347
2348 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2349 dev_info(dev, "probing port %u, using %u lanes\n",
2350 port->index, port->lanes);
2351
2352 tegra_pcie_port_enable(port);
2353 }
2354
2355 /* Start LTSSM from Tegra side */
2356 reset_control_deassert(pcie->pcie_xrst);
2357
2358 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2359 if (tegra_pcie_port_check_link(port))
2360 continue;
2361
2362 dev_info(dev, "link %u down, ignoring\n", port->index);
2363
2364 tegra_pcie_port_disable(port);
2365 tegra_pcie_port_free(port);
2366 }
2367
2368 if (pcie->soc->has_gen2)
2369 tegra_pcie_change_link_speed(pcie);
2370}
2371
2372static void tegra_pcie_disable_ports(struct tegra_pcie *pcie)
2373{
2374 struct tegra_pcie_port *port, *tmp;
2375
2376 reset_control_assert(pcie->pcie_xrst);
2377
2378 list_for_each_entry_safe(port, tmp, &pcie->ports, list)
2379 tegra_pcie_port_disable(port);
2380}
2381
2382static const struct tegra_pcie_port_soc tegra20_pcie_ports[] = {
2383 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2384 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2385};
2386
2387static const struct tegra_pcie_soc tegra20_pcie = {
2388 .num_ports = 2,
2389 .ports = tegra20_pcie_ports,
2390 .msi_base_shift = 0,
2391 .pads_pll_ctl = PADS_PLL_CTL_TEGRA20,
2392 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10,
2393 .pads_refclk_cfg0 = 0xfa5cfa5c,
2394 .has_pex_clkreq_en = false,
2395 .has_pex_bias_ctrl = false,
2396 .has_intr_prsnt_sense = false,
2397 .has_cml_clk = false,
2398 .has_gen2 = false,
2399 .force_pca_enable = false,
2400 .program_uphy = true,
2401 .update_clamp_threshold = false,
2402 .program_deskew_time = false,
2403 .update_fc_timer = false,
2404 .has_cache_bars = true,
2405 .ectl.enable = false,
2406};
2407
2408static const struct tegra_pcie_port_soc tegra30_pcie_ports[] = {
2409 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2410 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2411 { .pme.turnoff_bit = 16, .pme.ack_bit = 18 },
2412};
2413
2414static const struct tegra_pcie_soc tegra30_pcie = {
2415 .num_ports = 3,
2416 .ports = tegra30_pcie_ports,
2417 .msi_base_shift = 8,
2418 .afi_pex2_ctrl = 0x128,
2419 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2420 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2421 .pads_refclk_cfg0 = 0xfa5cfa5c,
2422 .pads_refclk_cfg1 = 0xfa5cfa5c,
2423 .has_pex_clkreq_en = true,
2424 .has_pex_bias_ctrl = true,
2425 .has_intr_prsnt_sense = true,
2426 .has_cml_clk = true,
2427 .has_gen2 = false,
2428 .force_pca_enable = false,
2429 .program_uphy = true,
2430 .update_clamp_threshold = false,
2431 .program_deskew_time = false,
2432 .update_fc_timer = false,
2433 .has_cache_bars = false,
2434 .ectl.enable = false,
2435};
2436
2437static const struct tegra_pcie_soc tegra124_pcie = {
2438 .num_ports = 2,
2439 .ports = tegra20_pcie_ports,
2440 .msi_base_shift = 8,
2441 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2442 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2443 .pads_refclk_cfg0 = 0x44ac44ac,
2444 .has_pex_clkreq_en = true,
2445 .has_pex_bias_ctrl = true,
2446 .has_intr_prsnt_sense = true,
2447 .has_cml_clk = true,
2448 .has_gen2 = true,
2449 .force_pca_enable = false,
2450 .program_uphy = true,
2451 .update_clamp_threshold = true,
2452 .program_deskew_time = false,
2453 .update_fc_timer = false,
2454 .has_cache_bars = false,
2455 .ectl.enable = false,
2456};
2457
2458static const struct tegra_pcie_soc tegra210_pcie = {
2459 .num_ports = 2,
2460 .ports = tegra20_pcie_ports,
2461 .msi_base_shift = 8,
2462 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2463 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2464 .pads_refclk_cfg0 = 0x90b890b8,
2465 /* FC threshold is bit[25:18] */
2466 .update_fc_threshold = 0x01800000,
2467 .has_pex_clkreq_en = true,
2468 .has_pex_bias_ctrl = true,
2469 .has_intr_prsnt_sense = true,
2470 .has_cml_clk = true,
2471 .has_gen2 = true,
2472 .force_pca_enable = true,
2473 .program_uphy = true,
2474 .update_clamp_threshold = true,
2475 .program_deskew_time = true,
2476 .update_fc_timer = true,
2477 .has_cache_bars = false,
2478 .ectl = {
2479 .regs = {
2480 .rp_ectl_2_r1 = 0x0000000f,
2481 .rp_ectl_4_r1 = 0x00000067,
2482 .rp_ectl_5_r1 = 0x55010000,
2483 .rp_ectl_6_r1 = 0x00000001,
2484 .rp_ectl_2_r2 = 0x0000008f,
2485 .rp_ectl_4_r2 = 0x000000c7,
2486 .rp_ectl_5_r2 = 0x55010000,
2487 .rp_ectl_6_r2 = 0x00000001,
2488 },
2489 .enable = true,
2490 },
2491};
2492
2493static const struct tegra_pcie_port_soc tegra186_pcie_ports[] = {
2494 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2495 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2496 { .pme.turnoff_bit = 12, .pme.ack_bit = 14 },
2497};
2498
2499static const struct tegra_pcie_soc tegra186_pcie = {
2500 .num_ports = 3,
2501 .ports = tegra186_pcie_ports,
2502 .msi_base_shift = 8,
2503 .afi_pex2_ctrl = 0x19c,
2504 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2505 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2506 .pads_refclk_cfg0 = 0x80b880b8,
2507 .pads_refclk_cfg1 = 0x000480b8,
2508 .has_pex_clkreq_en = true,
2509 .has_pex_bias_ctrl = true,
2510 .has_intr_prsnt_sense = true,
2511 .has_cml_clk = false,
2512 .has_gen2 = true,
2513 .force_pca_enable = false,
2514 .program_uphy = false,
2515 .update_clamp_threshold = false,
2516 .program_deskew_time = false,
2517 .update_fc_timer = false,
2518 .has_cache_bars = false,
2519 .ectl.enable = false,
2520};
2521
2522static const struct of_device_id tegra_pcie_of_match[] = {
2523 { .compatible = "nvidia,tegra186-pcie", .data = &tegra186_pcie },
2524 { .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie },
2525 { .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
2526 { .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
2527 { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
2528 { },
2529};
2530MODULE_DEVICE_TABLE(of, tegra_pcie_of_match);
2531
2532static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos)
2533{
2534 struct tegra_pcie *pcie = s->private;
2535
2536 if (list_empty(&pcie->ports))
2537 return NULL;
2538
2539 seq_puts(s, "Index Status\n");
2540
2541 return seq_list_start(&pcie->ports, *pos);
2542}
2543
2544static void *tegra_pcie_ports_seq_next(struct seq_file *s, void *v, loff_t *pos)
2545{
2546 struct tegra_pcie *pcie = s->private;
2547
2548 return seq_list_next(v, &pcie->ports, pos);
2549}
2550
2551static void tegra_pcie_ports_seq_stop(struct seq_file *s, void *v)
2552{
2553}
2554
2555static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v)
2556{
2557 bool up = false, active = false;
2558 struct tegra_pcie_port *port;
2559 unsigned int value;
2560
2561 port = list_entry(v, struct tegra_pcie_port, list);
2562
2563 value = readl(port->base + RP_VEND_XP);
2564
2565 if (value & RP_VEND_XP_DL_UP)
2566 up = true;
2567
2568 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2569
2570 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2571 active = true;
2572
2573 seq_printf(s, "%2u ", port->index);
2574
2575 if (up)
2576 seq_puts(s, "up");
2577
2578 if (active) {
2579 if (up)
2580 seq_puts(s, ", ");
2581
2582 seq_puts(s, "active");
2583 }
2584
2585 seq_puts(s, "\n");
2586 return 0;
2587}
2588
2589static const struct seq_operations tegra_pcie_ports_sops = {
2590 .start = tegra_pcie_ports_seq_start,
2591 .next = tegra_pcie_ports_seq_next,
2592 .stop = tegra_pcie_ports_seq_stop,
2593 .show = tegra_pcie_ports_seq_show,
2594};
2595
2596DEFINE_SEQ_ATTRIBUTE(tegra_pcie_ports);
2597
2598static void tegra_pcie_debugfs_exit(struct tegra_pcie *pcie)
2599{
2600 debugfs_remove_recursive(pcie->debugfs);
2601 pcie->debugfs = NULL;
2602}
2603
2604static void tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
2605{
2606 pcie->debugfs = debugfs_create_dir("pcie", NULL);
2607
2608 debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs, pcie,
2609 &tegra_pcie_ports_fops);
2610}
2611
2612static int tegra_pcie_probe(struct platform_device *pdev)
2613{
2614 struct device *dev = &pdev->dev;
2615 struct pci_host_bridge *host;
2616 struct tegra_pcie *pcie;
2617 int err;
2618
2619 host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
2620 if (!host)
2621 return -ENOMEM;
2622
2623 pcie = pci_host_bridge_priv(host);
2624 host->sysdata = pcie;
2625 platform_set_drvdata(pdev, pcie);
2626
2627 pcie->soc = of_device_get_match_data(dev);
2628 INIT_LIST_HEAD(&pcie->ports);
2629 pcie->dev = dev;
2630
2631 err = tegra_pcie_parse_dt(pcie);
2632 if (err < 0)
2633 return err;
2634
2635 err = tegra_pcie_get_resources(pcie);
2636 if (err < 0) {
2637 dev_err(dev, "failed to request resources: %d\n", err);
2638 return err;
2639 }
2640
2641 err = tegra_pcie_msi_setup(pcie);
2642 if (err < 0) {
2643 dev_err(dev, "failed to enable MSI support: %d\n", err);
2644 goto put_resources;
2645 }
2646
2647 pm_runtime_enable(pcie->dev);
2648 err = pm_runtime_get_sync(pcie->dev);
2649 if (err < 0) {
2650 dev_err(dev, "fail to enable pcie controller: %d\n", err);
2651 goto pm_runtime_put;
2652 }
2653
2654 host->ops = &tegra_pcie_ops;
2655 host->map_irq = tegra_pcie_map_irq;
2656
2657 err = pci_host_probe(host);
2658 if (err < 0) {
2659 dev_err(dev, "failed to register host: %d\n", err);
2660 goto pm_runtime_put;
2661 }
2662
2663 if (IS_ENABLED(CONFIG_DEBUG_FS))
2664 tegra_pcie_debugfs_init(pcie);
2665
2666 return 0;
2667
2668pm_runtime_put:
2669 pm_runtime_put_sync(pcie->dev);
2670 pm_runtime_disable(pcie->dev);
2671 tegra_pcie_msi_teardown(pcie);
2672put_resources:
2673 tegra_pcie_put_resources(pcie);
2674 return err;
2675}
2676
2677static void tegra_pcie_remove(struct platform_device *pdev)
2678{
2679 struct tegra_pcie *pcie = platform_get_drvdata(pdev);
2680 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
2681 struct tegra_pcie_port *port, *tmp;
2682
2683 if (IS_ENABLED(CONFIG_DEBUG_FS))
2684 tegra_pcie_debugfs_exit(pcie);
2685
2686 pci_stop_root_bus(host->bus);
2687 pci_remove_root_bus(host->bus);
2688 pm_runtime_put_sync(pcie->dev);
2689 pm_runtime_disable(pcie->dev);
2690
2691 if (IS_ENABLED(CONFIG_PCI_MSI))
2692 tegra_pcie_msi_teardown(pcie);
2693
2694 tegra_pcie_put_resources(pcie);
2695
2696 list_for_each_entry_safe(port, tmp, &pcie->ports, list)
2697 tegra_pcie_port_free(port);
2698}
2699
2700static int tegra_pcie_pm_suspend(struct device *dev)
2701{
2702 struct tegra_pcie *pcie = dev_get_drvdata(dev);
2703 struct tegra_pcie_port *port;
2704 int err;
2705
2706 list_for_each_entry(port, &pcie->ports, list)
2707 tegra_pcie_pme_turnoff(port);
2708
2709 tegra_pcie_disable_ports(pcie);
2710
2711 /*
2712 * AFI_INTR is unmasked in tegra_pcie_enable_controller(), mask it to
2713 * avoid unwanted interrupts raised by AFI after pex_rst is asserted.
2714 */
2715 tegra_pcie_disable_interrupts(pcie);
2716
2717 if (pcie->soc->program_uphy) {
2718 err = tegra_pcie_phy_power_off(pcie);
2719 if (err < 0)
2720 dev_err(dev, "failed to power off PHY(s): %d\n", err);
2721 }
2722
2723 reset_control_assert(pcie->pex_rst);
2724 clk_disable_unprepare(pcie->pex_clk);
2725
2726 if (IS_ENABLED(CONFIG_PCI_MSI))
2727 tegra_pcie_disable_msi(pcie);
2728
2729 pinctrl_pm_select_idle_state(dev);
2730 tegra_pcie_power_off(pcie);
2731
2732 return 0;
2733}
2734
2735static int tegra_pcie_pm_resume(struct device *dev)
2736{
2737 struct tegra_pcie *pcie = dev_get_drvdata(dev);
2738 int err;
2739
2740 err = tegra_pcie_power_on(pcie);
2741 if (err) {
2742 dev_err(dev, "tegra pcie power on fail: %d\n", err);
2743 return err;
2744 }
2745
2746 err = pinctrl_pm_select_default_state(dev);
2747 if (err < 0) {
2748 dev_err(dev, "failed to disable PCIe IO DPD: %d\n", err);
2749 goto poweroff;
2750 }
2751
2752 tegra_pcie_enable_controller(pcie);
2753 tegra_pcie_setup_translations(pcie);
2754
2755 if (IS_ENABLED(CONFIG_PCI_MSI))
2756 tegra_pcie_enable_msi(pcie);
2757
2758 err = clk_prepare_enable(pcie->pex_clk);
2759 if (err) {
2760 dev_err(dev, "failed to enable PEX clock: %d\n", err);
2761 goto pex_dpd_enable;
2762 }
2763
2764 reset_control_deassert(pcie->pex_rst);
2765
2766 if (pcie->soc->program_uphy) {
2767 err = tegra_pcie_phy_power_on(pcie);
2768 if (err < 0) {
2769 dev_err(dev, "failed to power on PHY(s): %d\n", err);
2770 goto disable_pex_clk;
2771 }
2772 }
2773
2774 tegra_pcie_apply_pad_settings(pcie);
2775 tegra_pcie_enable_ports(pcie);
2776
2777 return 0;
2778
2779disable_pex_clk:
2780 reset_control_assert(pcie->pex_rst);
2781 clk_disable_unprepare(pcie->pex_clk);
2782pex_dpd_enable:
2783 pinctrl_pm_select_idle_state(dev);
2784poweroff:
2785 tegra_pcie_power_off(pcie);
2786
2787 return err;
2788}
2789
2790static const struct dev_pm_ops tegra_pcie_pm_ops = {
2791 RUNTIME_PM_OPS(tegra_pcie_pm_suspend, tegra_pcie_pm_resume, NULL)
2792 NOIRQ_SYSTEM_SLEEP_PM_OPS(tegra_pcie_pm_suspend, tegra_pcie_pm_resume)
2793};
2794
2795static struct platform_driver tegra_pcie_driver = {
2796 .driver = {
2797 .name = "tegra-pcie",
2798 .of_match_table = tegra_pcie_of_match,
2799 .suppress_bind_attrs = true,
2800 .pm = &tegra_pcie_pm_ops,
2801 },
2802 .probe = tegra_pcie_probe,
2803 .remove = tegra_pcie_remove,
2804};
2805module_platform_driver(tegra_pcie_driver);
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PCIe host controller driver for Tegra SoCs
4 *
5 * Copyright (c) 2010, CompuLab, Ltd.
6 * Author: Mike Rapoport <mike@compulab.co.il>
7 *
8 * Based on NVIDIA PCIe driver
9 * Copyright (c) 2008-2009, NVIDIA Corporation.
10 *
11 * Bits taken from arch/arm/mach-dove/pcie.c
12 *
13 * Author: Thierry Reding <treding@nvidia.com>
14 */
15
16#include <linux/clk.h>
17#include <linux/debugfs.h>
18#include <linux/delay.h>
19#include <linux/export.h>
20#include <linux/gpio/consumer.h>
21#include <linux/interrupt.h>
22#include <linux/iopoll.h>
23#include <linux/irq.h>
24#include <linux/irqdomain.h>
25#include <linux/kernel.h>
26#include <linux/init.h>
27#include <linux/module.h>
28#include <linux/msi.h>
29#include <linux/of_address.h>
30#include <linux/of_pci.h>
31#include <linux/of_platform.h>
32#include <linux/pci.h>
33#include <linux/phy/phy.h>
34#include <linux/pinctrl/consumer.h>
35#include <linux/platform_device.h>
36#include <linux/reset.h>
37#include <linux/sizes.h>
38#include <linux/slab.h>
39#include <linux/vmalloc.h>
40#include <linux/regulator/consumer.h>
41
42#include <soc/tegra/cpuidle.h>
43#include <soc/tegra/pmc.h>
44
45#include "../pci.h"
46
47#define INT_PCI_MSI_NR (8 * 32)
48
49/* register definitions */
50
51#define AFI_AXI_BAR0_SZ 0x00
52#define AFI_AXI_BAR1_SZ 0x04
53#define AFI_AXI_BAR2_SZ 0x08
54#define AFI_AXI_BAR3_SZ 0x0c
55#define AFI_AXI_BAR4_SZ 0x10
56#define AFI_AXI_BAR5_SZ 0x14
57
58#define AFI_AXI_BAR0_START 0x18
59#define AFI_AXI_BAR1_START 0x1c
60#define AFI_AXI_BAR2_START 0x20
61#define AFI_AXI_BAR3_START 0x24
62#define AFI_AXI_BAR4_START 0x28
63#define AFI_AXI_BAR5_START 0x2c
64
65#define AFI_FPCI_BAR0 0x30
66#define AFI_FPCI_BAR1 0x34
67#define AFI_FPCI_BAR2 0x38
68#define AFI_FPCI_BAR3 0x3c
69#define AFI_FPCI_BAR4 0x40
70#define AFI_FPCI_BAR5 0x44
71
72#define AFI_CACHE_BAR0_SZ 0x48
73#define AFI_CACHE_BAR0_ST 0x4c
74#define AFI_CACHE_BAR1_SZ 0x50
75#define AFI_CACHE_BAR1_ST 0x54
76
77#define AFI_MSI_BAR_SZ 0x60
78#define AFI_MSI_FPCI_BAR_ST 0x64
79#define AFI_MSI_AXI_BAR_ST 0x68
80
81#define AFI_MSI_VEC0 0x6c
82#define AFI_MSI_VEC1 0x70
83#define AFI_MSI_VEC2 0x74
84#define AFI_MSI_VEC3 0x78
85#define AFI_MSI_VEC4 0x7c
86#define AFI_MSI_VEC5 0x80
87#define AFI_MSI_VEC6 0x84
88#define AFI_MSI_VEC7 0x88
89
90#define AFI_MSI_EN_VEC0 0x8c
91#define AFI_MSI_EN_VEC1 0x90
92#define AFI_MSI_EN_VEC2 0x94
93#define AFI_MSI_EN_VEC3 0x98
94#define AFI_MSI_EN_VEC4 0x9c
95#define AFI_MSI_EN_VEC5 0xa0
96#define AFI_MSI_EN_VEC6 0xa4
97#define AFI_MSI_EN_VEC7 0xa8
98
99#define AFI_CONFIGURATION 0xac
100#define AFI_CONFIGURATION_EN_FPCI (1 << 0)
101#define AFI_CONFIGURATION_CLKEN_OVERRIDE (1 << 31)
102
103#define AFI_FPCI_ERROR_MASKS 0xb0
104
105#define AFI_INTR_MASK 0xb4
106#define AFI_INTR_MASK_INT_MASK (1 << 0)
107#define AFI_INTR_MASK_MSI_MASK (1 << 8)
108
109#define AFI_INTR_CODE 0xb8
110#define AFI_INTR_CODE_MASK 0xf
111#define AFI_INTR_INI_SLAVE_ERROR 1
112#define AFI_INTR_INI_DECODE_ERROR 2
113#define AFI_INTR_TARGET_ABORT 3
114#define AFI_INTR_MASTER_ABORT 4
115#define AFI_INTR_INVALID_WRITE 5
116#define AFI_INTR_LEGACY 6
117#define AFI_INTR_FPCI_DECODE_ERROR 7
118#define AFI_INTR_AXI_DECODE_ERROR 8
119#define AFI_INTR_FPCI_TIMEOUT 9
120#define AFI_INTR_PE_PRSNT_SENSE 10
121#define AFI_INTR_PE_CLKREQ_SENSE 11
122#define AFI_INTR_CLKCLAMP_SENSE 12
123#define AFI_INTR_RDY4PD_SENSE 13
124#define AFI_INTR_P2P_ERROR 14
125
126#define AFI_INTR_SIGNATURE 0xbc
127#define AFI_UPPER_FPCI_ADDRESS 0xc0
128#define AFI_SM_INTR_ENABLE 0xc4
129#define AFI_SM_INTR_INTA_ASSERT (1 << 0)
130#define AFI_SM_INTR_INTB_ASSERT (1 << 1)
131#define AFI_SM_INTR_INTC_ASSERT (1 << 2)
132#define AFI_SM_INTR_INTD_ASSERT (1 << 3)
133#define AFI_SM_INTR_INTA_DEASSERT (1 << 4)
134#define AFI_SM_INTR_INTB_DEASSERT (1 << 5)
135#define AFI_SM_INTR_INTC_DEASSERT (1 << 6)
136#define AFI_SM_INTR_INTD_DEASSERT (1 << 7)
137
138#define AFI_AFI_INTR_ENABLE 0xc8
139#define AFI_INTR_EN_INI_SLVERR (1 << 0)
140#define AFI_INTR_EN_INI_DECERR (1 << 1)
141#define AFI_INTR_EN_TGT_SLVERR (1 << 2)
142#define AFI_INTR_EN_TGT_DECERR (1 << 3)
143#define AFI_INTR_EN_TGT_WRERR (1 << 4)
144#define AFI_INTR_EN_DFPCI_DECERR (1 << 5)
145#define AFI_INTR_EN_AXI_DECERR (1 << 6)
146#define AFI_INTR_EN_FPCI_TIMEOUT (1 << 7)
147#define AFI_INTR_EN_PRSNT_SENSE (1 << 8)
148
149#define AFI_PCIE_PME 0xf0
150
151#define AFI_PCIE_CONFIG 0x0f8
152#define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1))
153#define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0xe
154#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20)
155#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE (0x0 << 20)
156#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420 (0x0 << 20)
157#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1 (0x0 << 20)
158#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401 (0x0 << 20)
159#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL (0x1 << 20)
160#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222 (0x1 << 20)
161#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1 (0x1 << 20)
162#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211 (0x1 << 20)
163#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20)
164#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111 (0x2 << 20)
165#define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(x) (1 << ((x) + 29))
166#define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL (0x7 << 29)
167
168#define AFI_FUSE 0x104
169#define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2)
170
171#define AFI_PEX0_CTRL 0x110
172#define AFI_PEX1_CTRL 0x118
173#define AFI_PEX_CTRL_RST (1 << 0)
174#define AFI_PEX_CTRL_CLKREQ_EN (1 << 1)
175#define AFI_PEX_CTRL_REFCLK_EN (1 << 3)
176#define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4)
177
178#define AFI_PLLE_CONTROL 0x160
179#define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9)
180#define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1)
181
182#define AFI_PEXBIAS_CTRL_0 0x168
183
184#define RP_ECTL_2_R1 0x00000e84
185#define RP_ECTL_2_R1_RX_CTLE_1C_MASK 0xffff
186
187#define RP_ECTL_4_R1 0x00000e8c
188#define RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK (0xffff << 16)
189#define RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT 16
190
191#define RP_ECTL_5_R1 0x00000e90
192#define RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK 0xffffffff
193
194#define RP_ECTL_6_R1 0x00000e94
195#define RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK 0xffffffff
196
197#define RP_ECTL_2_R2 0x00000ea4
198#define RP_ECTL_2_R2_RX_CTLE_1C_MASK 0xffff
199
200#define RP_ECTL_4_R2 0x00000eac
201#define RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK (0xffff << 16)
202#define RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT 16
203
204#define RP_ECTL_5_R2 0x00000eb0
205#define RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK 0xffffffff
206
207#define RP_ECTL_6_R2 0x00000eb4
208#define RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK 0xffffffff
209
210#define RP_VEND_XP 0x00000f00
211#define RP_VEND_XP_DL_UP (1 << 30)
212#define RP_VEND_XP_OPPORTUNISTIC_ACK (1 << 27)
213#define RP_VEND_XP_OPPORTUNISTIC_UPDATEFC (1 << 28)
214#define RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK (0xff << 18)
215
216#define RP_VEND_CTL0 0x00000f44
217#define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK (0xf << 12)
218#define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH (0x9 << 12)
219
220#define RP_VEND_CTL1 0x00000f48
221#define RP_VEND_CTL1_ERPT (1 << 13)
222
223#define RP_VEND_XP_BIST 0x00000f4c
224#define RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE (1 << 28)
225
226#define RP_VEND_CTL2 0x00000fa8
227#define RP_VEND_CTL2_PCA_ENABLE (1 << 7)
228
229#define RP_PRIV_MISC 0x00000fe0
230#define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0)
231#define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0)
232#define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK (0x7f << 16)
233#define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD (0xf << 16)
234#define RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE (1 << 23)
235#define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK (0x7f << 24)
236#define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD (0xf << 24)
237#define RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE (1 << 31)
238
239#define RP_LINK_CONTROL_STATUS 0x00000090
240#define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000
241#define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000
242
243#define RP_LINK_CONTROL_STATUS_2 0x000000b0
244
245#define PADS_CTL_SEL 0x0000009c
246
247#define PADS_CTL 0x000000a0
248#define PADS_CTL_IDDQ_1L (1 << 0)
249#define PADS_CTL_TX_DATA_EN_1L (1 << 6)
250#define PADS_CTL_RX_DATA_EN_1L (1 << 10)
251
252#define PADS_PLL_CTL_TEGRA20 0x000000b8
253#define PADS_PLL_CTL_TEGRA30 0x000000b4
254#define PADS_PLL_CTL_RST_B4SM (1 << 1)
255#define PADS_PLL_CTL_LOCKDET (1 << 8)
256#define PADS_PLL_CTL_REFCLK_MASK (0x3 << 16)
257#define PADS_PLL_CTL_REFCLK_INTERNAL_CML (0 << 16)
258#define PADS_PLL_CTL_REFCLK_INTERNAL_CMOS (1 << 16)
259#define PADS_PLL_CTL_REFCLK_EXTERNAL (2 << 16)
260#define PADS_PLL_CTL_TXCLKREF_MASK (0x1 << 20)
261#define PADS_PLL_CTL_TXCLKREF_DIV10 (0 << 20)
262#define PADS_PLL_CTL_TXCLKREF_DIV5 (1 << 20)
263#define PADS_PLL_CTL_TXCLKREF_BUF_EN (1 << 22)
264
265#define PADS_REFCLK_CFG0 0x000000c8
266#define PADS_REFCLK_CFG1 0x000000cc
267#define PADS_REFCLK_BIAS 0x000000d0
268
269/*
270 * Fields in PADS_REFCLK_CFG*. Those registers form an array of 16-bit
271 * entries, one entry per PCIe port. These field definitions and desired
272 * values aren't in the TRM, but do come from NVIDIA.
273 */
274#define PADS_REFCLK_CFG_TERM_SHIFT 2 /* 6:2 */
275#define PADS_REFCLK_CFG_E_TERM_SHIFT 7
276#define PADS_REFCLK_CFG_PREDI_SHIFT 8 /* 11:8 */
277#define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */
278
279#define PME_ACK_TIMEOUT 10000
280#define LINK_RETRAIN_TIMEOUT 100000 /* in usec */
281
282struct tegra_msi {
283 struct msi_controller chip;
284 DECLARE_BITMAP(used, INT_PCI_MSI_NR);
285 struct irq_domain *domain;
286 struct mutex lock;
287 void *virt;
288 dma_addr_t phys;
289 int irq;
290};
291
292/* used to differentiate between Tegra SoC generations */
293struct tegra_pcie_port_soc {
294 struct {
295 u8 turnoff_bit;
296 u8 ack_bit;
297 } pme;
298};
299
300struct tegra_pcie_soc {
301 unsigned int num_ports;
302 const struct tegra_pcie_port_soc *ports;
303 unsigned int msi_base_shift;
304 unsigned long afi_pex2_ctrl;
305 u32 pads_pll_ctl;
306 u32 tx_ref_sel;
307 u32 pads_refclk_cfg0;
308 u32 pads_refclk_cfg1;
309 u32 update_fc_threshold;
310 bool has_pex_clkreq_en;
311 bool has_pex_bias_ctrl;
312 bool has_intr_prsnt_sense;
313 bool has_cml_clk;
314 bool has_gen2;
315 bool force_pca_enable;
316 bool program_uphy;
317 bool update_clamp_threshold;
318 bool program_deskew_time;
319 bool update_fc_timer;
320 bool has_cache_bars;
321 struct {
322 struct {
323 u32 rp_ectl_2_r1;
324 u32 rp_ectl_4_r1;
325 u32 rp_ectl_5_r1;
326 u32 rp_ectl_6_r1;
327 u32 rp_ectl_2_r2;
328 u32 rp_ectl_4_r2;
329 u32 rp_ectl_5_r2;
330 u32 rp_ectl_6_r2;
331 } regs;
332 bool enable;
333 } ectl;
334};
335
336static inline struct tegra_msi *to_tegra_msi(struct msi_controller *chip)
337{
338 return container_of(chip, struct tegra_msi, chip);
339}
340
341struct tegra_pcie {
342 struct device *dev;
343
344 void __iomem *pads;
345 void __iomem *afi;
346 void __iomem *cfg;
347 int irq;
348
349 struct resource cs;
350
351 struct clk *pex_clk;
352 struct clk *afi_clk;
353 struct clk *pll_e;
354 struct clk *cml_clk;
355
356 struct reset_control *pex_rst;
357 struct reset_control *afi_rst;
358 struct reset_control *pcie_xrst;
359
360 bool legacy_phy;
361 struct phy *phy;
362
363 struct tegra_msi msi;
364
365 struct list_head ports;
366 u32 xbar_config;
367
368 struct regulator_bulk_data *supplies;
369 unsigned int num_supplies;
370
371 const struct tegra_pcie_soc *soc;
372 struct dentry *debugfs;
373};
374
375struct tegra_pcie_port {
376 struct tegra_pcie *pcie;
377 struct device_node *np;
378 struct list_head list;
379 struct resource regs;
380 void __iomem *base;
381 unsigned int index;
382 unsigned int lanes;
383
384 struct phy **phys;
385
386 struct gpio_desc *reset_gpio;
387};
388
389struct tegra_pcie_bus {
390 struct list_head list;
391 unsigned int nr;
392};
393
394static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
395 unsigned long offset)
396{
397 writel(value, pcie->afi + offset);
398}
399
400static inline u32 afi_readl(struct tegra_pcie *pcie, unsigned long offset)
401{
402 return readl(pcie->afi + offset);
403}
404
405static inline void pads_writel(struct tegra_pcie *pcie, u32 value,
406 unsigned long offset)
407{
408 writel(value, pcie->pads + offset);
409}
410
411static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
412{
413 return readl(pcie->pads + offset);
414}
415
416/*
417 * The configuration space mapping on Tegra is somewhat similar to the ECAM
418 * defined by PCIe. However it deviates a bit in how the 4 bits for extended
419 * register accesses are mapped:
420 *
421 * [27:24] extended register number
422 * [23:16] bus number
423 * [15:11] device number
424 * [10: 8] function number
425 * [ 7: 0] register number
426 *
427 * Mapping the whole extended configuration space would require 256 MiB of
428 * virtual address space, only a small part of which will actually be used.
429 *
430 * To work around this, a 4 KiB region is used to generate the required
431 * configuration transaction with relevant B:D:F and register offset values.
432 * This is achieved by dynamically programming base address and size of
433 * AFI_AXI_BAR used for end point config space mapping to make sure that the
434 * address (access to which generates correct config transaction) falls in
435 * this 4 KiB region.
436 */
437static unsigned int tegra_pcie_conf_offset(u8 bus, unsigned int devfn,
438 unsigned int where)
439{
440 return ((where & 0xf00) << 16) | (bus << 16) | (PCI_SLOT(devfn) << 11) |
441 (PCI_FUNC(devfn) << 8) | (where & 0xff);
442}
443
444static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
445 unsigned int devfn,
446 int where)
447{
448 struct tegra_pcie *pcie = bus->sysdata;
449 void __iomem *addr = NULL;
450
451 if (bus->number == 0) {
452 unsigned int slot = PCI_SLOT(devfn);
453 struct tegra_pcie_port *port;
454
455 list_for_each_entry(port, &pcie->ports, list) {
456 if (port->index + 1 == slot) {
457 addr = port->base + (where & ~3);
458 break;
459 }
460 }
461 } else {
462 unsigned int offset;
463 u32 base;
464
465 offset = tegra_pcie_conf_offset(bus->number, devfn, where);
466
467 /* move 4 KiB window to offset within the FPCI region */
468 base = 0xfe100000 + ((offset & ~(SZ_4K - 1)) >> 8);
469 afi_writel(pcie, base, AFI_FPCI_BAR0);
470
471 /* move to correct offset within the 4 KiB page */
472 addr = pcie->cfg + (offset & (SZ_4K - 1));
473 }
474
475 return addr;
476}
477
478static int tegra_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
479 int where, int size, u32 *value)
480{
481 if (bus->number == 0)
482 return pci_generic_config_read32(bus, devfn, where, size,
483 value);
484
485 return pci_generic_config_read(bus, devfn, where, size, value);
486}
487
488static int tegra_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
489 int where, int size, u32 value)
490{
491 if (bus->number == 0)
492 return pci_generic_config_write32(bus, devfn, where, size,
493 value);
494
495 return pci_generic_config_write(bus, devfn, where, size, value);
496}
497
498static struct pci_ops tegra_pcie_ops = {
499 .map_bus = tegra_pcie_map_bus,
500 .read = tegra_pcie_config_read,
501 .write = tegra_pcie_config_write,
502};
503
504static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port)
505{
506 const struct tegra_pcie_soc *soc = port->pcie->soc;
507 unsigned long ret = 0;
508
509 switch (port->index) {
510 case 0:
511 ret = AFI_PEX0_CTRL;
512 break;
513
514 case 1:
515 ret = AFI_PEX1_CTRL;
516 break;
517
518 case 2:
519 ret = soc->afi_pex2_ctrl;
520 break;
521 }
522
523 return ret;
524}
525
526static void tegra_pcie_port_reset(struct tegra_pcie_port *port)
527{
528 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
529 unsigned long value;
530
531 /* pulse reset signal */
532 if (port->reset_gpio) {
533 gpiod_set_value(port->reset_gpio, 1);
534 } else {
535 value = afi_readl(port->pcie, ctrl);
536 value &= ~AFI_PEX_CTRL_RST;
537 afi_writel(port->pcie, value, ctrl);
538 }
539
540 usleep_range(1000, 2000);
541
542 if (port->reset_gpio) {
543 gpiod_set_value(port->reset_gpio, 0);
544 } else {
545 value = afi_readl(port->pcie, ctrl);
546 value |= AFI_PEX_CTRL_RST;
547 afi_writel(port->pcie, value, ctrl);
548 }
549}
550
551static void tegra_pcie_enable_rp_features(struct tegra_pcie_port *port)
552{
553 const struct tegra_pcie_soc *soc = port->pcie->soc;
554 u32 value;
555
556 /* Enable AER capability */
557 value = readl(port->base + RP_VEND_CTL1);
558 value |= RP_VEND_CTL1_ERPT;
559 writel(value, port->base + RP_VEND_CTL1);
560
561 /* Optimal settings to enhance bandwidth */
562 value = readl(port->base + RP_VEND_XP);
563 value |= RP_VEND_XP_OPPORTUNISTIC_ACK;
564 value |= RP_VEND_XP_OPPORTUNISTIC_UPDATEFC;
565 writel(value, port->base + RP_VEND_XP);
566
567 /*
568 * LTSSM will wait for DLLP to finish before entering L1 or L2,
569 * to avoid truncation of PM messages which results in receiver errors
570 */
571 value = readl(port->base + RP_VEND_XP_BIST);
572 value |= RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE;
573 writel(value, port->base + RP_VEND_XP_BIST);
574
575 value = readl(port->base + RP_PRIV_MISC);
576 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE;
577 value |= RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE;
578
579 if (soc->update_clamp_threshold) {
580 value &= ~(RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK |
581 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK);
582 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD |
583 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD;
584 }
585
586 writel(value, port->base + RP_PRIV_MISC);
587}
588
589static void tegra_pcie_program_ectl_settings(struct tegra_pcie_port *port)
590{
591 const struct tegra_pcie_soc *soc = port->pcie->soc;
592 u32 value;
593
594 value = readl(port->base + RP_ECTL_2_R1);
595 value &= ~RP_ECTL_2_R1_RX_CTLE_1C_MASK;
596 value |= soc->ectl.regs.rp_ectl_2_r1;
597 writel(value, port->base + RP_ECTL_2_R1);
598
599 value = readl(port->base + RP_ECTL_4_R1);
600 value &= ~RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK;
601 value |= soc->ectl.regs.rp_ectl_4_r1 <<
602 RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT;
603 writel(value, port->base + RP_ECTL_4_R1);
604
605 value = readl(port->base + RP_ECTL_5_R1);
606 value &= ~RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK;
607 value |= soc->ectl.regs.rp_ectl_5_r1;
608 writel(value, port->base + RP_ECTL_5_R1);
609
610 value = readl(port->base + RP_ECTL_6_R1);
611 value &= ~RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK;
612 value |= soc->ectl.regs.rp_ectl_6_r1;
613 writel(value, port->base + RP_ECTL_6_R1);
614
615 value = readl(port->base + RP_ECTL_2_R2);
616 value &= ~RP_ECTL_2_R2_RX_CTLE_1C_MASK;
617 value |= soc->ectl.regs.rp_ectl_2_r2;
618 writel(value, port->base + RP_ECTL_2_R2);
619
620 value = readl(port->base + RP_ECTL_4_R2);
621 value &= ~RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK;
622 value |= soc->ectl.regs.rp_ectl_4_r2 <<
623 RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT;
624 writel(value, port->base + RP_ECTL_4_R2);
625
626 value = readl(port->base + RP_ECTL_5_R2);
627 value &= ~RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK;
628 value |= soc->ectl.regs.rp_ectl_5_r2;
629 writel(value, port->base + RP_ECTL_5_R2);
630
631 value = readl(port->base + RP_ECTL_6_R2);
632 value &= ~RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK;
633 value |= soc->ectl.regs.rp_ectl_6_r2;
634 writel(value, port->base + RP_ECTL_6_R2);
635}
636
637static void tegra_pcie_apply_sw_fixup(struct tegra_pcie_port *port)
638{
639 const struct tegra_pcie_soc *soc = port->pcie->soc;
640 u32 value;
641
642 /*
643 * Sometimes link speed change from Gen2 to Gen1 fails due to
644 * instability in deskew logic on lane-0. Increase the deskew
645 * retry time to resolve this issue.
646 */
647 if (soc->program_deskew_time) {
648 value = readl(port->base + RP_VEND_CTL0);
649 value &= ~RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK;
650 value |= RP_VEND_CTL0_DSK_RST_PULSE_WIDTH;
651 writel(value, port->base + RP_VEND_CTL0);
652 }
653
654 if (soc->update_fc_timer) {
655 value = readl(port->base + RP_VEND_XP);
656 value &= ~RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK;
657 value |= soc->update_fc_threshold;
658 writel(value, port->base + RP_VEND_XP);
659 }
660
661 /*
662 * PCIe link doesn't come up with few legacy PCIe endpoints if
663 * root port advertises both Gen-1 and Gen-2 speeds in Tegra.
664 * Hence, the strategy followed here is to initially advertise
665 * only Gen-1 and after link is up, retrain link to Gen-2 speed
666 */
667 value = readl(port->base + RP_LINK_CONTROL_STATUS_2);
668 value &= ~PCI_EXP_LNKSTA_CLS;
669 value |= PCI_EXP_LNKSTA_CLS_2_5GB;
670 writel(value, port->base + RP_LINK_CONTROL_STATUS_2);
671}
672
673static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
674{
675 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
676 const struct tegra_pcie_soc *soc = port->pcie->soc;
677 unsigned long value;
678
679 /* enable reference clock */
680 value = afi_readl(port->pcie, ctrl);
681 value |= AFI_PEX_CTRL_REFCLK_EN;
682
683 if (soc->has_pex_clkreq_en)
684 value |= AFI_PEX_CTRL_CLKREQ_EN;
685
686 value |= AFI_PEX_CTRL_OVERRIDE_EN;
687
688 afi_writel(port->pcie, value, ctrl);
689
690 tegra_pcie_port_reset(port);
691
692 if (soc->force_pca_enable) {
693 value = readl(port->base + RP_VEND_CTL2);
694 value |= RP_VEND_CTL2_PCA_ENABLE;
695 writel(value, port->base + RP_VEND_CTL2);
696 }
697
698 tegra_pcie_enable_rp_features(port);
699
700 if (soc->ectl.enable)
701 tegra_pcie_program_ectl_settings(port);
702
703 tegra_pcie_apply_sw_fixup(port);
704}
705
706static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
707{
708 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
709 const struct tegra_pcie_soc *soc = port->pcie->soc;
710 unsigned long value;
711
712 /* assert port reset */
713 value = afi_readl(port->pcie, ctrl);
714 value &= ~AFI_PEX_CTRL_RST;
715 afi_writel(port->pcie, value, ctrl);
716
717 /* disable reference clock */
718 value = afi_readl(port->pcie, ctrl);
719
720 if (soc->has_pex_clkreq_en)
721 value &= ~AFI_PEX_CTRL_CLKREQ_EN;
722
723 value &= ~AFI_PEX_CTRL_REFCLK_EN;
724 afi_writel(port->pcie, value, ctrl);
725
726 /* disable PCIe port and set CLKREQ# as GPIO to allow PLLE power down */
727 value = afi_readl(port->pcie, AFI_PCIE_CONFIG);
728 value |= AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
729 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index);
730 afi_writel(port->pcie, value, AFI_PCIE_CONFIG);
731}
732
733static void tegra_pcie_port_free(struct tegra_pcie_port *port)
734{
735 struct tegra_pcie *pcie = port->pcie;
736 struct device *dev = pcie->dev;
737
738 devm_iounmap(dev, port->base);
739 devm_release_mem_region(dev, port->regs.start,
740 resource_size(&port->regs));
741 list_del(&port->list);
742 devm_kfree(dev, port);
743}
744
745/* Tegra PCIE root complex wrongly reports device class */
746static void tegra_pcie_fixup_class(struct pci_dev *dev)
747{
748 dev->class = PCI_CLASS_BRIDGE_PCI << 8;
749}
750DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_fixup_class);
751DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class);
752DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class);
753DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class);
754
755/* Tegra20 and Tegra30 PCIE requires relaxed ordering */
756static void tegra_pcie_relax_enable(struct pci_dev *dev)
757{
758 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
759}
760DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_relax_enable);
761DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_relax_enable);
762DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_relax_enable);
763DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_relax_enable);
764
765static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
766{
767 struct tegra_pcie *pcie = pdev->bus->sysdata;
768 int irq;
769
770 tegra_cpuidle_pcie_irqs_in_use();
771
772 irq = of_irq_parse_and_map_pci(pdev, slot, pin);
773 if (!irq)
774 irq = pcie->irq;
775
776 return irq;
777}
778
779static irqreturn_t tegra_pcie_isr(int irq, void *arg)
780{
781 const char *err_msg[] = {
782 "Unknown",
783 "AXI slave error",
784 "AXI decode error",
785 "Target abort",
786 "Master abort",
787 "Invalid write",
788 "Legacy interrupt",
789 "Response decoding error",
790 "AXI response decoding error",
791 "Transaction timeout",
792 "Slot present pin change",
793 "Slot clock request change",
794 "TMS clock ramp change",
795 "TMS ready for power down",
796 "Peer2Peer error",
797 };
798 struct tegra_pcie *pcie = arg;
799 struct device *dev = pcie->dev;
800 u32 code, signature;
801
802 code = afi_readl(pcie, AFI_INTR_CODE) & AFI_INTR_CODE_MASK;
803 signature = afi_readl(pcie, AFI_INTR_SIGNATURE);
804 afi_writel(pcie, 0, AFI_INTR_CODE);
805
806 if (code == AFI_INTR_LEGACY)
807 return IRQ_NONE;
808
809 if (code >= ARRAY_SIZE(err_msg))
810 code = 0;
811
812 /*
813 * do not pollute kernel log with master abort reports since they
814 * happen a lot during enumeration
815 */
816 if (code == AFI_INTR_MASTER_ABORT || code == AFI_INTR_PE_PRSNT_SENSE)
817 dev_dbg(dev, "%s, signature: %08x\n", err_msg[code], signature);
818 else
819 dev_err(dev, "%s, signature: %08x\n", err_msg[code], signature);
820
821 if (code == AFI_INTR_TARGET_ABORT || code == AFI_INTR_MASTER_ABORT ||
822 code == AFI_INTR_FPCI_DECODE_ERROR) {
823 u32 fpci = afi_readl(pcie, AFI_UPPER_FPCI_ADDRESS) & 0xff;
824 u64 address = (u64)fpci << 32 | (signature & 0xfffffffc);
825
826 if (code == AFI_INTR_MASTER_ABORT)
827 dev_dbg(dev, " FPCI address: %10llx\n", address);
828 else
829 dev_err(dev, " FPCI address: %10llx\n", address);
830 }
831
832 return IRQ_HANDLED;
833}
834
835/*
836 * FPCI map is as follows:
837 * - 0xfdfc000000: I/O space
838 * - 0xfdfe000000: type 0 configuration space
839 * - 0xfdff000000: type 1 configuration space
840 * - 0xfe00000000: type 0 extended configuration space
841 * - 0xfe10000000: type 1 extended configuration space
842 */
843static void tegra_pcie_setup_translations(struct tegra_pcie *pcie)
844{
845 u32 size;
846 struct resource_entry *entry;
847 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
848
849 /* Bar 0: type 1 extended configuration space */
850 size = resource_size(&pcie->cs);
851 afi_writel(pcie, pcie->cs.start, AFI_AXI_BAR0_START);
852 afi_writel(pcie, size >> 12, AFI_AXI_BAR0_SZ);
853
854 resource_list_for_each_entry(entry, &bridge->windows) {
855 u32 fpci_bar, axi_address;
856 struct resource *res = entry->res;
857
858 size = resource_size(res);
859
860 switch (resource_type(res)) {
861 case IORESOURCE_IO:
862 /* Bar 1: downstream IO bar */
863 fpci_bar = 0xfdfc0000;
864 axi_address = pci_pio_to_address(res->start);
865 afi_writel(pcie, axi_address, AFI_AXI_BAR1_START);
866 afi_writel(pcie, size >> 12, AFI_AXI_BAR1_SZ);
867 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR1);
868 break;
869 case IORESOURCE_MEM:
870 fpci_bar = (((res->start >> 12) & 0x0fffffff) << 4) | 0x1;
871 axi_address = res->start;
872
873 if (res->flags & IORESOURCE_PREFETCH) {
874 /* Bar 2: prefetchable memory BAR */
875 afi_writel(pcie, axi_address, AFI_AXI_BAR2_START);
876 afi_writel(pcie, size >> 12, AFI_AXI_BAR2_SZ);
877 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR2);
878
879 } else {
880 /* Bar 3: non prefetchable memory BAR */
881 afi_writel(pcie, axi_address, AFI_AXI_BAR3_START);
882 afi_writel(pcie, size >> 12, AFI_AXI_BAR3_SZ);
883 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR3);
884 }
885 break;
886 }
887 }
888
889 /* NULL out the remaining BARs as they are not used */
890 afi_writel(pcie, 0, AFI_AXI_BAR4_START);
891 afi_writel(pcie, 0, AFI_AXI_BAR4_SZ);
892 afi_writel(pcie, 0, AFI_FPCI_BAR4);
893
894 afi_writel(pcie, 0, AFI_AXI_BAR5_START);
895 afi_writel(pcie, 0, AFI_AXI_BAR5_SZ);
896 afi_writel(pcie, 0, AFI_FPCI_BAR5);
897
898 if (pcie->soc->has_cache_bars) {
899 /* map all upstream transactions as uncached */
900 afi_writel(pcie, 0, AFI_CACHE_BAR0_ST);
901 afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ);
902 afi_writel(pcie, 0, AFI_CACHE_BAR1_ST);
903 afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ);
904 }
905
906 /* MSI translations are setup only when needed */
907 afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST);
908 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
909 afi_writel(pcie, 0, AFI_MSI_AXI_BAR_ST);
910 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
911}
912
913static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout)
914{
915 const struct tegra_pcie_soc *soc = pcie->soc;
916 u32 value;
917
918 timeout = jiffies + msecs_to_jiffies(timeout);
919
920 while (time_before(jiffies, timeout)) {
921 value = pads_readl(pcie, soc->pads_pll_ctl);
922 if (value & PADS_PLL_CTL_LOCKDET)
923 return 0;
924 }
925
926 return -ETIMEDOUT;
927}
928
929static int tegra_pcie_phy_enable(struct tegra_pcie *pcie)
930{
931 struct device *dev = pcie->dev;
932 const struct tegra_pcie_soc *soc = pcie->soc;
933 u32 value;
934 int err;
935
936 /* initialize internal PHY, enable up to 16 PCIE lanes */
937 pads_writel(pcie, 0x0, PADS_CTL_SEL);
938
939 /* override IDDQ to 1 on all 4 lanes */
940 value = pads_readl(pcie, PADS_CTL);
941 value |= PADS_CTL_IDDQ_1L;
942 pads_writel(pcie, value, PADS_CTL);
943
944 /*
945 * Set up PHY PLL inputs select PLLE output as refclock,
946 * set TX ref sel to div10 (not div5).
947 */
948 value = pads_readl(pcie, soc->pads_pll_ctl);
949 value &= ~(PADS_PLL_CTL_REFCLK_MASK | PADS_PLL_CTL_TXCLKREF_MASK);
950 value |= PADS_PLL_CTL_REFCLK_INTERNAL_CML | soc->tx_ref_sel;
951 pads_writel(pcie, value, soc->pads_pll_ctl);
952
953 /* reset PLL */
954 value = pads_readl(pcie, soc->pads_pll_ctl);
955 value &= ~PADS_PLL_CTL_RST_B4SM;
956 pads_writel(pcie, value, soc->pads_pll_ctl);
957
958 usleep_range(20, 100);
959
960 /* take PLL out of reset */
961 value = pads_readl(pcie, soc->pads_pll_ctl);
962 value |= PADS_PLL_CTL_RST_B4SM;
963 pads_writel(pcie, value, soc->pads_pll_ctl);
964
965 /* wait for the PLL to lock */
966 err = tegra_pcie_pll_wait(pcie, 500);
967 if (err < 0) {
968 dev_err(dev, "PLL failed to lock: %d\n", err);
969 return err;
970 }
971
972 /* turn off IDDQ override */
973 value = pads_readl(pcie, PADS_CTL);
974 value &= ~PADS_CTL_IDDQ_1L;
975 pads_writel(pcie, value, PADS_CTL);
976
977 /* enable TX/RX data */
978 value = pads_readl(pcie, PADS_CTL);
979 value |= PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L;
980 pads_writel(pcie, value, PADS_CTL);
981
982 return 0;
983}
984
985static int tegra_pcie_phy_disable(struct tegra_pcie *pcie)
986{
987 const struct tegra_pcie_soc *soc = pcie->soc;
988 u32 value;
989
990 /* disable TX/RX data */
991 value = pads_readl(pcie, PADS_CTL);
992 value &= ~(PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L);
993 pads_writel(pcie, value, PADS_CTL);
994
995 /* override IDDQ */
996 value = pads_readl(pcie, PADS_CTL);
997 value |= PADS_CTL_IDDQ_1L;
998 pads_writel(pcie, value, PADS_CTL);
999
1000 /* reset PLL */
1001 value = pads_readl(pcie, soc->pads_pll_ctl);
1002 value &= ~PADS_PLL_CTL_RST_B4SM;
1003 pads_writel(pcie, value, soc->pads_pll_ctl);
1004
1005 usleep_range(20, 100);
1006
1007 return 0;
1008}
1009
1010static int tegra_pcie_port_phy_power_on(struct tegra_pcie_port *port)
1011{
1012 struct device *dev = port->pcie->dev;
1013 unsigned int i;
1014 int err;
1015
1016 for (i = 0; i < port->lanes; i++) {
1017 err = phy_power_on(port->phys[i]);
1018 if (err < 0) {
1019 dev_err(dev, "failed to power on PHY#%u: %d\n", i, err);
1020 return err;
1021 }
1022 }
1023
1024 return 0;
1025}
1026
1027static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port)
1028{
1029 struct device *dev = port->pcie->dev;
1030 unsigned int i;
1031 int err;
1032
1033 for (i = 0; i < port->lanes; i++) {
1034 err = phy_power_off(port->phys[i]);
1035 if (err < 0) {
1036 dev_err(dev, "failed to power off PHY#%u: %d\n", i,
1037 err);
1038 return err;
1039 }
1040 }
1041
1042 return 0;
1043}
1044
1045static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie)
1046{
1047 struct device *dev = pcie->dev;
1048 struct tegra_pcie_port *port;
1049 int err;
1050
1051 if (pcie->legacy_phy) {
1052 if (pcie->phy)
1053 err = phy_power_on(pcie->phy);
1054 else
1055 err = tegra_pcie_phy_enable(pcie);
1056
1057 if (err < 0)
1058 dev_err(dev, "failed to power on PHY: %d\n", err);
1059
1060 return err;
1061 }
1062
1063 list_for_each_entry(port, &pcie->ports, list) {
1064 err = tegra_pcie_port_phy_power_on(port);
1065 if (err < 0) {
1066 dev_err(dev,
1067 "failed to power on PCIe port %u PHY: %d\n",
1068 port->index, err);
1069 return err;
1070 }
1071 }
1072
1073 return 0;
1074}
1075
1076static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie)
1077{
1078 struct device *dev = pcie->dev;
1079 struct tegra_pcie_port *port;
1080 int err;
1081
1082 if (pcie->legacy_phy) {
1083 if (pcie->phy)
1084 err = phy_power_off(pcie->phy);
1085 else
1086 err = tegra_pcie_phy_disable(pcie);
1087
1088 if (err < 0)
1089 dev_err(dev, "failed to power off PHY: %d\n", err);
1090
1091 return err;
1092 }
1093
1094 list_for_each_entry(port, &pcie->ports, list) {
1095 err = tegra_pcie_port_phy_power_off(port);
1096 if (err < 0) {
1097 dev_err(dev,
1098 "failed to power off PCIe port %u PHY: %d\n",
1099 port->index, err);
1100 return err;
1101 }
1102 }
1103
1104 return 0;
1105}
1106
1107static void tegra_pcie_enable_controller(struct tegra_pcie *pcie)
1108{
1109 const struct tegra_pcie_soc *soc = pcie->soc;
1110 struct tegra_pcie_port *port;
1111 unsigned long value;
1112
1113 /* enable PLL power down */
1114 if (pcie->phy) {
1115 value = afi_readl(pcie, AFI_PLLE_CONTROL);
1116 value &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL;
1117 value |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN;
1118 afi_writel(pcie, value, AFI_PLLE_CONTROL);
1119 }
1120
1121 /* power down PCIe slot clock bias pad */
1122 if (soc->has_pex_bias_ctrl)
1123 afi_writel(pcie, 0, AFI_PEXBIAS_CTRL_0);
1124
1125 /* configure mode and disable all ports */
1126 value = afi_readl(pcie, AFI_PCIE_CONFIG);
1127 value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK;
1128 value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config;
1129 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL;
1130
1131 list_for_each_entry(port, &pcie->ports, list) {
1132 value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
1133 value &= ~AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index);
1134 }
1135
1136 afi_writel(pcie, value, AFI_PCIE_CONFIG);
1137
1138 if (soc->has_gen2) {
1139 value = afi_readl(pcie, AFI_FUSE);
1140 value &= ~AFI_FUSE_PCIE_T0_GEN2_DIS;
1141 afi_writel(pcie, value, AFI_FUSE);
1142 } else {
1143 value = afi_readl(pcie, AFI_FUSE);
1144 value |= AFI_FUSE_PCIE_T0_GEN2_DIS;
1145 afi_writel(pcie, value, AFI_FUSE);
1146 }
1147
1148 /* Disable AFI dynamic clock gating and enable PCIe */
1149 value = afi_readl(pcie, AFI_CONFIGURATION);
1150 value |= AFI_CONFIGURATION_EN_FPCI;
1151 value |= AFI_CONFIGURATION_CLKEN_OVERRIDE;
1152 afi_writel(pcie, value, AFI_CONFIGURATION);
1153
1154 value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR |
1155 AFI_INTR_EN_TGT_SLVERR | AFI_INTR_EN_TGT_DECERR |
1156 AFI_INTR_EN_TGT_WRERR | AFI_INTR_EN_DFPCI_DECERR;
1157
1158 if (soc->has_intr_prsnt_sense)
1159 value |= AFI_INTR_EN_PRSNT_SENSE;
1160
1161 afi_writel(pcie, value, AFI_AFI_INTR_ENABLE);
1162 afi_writel(pcie, 0xffffffff, AFI_SM_INTR_ENABLE);
1163
1164 /* don't enable MSI for now, only when needed */
1165 afi_writel(pcie, AFI_INTR_MASK_INT_MASK, AFI_INTR_MASK);
1166
1167 /* disable all exceptions */
1168 afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS);
1169}
1170
1171static void tegra_pcie_power_off(struct tegra_pcie *pcie)
1172{
1173 struct device *dev = pcie->dev;
1174 const struct tegra_pcie_soc *soc = pcie->soc;
1175 int err;
1176
1177 reset_control_assert(pcie->afi_rst);
1178
1179 clk_disable_unprepare(pcie->pll_e);
1180 if (soc->has_cml_clk)
1181 clk_disable_unprepare(pcie->cml_clk);
1182 clk_disable_unprepare(pcie->afi_clk);
1183
1184 if (!dev->pm_domain)
1185 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1186
1187 err = regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1188 if (err < 0)
1189 dev_warn(dev, "failed to disable regulators: %d\n", err);
1190}
1191
1192static int tegra_pcie_power_on(struct tegra_pcie *pcie)
1193{
1194 struct device *dev = pcie->dev;
1195 const struct tegra_pcie_soc *soc = pcie->soc;
1196 int err;
1197
1198 reset_control_assert(pcie->pcie_xrst);
1199 reset_control_assert(pcie->afi_rst);
1200 reset_control_assert(pcie->pex_rst);
1201
1202 if (!dev->pm_domain)
1203 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1204
1205 /* enable regulators */
1206 err = regulator_bulk_enable(pcie->num_supplies, pcie->supplies);
1207 if (err < 0)
1208 dev_err(dev, "failed to enable regulators: %d\n", err);
1209
1210 if (!dev->pm_domain) {
1211 err = tegra_powergate_power_on(TEGRA_POWERGATE_PCIE);
1212 if (err) {
1213 dev_err(dev, "failed to power ungate: %d\n", err);
1214 goto regulator_disable;
1215 }
1216 err = tegra_powergate_remove_clamping(TEGRA_POWERGATE_PCIE);
1217 if (err) {
1218 dev_err(dev, "failed to remove clamp: %d\n", err);
1219 goto powergate;
1220 }
1221 }
1222
1223 err = clk_prepare_enable(pcie->afi_clk);
1224 if (err < 0) {
1225 dev_err(dev, "failed to enable AFI clock: %d\n", err);
1226 goto powergate;
1227 }
1228
1229 if (soc->has_cml_clk) {
1230 err = clk_prepare_enable(pcie->cml_clk);
1231 if (err < 0) {
1232 dev_err(dev, "failed to enable CML clock: %d\n", err);
1233 goto disable_afi_clk;
1234 }
1235 }
1236
1237 err = clk_prepare_enable(pcie->pll_e);
1238 if (err < 0) {
1239 dev_err(dev, "failed to enable PLLE clock: %d\n", err);
1240 goto disable_cml_clk;
1241 }
1242
1243 reset_control_deassert(pcie->afi_rst);
1244
1245 return 0;
1246
1247disable_cml_clk:
1248 if (soc->has_cml_clk)
1249 clk_disable_unprepare(pcie->cml_clk);
1250disable_afi_clk:
1251 clk_disable_unprepare(pcie->afi_clk);
1252powergate:
1253 if (!dev->pm_domain)
1254 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1255regulator_disable:
1256 regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1257
1258 return err;
1259}
1260
1261static void tegra_pcie_apply_pad_settings(struct tegra_pcie *pcie)
1262{
1263 const struct tegra_pcie_soc *soc = pcie->soc;
1264
1265 /* Configure the reference clock driver */
1266 pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0);
1267
1268 if (soc->num_ports > 2)
1269 pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1);
1270}
1271
1272static int tegra_pcie_clocks_get(struct tegra_pcie *pcie)
1273{
1274 struct device *dev = pcie->dev;
1275 const struct tegra_pcie_soc *soc = pcie->soc;
1276
1277 pcie->pex_clk = devm_clk_get(dev, "pex");
1278 if (IS_ERR(pcie->pex_clk))
1279 return PTR_ERR(pcie->pex_clk);
1280
1281 pcie->afi_clk = devm_clk_get(dev, "afi");
1282 if (IS_ERR(pcie->afi_clk))
1283 return PTR_ERR(pcie->afi_clk);
1284
1285 pcie->pll_e = devm_clk_get(dev, "pll_e");
1286 if (IS_ERR(pcie->pll_e))
1287 return PTR_ERR(pcie->pll_e);
1288
1289 if (soc->has_cml_clk) {
1290 pcie->cml_clk = devm_clk_get(dev, "cml");
1291 if (IS_ERR(pcie->cml_clk))
1292 return PTR_ERR(pcie->cml_clk);
1293 }
1294
1295 return 0;
1296}
1297
1298static int tegra_pcie_resets_get(struct tegra_pcie *pcie)
1299{
1300 struct device *dev = pcie->dev;
1301
1302 pcie->pex_rst = devm_reset_control_get_exclusive(dev, "pex");
1303 if (IS_ERR(pcie->pex_rst))
1304 return PTR_ERR(pcie->pex_rst);
1305
1306 pcie->afi_rst = devm_reset_control_get_exclusive(dev, "afi");
1307 if (IS_ERR(pcie->afi_rst))
1308 return PTR_ERR(pcie->afi_rst);
1309
1310 pcie->pcie_xrst = devm_reset_control_get_exclusive(dev, "pcie_x");
1311 if (IS_ERR(pcie->pcie_xrst))
1312 return PTR_ERR(pcie->pcie_xrst);
1313
1314 return 0;
1315}
1316
1317static int tegra_pcie_phys_get_legacy(struct tegra_pcie *pcie)
1318{
1319 struct device *dev = pcie->dev;
1320 int err;
1321
1322 pcie->phy = devm_phy_optional_get(dev, "pcie");
1323 if (IS_ERR(pcie->phy)) {
1324 err = PTR_ERR(pcie->phy);
1325 dev_err(dev, "failed to get PHY: %d\n", err);
1326 return err;
1327 }
1328
1329 err = phy_init(pcie->phy);
1330 if (err < 0) {
1331 dev_err(dev, "failed to initialize PHY: %d\n", err);
1332 return err;
1333 }
1334
1335 pcie->legacy_phy = true;
1336
1337 return 0;
1338}
1339
1340static struct phy *devm_of_phy_optional_get_index(struct device *dev,
1341 struct device_node *np,
1342 const char *consumer,
1343 unsigned int index)
1344{
1345 struct phy *phy;
1346 char *name;
1347
1348 name = kasprintf(GFP_KERNEL, "%s-%u", consumer, index);
1349 if (!name)
1350 return ERR_PTR(-ENOMEM);
1351
1352 phy = devm_of_phy_get(dev, np, name);
1353 kfree(name);
1354
1355 if (PTR_ERR(phy) == -ENODEV)
1356 phy = NULL;
1357
1358 return phy;
1359}
1360
1361static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port)
1362{
1363 struct device *dev = port->pcie->dev;
1364 struct phy *phy;
1365 unsigned int i;
1366 int err;
1367
1368 port->phys = devm_kcalloc(dev, sizeof(phy), port->lanes, GFP_KERNEL);
1369 if (!port->phys)
1370 return -ENOMEM;
1371
1372 for (i = 0; i < port->lanes; i++) {
1373 phy = devm_of_phy_optional_get_index(dev, port->np, "pcie", i);
1374 if (IS_ERR(phy)) {
1375 dev_err(dev, "failed to get PHY#%u: %ld\n", i,
1376 PTR_ERR(phy));
1377 return PTR_ERR(phy);
1378 }
1379
1380 err = phy_init(phy);
1381 if (err < 0) {
1382 dev_err(dev, "failed to initialize PHY#%u: %d\n", i,
1383 err);
1384 return err;
1385 }
1386
1387 port->phys[i] = phy;
1388 }
1389
1390 return 0;
1391}
1392
1393static int tegra_pcie_phys_get(struct tegra_pcie *pcie)
1394{
1395 const struct tegra_pcie_soc *soc = pcie->soc;
1396 struct device_node *np = pcie->dev->of_node;
1397 struct tegra_pcie_port *port;
1398 int err;
1399
1400 if (!soc->has_gen2 || of_find_property(np, "phys", NULL) != NULL)
1401 return tegra_pcie_phys_get_legacy(pcie);
1402
1403 list_for_each_entry(port, &pcie->ports, list) {
1404 err = tegra_pcie_port_get_phys(port);
1405 if (err < 0)
1406 return err;
1407 }
1408
1409 return 0;
1410}
1411
1412static void tegra_pcie_phys_put(struct tegra_pcie *pcie)
1413{
1414 struct tegra_pcie_port *port;
1415 struct device *dev = pcie->dev;
1416 int err, i;
1417
1418 if (pcie->legacy_phy) {
1419 err = phy_exit(pcie->phy);
1420 if (err < 0)
1421 dev_err(dev, "failed to teardown PHY: %d\n", err);
1422 return;
1423 }
1424
1425 list_for_each_entry(port, &pcie->ports, list) {
1426 for (i = 0; i < port->lanes; i++) {
1427 err = phy_exit(port->phys[i]);
1428 if (err < 0)
1429 dev_err(dev, "failed to teardown PHY#%u: %d\n",
1430 i, err);
1431 }
1432 }
1433}
1434
1435
1436static int tegra_pcie_get_resources(struct tegra_pcie *pcie)
1437{
1438 struct device *dev = pcie->dev;
1439 struct platform_device *pdev = to_platform_device(dev);
1440 struct resource *res;
1441 const struct tegra_pcie_soc *soc = pcie->soc;
1442 int err;
1443
1444 err = tegra_pcie_clocks_get(pcie);
1445 if (err) {
1446 dev_err(dev, "failed to get clocks: %d\n", err);
1447 return err;
1448 }
1449
1450 err = tegra_pcie_resets_get(pcie);
1451 if (err) {
1452 dev_err(dev, "failed to get resets: %d\n", err);
1453 return err;
1454 }
1455
1456 if (soc->program_uphy) {
1457 err = tegra_pcie_phys_get(pcie);
1458 if (err < 0) {
1459 dev_err(dev, "failed to get PHYs: %d\n", err);
1460 return err;
1461 }
1462 }
1463
1464 pcie->pads = devm_platform_ioremap_resource_byname(pdev, "pads");
1465 if (IS_ERR(pcie->pads)) {
1466 err = PTR_ERR(pcie->pads);
1467 goto phys_put;
1468 }
1469
1470 pcie->afi = devm_platform_ioremap_resource_byname(pdev, "afi");
1471 if (IS_ERR(pcie->afi)) {
1472 err = PTR_ERR(pcie->afi);
1473 goto phys_put;
1474 }
1475
1476 /* request configuration space, but remap later, on demand */
1477 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs");
1478 if (!res) {
1479 err = -EADDRNOTAVAIL;
1480 goto phys_put;
1481 }
1482
1483 pcie->cs = *res;
1484
1485 /* constrain configuration space to 4 KiB */
1486 pcie->cs.end = pcie->cs.start + SZ_4K - 1;
1487
1488 pcie->cfg = devm_ioremap_resource(dev, &pcie->cs);
1489 if (IS_ERR(pcie->cfg)) {
1490 err = PTR_ERR(pcie->cfg);
1491 goto phys_put;
1492 }
1493
1494 /* request interrupt */
1495 err = platform_get_irq_byname(pdev, "intr");
1496 if (err < 0)
1497 goto phys_put;
1498
1499 pcie->irq = err;
1500
1501 err = request_irq(pcie->irq, tegra_pcie_isr, IRQF_SHARED, "PCIE", pcie);
1502 if (err) {
1503 dev_err(dev, "failed to register IRQ: %d\n", err);
1504 goto phys_put;
1505 }
1506
1507 return 0;
1508
1509phys_put:
1510 if (soc->program_uphy)
1511 tegra_pcie_phys_put(pcie);
1512 return err;
1513}
1514
1515static int tegra_pcie_put_resources(struct tegra_pcie *pcie)
1516{
1517 const struct tegra_pcie_soc *soc = pcie->soc;
1518
1519 if (pcie->irq > 0)
1520 free_irq(pcie->irq, pcie);
1521
1522 if (soc->program_uphy)
1523 tegra_pcie_phys_put(pcie);
1524
1525 return 0;
1526}
1527
1528static void tegra_pcie_pme_turnoff(struct tegra_pcie_port *port)
1529{
1530 struct tegra_pcie *pcie = port->pcie;
1531 const struct tegra_pcie_soc *soc = pcie->soc;
1532 int err;
1533 u32 val;
1534 u8 ack_bit;
1535
1536 val = afi_readl(pcie, AFI_PCIE_PME);
1537 val |= (0x1 << soc->ports[port->index].pme.turnoff_bit);
1538 afi_writel(pcie, val, AFI_PCIE_PME);
1539
1540 ack_bit = soc->ports[port->index].pme.ack_bit;
1541 err = readl_poll_timeout(pcie->afi + AFI_PCIE_PME, val,
1542 val & (0x1 << ack_bit), 1, PME_ACK_TIMEOUT);
1543 if (err)
1544 dev_err(pcie->dev, "PME Ack is not received on port: %d\n",
1545 port->index);
1546
1547 usleep_range(10000, 11000);
1548
1549 val = afi_readl(pcie, AFI_PCIE_PME);
1550 val &= ~(0x1 << soc->ports[port->index].pme.turnoff_bit);
1551 afi_writel(pcie, val, AFI_PCIE_PME);
1552}
1553
1554static int tegra_msi_alloc(struct tegra_msi *chip)
1555{
1556 int msi;
1557
1558 mutex_lock(&chip->lock);
1559
1560 msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR);
1561 if (msi < INT_PCI_MSI_NR)
1562 set_bit(msi, chip->used);
1563 else
1564 msi = -ENOSPC;
1565
1566 mutex_unlock(&chip->lock);
1567
1568 return msi;
1569}
1570
1571static void tegra_msi_free(struct tegra_msi *chip, unsigned long irq)
1572{
1573 struct device *dev = chip->chip.dev;
1574
1575 mutex_lock(&chip->lock);
1576
1577 if (!test_bit(irq, chip->used))
1578 dev_err(dev, "trying to free unused MSI#%lu\n", irq);
1579 else
1580 clear_bit(irq, chip->used);
1581
1582 mutex_unlock(&chip->lock);
1583}
1584
1585static irqreturn_t tegra_pcie_msi_irq(int irq, void *data)
1586{
1587 struct tegra_pcie *pcie = data;
1588 struct device *dev = pcie->dev;
1589 struct tegra_msi *msi = &pcie->msi;
1590 unsigned int i, processed = 0;
1591
1592 for (i = 0; i < 8; i++) {
1593 unsigned long reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4);
1594
1595 while (reg) {
1596 unsigned int offset = find_first_bit(®, 32);
1597 unsigned int index = i * 32 + offset;
1598 unsigned int irq;
1599
1600 /* clear the interrupt */
1601 afi_writel(pcie, 1 << offset, AFI_MSI_VEC0 + i * 4);
1602
1603 irq = irq_find_mapping(msi->domain, index);
1604 if (irq) {
1605 if (test_bit(index, msi->used))
1606 generic_handle_irq(irq);
1607 else
1608 dev_info(dev, "unhandled MSI\n");
1609 } else {
1610 /*
1611 * that's weird who triggered this?
1612 * just clear it
1613 */
1614 dev_info(dev, "unexpected MSI\n");
1615 }
1616
1617 /* see if there's any more pending in this vector */
1618 reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4);
1619
1620 processed++;
1621 }
1622 }
1623
1624 return processed > 0 ? IRQ_HANDLED : IRQ_NONE;
1625}
1626
1627static int tegra_msi_setup_irq(struct msi_controller *chip,
1628 struct pci_dev *pdev, struct msi_desc *desc)
1629{
1630 struct tegra_msi *msi = to_tegra_msi(chip);
1631 struct msi_msg msg;
1632 unsigned int irq;
1633 int hwirq;
1634
1635 hwirq = tegra_msi_alloc(msi);
1636 if (hwirq < 0)
1637 return hwirq;
1638
1639 irq = irq_create_mapping(msi->domain, hwirq);
1640 if (!irq) {
1641 tegra_msi_free(msi, hwirq);
1642 return -EINVAL;
1643 }
1644
1645 irq_set_msi_desc(irq, desc);
1646
1647 msg.address_lo = lower_32_bits(msi->phys);
1648 msg.address_hi = upper_32_bits(msi->phys);
1649 msg.data = hwirq;
1650
1651 pci_write_msi_msg(irq, &msg);
1652
1653 return 0;
1654}
1655
1656static void tegra_msi_teardown_irq(struct msi_controller *chip,
1657 unsigned int irq)
1658{
1659 struct tegra_msi *msi = to_tegra_msi(chip);
1660 struct irq_data *d = irq_get_irq_data(irq);
1661 irq_hw_number_t hwirq = irqd_to_hwirq(d);
1662
1663 irq_dispose_mapping(irq);
1664 tegra_msi_free(msi, hwirq);
1665}
1666
1667static struct irq_chip tegra_msi_irq_chip = {
1668 .name = "Tegra PCIe MSI",
1669 .irq_enable = pci_msi_unmask_irq,
1670 .irq_disable = pci_msi_mask_irq,
1671 .irq_mask = pci_msi_mask_irq,
1672 .irq_unmask = pci_msi_unmask_irq,
1673};
1674
1675static int tegra_msi_map(struct irq_domain *domain, unsigned int irq,
1676 irq_hw_number_t hwirq)
1677{
1678 irq_set_chip_and_handler(irq, &tegra_msi_irq_chip, handle_simple_irq);
1679 irq_set_chip_data(irq, domain->host_data);
1680
1681 tegra_cpuidle_pcie_irqs_in_use();
1682
1683 return 0;
1684}
1685
1686static const struct irq_domain_ops msi_domain_ops = {
1687 .map = tegra_msi_map,
1688};
1689
1690static int tegra_pcie_msi_setup(struct tegra_pcie *pcie)
1691{
1692 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
1693 struct platform_device *pdev = to_platform_device(pcie->dev);
1694 struct tegra_msi *msi = &pcie->msi;
1695 struct device *dev = pcie->dev;
1696 int err;
1697
1698 mutex_init(&msi->lock);
1699
1700 msi->chip.dev = dev;
1701 msi->chip.setup_irq = tegra_msi_setup_irq;
1702 msi->chip.teardown_irq = tegra_msi_teardown_irq;
1703
1704 msi->domain = irq_domain_add_linear(dev->of_node, INT_PCI_MSI_NR,
1705 &msi_domain_ops, &msi->chip);
1706 if (!msi->domain) {
1707 dev_err(dev, "failed to create IRQ domain\n");
1708 return -ENOMEM;
1709 }
1710
1711 err = platform_get_irq_byname(pdev, "msi");
1712 if (err < 0)
1713 goto free_irq_domain;
1714
1715 msi->irq = err;
1716
1717 err = request_irq(msi->irq, tegra_pcie_msi_irq, IRQF_NO_THREAD,
1718 tegra_msi_irq_chip.name, pcie);
1719 if (err < 0) {
1720 dev_err(dev, "failed to request IRQ: %d\n", err);
1721 goto free_irq_domain;
1722 }
1723
1724 /* Though the PCIe controller can address >32-bit address space, to
1725 * facilitate endpoints that support only 32-bit MSI target address,
1726 * the mask is set to 32-bit to make sure that MSI target address is
1727 * always a 32-bit address
1728 */
1729 err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
1730 if (err < 0) {
1731 dev_err(dev, "failed to set DMA coherent mask: %d\n", err);
1732 goto free_irq;
1733 }
1734
1735 msi->virt = dma_alloc_attrs(dev, PAGE_SIZE, &msi->phys, GFP_KERNEL,
1736 DMA_ATTR_NO_KERNEL_MAPPING);
1737 if (!msi->virt) {
1738 dev_err(dev, "failed to allocate DMA memory for MSI\n");
1739 err = -ENOMEM;
1740 goto free_irq;
1741 }
1742
1743 host->msi = &msi->chip;
1744
1745 return 0;
1746
1747free_irq:
1748 free_irq(msi->irq, pcie);
1749free_irq_domain:
1750 irq_domain_remove(msi->domain);
1751 return err;
1752}
1753
1754static void tegra_pcie_enable_msi(struct tegra_pcie *pcie)
1755{
1756 const struct tegra_pcie_soc *soc = pcie->soc;
1757 struct tegra_msi *msi = &pcie->msi;
1758 u32 reg;
1759
1760 afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
1761 afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST);
1762 /* this register is in 4K increments */
1763 afi_writel(pcie, 1, AFI_MSI_BAR_SZ);
1764
1765 /* enable all MSI vectors */
1766 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC0);
1767 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC1);
1768 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC2);
1769 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC3);
1770 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC4);
1771 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC5);
1772 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC6);
1773 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC7);
1774
1775 /* and unmask the MSI interrupt */
1776 reg = afi_readl(pcie, AFI_INTR_MASK);
1777 reg |= AFI_INTR_MASK_MSI_MASK;
1778 afi_writel(pcie, reg, AFI_INTR_MASK);
1779}
1780
1781static void tegra_pcie_msi_teardown(struct tegra_pcie *pcie)
1782{
1783 struct tegra_msi *msi = &pcie->msi;
1784 unsigned int i, irq;
1785
1786 dma_free_attrs(pcie->dev, PAGE_SIZE, msi->virt, msi->phys,
1787 DMA_ATTR_NO_KERNEL_MAPPING);
1788
1789 if (msi->irq > 0)
1790 free_irq(msi->irq, pcie);
1791
1792 for (i = 0; i < INT_PCI_MSI_NR; i++) {
1793 irq = irq_find_mapping(msi->domain, i);
1794 if (irq > 0)
1795 irq_dispose_mapping(irq);
1796 }
1797
1798 irq_domain_remove(msi->domain);
1799}
1800
1801static int tegra_pcie_disable_msi(struct tegra_pcie *pcie)
1802{
1803 u32 value;
1804
1805 /* mask the MSI interrupt */
1806 value = afi_readl(pcie, AFI_INTR_MASK);
1807 value &= ~AFI_INTR_MASK_MSI_MASK;
1808 afi_writel(pcie, value, AFI_INTR_MASK);
1809
1810 /* disable all MSI vectors */
1811 afi_writel(pcie, 0, AFI_MSI_EN_VEC0);
1812 afi_writel(pcie, 0, AFI_MSI_EN_VEC1);
1813 afi_writel(pcie, 0, AFI_MSI_EN_VEC2);
1814 afi_writel(pcie, 0, AFI_MSI_EN_VEC3);
1815 afi_writel(pcie, 0, AFI_MSI_EN_VEC4);
1816 afi_writel(pcie, 0, AFI_MSI_EN_VEC5);
1817 afi_writel(pcie, 0, AFI_MSI_EN_VEC6);
1818 afi_writel(pcie, 0, AFI_MSI_EN_VEC7);
1819
1820 return 0;
1821}
1822
1823static void tegra_pcie_disable_interrupts(struct tegra_pcie *pcie)
1824{
1825 u32 value;
1826
1827 value = afi_readl(pcie, AFI_INTR_MASK);
1828 value &= ~AFI_INTR_MASK_INT_MASK;
1829 afi_writel(pcie, value, AFI_INTR_MASK);
1830}
1831
1832static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
1833 u32 *xbar)
1834{
1835 struct device *dev = pcie->dev;
1836 struct device_node *np = dev->of_node;
1837
1838 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) {
1839 switch (lanes) {
1840 case 0x010004:
1841 dev_info(dev, "4x1, 1x1 configuration\n");
1842 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401;
1843 return 0;
1844
1845 case 0x010102:
1846 dev_info(dev, "2x1, 1X1, 1x1 configuration\n");
1847 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211;
1848 return 0;
1849
1850 case 0x010101:
1851 dev_info(dev, "1x1, 1x1, 1x1 configuration\n");
1852 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111;
1853 return 0;
1854
1855 default:
1856 dev_info(dev, "wrong configuration updated in DT, "
1857 "switching to default 2x1, 1x1, 1x1 "
1858 "configuration\n");
1859 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211;
1860 return 0;
1861 }
1862 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie") ||
1863 of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
1864 switch (lanes) {
1865 case 0x0000104:
1866 dev_info(dev, "4x1, 1x1 configuration\n");
1867 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1;
1868 return 0;
1869
1870 case 0x0000102:
1871 dev_info(dev, "2x1, 1x1 configuration\n");
1872 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1;
1873 return 0;
1874 }
1875 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
1876 switch (lanes) {
1877 case 0x00000204:
1878 dev_info(dev, "4x1, 2x1 configuration\n");
1879 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420;
1880 return 0;
1881
1882 case 0x00020202:
1883 dev_info(dev, "2x3 configuration\n");
1884 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222;
1885 return 0;
1886
1887 case 0x00010104:
1888 dev_info(dev, "4x1, 1x2 configuration\n");
1889 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411;
1890 return 0;
1891 }
1892 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
1893 switch (lanes) {
1894 case 0x00000004:
1895 dev_info(dev, "single-mode configuration\n");
1896 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE;
1897 return 0;
1898
1899 case 0x00000202:
1900 dev_info(dev, "dual-mode configuration\n");
1901 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL;
1902 return 0;
1903 }
1904 }
1905
1906 return -EINVAL;
1907}
1908
1909/*
1910 * Check whether a given set of supplies is available in a device tree node.
1911 * This is used to check whether the new or the legacy device tree bindings
1912 * should be used.
1913 */
1914static bool of_regulator_bulk_available(struct device_node *np,
1915 struct regulator_bulk_data *supplies,
1916 unsigned int num_supplies)
1917{
1918 char property[32];
1919 unsigned int i;
1920
1921 for (i = 0; i < num_supplies; i++) {
1922 snprintf(property, 32, "%s-supply", supplies[i].supply);
1923
1924 if (of_find_property(np, property, NULL) == NULL)
1925 return false;
1926 }
1927
1928 return true;
1929}
1930
1931/*
1932 * Old versions of the device tree binding for this device used a set of power
1933 * supplies that didn't match the hardware inputs. This happened to work for a
1934 * number of cases but is not future proof. However to preserve backwards-
1935 * compatibility with old device trees, this function will try to use the old
1936 * set of supplies.
1937 */
1938static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie)
1939{
1940 struct device *dev = pcie->dev;
1941 struct device_node *np = dev->of_node;
1942
1943 if (of_device_is_compatible(np, "nvidia,tegra30-pcie"))
1944 pcie->num_supplies = 3;
1945 else if (of_device_is_compatible(np, "nvidia,tegra20-pcie"))
1946 pcie->num_supplies = 2;
1947
1948 if (pcie->num_supplies == 0) {
1949 dev_err(dev, "device %pOF not supported in legacy mode\n", np);
1950 return -ENODEV;
1951 }
1952
1953 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
1954 sizeof(*pcie->supplies),
1955 GFP_KERNEL);
1956 if (!pcie->supplies)
1957 return -ENOMEM;
1958
1959 pcie->supplies[0].supply = "pex-clk";
1960 pcie->supplies[1].supply = "vdd";
1961
1962 if (pcie->num_supplies > 2)
1963 pcie->supplies[2].supply = "avdd";
1964
1965 return devm_regulator_bulk_get(dev, pcie->num_supplies, pcie->supplies);
1966}
1967
1968/*
1969 * Obtains the list of regulators required for a particular generation of the
1970 * IP block.
1971 *
1972 * This would've been nice to do simply by providing static tables for use
1973 * with the regulator_bulk_*() API, but unfortunately Tegra30 is a bit quirky
1974 * in that it has two pairs or AVDD_PEX and VDD_PEX supplies (PEXA and PEXB)
1975 * and either seems to be optional depending on which ports are being used.
1976 */
1977static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
1978{
1979 struct device *dev = pcie->dev;
1980 struct device_node *np = dev->of_node;
1981 unsigned int i = 0;
1982
1983 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) {
1984 pcie->num_supplies = 4;
1985
1986 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
1987 sizeof(*pcie->supplies),
1988 GFP_KERNEL);
1989 if (!pcie->supplies)
1990 return -ENOMEM;
1991
1992 pcie->supplies[i++].supply = "dvdd-pex";
1993 pcie->supplies[i++].supply = "hvdd-pex-pll";
1994 pcie->supplies[i++].supply = "hvdd-pex";
1995 pcie->supplies[i++].supply = "vddio-pexctl-aud";
1996 } else if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
1997 pcie->num_supplies = 3;
1998
1999 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
2000 sizeof(*pcie->supplies),
2001 GFP_KERNEL);
2002 if (!pcie->supplies)
2003 return -ENOMEM;
2004
2005 pcie->supplies[i++].supply = "hvddio-pex";
2006 pcie->supplies[i++].supply = "dvddio-pex";
2007 pcie->supplies[i++].supply = "vddio-pex-ctl";
2008 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
2009 pcie->num_supplies = 4;
2010
2011 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2012 sizeof(*pcie->supplies),
2013 GFP_KERNEL);
2014 if (!pcie->supplies)
2015 return -ENOMEM;
2016
2017 pcie->supplies[i++].supply = "avddio-pex";
2018 pcie->supplies[i++].supply = "dvddio-pex";
2019 pcie->supplies[i++].supply = "hvdd-pex";
2020 pcie->supplies[i++].supply = "vddio-pex-ctl";
2021 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
2022 bool need_pexa = false, need_pexb = false;
2023
2024 /* VDD_PEXA and AVDD_PEXA supply lanes 0 to 3 */
2025 if (lane_mask & 0x0f)
2026 need_pexa = true;
2027
2028 /* VDD_PEXB and AVDD_PEXB supply lanes 4 to 5 */
2029 if (lane_mask & 0x30)
2030 need_pexb = true;
2031
2032 pcie->num_supplies = 4 + (need_pexa ? 2 : 0) +
2033 (need_pexb ? 2 : 0);
2034
2035 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2036 sizeof(*pcie->supplies),
2037 GFP_KERNEL);
2038 if (!pcie->supplies)
2039 return -ENOMEM;
2040
2041 pcie->supplies[i++].supply = "avdd-pex-pll";
2042 pcie->supplies[i++].supply = "hvdd-pex";
2043 pcie->supplies[i++].supply = "vddio-pex-ctl";
2044 pcie->supplies[i++].supply = "avdd-plle";
2045
2046 if (need_pexa) {
2047 pcie->supplies[i++].supply = "avdd-pexa";
2048 pcie->supplies[i++].supply = "vdd-pexa";
2049 }
2050
2051 if (need_pexb) {
2052 pcie->supplies[i++].supply = "avdd-pexb";
2053 pcie->supplies[i++].supply = "vdd-pexb";
2054 }
2055 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
2056 pcie->num_supplies = 5;
2057
2058 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2059 sizeof(*pcie->supplies),
2060 GFP_KERNEL);
2061 if (!pcie->supplies)
2062 return -ENOMEM;
2063
2064 pcie->supplies[0].supply = "avdd-pex";
2065 pcie->supplies[1].supply = "vdd-pex";
2066 pcie->supplies[2].supply = "avdd-pex-pll";
2067 pcie->supplies[3].supply = "avdd-plle";
2068 pcie->supplies[4].supply = "vddio-pex-clk";
2069 }
2070
2071 if (of_regulator_bulk_available(dev->of_node, pcie->supplies,
2072 pcie->num_supplies))
2073 return devm_regulator_bulk_get(dev, pcie->num_supplies,
2074 pcie->supplies);
2075
2076 /*
2077 * If not all regulators are available for this new scheme, assume
2078 * that the device tree complies with an older version of the device
2079 * tree binding.
2080 */
2081 dev_info(dev, "using legacy DT binding for power supplies\n");
2082
2083 devm_kfree(dev, pcie->supplies);
2084 pcie->num_supplies = 0;
2085
2086 return tegra_pcie_get_legacy_regulators(pcie);
2087}
2088
2089static int tegra_pcie_parse_dt(struct tegra_pcie *pcie)
2090{
2091 struct device *dev = pcie->dev;
2092 struct device_node *np = dev->of_node, *port;
2093 const struct tegra_pcie_soc *soc = pcie->soc;
2094 u32 lanes = 0, mask = 0;
2095 unsigned int lane = 0;
2096 int err;
2097
2098 /* parse root ports */
2099 for_each_child_of_node(np, port) {
2100 struct tegra_pcie_port *rp;
2101 unsigned int index;
2102 u32 value;
2103 char *label;
2104
2105 err = of_pci_get_devfn(port);
2106 if (err < 0) {
2107 dev_err(dev, "failed to parse address: %d\n", err);
2108 goto err_node_put;
2109 }
2110
2111 index = PCI_SLOT(err);
2112
2113 if (index < 1 || index > soc->num_ports) {
2114 dev_err(dev, "invalid port number: %d\n", index);
2115 err = -EINVAL;
2116 goto err_node_put;
2117 }
2118
2119 index--;
2120
2121 err = of_property_read_u32(port, "nvidia,num-lanes", &value);
2122 if (err < 0) {
2123 dev_err(dev, "failed to parse # of lanes: %d\n",
2124 err);
2125 goto err_node_put;
2126 }
2127
2128 if (value > 16) {
2129 dev_err(dev, "invalid # of lanes: %u\n", value);
2130 err = -EINVAL;
2131 goto err_node_put;
2132 }
2133
2134 lanes |= value << (index << 3);
2135
2136 if (!of_device_is_available(port)) {
2137 lane += value;
2138 continue;
2139 }
2140
2141 mask |= ((1 << value) - 1) << lane;
2142 lane += value;
2143
2144 rp = devm_kzalloc(dev, sizeof(*rp), GFP_KERNEL);
2145 if (!rp) {
2146 err = -ENOMEM;
2147 goto err_node_put;
2148 }
2149
2150 err = of_address_to_resource(port, 0, &rp->regs);
2151 if (err < 0) {
2152 dev_err(dev, "failed to parse address: %d\n", err);
2153 goto err_node_put;
2154 }
2155
2156 INIT_LIST_HEAD(&rp->list);
2157 rp->index = index;
2158 rp->lanes = value;
2159 rp->pcie = pcie;
2160 rp->np = port;
2161
2162 rp->base = devm_pci_remap_cfg_resource(dev, &rp->regs);
2163 if (IS_ERR(rp->base))
2164 return PTR_ERR(rp->base);
2165
2166 label = devm_kasprintf(dev, GFP_KERNEL, "pex-reset-%u", index);
2167 if (!label) {
2168 dev_err(dev, "failed to create reset GPIO label\n");
2169 return -ENOMEM;
2170 }
2171
2172 /*
2173 * Returns -ENOENT if reset-gpios property is not populated
2174 * and in this case fall back to using AFI per port register
2175 * to toggle PERST# SFIO line.
2176 */
2177 rp->reset_gpio = devm_gpiod_get_from_of_node(dev, port,
2178 "reset-gpios", 0,
2179 GPIOD_OUT_LOW,
2180 label);
2181 if (IS_ERR(rp->reset_gpio)) {
2182 if (PTR_ERR(rp->reset_gpio) == -ENOENT) {
2183 rp->reset_gpio = NULL;
2184 } else {
2185 dev_err(dev, "failed to get reset GPIO: %ld\n",
2186 PTR_ERR(rp->reset_gpio));
2187 return PTR_ERR(rp->reset_gpio);
2188 }
2189 }
2190
2191 list_add_tail(&rp->list, &pcie->ports);
2192 }
2193
2194 err = tegra_pcie_get_xbar_config(pcie, lanes, &pcie->xbar_config);
2195 if (err < 0) {
2196 dev_err(dev, "invalid lane configuration\n");
2197 return err;
2198 }
2199
2200 err = tegra_pcie_get_regulators(pcie, mask);
2201 if (err < 0)
2202 return err;
2203
2204 return 0;
2205
2206err_node_put:
2207 of_node_put(port);
2208 return err;
2209}
2210
2211/*
2212 * FIXME: If there are no PCIe cards attached, then calling this function
2213 * can result in the increase of the bootup time as there are big timeout
2214 * loops.
2215 */
2216#define TEGRA_PCIE_LINKUP_TIMEOUT 200 /* up to 1.2 seconds */
2217static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
2218{
2219 struct device *dev = port->pcie->dev;
2220 unsigned int retries = 3;
2221 unsigned long value;
2222
2223 /* override presence detection */
2224 value = readl(port->base + RP_PRIV_MISC);
2225 value &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT;
2226 value |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT;
2227 writel(value, port->base + RP_PRIV_MISC);
2228
2229 do {
2230 unsigned int timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
2231
2232 do {
2233 value = readl(port->base + RP_VEND_XP);
2234
2235 if (value & RP_VEND_XP_DL_UP)
2236 break;
2237
2238 usleep_range(1000, 2000);
2239 } while (--timeout);
2240
2241 if (!timeout) {
2242 dev_dbg(dev, "link %u down, retrying\n", port->index);
2243 goto retry;
2244 }
2245
2246 timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
2247
2248 do {
2249 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2250
2251 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2252 return true;
2253
2254 usleep_range(1000, 2000);
2255 } while (--timeout);
2256
2257retry:
2258 tegra_pcie_port_reset(port);
2259 } while (--retries);
2260
2261 return false;
2262}
2263
2264static void tegra_pcie_change_link_speed(struct tegra_pcie *pcie)
2265{
2266 struct device *dev = pcie->dev;
2267 struct tegra_pcie_port *port;
2268 ktime_t deadline;
2269 u32 value;
2270
2271 list_for_each_entry(port, &pcie->ports, list) {
2272 /*
2273 * "Supported Link Speeds Vector" in "Link Capabilities 2"
2274 * is not supported by Tegra. tegra_pcie_change_link_speed()
2275 * is called only for Tegra chips which support Gen2.
2276 * So there no harm if supported link speed is not verified.
2277 */
2278 value = readl(port->base + RP_LINK_CONTROL_STATUS_2);
2279 value &= ~PCI_EXP_LNKSTA_CLS;
2280 value |= PCI_EXP_LNKSTA_CLS_5_0GB;
2281 writel(value, port->base + RP_LINK_CONTROL_STATUS_2);
2282
2283 /*
2284 * Poll until link comes back from recovery to avoid race
2285 * condition.
2286 */
2287 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT);
2288
2289 while (ktime_before(ktime_get(), deadline)) {
2290 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2291 if ((value & PCI_EXP_LNKSTA_LT) == 0)
2292 break;
2293
2294 usleep_range(2000, 3000);
2295 }
2296
2297 if (value & PCI_EXP_LNKSTA_LT)
2298 dev_warn(dev, "PCIe port %u link is in recovery\n",
2299 port->index);
2300
2301 /* Retrain the link */
2302 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2303 value |= PCI_EXP_LNKCTL_RL;
2304 writel(value, port->base + RP_LINK_CONTROL_STATUS);
2305
2306 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT);
2307
2308 while (ktime_before(ktime_get(), deadline)) {
2309 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2310 if ((value & PCI_EXP_LNKSTA_LT) == 0)
2311 break;
2312
2313 usleep_range(2000, 3000);
2314 }
2315
2316 if (value & PCI_EXP_LNKSTA_LT)
2317 dev_err(dev, "failed to retrain link of port %u\n",
2318 port->index);
2319 }
2320}
2321
2322static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
2323{
2324 struct device *dev = pcie->dev;
2325 struct tegra_pcie_port *port, *tmp;
2326
2327 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2328 dev_info(dev, "probing port %u, using %u lanes\n",
2329 port->index, port->lanes);
2330
2331 tegra_pcie_port_enable(port);
2332 }
2333
2334 /* Start LTSSM from Tegra side */
2335 reset_control_deassert(pcie->pcie_xrst);
2336
2337 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2338 if (tegra_pcie_port_check_link(port))
2339 continue;
2340
2341 dev_info(dev, "link %u down, ignoring\n", port->index);
2342
2343 tegra_pcie_port_disable(port);
2344 tegra_pcie_port_free(port);
2345 }
2346
2347 if (pcie->soc->has_gen2)
2348 tegra_pcie_change_link_speed(pcie);
2349}
2350
2351static void tegra_pcie_disable_ports(struct tegra_pcie *pcie)
2352{
2353 struct tegra_pcie_port *port, *tmp;
2354
2355 reset_control_assert(pcie->pcie_xrst);
2356
2357 list_for_each_entry_safe(port, tmp, &pcie->ports, list)
2358 tegra_pcie_port_disable(port);
2359}
2360
2361static const struct tegra_pcie_port_soc tegra20_pcie_ports[] = {
2362 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2363 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2364};
2365
2366static const struct tegra_pcie_soc tegra20_pcie = {
2367 .num_ports = 2,
2368 .ports = tegra20_pcie_ports,
2369 .msi_base_shift = 0,
2370 .pads_pll_ctl = PADS_PLL_CTL_TEGRA20,
2371 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10,
2372 .pads_refclk_cfg0 = 0xfa5cfa5c,
2373 .has_pex_clkreq_en = false,
2374 .has_pex_bias_ctrl = false,
2375 .has_intr_prsnt_sense = false,
2376 .has_cml_clk = false,
2377 .has_gen2 = false,
2378 .force_pca_enable = false,
2379 .program_uphy = true,
2380 .update_clamp_threshold = false,
2381 .program_deskew_time = false,
2382 .update_fc_timer = false,
2383 .has_cache_bars = true,
2384 .ectl.enable = false,
2385};
2386
2387static const struct tegra_pcie_port_soc tegra30_pcie_ports[] = {
2388 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2389 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2390 { .pme.turnoff_bit = 16, .pme.ack_bit = 18 },
2391};
2392
2393static const struct tegra_pcie_soc tegra30_pcie = {
2394 .num_ports = 3,
2395 .ports = tegra30_pcie_ports,
2396 .msi_base_shift = 8,
2397 .afi_pex2_ctrl = 0x128,
2398 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2399 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2400 .pads_refclk_cfg0 = 0xfa5cfa5c,
2401 .pads_refclk_cfg1 = 0xfa5cfa5c,
2402 .has_pex_clkreq_en = true,
2403 .has_pex_bias_ctrl = true,
2404 .has_intr_prsnt_sense = true,
2405 .has_cml_clk = true,
2406 .has_gen2 = false,
2407 .force_pca_enable = false,
2408 .program_uphy = true,
2409 .update_clamp_threshold = false,
2410 .program_deskew_time = false,
2411 .update_fc_timer = false,
2412 .has_cache_bars = false,
2413 .ectl.enable = false,
2414};
2415
2416static const struct tegra_pcie_soc tegra124_pcie = {
2417 .num_ports = 2,
2418 .ports = tegra20_pcie_ports,
2419 .msi_base_shift = 8,
2420 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2421 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2422 .pads_refclk_cfg0 = 0x44ac44ac,
2423 .has_pex_clkreq_en = true,
2424 .has_pex_bias_ctrl = true,
2425 .has_intr_prsnt_sense = true,
2426 .has_cml_clk = true,
2427 .has_gen2 = true,
2428 .force_pca_enable = false,
2429 .program_uphy = true,
2430 .update_clamp_threshold = true,
2431 .program_deskew_time = false,
2432 .update_fc_timer = false,
2433 .has_cache_bars = false,
2434 .ectl.enable = false,
2435};
2436
2437static const struct tegra_pcie_soc tegra210_pcie = {
2438 .num_ports = 2,
2439 .ports = tegra20_pcie_ports,
2440 .msi_base_shift = 8,
2441 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2442 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2443 .pads_refclk_cfg0 = 0x90b890b8,
2444 /* FC threshold is bit[25:18] */
2445 .update_fc_threshold = 0x01800000,
2446 .has_pex_clkreq_en = true,
2447 .has_pex_bias_ctrl = true,
2448 .has_intr_prsnt_sense = true,
2449 .has_cml_clk = true,
2450 .has_gen2 = true,
2451 .force_pca_enable = true,
2452 .program_uphy = true,
2453 .update_clamp_threshold = true,
2454 .program_deskew_time = true,
2455 .update_fc_timer = true,
2456 .has_cache_bars = false,
2457 .ectl = {
2458 .regs = {
2459 .rp_ectl_2_r1 = 0x0000000f,
2460 .rp_ectl_4_r1 = 0x00000067,
2461 .rp_ectl_5_r1 = 0x55010000,
2462 .rp_ectl_6_r1 = 0x00000001,
2463 .rp_ectl_2_r2 = 0x0000008f,
2464 .rp_ectl_4_r2 = 0x000000c7,
2465 .rp_ectl_5_r2 = 0x55010000,
2466 .rp_ectl_6_r2 = 0x00000001,
2467 },
2468 .enable = true,
2469 },
2470};
2471
2472static const struct tegra_pcie_port_soc tegra186_pcie_ports[] = {
2473 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2474 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2475 { .pme.turnoff_bit = 12, .pme.ack_bit = 14 },
2476};
2477
2478static const struct tegra_pcie_soc tegra186_pcie = {
2479 .num_ports = 3,
2480 .ports = tegra186_pcie_ports,
2481 .msi_base_shift = 8,
2482 .afi_pex2_ctrl = 0x19c,
2483 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2484 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2485 .pads_refclk_cfg0 = 0x80b880b8,
2486 .pads_refclk_cfg1 = 0x000480b8,
2487 .has_pex_clkreq_en = true,
2488 .has_pex_bias_ctrl = true,
2489 .has_intr_prsnt_sense = true,
2490 .has_cml_clk = false,
2491 .has_gen2 = true,
2492 .force_pca_enable = false,
2493 .program_uphy = false,
2494 .update_clamp_threshold = false,
2495 .program_deskew_time = false,
2496 .update_fc_timer = false,
2497 .has_cache_bars = false,
2498 .ectl.enable = false,
2499};
2500
2501static const struct of_device_id tegra_pcie_of_match[] = {
2502 { .compatible = "nvidia,tegra186-pcie", .data = &tegra186_pcie },
2503 { .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie },
2504 { .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
2505 { .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
2506 { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
2507 { },
2508};
2509
2510static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos)
2511{
2512 struct tegra_pcie *pcie = s->private;
2513
2514 if (list_empty(&pcie->ports))
2515 return NULL;
2516
2517 seq_printf(s, "Index Status\n");
2518
2519 return seq_list_start(&pcie->ports, *pos);
2520}
2521
2522static void *tegra_pcie_ports_seq_next(struct seq_file *s, void *v, loff_t *pos)
2523{
2524 struct tegra_pcie *pcie = s->private;
2525
2526 return seq_list_next(v, &pcie->ports, pos);
2527}
2528
2529static void tegra_pcie_ports_seq_stop(struct seq_file *s, void *v)
2530{
2531}
2532
2533static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v)
2534{
2535 bool up = false, active = false;
2536 struct tegra_pcie_port *port;
2537 unsigned int value;
2538
2539 port = list_entry(v, struct tegra_pcie_port, list);
2540
2541 value = readl(port->base + RP_VEND_XP);
2542
2543 if (value & RP_VEND_XP_DL_UP)
2544 up = true;
2545
2546 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2547
2548 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2549 active = true;
2550
2551 seq_printf(s, "%2u ", port->index);
2552
2553 if (up)
2554 seq_printf(s, "up");
2555
2556 if (active) {
2557 if (up)
2558 seq_printf(s, ", ");
2559
2560 seq_printf(s, "active");
2561 }
2562
2563 seq_printf(s, "\n");
2564 return 0;
2565}
2566
2567static const struct seq_operations tegra_pcie_ports_seq_ops = {
2568 .start = tegra_pcie_ports_seq_start,
2569 .next = tegra_pcie_ports_seq_next,
2570 .stop = tegra_pcie_ports_seq_stop,
2571 .show = tegra_pcie_ports_seq_show,
2572};
2573
2574static int tegra_pcie_ports_open(struct inode *inode, struct file *file)
2575{
2576 struct tegra_pcie *pcie = inode->i_private;
2577 struct seq_file *s;
2578 int err;
2579
2580 err = seq_open(file, &tegra_pcie_ports_seq_ops);
2581 if (err)
2582 return err;
2583
2584 s = file->private_data;
2585 s->private = pcie;
2586
2587 return 0;
2588}
2589
2590static const struct file_operations tegra_pcie_ports_ops = {
2591 .owner = THIS_MODULE,
2592 .open = tegra_pcie_ports_open,
2593 .read = seq_read,
2594 .llseek = seq_lseek,
2595 .release = seq_release,
2596};
2597
2598static void tegra_pcie_debugfs_exit(struct tegra_pcie *pcie)
2599{
2600 debugfs_remove_recursive(pcie->debugfs);
2601 pcie->debugfs = NULL;
2602}
2603
2604static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
2605{
2606 struct dentry *file;
2607
2608 pcie->debugfs = debugfs_create_dir("pcie", NULL);
2609 if (!pcie->debugfs)
2610 return -ENOMEM;
2611
2612 file = debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs,
2613 pcie, &tegra_pcie_ports_ops);
2614 if (!file)
2615 goto remove;
2616
2617 return 0;
2618
2619remove:
2620 tegra_pcie_debugfs_exit(pcie);
2621 return -ENOMEM;
2622}
2623
2624static int tegra_pcie_probe(struct platform_device *pdev)
2625{
2626 struct device *dev = &pdev->dev;
2627 struct pci_host_bridge *host;
2628 struct tegra_pcie *pcie;
2629 int err;
2630
2631 host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
2632 if (!host)
2633 return -ENOMEM;
2634
2635 pcie = pci_host_bridge_priv(host);
2636 host->sysdata = pcie;
2637 platform_set_drvdata(pdev, pcie);
2638
2639 pcie->soc = of_device_get_match_data(dev);
2640 INIT_LIST_HEAD(&pcie->ports);
2641 pcie->dev = dev;
2642
2643 err = tegra_pcie_parse_dt(pcie);
2644 if (err < 0)
2645 return err;
2646
2647 err = tegra_pcie_get_resources(pcie);
2648 if (err < 0) {
2649 dev_err(dev, "failed to request resources: %d\n", err);
2650 return err;
2651 }
2652
2653 err = tegra_pcie_msi_setup(pcie);
2654 if (err < 0) {
2655 dev_err(dev, "failed to enable MSI support: %d\n", err);
2656 goto put_resources;
2657 }
2658
2659 pm_runtime_enable(pcie->dev);
2660 err = pm_runtime_get_sync(pcie->dev);
2661 if (err < 0) {
2662 dev_err(dev, "fail to enable pcie controller: %d\n", err);
2663 goto pm_runtime_put;
2664 }
2665
2666 host->ops = &tegra_pcie_ops;
2667 host->map_irq = tegra_pcie_map_irq;
2668
2669 err = pci_host_probe(host);
2670 if (err < 0) {
2671 dev_err(dev, "failed to register host: %d\n", err);
2672 goto pm_runtime_put;
2673 }
2674
2675 if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2676 err = tegra_pcie_debugfs_init(pcie);
2677 if (err < 0)
2678 dev_err(dev, "failed to setup debugfs: %d\n", err);
2679 }
2680
2681 return 0;
2682
2683pm_runtime_put:
2684 pm_runtime_put_sync(pcie->dev);
2685 pm_runtime_disable(pcie->dev);
2686 tegra_pcie_msi_teardown(pcie);
2687put_resources:
2688 tegra_pcie_put_resources(pcie);
2689 return err;
2690}
2691
2692static int tegra_pcie_remove(struct platform_device *pdev)
2693{
2694 struct tegra_pcie *pcie = platform_get_drvdata(pdev);
2695 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
2696 struct tegra_pcie_port *port, *tmp;
2697
2698 if (IS_ENABLED(CONFIG_DEBUG_FS))
2699 tegra_pcie_debugfs_exit(pcie);
2700
2701 pci_stop_root_bus(host->bus);
2702 pci_remove_root_bus(host->bus);
2703 pm_runtime_put_sync(pcie->dev);
2704 pm_runtime_disable(pcie->dev);
2705
2706 if (IS_ENABLED(CONFIG_PCI_MSI))
2707 tegra_pcie_msi_teardown(pcie);
2708
2709 tegra_pcie_put_resources(pcie);
2710
2711 list_for_each_entry_safe(port, tmp, &pcie->ports, list)
2712 tegra_pcie_port_free(port);
2713
2714 return 0;
2715}
2716
2717static int __maybe_unused tegra_pcie_pm_suspend(struct device *dev)
2718{
2719 struct tegra_pcie *pcie = dev_get_drvdata(dev);
2720 struct tegra_pcie_port *port;
2721 int err;
2722
2723 list_for_each_entry(port, &pcie->ports, list)
2724 tegra_pcie_pme_turnoff(port);
2725
2726 tegra_pcie_disable_ports(pcie);
2727
2728 /*
2729 * AFI_INTR is unmasked in tegra_pcie_enable_controller(), mask it to
2730 * avoid unwanted interrupts raised by AFI after pex_rst is asserted.
2731 */
2732 tegra_pcie_disable_interrupts(pcie);
2733
2734 if (pcie->soc->program_uphy) {
2735 err = tegra_pcie_phy_power_off(pcie);
2736 if (err < 0)
2737 dev_err(dev, "failed to power off PHY(s): %d\n", err);
2738 }
2739
2740 reset_control_assert(pcie->pex_rst);
2741 clk_disable_unprepare(pcie->pex_clk);
2742
2743 if (IS_ENABLED(CONFIG_PCI_MSI))
2744 tegra_pcie_disable_msi(pcie);
2745
2746 pinctrl_pm_select_idle_state(dev);
2747 tegra_pcie_power_off(pcie);
2748
2749 return 0;
2750}
2751
2752static int __maybe_unused tegra_pcie_pm_resume(struct device *dev)
2753{
2754 struct tegra_pcie *pcie = dev_get_drvdata(dev);
2755 int err;
2756
2757 err = tegra_pcie_power_on(pcie);
2758 if (err) {
2759 dev_err(dev, "tegra pcie power on fail: %d\n", err);
2760 return err;
2761 }
2762
2763 err = pinctrl_pm_select_default_state(dev);
2764 if (err < 0) {
2765 dev_err(dev, "failed to disable PCIe IO DPD: %d\n", err);
2766 goto poweroff;
2767 }
2768
2769 tegra_pcie_enable_controller(pcie);
2770 tegra_pcie_setup_translations(pcie);
2771
2772 if (IS_ENABLED(CONFIG_PCI_MSI))
2773 tegra_pcie_enable_msi(pcie);
2774
2775 err = clk_prepare_enable(pcie->pex_clk);
2776 if (err) {
2777 dev_err(dev, "failed to enable PEX clock: %d\n", err);
2778 goto pex_dpd_enable;
2779 }
2780
2781 reset_control_deassert(pcie->pex_rst);
2782
2783 if (pcie->soc->program_uphy) {
2784 err = tegra_pcie_phy_power_on(pcie);
2785 if (err < 0) {
2786 dev_err(dev, "failed to power on PHY(s): %d\n", err);
2787 goto disable_pex_clk;
2788 }
2789 }
2790
2791 tegra_pcie_apply_pad_settings(pcie);
2792 tegra_pcie_enable_ports(pcie);
2793
2794 return 0;
2795
2796disable_pex_clk:
2797 reset_control_assert(pcie->pex_rst);
2798 clk_disable_unprepare(pcie->pex_clk);
2799pex_dpd_enable:
2800 pinctrl_pm_select_idle_state(dev);
2801poweroff:
2802 tegra_pcie_power_off(pcie);
2803
2804 return err;
2805}
2806
2807static const struct dev_pm_ops tegra_pcie_pm_ops = {
2808 SET_RUNTIME_PM_OPS(tegra_pcie_pm_suspend, tegra_pcie_pm_resume, NULL)
2809 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(tegra_pcie_pm_suspend,
2810 tegra_pcie_pm_resume)
2811};
2812
2813static struct platform_driver tegra_pcie_driver = {
2814 .driver = {
2815 .name = "tegra-pcie",
2816 .of_match_table = tegra_pcie_of_match,
2817 .suppress_bind_attrs = true,
2818 .pm = &tegra_pcie_pm_ops,
2819 },
2820 .probe = tegra_pcie_probe,
2821 .remove = tegra_pcie_remove,
2822};
2823module_platform_driver(tegra_pcie_driver);
2824MODULE_LICENSE("GPL");