Loading...
Note: File does not exist in v3.5.6.
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PCIe host controller driver for Tegra194 SoC
4 *
5 * Copyright (C) 2019 NVIDIA Corporation.
6 *
7 * Author: Vidya Sagar <vidyas@nvidia.com>
8 */
9
10#include <linux/clk.h>
11#include <linux/debugfs.h>
12#include <linux/delay.h>
13#include <linux/gpio.h>
14#include <linux/gpio/consumer.h>
15#include <linux/interrupt.h>
16#include <linux/iopoll.h>
17#include <linux/kernel.h>
18#include <linux/module.h>
19#include <linux/of.h>
20#include <linux/of_device.h>
21#include <linux/of_gpio.h>
22#include <linux/of_irq.h>
23#include <linux/of_pci.h>
24#include <linux/pci.h>
25#include <linux/phy/phy.h>
26#include <linux/pinctrl/consumer.h>
27#include <linux/platform_device.h>
28#include <linux/pm_runtime.h>
29#include <linux/random.h>
30#include <linux/reset.h>
31#include <linux/resource.h>
32#include <linux/types.h>
33#include "pcie-designware.h"
34#include <soc/tegra/bpmp.h>
35#include <soc/tegra/bpmp-abi.h>
36#include "../../pci.h"
37
38#define APPL_PINMUX 0x0
39#define APPL_PINMUX_PEX_RST BIT(0)
40#define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)
41#define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)
42#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)
43#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)
44
45#define APPL_CTRL 0x4
46#define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)
47#define APPL_CTRL_LTSSM_EN BIT(7)
48#define APPL_CTRL_HW_HOT_RST_EN BIT(20)
49#define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)
50#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22
51#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1
52
53#define APPL_INTR_EN_L0_0 0x8
54#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)
55#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)
56#define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)
57#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15)
58#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)
59#define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)
60#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)
61
62#define APPL_INTR_STATUS_L0 0xC
63#define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)
64#define APPL_INTR_STATUS_L0_INT_INT BIT(8)
65#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15)
66#define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16)
67#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)
68
69#define APPL_INTR_EN_L1_0_0 0x1C
70#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)
71#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3)
72#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30)
73
74#define APPL_INTR_STATUS_L1_0_0 0x20
75#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)
76#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3)
77#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30)
78
79#define APPL_INTR_STATUS_L1_1 0x2C
80#define APPL_INTR_STATUS_L1_2 0x30
81#define APPL_INTR_STATUS_L1_3 0x34
82#define APPL_INTR_STATUS_L1_6 0x3C
83#define APPL_INTR_STATUS_L1_7 0x40
84#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1)
85
86#define APPL_INTR_EN_L1_8_0 0x44
87#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)
88#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)
89#define APPL_INTR_EN_L1_8_INTX_EN BIT(11)
90#define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)
91
92#define APPL_INTR_STATUS_L1_8_0 0x4C
93#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)
94#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)
95#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)
96
97#define APPL_INTR_STATUS_L1_9 0x54
98#define APPL_INTR_STATUS_L1_10 0x58
99#define APPL_INTR_STATUS_L1_11 0x64
100#define APPL_INTR_STATUS_L1_13 0x74
101#define APPL_INTR_STATUS_L1_14 0x78
102#define APPL_INTR_STATUS_L1_15 0x7C
103#define APPL_INTR_STATUS_L1_17 0x88
104
105#define APPL_INTR_EN_L1_18 0x90
106#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)
107#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
108#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
109
110#define APPL_INTR_STATUS_L1_18 0x94
111#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)
112#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
113#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
114
115#define APPL_MSI_CTRL_1 0xAC
116
117#define APPL_MSI_CTRL_2 0xB0
118
119#define APPL_LEGACY_INTX 0xB8
120
121#define APPL_LTR_MSG_1 0xC4
122#define LTR_MSG_REQ BIT(15)
123#define LTR_MST_NO_SNOOP_SHIFT 16
124
125#define APPL_LTR_MSG_2 0xC8
126#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)
127
128#define APPL_LINK_STATUS 0xCC
129#define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)
130
131#define APPL_DEBUG 0xD0
132#define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)
133#define APPL_DEBUG_PM_LINKST_IN_L0 0x11
134#define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)
135#define APPL_DEBUG_LTSSM_STATE_SHIFT 3
136#define LTSSM_STATE_PRE_DETECT 5
137
138#define APPL_RADM_STATUS 0xE4
139#define APPL_PM_XMT_TURNOFF_STATE BIT(0)
140
141#define APPL_DM_TYPE 0x100
142#define APPL_DM_TYPE_MASK GENMASK(3, 0)
143#define APPL_DM_TYPE_RP 0x4
144#define APPL_DM_TYPE_EP 0x0
145
146#define APPL_CFG_BASE_ADDR 0x104
147#define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)
148
149#define APPL_CFG_IATU_DMA_BASE_ADDR 0x108
150#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)
151
152#define APPL_CFG_MISC 0x110
153#define APPL_CFG_MISC_SLV_EP_MODE BIT(14)
154#define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)
155#define APPL_CFG_MISC_ARCACHE_SHIFT 10
156#define APPL_CFG_MISC_ARCACHE_VAL 3
157
158#define APPL_CFG_SLCG_OVERRIDE 0x114
159#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)
160
161#define APPL_CAR_RESET_OVRD 0x12C
162#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)
163
164#define IO_BASE_IO_DECODE BIT(0)
165#define IO_BASE_IO_DECODE_BIT8 BIT(8)
166
167#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)
168#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)
169
170#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718
171#define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)
172
173#define EVENT_COUNTER_ALL_CLEAR 0x3
174#define EVENT_COUNTER_ENABLE_ALL 0x7
175#define EVENT_COUNTER_ENABLE_SHIFT 2
176#define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0)
177#define EVENT_COUNTER_EVENT_SEL_SHIFT 16
178#define EVENT_COUNTER_EVENT_Tx_L0S 0x2
179#define EVENT_COUNTER_EVENT_Rx_L0S 0x3
180#define EVENT_COUNTER_EVENT_L1 0x5
181#define EVENT_COUNTER_EVENT_L1_1 0x7
182#define EVENT_COUNTER_EVENT_L1_2 0x8
183#define EVENT_COUNTER_GROUP_SEL_SHIFT 24
184#define EVENT_COUNTER_GROUP_5 0x5
185
186#define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C
187#define ENTER_ASPM BIT(30)
188#define L0S_ENTRANCE_LAT_SHIFT 24
189#define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24)
190#define L1_ENTRANCE_LAT_SHIFT 27
191#define L1_ENTRANCE_LAT_MASK GENMASK(29, 27)
192#define N_FTS_SHIFT 8
193#define N_FTS_MASK GENMASK(7, 0)
194#define N_FTS_VAL 52
195
196#define PORT_LOGIC_GEN2_CTRL 0x80C
197#define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17)
198#define FTS_MASK GENMASK(7, 0)
199#define FTS_VAL 52
200
201#define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828
202
203#define GEN3_EQ_CONTROL_OFF 0x8a8
204#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8
205#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8)
206#define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0)
207
208#define GEN3_RELATED_OFF 0x890
209#define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0)
210#define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16)
211#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24
212#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24)
213
214#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0
215#define AMBA_ERROR_RESPONSE_CRS_SHIFT 3
216#define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0)
217#define AMBA_ERROR_RESPONSE_CRS_OKAY 0
218#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1
219#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2
220
221#define MSIX_ADDR_MATCH_LOW_OFF 0x940
222#define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0)
223#define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2)
224
225#define MSIX_ADDR_MATCH_HIGH_OFF 0x944
226#define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0)
227
228#define PORT_LOGIC_MSIX_DOORBELL 0x948
229
230#define CAP_SPCIE_CAP_OFF 0x154
231#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)
232#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)
233#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8
234
235#define PME_ACK_TIMEOUT 10000
236
237#define LTSSM_TIMEOUT 50000 /* 50ms */
238
239#define GEN3_GEN4_EQ_PRESET_INIT 5
240
241#define GEN1_CORE_CLK_FREQ 62500000
242#define GEN2_CORE_CLK_FREQ 125000000
243#define GEN3_CORE_CLK_FREQ 250000000
244#define GEN4_CORE_CLK_FREQ 500000000
245
246#define LTR_MSG_TIMEOUT (100 * 1000)
247
248#define PERST_DEBOUNCE_TIME (5 * 1000)
249
250#define EP_STATE_DISABLED 0
251#define EP_STATE_ENABLED 1
252
253static const unsigned int pcie_gen_freq[] = {
254 GEN1_CORE_CLK_FREQ,
255 GEN2_CORE_CLK_FREQ,
256 GEN3_CORE_CLK_FREQ,
257 GEN4_CORE_CLK_FREQ
258};
259
260static const u32 event_cntr_ctrl_offset[] = {
261 0x1d8,
262 0x1a8,
263 0x1a8,
264 0x1a8,
265 0x1c4,
266 0x1d8
267};
268
269static const u32 event_cntr_data_offset[] = {
270 0x1dc,
271 0x1ac,
272 0x1ac,
273 0x1ac,
274 0x1c8,
275 0x1dc
276};
277
278struct tegra_pcie_dw {
279 struct device *dev;
280 struct resource *appl_res;
281 struct resource *dbi_res;
282 struct resource *atu_dma_res;
283 void __iomem *appl_base;
284 struct clk *core_clk;
285 struct reset_control *core_apb_rst;
286 struct reset_control *core_rst;
287 struct dw_pcie pci;
288 struct tegra_bpmp *bpmp;
289
290 enum dw_pcie_device_mode mode;
291
292 bool supports_clkreq;
293 bool enable_cdm_check;
294 bool link_state;
295 bool update_fc_fixup;
296 u8 init_link_width;
297 u32 msi_ctrl_int;
298 u32 num_lanes;
299 u32 max_speed;
300 u32 cid;
301 u32 cfg_link_cap_l1sub;
302 u32 pcie_cap_base;
303 u32 aspm_cmrt;
304 u32 aspm_pwr_on_t;
305 u32 aspm_l0s_enter_lat;
306
307 struct regulator *pex_ctl_supply;
308 struct regulator *slot_ctl_3v3;
309 struct regulator *slot_ctl_12v;
310
311 unsigned int phy_count;
312 struct phy **phys;
313
314 struct dentry *debugfs;
315
316 /* Endpoint mode specific */
317 struct gpio_desc *pex_rst_gpiod;
318 struct gpio_desc *pex_refclk_sel_gpiod;
319 unsigned int pex_rst_irq;
320 int ep_state;
321};
322
323struct tegra_pcie_dw_of_data {
324 enum dw_pcie_device_mode mode;
325};
326
327static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
328{
329 return container_of(pci, struct tegra_pcie_dw, pci);
330}
331
332static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
333 const u32 reg)
334{
335 writel_relaxed(value, pcie->appl_base + reg);
336}
337
338static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
339{
340 return readl_relaxed(pcie->appl_base + reg);
341}
342
343struct tegra_pcie_soc {
344 enum dw_pcie_device_mode mode;
345};
346
347static void apply_bad_link_workaround(struct pcie_port *pp)
348{
349 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
350 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
351 u32 current_link_width;
352 u16 val;
353
354 /*
355 * NOTE:- Since this scenario is uncommon and link as such is not
356 * stable anyway, not waiting to confirm if link is really
357 * transitioning to Gen-2 speed
358 */
359 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
360 if (val & PCI_EXP_LNKSTA_LBMS) {
361 current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
362 PCI_EXP_LNKSTA_NLW_SHIFT;
363 if (pcie->init_link_width > current_link_width) {
364 dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
365 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
366 PCI_EXP_LNKCTL2);
367 val &= ~PCI_EXP_LNKCTL2_TLS;
368 val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
369 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
370 PCI_EXP_LNKCTL2, val);
371
372 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
373 PCI_EXP_LNKCTL);
374 val |= PCI_EXP_LNKCTL_RL;
375 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
376 PCI_EXP_LNKCTL, val);
377 }
378 }
379}
380
381static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
382{
383 struct tegra_pcie_dw *pcie = arg;
384 struct dw_pcie *pci = &pcie->pci;
385 struct pcie_port *pp = &pci->pp;
386 u32 val, tmp;
387 u16 val_w;
388
389 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
390 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
391 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
392 if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
393 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
394
395 /* SBR & Surprise Link Down WAR */
396 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
397 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
398 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
399 udelay(1);
400 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
401 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
402 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
403
404 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
405 val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE;
406 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
407 }
408 }
409
410 if (val & APPL_INTR_STATUS_L0_INT_INT) {
411 val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
412 if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
413 appl_writel(pcie,
414 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
415 APPL_INTR_STATUS_L1_8_0);
416 apply_bad_link_workaround(pp);
417 }
418 if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
419 appl_writel(pcie,
420 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
421 APPL_INTR_STATUS_L1_8_0);
422
423 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
424 PCI_EXP_LNKSTA);
425 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
426 PCI_EXP_LNKSTA_CLS);
427 }
428 }
429
430 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
431 if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
432 val = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
433 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
434 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
435 dev_info(pci->dev, "CDM check complete\n");
436 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
437 }
438 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
439 dev_err(pci->dev, "CDM comparison mismatch\n");
440 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
441 }
442 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
443 dev_err(pci->dev, "CDM Logic error\n");
444 tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
445 }
446 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp);
447 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
448 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp);
449 }
450
451 return IRQ_HANDLED;
452}
453
454static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
455{
456 u32 val;
457
458 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
459 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
460 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
461 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
462 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
463 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
464 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
465 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
466 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
467 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
468 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
469 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
470 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
471 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
472 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
473 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
474
475 val = appl_readl(pcie, APPL_CTRL);
476 val |= APPL_CTRL_LTSSM_EN;
477 appl_writel(pcie, val, APPL_CTRL);
478}
479
480static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
481{
482 struct tegra_pcie_dw *pcie = arg;
483 struct dw_pcie *pci = &pcie->pci;
484 u32 val, speed;
485
486 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
487 PCI_EXP_LNKSTA_CLS;
488 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
489
490 /* If EP doesn't advertise L1SS, just return */
491 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
492 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
493 return IRQ_HANDLED;
494
495 /* Check if BME is set to '1' */
496 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
497 if (val & PCI_COMMAND_MASTER) {
498 ktime_t timeout;
499
500 /* 110us for both snoop and no-snoop */
501 val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ;
502 val |= (val << LTR_MST_NO_SNOOP_SHIFT);
503 appl_writel(pcie, val, APPL_LTR_MSG_1);
504
505 /* Send LTR upstream */
506 val = appl_readl(pcie, APPL_LTR_MSG_2);
507 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
508 appl_writel(pcie, val, APPL_LTR_MSG_2);
509
510 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
511 for (;;) {
512 val = appl_readl(pcie, APPL_LTR_MSG_2);
513 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
514 break;
515 if (ktime_after(ktime_get(), timeout))
516 break;
517 usleep_range(1000, 1100);
518 }
519 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
520 dev_err(pcie->dev, "Failed to send LTR message\n");
521 }
522
523 return IRQ_HANDLED;
524}
525
526static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
527{
528 struct tegra_pcie_dw *pcie = arg;
529 struct dw_pcie_ep *ep = &pcie->pci.ep;
530 int spurious = 1;
531 u32 val, tmp;
532
533 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
534 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
535 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
536 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
537
538 if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
539 pex_ep_event_hot_rst_done(pcie);
540
541 if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
542 tmp = appl_readl(pcie, APPL_LINK_STATUS);
543 if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) {
544 dev_dbg(pcie->dev, "Link is up with Host\n");
545 dw_pcie_ep_linkup(ep);
546 }
547 }
548
549 spurious = 0;
550 }
551
552 if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
553 val = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
554 appl_writel(pcie, val, APPL_INTR_STATUS_L1_15);
555
556 if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
557 return IRQ_WAKE_THREAD;
558
559 spurious = 0;
560 }
561
562 if (spurious) {
563 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
564 val);
565 appl_writel(pcie, val, APPL_INTR_STATUS_L0);
566 }
567
568 return IRQ_HANDLED;
569}
570
571static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size,
572 u32 *val)
573{
574 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
575
576 /*
577 * This is an endpoint mode specific register happen to appear even
578 * when controller is operating in root port mode and system hangs
579 * when it is accessed with link being in ASPM-L1 state.
580 * So skip accessing it altogether
581 */
582 if (where == PORT_LOGIC_MSIX_DOORBELL) {
583 *val = 0x00000000;
584 return PCIBIOS_SUCCESSFUL;
585 }
586
587 return dw_pcie_read(pci->dbi_base + where, size, val);
588}
589
590static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size,
591 u32 val)
592{
593 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
594
595 /*
596 * This is an endpoint mode specific register happen to appear even
597 * when controller is operating in root port mode and system hangs
598 * when it is accessed with link being in ASPM-L1 state.
599 * So skip accessing it altogether
600 */
601 if (where == PORT_LOGIC_MSIX_DOORBELL)
602 return PCIBIOS_SUCCESSFUL;
603
604 return dw_pcie_write(pci->dbi_base + where, size, val);
605}
606
607#if defined(CONFIG_PCIEASPM)
608static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
609{
610 u32 val;
611
612 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
613 val &= ~PCI_L1SS_CAP_ASPM_L1_1;
614 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
615}
616
617static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
618{
619 u32 val;
620
621 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
622 val &= ~PCI_L1SS_CAP_ASPM_L1_2;
623 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
624}
625
626static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
627{
628 u32 val;
629
630 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]);
631 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
632 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
633 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
634 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
635 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
636 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]);
637
638 return val;
639}
640
641static int aspm_state_cnt(struct seq_file *s, void *data)
642{
643 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
644 dev_get_drvdata(s->private);
645 u32 val;
646
647 seq_printf(s, "Tx L0s entry count : %u\n",
648 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
649
650 seq_printf(s, "Rx L0s entry count : %u\n",
651 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
652
653 seq_printf(s, "Link L1 entry count : %u\n",
654 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
655
656 seq_printf(s, "Link L1.1 entry count : %u\n",
657 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
658
659 seq_printf(s, "Link L1.2 entry count : %u\n",
660 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
661
662 /* Clear all counters */
663 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid],
664 EVENT_COUNTER_ALL_CLEAR);
665
666 /* Re-enable counting */
667 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
668 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
669 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
670
671 return 0;
672}
673
674static void init_host_aspm(struct tegra_pcie_dw *pcie)
675{
676 struct dw_pcie *pci = &pcie->pci;
677 u32 val;
678
679 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
680 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
681
682 /* Enable ASPM counters */
683 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
684 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
685 dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val);
686
687 /* Program T_cmrt and T_pwr_on values */
688 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
689 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
690 val |= (pcie->aspm_cmrt << 8);
691 val |= (pcie->aspm_pwr_on_t << 19);
692 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
693
694 /* Program L0s and L1 entrance latencies */
695 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
696 val &= ~L0S_ENTRANCE_LAT_MASK;
697 val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT);
698 val |= ENTER_ASPM;
699 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
700}
701
702static int init_debugfs(struct tegra_pcie_dw *pcie)
703{
704 struct dentry *d;
705
706 d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt",
707 pcie->debugfs, aspm_state_cnt);
708 if (IS_ERR_OR_NULL(d))
709 dev_err(pcie->dev,
710 "Failed to create debugfs file \"aspm_state_cnt\"\n");
711
712 return 0;
713}
714#else
715static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
716static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
717static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
718static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; }
719#endif
720
721static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp)
722{
723 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
724 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
725 u32 val;
726 u16 val_w;
727
728 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
729 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
730 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
731
732 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
733 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
734 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
735
736 if (pcie->enable_cdm_check) {
737 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
738 val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN;
739 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
740
741 val = appl_readl(pcie, APPL_INTR_EN_L1_18);
742 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
743 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
744 appl_writel(pcie, val, APPL_INTR_EN_L1_18);
745 }
746
747 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
748 PCI_EXP_LNKSTA);
749 pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
750 PCI_EXP_LNKSTA_NLW_SHIFT;
751
752 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
753 PCI_EXP_LNKCTL);
754 val_w |= PCI_EXP_LNKCTL_LBMIE;
755 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
756 val_w);
757}
758
759static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp)
760{
761 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
762 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
763 u32 val;
764
765 /* Enable legacy interrupt generation */
766 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
767 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
768 val |= APPL_INTR_EN_L0_0_INT_INT_EN;
769 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
770
771 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
772 val |= APPL_INTR_EN_L1_8_INTX_EN;
773 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
774 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
775 if (IS_ENABLED(CONFIG_PCIEAER))
776 val |= APPL_INTR_EN_L1_8_AER_INT_EN;
777 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
778}
779
780static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp)
781{
782 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
783 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
784 u32 val;
785
786 dw_pcie_msi_init(pp);
787
788 /* Enable MSI interrupt generation */
789 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
790 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
791 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
792 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
793}
794
795static void tegra_pcie_enable_interrupts(struct pcie_port *pp)
796{
797 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
798 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
799
800 /* Clear interrupt statuses before enabling interrupts */
801 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
802 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
804 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
805 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
806 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
807 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
808 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
809 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
810 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
811 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
812 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
813 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
814 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
815 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
816
817 tegra_pcie_enable_system_interrupts(pp);
818 tegra_pcie_enable_legacy_interrupts(pp);
819 if (IS_ENABLED(CONFIG_PCI_MSI))
820 tegra_pcie_enable_msi_interrupts(pp);
821}
822
823static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
824{
825 struct dw_pcie *pci = &pcie->pci;
826 u32 val, offset, i;
827
828 /* Program init preset */
829 for (i = 0; i < pcie->num_lanes; i++) {
830 dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF
831 + (i * 2), 2, &val);
832 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
833 val |= GEN3_GEN4_EQ_PRESET_INIT;
834 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
835 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
836 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
837 dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF
838 + (i * 2), 2, val);
839
840 offset = dw_pcie_find_ext_capability(pci,
841 PCI_EXT_CAP_ID_PL_16GT) +
842 PCI_PL_16GT_LE_CTRL;
843 dw_pcie_read(pci->dbi_base + offset + i, 1, &val);
844 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
845 val |= GEN3_GEN4_EQ_PRESET_INIT;
846 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
847 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
848 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
849 dw_pcie_write(pci->dbi_base + offset + i, 1, val);
850 }
851
852 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
853 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
854 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
855
856 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
857 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
858 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
859 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
860 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
861
862 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
863 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
864 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
865 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
866
867 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
868 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
869 val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
870 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
871 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
872
873 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
874 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
875 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
876}
877
878static void tegra_pcie_prepare_host(struct pcie_port *pp)
879{
880 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
881 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
882 u32 val;
883
884 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
885 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
886 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
887
888 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
889 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
890 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
891 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
892
893 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
894
895 /* Configure FTS */
896 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
897 val &= ~(N_FTS_MASK << N_FTS_SHIFT);
898 val |= N_FTS_VAL << N_FTS_SHIFT;
899 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
900
901 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
902 val &= ~FTS_MASK;
903 val |= FTS_VAL;
904 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
905
906 /* Enable as 0xFFFF0001 response for CRS */
907 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
908 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
909 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
910 AMBA_ERROR_RESPONSE_CRS_SHIFT);
911 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
912
913 /* Configure Max Speed from DT */
914 if (pcie->max_speed && pcie->max_speed != -EINVAL) {
915 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base +
916 PCI_EXP_LNKCAP);
917 val &= ~PCI_EXP_LNKCAP_SLS;
918 val |= pcie->max_speed;
919 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP,
920 val);
921 }
922
923 /* Configure Max lane width from DT */
924 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
925 val &= ~PCI_EXP_LNKCAP_MLW;
926 val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
927 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
928
929 config_gen3_gen4_eq_presets(pcie);
930
931 init_host_aspm(pcie);
932
933 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
934 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
935 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
936
937 if (pcie->update_fc_fixup) {
938 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
939 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
940 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
941 }
942
943 dw_pcie_setup_rc(pp);
944
945 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
946
947 /* Assert RST */
948 val = appl_readl(pcie, APPL_PINMUX);
949 val &= ~APPL_PINMUX_PEX_RST;
950 appl_writel(pcie, val, APPL_PINMUX);
951
952 usleep_range(100, 200);
953
954 /* Enable LTSSM */
955 val = appl_readl(pcie, APPL_CTRL);
956 val |= APPL_CTRL_LTSSM_EN;
957 appl_writel(pcie, val, APPL_CTRL);
958
959 /* De-assert RST */
960 val = appl_readl(pcie, APPL_PINMUX);
961 val |= APPL_PINMUX_PEX_RST;
962 appl_writel(pcie, val, APPL_PINMUX);
963
964 msleep(100);
965}
966
967static int tegra_pcie_dw_host_init(struct pcie_port *pp)
968{
969 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
970 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
971 u32 val, tmp, offset, speed;
972
973 tegra_pcie_prepare_host(pp);
974
975 if (dw_pcie_wait_for_link(pci)) {
976 /*
977 * There are some endpoints which can't get the link up if
978 * root port has Data Link Feature (DLF) enabled.
979 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
980 * on Scaled Flow Control and DLF.
981 * So, need to confirm that is indeed the case here and attempt
982 * link up once again with DLF disabled.
983 */
984 val = appl_readl(pcie, APPL_DEBUG);
985 val &= APPL_DEBUG_LTSSM_STATE_MASK;
986 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
987 tmp = appl_readl(pcie, APPL_LINK_STATUS);
988 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
989 if (!(val == 0x11 && !tmp)) {
990 /* Link is down for all good reasons */
991 return 0;
992 }
993
994 dev_info(pci->dev, "Link is down in DLL");
995 dev_info(pci->dev, "Trying again with DLFE disabled\n");
996 /* Disable LTSSM */
997 val = appl_readl(pcie, APPL_CTRL);
998 val &= ~APPL_CTRL_LTSSM_EN;
999 appl_writel(pcie, val, APPL_CTRL);
1000
1001 reset_control_assert(pcie->core_rst);
1002 reset_control_deassert(pcie->core_rst);
1003
1004 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1005 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1006 val &= ~PCI_DLF_EXCHANGE_ENABLE;
1007 dw_pcie_writel_dbi(pci, offset, val);
1008
1009 tegra_pcie_prepare_host(pp);
1010
1011 if (dw_pcie_wait_for_link(pci))
1012 return 0;
1013 }
1014
1015 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
1016 PCI_EXP_LNKSTA_CLS;
1017 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
1018
1019 tegra_pcie_enable_interrupts(pp);
1020
1021 return 0;
1022}
1023
1024static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1025{
1026 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1027 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1028
1029 return !!(val & PCI_EXP_LNKSTA_DLLLA);
1030}
1031
1032static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp)
1033{
1034 pp->num_vectors = MAX_MSI_IRQS;
1035}
1036
1037static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
1038{
1039 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1040
1041 enable_irq(pcie->pex_rst_irq);
1042
1043 return 0;
1044}
1045
1046static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1047{
1048 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1049
1050 disable_irq(pcie->pex_rst_irq);
1051}
1052
1053static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1054 .link_up = tegra_pcie_dw_link_up,
1055 .start_link = tegra_pcie_dw_start_link,
1056 .stop_link = tegra_pcie_dw_stop_link,
1057};
1058
1059static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1060 .rd_own_conf = tegra_pcie_dw_rd_own_conf,
1061 .wr_own_conf = tegra_pcie_dw_wr_own_conf,
1062 .host_init = tegra_pcie_dw_host_init,
1063 .set_num_vectors = tegra_pcie_set_msi_vec_num,
1064};
1065
1066static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1067{
1068 unsigned int phy_count = pcie->phy_count;
1069
1070 while (phy_count--) {
1071 phy_power_off(pcie->phys[phy_count]);
1072 phy_exit(pcie->phys[phy_count]);
1073 }
1074}
1075
1076static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1077{
1078 unsigned int i;
1079 int ret;
1080
1081 for (i = 0; i < pcie->phy_count; i++) {
1082 ret = phy_init(pcie->phys[i]);
1083 if (ret < 0)
1084 goto phy_power_off;
1085
1086 ret = phy_power_on(pcie->phys[i]);
1087 if (ret < 0)
1088 goto phy_exit;
1089 }
1090
1091 return 0;
1092
1093phy_power_off:
1094 while (i--) {
1095 phy_power_off(pcie->phys[i]);
1096phy_exit:
1097 phy_exit(pcie->phys[i]);
1098 }
1099
1100 return ret;
1101}
1102
1103static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1104{
1105 struct device_node *np = pcie->dev->of_node;
1106 int ret;
1107
1108 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1109 if (ret < 0) {
1110 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1111 return ret;
1112 }
1113
1114 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1115 &pcie->aspm_pwr_on_t);
1116 if (ret < 0)
1117 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1118 ret);
1119
1120 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1121 &pcie->aspm_l0s_enter_lat);
1122 if (ret < 0)
1123 dev_info(pcie->dev,
1124 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1125
1126 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1127 if (ret < 0) {
1128 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1129 return ret;
1130 }
1131
1132 pcie->max_speed = of_pci_get_max_link_speed(np);
1133
1134 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1135 if (ret) {
1136 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1137 return ret;
1138 }
1139
1140 ret = of_property_count_strings(np, "phy-names");
1141 if (ret < 0) {
1142 dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1143 ret);
1144 return ret;
1145 }
1146 pcie->phy_count = ret;
1147
1148 if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1149 pcie->update_fc_fixup = true;
1150
1151 pcie->supports_clkreq =
1152 of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1153
1154 pcie->enable_cdm_check =
1155 of_property_read_bool(np, "snps,enable-cdm-check");
1156
1157 if (pcie->mode == DW_PCIE_RC_TYPE)
1158 return 0;
1159
1160 /* Endpoint mode specific DT entries */
1161 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1162 if (IS_ERR(pcie->pex_rst_gpiod)) {
1163 int err = PTR_ERR(pcie->pex_rst_gpiod);
1164 const char *level = KERN_ERR;
1165
1166 if (err == -EPROBE_DEFER)
1167 level = KERN_DEBUG;
1168
1169 dev_printk(level, pcie->dev,
1170 dev_fmt("Failed to get PERST GPIO: %d\n"),
1171 err);
1172 return err;
1173 }
1174
1175 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1176 "nvidia,refclk-select",
1177 GPIOD_OUT_HIGH);
1178 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1179 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1180 const char *level = KERN_ERR;
1181
1182 if (err == -EPROBE_DEFER)
1183 level = KERN_DEBUG;
1184
1185 dev_printk(level, pcie->dev,
1186 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1187 err);
1188 pcie->pex_refclk_sel_gpiod = NULL;
1189 }
1190
1191 return 0;
1192}
1193
1194static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1195 bool enable)
1196{
1197 struct mrq_uphy_response resp;
1198 struct tegra_bpmp_message msg;
1199 struct mrq_uphy_request req;
1200
1201 /* Controller-5 doesn't need to have its state set by BPMP-FW */
1202 if (pcie->cid == 5)
1203 return 0;
1204
1205 memset(&req, 0, sizeof(req));
1206 memset(&resp, 0, sizeof(resp));
1207
1208 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1209 req.controller_state.pcie_controller = pcie->cid;
1210 req.controller_state.enable = enable;
1211
1212 memset(&msg, 0, sizeof(msg));
1213 msg.mrq = MRQ_UPHY;
1214 msg.tx.data = &req;
1215 msg.tx.size = sizeof(req);
1216 msg.rx.data = &resp;
1217 msg.rx.size = sizeof(resp);
1218
1219 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1220}
1221
1222static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1223 bool enable)
1224{
1225 struct mrq_uphy_response resp;
1226 struct tegra_bpmp_message msg;
1227 struct mrq_uphy_request req;
1228
1229 memset(&req, 0, sizeof(req));
1230 memset(&resp, 0, sizeof(resp));
1231
1232 if (enable) {
1233 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1234 req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1235 } else {
1236 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1237 req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1238 }
1239
1240 memset(&msg, 0, sizeof(msg));
1241 msg.mrq = MRQ_UPHY;
1242 msg.tx.data = &req;
1243 msg.tx.size = sizeof(req);
1244 msg.rx.data = &resp;
1245 msg.rx.size = sizeof(resp);
1246
1247 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1248}
1249
1250static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1251{
1252 struct pcie_port *pp = &pcie->pci.pp;
1253 struct pci_bus *child, *root_bus = NULL;
1254 struct pci_dev *pdev;
1255
1256 /*
1257 * link doesn't go into L2 state with some of the endpoints with Tegra
1258 * if they are not in D0 state. So, need to make sure that immediate
1259 * downstream devices are in D0 state before sending PME_TurnOff to put
1260 * link into L2 state.
1261 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1262 * 5.2 Link State Power Management (Page #428).
1263 */
1264
1265 list_for_each_entry(child, &pp->root_bus->children, node) {
1266 /* Bring downstream devices to D0 if they are not already in */
1267 if (child->parent == pp->root_bus) {
1268 root_bus = child;
1269 break;
1270 }
1271 }
1272
1273 if (!root_bus) {
1274 dev_err(pcie->dev, "Failed to find downstream devices\n");
1275 return;
1276 }
1277
1278 list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1279 if (PCI_SLOT(pdev->devfn) == 0) {
1280 if (pci_set_power_state(pdev, PCI_D0))
1281 dev_err(pcie->dev,
1282 "Failed to transition %s to D0 state\n",
1283 dev_name(&pdev->dev));
1284 }
1285 }
1286}
1287
1288static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1289{
1290 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1291 if (IS_ERR(pcie->slot_ctl_3v3)) {
1292 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1293 return PTR_ERR(pcie->slot_ctl_3v3);
1294
1295 pcie->slot_ctl_3v3 = NULL;
1296 }
1297
1298 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1299 if (IS_ERR(pcie->slot_ctl_12v)) {
1300 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1301 return PTR_ERR(pcie->slot_ctl_12v);
1302
1303 pcie->slot_ctl_12v = NULL;
1304 }
1305
1306 return 0;
1307}
1308
1309static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1310{
1311 int ret;
1312
1313 if (pcie->slot_ctl_3v3) {
1314 ret = regulator_enable(pcie->slot_ctl_3v3);
1315 if (ret < 0) {
1316 dev_err(pcie->dev,
1317 "Failed to enable 3.3V slot supply: %d\n", ret);
1318 return ret;
1319 }
1320 }
1321
1322 if (pcie->slot_ctl_12v) {
1323 ret = regulator_enable(pcie->slot_ctl_12v);
1324 if (ret < 0) {
1325 dev_err(pcie->dev,
1326 "Failed to enable 12V slot supply: %d\n", ret);
1327 goto fail_12v_enable;
1328 }
1329 }
1330
1331 /*
1332 * According to PCI Express Card Electromechanical Specification
1333 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1334 * should be a minimum of 100ms.
1335 */
1336 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1337 msleep(100);
1338
1339 return 0;
1340
1341fail_12v_enable:
1342 if (pcie->slot_ctl_3v3)
1343 regulator_disable(pcie->slot_ctl_3v3);
1344 return ret;
1345}
1346
1347static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1348{
1349 if (pcie->slot_ctl_12v)
1350 regulator_disable(pcie->slot_ctl_12v);
1351 if (pcie->slot_ctl_3v3)
1352 regulator_disable(pcie->slot_ctl_3v3);
1353}
1354
1355static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1356 bool en_hw_hot_rst)
1357{
1358 int ret;
1359 u32 val;
1360
1361 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1362 if (ret) {
1363 dev_err(pcie->dev,
1364 "Failed to enable controller %u: %d\n", pcie->cid, ret);
1365 return ret;
1366 }
1367
1368 ret = tegra_pcie_enable_slot_regulators(pcie);
1369 if (ret < 0)
1370 goto fail_slot_reg_en;
1371
1372 ret = regulator_enable(pcie->pex_ctl_supply);
1373 if (ret < 0) {
1374 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1375 goto fail_reg_en;
1376 }
1377
1378 ret = clk_prepare_enable(pcie->core_clk);
1379 if (ret) {
1380 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1381 goto fail_core_clk;
1382 }
1383
1384 ret = reset_control_deassert(pcie->core_apb_rst);
1385 if (ret) {
1386 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1387 ret);
1388 goto fail_core_apb_rst;
1389 }
1390
1391 if (en_hw_hot_rst) {
1392 /* Enable HW_HOT_RST mode */
1393 val = appl_readl(pcie, APPL_CTRL);
1394 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1395 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1396 val |= APPL_CTRL_HW_HOT_RST_EN;
1397 appl_writel(pcie, val, APPL_CTRL);
1398 }
1399
1400 ret = tegra_pcie_enable_phy(pcie);
1401 if (ret) {
1402 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1403 goto fail_phy;
1404 }
1405
1406 /* Update CFG base address */
1407 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1408 APPL_CFG_BASE_ADDR);
1409
1410 /* Configure this core for RP mode operation */
1411 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1412
1413 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1414
1415 val = appl_readl(pcie, APPL_CTRL);
1416 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1417
1418 val = appl_readl(pcie, APPL_CFG_MISC);
1419 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1420 appl_writel(pcie, val, APPL_CFG_MISC);
1421
1422 if (!pcie->supports_clkreq) {
1423 val = appl_readl(pcie, APPL_PINMUX);
1424 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1425 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1426 appl_writel(pcie, val, APPL_PINMUX);
1427 }
1428
1429 /* Update iATU_DMA base address */
1430 appl_writel(pcie,
1431 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1432 APPL_CFG_IATU_DMA_BASE_ADDR);
1433
1434 reset_control_deassert(pcie->core_rst);
1435
1436 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1437 PCI_CAP_ID_EXP);
1438
1439 /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */
1440 if (!pcie->supports_clkreq) {
1441 disable_aspm_l11(pcie);
1442 disable_aspm_l12(pcie);
1443 }
1444
1445 return ret;
1446
1447fail_phy:
1448 reset_control_assert(pcie->core_apb_rst);
1449fail_core_apb_rst:
1450 clk_disable_unprepare(pcie->core_clk);
1451fail_core_clk:
1452 regulator_disable(pcie->pex_ctl_supply);
1453fail_reg_en:
1454 tegra_pcie_disable_slot_regulators(pcie);
1455fail_slot_reg_en:
1456 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1457
1458 return ret;
1459}
1460
1461static int __deinit_controller(struct tegra_pcie_dw *pcie)
1462{
1463 int ret;
1464
1465 ret = reset_control_assert(pcie->core_rst);
1466 if (ret) {
1467 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n",
1468 ret);
1469 return ret;
1470 }
1471
1472 tegra_pcie_disable_phy(pcie);
1473
1474 ret = reset_control_assert(pcie->core_apb_rst);
1475 if (ret) {
1476 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1477 return ret;
1478 }
1479
1480 clk_disable_unprepare(pcie->core_clk);
1481
1482 ret = regulator_disable(pcie->pex_ctl_supply);
1483 if (ret) {
1484 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1485 return ret;
1486 }
1487
1488 tegra_pcie_disable_slot_regulators(pcie);
1489
1490 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1491 if (ret) {
1492 dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1493 pcie->cid, ret);
1494 return ret;
1495 }
1496
1497 return ret;
1498}
1499
1500static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1501{
1502 struct dw_pcie *pci = &pcie->pci;
1503 struct pcie_port *pp = &pci->pp;
1504 int ret;
1505
1506 ret = tegra_pcie_config_controller(pcie, false);
1507 if (ret < 0)
1508 return ret;
1509
1510 pp->ops = &tegra_pcie_dw_host_ops;
1511
1512 ret = dw_pcie_host_init(pp);
1513 if (ret < 0) {
1514 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1515 goto fail_host_init;
1516 }
1517
1518 return 0;
1519
1520fail_host_init:
1521 return __deinit_controller(pcie);
1522}
1523
1524static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1525{
1526 u32 val;
1527
1528 if (!tegra_pcie_dw_link_up(&pcie->pci))
1529 return 0;
1530
1531 val = appl_readl(pcie, APPL_RADM_STATUS);
1532 val |= APPL_PM_XMT_TURNOFF_STATE;
1533 appl_writel(pcie, val, APPL_RADM_STATUS);
1534
1535 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1536 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1537 1, PME_ACK_TIMEOUT);
1538}
1539
1540static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1541{
1542 u32 data;
1543 int err;
1544
1545 if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1546 dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1547 return;
1548 }
1549
1550 if (tegra_pcie_try_link_l2(pcie)) {
1551 dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1552 /*
1553 * TX lane clock freq will reset to Gen1 only if link is in L2
1554 * or detect state.
1555 * So apply pex_rst to end point to force RP to go into detect
1556 * state
1557 */
1558 data = appl_readl(pcie, APPL_PINMUX);
1559 data &= ~APPL_PINMUX_PEX_RST;
1560 appl_writel(pcie, data, APPL_PINMUX);
1561
1562 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1563 data,
1564 ((data &
1565 APPL_DEBUG_LTSSM_STATE_MASK) >>
1566 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1567 LTSSM_STATE_PRE_DETECT,
1568 1, LTSSM_TIMEOUT);
1569 if (err) {
1570 dev_info(pcie->dev, "Link didn't go to detect state\n");
1571 } else {
1572 /* Disable LTSSM after link is in detect state */
1573 data = appl_readl(pcie, APPL_CTRL);
1574 data &= ~APPL_CTRL_LTSSM_EN;
1575 appl_writel(pcie, data, APPL_CTRL);
1576 }
1577 }
1578 /*
1579 * DBI registers may not be accessible after this as PLL-E would be
1580 * down depending on how CLKREQ is pulled by end point
1581 */
1582 data = appl_readl(pcie, APPL_PINMUX);
1583 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1584 /* Cut REFCLK to slot */
1585 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1586 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1587 appl_writel(pcie, data, APPL_PINMUX);
1588}
1589
1590static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1591{
1592 tegra_pcie_downstream_dev_to_D0(pcie);
1593 dw_pcie_host_deinit(&pcie->pci.pp);
1594 tegra_pcie_dw_pme_turnoff(pcie);
1595
1596 return __deinit_controller(pcie);
1597}
1598
1599static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1600{
1601 struct pcie_port *pp = &pcie->pci.pp;
1602 struct device *dev = pcie->dev;
1603 char *name;
1604 int ret;
1605
1606 if (IS_ENABLED(CONFIG_PCI_MSI)) {
1607 pp->msi_irq = of_irq_get_byname(dev->of_node, "msi");
1608 if (!pp->msi_irq) {
1609 dev_err(dev, "Failed to get MSI interrupt\n");
1610 return -ENODEV;
1611 }
1612 }
1613
1614 pm_runtime_enable(dev);
1615
1616 ret = pm_runtime_get_sync(dev);
1617 if (ret < 0) {
1618 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1619 ret);
1620 goto fail_pm_get_sync;
1621 }
1622
1623 ret = pinctrl_pm_select_default_state(dev);
1624 if (ret < 0) {
1625 dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1626 goto fail_pm_get_sync;
1627 }
1628
1629 tegra_pcie_init_controller(pcie);
1630
1631 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1632 if (!pcie->link_state) {
1633 ret = -ENOMEDIUM;
1634 goto fail_host_init;
1635 }
1636
1637 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1638 if (!name) {
1639 ret = -ENOMEM;
1640 goto fail_host_init;
1641 }
1642
1643 pcie->debugfs = debugfs_create_dir(name, NULL);
1644 if (!pcie->debugfs)
1645 dev_err(dev, "Failed to create debugfs\n");
1646 else
1647 init_debugfs(pcie);
1648
1649 return ret;
1650
1651fail_host_init:
1652 tegra_pcie_deinit_controller(pcie);
1653fail_pm_get_sync:
1654 pm_runtime_put_sync(dev);
1655 pm_runtime_disable(dev);
1656 return ret;
1657}
1658
1659static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1660{
1661 u32 val;
1662 int ret;
1663
1664 if (pcie->ep_state == EP_STATE_DISABLED)
1665 return;
1666
1667 /* Disable LTSSM */
1668 val = appl_readl(pcie, APPL_CTRL);
1669 val &= ~APPL_CTRL_LTSSM_EN;
1670 appl_writel(pcie, val, APPL_CTRL);
1671
1672 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1673 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1674 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1675 LTSSM_STATE_PRE_DETECT,
1676 1, LTSSM_TIMEOUT);
1677 if (ret)
1678 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1679
1680 reset_control_assert(pcie->core_rst);
1681
1682 tegra_pcie_disable_phy(pcie);
1683
1684 reset_control_assert(pcie->core_apb_rst);
1685
1686 clk_disable_unprepare(pcie->core_clk);
1687
1688 pm_runtime_put_sync(pcie->dev);
1689
1690 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1691 if (ret)
1692 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1693
1694 pcie->ep_state = EP_STATE_DISABLED;
1695 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1696}
1697
1698static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1699{
1700 struct dw_pcie *pci = &pcie->pci;
1701 struct dw_pcie_ep *ep = &pci->ep;
1702 struct device *dev = pcie->dev;
1703 u32 val;
1704 int ret;
1705
1706 if (pcie->ep_state == EP_STATE_ENABLED)
1707 return;
1708
1709 ret = pm_runtime_get_sync(dev);
1710 if (ret < 0) {
1711 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1712 ret);
1713 return;
1714 }
1715
1716 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1717 if (ret) {
1718 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret);
1719 goto fail_pll_init;
1720 }
1721
1722 ret = clk_prepare_enable(pcie->core_clk);
1723 if (ret) {
1724 dev_err(dev, "Failed to enable core clock: %d\n", ret);
1725 goto fail_core_clk_enable;
1726 }
1727
1728 ret = reset_control_deassert(pcie->core_apb_rst);
1729 if (ret) {
1730 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1731 goto fail_core_apb_rst;
1732 }
1733
1734 ret = tegra_pcie_enable_phy(pcie);
1735 if (ret) {
1736 dev_err(dev, "Failed to enable PHY: %d\n", ret);
1737 goto fail_phy;
1738 }
1739
1740 /* Clear any stale interrupt statuses */
1741 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1742 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1743 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1744 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1745 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1746 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1747 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1748 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1749 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1750 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1751 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1752 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1753 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1754 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1755 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1756
1757 /* configure this core for EP mode operation */
1758 val = appl_readl(pcie, APPL_DM_TYPE);
1759 val &= ~APPL_DM_TYPE_MASK;
1760 val |= APPL_DM_TYPE_EP;
1761 appl_writel(pcie, val, APPL_DM_TYPE);
1762
1763 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1764
1765 val = appl_readl(pcie, APPL_CTRL);
1766 val |= APPL_CTRL_SYS_PRE_DET_STATE;
1767 val |= APPL_CTRL_HW_HOT_RST_EN;
1768 appl_writel(pcie, val, APPL_CTRL);
1769
1770 val = appl_readl(pcie, APPL_CFG_MISC);
1771 val |= APPL_CFG_MISC_SLV_EP_MODE;
1772 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1773 appl_writel(pcie, val, APPL_CFG_MISC);
1774
1775 val = appl_readl(pcie, APPL_PINMUX);
1776 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1777 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1778 appl_writel(pcie, val, APPL_PINMUX);
1779
1780 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1781 APPL_CFG_BASE_ADDR);
1782
1783 appl_writel(pcie, pcie->atu_dma_res->start &
1784 APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1785 APPL_CFG_IATU_DMA_BASE_ADDR);
1786
1787 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1788 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1789 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1790 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1791 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1792
1793 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1794 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1795 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1796 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1797
1798 reset_control_deassert(pcie->core_rst);
1799
1800 if (pcie->update_fc_fixup) {
1801 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1802 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1803 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1804 }
1805
1806 config_gen3_gen4_eq_presets(pcie);
1807
1808 init_host_aspm(pcie);
1809
1810 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1811 if (!pcie->supports_clkreq) {
1812 disable_aspm_l11(pcie);
1813 disable_aspm_l12(pcie);
1814 }
1815
1816 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1817 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1818 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1819
1820 /* Configure N_FTS & FTS */
1821 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
1822 val &= ~(N_FTS_MASK << N_FTS_SHIFT);
1823 val |= N_FTS_VAL << N_FTS_SHIFT;
1824 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
1825
1826 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
1827 val &= ~FTS_MASK;
1828 val |= FTS_VAL;
1829 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
1830
1831 /* Configure Max Speed from DT */
1832 if (pcie->max_speed && pcie->max_speed != -EINVAL) {
1833 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base +
1834 PCI_EXP_LNKCAP);
1835 val &= ~PCI_EXP_LNKCAP_SLS;
1836 val |= pcie->max_speed;
1837 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP,
1838 val);
1839 }
1840
1841 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1842 PCI_CAP_ID_EXP);
1843 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1844
1845 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1846 val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1847 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1848 val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1849 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1850
1851 ret = dw_pcie_ep_init_complete(ep);
1852 if (ret) {
1853 dev_err(dev, "Failed to complete initialization: %d\n", ret);
1854 goto fail_init_complete;
1855 }
1856
1857 dw_pcie_ep_init_notify(ep);
1858
1859 /* Enable LTSSM */
1860 val = appl_readl(pcie, APPL_CTRL);
1861 val |= APPL_CTRL_LTSSM_EN;
1862 appl_writel(pcie, val, APPL_CTRL);
1863
1864 pcie->ep_state = EP_STATE_ENABLED;
1865 dev_dbg(dev, "Initialization of endpoint is completed\n");
1866
1867 return;
1868
1869fail_init_complete:
1870 reset_control_assert(pcie->core_rst);
1871 tegra_pcie_disable_phy(pcie);
1872fail_phy:
1873 reset_control_assert(pcie->core_apb_rst);
1874fail_core_apb_rst:
1875 clk_disable_unprepare(pcie->core_clk);
1876fail_core_clk_enable:
1877 tegra_pcie_bpmp_set_pll_state(pcie, false);
1878fail_pll_init:
1879 pm_runtime_put_sync(dev);
1880}
1881
1882static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1883{
1884 struct tegra_pcie_dw *pcie = arg;
1885
1886 if (gpiod_get_value(pcie->pex_rst_gpiod))
1887 pex_ep_event_pex_rst_assert(pcie);
1888 else
1889 pex_ep_event_pex_rst_deassert(pcie);
1890
1891 return IRQ_HANDLED;
1892}
1893
1894static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1895{
1896 /* Tegra194 supports only INTA */
1897 if (irq > 1)
1898 return -EINVAL;
1899
1900 appl_writel(pcie, 1, APPL_LEGACY_INTX);
1901 usleep_range(1000, 2000);
1902 appl_writel(pcie, 0, APPL_LEGACY_INTX);
1903 return 0;
1904}
1905
1906static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1907{
1908 if (unlikely(irq > 31))
1909 return -EINVAL;
1910
1911 appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1);
1912
1913 return 0;
1914}
1915
1916static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1917{
1918 struct dw_pcie_ep *ep = &pcie->pci.ep;
1919
1920 writel(irq, ep->msi_mem);
1921
1922 return 0;
1923}
1924
1925static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1926 enum pci_epc_irq_type type,
1927 u16 interrupt_num)
1928{
1929 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1930 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1931
1932 switch (type) {
1933 case PCI_EPC_IRQ_LEGACY:
1934 return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
1935
1936 case PCI_EPC_IRQ_MSI:
1937 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1938
1939 case PCI_EPC_IRQ_MSIX:
1940 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1941
1942 default:
1943 dev_err(pci->dev, "Unknown IRQ type\n");
1944 return -EPERM;
1945 }
1946
1947 return 0;
1948}
1949
1950static const struct pci_epc_features tegra_pcie_epc_features = {
1951 .linkup_notifier = true,
1952 .core_init_notifier = true,
1953 .msi_capable = false,
1954 .msix_capable = false,
1955 .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
1956 .bar_fixed_64bit = 1 << BAR_0,
1957 .bar_fixed_size[0] = SZ_1M,
1958};
1959
1960static const struct pci_epc_features*
1961tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
1962{
1963 return &tegra_pcie_epc_features;
1964}
1965
1966static struct dw_pcie_ep_ops pcie_ep_ops = {
1967 .raise_irq = tegra_pcie_ep_raise_irq,
1968 .get_features = tegra_pcie_ep_get_features,
1969};
1970
1971static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
1972 struct platform_device *pdev)
1973{
1974 struct dw_pcie *pci = &pcie->pci;
1975 struct device *dev = pcie->dev;
1976 struct dw_pcie_ep *ep;
1977 struct resource *res;
1978 char *name;
1979 int ret;
1980
1981 ep = &pci->ep;
1982 ep->ops = &pcie_ep_ops;
1983
1984 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
1985 if (!res)
1986 return -EINVAL;
1987
1988 ep->phys_base = res->start;
1989 ep->addr_size = resource_size(res);
1990 ep->page_size = SZ_64K;
1991
1992 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
1993 if (ret < 0) {
1994 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
1995 ret);
1996 return ret;
1997 }
1998
1999 ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2000 if (ret < 0) {
2001 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2002 return ret;
2003 }
2004 pcie->pex_rst_irq = (unsigned int)ret;
2005
2006 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2007 pcie->cid);
2008 if (!name) {
2009 dev_err(dev, "Failed to create PERST IRQ string\n");
2010 return -ENOMEM;
2011 }
2012
2013 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2014
2015 pcie->ep_state = EP_STATE_DISABLED;
2016
2017 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2018 tegra_pcie_ep_pex_rst_irq,
2019 IRQF_TRIGGER_RISING |
2020 IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2021 name, (void *)pcie);
2022 if (ret < 0) {
2023 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2024 return ret;
2025 }
2026
2027 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work",
2028 pcie->cid);
2029 if (!name) {
2030 dev_err(dev, "Failed to create PCIe EP work thread string\n");
2031 return -ENOMEM;
2032 }
2033
2034 pm_runtime_enable(dev);
2035
2036 ret = dw_pcie_ep_init(ep);
2037 if (ret) {
2038 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2039 ret);
2040 return ret;
2041 }
2042
2043 return 0;
2044}
2045
2046static int tegra_pcie_dw_probe(struct platform_device *pdev)
2047{
2048 const struct tegra_pcie_dw_of_data *data;
2049 struct device *dev = &pdev->dev;
2050 struct resource *atu_dma_res;
2051 struct tegra_pcie_dw *pcie;
2052 struct resource *dbi_res;
2053 struct pcie_port *pp;
2054 struct dw_pcie *pci;
2055 struct phy **phys;
2056 char *name;
2057 int ret;
2058 u32 i;
2059
2060 data = of_device_get_match_data(dev);
2061
2062 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2063 if (!pcie)
2064 return -ENOMEM;
2065
2066 pci = &pcie->pci;
2067 pci->dev = &pdev->dev;
2068 pci->ops = &tegra_dw_pcie_ops;
2069 pp = &pci->pp;
2070 pcie->dev = &pdev->dev;
2071 pcie->mode = (enum dw_pcie_device_mode)data->mode;
2072
2073 ret = tegra_pcie_dw_parse_dt(pcie);
2074 if (ret < 0) {
2075 const char *level = KERN_ERR;
2076
2077 if (ret == -EPROBE_DEFER)
2078 level = KERN_DEBUG;
2079
2080 dev_printk(level, dev,
2081 dev_fmt("Failed to parse device tree: %d\n"),
2082 ret);
2083 return ret;
2084 }
2085
2086 ret = tegra_pcie_get_slot_regulators(pcie);
2087 if (ret < 0) {
2088 const char *level = KERN_ERR;
2089
2090 if (ret == -EPROBE_DEFER)
2091 level = KERN_DEBUG;
2092
2093 dev_printk(level, dev,
2094 dev_fmt("Failed to get slot regulators: %d\n"),
2095 ret);
2096 return ret;
2097 }
2098
2099 if (pcie->pex_refclk_sel_gpiod)
2100 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2101
2102 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2103 if (IS_ERR(pcie->pex_ctl_supply)) {
2104 ret = PTR_ERR(pcie->pex_ctl_supply);
2105 if (ret != -EPROBE_DEFER)
2106 dev_err(dev, "Failed to get regulator: %ld\n",
2107 PTR_ERR(pcie->pex_ctl_supply));
2108 return ret;
2109 }
2110
2111 pcie->core_clk = devm_clk_get(dev, "core");
2112 if (IS_ERR(pcie->core_clk)) {
2113 dev_err(dev, "Failed to get core clock: %ld\n",
2114 PTR_ERR(pcie->core_clk));
2115 return PTR_ERR(pcie->core_clk);
2116 }
2117
2118 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2119 "appl");
2120 if (!pcie->appl_res) {
2121 dev_err(dev, "Failed to find \"appl\" region\n");
2122 return -ENODEV;
2123 }
2124
2125 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2126 if (IS_ERR(pcie->appl_base))
2127 return PTR_ERR(pcie->appl_base);
2128
2129 pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2130 if (IS_ERR(pcie->core_apb_rst)) {
2131 dev_err(dev, "Failed to get APB reset: %ld\n",
2132 PTR_ERR(pcie->core_apb_rst));
2133 return PTR_ERR(pcie->core_apb_rst);
2134 }
2135
2136 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2137 if (!phys)
2138 return -ENOMEM;
2139
2140 for (i = 0; i < pcie->phy_count; i++) {
2141 name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2142 if (!name) {
2143 dev_err(dev, "Failed to create P2U string\n");
2144 return -ENOMEM;
2145 }
2146 phys[i] = devm_phy_get(dev, name);
2147 kfree(name);
2148 if (IS_ERR(phys[i])) {
2149 ret = PTR_ERR(phys[i]);
2150 if (ret != -EPROBE_DEFER)
2151 dev_err(dev, "Failed to get PHY: %d\n", ret);
2152 return ret;
2153 }
2154 }
2155
2156 pcie->phys = phys;
2157
2158 dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
2159 if (!dbi_res) {
2160 dev_err(dev, "Failed to find \"dbi\" region\n");
2161 return -ENODEV;
2162 }
2163 pcie->dbi_res = dbi_res;
2164
2165 pci->dbi_base = devm_ioremap_resource(dev, dbi_res);
2166 if (IS_ERR(pci->dbi_base))
2167 return PTR_ERR(pci->dbi_base);
2168
2169 /* Tegra HW locates DBI2 at a fixed offset from DBI */
2170 pci->dbi_base2 = pci->dbi_base + 0x1000;
2171
2172 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2173 "atu_dma");
2174 if (!atu_dma_res) {
2175 dev_err(dev, "Failed to find \"atu_dma\" region\n");
2176 return -ENODEV;
2177 }
2178 pcie->atu_dma_res = atu_dma_res;
2179
2180 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2181 if (IS_ERR(pci->atu_base))
2182 return PTR_ERR(pci->atu_base);
2183
2184 pcie->core_rst = devm_reset_control_get(dev, "core");
2185 if (IS_ERR(pcie->core_rst)) {
2186 dev_err(dev, "Failed to get core reset: %ld\n",
2187 PTR_ERR(pcie->core_rst));
2188 return PTR_ERR(pcie->core_rst);
2189 }
2190
2191 pp->irq = platform_get_irq_byname(pdev, "intr");
2192 if (pp->irq < 0)
2193 return pp->irq;
2194
2195 pcie->bpmp = tegra_bpmp_get(dev);
2196 if (IS_ERR(pcie->bpmp))
2197 return PTR_ERR(pcie->bpmp);
2198
2199 platform_set_drvdata(pdev, pcie);
2200
2201 switch (pcie->mode) {
2202 case DW_PCIE_RC_TYPE:
2203 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2204 IRQF_SHARED, "tegra-pcie-intr", pcie);
2205 if (ret) {
2206 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2207 ret);
2208 goto fail;
2209 }
2210
2211 ret = tegra_pcie_config_rp(pcie);
2212 if (ret && ret != -ENOMEDIUM)
2213 goto fail;
2214 else
2215 return 0;
2216 break;
2217
2218 case DW_PCIE_EP_TYPE:
2219 ret = devm_request_threaded_irq(dev, pp->irq,
2220 tegra_pcie_ep_hard_irq,
2221 tegra_pcie_ep_irq_thread,
2222 IRQF_SHARED | IRQF_ONESHOT,
2223 "tegra-pcie-ep-intr", pcie);
2224 if (ret) {
2225 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2226 ret);
2227 goto fail;
2228 }
2229
2230 ret = tegra_pcie_config_ep(pcie, pdev);
2231 if (ret < 0)
2232 goto fail;
2233 break;
2234
2235 default:
2236 dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode);
2237 }
2238
2239fail:
2240 tegra_bpmp_put(pcie->bpmp);
2241 return ret;
2242}
2243
2244static int tegra_pcie_dw_remove(struct platform_device *pdev)
2245{
2246 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2247
2248 if (!pcie->link_state)
2249 return 0;
2250
2251 debugfs_remove_recursive(pcie->debugfs);
2252 tegra_pcie_deinit_controller(pcie);
2253 pm_runtime_put_sync(pcie->dev);
2254 pm_runtime_disable(pcie->dev);
2255 tegra_bpmp_put(pcie->bpmp);
2256 if (pcie->pex_refclk_sel_gpiod)
2257 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2258
2259 return 0;
2260}
2261
2262static int tegra_pcie_dw_suspend_late(struct device *dev)
2263{
2264 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2265 u32 val;
2266
2267 if (!pcie->link_state)
2268 return 0;
2269
2270 /* Enable HW_HOT_RST mode */
2271 val = appl_readl(pcie, APPL_CTRL);
2272 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2273 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2274 val |= APPL_CTRL_HW_HOT_RST_EN;
2275 appl_writel(pcie, val, APPL_CTRL);
2276
2277 return 0;
2278}
2279
2280static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2281{
2282 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2283
2284 if (!pcie->link_state)
2285 return 0;
2286
2287 /* Save MSI interrupt vector */
2288 pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci,
2289 PORT_LOGIC_MSI_CTRL_INT_0_EN);
2290 tegra_pcie_downstream_dev_to_D0(pcie);
2291 tegra_pcie_dw_pme_turnoff(pcie);
2292
2293 return __deinit_controller(pcie);
2294}
2295
2296static int tegra_pcie_dw_resume_noirq(struct device *dev)
2297{
2298 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2299 int ret;
2300
2301 if (!pcie->link_state)
2302 return 0;
2303
2304 ret = tegra_pcie_config_controller(pcie, true);
2305 if (ret < 0)
2306 return ret;
2307
2308 ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2309 if (ret < 0) {
2310 dev_err(dev, "Failed to init host: %d\n", ret);
2311 goto fail_host_init;
2312 }
2313
2314 /* Restore MSI interrupt vector */
2315 dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN,
2316 pcie->msi_ctrl_int);
2317
2318 return 0;
2319
2320fail_host_init:
2321 return __deinit_controller(pcie);
2322}
2323
2324static int tegra_pcie_dw_resume_early(struct device *dev)
2325{
2326 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2327 u32 val;
2328
2329 if (!pcie->link_state)
2330 return 0;
2331
2332 /* Disable HW_HOT_RST mode */
2333 val = appl_readl(pcie, APPL_CTRL);
2334 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2335 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2336 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2337 APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2338 val &= ~APPL_CTRL_HW_HOT_RST_EN;
2339 appl_writel(pcie, val, APPL_CTRL);
2340
2341 return 0;
2342}
2343
2344static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2345{
2346 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2347
2348 if (!pcie->link_state)
2349 return;
2350
2351 debugfs_remove_recursive(pcie->debugfs);
2352 tegra_pcie_downstream_dev_to_D0(pcie);
2353
2354 disable_irq(pcie->pci.pp.irq);
2355 if (IS_ENABLED(CONFIG_PCI_MSI))
2356 disable_irq(pcie->pci.pp.msi_irq);
2357
2358 tegra_pcie_dw_pme_turnoff(pcie);
2359 __deinit_controller(pcie);
2360}
2361
2362static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = {
2363 .mode = DW_PCIE_RC_TYPE,
2364};
2365
2366static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = {
2367 .mode = DW_PCIE_EP_TYPE,
2368};
2369
2370static const struct of_device_id tegra_pcie_dw_of_match[] = {
2371 {
2372 .compatible = "nvidia,tegra194-pcie",
2373 .data = &tegra_pcie_dw_rc_of_data,
2374 },
2375 {
2376 .compatible = "nvidia,tegra194-pcie-ep",
2377 .data = &tegra_pcie_dw_ep_of_data,
2378 },
2379 {},
2380};
2381
2382static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2383 .suspend_late = tegra_pcie_dw_suspend_late,
2384 .suspend_noirq = tegra_pcie_dw_suspend_noirq,
2385 .resume_noirq = tegra_pcie_dw_resume_noirq,
2386 .resume_early = tegra_pcie_dw_resume_early,
2387};
2388
2389static struct platform_driver tegra_pcie_dw_driver = {
2390 .probe = tegra_pcie_dw_probe,
2391 .remove = tegra_pcie_dw_remove,
2392 .shutdown = tegra_pcie_dw_shutdown,
2393 .driver = {
2394 .name = "tegra194-pcie",
2395 .pm = &tegra_pcie_dw_pm_ops,
2396 .of_match_table = tegra_pcie_dw_of_match,
2397 },
2398};
2399module_platform_driver(tegra_pcie_dw_driver);
2400
2401MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2402
2403MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2404MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2405MODULE_LICENSE("GPL v2");