Loading...
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PCIe host controller driver for the following SoCs
4 * Tegra194
5 * Tegra234
6 *
7 * Copyright (C) 2019-2022 NVIDIA Corporation.
8 *
9 * Author: Vidya Sagar <vidyas@nvidia.com>
10 */
11
12#include <linux/bitfield.h>
13#include <linux/clk.h>
14#include <linux/debugfs.h>
15#include <linux/delay.h>
16#include <linux/gpio/consumer.h>
17#include <linux/interconnect.h>
18#include <linux/interrupt.h>
19#include <linux/iopoll.h>
20#include <linux/kernel.h>
21#include <linux/module.h>
22#include <linux/of.h>
23#include <linux/of_pci.h>
24#include <linux/pci.h>
25#include <linux/phy/phy.h>
26#include <linux/pinctrl/consumer.h>
27#include <linux/platform_device.h>
28#include <linux/pm_runtime.h>
29#include <linux/random.h>
30#include <linux/reset.h>
31#include <linux/resource.h>
32#include <linux/types.h>
33#include "pcie-designware.h"
34#include <soc/tegra/bpmp.h>
35#include <soc/tegra/bpmp-abi.h>
36#include "../../pci.h"
37
38#define TEGRA194_DWC_IP_VER 0x490A
39#define TEGRA234_DWC_IP_VER 0x562A
40
41#define APPL_PINMUX 0x0
42#define APPL_PINMUX_PEX_RST BIT(0)
43#define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)
44#define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)
45#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)
46#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)
47
48#define APPL_CTRL 0x4
49#define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)
50#define APPL_CTRL_LTSSM_EN BIT(7)
51#define APPL_CTRL_HW_HOT_RST_EN BIT(20)
52#define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)
53#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22
54#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1
55#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN 0x2
56
57#define APPL_INTR_EN_L0_0 0x8
58#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)
59#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)
60#define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)
61#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15)
62#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)
63#define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)
64#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)
65
66#define APPL_INTR_STATUS_L0 0xC
67#define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)
68#define APPL_INTR_STATUS_L0_INT_INT BIT(8)
69#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15)
70#define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16)
71#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)
72
73#define APPL_INTR_EN_L1_0_0 0x1C
74#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)
75#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3)
76#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30)
77
78#define APPL_INTR_STATUS_L1_0_0 0x20
79#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)
80#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3)
81#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30)
82
83#define APPL_INTR_STATUS_L1_1 0x2C
84#define APPL_INTR_STATUS_L1_2 0x30
85#define APPL_INTR_STATUS_L1_3 0x34
86#define APPL_INTR_STATUS_L1_6 0x3C
87#define APPL_INTR_STATUS_L1_7 0x40
88#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1)
89
90#define APPL_INTR_EN_L1_8_0 0x44
91#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)
92#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)
93#define APPL_INTR_EN_L1_8_INTX_EN BIT(11)
94#define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)
95
96#define APPL_INTR_STATUS_L1_8_0 0x4C
97#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)
98#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)
99#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)
100
101#define APPL_INTR_STATUS_L1_9 0x54
102#define APPL_INTR_STATUS_L1_10 0x58
103#define APPL_INTR_STATUS_L1_11 0x64
104#define APPL_INTR_STATUS_L1_13 0x74
105#define APPL_INTR_STATUS_L1_14 0x78
106#define APPL_INTR_STATUS_L1_15 0x7C
107#define APPL_INTR_STATUS_L1_17 0x88
108
109#define APPL_INTR_EN_L1_18 0x90
110#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)
111#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
112#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
113
114#define APPL_INTR_STATUS_L1_18 0x94
115#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)
116#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
117#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
118
119#define APPL_MSI_CTRL_1 0xAC
120
121#define APPL_MSI_CTRL_2 0xB0
122
123#define APPL_LEGACY_INTX 0xB8
124
125#define APPL_LTR_MSG_1 0xC4
126#define LTR_MSG_REQ BIT(15)
127#define LTR_NOSNOOP_MSG_REQ BIT(31)
128
129#define APPL_LTR_MSG_2 0xC8
130#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)
131
132#define APPL_LINK_STATUS 0xCC
133#define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)
134
135#define APPL_DEBUG 0xD0
136#define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)
137#define APPL_DEBUG_PM_LINKST_IN_L0 0x11
138#define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)
139#define APPL_DEBUG_LTSSM_STATE_SHIFT 3
140#define LTSSM_STATE_PRE_DETECT 5
141
142#define APPL_RADM_STATUS 0xE4
143#define APPL_PM_XMT_TURNOFF_STATE BIT(0)
144
145#define APPL_DM_TYPE 0x100
146#define APPL_DM_TYPE_MASK GENMASK(3, 0)
147#define APPL_DM_TYPE_RP 0x4
148#define APPL_DM_TYPE_EP 0x0
149
150#define APPL_CFG_BASE_ADDR 0x104
151#define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)
152
153#define APPL_CFG_IATU_DMA_BASE_ADDR 0x108
154#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)
155
156#define APPL_CFG_MISC 0x110
157#define APPL_CFG_MISC_SLV_EP_MODE BIT(14)
158#define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)
159#define APPL_CFG_MISC_ARCACHE_SHIFT 10
160#define APPL_CFG_MISC_ARCACHE_VAL 3
161
162#define APPL_CFG_SLCG_OVERRIDE 0x114
163#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)
164
165#define APPL_CAR_RESET_OVRD 0x12C
166#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)
167
168#define IO_BASE_IO_DECODE BIT(0)
169#define IO_BASE_IO_DECODE_BIT8 BIT(8)
170
171#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)
172#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)
173
174#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718
175#define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)
176
177#define N_FTS_VAL 52
178#define FTS_VAL 52
179
180#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0
181#define AMBA_ERROR_RESPONSE_RRS_SHIFT 3
182#define AMBA_ERROR_RESPONSE_RRS_MASK GENMASK(1, 0)
183#define AMBA_ERROR_RESPONSE_RRS_OKAY 0
184#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFFFFFF 1
185#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 2
186
187#define MSIX_ADDR_MATCH_LOW_OFF 0x940
188#define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0)
189#define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2)
190
191#define MSIX_ADDR_MATCH_HIGH_OFF 0x944
192#define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0)
193
194#define PORT_LOGIC_MSIX_DOORBELL 0x948
195
196#define CAP_SPCIE_CAP_OFF 0x154
197#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)
198#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)
199#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8
200
201#define PME_ACK_TIMEOUT 10000
202
203#define LTSSM_TIMEOUT 50000 /* 50ms */
204
205#define GEN3_GEN4_EQ_PRESET_INIT 5
206
207#define GEN1_CORE_CLK_FREQ 62500000
208#define GEN2_CORE_CLK_FREQ 125000000
209#define GEN3_CORE_CLK_FREQ 250000000
210#define GEN4_CORE_CLK_FREQ 500000000
211
212#define LTR_MSG_TIMEOUT (100 * 1000)
213
214#define PERST_DEBOUNCE_TIME (5 * 1000)
215
216#define EP_STATE_DISABLED 0
217#define EP_STATE_ENABLED 1
218
219static const unsigned int pcie_gen_freq[] = {
220 GEN1_CORE_CLK_FREQ, /* PCI_EXP_LNKSTA_CLS == 0; undefined */
221 GEN1_CORE_CLK_FREQ,
222 GEN2_CORE_CLK_FREQ,
223 GEN3_CORE_CLK_FREQ,
224 GEN4_CORE_CLK_FREQ
225};
226
227struct tegra_pcie_dw_of_data {
228 u32 version;
229 enum dw_pcie_device_mode mode;
230 bool has_msix_doorbell_access_fix;
231 bool has_sbr_reset_fix;
232 bool has_l1ss_exit_fix;
233 bool has_ltr_req_fix;
234 u32 cdm_chk_int_en_bit;
235 u32 gen4_preset_vec;
236 u8 n_fts[2];
237};
238
239struct tegra_pcie_dw {
240 struct device *dev;
241 struct resource *appl_res;
242 struct resource *dbi_res;
243 struct resource *atu_dma_res;
244 void __iomem *appl_base;
245 struct clk *core_clk;
246 struct reset_control *core_apb_rst;
247 struct reset_control *core_rst;
248 struct dw_pcie pci;
249 struct tegra_bpmp *bpmp;
250
251 struct tegra_pcie_dw_of_data *of_data;
252
253 bool supports_clkreq;
254 bool enable_cdm_check;
255 bool enable_srns;
256 bool link_state;
257 bool update_fc_fixup;
258 bool enable_ext_refclk;
259 u8 init_link_width;
260 u32 msi_ctrl_int;
261 u32 num_lanes;
262 u32 cid;
263 u32 cfg_link_cap_l1sub;
264 u32 ras_des_cap;
265 u32 pcie_cap_base;
266 u32 aspm_cmrt;
267 u32 aspm_pwr_on_t;
268 u32 aspm_l0s_enter_lat;
269
270 struct regulator *pex_ctl_supply;
271 struct regulator *slot_ctl_3v3;
272 struct regulator *slot_ctl_12v;
273
274 unsigned int phy_count;
275 struct phy **phys;
276
277 struct dentry *debugfs;
278
279 /* Endpoint mode specific */
280 struct gpio_desc *pex_rst_gpiod;
281 struct gpio_desc *pex_refclk_sel_gpiod;
282 unsigned int pex_rst_irq;
283 int ep_state;
284 long link_status;
285 struct icc_path *icc_path;
286};
287
288static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
289{
290 return container_of(pci, struct tegra_pcie_dw, pci);
291}
292
293static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
294 const u32 reg)
295{
296 writel_relaxed(value, pcie->appl_base + reg);
297}
298
299static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
300{
301 return readl_relaxed(pcie->appl_base + reg);
302}
303
304static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie)
305{
306 struct dw_pcie *pci = &pcie->pci;
307 u32 val, speed, width;
308
309 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
310
311 speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val);
312 width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
313
314 val = width * PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]);
315
316 if (icc_set_bw(pcie->icc_path, Mbps_to_icc(val), 0))
317 dev_err(pcie->dev, "can't set bw[%u]\n", val);
318
319 if (speed >= ARRAY_SIZE(pcie_gen_freq))
320 speed = 0;
321
322 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed]);
323}
324
325static void apply_bad_link_workaround(struct dw_pcie_rp *pp)
326{
327 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
328 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
329 u32 current_link_width;
330 u16 val;
331
332 /*
333 * NOTE:- Since this scenario is uncommon and link as such is not
334 * stable anyway, not waiting to confirm if link is really
335 * transitioning to Gen-2 speed
336 */
337 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
338 if (val & PCI_EXP_LNKSTA_LBMS) {
339 current_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
340 if (pcie->init_link_width > current_link_width) {
341 dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
342 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
343 PCI_EXP_LNKCTL2);
344 val &= ~PCI_EXP_LNKCTL2_TLS;
345 val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
346 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
347 PCI_EXP_LNKCTL2, val);
348
349 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
350 PCI_EXP_LNKCTL);
351 val |= PCI_EXP_LNKCTL_RL;
352 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
353 PCI_EXP_LNKCTL, val);
354 }
355 }
356}
357
358static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
359{
360 struct tegra_pcie_dw *pcie = arg;
361 struct dw_pcie *pci = &pcie->pci;
362 struct dw_pcie_rp *pp = &pci->pp;
363 u32 val, status_l0, status_l1;
364 u16 val_w;
365
366 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
367 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
368 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
369 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
370 if (!pcie->of_data->has_sbr_reset_fix &&
371 status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
372 /* SBR & Surprise Link Down WAR */
373 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
374 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
375 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
376 udelay(1);
377 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
378 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
379 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
380
381 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
382 val |= PORT_LOGIC_SPEED_CHANGE;
383 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
384 }
385 }
386
387 if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) {
388 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
389 if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
390 appl_writel(pcie,
391 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
392 APPL_INTR_STATUS_L1_8_0);
393 apply_bad_link_workaround(pp);
394 }
395 if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
396 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
397 PCI_EXP_LNKSTA);
398 val_w |= PCI_EXP_LNKSTA_LBMS;
399 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
400 PCI_EXP_LNKSTA, val_w);
401
402 appl_writel(pcie,
403 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
404 APPL_INTR_STATUS_L1_8_0);
405
406 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
407 PCI_EXP_LNKSTA);
408 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
409 PCI_EXP_LNKSTA_CLS);
410 }
411 }
412
413 if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
414 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
415 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
416 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
417 dev_info(pci->dev, "CDM check complete\n");
418 val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
419 }
420 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
421 dev_err(pci->dev, "CDM comparison mismatch\n");
422 val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
423 }
424 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
425 dev_err(pci->dev, "CDM Logic error\n");
426 val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
427 }
428 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val);
429 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
430 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val);
431 }
432
433 return IRQ_HANDLED;
434}
435
436static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
437{
438 u32 val;
439
440 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
441 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
442 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
443 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
444 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
445 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
446 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
447 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
448 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
449 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
450 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
451 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
452 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
453 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
454 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
455 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
456
457 val = appl_readl(pcie, APPL_CTRL);
458 val |= APPL_CTRL_LTSSM_EN;
459 appl_writel(pcie, val, APPL_CTRL);
460}
461
462static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
463{
464 struct tegra_pcie_dw *pcie = arg;
465 struct dw_pcie_ep *ep = &pcie->pci.ep;
466 struct dw_pcie *pci = &pcie->pci;
467 u32 val;
468
469 if (test_and_clear_bit(0, &pcie->link_status))
470 dw_pcie_ep_linkup(ep);
471
472 tegra_pcie_icc_set(pcie);
473
474 if (pcie->of_data->has_ltr_req_fix)
475 return IRQ_HANDLED;
476
477 /* If EP doesn't advertise L1SS, just return */
478 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
479 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
480 return IRQ_HANDLED;
481
482 /* Check if BME is set to '1' */
483 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
484 if (val & PCI_COMMAND_MASTER) {
485 ktime_t timeout;
486
487 /* 110us for both snoop and no-snoop */
488 val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) |
489 FIELD_PREP(PCI_LTR_SCALE_MASK, 2) |
490 LTR_MSG_REQ |
491 FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) |
492 FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) |
493 LTR_NOSNOOP_MSG_REQ;
494 appl_writel(pcie, val, APPL_LTR_MSG_1);
495
496 /* Send LTR upstream */
497 val = appl_readl(pcie, APPL_LTR_MSG_2);
498 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
499 appl_writel(pcie, val, APPL_LTR_MSG_2);
500
501 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
502 for (;;) {
503 val = appl_readl(pcie, APPL_LTR_MSG_2);
504 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
505 break;
506 if (ktime_after(ktime_get(), timeout))
507 break;
508 usleep_range(1000, 1100);
509 }
510 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
511 dev_err(pcie->dev, "Failed to send LTR message\n");
512 }
513
514 return IRQ_HANDLED;
515}
516
517static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
518{
519 struct tegra_pcie_dw *pcie = arg;
520 int spurious = 1;
521 u32 status_l0, status_l1, link_status;
522
523 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
524 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
525 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
526 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
527
528 if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
529 pex_ep_event_hot_rst_done(pcie);
530
531 if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
532 link_status = appl_readl(pcie, APPL_LINK_STATUS);
533 if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {
534 dev_dbg(pcie->dev, "Link is up with Host\n");
535 set_bit(0, &pcie->link_status);
536 return IRQ_WAKE_THREAD;
537 }
538 }
539
540 spurious = 0;
541 }
542
543 if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
544 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
545 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15);
546
547 if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
548 return IRQ_WAKE_THREAD;
549
550 spurious = 0;
551 }
552
553 if (spurious) {
554 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
555 status_l0);
556 appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0);
557 }
558
559 return IRQ_HANDLED;
560}
561
562static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
563 int size, u32 *val)
564{
565 struct dw_pcie_rp *pp = bus->sysdata;
566 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
567 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
568
569 /*
570 * This is an endpoint mode specific register happen to appear even
571 * when controller is operating in root port mode and system hangs
572 * when it is accessed with link being in ASPM-L1 state.
573 * So skip accessing it altogether
574 */
575 if (!pcie->of_data->has_msix_doorbell_access_fix &&
576 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
577 *val = 0x00000000;
578 return PCIBIOS_SUCCESSFUL;
579 }
580
581 return pci_generic_config_read(bus, devfn, where, size, val);
582}
583
584static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
585 int size, u32 val)
586{
587 struct dw_pcie_rp *pp = bus->sysdata;
588 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
589 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
590
591 /*
592 * This is an endpoint mode specific register happen to appear even
593 * when controller is operating in root port mode and system hangs
594 * when it is accessed with link being in ASPM-L1 state.
595 * So skip accessing it altogether
596 */
597 if (!pcie->of_data->has_msix_doorbell_access_fix &&
598 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
599 return PCIBIOS_SUCCESSFUL;
600
601 return pci_generic_config_write(bus, devfn, where, size, val);
602}
603
604static struct pci_ops tegra_pci_ops = {
605 .map_bus = dw_pcie_own_conf_map_bus,
606 .read = tegra_pcie_dw_rd_own_conf,
607 .write = tegra_pcie_dw_wr_own_conf,
608};
609
610#if defined(CONFIG_PCIEASPM)
611static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
612{
613 u32 val;
614
615 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
616 val &= ~PCI_L1SS_CAP_ASPM_L1_1;
617 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
618}
619
620static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
621{
622 u32 val;
623
624 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
625 val &= ~PCI_L1SS_CAP_ASPM_L1_2;
626 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
627}
628
629static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
630{
631 u32 val;
632
633 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
634 PCIE_RAS_DES_EVENT_COUNTER_CONTROL);
635 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
636 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
637 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
638 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
639 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
640 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
641 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
642 PCIE_RAS_DES_EVENT_COUNTER_DATA);
643
644 return val;
645}
646
647static int aspm_state_cnt(struct seq_file *s, void *data)
648{
649 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
650 dev_get_drvdata(s->private);
651 u32 val;
652
653 seq_printf(s, "Tx L0s entry count : %u\n",
654 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
655
656 seq_printf(s, "Rx L0s entry count : %u\n",
657 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
658
659 seq_printf(s, "Link L1 entry count : %u\n",
660 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
661
662 seq_printf(s, "Link L1.1 entry count : %u\n",
663 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
664
665 seq_printf(s, "Link L1.2 entry count : %u\n",
666 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
667
668 /* Clear all counters */
669 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
670 PCIE_RAS_DES_EVENT_COUNTER_CONTROL,
671 EVENT_COUNTER_ALL_CLEAR);
672
673 /* Re-enable counting */
674 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
675 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
676 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
677 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
678
679 return 0;
680}
681
682static void init_host_aspm(struct tegra_pcie_dw *pcie)
683{
684 struct dw_pcie *pci = &pcie->pci;
685 u32 val;
686
687 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
688 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
689
690 pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci,
691 PCI_EXT_CAP_ID_VNDR);
692
693 /* Enable ASPM counters */
694 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
695 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
696 dw_pcie_writel_dbi(pci, pcie->ras_des_cap +
697 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
698
699 /* Program T_cmrt and T_pwr_on values */
700 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
701 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
702 val |= (pcie->aspm_cmrt << 8);
703 val |= (pcie->aspm_pwr_on_t << 19);
704 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
705
706 /* Program L0s and L1 entrance latencies */
707 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
708 val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
709 val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
710 val |= PORT_AFR_ENTER_ASPM;
711 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
712}
713
714static void init_debugfs(struct tegra_pcie_dw *pcie)
715{
716 debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,
717 aspm_state_cnt);
718}
719#else
720static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
721static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
722static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
723static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
724#endif
725
726static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp)
727{
728 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
729 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
730 u32 val;
731 u16 val_w;
732
733 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
734 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
735 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
736
737 if (!pcie->of_data->has_sbr_reset_fix) {
738 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
739 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
740 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
741 }
742
743 if (pcie->enable_cdm_check) {
744 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
745 val |= pcie->of_data->cdm_chk_int_en_bit;
746 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
747
748 val = appl_readl(pcie, APPL_INTR_EN_L1_18);
749 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
750 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
751 appl_writel(pcie, val, APPL_INTR_EN_L1_18);
752 }
753
754 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
755 PCI_EXP_LNKSTA);
756 pcie->init_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val_w);
757
758 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
759 PCI_EXP_LNKCTL);
760 val_w |= PCI_EXP_LNKCTL_LBMIE;
761 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
762 val_w);
763}
764
765static void tegra_pcie_enable_intx_interrupts(struct dw_pcie_rp *pp)
766{
767 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
768 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
769 u32 val;
770
771 /* Enable INTX interrupt generation */
772 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
773 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
774 val |= APPL_INTR_EN_L0_0_INT_INT_EN;
775 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
776
777 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
778 val |= APPL_INTR_EN_L1_8_INTX_EN;
779 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
780 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
781 if (IS_ENABLED(CONFIG_PCIEAER))
782 val |= APPL_INTR_EN_L1_8_AER_INT_EN;
783 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
784}
785
786static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp)
787{
788 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
789 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
790 u32 val;
791
792 /* Enable MSI interrupt generation */
793 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
794 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
795 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
796 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
797}
798
799static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp)
800{
801 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
802 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
803
804 /* Clear interrupt statuses before enabling interrupts */
805 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
806 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
807 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
808 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
809 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
810 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
811 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
812 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
813 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
814 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
815 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
816 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
817 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
818 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
819 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
820
821 tegra_pcie_enable_system_interrupts(pp);
822 tegra_pcie_enable_intx_interrupts(pp);
823 if (IS_ENABLED(CONFIG_PCI_MSI))
824 tegra_pcie_enable_msi_interrupts(pp);
825}
826
827static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
828{
829 struct dw_pcie *pci = &pcie->pci;
830 u32 val, offset, i;
831
832 /* Program init preset */
833 for (i = 0; i < pcie->num_lanes; i++) {
834 val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
835 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
836 val |= GEN3_GEN4_EQ_PRESET_INIT;
837 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
838 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
839 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
840 dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
841
842 offset = dw_pcie_find_ext_capability(pci,
843 PCI_EXT_CAP_ID_PL_16GT) +
844 PCI_PL_16GT_LE_CTRL;
845 val = dw_pcie_readb_dbi(pci, offset + i);
846 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
847 val |= GEN3_GEN4_EQ_PRESET_INIT;
848 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
849 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
850 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
851 dw_pcie_writeb_dbi(pci, offset + i, val);
852 }
853
854 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
855 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
856 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
857
858 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
859 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC;
860 val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, 0x3ff);
861 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE;
862 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
863
864 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
865 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
866 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
867 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
868
869 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
870 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC;
871 val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC,
872 pcie->of_data->gen4_preset_vec);
873 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE;
874 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
875
876 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
877 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
878 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
879}
880
881static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp)
882{
883 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
884 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
885 u32 val;
886 u16 val_16;
887
888 pp->bridge->ops = &tegra_pci_ops;
889
890 if (!pcie->pcie_cap_base)
891 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
892 PCI_CAP_ID_EXP);
893
894 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
895 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
896 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
897
898 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
899 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
900 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
901 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
902
903 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
904
905 /* Enable as 0xFFFF0001 response for RRS */
906 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
907 val &= ~(AMBA_ERROR_RESPONSE_RRS_MASK << AMBA_ERROR_RESPONSE_RRS_SHIFT);
908 val |= (AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 <<
909 AMBA_ERROR_RESPONSE_RRS_SHIFT);
910 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
911
912 /* Clear Slot Clock Configuration bit if SRNS configuration */
913 if (pcie->enable_srns) {
914 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
915 PCI_EXP_LNKSTA);
916 val_16 &= ~PCI_EXP_LNKSTA_SLC;
917 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
918 val_16);
919 }
920
921 config_gen3_gen4_eq_presets(pcie);
922
923 init_host_aspm(pcie);
924
925 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
926 if (!pcie->supports_clkreq) {
927 disable_aspm_l11(pcie);
928 disable_aspm_l12(pcie);
929 }
930
931 if (!pcie->of_data->has_l1ss_exit_fix) {
932 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
933 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
934 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
935 }
936
937 if (pcie->update_fc_fixup) {
938 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
939 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
940 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
941 }
942
943 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
944
945 return 0;
946}
947
948static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
949{
950 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
951 struct dw_pcie_rp *pp = &pci->pp;
952 u32 val, offset, tmp;
953 bool retry = true;
954
955 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
956 enable_irq(pcie->pex_rst_irq);
957 return 0;
958 }
959
960retry_link:
961 /* Assert RST */
962 val = appl_readl(pcie, APPL_PINMUX);
963 val &= ~APPL_PINMUX_PEX_RST;
964 appl_writel(pcie, val, APPL_PINMUX);
965
966 usleep_range(100, 200);
967
968 /* Enable LTSSM */
969 val = appl_readl(pcie, APPL_CTRL);
970 val |= APPL_CTRL_LTSSM_EN;
971 appl_writel(pcie, val, APPL_CTRL);
972
973 /* De-assert RST */
974 val = appl_readl(pcie, APPL_PINMUX);
975 val |= APPL_PINMUX_PEX_RST;
976 appl_writel(pcie, val, APPL_PINMUX);
977
978 msleep(100);
979
980 if (dw_pcie_wait_for_link(pci)) {
981 if (!retry)
982 return 0;
983 /*
984 * There are some endpoints which can't get the link up if
985 * root port has Data Link Feature (DLF) enabled.
986 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
987 * on Scaled Flow Control and DLF.
988 * So, need to confirm that is indeed the case here and attempt
989 * link up once again with DLF disabled.
990 */
991 val = appl_readl(pcie, APPL_DEBUG);
992 val &= APPL_DEBUG_LTSSM_STATE_MASK;
993 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
994 tmp = appl_readl(pcie, APPL_LINK_STATUS);
995 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
996 if (!(val == 0x11 && !tmp)) {
997 /* Link is down for all good reasons */
998 return 0;
999 }
1000
1001 dev_info(pci->dev, "Link is down in DLL");
1002 dev_info(pci->dev, "Trying again with DLFE disabled\n");
1003 /* Disable LTSSM */
1004 val = appl_readl(pcie, APPL_CTRL);
1005 val &= ~APPL_CTRL_LTSSM_EN;
1006 appl_writel(pcie, val, APPL_CTRL);
1007
1008 reset_control_assert(pcie->core_rst);
1009 reset_control_deassert(pcie->core_rst);
1010
1011 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1012 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1013 val &= ~PCI_DLF_EXCHANGE_ENABLE;
1014 dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val);
1015
1016 tegra_pcie_dw_host_init(pp);
1017 dw_pcie_setup_rc(pp);
1018
1019 retry = false;
1020 goto retry_link;
1021 }
1022
1023 tegra_pcie_icc_set(pcie);
1024
1025 tegra_pcie_enable_interrupts(pp);
1026
1027 return 0;
1028}
1029
1030static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1031{
1032 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1033 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1034
1035 return !!(val & PCI_EXP_LNKSTA_DLLLA);
1036}
1037
1038static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1039{
1040 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1041
1042 disable_irq(pcie->pex_rst_irq);
1043}
1044
1045static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1046 .link_up = tegra_pcie_dw_link_up,
1047 .start_link = tegra_pcie_dw_start_link,
1048 .stop_link = tegra_pcie_dw_stop_link,
1049};
1050
1051static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1052 .init = tegra_pcie_dw_host_init,
1053};
1054
1055static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1056{
1057 unsigned int phy_count = pcie->phy_count;
1058
1059 while (phy_count--) {
1060 phy_power_off(pcie->phys[phy_count]);
1061 phy_exit(pcie->phys[phy_count]);
1062 }
1063}
1064
1065static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1066{
1067 unsigned int i;
1068 int ret;
1069
1070 for (i = 0; i < pcie->phy_count; i++) {
1071 ret = phy_init(pcie->phys[i]);
1072 if (ret < 0)
1073 goto phy_power_off;
1074
1075 ret = phy_power_on(pcie->phys[i]);
1076 if (ret < 0)
1077 goto phy_exit;
1078 }
1079
1080 return 0;
1081
1082phy_power_off:
1083 while (i--) {
1084 phy_power_off(pcie->phys[i]);
1085phy_exit:
1086 phy_exit(pcie->phys[i]);
1087 }
1088
1089 return ret;
1090}
1091
1092static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1093{
1094 struct platform_device *pdev = to_platform_device(pcie->dev);
1095 struct device_node *np = pcie->dev->of_node;
1096 int ret;
1097
1098 pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1099 if (!pcie->dbi_res) {
1100 dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1101 return -ENODEV;
1102 }
1103
1104 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1105 if (ret < 0) {
1106 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1107 return ret;
1108 }
1109
1110 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1111 &pcie->aspm_pwr_on_t);
1112 if (ret < 0)
1113 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1114 ret);
1115
1116 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1117 &pcie->aspm_l0s_enter_lat);
1118 if (ret < 0)
1119 dev_info(pcie->dev,
1120 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1121
1122 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1123 if (ret < 0) {
1124 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1125 return ret;
1126 }
1127
1128 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1129 if (ret) {
1130 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1131 return ret;
1132 }
1133
1134 ret = of_property_count_strings(np, "phy-names");
1135 if (ret < 0) {
1136 dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1137 ret);
1138 return ret;
1139 }
1140 pcie->phy_count = ret;
1141
1142 if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1143 pcie->update_fc_fixup = true;
1144
1145 /* RP using an external REFCLK is supported only in Tegra234 */
1146 if (pcie->of_data->version == TEGRA194_DWC_IP_VER) {
1147 if (pcie->of_data->mode == DW_PCIE_EP_TYPE)
1148 pcie->enable_ext_refclk = true;
1149 } else {
1150 pcie->enable_ext_refclk =
1151 of_property_read_bool(pcie->dev->of_node,
1152 "nvidia,enable-ext-refclk");
1153 }
1154
1155 pcie->supports_clkreq =
1156 of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1157
1158 pcie->enable_cdm_check =
1159 of_property_read_bool(np, "snps,enable-cdm-check");
1160
1161 if (pcie->of_data->version == TEGRA234_DWC_IP_VER)
1162 pcie->enable_srns =
1163 of_property_read_bool(np, "nvidia,enable-srns");
1164
1165 if (pcie->of_data->mode == DW_PCIE_RC_TYPE)
1166 return 0;
1167
1168 /* Endpoint mode specific DT entries */
1169 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1170 if (IS_ERR(pcie->pex_rst_gpiod)) {
1171 int err = PTR_ERR(pcie->pex_rst_gpiod);
1172 const char *level = KERN_ERR;
1173
1174 if (err == -EPROBE_DEFER)
1175 level = KERN_DEBUG;
1176
1177 dev_printk(level, pcie->dev,
1178 dev_fmt("Failed to get PERST GPIO: %d\n"),
1179 err);
1180 return err;
1181 }
1182
1183 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1184 "nvidia,refclk-select",
1185 GPIOD_OUT_HIGH);
1186 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1187 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1188 const char *level = KERN_ERR;
1189
1190 if (err == -EPROBE_DEFER)
1191 level = KERN_DEBUG;
1192
1193 dev_printk(level, pcie->dev,
1194 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1195 err);
1196 pcie->pex_refclk_sel_gpiod = NULL;
1197 }
1198
1199 return 0;
1200}
1201
1202static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1203 bool enable)
1204{
1205 struct mrq_uphy_response resp;
1206 struct tegra_bpmp_message msg;
1207 struct mrq_uphy_request req;
1208
1209 /*
1210 * Controller-5 doesn't need to have its state set by BPMP-FW in
1211 * Tegra194
1212 */
1213 if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5)
1214 return 0;
1215
1216 memset(&req, 0, sizeof(req));
1217 memset(&resp, 0, sizeof(resp));
1218
1219 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1220 req.controller_state.pcie_controller = pcie->cid;
1221 req.controller_state.enable = enable;
1222
1223 memset(&msg, 0, sizeof(msg));
1224 msg.mrq = MRQ_UPHY;
1225 msg.tx.data = &req;
1226 msg.tx.size = sizeof(req);
1227 msg.rx.data = &resp;
1228 msg.rx.size = sizeof(resp);
1229
1230 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1231}
1232
1233static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1234 bool enable)
1235{
1236 struct mrq_uphy_response resp;
1237 struct tegra_bpmp_message msg;
1238 struct mrq_uphy_request req;
1239
1240 memset(&req, 0, sizeof(req));
1241 memset(&resp, 0, sizeof(resp));
1242
1243 if (enable) {
1244 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1245 req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1246 } else {
1247 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1248 req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1249 }
1250
1251 memset(&msg, 0, sizeof(msg));
1252 msg.mrq = MRQ_UPHY;
1253 msg.tx.data = &req;
1254 msg.tx.size = sizeof(req);
1255 msg.rx.data = &resp;
1256 msg.rx.size = sizeof(resp);
1257
1258 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1259}
1260
1261static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1262{
1263 struct dw_pcie_rp *pp = &pcie->pci.pp;
1264 struct pci_bus *child, *root_bus = NULL;
1265 struct pci_dev *pdev;
1266
1267 /*
1268 * link doesn't go into L2 state with some of the endpoints with Tegra
1269 * if they are not in D0 state. So, need to make sure that immediate
1270 * downstream devices are in D0 state before sending PME_TurnOff to put
1271 * link into L2 state.
1272 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1273 * 5.2 Link State Power Management (Page #428).
1274 */
1275
1276 list_for_each_entry(child, &pp->bridge->bus->children, node) {
1277 /* Bring downstream devices to D0 if they are not already in */
1278 if (child->parent == pp->bridge->bus) {
1279 root_bus = child;
1280 break;
1281 }
1282 }
1283
1284 if (!root_bus) {
1285 dev_err(pcie->dev, "Failed to find downstream devices\n");
1286 return;
1287 }
1288
1289 list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1290 if (PCI_SLOT(pdev->devfn) == 0) {
1291 if (pci_set_power_state(pdev, PCI_D0))
1292 dev_err(pcie->dev,
1293 "Failed to transition %s to D0 state\n",
1294 dev_name(&pdev->dev));
1295 }
1296 }
1297}
1298
1299static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1300{
1301 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1302 if (IS_ERR(pcie->slot_ctl_3v3)) {
1303 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1304 return PTR_ERR(pcie->slot_ctl_3v3);
1305
1306 pcie->slot_ctl_3v3 = NULL;
1307 }
1308
1309 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1310 if (IS_ERR(pcie->slot_ctl_12v)) {
1311 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1312 return PTR_ERR(pcie->slot_ctl_12v);
1313
1314 pcie->slot_ctl_12v = NULL;
1315 }
1316
1317 return 0;
1318}
1319
1320static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1321{
1322 int ret;
1323
1324 if (pcie->slot_ctl_3v3) {
1325 ret = regulator_enable(pcie->slot_ctl_3v3);
1326 if (ret < 0) {
1327 dev_err(pcie->dev,
1328 "Failed to enable 3.3V slot supply: %d\n", ret);
1329 return ret;
1330 }
1331 }
1332
1333 if (pcie->slot_ctl_12v) {
1334 ret = regulator_enable(pcie->slot_ctl_12v);
1335 if (ret < 0) {
1336 dev_err(pcie->dev,
1337 "Failed to enable 12V slot supply: %d\n", ret);
1338 goto fail_12v_enable;
1339 }
1340 }
1341
1342 /*
1343 * According to PCI Express Card Electromechanical Specification
1344 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1345 * should be a minimum of 100ms.
1346 */
1347 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1348 msleep(100);
1349
1350 return 0;
1351
1352fail_12v_enable:
1353 if (pcie->slot_ctl_3v3)
1354 regulator_disable(pcie->slot_ctl_3v3);
1355 return ret;
1356}
1357
1358static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1359{
1360 if (pcie->slot_ctl_12v)
1361 regulator_disable(pcie->slot_ctl_12v);
1362 if (pcie->slot_ctl_3v3)
1363 regulator_disable(pcie->slot_ctl_3v3);
1364}
1365
1366static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1367 bool en_hw_hot_rst)
1368{
1369 int ret;
1370 u32 val;
1371
1372 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1373 if (ret) {
1374 dev_err(pcie->dev,
1375 "Failed to enable controller %u: %d\n", pcie->cid, ret);
1376 return ret;
1377 }
1378
1379 if (pcie->enable_ext_refclk) {
1380 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1381 if (ret) {
1382 dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret);
1383 goto fail_pll_init;
1384 }
1385 }
1386
1387 ret = tegra_pcie_enable_slot_regulators(pcie);
1388 if (ret < 0)
1389 goto fail_slot_reg_en;
1390
1391 ret = regulator_enable(pcie->pex_ctl_supply);
1392 if (ret < 0) {
1393 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1394 goto fail_reg_en;
1395 }
1396
1397 ret = clk_prepare_enable(pcie->core_clk);
1398 if (ret) {
1399 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1400 goto fail_core_clk;
1401 }
1402
1403 ret = reset_control_deassert(pcie->core_apb_rst);
1404 if (ret) {
1405 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1406 ret);
1407 goto fail_core_apb_rst;
1408 }
1409
1410 if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) {
1411 /* Enable HW_HOT_RST mode */
1412 val = appl_readl(pcie, APPL_CTRL);
1413 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1414 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1415 val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN <<
1416 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1417 val |= APPL_CTRL_HW_HOT_RST_EN;
1418 appl_writel(pcie, val, APPL_CTRL);
1419 }
1420
1421 ret = tegra_pcie_enable_phy(pcie);
1422 if (ret) {
1423 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1424 goto fail_phy;
1425 }
1426
1427 /* Update CFG base address */
1428 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1429 APPL_CFG_BASE_ADDR);
1430
1431 /* Configure this core for RP mode operation */
1432 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1433
1434 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1435
1436 val = appl_readl(pcie, APPL_CTRL);
1437 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1438
1439 val = appl_readl(pcie, APPL_CFG_MISC);
1440 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1441 appl_writel(pcie, val, APPL_CFG_MISC);
1442
1443 if (pcie->enable_srns || pcie->enable_ext_refclk) {
1444 /*
1445 * When Tegra PCIe RP is using external clock, it cannot supply
1446 * same clock to its downstream hierarchy. Hence, gate PCIe RP
1447 * REFCLK out pads when RP & EP are using separate clocks or RP
1448 * is using an external REFCLK.
1449 */
1450 val = appl_readl(pcie, APPL_PINMUX);
1451 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1452 val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1453 appl_writel(pcie, val, APPL_PINMUX);
1454 }
1455
1456 if (!pcie->supports_clkreq) {
1457 val = appl_readl(pcie, APPL_PINMUX);
1458 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1459 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1460 appl_writel(pcie, val, APPL_PINMUX);
1461 }
1462
1463 /* Update iATU_DMA base address */
1464 appl_writel(pcie,
1465 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1466 APPL_CFG_IATU_DMA_BASE_ADDR);
1467
1468 reset_control_deassert(pcie->core_rst);
1469
1470 return ret;
1471
1472fail_phy:
1473 reset_control_assert(pcie->core_apb_rst);
1474fail_core_apb_rst:
1475 clk_disable_unprepare(pcie->core_clk);
1476fail_core_clk:
1477 regulator_disable(pcie->pex_ctl_supply);
1478fail_reg_en:
1479 tegra_pcie_disable_slot_regulators(pcie);
1480fail_slot_reg_en:
1481 if (pcie->enable_ext_refclk)
1482 tegra_pcie_bpmp_set_pll_state(pcie, false);
1483fail_pll_init:
1484 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1485
1486 return ret;
1487}
1488
1489static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1490{
1491 int ret;
1492
1493 ret = reset_control_assert(pcie->core_rst);
1494 if (ret)
1495 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1496
1497 tegra_pcie_disable_phy(pcie);
1498
1499 ret = reset_control_assert(pcie->core_apb_rst);
1500 if (ret)
1501 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1502
1503 clk_disable_unprepare(pcie->core_clk);
1504
1505 ret = regulator_disable(pcie->pex_ctl_supply);
1506 if (ret)
1507 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1508
1509 tegra_pcie_disable_slot_regulators(pcie);
1510
1511 if (pcie->enable_ext_refclk) {
1512 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1513 if (ret)
1514 dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret);
1515 }
1516
1517 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1518 if (ret)
1519 dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1520 pcie->cid, ret);
1521}
1522
1523static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1524{
1525 struct dw_pcie *pci = &pcie->pci;
1526 struct dw_pcie_rp *pp = &pci->pp;
1527 int ret;
1528
1529 ret = tegra_pcie_config_controller(pcie, false);
1530 if (ret < 0)
1531 return ret;
1532
1533 pp->ops = &tegra_pcie_dw_host_ops;
1534
1535 ret = dw_pcie_host_init(pp);
1536 if (ret < 0) {
1537 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1538 goto fail_host_init;
1539 }
1540
1541 return 0;
1542
1543fail_host_init:
1544 tegra_pcie_unconfig_controller(pcie);
1545 return ret;
1546}
1547
1548static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1549{
1550 u32 val;
1551
1552 if (!tegra_pcie_dw_link_up(&pcie->pci))
1553 return 0;
1554
1555 val = appl_readl(pcie, APPL_RADM_STATUS);
1556 val |= APPL_PM_XMT_TURNOFF_STATE;
1557 appl_writel(pcie, val, APPL_RADM_STATUS);
1558
1559 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1560 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1561 1, PME_ACK_TIMEOUT);
1562}
1563
1564static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1565{
1566 u32 data;
1567 int err;
1568
1569 if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1570 dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1571 return;
1572 }
1573
1574 /*
1575 * PCIe controller exits from L2 only if reset is applied, so
1576 * controller doesn't handle interrupts. But in cases where
1577 * L2 entry fails, PERST# is asserted which can trigger surprise
1578 * link down AER. However this function call happens in
1579 * suspend_noirq(), so AER interrupt will not be processed.
1580 * Disable all interrupts to avoid such a scenario.
1581 */
1582 appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0);
1583
1584 if (tegra_pcie_try_link_l2(pcie)) {
1585 dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1586 /*
1587 * TX lane clock freq will reset to Gen1 only if link is in L2
1588 * or detect state.
1589 * So apply pex_rst to end point to force RP to go into detect
1590 * state
1591 */
1592 data = appl_readl(pcie, APPL_PINMUX);
1593 data &= ~APPL_PINMUX_PEX_RST;
1594 appl_writel(pcie, data, APPL_PINMUX);
1595
1596 /*
1597 * Some cards do not go to detect state even after de-asserting
1598 * PERST#. So, de-assert LTSSM to bring link to detect state.
1599 */
1600 data = readl(pcie->appl_base + APPL_CTRL);
1601 data &= ~APPL_CTRL_LTSSM_EN;
1602 writel(data, pcie->appl_base + APPL_CTRL);
1603
1604 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1605 data,
1606 ((data &
1607 APPL_DEBUG_LTSSM_STATE_MASK) >>
1608 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1609 LTSSM_STATE_PRE_DETECT,
1610 1, LTSSM_TIMEOUT);
1611 if (err)
1612 dev_info(pcie->dev, "Link didn't go to detect state\n");
1613 }
1614 /*
1615 * DBI registers may not be accessible after this as PLL-E would be
1616 * down depending on how CLKREQ is pulled by end point
1617 */
1618 data = appl_readl(pcie, APPL_PINMUX);
1619 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1620 /* Cut REFCLK to slot */
1621 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1622 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1623 appl_writel(pcie, data, APPL_PINMUX);
1624}
1625
1626static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1627{
1628 tegra_pcie_downstream_dev_to_D0(pcie);
1629 dw_pcie_host_deinit(&pcie->pci.pp);
1630 tegra_pcie_dw_pme_turnoff(pcie);
1631 tegra_pcie_unconfig_controller(pcie);
1632}
1633
1634static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1635{
1636 struct device *dev = pcie->dev;
1637 char *name;
1638 int ret;
1639
1640 pm_runtime_enable(dev);
1641
1642 ret = pm_runtime_get_sync(dev);
1643 if (ret < 0) {
1644 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1645 ret);
1646 goto fail_pm_get_sync;
1647 }
1648
1649 ret = pinctrl_pm_select_default_state(dev);
1650 if (ret < 0) {
1651 dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1652 goto fail_pm_get_sync;
1653 }
1654
1655 ret = tegra_pcie_init_controller(pcie);
1656 if (ret < 0) {
1657 dev_err(dev, "Failed to initialize controller: %d\n", ret);
1658 goto fail_pm_get_sync;
1659 }
1660
1661 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1662 if (!pcie->link_state) {
1663 ret = -ENOMEDIUM;
1664 goto fail_host_init;
1665 }
1666
1667 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1668 if (!name) {
1669 ret = -ENOMEM;
1670 goto fail_host_init;
1671 }
1672
1673 pcie->debugfs = debugfs_create_dir(name, NULL);
1674 init_debugfs(pcie);
1675
1676 return ret;
1677
1678fail_host_init:
1679 tegra_pcie_deinit_controller(pcie);
1680fail_pm_get_sync:
1681 pm_runtime_put_sync(dev);
1682 pm_runtime_disable(dev);
1683 return ret;
1684}
1685
1686static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1687{
1688 u32 val;
1689 int ret;
1690
1691 if (pcie->ep_state == EP_STATE_DISABLED)
1692 return;
1693
1694 /* Disable LTSSM */
1695 val = appl_readl(pcie, APPL_CTRL);
1696 val &= ~APPL_CTRL_LTSSM_EN;
1697 appl_writel(pcie, val, APPL_CTRL);
1698
1699 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1700 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1701 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1702 LTSSM_STATE_PRE_DETECT,
1703 1, LTSSM_TIMEOUT);
1704 if (ret)
1705 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1706
1707 reset_control_assert(pcie->core_rst);
1708
1709 tegra_pcie_disable_phy(pcie);
1710
1711 reset_control_assert(pcie->core_apb_rst);
1712
1713 clk_disable_unprepare(pcie->core_clk);
1714
1715 pm_runtime_put_sync(pcie->dev);
1716
1717 if (pcie->enable_ext_refclk) {
1718 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1719 if (ret)
1720 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n",
1721 ret);
1722 }
1723
1724 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1725 if (ret)
1726 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1727
1728 pcie->ep_state = EP_STATE_DISABLED;
1729 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1730}
1731
1732static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1733{
1734 struct dw_pcie *pci = &pcie->pci;
1735 struct dw_pcie_ep *ep = &pci->ep;
1736 struct device *dev = pcie->dev;
1737 u32 val;
1738 int ret;
1739 u16 val_16;
1740
1741 if (pcie->ep_state == EP_STATE_ENABLED)
1742 return;
1743
1744 ret = pm_runtime_resume_and_get(dev);
1745 if (ret < 0) {
1746 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1747 ret);
1748 return;
1749 }
1750
1751 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1752 if (ret) {
1753 dev_err(pcie->dev, "Failed to enable controller %u: %d\n",
1754 pcie->cid, ret);
1755 goto fail_set_ctrl_state;
1756 }
1757
1758 if (pcie->enable_ext_refclk) {
1759 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1760 if (ret) {
1761 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n",
1762 ret);
1763 goto fail_pll_init;
1764 }
1765 }
1766
1767 ret = clk_prepare_enable(pcie->core_clk);
1768 if (ret) {
1769 dev_err(dev, "Failed to enable core clock: %d\n", ret);
1770 goto fail_core_clk_enable;
1771 }
1772
1773 ret = reset_control_deassert(pcie->core_apb_rst);
1774 if (ret) {
1775 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1776 goto fail_core_apb_rst;
1777 }
1778
1779 ret = tegra_pcie_enable_phy(pcie);
1780 if (ret) {
1781 dev_err(dev, "Failed to enable PHY: %d\n", ret);
1782 goto fail_phy;
1783 }
1784
1785 /* Perform cleanup that requires refclk */
1786 pci_epc_deinit_notify(pcie->pci.ep.epc);
1787 dw_pcie_ep_cleanup(&pcie->pci.ep);
1788
1789 /* Clear any stale interrupt statuses */
1790 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1791 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1792 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1793 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1794 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1795 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1796 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1797 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1798 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1799 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1800 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1801 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1802 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1804 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1805
1806 /* configure this core for EP mode operation */
1807 val = appl_readl(pcie, APPL_DM_TYPE);
1808 val &= ~APPL_DM_TYPE_MASK;
1809 val |= APPL_DM_TYPE_EP;
1810 appl_writel(pcie, val, APPL_DM_TYPE);
1811
1812 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1813
1814 val = appl_readl(pcie, APPL_CTRL);
1815 val |= APPL_CTRL_SYS_PRE_DET_STATE;
1816 val |= APPL_CTRL_HW_HOT_RST_EN;
1817 appl_writel(pcie, val, APPL_CTRL);
1818
1819 val = appl_readl(pcie, APPL_CFG_MISC);
1820 val |= APPL_CFG_MISC_SLV_EP_MODE;
1821 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1822 appl_writel(pcie, val, APPL_CFG_MISC);
1823
1824 val = appl_readl(pcie, APPL_PINMUX);
1825 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1826 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1827 appl_writel(pcie, val, APPL_PINMUX);
1828
1829 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1830 APPL_CFG_BASE_ADDR);
1831
1832 appl_writel(pcie, pcie->atu_dma_res->start &
1833 APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1834 APPL_CFG_IATU_DMA_BASE_ADDR);
1835
1836 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1837 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1838 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1839 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1840 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1841
1842 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1843 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1844 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1845 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1846
1847 reset_control_deassert(pcie->core_rst);
1848
1849 if (pcie->update_fc_fixup) {
1850 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1851 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1852 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1853 }
1854
1855 config_gen3_gen4_eq_presets(pcie);
1856
1857 init_host_aspm(pcie);
1858
1859 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1860 if (!pcie->supports_clkreq) {
1861 disable_aspm_l11(pcie);
1862 disable_aspm_l12(pcie);
1863 }
1864
1865 if (!pcie->of_data->has_l1ss_exit_fix) {
1866 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1867 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1868 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1869 }
1870
1871 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1872 PCI_CAP_ID_EXP);
1873
1874 /* Clear Slot Clock Configuration bit if SRNS configuration */
1875 if (pcie->enable_srns) {
1876 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
1877 PCI_EXP_LNKSTA);
1878 val_16 &= ~PCI_EXP_LNKSTA_SLC;
1879 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
1880 val_16);
1881 }
1882
1883 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1884
1885 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1886 val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1887 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1888 val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1889 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1890
1891 ret = dw_pcie_ep_init_registers(ep);
1892 if (ret) {
1893 dev_err(dev, "Failed to complete initialization: %d\n", ret);
1894 goto fail_init_complete;
1895 }
1896
1897 pci_epc_init_notify(ep->epc);
1898
1899 /* Program the private control to allow sending LTR upstream */
1900 if (pcie->of_data->has_ltr_req_fix) {
1901 val = appl_readl(pcie, APPL_LTR_MSG_2);
1902 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
1903 appl_writel(pcie, val, APPL_LTR_MSG_2);
1904 }
1905
1906 /* Enable LTSSM */
1907 val = appl_readl(pcie, APPL_CTRL);
1908 val |= APPL_CTRL_LTSSM_EN;
1909 appl_writel(pcie, val, APPL_CTRL);
1910
1911 pcie->ep_state = EP_STATE_ENABLED;
1912 dev_dbg(dev, "Initialization of endpoint is completed\n");
1913
1914 return;
1915
1916fail_init_complete:
1917 reset_control_assert(pcie->core_rst);
1918 tegra_pcie_disable_phy(pcie);
1919fail_phy:
1920 reset_control_assert(pcie->core_apb_rst);
1921fail_core_apb_rst:
1922 clk_disable_unprepare(pcie->core_clk);
1923fail_core_clk_enable:
1924 tegra_pcie_bpmp_set_pll_state(pcie, false);
1925fail_pll_init:
1926 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1927fail_set_ctrl_state:
1928 pm_runtime_put_sync(dev);
1929}
1930
1931static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1932{
1933 struct tegra_pcie_dw *pcie = arg;
1934
1935 if (gpiod_get_value(pcie->pex_rst_gpiod))
1936 pex_ep_event_pex_rst_assert(pcie);
1937 else
1938 pex_ep_event_pex_rst_deassert(pcie);
1939
1940 return IRQ_HANDLED;
1941}
1942
1943static int tegra_pcie_ep_raise_intx_irq(struct tegra_pcie_dw *pcie, u16 irq)
1944{
1945 /* Tegra194 supports only INTA */
1946 if (irq > 1)
1947 return -EINVAL;
1948
1949 appl_writel(pcie, 1, APPL_LEGACY_INTX);
1950 usleep_range(1000, 2000);
1951 appl_writel(pcie, 0, APPL_LEGACY_INTX);
1952 return 0;
1953}
1954
1955static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1956{
1957 if (unlikely(irq > 31))
1958 return -EINVAL;
1959
1960 appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1961
1962 return 0;
1963}
1964
1965static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1966{
1967 struct dw_pcie_ep *ep = &pcie->pci.ep;
1968
1969 writel(irq, ep->msi_mem);
1970
1971 return 0;
1972}
1973
1974static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1975 unsigned int type, u16 interrupt_num)
1976{
1977 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1978 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1979
1980 switch (type) {
1981 case PCI_IRQ_INTX:
1982 return tegra_pcie_ep_raise_intx_irq(pcie, interrupt_num);
1983
1984 case PCI_IRQ_MSI:
1985 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1986
1987 case PCI_IRQ_MSIX:
1988 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1989
1990 default:
1991 dev_err(pci->dev, "Unknown IRQ type\n");
1992 return -EPERM;
1993 }
1994
1995 return 0;
1996}
1997
1998static const struct pci_epc_features tegra_pcie_epc_features = {
1999 .linkup_notifier = true,
2000 .msi_capable = false,
2001 .msix_capable = false,
2002 .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M,
2003 .only_64bit = true, },
2004 .bar[BAR_1] = { .type = BAR_RESERVED, },
2005 .bar[BAR_2] = { .type = BAR_RESERVED, },
2006 .bar[BAR_3] = { .type = BAR_RESERVED, },
2007 .bar[BAR_4] = { .type = BAR_RESERVED, },
2008 .bar[BAR_5] = { .type = BAR_RESERVED, },
2009 .align = SZ_64K,
2010};
2011
2012static const struct pci_epc_features*
2013tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
2014{
2015 return &tegra_pcie_epc_features;
2016}
2017
2018static const struct dw_pcie_ep_ops pcie_ep_ops = {
2019 .raise_irq = tegra_pcie_ep_raise_irq,
2020 .get_features = tegra_pcie_ep_get_features,
2021};
2022
2023static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
2024 struct platform_device *pdev)
2025{
2026 struct dw_pcie *pci = &pcie->pci;
2027 struct device *dev = pcie->dev;
2028 struct dw_pcie_ep *ep;
2029 char *name;
2030 int ret;
2031
2032 ep = &pci->ep;
2033 ep->ops = &pcie_ep_ops;
2034
2035 ep->page_size = SZ_64K;
2036
2037 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
2038 if (ret < 0) {
2039 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
2040 ret);
2041 return ret;
2042 }
2043
2044 ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2045 if (ret < 0) {
2046 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2047 return ret;
2048 }
2049 pcie->pex_rst_irq = (unsigned int)ret;
2050
2051 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2052 pcie->cid);
2053 if (!name) {
2054 dev_err(dev, "Failed to create PERST IRQ string\n");
2055 return -ENOMEM;
2056 }
2057
2058 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2059
2060 pcie->ep_state = EP_STATE_DISABLED;
2061
2062 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2063 tegra_pcie_ep_pex_rst_irq,
2064 IRQF_TRIGGER_RISING |
2065 IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2066 name, (void *)pcie);
2067 if (ret < 0) {
2068 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2069 return ret;
2070 }
2071
2072 pm_runtime_enable(dev);
2073
2074 ret = dw_pcie_ep_init(ep);
2075 if (ret) {
2076 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2077 ret);
2078 pm_runtime_disable(dev);
2079 return ret;
2080 }
2081
2082 return 0;
2083}
2084
2085static int tegra_pcie_dw_probe(struct platform_device *pdev)
2086{
2087 const struct tegra_pcie_dw_of_data *data;
2088 struct device *dev = &pdev->dev;
2089 struct resource *atu_dma_res;
2090 struct tegra_pcie_dw *pcie;
2091 struct dw_pcie_rp *pp;
2092 struct dw_pcie *pci;
2093 struct phy **phys;
2094 char *name;
2095 int ret;
2096 u32 i;
2097
2098 data = of_device_get_match_data(dev);
2099
2100 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2101 if (!pcie)
2102 return -ENOMEM;
2103
2104 pci = &pcie->pci;
2105 pci->dev = &pdev->dev;
2106 pci->ops = &tegra_dw_pcie_ops;
2107 pcie->dev = &pdev->dev;
2108 pcie->of_data = (struct tegra_pcie_dw_of_data *)data;
2109 pci->n_fts[0] = pcie->of_data->n_fts[0];
2110 pci->n_fts[1] = pcie->of_data->n_fts[1];
2111 pp = &pci->pp;
2112 pp->num_vectors = MAX_MSI_IRQS;
2113
2114 ret = tegra_pcie_dw_parse_dt(pcie);
2115 if (ret < 0) {
2116 const char *level = KERN_ERR;
2117
2118 if (ret == -EPROBE_DEFER)
2119 level = KERN_DEBUG;
2120
2121 dev_printk(level, dev,
2122 dev_fmt("Failed to parse device tree: %d\n"),
2123 ret);
2124 return ret;
2125 }
2126
2127 ret = tegra_pcie_get_slot_regulators(pcie);
2128 if (ret < 0) {
2129 const char *level = KERN_ERR;
2130
2131 if (ret == -EPROBE_DEFER)
2132 level = KERN_DEBUG;
2133
2134 dev_printk(level, dev,
2135 dev_fmt("Failed to get slot regulators: %d\n"),
2136 ret);
2137 return ret;
2138 }
2139
2140 if (pcie->pex_refclk_sel_gpiod)
2141 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2142
2143 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2144 if (IS_ERR(pcie->pex_ctl_supply)) {
2145 ret = PTR_ERR(pcie->pex_ctl_supply);
2146 if (ret != -EPROBE_DEFER)
2147 dev_err(dev, "Failed to get regulator: %ld\n",
2148 PTR_ERR(pcie->pex_ctl_supply));
2149 return ret;
2150 }
2151
2152 pcie->core_clk = devm_clk_get(dev, "core");
2153 if (IS_ERR(pcie->core_clk)) {
2154 dev_err(dev, "Failed to get core clock: %ld\n",
2155 PTR_ERR(pcie->core_clk));
2156 return PTR_ERR(pcie->core_clk);
2157 }
2158
2159 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2160 "appl");
2161 if (!pcie->appl_res) {
2162 dev_err(dev, "Failed to find \"appl\" region\n");
2163 return -ENODEV;
2164 }
2165
2166 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2167 if (IS_ERR(pcie->appl_base))
2168 return PTR_ERR(pcie->appl_base);
2169
2170 pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2171 if (IS_ERR(pcie->core_apb_rst)) {
2172 dev_err(dev, "Failed to get APB reset: %ld\n",
2173 PTR_ERR(pcie->core_apb_rst));
2174 return PTR_ERR(pcie->core_apb_rst);
2175 }
2176
2177 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2178 if (!phys)
2179 return -ENOMEM;
2180
2181 for (i = 0; i < pcie->phy_count; i++) {
2182 name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2183 if (!name) {
2184 dev_err(dev, "Failed to create P2U string\n");
2185 return -ENOMEM;
2186 }
2187 phys[i] = devm_phy_get(dev, name);
2188 kfree(name);
2189 if (IS_ERR(phys[i])) {
2190 ret = PTR_ERR(phys[i]);
2191 if (ret != -EPROBE_DEFER)
2192 dev_err(dev, "Failed to get PHY: %d\n", ret);
2193 return ret;
2194 }
2195 }
2196
2197 pcie->phys = phys;
2198
2199 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2200 "atu_dma");
2201 if (!atu_dma_res) {
2202 dev_err(dev, "Failed to find \"atu_dma\" region\n");
2203 return -ENODEV;
2204 }
2205 pcie->atu_dma_res = atu_dma_res;
2206
2207 pci->atu_size = resource_size(atu_dma_res);
2208 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2209 if (IS_ERR(pci->atu_base))
2210 return PTR_ERR(pci->atu_base);
2211
2212 pcie->core_rst = devm_reset_control_get(dev, "core");
2213 if (IS_ERR(pcie->core_rst)) {
2214 dev_err(dev, "Failed to get core reset: %ld\n",
2215 PTR_ERR(pcie->core_rst));
2216 return PTR_ERR(pcie->core_rst);
2217 }
2218
2219 pp->irq = platform_get_irq_byname(pdev, "intr");
2220 if (pp->irq < 0)
2221 return pp->irq;
2222
2223 pcie->bpmp = tegra_bpmp_get(dev);
2224 if (IS_ERR(pcie->bpmp))
2225 return PTR_ERR(pcie->bpmp);
2226
2227 platform_set_drvdata(pdev, pcie);
2228
2229 pcie->icc_path = devm_of_icc_get(&pdev->dev, "write");
2230 ret = PTR_ERR_OR_ZERO(pcie->icc_path);
2231 if (ret) {
2232 tegra_bpmp_put(pcie->bpmp);
2233 dev_err_probe(&pdev->dev, ret, "failed to get write interconnect\n");
2234 return ret;
2235 }
2236
2237 switch (pcie->of_data->mode) {
2238 case DW_PCIE_RC_TYPE:
2239 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2240 IRQF_SHARED, "tegra-pcie-intr", pcie);
2241 if (ret) {
2242 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2243 ret);
2244 goto fail;
2245 }
2246
2247 ret = tegra_pcie_config_rp(pcie);
2248 if (ret && ret != -ENOMEDIUM)
2249 goto fail;
2250 else
2251 return 0;
2252 break;
2253
2254 case DW_PCIE_EP_TYPE:
2255 ret = devm_request_threaded_irq(dev, pp->irq,
2256 tegra_pcie_ep_hard_irq,
2257 tegra_pcie_ep_irq_thread,
2258 IRQF_SHARED | IRQF_ONESHOT,
2259 "tegra-pcie-ep-intr", pcie);
2260 if (ret) {
2261 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2262 ret);
2263 goto fail;
2264 }
2265
2266 ret = tegra_pcie_config_ep(pcie, pdev);
2267 if (ret < 0)
2268 goto fail;
2269 else
2270 return 0;
2271 break;
2272
2273 default:
2274 dev_err(dev, "Invalid PCIe device type %d\n",
2275 pcie->of_data->mode);
2276 ret = -EINVAL;
2277 }
2278
2279fail:
2280 tegra_bpmp_put(pcie->bpmp);
2281 return ret;
2282}
2283
2284static void tegra_pcie_dw_remove(struct platform_device *pdev)
2285{
2286 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2287
2288 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2289 if (!pcie->link_state)
2290 return;
2291
2292 debugfs_remove_recursive(pcie->debugfs);
2293 tegra_pcie_deinit_controller(pcie);
2294 pm_runtime_put_sync(pcie->dev);
2295 } else {
2296 disable_irq(pcie->pex_rst_irq);
2297 pex_ep_event_pex_rst_assert(pcie);
2298 }
2299
2300 pm_runtime_disable(pcie->dev);
2301 tegra_bpmp_put(pcie->bpmp);
2302 if (pcie->pex_refclk_sel_gpiod)
2303 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2304}
2305
2306static int tegra_pcie_dw_suspend_late(struct device *dev)
2307{
2308 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2309 u32 val;
2310
2311 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2312 dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n");
2313 return -EPERM;
2314 }
2315
2316 if (!pcie->link_state)
2317 return 0;
2318
2319 /* Enable HW_HOT_RST mode */
2320 if (!pcie->of_data->has_sbr_reset_fix) {
2321 val = appl_readl(pcie, APPL_CTRL);
2322 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2323 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2324 val |= APPL_CTRL_HW_HOT_RST_EN;
2325 appl_writel(pcie, val, APPL_CTRL);
2326 }
2327
2328 return 0;
2329}
2330
2331static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2332{
2333 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2334
2335 if (!pcie->link_state)
2336 return 0;
2337
2338 tegra_pcie_downstream_dev_to_D0(pcie);
2339 tegra_pcie_dw_pme_turnoff(pcie);
2340 tegra_pcie_unconfig_controller(pcie);
2341
2342 return 0;
2343}
2344
2345static int tegra_pcie_dw_resume_noirq(struct device *dev)
2346{
2347 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2348 int ret;
2349
2350 if (!pcie->link_state)
2351 return 0;
2352
2353 ret = tegra_pcie_config_controller(pcie, true);
2354 if (ret < 0)
2355 return ret;
2356
2357 ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2358 if (ret < 0) {
2359 dev_err(dev, "Failed to init host: %d\n", ret);
2360 goto fail_host_init;
2361 }
2362
2363 dw_pcie_setup_rc(&pcie->pci.pp);
2364
2365 ret = tegra_pcie_dw_start_link(&pcie->pci);
2366 if (ret < 0)
2367 goto fail_host_init;
2368
2369 return 0;
2370
2371fail_host_init:
2372 tegra_pcie_unconfig_controller(pcie);
2373 return ret;
2374}
2375
2376static int tegra_pcie_dw_resume_early(struct device *dev)
2377{
2378 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2379 u32 val;
2380
2381 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2382 dev_err(dev, "Suspend is not supported in EP mode");
2383 return -ENOTSUPP;
2384 }
2385
2386 if (!pcie->link_state)
2387 return 0;
2388
2389 /* Disable HW_HOT_RST mode */
2390 if (!pcie->of_data->has_sbr_reset_fix) {
2391 val = appl_readl(pcie, APPL_CTRL);
2392 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2393 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2394 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2395 APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2396 val &= ~APPL_CTRL_HW_HOT_RST_EN;
2397 appl_writel(pcie, val, APPL_CTRL);
2398 }
2399
2400 return 0;
2401}
2402
2403static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2404{
2405 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2406
2407 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2408 if (!pcie->link_state)
2409 return;
2410
2411 debugfs_remove_recursive(pcie->debugfs);
2412 tegra_pcie_downstream_dev_to_D0(pcie);
2413
2414 disable_irq(pcie->pci.pp.irq);
2415 if (IS_ENABLED(CONFIG_PCI_MSI))
2416 disable_irq(pcie->pci.pp.msi_irq[0]);
2417
2418 tegra_pcie_dw_pme_turnoff(pcie);
2419 tegra_pcie_unconfig_controller(pcie);
2420 pm_runtime_put_sync(pcie->dev);
2421 } else {
2422 disable_irq(pcie->pex_rst_irq);
2423 pex_ep_event_pex_rst_assert(pcie);
2424 }
2425}
2426
2427static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = {
2428 .version = TEGRA194_DWC_IP_VER,
2429 .mode = DW_PCIE_RC_TYPE,
2430 .cdm_chk_int_en_bit = BIT(19),
2431 /* Gen4 - 5, 6, 8 and 9 presets enabled */
2432 .gen4_preset_vec = 0x360,
2433 .n_fts = { 52, 52 },
2434};
2435
2436static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = {
2437 .version = TEGRA194_DWC_IP_VER,
2438 .mode = DW_PCIE_EP_TYPE,
2439 .cdm_chk_int_en_bit = BIT(19),
2440 /* Gen4 - 5, 6, 8 and 9 presets enabled */
2441 .gen4_preset_vec = 0x360,
2442 .n_fts = { 52, 52 },
2443};
2444
2445static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = {
2446 .version = TEGRA234_DWC_IP_VER,
2447 .mode = DW_PCIE_RC_TYPE,
2448 .has_msix_doorbell_access_fix = true,
2449 .has_sbr_reset_fix = true,
2450 .has_l1ss_exit_fix = true,
2451 .cdm_chk_int_en_bit = BIT(18),
2452 /* Gen4 - 6, 8 and 9 presets enabled */
2453 .gen4_preset_vec = 0x340,
2454 .n_fts = { 52, 80 },
2455};
2456
2457static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = {
2458 .version = TEGRA234_DWC_IP_VER,
2459 .mode = DW_PCIE_EP_TYPE,
2460 .has_l1ss_exit_fix = true,
2461 .has_ltr_req_fix = true,
2462 .cdm_chk_int_en_bit = BIT(18),
2463 /* Gen4 - 6, 8 and 9 presets enabled */
2464 .gen4_preset_vec = 0x340,
2465 .n_fts = { 52, 80 },
2466};
2467
2468static const struct of_device_id tegra_pcie_dw_of_match[] = {
2469 {
2470 .compatible = "nvidia,tegra194-pcie",
2471 .data = &tegra194_pcie_dw_rc_of_data,
2472 },
2473 {
2474 .compatible = "nvidia,tegra194-pcie-ep",
2475 .data = &tegra194_pcie_dw_ep_of_data,
2476 },
2477 {
2478 .compatible = "nvidia,tegra234-pcie",
2479 .data = &tegra234_pcie_dw_rc_of_data,
2480 },
2481 {
2482 .compatible = "nvidia,tegra234-pcie-ep",
2483 .data = &tegra234_pcie_dw_ep_of_data,
2484 },
2485 {}
2486};
2487
2488static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2489 .suspend_late = tegra_pcie_dw_suspend_late,
2490 .suspend_noirq = tegra_pcie_dw_suspend_noirq,
2491 .resume_noirq = tegra_pcie_dw_resume_noirq,
2492 .resume_early = tegra_pcie_dw_resume_early,
2493};
2494
2495static struct platform_driver tegra_pcie_dw_driver = {
2496 .probe = tegra_pcie_dw_probe,
2497 .remove = tegra_pcie_dw_remove,
2498 .shutdown = tegra_pcie_dw_shutdown,
2499 .driver = {
2500 .name = "tegra194-pcie",
2501 .pm = &tegra_pcie_dw_pm_ops,
2502 .of_match_table = tegra_pcie_dw_of_match,
2503 },
2504};
2505module_platform_driver(tegra_pcie_dw_driver);
2506
2507MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2508
2509MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2510MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2511MODULE_LICENSE("GPL v2");
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PCIe host controller driver for Tegra194 SoC
4 *
5 * Copyright (C) 2019 NVIDIA Corporation.
6 *
7 * Author: Vidya Sagar <vidyas@nvidia.com>
8 */
9
10#include <linux/clk.h>
11#include <linux/debugfs.h>
12#include <linux/delay.h>
13#include <linux/gpio.h>
14#include <linux/gpio/consumer.h>
15#include <linux/interrupt.h>
16#include <linux/iopoll.h>
17#include <linux/kernel.h>
18#include <linux/module.h>
19#include <linux/of.h>
20#include <linux/of_device.h>
21#include <linux/of_gpio.h>
22#include <linux/of_irq.h>
23#include <linux/of_pci.h>
24#include <linux/pci.h>
25#include <linux/phy/phy.h>
26#include <linux/pinctrl/consumer.h>
27#include <linux/platform_device.h>
28#include <linux/pm_runtime.h>
29#include <linux/random.h>
30#include <linux/reset.h>
31#include <linux/resource.h>
32#include <linux/types.h>
33#include "pcie-designware.h"
34#include <soc/tegra/bpmp.h>
35#include <soc/tegra/bpmp-abi.h>
36#include "../../pci.h"
37
38#define APPL_PINMUX 0x0
39#define APPL_PINMUX_PEX_RST BIT(0)
40#define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)
41#define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)
42#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)
43#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)
44
45#define APPL_CTRL 0x4
46#define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)
47#define APPL_CTRL_LTSSM_EN BIT(7)
48#define APPL_CTRL_HW_HOT_RST_EN BIT(20)
49#define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)
50#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22
51#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1
52
53#define APPL_INTR_EN_L0_0 0x8
54#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)
55#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)
56#define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)
57#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15)
58#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)
59#define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)
60#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)
61
62#define APPL_INTR_STATUS_L0 0xC
63#define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)
64#define APPL_INTR_STATUS_L0_INT_INT BIT(8)
65#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15)
66#define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16)
67#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)
68
69#define APPL_INTR_EN_L1_0_0 0x1C
70#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)
71#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3)
72#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30)
73
74#define APPL_INTR_STATUS_L1_0_0 0x20
75#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)
76#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3)
77#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30)
78
79#define APPL_INTR_STATUS_L1_1 0x2C
80#define APPL_INTR_STATUS_L1_2 0x30
81#define APPL_INTR_STATUS_L1_3 0x34
82#define APPL_INTR_STATUS_L1_6 0x3C
83#define APPL_INTR_STATUS_L1_7 0x40
84#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1)
85
86#define APPL_INTR_EN_L1_8_0 0x44
87#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)
88#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)
89#define APPL_INTR_EN_L1_8_INTX_EN BIT(11)
90#define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)
91
92#define APPL_INTR_STATUS_L1_8_0 0x4C
93#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)
94#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)
95#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)
96
97#define APPL_INTR_STATUS_L1_9 0x54
98#define APPL_INTR_STATUS_L1_10 0x58
99#define APPL_INTR_STATUS_L1_11 0x64
100#define APPL_INTR_STATUS_L1_13 0x74
101#define APPL_INTR_STATUS_L1_14 0x78
102#define APPL_INTR_STATUS_L1_15 0x7C
103#define APPL_INTR_STATUS_L1_17 0x88
104
105#define APPL_INTR_EN_L1_18 0x90
106#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)
107#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
108#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
109
110#define APPL_INTR_STATUS_L1_18 0x94
111#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)
112#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
113#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
114
115#define APPL_MSI_CTRL_1 0xAC
116
117#define APPL_MSI_CTRL_2 0xB0
118
119#define APPL_LEGACY_INTX 0xB8
120
121#define APPL_LTR_MSG_1 0xC4
122#define LTR_MSG_REQ BIT(15)
123#define LTR_MST_NO_SNOOP_SHIFT 16
124
125#define APPL_LTR_MSG_2 0xC8
126#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)
127
128#define APPL_LINK_STATUS 0xCC
129#define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)
130
131#define APPL_DEBUG 0xD0
132#define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)
133#define APPL_DEBUG_PM_LINKST_IN_L0 0x11
134#define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)
135#define APPL_DEBUG_LTSSM_STATE_SHIFT 3
136#define LTSSM_STATE_PRE_DETECT 5
137
138#define APPL_RADM_STATUS 0xE4
139#define APPL_PM_XMT_TURNOFF_STATE BIT(0)
140
141#define APPL_DM_TYPE 0x100
142#define APPL_DM_TYPE_MASK GENMASK(3, 0)
143#define APPL_DM_TYPE_RP 0x4
144#define APPL_DM_TYPE_EP 0x0
145
146#define APPL_CFG_BASE_ADDR 0x104
147#define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)
148
149#define APPL_CFG_IATU_DMA_BASE_ADDR 0x108
150#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)
151
152#define APPL_CFG_MISC 0x110
153#define APPL_CFG_MISC_SLV_EP_MODE BIT(14)
154#define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)
155#define APPL_CFG_MISC_ARCACHE_SHIFT 10
156#define APPL_CFG_MISC_ARCACHE_VAL 3
157
158#define APPL_CFG_SLCG_OVERRIDE 0x114
159#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)
160
161#define APPL_CAR_RESET_OVRD 0x12C
162#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)
163
164#define IO_BASE_IO_DECODE BIT(0)
165#define IO_BASE_IO_DECODE_BIT8 BIT(8)
166
167#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)
168#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)
169
170#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718
171#define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)
172
173#define EVENT_COUNTER_ALL_CLEAR 0x3
174#define EVENT_COUNTER_ENABLE_ALL 0x7
175#define EVENT_COUNTER_ENABLE_SHIFT 2
176#define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0)
177#define EVENT_COUNTER_EVENT_SEL_SHIFT 16
178#define EVENT_COUNTER_EVENT_Tx_L0S 0x2
179#define EVENT_COUNTER_EVENT_Rx_L0S 0x3
180#define EVENT_COUNTER_EVENT_L1 0x5
181#define EVENT_COUNTER_EVENT_L1_1 0x7
182#define EVENT_COUNTER_EVENT_L1_2 0x8
183#define EVENT_COUNTER_GROUP_SEL_SHIFT 24
184#define EVENT_COUNTER_GROUP_5 0x5
185
186#define N_FTS_VAL 52
187#define FTS_VAL 52
188
189#define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828
190
191#define GEN3_EQ_CONTROL_OFF 0x8a8
192#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8
193#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8)
194#define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0)
195
196#define GEN3_RELATED_OFF 0x890
197#define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0)
198#define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16)
199#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24
200#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24)
201
202#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0
203#define AMBA_ERROR_RESPONSE_CRS_SHIFT 3
204#define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0)
205#define AMBA_ERROR_RESPONSE_CRS_OKAY 0
206#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1
207#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2
208
209#define MSIX_ADDR_MATCH_LOW_OFF 0x940
210#define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0)
211#define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2)
212
213#define MSIX_ADDR_MATCH_HIGH_OFF 0x944
214#define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0)
215
216#define PORT_LOGIC_MSIX_DOORBELL 0x948
217
218#define CAP_SPCIE_CAP_OFF 0x154
219#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)
220#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)
221#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8
222
223#define PME_ACK_TIMEOUT 10000
224
225#define LTSSM_TIMEOUT 50000 /* 50ms */
226
227#define GEN3_GEN4_EQ_PRESET_INIT 5
228
229#define GEN1_CORE_CLK_FREQ 62500000
230#define GEN2_CORE_CLK_FREQ 125000000
231#define GEN3_CORE_CLK_FREQ 250000000
232#define GEN4_CORE_CLK_FREQ 500000000
233
234#define LTR_MSG_TIMEOUT (100 * 1000)
235
236#define PERST_DEBOUNCE_TIME (5 * 1000)
237
238#define EP_STATE_DISABLED 0
239#define EP_STATE_ENABLED 1
240
241static const unsigned int pcie_gen_freq[] = {
242 GEN1_CORE_CLK_FREQ,
243 GEN2_CORE_CLK_FREQ,
244 GEN3_CORE_CLK_FREQ,
245 GEN4_CORE_CLK_FREQ
246};
247
248struct tegra_pcie_dw {
249 struct device *dev;
250 struct resource *appl_res;
251 struct resource *dbi_res;
252 struct resource *atu_dma_res;
253 void __iomem *appl_base;
254 struct clk *core_clk;
255 struct reset_control *core_apb_rst;
256 struct reset_control *core_rst;
257 struct dw_pcie pci;
258 struct tegra_bpmp *bpmp;
259
260 enum dw_pcie_device_mode mode;
261
262 bool supports_clkreq;
263 bool enable_cdm_check;
264 bool link_state;
265 bool update_fc_fixup;
266 u8 init_link_width;
267 u32 msi_ctrl_int;
268 u32 num_lanes;
269 u32 cid;
270 u32 cfg_link_cap_l1sub;
271 u32 pcie_cap_base;
272 u32 aspm_cmrt;
273 u32 aspm_pwr_on_t;
274 u32 aspm_l0s_enter_lat;
275
276 struct regulator *pex_ctl_supply;
277 struct regulator *slot_ctl_3v3;
278 struct regulator *slot_ctl_12v;
279
280 unsigned int phy_count;
281 struct phy **phys;
282
283 struct dentry *debugfs;
284
285 /* Endpoint mode specific */
286 struct gpio_desc *pex_rst_gpiod;
287 struct gpio_desc *pex_refclk_sel_gpiod;
288 unsigned int pex_rst_irq;
289 int ep_state;
290};
291
292struct tegra_pcie_dw_of_data {
293 enum dw_pcie_device_mode mode;
294};
295
296static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
297{
298 return container_of(pci, struct tegra_pcie_dw, pci);
299}
300
301static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
302 const u32 reg)
303{
304 writel_relaxed(value, pcie->appl_base + reg);
305}
306
307static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
308{
309 return readl_relaxed(pcie->appl_base + reg);
310}
311
312struct tegra_pcie_soc {
313 enum dw_pcie_device_mode mode;
314};
315
316static void apply_bad_link_workaround(struct pcie_port *pp)
317{
318 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
319 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
320 u32 current_link_width;
321 u16 val;
322
323 /*
324 * NOTE:- Since this scenario is uncommon and link as such is not
325 * stable anyway, not waiting to confirm if link is really
326 * transitioning to Gen-2 speed
327 */
328 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
329 if (val & PCI_EXP_LNKSTA_LBMS) {
330 current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
331 PCI_EXP_LNKSTA_NLW_SHIFT;
332 if (pcie->init_link_width > current_link_width) {
333 dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
334 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
335 PCI_EXP_LNKCTL2);
336 val &= ~PCI_EXP_LNKCTL2_TLS;
337 val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
338 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
339 PCI_EXP_LNKCTL2, val);
340
341 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
342 PCI_EXP_LNKCTL);
343 val |= PCI_EXP_LNKCTL_RL;
344 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
345 PCI_EXP_LNKCTL, val);
346 }
347 }
348}
349
350static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
351{
352 struct tegra_pcie_dw *pcie = arg;
353 struct dw_pcie *pci = &pcie->pci;
354 struct pcie_port *pp = &pci->pp;
355 u32 val, tmp;
356 u16 val_w;
357
358 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
359 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
360 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
361 if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
362 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
363
364 /* SBR & Surprise Link Down WAR */
365 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
366 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
367 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
368 udelay(1);
369 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
370 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
371 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
372
373 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
374 val |= PORT_LOGIC_SPEED_CHANGE;
375 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
376 }
377 }
378
379 if (val & APPL_INTR_STATUS_L0_INT_INT) {
380 val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
381 if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
382 appl_writel(pcie,
383 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
384 APPL_INTR_STATUS_L1_8_0);
385 apply_bad_link_workaround(pp);
386 }
387 if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
388 appl_writel(pcie,
389 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
390 APPL_INTR_STATUS_L1_8_0);
391
392 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
393 PCI_EXP_LNKSTA);
394 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
395 PCI_EXP_LNKSTA_CLS);
396 }
397 }
398
399 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
400 if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
401 val = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
402 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
403 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
404 dev_info(pci->dev, "CDM check complete\n");
405 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
406 }
407 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
408 dev_err(pci->dev, "CDM comparison mismatch\n");
409 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
410 }
411 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
412 dev_err(pci->dev, "CDM Logic error\n");
413 tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
414 }
415 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp);
416 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
417 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp);
418 }
419
420 return IRQ_HANDLED;
421}
422
423static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
424{
425 u32 val;
426
427 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
428 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
429 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
430 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
431 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
432 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
433 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
434 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
435 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
436 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
437 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
438 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
439 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
440 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
441 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
442 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
443
444 val = appl_readl(pcie, APPL_CTRL);
445 val |= APPL_CTRL_LTSSM_EN;
446 appl_writel(pcie, val, APPL_CTRL);
447}
448
449static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
450{
451 struct tegra_pcie_dw *pcie = arg;
452 struct dw_pcie *pci = &pcie->pci;
453 u32 val, speed;
454
455 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
456 PCI_EXP_LNKSTA_CLS;
457 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
458
459 /* If EP doesn't advertise L1SS, just return */
460 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
461 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
462 return IRQ_HANDLED;
463
464 /* Check if BME is set to '1' */
465 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
466 if (val & PCI_COMMAND_MASTER) {
467 ktime_t timeout;
468
469 /* 110us for both snoop and no-snoop */
470 val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ;
471 val |= (val << LTR_MST_NO_SNOOP_SHIFT);
472 appl_writel(pcie, val, APPL_LTR_MSG_1);
473
474 /* Send LTR upstream */
475 val = appl_readl(pcie, APPL_LTR_MSG_2);
476 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
477 appl_writel(pcie, val, APPL_LTR_MSG_2);
478
479 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
480 for (;;) {
481 val = appl_readl(pcie, APPL_LTR_MSG_2);
482 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
483 break;
484 if (ktime_after(ktime_get(), timeout))
485 break;
486 usleep_range(1000, 1100);
487 }
488 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
489 dev_err(pcie->dev, "Failed to send LTR message\n");
490 }
491
492 return IRQ_HANDLED;
493}
494
495static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
496{
497 struct tegra_pcie_dw *pcie = arg;
498 struct dw_pcie_ep *ep = &pcie->pci.ep;
499 int spurious = 1;
500 u32 status_l0, status_l1, link_status;
501
502 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
503 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
504 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
505 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
506
507 if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
508 pex_ep_event_hot_rst_done(pcie);
509
510 if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
511 link_status = appl_readl(pcie, APPL_LINK_STATUS);
512 if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {
513 dev_dbg(pcie->dev, "Link is up with Host\n");
514 dw_pcie_ep_linkup(ep);
515 }
516 }
517
518 spurious = 0;
519 }
520
521 if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
522 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
523 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15);
524
525 if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
526 return IRQ_WAKE_THREAD;
527
528 spurious = 0;
529 }
530
531 if (spurious) {
532 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
533 status_l0);
534 appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0);
535 }
536
537 return IRQ_HANDLED;
538}
539
540static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
541 int size, u32 *val)
542{
543 /*
544 * This is an endpoint mode specific register happen to appear even
545 * when controller is operating in root port mode and system hangs
546 * when it is accessed with link being in ASPM-L1 state.
547 * So skip accessing it altogether
548 */
549 if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
550 *val = 0x00000000;
551 return PCIBIOS_SUCCESSFUL;
552 }
553
554 return pci_generic_config_read(bus, devfn, where, size, val);
555}
556
557static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
558 int size, u32 val)
559{
560 /*
561 * This is an endpoint mode specific register happen to appear even
562 * when controller is operating in root port mode and system hangs
563 * when it is accessed with link being in ASPM-L1 state.
564 * So skip accessing it altogether
565 */
566 if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
567 return PCIBIOS_SUCCESSFUL;
568
569 return pci_generic_config_write(bus, devfn, where, size, val);
570}
571
572static struct pci_ops tegra_pci_ops = {
573 .map_bus = dw_pcie_own_conf_map_bus,
574 .read = tegra_pcie_dw_rd_own_conf,
575 .write = tegra_pcie_dw_wr_own_conf,
576};
577
578#if defined(CONFIG_PCIEASPM)
579static const u32 event_cntr_ctrl_offset[] = {
580 0x1d8,
581 0x1a8,
582 0x1a8,
583 0x1a8,
584 0x1c4,
585 0x1d8
586};
587
588static const u32 event_cntr_data_offset[] = {
589 0x1dc,
590 0x1ac,
591 0x1ac,
592 0x1ac,
593 0x1c8,
594 0x1dc
595};
596
597static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
598{
599 u32 val;
600
601 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
602 val &= ~PCI_L1SS_CAP_ASPM_L1_1;
603 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
604}
605
606static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
607{
608 u32 val;
609
610 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
611 val &= ~PCI_L1SS_CAP_ASPM_L1_2;
612 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
613}
614
615static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
616{
617 u32 val;
618
619 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]);
620 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
621 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
622 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
623 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
624 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
625 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]);
626
627 return val;
628}
629
630static int aspm_state_cnt(struct seq_file *s, void *data)
631{
632 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
633 dev_get_drvdata(s->private);
634 u32 val;
635
636 seq_printf(s, "Tx L0s entry count : %u\n",
637 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
638
639 seq_printf(s, "Rx L0s entry count : %u\n",
640 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
641
642 seq_printf(s, "Link L1 entry count : %u\n",
643 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
644
645 seq_printf(s, "Link L1.1 entry count : %u\n",
646 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
647
648 seq_printf(s, "Link L1.2 entry count : %u\n",
649 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
650
651 /* Clear all counters */
652 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid],
653 EVENT_COUNTER_ALL_CLEAR);
654
655 /* Re-enable counting */
656 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
657 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
658 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
659
660 return 0;
661}
662
663static void init_host_aspm(struct tegra_pcie_dw *pcie)
664{
665 struct dw_pcie *pci = &pcie->pci;
666 u32 val;
667
668 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
669 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
670
671 /* Enable ASPM counters */
672 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
673 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
674 dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val);
675
676 /* Program T_cmrt and T_pwr_on values */
677 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
678 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
679 val |= (pcie->aspm_cmrt << 8);
680 val |= (pcie->aspm_pwr_on_t << 19);
681 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
682
683 /* Program L0s and L1 entrance latencies */
684 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
685 val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
686 val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
687 val |= PORT_AFR_ENTER_ASPM;
688 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
689}
690
691static void init_debugfs(struct tegra_pcie_dw *pcie)
692{
693 debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,
694 aspm_state_cnt);
695}
696#else
697static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
698static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
699static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
700static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
701#endif
702
703static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp)
704{
705 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
706 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
707 u32 val;
708 u16 val_w;
709
710 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
711 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
712 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
713
714 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
715 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
716 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
717
718 if (pcie->enable_cdm_check) {
719 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
720 val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN;
721 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
722
723 val = appl_readl(pcie, APPL_INTR_EN_L1_18);
724 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
725 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
726 appl_writel(pcie, val, APPL_INTR_EN_L1_18);
727 }
728
729 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
730 PCI_EXP_LNKSTA);
731 pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
732 PCI_EXP_LNKSTA_NLW_SHIFT;
733
734 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
735 PCI_EXP_LNKCTL);
736 val_w |= PCI_EXP_LNKCTL_LBMIE;
737 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
738 val_w);
739}
740
741static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp)
742{
743 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
744 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
745 u32 val;
746
747 /* Enable legacy interrupt generation */
748 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
749 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
750 val |= APPL_INTR_EN_L0_0_INT_INT_EN;
751 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
752
753 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
754 val |= APPL_INTR_EN_L1_8_INTX_EN;
755 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
756 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
757 if (IS_ENABLED(CONFIG_PCIEAER))
758 val |= APPL_INTR_EN_L1_8_AER_INT_EN;
759 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
760}
761
762static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp)
763{
764 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
765 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
766 u32 val;
767
768 /* Enable MSI interrupt generation */
769 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
770 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
771 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
772 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
773}
774
775static void tegra_pcie_enable_interrupts(struct pcie_port *pp)
776{
777 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
778 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
779
780 /* Clear interrupt statuses before enabling interrupts */
781 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
782 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
783 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
784 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
785 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
786 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
787 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
788 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
789 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
790 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
791 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
792 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
793 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
794 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
795 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
796
797 tegra_pcie_enable_system_interrupts(pp);
798 tegra_pcie_enable_legacy_interrupts(pp);
799 if (IS_ENABLED(CONFIG_PCI_MSI))
800 tegra_pcie_enable_msi_interrupts(pp);
801}
802
803static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
804{
805 struct dw_pcie *pci = &pcie->pci;
806 u32 val, offset, i;
807
808 /* Program init preset */
809 for (i = 0; i < pcie->num_lanes; i++) {
810 val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
811 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
812 val |= GEN3_GEN4_EQ_PRESET_INIT;
813 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
814 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
815 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
816 dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
817
818 offset = dw_pcie_find_ext_capability(pci,
819 PCI_EXT_CAP_ID_PL_16GT) +
820 PCI_PL_16GT_LE_CTRL;
821 val = dw_pcie_readb_dbi(pci, offset + i);
822 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
823 val |= GEN3_GEN4_EQ_PRESET_INIT;
824 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
825 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
826 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
827 dw_pcie_writeb_dbi(pci, offset + i, val);
828 }
829
830 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
831 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
832 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
833
834 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
835 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
836 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
837 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
838 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
839
840 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
841 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
842 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
843 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
844
845 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
846 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
847 val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
848 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
849 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
850
851 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
852 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
853 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
854}
855
856static int tegra_pcie_dw_host_init(struct pcie_port *pp)
857{
858 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
859 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
860 u32 val;
861
862 pp->bridge->ops = &tegra_pci_ops;
863
864 if (!pcie->pcie_cap_base)
865 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
866 PCI_CAP_ID_EXP);
867
868 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
869 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
870 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
871
872 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
873 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
874 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
875 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
876
877 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
878
879 /* Enable as 0xFFFF0001 response for CRS */
880 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
881 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
882 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
883 AMBA_ERROR_RESPONSE_CRS_SHIFT);
884 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
885
886 /* Configure Max lane width from DT */
887 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
888 val &= ~PCI_EXP_LNKCAP_MLW;
889 val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
890 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
891
892 config_gen3_gen4_eq_presets(pcie);
893
894 init_host_aspm(pcie);
895
896 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
897 if (!pcie->supports_clkreq) {
898 disable_aspm_l11(pcie);
899 disable_aspm_l12(pcie);
900 }
901
902 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
903 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
904 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
905
906 if (pcie->update_fc_fixup) {
907 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
908 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
909 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
910 }
911
912 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
913
914 return 0;
915}
916
917static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
918{
919 u32 val, offset, speed, tmp;
920 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
921 struct pcie_port *pp = &pci->pp;
922 bool retry = true;
923
924 if (pcie->mode == DW_PCIE_EP_TYPE) {
925 enable_irq(pcie->pex_rst_irq);
926 return 0;
927 }
928
929retry_link:
930 /* Assert RST */
931 val = appl_readl(pcie, APPL_PINMUX);
932 val &= ~APPL_PINMUX_PEX_RST;
933 appl_writel(pcie, val, APPL_PINMUX);
934
935 usleep_range(100, 200);
936
937 /* Enable LTSSM */
938 val = appl_readl(pcie, APPL_CTRL);
939 val |= APPL_CTRL_LTSSM_EN;
940 appl_writel(pcie, val, APPL_CTRL);
941
942 /* De-assert RST */
943 val = appl_readl(pcie, APPL_PINMUX);
944 val |= APPL_PINMUX_PEX_RST;
945 appl_writel(pcie, val, APPL_PINMUX);
946
947 msleep(100);
948
949 if (dw_pcie_wait_for_link(pci)) {
950 if (!retry)
951 return 0;
952 /*
953 * There are some endpoints which can't get the link up if
954 * root port has Data Link Feature (DLF) enabled.
955 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
956 * on Scaled Flow Control and DLF.
957 * So, need to confirm that is indeed the case here and attempt
958 * link up once again with DLF disabled.
959 */
960 val = appl_readl(pcie, APPL_DEBUG);
961 val &= APPL_DEBUG_LTSSM_STATE_MASK;
962 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
963 tmp = appl_readl(pcie, APPL_LINK_STATUS);
964 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
965 if (!(val == 0x11 && !tmp)) {
966 /* Link is down for all good reasons */
967 return 0;
968 }
969
970 dev_info(pci->dev, "Link is down in DLL");
971 dev_info(pci->dev, "Trying again with DLFE disabled\n");
972 /* Disable LTSSM */
973 val = appl_readl(pcie, APPL_CTRL);
974 val &= ~APPL_CTRL_LTSSM_EN;
975 appl_writel(pcie, val, APPL_CTRL);
976
977 reset_control_assert(pcie->core_rst);
978 reset_control_deassert(pcie->core_rst);
979
980 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
981 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
982 val &= ~PCI_DLF_EXCHANGE_ENABLE;
983 dw_pcie_writel_dbi(pci, offset, val);
984
985 tegra_pcie_dw_host_init(pp);
986 dw_pcie_setup_rc(pp);
987
988 retry = false;
989 goto retry_link;
990 }
991
992 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
993 PCI_EXP_LNKSTA_CLS;
994 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
995
996 tegra_pcie_enable_interrupts(pp);
997
998 return 0;
999}
1000
1001static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1002{
1003 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1004 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1005
1006 return !!(val & PCI_EXP_LNKSTA_DLLLA);
1007}
1008
1009static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1010{
1011 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1012
1013 disable_irq(pcie->pex_rst_irq);
1014}
1015
1016static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1017 .link_up = tegra_pcie_dw_link_up,
1018 .start_link = tegra_pcie_dw_start_link,
1019 .stop_link = tegra_pcie_dw_stop_link,
1020};
1021
1022static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1023 .host_init = tegra_pcie_dw_host_init,
1024};
1025
1026static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1027{
1028 unsigned int phy_count = pcie->phy_count;
1029
1030 while (phy_count--) {
1031 phy_power_off(pcie->phys[phy_count]);
1032 phy_exit(pcie->phys[phy_count]);
1033 }
1034}
1035
1036static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1037{
1038 unsigned int i;
1039 int ret;
1040
1041 for (i = 0; i < pcie->phy_count; i++) {
1042 ret = phy_init(pcie->phys[i]);
1043 if (ret < 0)
1044 goto phy_power_off;
1045
1046 ret = phy_power_on(pcie->phys[i]);
1047 if (ret < 0)
1048 goto phy_exit;
1049 }
1050
1051 return 0;
1052
1053phy_power_off:
1054 while (i--) {
1055 phy_power_off(pcie->phys[i]);
1056phy_exit:
1057 phy_exit(pcie->phys[i]);
1058 }
1059
1060 return ret;
1061}
1062
1063static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1064{
1065 struct platform_device *pdev = to_platform_device(pcie->dev);
1066 struct device_node *np = pcie->dev->of_node;
1067 int ret;
1068
1069 pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1070 if (!pcie->dbi_res) {
1071 dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1072 return -ENODEV;
1073 }
1074
1075 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1076 if (ret < 0) {
1077 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1078 return ret;
1079 }
1080
1081 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1082 &pcie->aspm_pwr_on_t);
1083 if (ret < 0)
1084 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1085 ret);
1086
1087 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1088 &pcie->aspm_l0s_enter_lat);
1089 if (ret < 0)
1090 dev_info(pcie->dev,
1091 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1092
1093 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1094 if (ret < 0) {
1095 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1096 return ret;
1097 }
1098
1099 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1100 if (ret) {
1101 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1102 return ret;
1103 }
1104
1105 ret = of_property_count_strings(np, "phy-names");
1106 if (ret < 0) {
1107 dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1108 ret);
1109 return ret;
1110 }
1111 pcie->phy_count = ret;
1112
1113 if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1114 pcie->update_fc_fixup = true;
1115
1116 pcie->supports_clkreq =
1117 of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1118
1119 pcie->enable_cdm_check =
1120 of_property_read_bool(np, "snps,enable-cdm-check");
1121
1122 if (pcie->mode == DW_PCIE_RC_TYPE)
1123 return 0;
1124
1125 /* Endpoint mode specific DT entries */
1126 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1127 if (IS_ERR(pcie->pex_rst_gpiod)) {
1128 int err = PTR_ERR(pcie->pex_rst_gpiod);
1129 const char *level = KERN_ERR;
1130
1131 if (err == -EPROBE_DEFER)
1132 level = KERN_DEBUG;
1133
1134 dev_printk(level, pcie->dev,
1135 dev_fmt("Failed to get PERST GPIO: %d\n"),
1136 err);
1137 return err;
1138 }
1139
1140 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1141 "nvidia,refclk-select",
1142 GPIOD_OUT_HIGH);
1143 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1144 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1145 const char *level = KERN_ERR;
1146
1147 if (err == -EPROBE_DEFER)
1148 level = KERN_DEBUG;
1149
1150 dev_printk(level, pcie->dev,
1151 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1152 err);
1153 pcie->pex_refclk_sel_gpiod = NULL;
1154 }
1155
1156 return 0;
1157}
1158
1159static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1160 bool enable)
1161{
1162 struct mrq_uphy_response resp;
1163 struct tegra_bpmp_message msg;
1164 struct mrq_uphy_request req;
1165
1166 /* Controller-5 doesn't need to have its state set by BPMP-FW */
1167 if (pcie->cid == 5)
1168 return 0;
1169
1170 memset(&req, 0, sizeof(req));
1171 memset(&resp, 0, sizeof(resp));
1172
1173 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1174 req.controller_state.pcie_controller = pcie->cid;
1175 req.controller_state.enable = enable;
1176
1177 memset(&msg, 0, sizeof(msg));
1178 msg.mrq = MRQ_UPHY;
1179 msg.tx.data = &req;
1180 msg.tx.size = sizeof(req);
1181 msg.rx.data = &resp;
1182 msg.rx.size = sizeof(resp);
1183
1184 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1185}
1186
1187static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1188 bool enable)
1189{
1190 struct mrq_uphy_response resp;
1191 struct tegra_bpmp_message msg;
1192 struct mrq_uphy_request req;
1193
1194 memset(&req, 0, sizeof(req));
1195 memset(&resp, 0, sizeof(resp));
1196
1197 if (enable) {
1198 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1199 req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1200 } else {
1201 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1202 req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1203 }
1204
1205 memset(&msg, 0, sizeof(msg));
1206 msg.mrq = MRQ_UPHY;
1207 msg.tx.data = &req;
1208 msg.tx.size = sizeof(req);
1209 msg.rx.data = &resp;
1210 msg.rx.size = sizeof(resp);
1211
1212 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1213}
1214
1215static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1216{
1217 struct pcie_port *pp = &pcie->pci.pp;
1218 struct pci_bus *child, *root_bus = NULL;
1219 struct pci_dev *pdev;
1220
1221 /*
1222 * link doesn't go into L2 state with some of the endpoints with Tegra
1223 * if they are not in D0 state. So, need to make sure that immediate
1224 * downstream devices are in D0 state before sending PME_TurnOff to put
1225 * link into L2 state.
1226 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1227 * 5.2 Link State Power Management (Page #428).
1228 */
1229
1230 list_for_each_entry(child, &pp->bridge->bus->children, node) {
1231 /* Bring downstream devices to D0 if they are not already in */
1232 if (child->parent == pp->bridge->bus) {
1233 root_bus = child;
1234 break;
1235 }
1236 }
1237
1238 if (!root_bus) {
1239 dev_err(pcie->dev, "Failed to find downstream devices\n");
1240 return;
1241 }
1242
1243 list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1244 if (PCI_SLOT(pdev->devfn) == 0) {
1245 if (pci_set_power_state(pdev, PCI_D0))
1246 dev_err(pcie->dev,
1247 "Failed to transition %s to D0 state\n",
1248 dev_name(&pdev->dev));
1249 }
1250 }
1251}
1252
1253static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1254{
1255 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1256 if (IS_ERR(pcie->slot_ctl_3v3)) {
1257 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1258 return PTR_ERR(pcie->slot_ctl_3v3);
1259
1260 pcie->slot_ctl_3v3 = NULL;
1261 }
1262
1263 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1264 if (IS_ERR(pcie->slot_ctl_12v)) {
1265 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1266 return PTR_ERR(pcie->slot_ctl_12v);
1267
1268 pcie->slot_ctl_12v = NULL;
1269 }
1270
1271 return 0;
1272}
1273
1274static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1275{
1276 int ret;
1277
1278 if (pcie->slot_ctl_3v3) {
1279 ret = regulator_enable(pcie->slot_ctl_3v3);
1280 if (ret < 0) {
1281 dev_err(pcie->dev,
1282 "Failed to enable 3.3V slot supply: %d\n", ret);
1283 return ret;
1284 }
1285 }
1286
1287 if (pcie->slot_ctl_12v) {
1288 ret = regulator_enable(pcie->slot_ctl_12v);
1289 if (ret < 0) {
1290 dev_err(pcie->dev,
1291 "Failed to enable 12V slot supply: %d\n", ret);
1292 goto fail_12v_enable;
1293 }
1294 }
1295
1296 /*
1297 * According to PCI Express Card Electromechanical Specification
1298 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1299 * should be a minimum of 100ms.
1300 */
1301 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1302 msleep(100);
1303
1304 return 0;
1305
1306fail_12v_enable:
1307 if (pcie->slot_ctl_3v3)
1308 regulator_disable(pcie->slot_ctl_3v3);
1309 return ret;
1310}
1311
1312static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1313{
1314 if (pcie->slot_ctl_12v)
1315 regulator_disable(pcie->slot_ctl_12v);
1316 if (pcie->slot_ctl_3v3)
1317 regulator_disable(pcie->slot_ctl_3v3);
1318}
1319
1320static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1321 bool en_hw_hot_rst)
1322{
1323 int ret;
1324 u32 val;
1325
1326 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1327 if (ret) {
1328 dev_err(pcie->dev,
1329 "Failed to enable controller %u: %d\n", pcie->cid, ret);
1330 return ret;
1331 }
1332
1333 ret = tegra_pcie_enable_slot_regulators(pcie);
1334 if (ret < 0)
1335 goto fail_slot_reg_en;
1336
1337 ret = regulator_enable(pcie->pex_ctl_supply);
1338 if (ret < 0) {
1339 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1340 goto fail_reg_en;
1341 }
1342
1343 ret = clk_prepare_enable(pcie->core_clk);
1344 if (ret) {
1345 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1346 goto fail_core_clk;
1347 }
1348
1349 ret = reset_control_deassert(pcie->core_apb_rst);
1350 if (ret) {
1351 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1352 ret);
1353 goto fail_core_apb_rst;
1354 }
1355
1356 if (en_hw_hot_rst) {
1357 /* Enable HW_HOT_RST mode */
1358 val = appl_readl(pcie, APPL_CTRL);
1359 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1360 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1361 val |= APPL_CTRL_HW_HOT_RST_EN;
1362 appl_writel(pcie, val, APPL_CTRL);
1363 }
1364
1365 ret = tegra_pcie_enable_phy(pcie);
1366 if (ret) {
1367 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1368 goto fail_phy;
1369 }
1370
1371 /* Update CFG base address */
1372 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1373 APPL_CFG_BASE_ADDR);
1374
1375 /* Configure this core for RP mode operation */
1376 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1377
1378 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1379
1380 val = appl_readl(pcie, APPL_CTRL);
1381 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1382
1383 val = appl_readl(pcie, APPL_CFG_MISC);
1384 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1385 appl_writel(pcie, val, APPL_CFG_MISC);
1386
1387 if (!pcie->supports_clkreq) {
1388 val = appl_readl(pcie, APPL_PINMUX);
1389 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1390 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1391 appl_writel(pcie, val, APPL_PINMUX);
1392 }
1393
1394 /* Update iATU_DMA base address */
1395 appl_writel(pcie,
1396 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1397 APPL_CFG_IATU_DMA_BASE_ADDR);
1398
1399 reset_control_deassert(pcie->core_rst);
1400
1401 return ret;
1402
1403fail_phy:
1404 reset_control_assert(pcie->core_apb_rst);
1405fail_core_apb_rst:
1406 clk_disable_unprepare(pcie->core_clk);
1407fail_core_clk:
1408 regulator_disable(pcie->pex_ctl_supply);
1409fail_reg_en:
1410 tegra_pcie_disable_slot_regulators(pcie);
1411fail_slot_reg_en:
1412 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1413
1414 return ret;
1415}
1416
1417static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1418{
1419 int ret;
1420
1421 ret = reset_control_assert(pcie->core_rst);
1422 if (ret)
1423 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1424
1425 tegra_pcie_disable_phy(pcie);
1426
1427 ret = reset_control_assert(pcie->core_apb_rst);
1428 if (ret)
1429 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1430
1431 clk_disable_unprepare(pcie->core_clk);
1432
1433 ret = regulator_disable(pcie->pex_ctl_supply);
1434 if (ret)
1435 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1436
1437 tegra_pcie_disable_slot_regulators(pcie);
1438
1439 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1440 if (ret)
1441 dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1442 pcie->cid, ret);
1443}
1444
1445static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1446{
1447 struct dw_pcie *pci = &pcie->pci;
1448 struct pcie_port *pp = &pci->pp;
1449 int ret;
1450
1451 ret = tegra_pcie_config_controller(pcie, false);
1452 if (ret < 0)
1453 return ret;
1454
1455 pp->ops = &tegra_pcie_dw_host_ops;
1456
1457 ret = dw_pcie_host_init(pp);
1458 if (ret < 0) {
1459 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1460 goto fail_host_init;
1461 }
1462
1463 return 0;
1464
1465fail_host_init:
1466 tegra_pcie_unconfig_controller(pcie);
1467 return ret;
1468}
1469
1470static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1471{
1472 u32 val;
1473
1474 if (!tegra_pcie_dw_link_up(&pcie->pci))
1475 return 0;
1476
1477 val = appl_readl(pcie, APPL_RADM_STATUS);
1478 val |= APPL_PM_XMT_TURNOFF_STATE;
1479 appl_writel(pcie, val, APPL_RADM_STATUS);
1480
1481 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1482 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1483 1, PME_ACK_TIMEOUT);
1484}
1485
1486static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1487{
1488 u32 data;
1489 int err;
1490
1491 if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1492 dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1493 return;
1494 }
1495
1496 if (tegra_pcie_try_link_l2(pcie)) {
1497 dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1498 /*
1499 * TX lane clock freq will reset to Gen1 only if link is in L2
1500 * or detect state.
1501 * So apply pex_rst to end point to force RP to go into detect
1502 * state
1503 */
1504 data = appl_readl(pcie, APPL_PINMUX);
1505 data &= ~APPL_PINMUX_PEX_RST;
1506 appl_writel(pcie, data, APPL_PINMUX);
1507
1508 /*
1509 * Some cards do not go to detect state even after de-asserting
1510 * PERST#. So, de-assert LTSSM to bring link to detect state.
1511 */
1512 data = readl(pcie->appl_base + APPL_CTRL);
1513 data &= ~APPL_CTRL_LTSSM_EN;
1514 writel(data, pcie->appl_base + APPL_CTRL);
1515
1516 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1517 data,
1518 ((data &
1519 APPL_DEBUG_LTSSM_STATE_MASK) >>
1520 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1521 LTSSM_STATE_PRE_DETECT,
1522 1, LTSSM_TIMEOUT);
1523 if (err)
1524 dev_info(pcie->dev, "Link didn't go to detect state\n");
1525 }
1526 /*
1527 * DBI registers may not be accessible after this as PLL-E would be
1528 * down depending on how CLKREQ is pulled by end point
1529 */
1530 data = appl_readl(pcie, APPL_PINMUX);
1531 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1532 /* Cut REFCLK to slot */
1533 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1534 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1535 appl_writel(pcie, data, APPL_PINMUX);
1536}
1537
1538static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1539{
1540 tegra_pcie_downstream_dev_to_D0(pcie);
1541 dw_pcie_host_deinit(&pcie->pci.pp);
1542 tegra_pcie_dw_pme_turnoff(pcie);
1543 tegra_pcie_unconfig_controller(pcie);
1544}
1545
1546static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1547{
1548 struct device *dev = pcie->dev;
1549 char *name;
1550 int ret;
1551
1552 pm_runtime_enable(dev);
1553
1554 ret = pm_runtime_get_sync(dev);
1555 if (ret < 0) {
1556 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1557 ret);
1558 goto fail_pm_get_sync;
1559 }
1560
1561 ret = pinctrl_pm_select_default_state(dev);
1562 if (ret < 0) {
1563 dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1564 goto fail_pm_get_sync;
1565 }
1566
1567 ret = tegra_pcie_init_controller(pcie);
1568 if (ret < 0) {
1569 dev_err(dev, "Failed to initialize controller: %d\n", ret);
1570 goto fail_pm_get_sync;
1571 }
1572
1573 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1574 if (!pcie->link_state) {
1575 ret = -ENOMEDIUM;
1576 goto fail_host_init;
1577 }
1578
1579 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1580 if (!name) {
1581 ret = -ENOMEM;
1582 goto fail_host_init;
1583 }
1584
1585 pcie->debugfs = debugfs_create_dir(name, NULL);
1586 init_debugfs(pcie);
1587
1588 return ret;
1589
1590fail_host_init:
1591 tegra_pcie_deinit_controller(pcie);
1592fail_pm_get_sync:
1593 pm_runtime_put_sync(dev);
1594 pm_runtime_disable(dev);
1595 return ret;
1596}
1597
1598static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1599{
1600 u32 val;
1601 int ret;
1602
1603 if (pcie->ep_state == EP_STATE_DISABLED)
1604 return;
1605
1606 /* Disable LTSSM */
1607 val = appl_readl(pcie, APPL_CTRL);
1608 val &= ~APPL_CTRL_LTSSM_EN;
1609 appl_writel(pcie, val, APPL_CTRL);
1610
1611 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1612 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1613 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1614 LTSSM_STATE_PRE_DETECT,
1615 1, LTSSM_TIMEOUT);
1616 if (ret)
1617 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1618
1619 reset_control_assert(pcie->core_rst);
1620
1621 tegra_pcie_disable_phy(pcie);
1622
1623 reset_control_assert(pcie->core_apb_rst);
1624
1625 clk_disable_unprepare(pcie->core_clk);
1626
1627 pm_runtime_put_sync(pcie->dev);
1628
1629 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1630 if (ret)
1631 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1632
1633 pcie->ep_state = EP_STATE_DISABLED;
1634 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1635}
1636
1637static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1638{
1639 struct dw_pcie *pci = &pcie->pci;
1640 struct dw_pcie_ep *ep = &pci->ep;
1641 struct device *dev = pcie->dev;
1642 u32 val;
1643 int ret;
1644
1645 if (pcie->ep_state == EP_STATE_ENABLED)
1646 return;
1647
1648 ret = pm_runtime_resume_and_get(dev);
1649 if (ret < 0) {
1650 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1651 ret);
1652 return;
1653 }
1654
1655 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1656 if (ret) {
1657 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret);
1658 goto fail_pll_init;
1659 }
1660
1661 ret = clk_prepare_enable(pcie->core_clk);
1662 if (ret) {
1663 dev_err(dev, "Failed to enable core clock: %d\n", ret);
1664 goto fail_core_clk_enable;
1665 }
1666
1667 ret = reset_control_deassert(pcie->core_apb_rst);
1668 if (ret) {
1669 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1670 goto fail_core_apb_rst;
1671 }
1672
1673 ret = tegra_pcie_enable_phy(pcie);
1674 if (ret) {
1675 dev_err(dev, "Failed to enable PHY: %d\n", ret);
1676 goto fail_phy;
1677 }
1678
1679 /* Clear any stale interrupt statuses */
1680 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1681 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1682 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1683 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1684 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1685 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1686 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1687 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1688 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1689 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1690 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1691 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1692 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1693 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1694 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1695
1696 /* configure this core for EP mode operation */
1697 val = appl_readl(pcie, APPL_DM_TYPE);
1698 val &= ~APPL_DM_TYPE_MASK;
1699 val |= APPL_DM_TYPE_EP;
1700 appl_writel(pcie, val, APPL_DM_TYPE);
1701
1702 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1703
1704 val = appl_readl(pcie, APPL_CTRL);
1705 val |= APPL_CTRL_SYS_PRE_DET_STATE;
1706 val |= APPL_CTRL_HW_HOT_RST_EN;
1707 appl_writel(pcie, val, APPL_CTRL);
1708
1709 val = appl_readl(pcie, APPL_CFG_MISC);
1710 val |= APPL_CFG_MISC_SLV_EP_MODE;
1711 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1712 appl_writel(pcie, val, APPL_CFG_MISC);
1713
1714 val = appl_readl(pcie, APPL_PINMUX);
1715 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1716 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1717 appl_writel(pcie, val, APPL_PINMUX);
1718
1719 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1720 APPL_CFG_BASE_ADDR);
1721
1722 appl_writel(pcie, pcie->atu_dma_res->start &
1723 APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1724 APPL_CFG_IATU_DMA_BASE_ADDR);
1725
1726 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1727 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1728 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1729 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1730 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1731
1732 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1733 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1734 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1735 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1736
1737 reset_control_deassert(pcie->core_rst);
1738
1739 if (pcie->update_fc_fixup) {
1740 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1741 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1742 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1743 }
1744
1745 config_gen3_gen4_eq_presets(pcie);
1746
1747 init_host_aspm(pcie);
1748
1749 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1750 if (!pcie->supports_clkreq) {
1751 disable_aspm_l11(pcie);
1752 disable_aspm_l12(pcie);
1753 }
1754
1755 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1756 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1757 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1758
1759 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1760 PCI_CAP_ID_EXP);
1761 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1762
1763 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1764 val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1765 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1766 val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1767 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1768
1769 ret = dw_pcie_ep_init_complete(ep);
1770 if (ret) {
1771 dev_err(dev, "Failed to complete initialization: %d\n", ret);
1772 goto fail_init_complete;
1773 }
1774
1775 dw_pcie_ep_init_notify(ep);
1776
1777 /* Enable LTSSM */
1778 val = appl_readl(pcie, APPL_CTRL);
1779 val |= APPL_CTRL_LTSSM_EN;
1780 appl_writel(pcie, val, APPL_CTRL);
1781
1782 pcie->ep_state = EP_STATE_ENABLED;
1783 dev_dbg(dev, "Initialization of endpoint is completed\n");
1784
1785 return;
1786
1787fail_init_complete:
1788 reset_control_assert(pcie->core_rst);
1789 tegra_pcie_disable_phy(pcie);
1790fail_phy:
1791 reset_control_assert(pcie->core_apb_rst);
1792fail_core_apb_rst:
1793 clk_disable_unprepare(pcie->core_clk);
1794fail_core_clk_enable:
1795 tegra_pcie_bpmp_set_pll_state(pcie, false);
1796fail_pll_init:
1797 pm_runtime_put_sync(dev);
1798}
1799
1800static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1801{
1802 struct tegra_pcie_dw *pcie = arg;
1803
1804 if (gpiod_get_value(pcie->pex_rst_gpiod))
1805 pex_ep_event_pex_rst_assert(pcie);
1806 else
1807 pex_ep_event_pex_rst_deassert(pcie);
1808
1809 return IRQ_HANDLED;
1810}
1811
1812static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1813{
1814 /* Tegra194 supports only INTA */
1815 if (irq > 1)
1816 return -EINVAL;
1817
1818 appl_writel(pcie, 1, APPL_LEGACY_INTX);
1819 usleep_range(1000, 2000);
1820 appl_writel(pcie, 0, APPL_LEGACY_INTX);
1821 return 0;
1822}
1823
1824static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1825{
1826 if (unlikely(irq > 31))
1827 return -EINVAL;
1828
1829 appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1830
1831 return 0;
1832}
1833
1834static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1835{
1836 struct dw_pcie_ep *ep = &pcie->pci.ep;
1837
1838 writel(irq, ep->msi_mem);
1839
1840 return 0;
1841}
1842
1843static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1844 enum pci_epc_irq_type type,
1845 u16 interrupt_num)
1846{
1847 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1848 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1849
1850 switch (type) {
1851 case PCI_EPC_IRQ_LEGACY:
1852 return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
1853
1854 case PCI_EPC_IRQ_MSI:
1855 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1856
1857 case PCI_EPC_IRQ_MSIX:
1858 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1859
1860 default:
1861 dev_err(pci->dev, "Unknown IRQ type\n");
1862 return -EPERM;
1863 }
1864
1865 return 0;
1866}
1867
1868static const struct pci_epc_features tegra_pcie_epc_features = {
1869 .linkup_notifier = true,
1870 .core_init_notifier = true,
1871 .msi_capable = false,
1872 .msix_capable = false,
1873 .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
1874 .bar_fixed_64bit = 1 << BAR_0,
1875 .bar_fixed_size[0] = SZ_1M,
1876};
1877
1878static const struct pci_epc_features*
1879tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
1880{
1881 return &tegra_pcie_epc_features;
1882}
1883
1884static const struct dw_pcie_ep_ops pcie_ep_ops = {
1885 .raise_irq = tegra_pcie_ep_raise_irq,
1886 .get_features = tegra_pcie_ep_get_features,
1887};
1888
1889static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
1890 struct platform_device *pdev)
1891{
1892 struct dw_pcie *pci = &pcie->pci;
1893 struct device *dev = pcie->dev;
1894 struct dw_pcie_ep *ep;
1895 char *name;
1896 int ret;
1897
1898 ep = &pci->ep;
1899 ep->ops = &pcie_ep_ops;
1900
1901 ep->page_size = SZ_64K;
1902
1903 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
1904 if (ret < 0) {
1905 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
1906 ret);
1907 return ret;
1908 }
1909
1910 ret = gpiod_to_irq(pcie->pex_rst_gpiod);
1911 if (ret < 0) {
1912 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
1913 return ret;
1914 }
1915 pcie->pex_rst_irq = (unsigned int)ret;
1916
1917 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
1918 pcie->cid);
1919 if (!name) {
1920 dev_err(dev, "Failed to create PERST IRQ string\n");
1921 return -ENOMEM;
1922 }
1923
1924 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
1925
1926 pcie->ep_state = EP_STATE_DISABLED;
1927
1928 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
1929 tegra_pcie_ep_pex_rst_irq,
1930 IRQF_TRIGGER_RISING |
1931 IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
1932 name, (void *)pcie);
1933 if (ret < 0) {
1934 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
1935 return ret;
1936 }
1937
1938 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work",
1939 pcie->cid);
1940 if (!name) {
1941 dev_err(dev, "Failed to create PCIe EP work thread string\n");
1942 return -ENOMEM;
1943 }
1944
1945 pm_runtime_enable(dev);
1946
1947 ret = dw_pcie_ep_init(ep);
1948 if (ret) {
1949 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
1950 ret);
1951 return ret;
1952 }
1953
1954 return 0;
1955}
1956
1957static int tegra_pcie_dw_probe(struct platform_device *pdev)
1958{
1959 const struct tegra_pcie_dw_of_data *data;
1960 struct device *dev = &pdev->dev;
1961 struct resource *atu_dma_res;
1962 struct tegra_pcie_dw *pcie;
1963 struct pcie_port *pp;
1964 struct dw_pcie *pci;
1965 struct phy **phys;
1966 char *name;
1967 int ret;
1968 u32 i;
1969
1970 data = of_device_get_match_data(dev);
1971
1972 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
1973 if (!pcie)
1974 return -ENOMEM;
1975
1976 pci = &pcie->pci;
1977 pci->dev = &pdev->dev;
1978 pci->ops = &tegra_dw_pcie_ops;
1979 pci->n_fts[0] = N_FTS_VAL;
1980 pci->n_fts[1] = FTS_VAL;
1981 pci->version = 0x490A;
1982
1983 pp = &pci->pp;
1984 pp->num_vectors = MAX_MSI_IRQS;
1985 pcie->dev = &pdev->dev;
1986 pcie->mode = (enum dw_pcie_device_mode)data->mode;
1987
1988 ret = tegra_pcie_dw_parse_dt(pcie);
1989 if (ret < 0) {
1990 const char *level = KERN_ERR;
1991
1992 if (ret == -EPROBE_DEFER)
1993 level = KERN_DEBUG;
1994
1995 dev_printk(level, dev,
1996 dev_fmt("Failed to parse device tree: %d\n"),
1997 ret);
1998 return ret;
1999 }
2000
2001 ret = tegra_pcie_get_slot_regulators(pcie);
2002 if (ret < 0) {
2003 const char *level = KERN_ERR;
2004
2005 if (ret == -EPROBE_DEFER)
2006 level = KERN_DEBUG;
2007
2008 dev_printk(level, dev,
2009 dev_fmt("Failed to get slot regulators: %d\n"),
2010 ret);
2011 return ret;
2012 }
2013
2014 if (pcie->pex_refclk_sel_gpiod)
2015 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2016
2017 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2018 if (IS_ERR(pcie->pex_ctl_supply)) {
2019 ret = PTR_ERR(pcie->pex_ctl_supply);
2020 if (ret != -EPROBE_DEFER)
2021 dev_err(dev, "Failed to get regulator: %ld\n",
2022 PTR_ERR(pcie->pex_ctl_supply));
2023 return ret;
2024 }
2025
2026 pcie->core_clk = devm_clk_get(dev, "core");
2027 if (IS_ERR(pcie->core_clk)) {
2028 dev_err(dev, "Failed to get core clock: %ld\n",
2029 PTR_ERR(pcie->core_clk));
2030 return PTR_ERR(pcie->core_clk);
2031 }
2032
2033 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2034 "appl");
2035 if (!pcie->appl_res) {
2036 dev_err(dev, "Failed to find \"appl\" region\n");
2037 return -ENODEV;
2038 }
2039
2040 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2041 if (IS_ERR(pcie->appl_base))
2042 return PTR_ERR(pcie->appl_base);
2043
2044 pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2045 if (IS_ERR(pcie->core_apb_rst)) {
2046 dev_err(dev, "Failed to get APB reset: %ld\n",
2047 PTR_ERR(pcie->core_apb_rst));
2048 return PTR_ERR(pcie->core_apb_rst);
2049 }
2050
2051 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2052 if (!phys)
2053 return -ENOMEM;
2054
2055 for (i = 0; i < pcie->phy_count; i++) {
2056 name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2057 if (!name) {
2058 dev_err(dev, "Failed to create P2U string\n");
2059 return -ENOMEM;
2060 }
2061 phys[i] = devm_phy_get(dev, name);
2062 kfree(name);
2063 if (IS_ERR(phys[i])) {
2064 ret = PTR_ERR(phys[i]);
2065 if (ret != -EPROBE_DEFER)
2066 dev_err(dev, "Failed to get PHY: %d\n", ret);
2067 return ret;
2068 }
2069 }
2070
2071 pcie->phys = phys;
2072
2073 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2074 "atu_dma");
2075 if (!atu_dma_res) {
2076 dev_err(dev, "Failed to find \"atu_dma\" region\n");
2077 return -ENODEV;
2078 }
2079 pcie->atu_dma_res = atu_dma_res;
2080
2081 pci->atu_size = resource_size(atu_dma_res);
2082 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2083 if (IS_ERR(pci->atu_base))
2084 return PTR_ERR(pci->atu_base);
2085
2086 pcie->core_rst = devm_reset_control_get(dev, "core");
2087 if (IS_ERR(pcie->core_rst)) {
2088 dev_err(dev, "Failed to get core reset: %ld\n",
2089 PTR_ERR(pcie->core_rst));
2090 return PTR_ERR(pcie->core_rst);
2091 }
2092
2093 pp->irq = platform_get_irq_byname(pdev, "intr");
2094 if (pp->irq < 0)
2095 return pp->irq;
2096
2097 pcie->bpmp = tegra_bpmp_get(dev);
2098 if (IS_ERR(pcie->bpmp))
2099 return PTR_ERR(pcie->bpmp);
2100
2101 platform_set_drvdata(pdev, pcie);
2102
2103 switch (pcie->mode) {
2104 case DW_PCIE_RC_TYPE:
2105 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2106 IRQF_SHARED, "tegra-pcie-intr", pcie);
2107 if (ret) {
2108 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2109 ret);
2110 goto fail;
2111 }
2112
2113 ret = tegra_pcie_config_rp(pcie);
2114 if (ret && ret != -ENOMEDIUM)
2115 goto fail;
2116 else
2117 return 0;
2118 break;
2119
2120 case DW_PCIE_EP_TYPE:
2121 ret = devm_request_threaded_irq(dev, pp->irq,
2122 tegra_pcie_ep_hard_irq,
2123 tegra_pcie_ep_irq_thread,
2124 IRQF_SHARED | IRQF_ONESHOT,
2125 "tegra-pcie-ep-intr", pcie);
2126 if (ret) {
2127 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2128 ret);
2129 goto fail;
2130 }
2131
2132 ret = tegra_pcie_config_ep(pcie, pdev);
2133 if (ret < 0)
2134 goto fail;
2135 break;
2136
2137 default:
2138 dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode);
2139 }
2140
2141fail:
2142 tegra_bpmp_put(pcie->bpmp);
2143 return ret;
2144}
2145
2146static int tegra_pcie_dw_remove(struct platform_device *pdev)
2147{
2148 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2149
2150 if (!pcie->link_state)
2151 return 0;
2152
2153 debugfs_remove_recursive(pcie->debugfs);
2154 tegra_pcie_deinit_controller(pcie);
2155 pm_runtime_put_sync(pcie->dev);
2156 pm_runtime_disable(pcie->dev);
2157 tegra_bpmp_put(pcie->bpmp);
2158 if (pcie->pex_refclk_sel_gpiod)
2159 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2160
2161 return 0;
2162}
2163
2164static int tegra_pcie_dw_suspend_late(struct device *dev)
2165{
2166 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2167 u32 val;
2168
2169 if (!pcie->link_state)
2170 return 0;
2171
2172 /* Enable HW_HOT_RST mode */
2173 val = appl_readl(pcie, APPL_CTRL);
2174 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2175 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2176 val |= APPL_CTRL_HW_HOT_RST_EN;
2177 appl_writel(pcie, val, APPL_CTRL);
2178
2179 return 0;
2180}
2181
2182static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2183{
2184 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2185
2186 if (!pcie->link_state)
2187 return 0;
2188
2189 /* Save MSI interrupt vector */
2190 pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci,
2191 PORT_LOGIC_MSI_CTRL_INT_0_EN);
2192 tegra_pcie_downstream_dev_to_D0(pcie);
2193 tegra_pcie_dw_pme_turnoff(pcie);
2194 tegra_pcie_unconfig_controller(pcie);
2195
2196 return 0;
2197}
2198
2199static int tegra_pcie_dw_resume_noirq(struct device *dev)
2200{
2201 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2202 int ret;
2203
2204 if (!pcie->link_state)
2205 return 0;
2206
2207 ret = tegra_pcie_config_controller(pcie, true);
2208 if (ret < 0)
2209 return ret;
2210
2211 ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2212 if (ret < 0) {
2213 dev_err(dev, "Failed to init host: %d\n", ret);
2214 goto fail_host_init;
2215 }
2216
2217 dw_pcie_setup_rc(&pcie->pci.pp);
2218
2219 ret = tegra_pcie_dw_start_link(&pcie->pci);
2220 if (ret < 0)
2221 goto fail_host_init;
2222
2223 /* Restore MSI interrupt vector */
2224 dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN,
2225 pcie->msi_ctrl_int);
2226
2227 return 0;
2228
2229fail_host_init:
2230 tegra_pcie_unconfig_controller(pcie);
2231 return ret;
2232}
2233
2234static int tegra_pcie_dw_resume_early(struct device *dev)
2235{
2236 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2237 u32 val;
2238
2239 if (!pcie->link_state)
2240 return 0;
2241
2242 /* Disable HW_HOT_RST mode */
2243 val = appl_readl(pcie, APPL_CTRL);
2244 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2245 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2246 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2247 APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2248 val &= ~APPL_CTRL_HW_HOT_RST_EN;
2249 appl_writel(pcie, val, APPL_CTRL);
2250
2251 return 0;
2252}
2253
2254static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2255{
2256 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2257
2258 if (!pcie->link_state)
2259 return;
2260
2261 debugfs_remove_recursive(pcie->debugfs);
2262 tegra_pcie_downstream_dev_to_D0(pcie);
2263
2264 disable_irq(pcie->pci.pp.irq);
2265 if (IS_ENABLED(CONFIG_PCI_MSI))
2266 disable_irq(pcie->pci.pp.msi_irq);
2267
2268 tegra_pcie_dw_pme_turnoff(pcie);
2269 tegra_pcie_unconfig_controller(pcie);
2270}
2271
2272static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = {
2273 .mode = DW_PCIE_RC_TYPE,
2274};
2275
2276static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = {
2277 .mode = DW_PCIE_EP_TYPE,
2278};
2279
2280static const struct of_device_id tegra_pcie_dw_of_match[] = {
2281 {
2282 .compatible = "nvidia,tegra194-pcie",
2283 .data = &tegra_pcie_dw_rc_of_data,
2284 },
2285 {
2286 .compatible = "nvidia,tegra194-pcie-ep",
2287 .data = &tegra_pcie_dw_ep_of_data,
2288 },
2289 {},
2290};
2291
2292static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2293 .suspend_late = tegra_pcie_dw_suspend_late,
2294 .suspend_noirq = tegra_pcie_dw_suspend_noirq,
2295 .resume_noirq = tegra_pcie_dw_resume_noirq,
2296 .resume_early = tegra_pcie_dw_resume_early,
2297};
2298
2299static struct platform_driver tegra_pcie_dw_driver = {
2300 .probe = tegra_pcie_dw_probe,
2301 .remove = tegra_pcie_dw_remove,
2302 .shutdown = tegra_pcie_dw_shutdown,
2303 .driver = {
2304 .name = "tegra194-pcie",
2305 .pm = &tegra_pcie_dw_pm_ops,
2306 .of_match_table = tegra_pcie_dw_of_match,
2307 },
2308};
2309module_platform_driver(tegra_pcie_dw_driver);
2310
2311MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2312
2313MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2314MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2315MODULE_LICENSE("GPL v2");