Loading...
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PCIe host controller driver for the following SoCs
4 * Tegra194
5 * Tegra234
6 *
7 * Copyright (C) 2019-2022 NVIDIA Corporation.
8 *
9 * Author: Vidya Sagar <vidyas@nvidia.com>
10 */
11
12#include <linux/bitfield.h>
13#include <linux/clk.h>
14#include <linux/debugfs.h>
15#include <linux/delay.h>
16#include <linux/gpio.h>
17#include <linux/gpio/consumer.h>
18#include <linux/interconnect.h>
19#include <linux/interrupt.h>
20#include <linux/iopoll.h>
21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/of.h>
24#include <linux/of_gpio.h>
25#include <linux/of_pci.h>
26#include <linux/pci.h>
27#include <linux/phy/phy.h>
28#include <linux/pinctrl/consumer.h>
29#include <linux/platform_device.h>
30#include <linux/pm_runtime.h>
31#include <linux/random.h>
32#include <linux/reset.h>
33#include <linux/resource.h>
34#include <linux/types.h>
35#include "pcie-designware.h"
36#include <soc/tegra/bpmp.h>
37#include <soc/tegra/bpmp-abi.h>
38#include "../../pci.h"
39
40#define TEGRA194_DWC_IP_VER 0x490A
41#define TEGRA234_DWC_IP_VER 0x562A
42
43#define APPL_PINMUX 0x0
44#define APPL_PINMUX_PEX_RST BIT(0)
45#define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)
46#define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)
47#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)
48#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)
49
50#define APPL_CTRL 0x4
51#define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)
52#define APPL_CTRL_LTSSM_EN BIT(7)
53#define APPL_CTRL_HW_HOT_RST_EN BIT(20)
54#define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)
55#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22
56#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1
57#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN 0x2
58
59#define APPL_INTR_EN_L0_0 0x8
60#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)
61#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)
62#define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)
63#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15)
64#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)
65#define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)
66#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)
67
68#define APPL_INTR_STATUS_L0 0xC
69#define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)
70#define APPL_INTR_STATUS_L0_INT_INT BIT(8)
71#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15)
72#define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16)
73#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)
74
75#define APPL_INTR_EN_L1_0_0 0x1C
76#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)
77#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3)
78#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30)
79
80#define APPL_INTR_STATUS_L1_0_0 0x20
81#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)
82#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3)
83#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30)
84
85#define APPL_INTR_STATUS_L1_1 0x2C
86#define APPL_INTR_STATUS_L1_2 0x30
87#define APPL_INTR_STATUS_L1_3 0x34
88#define APPL_INTR_STATUS_L1_6 0x3C
89#define APPL_INTR_STATUS_L1_7 0x40
90#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1)
91
92#define APPL_INTR_EN_L1_8_0 0x44
93#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)
94#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)
95#define APPL_INTR_EN_L1_8_INTX_EN BIT(11)
96#define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)
97
98#define APPL_INTR_STATUS_L1_8_0 0x4C
99#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)
100#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)
101#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)
102
103#define APPL_INTR_STATUS_L1_9 0x54
104#define APPL_INTR_STATUS_L1_10 0x58
105#define APPL_INTR_STATUS_L1_11 0x64
106#define APPL_INTR_STATUS_L1_13 0x74
107#define APPL_INTR_STATUS_L1_14 0x78
108#define APPL_INTR_STATUS_L1_15 0x7C
109#define APPL_INTR_STATUS_L1_17 0x88
110
111#define APPL_INTR_EN_L1_18 0x90
112#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)
113#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
114#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
115
116#define APPL_INTR_STATUS_L1_18 0x94
117#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)
118#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
119#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
120
121#define APPL_MSI_CTRL_1 0xAC
122
123#define APPL_MSI_CTRL_2 0xB0
124
125#define APPL_LEGACY_INTX 0xB8
126
127#define APPL_LTR_MSG_1 0xC4
128#define LTR_MSG_REQ BIT(15)
129#define LTR_NOSNOOP_MSG_REQ BIT(31)
130
131#define APPL_LTR_MSG_2 0xC8
132#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)
133
134#define APPL_LINK_STATUS 0xCC
135#define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)
136
137#define APPL_DEBUG 0xD0
138#define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)
139#define APPL_DEBUG_PM_LINKST_IN_L0 0x11
140#define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)
141#define APPL_DEBUG_LTSSM_STATE_SHIFT 3
142#define LTSSM_STATE_PRE_DETECT 5
143
144#define APPL_RADM_STATUS 0xE4
145#define APPL_PM_XMT_TURNOFF_STATE BIT(0)
146
147#define APPL_DM_TYPE 0x100
148#define APPL_DM_TYPE_MASK GENMASK(3, 0)
149#define APPL_DM_TYPE_RP 0x4
150#define APPL_DM_TYPE_EP 0x0
151
152#define APPL_CFG_BASE_ADDR 0x104
153#define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)
154
155#define APPL_CFG_IATU_DMA_BASE_ADDR 0x108
156#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)
157
158#define APPL_CFG_MISC 0x110
159#define APPL_CFG_MISC_SLV_EP_MODE BIT(14)
160#define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)
161#define APPL_CFG_MISC_ARCACHE_SHIFT 10
162#define APPL_CFG_MISC_ARCACHE_VAL 3
163
164#define APPL_CFG_SLCG_OVERRIDE 0x114
165#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)
166
167#define APPL_CAR_RESET_OVRD 0x12C
168#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)
169
170#define IO_BASE_IO_DECODE BIT(0)
171#define IO_BASE_IO_DECODE_BIT8 BIT(8)
172
173#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)
174#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)
175
176#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718
177#define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)
178
179#define N_FTS_VAL 52
180#define FTS_VAL 52
181
182#define GEN3_EQ_CONTROL_OFF 0x8a8
183#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8
184#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8)
185#define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0)
186
187#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0
188#define AMBA_ERROR_RESPONSE_CRS_SHIFT 3
189#define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0)
190#define AMBA_ERROR_RESPONSE_CRS_OKAY 0
191#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1
192#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2
193
194#define MSIX_ADDR_MATCH_LOW_OFF 0x940
195#define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0)
196#define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2)
197
198#define MSIX_ADDR_MATCH_HIGH_OFF 0x944
199#define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0)
200
201#define PORT_LOGIC_MSIX_DOORBELL 0x948
202
203#define CAP_SPCIE_CAP_OFF 0x154
204#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)
205#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)
206#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8
207
208#define PME_ACK_TIMEOUT 10000
209
210#define LTSSM_TIMEOUT 50000 /* 50ms */
211
212#define GEN3_GEN4_EQ_PRESET_INIT 5
213
214#define GEN1_CORE_CLK_FREQ 62500000
215#define GEN2_CORE_CLK_FREQ 125000000
216#define GEN3_CORE_CLK_FREQ 250000000
217#define GEN4_CORE_CLK_FREQ 500000000
218
219#define LTR_MSG_TIMEOUT (100 * 1000)
220
221#define PERST_DEBOUNCE_TIME (5 * 1000)
222
223#define EP_STATE_DISABLED 0
224#define EP_STATE_ENABLED 1
225
226static const unsigned int pcie_gen_freq[] = {
227 GEN1_CORE_CLK_FREQ, /* PCI_EXP_LNKSTA_CLS == 0; undefined */
228 GEN1_CORE_CLK_FREQ,
229 GEN2_CORE_CLK_FREQ,
230 GEN3_CORE_CLK_FREQ,
231 GEN4_CORE_CLK_FREQ
232};
233
234struct tegra_pcie_dw_of_data {
235 u32 version;
236 enum dw_pcie_device_mode mode;
237 bool has_msix_doorbell_access_fix;
238 bool has_sbr_reset_fix;
239 bool has_l1ss_exit_fix;
240 bool has_ltr_req_fix;
241 u32 cdm_chk_int_en_bit;
242 u32 gen4_preset_vec;
243 u8 n_fts[2];
244};
245
246struct tegra_pcie_dw {
247 struct device *dev;
248 struct resource *appl_res;
249 struct resource *dbi_res;
250 struct resource *atu_dma_res;
251 void __iomem *appl_base;
252 struct clk *core_clk;
253 struct reset_control *core_apb_rst;
254 struct reset_control *core_rst;
255 struct dw_pcie pci;
256 struct tegra_bpmp *bpmp;
257
258 struct tegra_pcie_dw_of_data *of_data;
259
260 bool supports_clkreq;
261 bool enable_cdm_check;
262 bool enable_srns;
263 bool link_state;
264 bool update_fc_fixup;
265 bool enable_ext_refclk;
266 u8 init_link_width;
267 u32 msi_ctrl_int;
268 u32 num_lanes;
269 u32 cid;
270 u32 cfg_link_cap_l1sub;
271 u32 ras_des_cap;
272 u32 pcie_cap_base;
273 u32 aspm_cmrt;
274 u32 aspm_pwr_on_t;
275 u32 aspm_l0s_enter_lat;
276
277 struct regulator *pex_ctl_supply;
278 struct regulator *slot_ctl_3v3;
279 struct regulator *slot_ctl_12v;
280
281 unsigned int phy_count;
282 struct phy **phys;
283
284 struct dentry *debugfs;
285
286 /* Endpoint mode specific */
287 struct gpio_desc *pex_rst_gpiod;
288 struct gpio_desc *pex_refclk_sel_gpiod;
289 unsigned int pex_rst_irq;
290 int ep_state;
291 long link_status;
292 struct icc_path *icc_path;
293};
294
295static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
296{
297 return container_of(pci, struct tegra_pcie_dw, pci);
298}
299
300static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
301 const u32 reg)
302{
303 writel_relaxed(value, pcie->appl_base + reg);
304}
305
306static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
307{
308 return readl_relaxed(pcie->appl_base + reg);
309}
310
311struct tegra_pcie_soc {
312 enum dw_pcie_device_mode mode;
313};
314
315static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie)
316{
317 struct dw_pcie *pci = &pcie->pci;
318 u32 val, speed, width;
319
320 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
321
322 speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val);
323 width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
324
325 val = width * PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]);
326
327 if (icc_set_bw(pcie->icc_path, Mbps_to_icc(val), 0))
328 dev_err(pcie->dev, "can't set bw[%u]\n", val);
329
330 if (speed >= ARRAY_SIZE(pcie_gen_freq))
331 speed = 0;
332
333 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed]);
334}
335
336static void apply_bad_link_workaround(struct dw_pcie_rp *pp)
337{
338 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
339 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
340 u32 current_link_width;
341 u16 val;
342
343 /*
344 * NOTE:- Since this scenario is uncommon and link as such is not
345 * stable anyway, not waiting to confirm if link is really
346 * transitioning to Gen-2 speed
347 */
348 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
349 if (val & PCI_EXP_LNKSTA_LBMS) {
350 current_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
351 if (pcie->init_link_width > current_link_width) {
352 dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
353 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
354 PCI_EXP_LNKCTL2);
355 val &= ~PCI_EXP_LNKCTL2_TLS;
356 val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
357 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
358 PCI_EXP_LNKCTL2, val);
359
360 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
361 PCI_EXP_LNKCTL);
362 val |= PCI_EXP_LNKCTL_RL;
363 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
364 PCI_EXP_LNKCTL, val);
365 }
366 }
367}
368
369static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
370{
371 struct tegra_pcie_dw *pcie = arg;
372 struct dw_pcie *pci = &pcie->pci;
373 struct dw_pcie_rp *pp = &pci->pp;
374 u32 val, status_l0, status_l1;
375 u16 val_w;
376
377 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
378 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
379 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
380 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
381 if (!pcie->of_data->has_sbr_reset_fix &&
382 status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
383 /* SBR & Surprise Link Down WAR */
384 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
385 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
386 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
387 udelay(1);
388 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
389 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
390 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
391
392 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
393 val |= PORT_LOGIC_SPEED_CHANGE;
394 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
395 }
396 }
397
398 if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) {
399 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
400 if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
401 appl_writel(pcie,
402 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
403 APPL_INTR_STATUS_L1_8_0);
404 apply_bad_link_workaround(pp);
405 }
406 if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
407 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
408 PCI_EXP_LNKSTA);
409 val_w |= PCI_EXP_LNKSTA_LBMS;
410 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
411 PCI_EXP_LNKSTA, val_w);
412
413 appl_writel(pcie,
414 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
415 APPL_INTR_STATUS_L1_8_0);
416
417 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
418 PCI_EXP_LNKSTA);
419 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
420 PCI_EXP_LNKSTA_CLS);
421 }
422 }
423
424 if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
425 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
426 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
427 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
428 dev_info(pci->dev, "CDM check complete\n");
429 val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
430 }
431 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
432 dev_err(pci->dev, "CDM comparison mismatch\n");
433 val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
434 }
435 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
436 dev_err(pci->dev, "CDM Logic error\n");
437 val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
438 }
439 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val);
440 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
441 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val);
442 }
443
444 return IRQ_HANDLED;
445}
446
447static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
448{
449 u32 val;
450
451 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
452 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
453 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
454 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
455 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
456 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
457 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
458 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
459 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
460 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
461 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
462 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
463 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
464 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
465 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
466 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
467
468 val = appl_readl(pcie, APPL_CTRL);
469 val |= APPL_CTRL_LTSSM_EN;
470 appl_writel(pcie, val, APPL_CTRL);
471}
472
473static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
474{
475 struct tegra_pcie_dw *pcie = arg;
476 struct dw_pcie_ep *ep = &pcie->pci.ep;
477 struct dw_pcie *pci = &pcie->pci;
478 u32 val;
479
480 if (test_and_clear_bit(0, &pcie->link_status))
481 dw_pcie_ep_linkup(ep);
482
483 tegra_pcie_icc_set(pcie);
484
485 if (pcie->of_data->has_ltr_req_fix)
486 return IRQ_HANDLED;
487
488 /* If EP doesn't advertise L1SS, just return */
489 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
490 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
491 return IRQ_HANDLED;
492
493 /* Check if BME is set to '1' */
494 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
495 if (val & PCI_COMMAND_MASTER) {
496 ktime_t timeout;
497
498 /* 110us for both snoop and no-snoop */
499 val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) |
500 FIELD_PREP(PCI_LTR_SCALE_MASK, 2) |
501 LTR_MSG_REQ |
502 FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) |
503 FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) |
504 LTR_NOSNOOP_MSG_REQ;
505 appl_writel(pcie, val, APPL_LTR_MSG_1);
506
507 /* Send LTR upstream */
508 val = appl_readl(pcie, APPL_LTR_MSG_2);
509 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
510 appl_writel(pcie, val, APPL_LTR_MSG_2);
511
512 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
513 for (;;) {
514 val = appl_readl(pcie, APPL_LTR_MSG_2);
515 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
516 break;
517 if (ktime_after(ktime_get(), timeout))
518 break;
519 usleep_range(1000, 1100);
520 }
521 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
522 dev_err(pcie->dev, "Failed to send LTR message\n");
523 }
524
525 return IRQ_HANDLED;
526}
527
528static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
529{
530 struct tegra_pcie_dw *pcie = arg;
531 int spurious = 1;
532 u32 status_l0, status_l1, link_status;
533
534 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
535 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
536 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
537 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
538
539 if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
540 pex_ep_event_hot_rst_done(pcie);
541
542 if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
543 link_status = appl_readl(pcie, APPL_LINK_STATUS);
544 if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {
545 dev_dbg(pcie->dev, "Link is up with Host\n");
546 set_bit(0, &pcie->link_status);
547 return IRQ_WAKE_THREAD;
548 }
549 }
550
551 spurious = 0;
552 }
553
554 if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
555 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
556 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15);
557
558 if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
559 return IRQ_WAKE_THREAD;
560
561 spurious = 0;
562 }
563
564 if (spurious) {
565 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
566 status_l0);
567 appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0);
568 }
569
570 return IRQ_HANDLED;
571}
572
573static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
574 int size, u32 *val)
575{
576 struct dw_pcie_rp *pp = bus->sysdata;
577 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
578 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
579
580 /*
581 * This is an endpoint mode specific register happen to appear even
582 * when controller is operating in root port mode and system hangs
583 * when it is accessed with link being in ASPM-L1 state.
584 * So skip accessing it altogether
585 */
586 if (!pcie->of_data->has_msix_doorbell_access_fix &&
587 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
588 *val = 0x00000000;
589 return PCIBIOS_SUCCESSFUL;
590 }
591
592 return pci_generic_config_read(bus, devfn, where, size, val);
593}
594
595static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
596 int size, u32 val)
597{
598 struct dw_pcie_rp *pp = bus->sysdata;
599 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
600 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
601
602 /*
603 * This is an endpoint mode specific register happen to appear even
604 * when controller is operating in root port mode and system hangs
605 * when it is accessed with link being in ASPM-L1 state.
606 * So skip accessing it altogether
607 */
608 if (!pcie->of_data->has_msix_doorbell_access_fix &&
609 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
610 return PCIBIOS_SUCCESSFUL;
611
612 return pci_generic_config_write(bus, devfn, where, size, val);
613}
614
615static struct pci_ops tegra_pci_ops = {
616 .map_bus = dw_pcie_own_conf_map_bus,
617 .read = tegra_pcie_dw_rd_own_conf,
618 .write = tegra_pcie_dw_wr_own_conf,
619};
620
621#if defined(CONFIG_PCIEASPM)
622static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
623{
624 u32 val;
625
626 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
627 val &= ~PCI_L1SS_CAP_ASPM_L1_1;
628 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
629}
630
631static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
632{
633 u32 val;
634
635 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
636 val &= ~PCI_L1SS_CAP_ASPM_L1_2;
637 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
638}
639
640static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
641{
642 u32 val;
643
644 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
645 PCIE_RAS_DES_EVENT_COUNTER_CONTROL);
646 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
647 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
648 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
649 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
650 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
651 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
652 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
653 PCIE_RAS_DES_EVENT_COUNTER_DATA);
654
655 return val;
656}
657
658static int aspm_state_cnt(struct seq_file *s, void *data)
659{
660 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
661 dev_get_drvdata(s->private);
662 u32 val;
663
664 seq_printf(s, "Tx L0s entry count : %u\n",
665 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
666
667 seq_printf(s, "Rx L0s entry count : %u\n",
668 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
669
670 seq_printf(s, "Link L1 entry count : %u\n",
671 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
672
673 seq_printf(s, "Link L1.1 entry count : %u\n",
674 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
675
676 seq_printf(s, "Link L1.2 entry count : %u\n",
677 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
678
679 /* Clear all counters */
680 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
681 PCIE_RAS_DES_EVENT_COUNTER_CONTROL,
682 EVENT_COUNTER_ALL_CLEAR);
683
684 /* Re-enable counting */
685 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
686 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
687 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
688 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
689
690 return 0;
691}
692
693static void init_host_aspm(struct tegra_pcie_dw *pcie)
694{
695 struct dw_pcie *pci = &pcie->pci;
696 u32 val;
697
698 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
699 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
700
701 pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci,
702 PCI_EXT_CAP_ID_VNDR);
703
704 /* Enable ASPM counters */
705 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
706 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
707 dw_pcie_writel_dbi(pci, pcie->ras_des_cap +
708 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
709
710 /* Program T_cmrt and T_pwr_on values */
711 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
712 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
713 val |= (pcie->aspm_cmrt << 8);
714 val |= (pcie->aspm_pwr_on_t << 19);
715 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
716
717 /* Program L0s and L1 entrance latencies */
718 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
719 val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
720 val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
721 val |= PORT_AFR_ENTER_ASPM;
722 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
723}
724
725static void init_debugfs(struct tegra_pcie_dw *pcie)
726{
727 debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,
728 aspm_state_cnt);
729}
730#else
731static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
732static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
733static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
734static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
735#endif
736
737static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp)
738{
739 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
740 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
741 u32 val;
742 u16 val_w;
743
744 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
745 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
746 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
747
748 if (!pcie->of_data->has_sbr_reset_fix) {
749 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
750 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
751 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
752 }
753
754 if (pcie->enable_cdm_check) {
755 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
756 val |= pcie->of_data->cdm_chk_int_en_bit;
757 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
758
759 val = appl_readl(pcie, APPL_INTR_EN_L1_18);
760 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
761 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
762 appl_writel(pcie, val, APPL_INTR_EN_L1_18);
763 }
764
765 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
766 PCI_EXP_LNKSTA);
767 pcie->init_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val_w);
768
769 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
770 PCI_EXP_LNKCTL);
771 val_w |= PCI_EXP_LNKCTL_LBMIE;
772 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
773 val_w);
774}
775
776static void tegra_pcie_enable_intx_interrupts(struct dw_pcie_rp *pp)
777{
778 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
779 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
780 u32 val;
781
782 /* Enable INTX interrupt generation */
783 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
784 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
785 val |= APPL_INTR_EN_L0_0_INT_INT_EN;
786 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
787
788 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
789 val |= APPL_INTR_EN_L1_8_INTX_EN;
790 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
791 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
792 if (IS_ENABLED(CONFIG_PCIEAER))
793 val |= APPL_INTR_EN_L1_8_AER_INT_EN;
794 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
795}
796
797static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp)
798{
799 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
800 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
801 u32 val;
802
803 /* Enable MSI interrupt generation */
804 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
805 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
806 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
807 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
808}
809
810static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp)
811{
812 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
813 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
814
815 /* Clear interrupt statuses before enabling interrupts */
816 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
817 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
818 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
819 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
820 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
821 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
822 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
823 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
824 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
825 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
826 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
827 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
828 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
829 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
830 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
831
832 tegra_pcie_enable_system_interrupts(pp);
833 tegra_pcie_enable_intx_interrupts(pp);
834 if (IS_ENABLED(CONFIG_PCI_MSI))
835 tegra_pcie_enable_msi_interrupts(pp);
836}
837
838static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
839{
840 struct dw_pcie *pci = &pcie->pci;
841 u32 val, offset, i;
842
843 /* Program init preset */
844 for (i = 0; i < pcie->num_lanes; i++) {
845 val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
846 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
847 val |= GEN3_GEN4_EQ_PRESET_INIT;
848 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
849 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
850 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
851 dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
852
853 offset = dw_pcie_find_ext_capability(pci,
854 PCI_EXT_CAP_ID_PL_16GT) +
855 PCI_PL_16GT_LE_CTRL;
856 val = dw_pcie_readb_dbi(pci, offset + i);
857 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
858 val |= GEN3_GEN4_EQ_PRESET_INIT;
859 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
860 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
861 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
862 dw_pcie_writeb_dbi(pci, offset + i, val);
863 }
864
865 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
866 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
867 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
868
869 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
870 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
871 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
872 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
873 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
874
875 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
876 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
877 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
878 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
879
880 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
881 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
882 val |= (pcie->of_data->gen4_preset_vec <<
883 GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
884 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
885 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
886
887 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
888 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
889 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
890}
891
892static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp)
893{
894 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
895 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
896 u32 val;
897 u16 val_16;
898
899 pp->bridge->ops = &tegra_pci_ops;
900
901 if (!pcie->pcie_cap_base)
902 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
903 PCI_CAP_ID_EXP);
904
905 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
906 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
907 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
908
909 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
910 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
911 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
912 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
913
914 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
915
916 /* Enable as 0xFFFF0001 response for CRS */
917 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
918 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
919 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
920 AMBA_ERROR_RESPONSE_CRS_SHIFT);
921 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
922
923 /* Clear Slot Clock Configuration bit if SRNS configuration */
924 if (pcie->enable_srns) {
925 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
926 PCI_EXP_LNKSTA);
927 val_16 &= ~PCI_EXP_LNKSTA_SLC;
928 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
929 val_16);
930 }
931
932 config_gen3_gen4_eq_presets(pcie);
933
934 init_host_aspm(pcie);
935
936 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
937 if (!pcie->supports_clkreq) {
938 disable_aspm_l11(pcie);
939 disable_aspm_l12(pcie);
940 }
941
942 if (!pcie->of_data->has_l1ss_exit_fix) {
943 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
944 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
945 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
946 }
947
948 if (pcie->update_fc_fixup) {
949 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
950 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
951 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
952 }
953
954 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
955
956 return 0;
957}
958
959static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
960{
961 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
962 struct dw_pcie_rp *pp = &pci->pp;
963 u32 val, offset, tmp;
964 bool retry = true;
965
966 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
967 enable_irq(pcie->pex_rst_irq);
968 return 0;
969 }
970
971retry_link:
972 /* Assert RST */
973 val = appl_readl(pcie, APPL_PINMUX);
974 val &= ~APPL_PINMUX_PEX_RST;
975 appl_writel(pcie, val, APPL_PINMUX);
976
977 usleep_range(100, 200);
978
979 /* Enable LTSSM */
980 val = appl_readl(pcie, APPL_CTRL);
981 val |= APPL_CTRL_LTSSM_EN;
982 appl_writel(pcie, val, APPL_CTRL);
983
984 /* De-assert RST */
985 val = appl_readl(pcie, APPL_PINMUX);
986 val |= APPL_PINMUX_PEX_RST;
987 appl_writel(pcie, val, APPL_PINMUX);
988
989 msleep(100);
990
991 if (dw_pcie_wait_for_link(pci)) {
992 if (!retry)
993 return 0;
994 /*
995 * There are some endpoints which can't get the link up if
996 * root port has Data Link Feature (DLF) enabled.
997 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
998 * on Scaled Flow Control and DLF.
999 * So, need to confirm that is indeed the case here and attempt
1000 * link up once again with DLF disabled.
1001 */
1002 val = appl_readl(pcie, APPL_DEBUG);
1003 val &= APPL_DEBUG_LTSSM_STATE_MASK;
1004 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
1005 tmp = appl_readl(pcie, APPL_LINK_STATUS);
1006 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
1007 if (!(val == 0x11 && !tmp)) {
1008 /* Link is down for all good reasons */
1009 return 0;
1010 }
1011
1012 dev_info(pci->dev, "Link is down in DLL");
1013 dev_info(pci->dev, "Trying again with DLFE disabled\n");
1014 /* Disable LTSSM */
1015 val = appl_readl(pcie, APPL_CTRL);
1016 val &= ~APPL_CTRL_LTSSM_EN;
1017 appl_writel(pcie, val, APPL_CTRL);
1018
1019 reset_control_assert(pcie->core_rst);
1020 reset_control_deassert(pcie->core_rst);
1021
1022 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1023 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1024 val &= ~PCI_DLF_EXCHANGE_ENABLE;
1025 dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val);
1026
1027 tegra_pcie_dw_host_init(pp);
1028 dw_pcie_setup_rc(pp);
1029
1030 retry = false;
1031 goto retry_link;
1032 }
1033
1034 tegra_pcie_icc_set(pcie);
1035
1036 tegra_pcie_enable_interrupts(pp);
1037
1038 return 0;
1039}
1040
1041static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1042{
1043 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1044 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1045
1046 return !!(val & PCI_EXP_LNKSTA_DLLLA);
1047}
1048
1049static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1050{
1051 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1052
1053 disable_irq(pcie->pex_rst_irq);
1054}
1055
1056static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1057 .link_up = tegra_pcie_dw_link_up,
1058 .start_link = tegra_pcie_dw_start_link,
1059 .stop_link = tegra_pcie_dw_stop_link,
1060};
1061
1062static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1063 .init = tegra_pcie_dw_host_init,
1064};
1065
1066static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1067{
1068 unsigned int phy_count = pcie->phy_count;
1069
1070 while (phy_count--) {
1071 phy_power_off(pcie->phys[phy_count]);
1072 phy_exit(pcie->phys[phy_count]);
1073 }
1074}
1075
1076static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1077{
1078 unsigned int i;
1079 int ret;
1080
1081 for (i = 0; i < pcie->phy_count; i++) {
1082 ret = phy_init(pcie->phys[i]);
1083 if (ret < 0)
1084 goto phy_power_off;
1085
1086 ret = phy_power_on(pcie->phys[i]);
1087 if (ret < 0)
1088 goto phy_exit;
1089 }
1090
1091 return 0;
1092
1093phy_power_off:
1094 while (i--) {
1095 phy_power_off(pcie->phys[i]);
1096phy_exit:
1097 phy_exit(pcie->phys[i]);
1098 }
1099
1100 return ret;
1101}
1102
1103static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1104{
1105 struct platform_device *pdev = to_platform_device(pcie->dev);
1106 struct device_node *np = pcie->dev->of_node;
1107 int ret;
1108
1109 pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1110 if (!pcie->dbi_res) {
1111 dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1112 return -ENODEV;
1113 }
1114
1115 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1116 if (ret < 0) {
1117 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1118 return ret;
1119 }
1120
1121 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1122 &pcie->aspm_pwr_on_t);
1123 if (ret < 0)
1124 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1125 ret);
1126
1127 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1128 &pcie->aspm_l0s_enter_lat);
1129 if (ret < 0)
1130 dev_info(pcie->dev,
1131 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1132
1133 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1134 if (ret < 0) {
1135 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1136 return ret;
1137 }
1138
1139 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1140 if (ret) {
1141 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1142 return ret;
1143 }
1144
1145 ret = of_property_count_strings(np, "phy-names");
1146 if (ret < 0) {
1147 dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1148 ret);
1149 return ret;
1150 }
1151 pcie->phy_count = ret;
1152
1153 if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1154 pcie->update_fc_fixup = true;
1155
1156 /* RP using an external REFCLK is supported only in Tegra234 */
1157 if (pcie->of_data->version == TEGRA194_DWC_IP_VER) {
1158 if (pcie->of_data->mode == DW_PCIE_EP_TYPE)
1159 pcie->enable_ext_refclk = true;
1160 } else {
1161 pcie->enable_ext_refclk =
1162 of_property_read_bool(pcie->dev->of_node,
1163 "nvidia,enable-ext-refclk");
1164 }
1165
1166 pcie->supports_clkreq =
1167 of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1168
1169 pcie->enable_cdm_check =
1170 of_property_read_bool(np, "snps,enable-cdm-check");
1171
1172 if (pcie->of_data->version == TEGRA234_DWC_IP_VER)
1173 pcie->enable_srns =
1174 of_property_read_bool(np, "nvidia,enable-srns");
1175
1176 if (pcie->of_data->mode == DW_PCIE_RC_TYPE)
1177 return 0;
1178
1179 /* Endpoint mode specific DT entries */
1180 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1181 if (IS_ERR(pcie->pex_rst_gpiod)) {
1182 int err = PTR_ERR(pcie->pex_rst_gpiod);
1183 const char *level = KERN_ERR;
1184
1185 if (err == -EPROBE_DEFER)
1186 level = KERN_DEBUG;
1187
1188 dev_printk(level, pcie->dev,
1189 dev_fmt("Failed to get PERST GPIO: %d\n"),
1190 err);
1191 return err;
1192 }
1193
1194 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1195 "nvidia,refclk-select",
1196 GPIOD_OUT_HIGH);
1197 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1198 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1199 const char *level = KERN_ERR;
1200
1201 if (err == -EPROBE_DEFER)
1202 level = KERN_DEBUG;
1203
1204 dev_printk(level, pcie->dev,
1205 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1206 err);
1207 pcie->pex_refclk_sel_gpiod = NULL;
1208 }
1209
1210 return 0;
1211}
1212
1213static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1214 bool enable)
1215{
1216 struct mrq_uphy_response resp;
1217 struct tegra_bpmp_message msg;
1218 struct mrq_uphy_request req;
1219
1220 /*
1221 * Controller-5 doesn't need to have its state set by BPMP-FW in
1222 * Tegra194
1223 */
1224 if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5)
1225 return 0;
1226
1227 memset(&req, 0, sizeof(req));
1228 memset(&resp, 0, sizeof(resp));
1229
1230 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1231 req.controller_state.pcie_controller = pcie->cid;
1232 req.controller_state.enable = enable;
1233
1234 memset(&msg, 0, sizeof(msg));
1235 msg.mrq = MRQ_UPHY;
1236 msg.tx.data = &req;
1237 msg.tx.size = sizeof(req);
1238 msg.rx.data = &resp;
1239 msg.rx.size = sizeof(resp);
1240
1241 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1242}
1243
1244static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1245 bool enable)
1246{
1247 struct mrq_uphy_response resp;
1248 struct tegra_bpmp_message msg;
1249 struct mrq_uphy_request req;
1250
1251 memset(&req, 0, sizeof(req));
1252 memset(&resp, 0, sizeof(resp));
1253
1254 if (enable) {
1255 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1256 req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1257 } else {
1258 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1259 req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1260 }
1261
1262 memset(&msg, 0, sizeof(msg));
1263 msg.mrq = MRQ_UPHY;
1264 msg.tx.data = &req;
1265 msg.tx.size = sizeof(req);
1266 msg.rx.data = &resp;
1267 msg.rx.size = sizeof(resp);
1268
1269 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1270}
1271
1272static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1273{
1274 struct dw_pcie_rp *pp = &pcie->pci.pp;
1275 struct pci_bus *child, *root_bus = NULL;
1276 struct pci_dev *pdev;
1277
1278 /*
1279 * link doesn't go into L2 state with some of the endpoints with Tegra
1280 * if they are not in D0 state. So, need to make sure that immediate
1281 * downstream devices are in D0 state before sending PME_TurnOff to put
1282 * link into L2 state.
1283 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1284 * 5.2 Link State Power Management (Page #428).
1285 */
1286
1287 list_for_each_entry(child, &pp->bridge->bus->children, node) {
1288 /* Bring downstream devices to D0 if they are not already in */
1289 if (child->parent == pp->bridge->bus) {
1290 root_bus = child;
1291 break;
1292 }
1293 }
1294
1295 if (!root_bus) {
1296 dev_err(pcie->dev, "Failed to find downstream devices\n");
1297 return;
1298 }
1299
1300 list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1301 if (PCI_SLOT(pdev->devfn) == 0) {
1302 if (pci_set_power_state(pdev, PCI_D0))
1303 dev_err(pcie->dev,
1304 "Failed to transition %s to D0 state\n",
1305 dev_name(&pdev->dev));
1306 }
1307 }
1308}
1309
1310static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1311{
1312 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1313 if (IS_ERR(pcie->slot_ctl_3v3)) {
1314 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1315 return PTR_ERR(pcie->slot_ctl_3v3);
1316
1317 pcie->slot_ctl_3v3 = NULL;
1318 }
1319
1320 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1321 if (IS_ERR(pcie->slot_ctl_12v)) {
1322 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1323 return PTR_ERR(pcie->slot_ctl_12v);
1324
1325 pcie->slot_ctl_12v = NULL;
1326 }
1327
1328 return 0;
1329}
1330
1331static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1332{
1333 int ret;
1334
1335 if (pcie->slot_ctl_3v3) {
1336 ret = regulator_enable(pcie->slot_ctl_3v3);
1337 if (ret < 0) {
1338 dev_err(pcie->dev,
1339 "Failed to enable 3.3V slot supply: %d\n", ret);
1340 return ret;
1341 }
1342 }
1343
1344 if (pcie->slot_ctl_12v) {
1345 ret = regulator_enable(pcie->slot_ctl_12v);
1346 if (ret < 0) {
1347 dev_err(pcie->dev,
1348 "Failed to enable 12V slot supply: %d\n", ret);
1349 goto fail_12v_enable;
1350 }
1351 }
1352
1353 /*
1354 * According to PCI Express Card Electromechanical Specification
1355 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1356 * should be a minimum of 100ms.
1357 */
1358 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1359 msleep(100);
1360
1361 return 0;
1362
1363fail_12v_enable:
1364 if (pcie->slot_ctl_3v3)
1365 regulator_disable(pcie->slot_ctl_3v3);
1366 return ret;
1367}
1368
1369static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1370{
1371 if (pcie->slot_ctl_12v)
1372 regulator_disable(pcie->slot_ctl_12v);
1373 if (pcie->slot_ctl_3v3)
1374 regulator_disable(pcie->slot_ctl_3v3);
1375}
1376
1377static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1378 bool en_hw_hot_rst)
1379{
1380 int ret;
1381 u32 val;
1382
1383 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1384 if (ret) {
1385 dev_err(pcie->dev,
1386 "Failed to enable controller %u: %d\n", pcie->cid, ret);
1387 return ret;
1388 }
1389
1390 if (pcie->enable_ext_refclk) {
1391 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1392 if (ret) {
1393 dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret);
1394 goto fail_pll_init;
1395 }
1396 }
1397
1398 ret = tegra_pcie_enable_slot_regulators(pcie);
1399 if (ret < 0)
1400 goto fail_slot_reg_en;
1401
1402 ret = regulator_enable(pcie->pex_ctl_supply);
1403 if (ret < 0) {
1404 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1405 goto fail_reg_en;
1406 }
1407
1408 ret = clk_prepare_enable(pcie->core_clk);
1409 if (ret) {
1410 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1411 goto fail_core_clk;
1412 }
1413
1414 ret = reset_control_deassert(pcie->core_apb_rst);
1415 if (ret) {
1416 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1417 ret);
1418 goto fail_core_apb_rst;
1419 }
1420
1421 if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) {
1422 /* Enable HW_HOT_RST mode */
1423 val = appl_readl(pcie, APPL_CTRL);
1424 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1425 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1426 val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN <<
1427 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1428 val |= APPL_CTRL_HW_HOT_RST_EN;
1429 appl_writel(pcie, val, APPL_CTRL);
1430 }
1431
1432 ret = tegra_pcie_enable_phy(pcie);
1433 if (ret) {
1434 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1435 goto fail_phy;
1436 }
1437
1438 /* Update CFG base address */
1439 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1440 APPL_CFG_BASE_ADDR);
1441
1442 /* Configure this core for RP mode operation */
1443 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1444
1445 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1446
1447 val = appl_readl(pcie, APPL_CTRL);
1448 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1449
1450 val = appl_readl(pcie, APPL_CFG_MISC);
1451 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1452 appl_writel(pcie, val, APPL_CFG_MISC);
1453
1454 if (pcie->enable_srns || pcie->enable_ext_refclk) {
1455 /*
1456 * When Tegra PCIe RP is using external clock, it cannot supply
1457 * same clock to its downstream hierarchy. Hence, gate PCIe RP
1458 * REFCLK out pads when RP & EP are using separate clocks or RP
1459 * is using an external REFCLK.
1460 */
1461 val = appl_readl(pcie, APPL_PINMUX);
1462 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1463 val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1464 appl_writel(pcie, val, APPL_PINMUX);
1465 }
1466
1467 if (!pcie->supports_clkreq) {
1468 val = appl_readl(pcie, APPL_PINMUX);
1469 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1470 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1471 appl_writel(pcie, val, APPL_PINMUX);
1472 }
1473
1474 /* Update iATU_DMA base address */
1475 appl_writel(pcie,
1476 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1477 APPL_CFG_IATU_DMA_BASE_ADDR);
1478
1479 reset_control_deassert(pcie->core_rst);
1480
1481 return ret;
1482
1483fail_phy:
1484 reset_control_assert(pcie->core_apb_rst);
1485fail_core_apb_rst:
1486 clk_disable_unprepare(pcie->core_clk);
1487fail_core_clk:
1488 regulator_disable(pcie->pex_ctl_supply);
1489fail_reg_en:
1490 tegra_pcie_disable_slot_regulators(pcie);
1491fail_slot_reg_en:
1492 if (pcie->enable_ext_refclk)
1493 tegra_pcie_bpmp_set_pll_state(pcie, false);
1494fail_pll_init:
1495 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1496
1497 return ret;
1498}
1499
1500static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1501{
1502 int ret;
1503
1504 ret = reset_control_assert(pcie->core_rst);
1505 if (ret)
1506 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1507
1508 tegra_pcie_disable_phy(pcie);
1509
1510 ret = reset_control_assert(pcie->core_apb_rst);
1511 if (ret)
1512 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1513
1514 clk_disable_unprepare(pcie->core_clk);
1515
1516 ret = regulator_disable(pcie->pex_ctl_supply);
1517 if (ret)
1518 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1519
1520 tegra_pcie_disable_slot_regulators(pcie);
1521
1522 if (pcie->enable_ext_refclk) {
1523 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1524 if (ret)
1525 dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret);
1526 }
1527
1528 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1529 if (ret)
1530 dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1531 pcie->cid, ret);
1532}
1533
1534static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1535{
1536 struct dw_pcie *pci = &pcie->pci;
1537 struct dw_pcie_rp *pp = &pci->pp;
1538 int ret;
1539
1540 ret = tegra_pcie_config_controller(pcie, false);
1541 if (ret < 0)
1542 return ret;
1543
1544 pp->ops = &tegra_pcie_dw_host_ops;
1545
1546 ret = dw_pcie_host_init(pp);
1547 if (ret < 0) {
1548 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1549 goto fail_host_init;
1550 }
1551
1552 return 0;
1553
1554fail_host_init:
1555 tegra_pcie_unconfig_controller(pcie);
1556 return ret;
1557}
1558
1559static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1560{
1561 u32 val;
1562
1563 if (!tegra_pcie_dw_link_up(&pcie->pci))
1564 return 0;
1565
1566 val = appl_readl(pcie, APPL_RADM_STATUS);
1567 val |= APPL_PM_XMT_TURNOFF_STATE;
1568 appl_writel(pcie, val, APPL_RADM_STATUS);
1569
1570 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1571 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1572 1, PME_ACK_TIMEOUT);
1573}
1574
1575static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1576{
1577 u32 data;
1578 int err;
1579
1580 if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1581 dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1582 return;
1583 }
1584
1585 /*
1586 * PCIe controller exits from L2 only if reset is applied, so
1587 * controller doesn't handle interrupts. But in cases where
1588 * L2 entry fails, PERST# is asserted which can trigger surprise
1589 * link down AER. However this function call happens in
1590 * suspend_noirq(), so AER interrupt will not be processed.
1591 * Disable all interrupts to avoid such a scenario.
1592 */
1593 appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0);
1594
1595 if (tegra_pcie_try_link_l2(pcie)) {
1596 dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1597 /*
1598 * TX lane clock freq will reset to Gen1 only if link is in L2
1599 * or detect state.
1600 * So apply pex_rst to end point to force RP to go into detect
1601 * state
1602 */
1603 data = appl_readl(pcie, APPL_PINMUX);
1604 data &= ~APPL_PINMUX_PEX_RST;
1605 appl_writel(pcie, data, APPL_PINMUX);
1606
1607 /*
1608 * Some cards do not go to detect state even after de-asserting
1609 * PERST#. So, de-assert LTSSM to bring link to detect state.
1610 */
1611 data = readl(pcie->appl_base + APPL_CTRL);
1612 data &= ~APPL_CTRL_LTSSM_EN;
1613 writel(data, pcie->appl_base + APPL_CTRL);
1614
1615 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1616 data,
1617 ((data &
1618 APPL_DEBUG_LTSSM_STATE_MASK) >>
1619 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1620 LTSSM_STATE_PRE_DETECT,
1621 1, LTSSM_TIMEOUT);
1622 if (err)
1623 dev_info(pcie->dev, "Link didn't go to detect state\n");
1624 }
1625 /*
1626 * DBI registers may not be accessible after this as PLL-E would be
1627 * down depending on how CLKREQ is pulled by end point
1628 */
1629 data = appl_readl(pcie, APPL_PINMUX);
1630 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1631 /* Cut REFCLK to slot */
1632 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1633 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1634 appl_writel(pcie, data, APPL_PINMUX);
1635}
1636
1637static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1638{
1639 tegra_pcie_downstream_dev_to_D0(pcie);
1640 dw_pcie_host_deinit(&pcie->pci.pp);
1641 tegra_pcie_dw_pme_turnoff(pcie);
1642 tegra_pcie_unconfig_controller(pcie);
1643}
1644
1645static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1646{
1647 struct device *dev = pcie->dev;
1648 char *name;
1649 int ret;
1650
1651 pm_runtime_enable(dev);
1652
1653 ret = pm_runtime_get_sync(dev);
1654 if (ret < 0) {
1655 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1656 ret);
1657 goto fail_pm_get_sync;
1658 }
1659
1660 ret = pinctrl_pm_select_default_state(dev);
1661 if (ret < 0) {
1662 dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1663 goto fail_pm_get_sync;
1664 }
1665
1666 ret = tegra_pcie_init_controller(pcie);
1667 if (ret < 0) {
1668 dev_err(dev, "Failed to initialize controller: %d\n", ret);
1669 goto fail_pm_get_sync;
1670 }
1671
1672 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1673 if (!pcie->link_state) {
1674 ret = -ENOMEDIUM;
1675 goto fail_host_init;
1676 }
1677
1678 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1679 if (!name) {
1680 ret = -ENOMEM;
1681 goto fail_host_init;
1682 }
1683
1684 pcie->debugfs = debugfs_create_dir(name, NULL);
1685 init_debugfs(pcie);
1686
1687 return ret;
1688
1689fail_host_init:
1690 tegra_pcie_deinit_controller(pcie);
1691fail_pm_get_sync:
1692 pm_runtime_put_sync(dev);
1693 pm_runtime_disable(dev);
1694 return ret;
1695}
1696
1697static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1698{
1699 u32 val;
1700 int ret;
1701
1702 if (pcie->ep_state == EP_STATE_DISABLED)
1703 return;
1704
1705 /* Disable LTSSM */
1706 val = appl_readl(pcie, APPL_CTRL);
1707 val &= ~APPL_CTRL_LTSSM_EN;
1708 appl_writel(pcie, val, APPL_CTRL);
1709
1710 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1711 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1712 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1713 LTSSM_STATE_PRE_DETECT,
1714 1, LTSSM_TIMEOUT);
1715 if (ret)
1716 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1717
1718 reset_control_assert(pcie->core_rst);
1719
1720 tegra_pcie_disable_phy(pcie);
1721
1722 reset_control_assert(pcie->core_apb_rst);
1723
1724 clk_disable_unprepare(pcie->core_clk);
1725
1726 pm_runtime_put_sync(pcie->dev);
1727
1728 if (pcie->enable_ext_refclk) {
1729 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1730 if (ret)
1731 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n",
1732 ret);
1733 }
1734
1735 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1736 if (ret)
1737 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1738
1739 pcie->ep_state = EP_STATE_DISABLED;
1740 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1741}
1742
1743static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1744{
1745 struct dw_pcie *pci = &pcie->pci;
1746 struct dw_pcie_ep *ep = &pci->ep;
1747 struct device *dev = pcie->dev;
1748 u32 val;
1749 int ret;
1750 u16 val_16;
1751
1752 if (pcie->ep_state == EP_STATE_ENABLED)
1753 return;
1754
1755 ret = pm_runtime_resume_and_get(dev);
1756 if (ret < 0) {
1757 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1758 ret);
1759 return;
1760 }
1761
1762 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1763 if (ret) {
1764 dev_err(pcie->dev, "Failed to enable controller %u: %d\n",
1765 pcie->cid, ret);
1766 goto fail_set_ctrl_state;
1767 }
1768
1769 if (pcie->enable_ext_refclk) {
1770 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1771 if (ret) {
1772 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n",
1773 ret);
1774 goto fail_pll_init;
1775 }
1776 }
1777
1778 ret = clk_prepare_enable(pcie->core_clk);
1779 if (ret) {
1780 dev_err(dev, "Failed to enable core clock: %d\n", ret);
1781 goto fail_core_clk_enable;
1782 }
1783
1784 ret = reset_control_deassert(pcie->core_apb_rst);
1785 if (ret) {
1786 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1787 goto fail_core_apb_rst;
1788 }
1789
1790 ret = tegra_pcie_enable_phy(pcie);
1791 if (ret) {
1792 dev_err(dev, "Failed to enable PHY: %d\n", ret);
1793 goto fail_phy;
1794 }
1795
1796 /* Clear any stale interrupt statuses */
1797 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1798 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1799 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1800 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1801 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1802 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1804 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1805 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1806 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1807 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1808 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1809 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1810 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1811 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1812
1813 /* configure this core for EP mode operation */
1814 val = appl_readl(pcie, APPL_DM_TYPE);
1815 val &= ~APPL_DM_TYPE_MASK;
1816 val |= APPL_DM_TYPE_EP;
1817 appl_writel(pcie, val, APPL_DM_TYPE);
1818
1819 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1820
1821 val = appl_readl(pcie, APPL_CTRL);
1822 val |= APPL_CTRL_SYS_PRE_DET_STATE;
1823 val |= APPL_CTRL_HW_HOT_RST_EN;
1824 appl_writel(pcie, val, APPL_CTRL);
1825
1826 val = appl_readl(pcie, APPL_CFG_MISC);
1827 val |= APPL_CFG_MISC_SLV_EP_MODE;
1828 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1829 appl_writel(pcie, val, APPL_CFG_MISC);
1830
1831 val = appl_readl(pcie, APPL_PINMUX);
1832 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1833 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1834 appl_writel(pcie, val, APPL_PINMUX);
1835
1836 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1837 APPL_CFG_BASE_ADDR);
1838
1839 appl_writel(pcie, pcie->atu_dma_res->start &
1840 APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1841 APPL_CFG_IATU_DMA_BASE_ADDR);
1842
1843 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1844 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1845 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1846 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1847 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1848
1849 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1850 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1851 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1852 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1853
1854 reset_control_deassert(pcie->core_rst);
1855
1856 if (pcie->update_fc_fixup) {
1857 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1858 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1859 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1860 }
1861
1862 config_gen3_gen4_eq_presets(pcie);
1863
1864 init_host_aspm(pcie);
1865
1866 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1867 if (!pcie->supports_clkreq) {
1868 disable_aspm_l11(pcie);
1869 disable_aspm_l12(pcie);
1870 }
1871
1872 if (!pcie->of_data->has_l1ss_exit_fix) {
1873 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1874 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1875 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1876 }
1877
1878 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1879 PCI_CAP_ID_EXP);
1880
1881 /* Clear Slot Clock Configuration bit if SRNS configuration */
1882 if (pcie->enable_srns) {
1883 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
1884 PCI_EXP_LNKSTA);
1885 val_16 &= ~PCI_EXP_LNKSTA_SLC;
1886 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
1887 val_16);
1888 }
1889
1890 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1891
1892 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1893 val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1894 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1895 val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1896 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1897
1898 ret = dw_pcie_ep_init_complete(ep);
1899 if (ret) {
1900 dev_err(dev, "Failed to complete initialization: %d\n", ret);
1901 goto fail_init_complete;
1902 }
1903
1904 dw_pcie_ep_init_notify(ep);
1905
1906 /* Program the private control to allow sending LTR upstream */
1907 if (pcie->of_data->has_ltr_req_fix) {
1908 val = appl_readl(pcie, APPL_LTR_MSG_2);
1909 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
1910 appl_writel(pcie, val, APPL_LTR_MSG_2);
1911 }
1912
1913 /* Enable LTSSM */
1914 val = appl_readl(pcie, APPL_CTRL);
1915 val |= APPL_CTRL_LTSSM_EN;
1916 appl_writel(pcie, val, APPL_CTRL);
1917
1918 pcie->ep_state = EP_STATE_ENABLED;
1919 dev_dbg(dev, "Initialization of endpoint is completed\n");
1920
1921 return;
1922
1923fail_init_complete:
1924 reset_control_assert(pcie->core_rst);
1925 tegra_pcie_disable_phy(pcie);
1926fail_phy:
1927 reset_control_assert(pcie->core_apb_rst);
1928fail_core_apb_rst:
1929 clk_disable_unprepare(pcie->core_clk);
1930fail_core_clk_enable:
1931 tegra_pcie_bpmp_set_pll_state(pcie, false);
1932fail_pll_init:
1933 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1934fail_set_ctrl_state:
1935 pm_runtime_put_sync(dev);
1936}
1937
1938static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1939{
1940 struct tegra_pcie_dw *pcie = arg;
1941
1942 if (gpiod_get_value(pcie->pex_rst_gpiod))
1943 pex_ep_event_pex_rst_assert(pcie);
1944 else
1945 pex_ep_event_pex_rst_deassert(pcie);
1946
1947 return IRQ_HANDLED;
1948}
1949
1950static int tegra_pcie_ep_raise_intx_irq(struct tegra_pcie_dw *pcie, u16 irq)
1951{
1952 /* Tegra194 supports only INTA */
1953 if (irq > 1)
1954 return -EINVAL;
1955
1956 appl_writel(pcie, 1, APPL_LEGACY_INTX);
1957 usleep_range(1000, 2000);
1958 appl_writel(pcie, 0, APPL_LEGACY_INTX);
1959 return 0;
1960}
1961
1962static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1963{
1964 if (unlikely(irq > 31))
1965 return -EINVAL;
1966
1967 appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1968
1969 return 0;
1970}
1971
1972static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1973{
1974 struct dw_pcie_ep *ep = &pcie->pci.ep;
1975
1976 writel(irq, ep->msi_mem);
1977
1978 return 0;
1979}
1980
1981static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1982 unsigned int type, u16 interrupt_num)
1983{
1984 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1985 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1986
1987 switch (type) {
1988 case PCI_IRQ_INTX:
1989 return tegra_pcie_ep_raise_intx_irq(pcie, interrupt_num);
1990
1991 case PCI_IRQ_MSI:
1992 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1993
1994 case PCI_IRQ_MSIX:
1995 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1996
1997 default:
1998 dev_err(pci->dev, "Unknown IRQ type\n");
1999 return -EPERM;
2000 }
2001
2002 return 0;
2003}
2004
2005static const struct pci_epc_features tegra_pcie_epc_features = {
2006 .linkup_notifier = true,
2007 .core_init_notifier = true,
2008 .msi_capable = false,
2009 .msix_capable = false,
2010 .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
2011 .bar_fixed_64bit = 1 << BAR_0,
2012 .bar_fixed_size[0] = SZ_1M,
2013};
2014
2015static const struct pci_epc_features*
2016tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
2017{
2018 return &tegra_pcie_epc_features;
2019}
2020
2021static const struct dw_pcie_ep_ops pcie_ep_ops = {
2022 .raise_irq = tegra_pcie_ep_raise_irq,
2023 .get_features = tegra_pcie_ep_get_features,
2024};
2025
2026static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
2027 struct platform_device *pdev)
2028{
2029 struct dw_pcie *pci = &pcie->pci;
2030 struct device *dev = pcie->dev;
2031 struct dw_pcie_ep *ep;
2032 char *name;
2033 int ret;
2034
2035 ep = &pci->ep;
2036 ep->ops = &pcie_ep_ops;
2037
2038 ep->page_size = SZ_64K;
2039
2040 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
2041 if (ret < 0) {
2042 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
2043 ret);
2044 return ret;
2045 }
2046
2047 ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2048 if (ret < 0) {
2049 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2050 return ret;
2051 }
2052 pcie->pex_rst_irq = (unsigned int)ret;
2053
2054 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2055 pcie->cid);
2056 if (!name) {
2057 dev_err(dev, "Failed to create PERST IRQ string\n");
2058 return -ENOMEM;
2059 }
2060
2061 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2062
2063 pcie->ep_state = EP_STATE_DISABLED;
2064
2065 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2066 tegra_pcie_ep_pex_rst_irq,
2067 IRQF_TRIGGER_RISING |
2068 IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2069 name, (void *)pcie);
2070 if (ret < 0) {
2071 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2072 return ret;
2073 }
2074
2075 pm_runtime_enable(dev);
2076
2077 ret = dw_pcie_ep_init(ep);
2078 if (ret) {
2079 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2080 ret);
2081 pm_runtime_disable(dev);
2082 return ret;
2083 }
2084
2085 return 0;
2086}
2087
2088static int tegra_pcie_dw_probe(struct platform_device *pdev)
2089{
2090 const struct tegra_pcie_dw_of_data *data;
2091 struct device *dev = &pdev->dev;
2092 struct resource *atu_dma_res;
2093 struct tegra_pcie_dw *pcie;
2094 struct dw_pcie_rp *pp;
2095 struct dw_pcie *pci;
2096 struct phy **phys;
2097 char *name;
2098 int ret;
2099 u32 i;
2100
2101 data = of_device_get_match_data(dev);
2102
2103 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2104 if (!pcie)
2105 return -ENOMEM;
2106
2107 pci = &pcie->pci;
2108 pci->dev = &pdev->dev;
2109 pci->ops = &tegra_dw_pcie_ops;
2110 pcie->dev = &pdev->dev;
2111 pcie->of_data = (struct tegra_pcie_dw_of_data *)data;
2112 pci->n_fts[0] = pcie->of_data->n_fts[0];
2113 pci->n_fts[1] = pcie->of_data->n_fts[1];
2114 pp = &pci->pp;
2115 pp->num_vectors = MAX_MSI_IRQS;
2116
2117 ret = tegra_pcie_dw_parse_dt(pcie);
2118 if (ret < 0) {
2119 const char *level = KERN_ERR;
2120
2121 if (ret == -EPROBE_DEFER)
2122 level = KERN_DEBUG;
2123
2124 dev_printk(level, dev,
2125 dev_fmt("Failed to parse device tree: %d\n"),
2126 ret);
2127 return ret;
2128 }
2129
2130 ret = tegra_pcie_get_slot_regulators(pcie);
2131 if (ret < 0) {
2132 const char *level = KERN_ERR;
2133
2134 if (ret == -EPROBE_DEFER)
2135 level = KERN_DEBUG;
2136
2137 dev_printk(level, dev,
2138 dev_fmt("Failed to get slot regulators: %d\n"),
2139 ret);
2140 return ret;
2141 }
2142
2143 if (pcie->pex_refclk_sel_gpiod)
2144 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2145
2146 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2147 if (IS_ERR(pcie->pex_ctl_supply)) {
2148 ret = PTR_ERR(pcie->pex_ctl_supply);
2149 if (ret != -EPROBE_DEFER)
2150 dev_err(dev, "Failed to get regulator: %ld\n",
2151 PTR_ERR(pcie->pex_ctl_supply));
2152 return ret;
2153 }
2154
2155 pcie->core_clk = devm_clk_get(dev, "core");
2156 if (IS_ERR(pcie->core_clk)) {
2157 dev_err(dev, "Failed to get core clock: %ld\n",
2158 PTR_ERR(pcie->core_clk));
2159 return PTR_ERR(pcie->core_clk);
2160 }
2161
2162 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2163 "appl");
2164 if (!pcie->appl_res) {
2165 dev_err(dev, "Failed to find \"appl\" region\n");
2166 return -ENODEV;
2167 }
2168
2169 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2170 if (IS_ERR(pcie->appl_base))
2171 return PTR_ERR(pcie->appl_base);
2172
2173 pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2174 if (IS_ERR(pcie->core_apb_rst)) {
2175 dev_err(dev, "Failed to get APB reset: %ld\n",
2176 PTR_ERR(pcie->core_apb_rst));
2177 return PTR_ERR(pcie->core_apb_rst);
2178 }
2179
2180 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2181 if (!phys)
2182 return -ENOMEM;
2183
2184 for (i = 0; i < pcie->phy_count; i++) {
2185 name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2186 if (!name) {
2187 dev_err(dev, "Failed to create P2U string\n");
2188 return -ENOMEM;
2189 }
2190 phys[i] = devm_phy_get(dev, name);
2191 kfree(name);
2192 if (IS_ERR(phys[i])) {
2193 ret = PTR_ERR(phys[i]);
2194 if (ret != -EPROBE_DEFER)
2195 dev_err(dev, "Failed to get PHY: %d\n", ret);
2196 return ret;
2197 }
2198 }
2199
2200 pcie->phys = phys;
2201
2202 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2203 "atu_dma");
2204 if (!atu_dma_res) {
2205 dev_err(dev, "Failed to find \"atu_dma\" region\n");
2206 return -ENODEV;
2207 }
2208 pcie->atu_dma_res = atu_dma_res;
2209
2210 pci->atu_size = resource_size(atu_dma_res);
2211 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2212 if (IS_ERR(pci->atu_base))
2213 return PTR_ERR(pci->atu_base);
2214
2215 pcie->core_rst = devm_reset_control_get(dev, "core");
2216 if (IS_ERR(pcie->core_rst)) {
2217 dev_err(dev, "Failed to get core reset: %ld\n",
2218 PTR_ERR(pcie->core_rst));
2219 return PTR_ERR(pcie->core_rst);
2220 }
2221
2222 pp->irq = platform_get_irq_byname(pdev, "intr");
2223 if (pp->irq < 0)
2224 return pp->irq;
2225
2226 pcie->bpmp = tegra_bpmp_get(dev);
2227 if (IS_ERR(pcie->bpmp))
2228 return PTR_ERR(pcie->bpmp);
2229
2230 platform_set_drvdata(pdev, pcie);
2231
2232 pcie->icc_path = devm_of_icc_get(&pdev->dev, "write");
2233 ret = PTR_ERR_OR_ZERO(pcie->icc_path);
2234 if (ret) {
2235 tegra_bpmp_put(pcie->bpmp);
2236 dev_err_probe(&pdev->dev, ret, "failed to get write interconnect\n");
2237 return ret;
2238 }
2239
2240 switch (pcie->of_data->mode) {
2241 case DW_PCIE_RC_TYPE:
2242 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2243 IRQF_SHARED, "tegra-pcie-intr", pcie);
2244 if (ret) {
2245 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2246 ret);
2247 goto fail;
2248 }
2249
2250 ret = tegra_pcie_config_rp(pcie);
2251 if (ret && ret != -ENOMEDIUM)
2252 goto fail;
2253 else
2254 return 0;
2255 break;
2256
2257 case DW_PCIE_EP_TYPE:
2258 ret = devm_request_threaded_irq(dev, pp->irq,
2259 tegra_pcie_ep_hard_irq,
2260 tegra_pcie_ep_irq_thread,
2261 IRQF_SHARED | IRQF_ONESHOT,
2262 "tegra-pcie-ep-intr", pcie);
2263 if (ret) {
2264 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2265 ret);
2266 goto fail;
2267 }
2268
2269 ret = tegra_pcie_config_ep(pcie, pdev);
2270 if (ret < 0)
2271 goto fail;
2272 break;
2273
2274 default:
2275 dev_err(dev, "Invalid PCIe device type %d\n",
2276 pcie->of_data->mode);
2277 }
2278
2279fail:
2280 tegra_bpmp_put(pcie->bpmp);
2281 return ret;
2282}
2283
2284static void tegra_pcie_dw_remove(struct platform_device *pdev)
2285{
2286 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2287
2288 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2289 if (!pcie->link_state)
2290 return;
2291
2292 debugfs_remove_recursive(pcie->debugfs);
2293 tegra_pcie_deinit_controller(pcie);
2294 pm_runtime_put_sync(pcie->dev);
2295 } else {
2296 disable_irq(pcie->pex_rst_irq);
2297 pex_ep_event_pex_rst_assert(pcie);
2298 }
2299
2300 pm_runtime_disable(pcie->dev);
2301 tegra_bpmp_put(pcie->bpmp);
2302 if (pcie->pex_refclk_sel_gpiod)
2303 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2304}
2305
2306static int tegra_pcie_dw_suspend_late(struct device *dev)
2307{
2308 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2309 u32 val;
2310
2311 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2312 dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n");
2313 return -EPERM;
2314 }
2315
2316 if (!pcie->link_state)
2317 return 0;
2318
2319 /* Enable HW_HOT_RST mode */
2320 if (!pcie->of_data->has_sbr_reset_fix) {
2321 val = appl_readl(pcie, APPL_CTRL);
2322 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2323 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2324 val |= APPL_CTRL_HW_HOT_RST_EN;
2325 appl_writel(pcie, val, APPL_CTRL);
2326 }
2327
2328 return 0;
2329}
2330
2331static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2332{
2333 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2334
2335 if (!pcie->link_state)
2336 return 0;
2337
2338 tegra_pcie_downstream_dev_to_D0(pcie);
2339 tegra_pcie_dw_pme_turnoff(pcie);
2340 tegra_pcie_unconfig_controller(pcie);
2341
2342 return 0;
2343}
2344
2345static int tegra_pcie_dw_resume_noirq(struct device *dev)
2346{
2347 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2348 int ret;
2349
2350 if (!pcie->link_state)
2351 return 0;
2352
2353 ret = tegra_pcie_config_controller(pcie, true);
2354 if (ret < 0)
2355 return ret;
2356
2357 ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2358 if (ret < 0) {
2359 dev_err(dev, "Failed to init host: %d\n", ret);
2360 goto fail_host_init;
2361 }
2362
2363 dw_pcie_setup_rc(&pcie->pci.pp);
2364
2365 ret = tegra_pcie_dw_start_link(&pcie->pci);
2366 if (ret < 0)
2367 goto fail_host_init;
2368
2369 return 0;
2370
2371fail_host_init:
2372 tegra_pcie_unconfig_controller(pcie);
2373 return ret;
2374}
2375
2376static int tegra_pcie_dw_resume_early(struct device *dev)
2377{
2378 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2379 u32 val;
2380
2381 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2382 dev_err(dev, "Suspend is not supported in EP mode");
2383 return -ENOTSUPP;
2384 }
2385
2386 if (!pcie->link_state)
2387 return 0;
2388
2389 /* Disable HW_HOT_RST mode */
2390 if (!pcie->of_data->has_sbr_reset_fix) {
2391 val = appl_readl(pcie, APPL_CTRL);
2392 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2393 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2394 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2395 APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2396 val &= ~APPL_CTRL_HW_HOT_RST_EN;
2397 appl_writel(pcie, val, APPL_CTRL);
2398 }
2399
2400 return 0;
2401}
2402
2403static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2404{
2405 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2406
2407 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2408 if (!pcie->link_state)
2409 return;
2410
2411 debugfs_remove_recursive(pcie->debugfs);
2412 tegra_pcie_downstream_dev_to_D0(pcie);
2413
2414 disable_irq(pcie->pci.pp.irq);
2415 if (IS_ENABLED(CONFIG_PCI_MSI))
2416 disable_irq(pcie->pci.pp.msi_irq[0]);
2417
2418 tegra_pcie_dw_pme_turnoff(pcie);
2419 tegra_pcie_unconfig_controller(pcie);
2420 pm_runtime_put_sync(pcie->dev);
2421 } else {
2422 disable_irq(pcie->pex_rst_irq);
2423 pex_ep_event_pex_rst_assert(pcie);
2424 }
2425}
2426
2427static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = {
2428 .version = TEGRA194_DWC_IP_VER,
2429 .mode = DW_PCIE_RC_TYPE,
2430 .cdm_chk_int_en_bit = BIT(19),
2431 /* Gen4 - 5, 6, 8 and 9 presets enabled */
2432 .gen4_preset_vec = 0x360,
2433 .n_fts = { 52, 52 },
2434};
2435
2436static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = {
2437 .version = TEGRA194_DWC_IP_VER,
2438 .mode = DW_PCIE_EP_TYPE,
2439 .cdm_chk_int_en_bit = BIT(19),
2440 /* Gen4 - 5, 6, 8 and 9 presets enabled */
2441 .gen4_preset_vec = 0x360,
2442 .n_fts = { 52, 52 },
2443};
2444
2445static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = {
2446 .version = TEGRA234_DWC_IP_VER,
2447 .mode = DW_PCIE_RC_TYPE,
2448 .has_msix_doorbell_access_fix = true,
2449 .has_sbr_reset_fix = true,
2450 .has_l1ss_exit_fix = true,
2451 .cdm_chk_int_en_bit = BIT(18),
2452 /* Gen4 - 6, 8 and 9 presets enabled */
2453 .gen4_preset_vec = 0x340,
2454 .n_fts = { 52, 80 },
2455};
2456
2457static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = {
2458 .version = TEGRA234_DWC_IP_VER,
2459 .mode = DW_PCIE_EP_TYPE,
2460 .has_l1ss_exit_fix = true,
2461 .has_ltr_req_fix = true,
2462 .cdm_chk_int_en_bit = BIT(18),
2463 /* Gen4 - 6, 8 and 9 presets enabled */
2464 .gen4_preset_vec = 0x340,
2465 .n_fts = { 52, 80 },
2466};
2467
2468static const struct of_device_id tegra_pcie_dw_of_match[] = {
2469 {
2470 .compatible = "nvidia,tegra194-pcie",
2471 .data = &tegra194_pcie_dw_rc_of_data,
2472 },
2473 {
2474 .compatible = "nvidia,tegra194-pcie-ep",
2475 .data = &tegra194_pcie_dw_ep_of_data,
2476 },
2477 {
2478 .compatible = "nvidia,tegra234-pcie",
2479 .data = &tegra234_pcie_dw_rc_of_data,
2480 },
2481 {
2482 .compatible = "nvidia,tegra234-pcie-ep",
2483 .data = &tegra234_pcie_dw_ep_of_data,
2484 },
2485 {}
2486};
2487
2488static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2489 .suspend_late = tegra_pcie_dw_suspend_late,
2490 .suspend_noirq = tegra_pcie_dw_suspend_noirq,
2491 .resume_noirq = tegra_pcie_dw_resume_noirq,
2492 .resume_early = tegra_pcie_dw_resume_early,
2493};
2494
2495static struct platform_driver tegra_pcie_dw_driver = {
2496 .probe = tegra_pcie_dw_probe,
2497 .remove_new = tegra_pcie_dw_remove,
2498 .shutdown = tegra_pcie_dw_shutdown,
2499 .driver = {
2500 .name = "tegra194-pcie",
2501 .pm = &tegra_pcie_dw_pm_ops,
2502 .of_match_table = tegra_pcie_dw_of_match,
2503 },
2504};
2505module_platform_driver(tegra_pcie_dw_driver);
2506
2507MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2508
2509MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2510MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2511MODULE_LICENSE("GPL v2");
1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PCIe host controller driver for the following SoCs
4 * Tegra194
5 * Tegra234
6 *
7 * Copyright (C) 2019-2022 NVIDIA Corporation.
8 *
9 * Author: Vidya Sagar <vidyas@nvidia.com>
10 */
11
12#include <linux/clk.h>
13#include <linux/debugfs.h>
14#include <linux/delay.h>
15#include <linux/gpio.h>
16#include <linux/gpio/consumer.h>
17#include <linux/interrupt.h>
18#include <linux/iopoll.h>
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/of.h>
22#include <linux/of_device.h>
23#include <linux/of_gpio.h>
24#include <linux/of_pci.h>
25#include <linux/pci.h>
26#include <linux/phy/phy.h>
27#include <linux/pinctrl/consumer.h>
28#include <linux/platform_device.h>
29#include <linux/pm_runtime.h>
30#include <linux/random.h>
31#include <linux/reset.h>
32#include <linux/resource.h>
33#include <linux/types.h>
34#include "pcie-designware.h"
35#include <soc/tegra/bpmp.h>
36#include <soc/tegra/bpmp-abi.h>
37#include "../../pci.h"
38
39#define TEGRA194_DWC_IP_VER 0x490A
40#define TEGRA234_DWC_IP_VER 0x562A
41
42#define APPL_PINMUX 0x0
43#define APPL_PINMUX_PEX_RST BIT(0)
44#define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)
45#define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)
46#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)
47#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)
48
49#define APPL_CTRL 0x4
50#define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)
51#define APPL_CTRL_LTSSM_EN BIT(7)
52#define APPL_CTRL_HW_HOT_RST_EN BIT(20)
53#define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)
54#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22
55#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1
56#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN 0x2
57
58#define APPL_INTR_EN_L0_0 0x8
59#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)
60#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)
61#define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)
62#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15)
63#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)
64#define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)
65#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)
66
67#define APPL_INTR_STATUS_L0 0xC
68#define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)
69#define APPL_INTR_STATUS_L0_INT_INT BIT(8)
70#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15)
71#define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16)
72#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)
73
74#define APPL_INTR_EN_L1_0_0 0x1C
75#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)
76#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3)
77#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30)
78
79#define APPL_INTR_STATUS_L1_0_0 0x20
80#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)
81#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3)
82#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30)
83
84#define APPL_INTR_STATUS_L1_1 0x2C
85#define APPL_INTR_STATUS_L1_2 0x30
86#define APPL_INTR_STATUS_L1_3 0x34
87#define APPL_INTR_STATUS_L1_6 0x3C
88#define APPL_INTR_STATUS_L1_7 0x40
89#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1)
90
91#define APPL_INTR_EN_L1_8_0 0x44
92#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)
93#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)
94#define APPL_INTR_EN_L1_8_INTX_EN BIT(11)
95#define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)
96
97#define APPL_INTR_STATUS_L1_8_0 0x4C
98#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)
99#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)
100#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)
101
102#define APPL_INTR_STATUS_L1_9 0x54
103#define APPL_INTR_STATUS_L1_10 0x58
104#define APPL_INTR_STATUS_L1_11 0x64
105#define APPL_INTR_STATUS_L1_13 0x74
106#define APPL_INTR_STATUS_L1_14 0x78
107#define APPL_INTR_STATUS_L1_15 0x7C
108#define APPL_INTR_STATUS_L1_17 0x88
109
110#define APPL_INTR_EN_L1_18 0x90
111#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)
112#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
113#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
114
115#define APPL_INTR_STATUS_L1_18 0x94
116#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)
117#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
118#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
119
120#define APPL_MSI_CTRL_1 0xAC
121
122#define APPL_MSI_CTRL_2 0xB0
123
124#define APPL_LEGACY_INTX 0xB8
125
126#define APPL_LTR_MSG_1 0xC4
127#define LTR_MSG_REQ BIT(15)
128#define LTR_MST_NO_SNOOP_SHIFT 16
129
130#define APPL_LTR_MSG_2 0xC8
131#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)
132
133#define APPL_LINK_STATUS 0xCC
134#define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)
135
136#define APPL_DEBUG 0xD0
137#define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)
138#define APPL_DEBUG_PM_LINKST_IN_L0 0x11
139#define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)
140#define APPL_DEBUG_LTSSM_STATE_SHIFT 3
141#define LTSSM_STATE_PRE_DETECT 5
142
143#define APPL_RADM_STATUS 0xE4
144#define APPL_PM_XMT_TURNOFF_STATE BIT(0)
145
146#define APPL_DM_TYPE 0x100
147#define APPL_DM_TYPE_MASK GENMASK(3, 0)
148#define APPL_DM_TYPE_RP 0x4
149#define APPL_DM_TYPE_EP 0x0
150
151#define APPL_CFG_BASE_ADDR 0x104
152#define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)
153
154#define APPL_CFG_IATU_DMA_BASE_ADDR 0x108
155#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)
156
157#define APPL_CFG_MISC 0x110
158#define APPL_CFG_MISC_SLV_EP_MODE BIT(14)
159#define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)
160#define APPL_CFG_MISC_ARCACHE_SHIFT 10
161#define APPL_CFG_MISC_ARCACHE_VAL 3
162
163#define APPL_CFG_SLCG_OVERRIDE 0x114
164#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)
165
166#define APPL_CAR_RESET_OVRD 0x12C
167#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)
168
169#define IO_BASE_IO_DECODE BIT(0)
170#define IO_BASE_IO_DECODE_BIT8 BIT(8)
171
172#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)
173#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)
174
175#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718
176#define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)
177
178#define N_FTS_VAL 52
179#define FTS_VAL 52
180
181#define GEN3_EQ_CONTROL_OFF 0x8a8
182#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8
183#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8)
184#define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0)
185
186#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0
187#define AMBA_ERROR_RESPONSE_CRS_SHIFT 3
188#define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0)
189#define AMBA_ERROR_RESPONSE_CRS_OKAY 0
190#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1
191#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2
192
193#define MSIX_ADDR_MATCH_LOW_OFF 0x940
194#define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0)
195#define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2)
196
197#define MSIX_ADDR_MATCH_HIGH_OFF 0x944
198#define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0)
199
200#define PORT_LOGIC_MSIX_DOORBELL 0x948
201
202#define CAP_SPCIE_CAP_OFF 0x154
203#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)
204#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)
205#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8
206
207#define PME_ACK_TIMEOUT 10000
208
209#define LTSSM_TIMEOUT 50000 /* 50ms */
210
211#define GEN3_GEN4_EQ_PRESET_INIT 5
212
213#define GEN1_CORE_CLK_FREQ 62500000
214#define GEN2_CORE_CLK_FREQ 125000000
215#define GEN3_CORE_CLK_FREQ 250000000
216#define GEN4_CORE_CLK_FREQ 500000000
217
218#define LTR_MSG_TIMEOUT (100 * 1000)
219
220#define PERST_DEBOUNCE_TIME (5 * 1000)
221
222#define EP_STATE_DISABLED 0
223#define EP_STATE_ENABLED 1
224
225static const unsigned int pcie_gen_freq[] = {
226 GEN1_CORE_CLK_FREQ,
227 GEN2_CORE_CLK_FREQ,
228 GEN3_CORE_CLK_FREQ,
229 GEN4_CORE_CLK_FREQ
230};
231
232struct tegra_pcie_dw_of_data {
233 u32 version;
234 enum dw_pcie_device_mode mode;
235 bool has_msix_doorbell_access_fix;
236 bool has_sbr_reset_fix;
237 bool has_l1ss_exit_fix;
238 bool has_ltr_req_fix;
239 u32 cdm_chk_int_en_bit;
240 u32 gen4_preset_vec;
241 u8 n_fts[2];
242};
243
244struct tegra_pcie_dw {
245 struct device *dev;
246 struct resource *appl_res;
247 struct resource *dbi_res;
248 struct resource *atu_dma_res;
249 void __iomem *appl_base;
250 struct clk *core_clk;
251 struct reset_control *core_apb_rst;
252 struct reset_control *core_rst;
253 struct dw_pcie pci;
254 struct tegra_bpmp *bpmp;
255
256 struct tegra_pcie_dw_of_data *of_data;
257
258 bool supports_clkreq;
259 bool enable_cdm_check;
260 bool enable_srns;
261 bool link_state;
262 bool update_fc_fixup;
263 bool enable_ext_refclk;
264 u8 init_link_width;
265 u32 msi_ctrl_int;
266 u32 num_lanes;
267 u32 cid;
268 u32 cfg_link_cap_l1sub;
269 u32 ras_des_cap;
270 u32 pcie_cap_base;
271 u32 aspm_cmrt;
272 u32 aspm_pwr_on_t;
273 u32 aspm_l0s_enter_lat;
274
275 struct regulator *pex_ctl_supply;
276 struct regulator *slot_ctl_3v3;
277 struct regulator *slot_ctl_12v;
278
279 unsigned int phy_count;
280 struct phy **phys;
281
282 struct dentry *debugfs;
283
284 /* Endpoint mode specific */
285 struct gpio_desc *pex_rst_gpiod;
286 struct gpio_desc *pex_refclk_sel_gpiod;
287 unsigned int pex_rst_irq;
288 int ep_state;
289};
290
291static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
292{
293 return container_of(pci, struct tegra_pcie_dw, pci);
294}
295
296static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
297 const u32 reg)
298{
299 writel_relaxed(value, pcie->appl_base + reg);
300}
301
302static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
303{
304 return readl_relaxed(pcie->appl_base + reg);
305}
306
307struct tegra_pcie_soc {
308 enum dw_pcie_device_mode mode;
309};
310
311static void apply_bad_link_workaround(struct dw_pcie_rp *pp)
312{
313 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
314 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
315 u32 current_link_width;
316 u16 val;
317
318 /*
319 * NOTE:- Since this scenario is uncommon and link as such is not
320 * stable anyway, not waiting to confirm if link is really
321 * transitioning to Gen-2 speed
322 */
323 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
324 if (val & PCI_EXP_LNKSTA_LBMS) {
325 current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
326 PCI_EXP_LNKSTA_NLW_SHIFT;
327 if (pcie->init_link_width > current_link_width) {
328 dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
329 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
330 PCI_EXP_LNKCTL2);
331 val &= ~PCI_EXP_LNKCTL2_TLS;
332 val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
333 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
334 PCI_EXP_LNKCTL2, val);
335
336 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
337 PCI_EXP_LNKCTL);
338 val |= PCI_EXP_LNKCTL_RL;
339 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
340 PCI_EXP_LNKCTL, val);
341 }
342 }
343}
344
345static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
346{
347 struct tegra_pcie_dw *pcie = arg;
348 struct dw_pcie *pci = &pcie->pci;
349 struct dw_pcie_rp *pp = &pci->pp;
350 u32 val, status_l0, status_l1;
351 u16 val_w;
352
353 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
354 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
355 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
356 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
357 if (!pcie->of_data->has_sbr_reset_fix &&
358 status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
359 /* SBR & Surprise Link Down WAR */
360 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
361 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
362 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
363 udelay(1);
364 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
365 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
366 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
367
368 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
369 val |= PORT_LOGIC_SPEED_CHANGE;
370 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
371 }
372 }
373
374 if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) {
375 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
376 if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
377 appl_writel(pcie,
378 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
379 APPL_INTR_STATUS_L1_8_0);
380 apply_bad_link_workaround(pp);
381 }
382 if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
383 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
384 PCI_EXP_LNKSTA);
385 val_w |= PCI_EXP_LNKSTA_LBMS;
386 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
387 PCI_EXP_LNKSTA, val_w);
388
389 appl_writel(pcie,
390 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
391 APPL_INTR_STATUS_L1_8_0);
392
393 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
394 PCI_EXP_LNKSTA);
395 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
396 PCI_EXP_LNKSTA_CLS);
397 }
398 }
399
400 if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
401 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
402 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
403 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
404 dev_info(pci->dev, "CDM check complete\n");
405 val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
406 }
407 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
408 dev_err(pci->dev, "CDM comparison mismatch\n");
409 val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
410 }
411 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
412 dev_err(pci->dev, "CDM Logic error\n");
413 val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
414 }
415 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val);
416 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
417 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val);
418 }
419
420 return IRQ_HANDLED;
421}
422
423static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
424{
425 u32 val;
426
427 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
428 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
429 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
430 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
431 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
432 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
433 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
434 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
435 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
436 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
437 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
438 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
439 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
440 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
441 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
442 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
443
444 val = appl_readl(pcie, APPL_CTRL);
445 val |= APPL_CTRL_LTSSM_EN;
446 appl_writel(pcie, val, APPL_CTRL);
447}
448
449static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
450{
451 struct tegra_pcie_dw *pcie = arg;
452 struct dw_pcie *pci = &pcie->pci;
453 u32 val, speed;
454
455 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
456 PCI_EXP_LNKSTA_CLS;
457 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
458
459 if (pcie->of_data->has_ltr_req_fix)
460 return IRQ_HANDLED;
461
462 /* If EP doesn't advertise L1SS, just return */
463 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
464 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
465 return IRQ_HANDLED;
466
467 /* Check if BME is set to '1' */
468 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
469 if (val & PCI_COMMAND_MASTER) {
470 ktime_t timeout;
471
472 /* 110us for both snoop and no-snoop */
473 val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ;
474 val |= (val << LTR_MST_NO_SNOOP_SHIFT);
475 appl_writel(pcie, val, APPL_LTR_MSG_1);
476
477 /* Send LTR upstream */
478 val = appl_readl(pcie, APPL_LTR_MSG_2);
479 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
480 appl_writel(pcie, val, APPL_LTR_MSG_2);
481
482 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
483 for (;;) {
484 val = appl_readl(pcie, APPL_LTR_MSG_2);
485 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
486 break;
487 if (ktime_after(ktime_get(), timeout))
488 break;
489 usleep_range(1000, 1100);
490 }
491 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
492 dev_err(pcie->dev, "Failed to send LTR message\n");
493 }
494
495 return IRQ_HANDLED;
496}
497
498static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
499{
500 struct tegra_pcie_dw *pcie = arg;
501 struct dw_pcie_ep *ep = &pcie->pci.ep;
502 int spurious = 1;
503 u32 status_l0, status_l1, link_status;
504
505 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
506 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
507 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
508 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
509
510 if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
511 pex_ep_event_hot_rst_done(pcie);
512
513 if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
514 link_status = appl_readl(pcie, APPL_LINK_STATUS);
515 if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {
516 dev_dbg(pcie->dev, "Link is up with Host\n");
517 dw_pcie_ep_linkup(ep);
518 }
519 }
520
521 spurious = 0;
522 }
523
524 if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
525 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
526 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15);
527
528 if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
529 return IRQ_WAKE_THREAD;
530
531 spurious = 0;
532 }
533
534 if (spurious) {
535 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
536 status_l0);
537 appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0);
538 }
539
540 return IRQ_HANDLED;
541}
542
543static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
544 int size, u32 *val)
545{
546 struct dw_pcie_rp *pp = bus->sysdata;
547 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
548 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
549
550 /*
551 * This is an endpoint mode specific register happen to appear even
552 * when controller is operating in root port mode and system hangs
553 * when it is accessed with link being in ASPM-L1 state.
554 * So skip accessing it altogether
555 */
556 if (!pcie->of_data->has_msix_doorbell_access_fix &&
557 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
558 *val = 0x00000000;
559 return PCIBIOS_SUCCESSFUL;
560 }
561
562 return pci_generic_config_read(bus, devfn, where, size, val);
563}
564
565static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
566 int size, u32 val)
567{
568 struct dw_pcie_rp *pp = bus->sysdata;
569 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
570 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
571
572 /*
573 * This is an endpoint mode specific register happen to appear even
574 * when controller is operating in root port mode and system hangs
575 * when it is accessed with link being in ASPM-L1 state.
576 * So skip accessing it altogether
577 */
578 if (!pcie->of_data->has_msix_doorbell_access_fix &&
579 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
580 return PCIBIOS_SUCCESSFUL;
581
582 return pci_generic_config_write(bus, devfn, where, size, val);
583}
584
585static struct pci_ops tegra_pci_ops = {
586 .map_bus = dw_pcie_own_conf_map_bus,
587 .read = tegra_pcie_dw_rd_own_conf,
588 .write = tegra_pcie_dw_wr_own_conf,
589};
590
591#if defined(CONFIG_PCIEASPM)
592static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
593{
594 u32 val;
595
596 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
597 val &= ~PCI_L1SS_CAP_ASPM_L1_1;
598 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
599}
600
601static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
602{
603 u32 val;
604
605 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
606 val &= ~PCI_L1SS_CAP_ASPM_L1_2;
607 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
608}
609
610static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
611{
612 u32 val;
613
614 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
615 PCIE_RAS_DES_EVENT_COUNTER_CONTROL);
616 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
617 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
618 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
619 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
620 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
621 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
622 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
623 PCIE_RAS_DES_EVENT_COUNTER_DATA);
624
625 return val;
626}
627
628static int aspm_state_cnt(struct seq_file *s, void *data)
629{
630 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
631 dev_get_drvdata(s->private);
632 u32 val;
633
634 seq_printf(s, "Tx L0s entry count : %u\n",
635 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
636
637 seq_printf(s, "Rx L0s entry count : %u\n",
638 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
639
640 seq_printf(s, "Link L1 entry count : %u\n",
641 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
642
643 seq_printf(s, "Link L1.1 entry count : %u\n",
644 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
645
646 seq_printf(s, "Link L1.2 entry count : %u\n",
647 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
648
649 /* Clear all counters */
650 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
651 PCIE_RAS_DES_EVENT_COUNTER_CONTROL,
652 EVENT_COUNTER_ALL_CLEAR);
653
654 /* Re-enable counting */
655 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
656 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
657 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
658 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
659
660 return 0;
661}
662
663static void init_host_aspm(struct tegra_pcie_dw *pcie)
664{
665 struct dw_pcie *pci = &pcie->pci;
666 u32 val;
667
668 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
669 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
670
671 pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci,
672 PCI_EXT_CAP_ID_VNDR);
673
674 /* Enable ASPM counters */
675 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
676 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
677 dw_pcie_writel_dbi(pci, pcie->ras_des_cap +
678 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
679
680 /* Program T_cmrt and T_pwr_on values */
681 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
682 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
683 val |= (pcie->aspm_cmrt << 8);
684 val |= (pcie->aspm_pwr_on_t << 19);
685 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
686
687 /* Program L0s and L1 entrance latencies */
688 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
689 val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
690 val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
691 val |= PORT_AFR_ENTER_ASPM;
692 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
693}
694
695static void init_debugfs(struct tegra_pcie_dw *pcie)
696{
697 debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,
698 aspm_state_cnt);
699}
700#else
701static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
702static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
703static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
704static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
705#endif
706
707static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp)
708{
709 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
710 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
711 u32 val;
712 u16 val_w;
713
714 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
715 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
716 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
717
718 if (!pcie->of_data->has_sbr_reset_fix) {
719 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
720 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
721 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
722 }
723
724 if (pcie->enable_cdm_check) {
725 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
726 val |= pcie->of_data->cdm_chk_int_en_bit;
727 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
728
729 val = appl_readl(pcie, APPL_INTR_EN_L1_18);
730 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
731 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
732 appl_writel(pcie, val, APPL_INTR_EN_L1_18);
733 }
734
735 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
736 PCI_EXP_LNKSTA);
737 pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
738 PCI_EXP_LNKSTA_NLW_SHIFT;
739
740 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
741 PCI_EXP_LNKCTL);
742 val_w |= PCI_EXP_LNKCTL_LBMIE;
743 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
744 val_w);
745}
746
747static void tegra_pcie_enable_legacy_interrupts(struct dw_pcie_rp *pp)
748{
749 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
750 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
751 u32 val;
752
753 /* Enable legacy interrupt generation */
754 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
755 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
756 val |= APPL_INTR_EN_L0_0_INT_INT_EN;
757 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
758
759 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
760 val |= APPL_INTR_EN_L1_8_INTX_EN;
761 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
762 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
763 if (IS_ENABLED(CONFIG_PCIEAER))
764 val |= APPL_INTR_EN_L1_8_AER_INT_EN;
765 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
766}
767
768static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp)
769{
770 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
771 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
772 u32 val;
773
774 /* Enable MSI interrupt generation */
775 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
776 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
777 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
778 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
779}
780
781static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp)
782{
783 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
784 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
785
786 /* Clear interrupt statuses before enabling interrupts */
787 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
788 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
789 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
790 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
791 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
792 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
793 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
794 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
795 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
796 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
797 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
798 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
799 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
800 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
801 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
802
803 tegra_pcie_enable_system_interrupts(pp);
804 tegra_pcie_enable_legacy_interrupts(pp);
805 if (IS_ENABLED(CONFIG_PCI_MSI))
806 tegra_pcie_enable_msi_interrupts(pp);
807}
808
809static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
810{
811 struct dw_pcie *pci = &pcie->pci;
812 u32 val, offset, i;
813
814 /* Program init preset */
815 for (i = 0; i < pcie->num_lanes; i++) {
816 val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
817 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
818 val |= GEN3_GEN4_EQ_PRESET_INIT;
819 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
820 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
821 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
822 dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
823
824 offset = dw_pcie_find_ext_capability(pci,
825 PCI_EXT_CAP_ID_PL_16GT) +
826 PCI_PL_16GT_LE_CTRL;
827 val = dw_pcie_readb_dbi(pci, offset + i);
828 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
829 val |= GEN3_GEN4_EQ_PRESET_INIT;
830 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
831 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
832 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
833 dw_pcie_writeb_dbi(pci, offset + i, val);
834 }
835
836 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
837 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
838 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
839
840 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
841 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
842 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
843 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
844 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
845
846 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
847 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
848 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
849 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
850
851 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
852 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
853 val |= (pcie->of_data->gen4_preset_vec <<
854 GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
855 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
856 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
857
858 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
859 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
860 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
861}
862
863static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp)
864{
865 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
866 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
867 u32 val;
868 u16 val_16;
869
870 pp->bridge->ops = &tegra_pci_ops;
871
872 if (!pcie->pcie_cap_base)
873 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
874 PCI_CAP_ID_EXP);
875
876 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_DEVCTL);
877 val_16 &= ~PCI_EXP_DEVCTL_PAYLOAD;
878 val_16 |= PCI_EXP_DEVCTL_PAYLOAD_256B;
879 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_DEVCTL, val_16);
880
881 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
882 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
883 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
884
885 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
886 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
887 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
888 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
889
890 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
891
892 /* Enable as 0xFFFF0001 response for CRS */
893 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
894 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
895 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
896 AMBA_ERROR_RESPONSE_CRS_SHIFT);
897 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
898
899 /* Configure Max lane width from DT */
900 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
901 val &= ~PCI_EXP_LNKCAP_MLW;
902 val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
903 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
904
905 /* Clear Slot Clock Configuration bit if SRNS configuration */
906 if (pcie->enable_srns) {
907 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
908 PCI_EXP_LNKSTA);
909 val_16 &= ~PCI_EXP_LNKSTA_SLC;
910 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
911 val_16);
912 }
913
914 config_gen3_gen4_eq_presets(pcie);
915
916 init_host_aspm(pcie);
917
918 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
919 if (!pcie->supports_clkreq) {
920 disable_aspm_l11(pcie);
921 disable_aspm_l12(pcie);
922 }
923
924 if (!pcie->of_data->has_l1ss_exit_fix) {
925 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
926 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
927 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
928 }
929
930 if (pcie->update_fc_fixup) {
931 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
932 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
933 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
934 }
935
936 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
937
938 return 0;
939}
940
941static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
942{
943 u32 val, offset, speed, tmp;
944 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
945 struct dw_pcie_rp *pp = &pci->pp;
946 bool retry = true;
947
948 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
949 enable_irq(pcie->pex_rst_irq);
950 return 0;
951 }
952
953retry_link:
954 /* Assert RST */
955 val = appl_readl(pcie, APPL_PINMUX);
956 val &= ~APPL_PINMUX_PEX_RST;
957 appl_writel(pcie, val, APPL_PINMUX);
958
959 usleep_range(100, 200);
960
961 /* Enable LTSSM */
962 val = appl_readl(pcie, APPL_CTRL);
963 val |= APPL_CTRL_LTSSM_EN;
964 appl_writel(pcie, val, APPL_CTRL);
965
966 /* De-assert RST */
967 val = appl_readl(pcie, APPL_PINMUX);
968 val |= APPL_PINMUX_PEX_RST;
969 appl_writel(pcie, val, APPL_PINMUX);
970
971 msleep(100);
972
973 if (dw_pcie_wait_for_link(pci)) {
974 if (!retry)
975 return 0;
976 /*
977 * There are some endpoints which can't get the link up if
978 * root port has Data Link Feature (DLF) enabled.
979 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
980 * on Scaled Flow Control and DLF.
981 * So, need to confirm that is indeed the case here and attempt
982 * link up once again with DLF disabled.
983 */
984 val = appl_readl(pcie, APPL_DEBUG);
985 val &= APPL_DEBUG_LTSSM_STATE_MASK;
986 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
987 tmp = appl_readl(pcie, APPL_LINK_STATUS);
988 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
989 if (!(val == 0x11 && !tmp)) {
990 /* Link is down for all good reasons */
991 return 0;
992 }
993
994 dev_info(pci->dev, "Link is down in DLL");
995 dev_info(pci->dev, "Trying again with DLFE disabled\n");
996 /* Disable LTSSM */
997 val = appl_readl(pcie, APPL_CTRL);
998 val &= ~APPL_CTRL_LTSSM_EN;
999 appl_writel(pcie, val, APPL_CTRL);
1000
1001 reset_control_assert(pcie->core_rst);
1002 reset_control_deassert(pcie->core_rst);
1003
1004 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1005 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1006 val &= ~PCI_DLF_EXCHANGE_ENABLE;
1007 dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val);
1008
1009 tegra_pcie_dw_host_init(pp);
1010 dw_pcie_setup_rc(pp);
1011
1012 retry = false;
1013 goto retry_link;
1014 }
1015
1016 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
1017 PCI_EXP_LNKSTA_CLS;
1018 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
1019
1020 tegra_pcie_enable_interrupts(pp);
1021
1022 return 0;
1023}
1024
1025static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1026{
1027 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1028 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1029
1030 return !!(val & PCI_EXP_LNKSTA_DLLLA);
1031}
1032
1033static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1034{
1035 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1036
1037 disable_irq(pcie->pex_rst_irq);
1038}
1039
1040static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1041 .link_up = tegra_pcie_dw_link_up,
1042 .start_link = tegra_pcie_dw_start_link,
1043 .stop_link = tegra_pcie_dw_stop_link,
1044};
1045
1046static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1047 .host_init = tegra_pcie_dw_host_init,
1048};
1049
1050static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1051{
1052 unsigned int phy_count = pcie->phy_count;
1053
1054 while (phy_count--) {
1055 phy_power_off(pcie->phys[phy_count]);
1056 phy_exit(pcie->phys[phy_count]);
1057 }
1058}
1059
1060static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1061{
1062 unsigned int i;
1063 int ret;
1064
1065 for (i = 0; i < pcie->phy_count; i++) {
1066 ret = phy_init(pcie->phys[i]);
1067 if (ret < 0)
1068 goto phy_power_off;
1069
1070 ret = phy_power_on(pcie->phys[i]);
1071 if (ret < 0)
1072 goto phy_exit;
1073 }
1074
1075 return 0;
1076
1077phy_power_off:
1078 while (i--) {
1079 phy_power_off(pcie->phys[i]);
1080phy_exit:
1081 phy_exit(pcie->phys[i]);
1082 }
1083
1084 return ret;
1085}
1086
1087static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1088{
1089 struct platform_device *pdev = to_platform_device(pcie->dev);
1090 struct device_node *np = pcie->dev->of_node;
1091 int ret;
1092
1093 pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1094 if (!pcie->dbi_res) {
1095 dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1096 return -ENODEV;
1097 }
1098
1099 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1100 if (ret < 0) {
1101 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1102 return ret;
1103 }
1104
1105 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1106 &pcie->aspm_pwr_on_t);
1107 if (ret < 0)
1108 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1109 ret);
1110
1111 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1112 &pcie->aspm_l0s_enter_lat);
1113 if (ret < 0)
1114 dev_info(pcie->dev,
1115 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1116
1117 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1118 if (ret < 0) {
1119 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1120 return ret;
1121 }
1122
1123 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1124 if (ret) {
1125 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1126 return ret;
1127 }
1128
1129 ret = of_property_count_strings(np, "phy-names");
1130 if (ret < 0) {
1131 dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1132 ret);
1133 return ret;
1134 }
1135 pcie->phy_count = ret;
1136
1137 if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1138 pcie->update_fc_fixup = true;
1139
1140 /* RP using an external REFCLK is supported only in Tegra234 */
1141 if (pcie->of_data->version == TEGRA194_DWC_IP_VER) {
1142 if (pcie->of_data->mode == DW_PCIE_EP_TYPE)
1143 pcie->enable_ext_refclk = true;
1144 } else {
1145 pcie->enable_ext_refclk =
1146 of_property_read_bool(pcie->dev->of_node,
1147 "nvidia,enable-ext-refclk");
1148 }
1149
1150 pcie->supports_clkreq =
1151 of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1152
1153 pcie->enable_cdm_check =
1154 of_property_read_bool(np, "snps,enable-cdm-check");
1155
1156 if (pcie->of_data->version == TEGRA234_DWC_IP_VER)
1157 pcie->enable_srns =
1158 of_property_read_bool(np, "nvidia,enable-srns");
1159
1160 if (pcie->of_data->mode == DW_PCIE_RC_TYPE)
1161 return 0;
1162
1163 /* Endpoint mode specific DT entries */
1164 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1165 if (IS_ERR(pcie->pex_rst_gpiod)) {
1166 int err = PTR_ERR(pcie->pex_rst_gpiod);
1167 const char *level = KERN_ERR;
1168
1169 if (err == -EPROBE_DEFER)
1170 level = KERN_DEBUG;
1171
1172 dev_printk(level, pcie->dev,
1173 dev_fmt("Failed to get PERST GPIO: %d\n"),
1174 err);
1175 return err;
1176 }
1177
1178 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1179 "nvidia,refclk-select",
1180 GPIOD_OUT_HIGH);
1181 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1182 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1183 const char *level = KERN_ERR;
1184
1185 if (err == -EPROBE_DEFER)
1186 level = KERN_DEBUG;
1187
1188 dev_printk(level, pcie->dev,
1189 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1190 err);
1191 pcie->pex_refclk_sel_gpiod = NULL;
1192 }
1193
1194 return 0;
1195}
1196
1197static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1198 bool enable)
1199{
1200 struct mrq_uphy_response resp;
1201 struct tegra_bpmp_message msg;
1202 struct mrq_uphy_request req;
1203
1204 /*
1205 * Controller-5 doesn't need to have its state set by BPMP-FW in
1206 * Tegra194
1207 */
1208 if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5)
1209 return 0;
1210
1211 memset(&req, 0, sizeof(req));
1212 memset(&resp, 0, sizeof(resp));
1213
1214 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1215 req.controller_state.pcie_controller = pcie->cid;
1216 req.controller_state.enable = enable;
1217
1218 memset(&msg, 0, sizeof(msg));
1219 msg.mrq = MRQ_UPHY;
1220 msg.tx.data = &req;
1221 msg.tx.size = sizeof(req);
1222 msg.rx.data = &resp;
1223 msg.rx.size = sizeof(resp);
1224
1225 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1226}
1227
1228static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1229 bool enable)
1230{
1231 struct mrq_uphy_response resp;
1232 struct tegra_bpmp_message msg;
1233 struct mrq_uphy_request req;
1234
1235 memset(&req, 0, sizeof(req));
1236 memset(&resp, 0, sizeof(resp));
1237
1238 if (enable) {
1239 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1240 req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1241 } else {
1242 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1243 req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1244 }
1245
1246 memset(&msg, 0, sizeof(msg));
1247 msg.mrq = MRQ_UPHY;
1248 msg.tx.data = &req;
1249 msg.tx.size = sizeof(req);
1250 msg.rx.data = &resp;
1251 msg.rx.size = sizeof(resp);
1252
1253 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1254}
1255
1256static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1257{
1258 struct dw_pcie_rp *pp = &pcie->pci.pp;
1259 struct pci_bus *child, *root_bus = NULL;
1260 struct pci_dev *pdev;
1261
1262 /*
1263 * link doesn't go into L2 state with some of the endpoints with Tegra
1264 * if they are not in D0 state. So, need to make sure that immediate
1265 * downstream devices are in D0 state before sending PME_TurnOff to put
1266 * link into L2 state.
1267 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1268 * 5.2 Link State Power Management (Page #428).
1269 */
1270
1271 list_for_each_entry(child, &pp->bridge->bus->children, node) {
1272 /* Bring downstream devices to D0 if they are not already in */
1273 if (child->parent == pp->bridge->bus) {
1274 root_bus = child;
1275 break;
1276 }
1277 }
1278
1279 if (!root_bus) {
1280 dev_err(pcie->dev, "Failed to find downstream devices\n");
1281 return;
1282 }
1283
1284 list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1285 if (PCI_SLOT(pdev->devfn) == 0) {
1286 if (pci_set_power_state(pdev, PCI_D0))
1287 dev_err(pcie->dev,
1288 "Failed to transition %s to D0 state\n",
1289 dev_name(&pdev->dev));
1290 }
1291 }
1292}
1293
1294static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1295{
1296 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1297 if (IS_ERR(pcie->slot_ctl_3v3)) {
1298 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1299 return PTR_ERR(pcie->slot_ctl_3v3);
1300
1301 pcie->slot_ctl_3v3 = NULL;
1302 }
1303
1304 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1305 if (IS_ERR(pcie->slot_ctl_12v)) {
1306 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1307 return PTR_ERR(pcie->slot_ctl_12v);
1308
1309 pcie->slot_ctl_12v = NULL;
1310 }
1311
1312 return 0;
1313}
1314
1315static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1316{
1317 int ret;
1318
1319 if (pcie->slot_ctl_3v3) {
1320 ret = regulator_enable(pcie->slot_ctl_3v3);
1321 if (ret < 0) {
1322 dev_err(pcie->dev,
1323 "Failed to enable 3.3V slot supply: %d\n", ret);
1324 return ret;
1325 }
1326 }
1327
1328 if (pcie->slot_ctl_12v) {
1329 ret = regulator_enable(pcie->slot_ctl_12v);
1330 if (ret < 0) {
1331 dev_err(pcie->dev,
1332 "Failed to enable 12V slot supply: %d\n", ret);
1333 goto fail_12v_enable;
1334 }
1335 }
1336
1337 /*
1338 * According to PCI Express Card Electromechanical Specification
1339 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1340 * should be a minimum of 100ms.
1341 */
1342 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1343 msleep(100);
1344
1345 return 0;
1346
1347fail_12v_enable:
1348 if (pcie->slot_ctl_3v3)
1349 regulator_disable(pcie->slot_ctl_3v3);
1350 return ret;
1351}
1352
1353static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1354{
1355 if (pcie->slot_ctl_12v)
1356 regulator_disable(pcie->slot_ctl_12v);
1357 if (pcie->slot_ctl_3v3)
1358 regulator_disable(pcie->slot_ctl_3v3);
1359}
1360
1361static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1362 bool en_hw_hot_rst)
1363{
1364 int ret;
1365 u32 val;
1366
1367 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1368 if (ret) {
1369 dev_err(pcie->dev,
1370 "Failed to enable controller %u: %d\n", pcie->cid, ret);
1371 return ret;
1372 }
1373
1374 if (pcie->enable_ext_refclk) {
1375 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1376 if (ret) {
1377 dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret);
1378 goto fail_pll_init;
1379 }
1380 }
1381
1382 ret = tegra_pcie_enable_slot_regulators(pcie);
1383 if (ret < 0)
1384 goto fail_slot_reg_en;
1385
1386 ret = regulator_enable(pcie->pex_ctl_supply);
1387 if (ret < 0) {
1388 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1389 goto fail_reg_en;
1390 }
1391
1392 ret = clk_prepare_enable(pcie->core_clk);
1393 if (ret) {
1394 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1395 goto fail_core_clk;
1396 }
1397
1398 ret = reset_control_deassert(pcie->core_apb_rst);
1399 if (ret) {
1400 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1401 ret);
1402 goto fail_core_apb_rst;
1403 }
1404
1405 if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) {
1406 /* Enable HW_HOT_RST mode */
1407 val = appl_readl(pcie, APPL_CTRL);
1408 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1409 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1410 val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN <<
1411 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1412 val |= APPL_CTRL_HW_HOT_RST_EN;
1413 appl_writel(pcie, val, APPL_CTRL);
1414 }
1415
1416 ret = tegra_pcie_enable_phy(pcie);
1417 if (ret) {
1418 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1419 goto fail_phy;
1420 }
1421
1422 /* Update CFG base address */
1423 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1424 APPL_CFG_BASE_ADDR);
1425
1426 /* Configure this core for RP mode operation */
1427 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1428
1429 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1430
1431 val = appl_readl(pcie, APPL_CTRL);
1432 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1433
1434 val = appl_readl(pcie, APPL_CFG_MISC);
1435 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1436 appl_writel(pcie, val, APPL_CFG_MISC);
1437
1438 if (pcie->enable_srns || pcie->enable_ext_refclk) {
1439 /*
1440 * When Tegra PCIe RP is using external clock, it cannot supply
1441 * same clock to its downstream hierarchy. Hence, gate PCIe RP
1442 * REFCLK out pads when RP & EP are using separate clocks or RP
1443 * is using an external REFCLK.
1444 */
1445 val = appl_readl(pcie, APPL_PINMUX);
1446 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1447 val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1448 appl_writel(pcie, val, APPL_PINMUX);
1449 }
1450
1451 if (!pcie->supports_clkreq) {
1452 val = appl_readl(pcie, APPL_PINMUX);
1453 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1454 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1455 appl_writel(pcie, val, APPL_PINMUX);
1456 }
1457
1458 /* Update iATU_DMA base address */
1459 appl_writel(pcie,
1460 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1461 APPL_CFG_IATU_DMA_BASE_ADDR);
1462
1463 reset_control_deassert(pcie->core_rst);
1464
1465 return ret;
1466
1467fail_phy:
1468 reset_control_assert(pcie->core_apb_rst);
1469fail_core_apb_rst:
1470 clk_disable_unprepare(pcie->core_clk);
1471fail_core_clk:
1472 regulator_disable(pcie->pex_ctl_supply);
1473fail_reg_en:
1474 tegra_pcie_disable_slot_regulators(pcie);
1475fail_slot_reg_en:
1476 if (pcie->enable_ext_refclk)
1477 tegra_pcie_bpmp_set_pll_state(pcie, false);
1478fail_pll_init:
1479 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1480
1481 return ret;
1482}
1483
1484static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1485{
1486 int ret;
1487
1488 ret = reset_control_assert(pcie->core_rst);
1489 if (ret)
1490 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1491
1492 tegra_pcie_disable_phy(pcie);
1493
1494 ret = reset_control_assert(pcie->core_apb_rst);
1495 if (ret)
1496 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1497
1498 clk_disable_unprepare(pcie->core_clk);
1499
1500 ret = regulator_disable(pcie->pex_ctl_supply);
1501 if (ret)
1502 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1503
1504 tegra_pcie_disable_slot_regulators(pcie);
1505
1506 if (pcie->enable_ext_refclk) {
1507 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1508 if (ret)
1509 dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret);
1510 }
1511
1512 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1513 if (ret)
1514 dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1515 pcie->cid, ret);
1516}
1517
1518static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1519{
1520 struct dw_pcie *pci = &pcie->pci;
1521 struct dw_pcie_rp *pp = &pci->pp;
1522 int ret;
1523
1524 ret = tegra_pcie_config_controller(pcie, false);
1525 if (ret < 0)
1526 return ret;
1527
1528 pp->ops = &tegra_pcie_dw_host_ops;
1529
1530 ret = dw_pcie_host_init(pp);
1531 if (ret < 0) {
1532 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1533 goto fail_host_init;
1534 }
1535
1536 return 0;
1537
1538fail_host_init:
1539 tegra_pcie_unconfig_controller(pcie);
1540 return ret;
1541}
1542
1543static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1544{
1545 u32 val;
1546
1547 if (!tegra_pcie_dw_link_up(&pcie->pci))
1548 return 0;
1549
1550 val = appl_readl(pcie, APPL_RADM_STATUS);
1551 val |= APPL_PM_XMT_TURNOFF_STATE;
1552 appl_writel(pcie, val, APPL_RADM_STATUS);
1553
1554 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1555 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1556 1, PME_ACK_TIMEOUT);
1557}
1558
1559static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1560{
1561 u32 data;
1562 int err;
1563
1564 if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1565 dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1566 return;
1567 }
1568
1569 /*
1570 * PCIe controller exits from L2 only if reset is applied, so
1571 * controller doesn't handle interrupts. But in cases where
1572 * L2 entry fails, PERST# is asserted which can trigger surprise
1573 * link down AER. However this function call happens in
1574 * suspend_noirq(), so AER interrupt will not be processed.
1575 * Disable all interrupts to avoid such a scenario.
1576 */
1577 appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0);
1578
1579 if (tegra_pcie_try_link_l2(pcie)) {
1580 dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1581 /*
1582 * TX lane clock freq will reset to Gen1 only if link is in L2
1583 * or detect state.
1584 * So apply pex_rst to end point to force RP to go into detect
1585 * state
1586 */
1587 data = appl_readl(pcie, APPL_PINMUX);
1588 data &= ~APPL_PINMUX_PEX_RST;
1589 appl_writel(pcie, data, APPL_PINMUX);
1590
1591 /*
1592 * Some cards do not go to detect state even after de-asserting
1593 * PERST#. So, de-assert LTSSM to bring link to detect state.
1594 */
1595 data = readl(pcie->appl_base + APPL_CTRL);
1596 data &= ~APPL_CTRL_LTSSM_EN;
1597 writel(data, pcie->appl_base + APPL_CTRL);
1598
1599 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1600 data,
1601 ((data &
1602 APPL_DEBUG_LTSSM_STATE_MASK) >>
1603 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1604 LTSSM_STATE_PRE_DETECT,
1605 1, LTSSM_TIMEOUT);
1606 if (err)
1607 dev_info(pcie->dev, "Link didn't go to detect state\n");
1608 }
1609 /*
1610 * DBI registers may not be accessible after this as PLL-E would be
1611 * down depending on how CLKREQ is pulled by end point
1612 */
1613 data = appl_readl(pcie, APPL_PINMUX);
1614 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1615 /* Cut REFCLK to slot */
1616 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1617 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1618 appl_writel(pcie, data, APPL_PINMUX);
1619}
1620
1621static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1622{
1623 tegra_pcie_downstream_dev_to_D0(pcie);
1624 dw_pcie_host_deinit(&pcie->pci.pp);
1625 tegra_pcie_dw_pme_turnoff(pcie);
1626 tegra_pcie_unconfig_controller(pcie);
1627}
1628
1629static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1630{
1631 struct device *dev = pcie->dev;
1632 char *name;
1633 int ret;
1634
1635 pm_runtime_enable(dev);
1636
1637 ret = pm_runtime_get_sync(dev);
1638 if (ret < 0) {
1639 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1640 ret);
1641 goto fail_pm_get_sync;
1642 }
1643
1644 ret = pinctrl_pm_select_default_state(dev);
1645 if (ret < 0) {
1646 dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1647 goto fail_pm_get_sync;
1648 }
1649
1650 ret = tegra_pcie_init_controller(pcie);
1651 if (ret < 0) {
1652 dev_err(dev, "Failed to initialize controller: %d\n", ret);
1653 goto fail_pm_get_sync;
1654 }
1655
1656 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1657 if (!pcie->link_state) {
1658 ret = -ENOMEDIUM;
1659 goto fail_host_init;
1660 }
1661
1662 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1663 if (!name) {
1664 ret = -ENOMEM;
1665 goto fail_host_init;
1666 }
1667
1668 pcie->debugfs = debugfs_create_dir(name, NULL);
1669 init_debugfs(pcie);
1670
1671 return ret;
1672
1673fail_host_init:
1674 tegra_pcie_deinit_controller(pcie);
1675fail_pm_get_sync:
1676 pm_runtime_put_sync(dev);
1677 pm_runtime_disable(dev);
1678 return ret;
1679}
1680
1681static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1682{
1683 u32 val;
1684 int ret;
1685
1686 if (pcie->ep_state == EP_STATE_DISABLED)
1687 return;
1688
1689 /* Disable LTSSM */
1690 val = appl_readl(pcie, APPL_CTRL);
1691 val &= ~APPL_CTRL_LTSSM_EN;
1692 appl_writel(pcie, val, APPL_CTRL);
1693
1694 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1695 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1696 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1697 LTSSM_STATE_PRE_DETECT,
1698 1, LTSSM_TIMEOUT);
1699 if (ret)
1700 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1701
1702 reset_control_assert(pcie->core_rst);
1703
1704 tegra_pcie_disable_phy(pcie);
1705
1706 reset_control_assert(pcie->core_apb_rst);
1707
1708 clk_disable_unprepare(pcie->core_clk);
1709
1710 pm_runtime_put_sync(pcie->dev);
1711
1712 if (pcie->enable_ext_refclk) {
1713 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1714 if (ret)
1715 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n",
1716 ret);
1717 }
1718
1719 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1720 if (ret)
1721 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1722
1723 pcie->ep_state = EP_STATE_DISABLED;
1724 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1725}
1726
1727static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1728{
1729 struct dw_pcie *pci = &pcie->pci;
1730 struct dw_pcie_ep *ep = &pci->ep;
1731 struct device *dev = pcie->dev;
1732 u32 val;
1733 int ret;
1734 u16 val_16;
1735
1736 if (pcie->ep_state == EP_STATE_ENABLED)
1737 return;
1738
1739 ret = pm_runtime_resume_and_get(dev);
1740 if (ret < 0) {
1741 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1742 ret);
1743 return;
1744 }
1745
1746 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1747 if (ret) {
1748 dev_err(pcie->dev, "Failed to enable controller %u: %d\n",
1749 pcie->cid, ret);
1750 goto fail_set_ctrl_state;
1751 }
1752
1753 if (pcie->enable_ext_refclk) {
1754 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1755 if (ret) {
1756 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n",
1757 ret);
1758 goto fail_pll_init;
1759 }
1760 }
1761
1762 ret = clk_prepare_enable(pcie->core_clk);
1763 if (ret) {
1764 dev_err(dev, "Failed to enable core clock: %d\n", ret);
1765 goto fail_core_clk_enable;
1766 }
1767
1768 ret = reset_control_deassert(pcie->core_apb_rst);
1769 if (ret) {
1770 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1771 goto fail_core_apb_rst;
1772 }
1773
1774 ret = tegra_pcie_enable_phy(pcie);
1775 if (ret) {
1776 dev_err(dev, "Failed to enable PHY: %d\n", ret);
1777 goto fail_phy;
1778 }
1779
1780 /* Clear any stale interrupt statuses */
1781 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1782 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1783 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1784 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1785 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1786 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1787 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1788 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1789 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1790 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1791 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1792 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1793 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1794 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1795 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1796
1797 /* configure this core for EP mode operation */
1798 val = appl_readl(pcie, APPL_DM_TYPE);
1799 val &= ~APPL_DM_TYPE_MASK;
1800 val |= APPL_DM_TYPE_EP;
1801 appl_writel(pcie, val, APPL_DM_TYPE);
1802
1803 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1804
1805 val = appl_readl(pcie, APPL_CTRL);
1806 val |= APPL_CTRL_SYS_PRE_DET_STATE;
1807 val |= APPL_CTRL_HW_HOT_RST_EN;
1808 appl_writel(pcie, val, APPL_CTRL);
1809
1810 val = appl_readl(pcie, APPL_CFG_MISC);
1811 val |= APPL_CFG_MISC_SLV_EP_MODE;
1812 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1813 appl_writel(pcie, val, APPL_CFG_MISC);
1814
1815 val = appl_readl(pcie, APPL_PINMUX);
1816 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1817 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1818 appl_writel(pcie, val, APPL_PINMUX);
1819
1820 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1821 APPL_CFG_BASE_ADDR);
1822
1823 appl_writel(pcie, pcie->atu_dma_res->start &
1824 APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1825 APPL_CFG_IATU_DMA_BASE_ADDR);
1826
1827 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1828 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1829 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1830 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1831 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1832
1833 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1834 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1835 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1836 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1837
1838 reset_control_deassert(pcie->core_rst);
1839
1840 if (pcie->update_fc_fixup) {
1841 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1842 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1843 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1844 }
1845
1846 config_gen3_gen4_eq_presets(pcie);
1847
1848 init_host_aspm(pcie);
1849
1850 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1851 if (!pcie->supports_clkreq) {
1852 disable_aspm_l11(pcie);
1853 disable_aspm_l12(pcie);
1854 }
1855
1856 if (!pcie->of_data->has_l1ss_exit_fix) {
1857 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1858 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1859 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1860 }
1861
1862 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1863 PCI_CAP_ID_EXP);
1864
1865 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_DEVCTL);
1866 val_16 &= ~PCI_EXP_DEVCTL_PAYLOAD;
1867 val_16 |= PCI_EXP_DEVCTL_PAYLOAD_256B;
1868 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_DEVCTL, val_16);
1869
1870 /* Clear Slot Clock Configuration bit if SRNS configuration */
1871 if (pcie->enable_srns) {
1872 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
1873 PCI_EXP_LNKSTA);
1874 val_16 &= ~PCI_EXP_LNKSTA_SLC;
1875 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
1876 val_16);
1877 }
1878
1879 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1880
1881 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1882 val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1883 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1884 val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1885 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1886
1887 ret = dw_pcie_ep_init_complete(ep);
1888 if (ret) {
1889 dev_err(dev, "Failed to complete initialization: %d\n", ret);
1890 goto fail_init_complete;
1891 }
1892
1893 dw_pcie_ep_init_notify(ep);
1894
1895 /* Program the private control to allow sending LTR upstream */
1896 if (pcie->of_data->has_ltr_req_fix) {
1897 val = appl_readl(pcie, APPL_LTR_MSG_2);
1898 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
1899 appl_writel(pcie, val, APPL_LTR_MSG_2);
1900 }
1901
1902 /* Enable LTSSM */
1903 val = appl_readl(pcie, APPL_CTRL);
1904 val |= APPL_CTRL_LTSSM_EN;
1905 appl_writel(pcie, val, APPL_CTRL);
1906
1907 pcie->ep_state = EP_STATE_ENABLED;
1908 dev_dbg(dev, "Initialization of endpoint is completed\n");
1909
1910 return;
1911
1912fail_init_complete:
1913 reset_control_assert(pcie->core_rst);
1914 tegra_pcie_disable_phy(pcie);
1915fail_phy:
1916 reset_control_assert(pcie->core_apb_rst);
1917fail_core_apb_rst:
1918 clk_disable_unprepare(pcie->core_clk);
1919fail_core_clk_enable:
1920 tegra_pcie_bpmp_set_pll_state(pcie, false);
1921fail_pll_init:
1922 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1923fail_set_ctrl_state:
1924 pm_runtime_put_sync(dev);
1925}
1926
1927static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1928{
1929 struct tegra_pcie_dw *pcie = arg;
1930
1931 if (gpiod_get_value(pcie->pex_rst_gpiod))
1932 pex_ep_event_pex_rst_assert(pcie);
1933 else
1934 pex_ep_event_pex_rst_deassert(pcie);
1935
1936 return IRQ_HANDLED;
1937}
1938
1939static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1940{
1941 /* Tegra194 supports only INTA */
1942 if (irq > 1)
1943 return -EINVAL;
1944
1945 appl_writel(pcie, 1, APPL_LEGACY_INTX);
1946 usleep_range(1000, 2000);
1947 appl_writel(pcie, 0, APPL_LEGACY_INTX);
1948 return 0;
1949}
1950
1951static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1952{
1953 if (unlikely(irq > 31))
1954 return -EINVAL;
1955
1956 appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1957
1958 return 0;
1959}
1960
1961static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1962{
1963 struct dw_pcie_ep *ep = &pcie->pci.ep;
1964
1965 writel(irq, ep->msi_mem);
1966
1967 return 0;
1968}
1969
1970static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1971 enum pci_epc_irq_type type,
1972 u16 interrupt_num)
1973{
1974 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1975 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1976
1977 switch (type) {
1978 case PCI_EPC_IRQ_LEGACY:
1979 return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
1980
1981 case PCI_EPC_IRQ_MSI:
1982 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1983
1984 case PCI_EPC_IRQ_MSIX:
1985 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1986
1987 default:
1988 dev_err(pci->dev, "Unknown IRQ type\n");
1989 return -EPERM;
1990 }
1991
1992 return 0;
1993}
1994
1995static const struct pci_epc_features tegra_pcie_epc_features = {
1996 .linkup_notifier = true,
1997 .core_init_notifier = true,
1998 .msi_capable = false,
1999 .msix_capable = false,
2000 .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
2001 .bar_fixed_64bit = 1 << BAR_0,
2002 .bar_fixed_size[0] = SZ_1M,
2003};
2004
2005static const struct pci_epc_features*
2006tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
2007{
2008 return &tegra_pcie_epc_features;
2009}
2010
2011static const struct dw_pcie_ep_ops pcie_ep_ops = {
2012 .raise_irq = tegra_pcie_ep_raise_irq,
2013 .get_features = tegra_pcie_ep_get_features,
2014};
2015
2016static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
2017 struct platform_device *pdev)
2018{
2019 struct dw_pcie *pci = &pcie->pci;
2020 struct device *dev = pcie->dev;
2021 struct dw_pcie_ep *ep;
2022 char *name;
2023 int ret;
2024
2025 ep = &pci->ep;
2026 ep->ops = &pcie_ep_ops;
2027
2028 ep->page_size = SZ_64K;
2029
2030 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
2031 if (ret < 0) {
2032 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
2033 ret);
2034 return ret;
2035 }
2036
2037 ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2038 if (ret < 0) {
2039 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2040 return ret;
2041 }
2042 pcie->pex_rst_irq = (unsigned int)ret;
2043
2044 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2045 pcie->cid);
2046 if (!name) {
2047 dev_err(dev, "Failed to create PERST IRQ string\n");
2048 return -ENOMEM;
2049 }
2050
2051 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2052
2053 pcie->ep_state = EP_STATE_DISABLED;
2054
2055 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2056 tegra_pcie_ep_pex_rst_irq,
2057 IRQF_TRIGGER_RISING |
2058 IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2059 name, (void *)pcie);
2060 if (ret < 0) {
2061 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2062 return ret;
2063 }
2064
2065 pm_runtime_enable(dev);
2066
2067 ret = dw_pcie_ep_init(ep);
2068 if (ret) {
2069 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2070 ret);
2071 pm_runtime_disable(dev);
2072 return ret;
2073 }
2074
2075 return 0;
2076}
2077
2078static int tegra_pcie_dw_probe(struct platform_device *pdev)
2079{
2080 const struct tegra_pcie_dw_of_data *data;
2081 struct device *dev = &pdev->dev;
2082 struct resource *atu_dma_res;
2083 struct tegra_pcie_dw *pcie;
2084 struct dw_pcie_rp *pp;
2085 struct dw_pcie *pci;
2086 struct phy **phys;
2087 char *name;
2088 int ret;
2089 u32 i;
2090
2091 data = of_device_get_match_data(dev);
2092
2093 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2094 if (!pcie)
2095 return -ENOMEM;
2096
2097 pci = &pcie->pci;
2098 pci->dev = &pdev->dev;
2099 pci->ops = &tegra_dw_pcie_ops;
2100 pcie->dev = &pdev->dev;
2101 pcie->of_data = (struct tegra_pcie_dw_of_data *)data;
2102 pci->n_fts[0] = pcie->of_data->n_fts[0];
2103 pci->n_fts[1] = pcie->of_data->n_fts[1];
2104 pp = &pci->pp;
2105 pp->num_vectors = MAX_MSI_IRQS;
2106
2107 ret = tegra_pcie_dw_parse_dt(pcie);
2108 if (ret < 0) {
2109 const char *level = KERN_ERR;
2110
2111 if (ret == -EPROBE_DEFER)
2112 level = KERN_DEBUG;
2113
2114 dev_printk(level, dev,
2115 dev_fmt("Failed to parse device tree: %d\n"),
2116 ret);
2117 return ret;
2118 }
2119
2120 ret = tegra_pcie_get_slot_regulators(pcie);
2121 if (ret < 0) {
2122 const char *level = KERN_ERR;
2123
2124 if (ret == -EPROBE_DEFER)
2125 level = KERN_DEBUG;
2126
2127 dev_printk(level, dev,
2128 dev_fmt("Failed to get slot regulators: %d\n"),
2129 ret);
2130 return ret;
2131 }
2132
2133 if (pcie->pex_refclk_sel_gpiod)
2134 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2135
2136 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2137 if (IS_ERR(pcie->pex_ctl_supply)) {
2138 ret = PTR_ERR(pcie->pex_ctl_supply);
2139 if (ret != -EPROBE_DEFER)
2140 dev_err(dev, "Failed to get regulator: %ld\n",
2141 PTR_ERR(pcie->pex_ctl_supply));
2142 return ret;
2143 }
2144
2145 pcie->core_clk = devm_clk_get(dev, "core");
2146 if (IS_ERR(pcie->core_clk)) {
2147 dev_err(dev, "Failed to get core clock: %ld\n",
2148 PTR_ERR(pcie->core_clk));
2149 return PTR_ERR(pcie->core_clk);
2150 }
2151
2152 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2153 "appl");
2154 if (!pcie->appl_res) {
2155 dev_err(dev, "Failed to find \"appl\" region\n");
2156 return -ENODEV;
2157 }
2158
2159 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2160 if (IS_ERR(pcie->appl_base))
2161 return PTR_ERR(pcie->appl_base);
2162
2163 pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2164 if (IS_ERR(pcie->core_apb_rst)) {
2165 dev_err(dev, "Failed to get APB reset: %ld\n",
2166 PTR_ERR(pcie->core_apb_rst));
2167 return PTR_ERR(pcie->core_apb_rst);
2168 }
2169
2170 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2171 if (!phys)
2172 return -ENOMEM;
2173
2174 for (i = 0; i < pcie->phy_count; i++) {
2175 name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2176 if (!name) {
2177 dev_err(dev, "Failed to create P2U string\n");
2178 return -ENOMEM;
2179 }
2180 phys[i] = devm_phy_get(dev, name);
2181 kfree(name);
2182 if (IS_ERR(phys[i])) {
2183 ret = PTR_ERR(phys[i]);
2184 if (ret != -EPROBE_DEFER)
2185 dev_err(dev, "Failed to get PHY: %d\n", ret);
2186 return ret;
2187 }
2188 }
2189
2190 pcie->phys = phys;
2191
2192 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2193 "atu_dma");
2194 if (!atu_dma_res) {
2195 dev_err(dev, "Failed to find \"atu_dma\" region\n");
2196 return -ENODEV;
2197 }
2198 pcie->atu_dma_res = atu_dma_res;
2199
2200 pci->atu_size = resource_size(atu_dma_res);
2201 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2202 if (IS_ERR(pci->atu_base))
2203 return PTR_ERR(pci->atu_base);
2204
2205 pcie->core_rst = devm_reset_control_get(dev, "core");
2206 if (IS_ERR(pcie->core_rst)) {
2207 dev_err(dev, "Failed to get core reset: %ld\n",
2208 PTR_ERR(pcie->core_rst));
2209 return PTR_ERR(pcie->core_rst);
2210 }
2211
2212 pp->irq = platform_get_irq_byname(pdev, "intr");
2213 if (pp->irq < 0)
2214 return pp->irq;
2215
2216 pcie->bpmp = tegra_bpmp_get(dev);
2217 if (IS_ERR(pcie->bpmp))
2218 return PTR_ERR(pcie->bpmp);
2219
2220 platform_set_drvdata(pdev, pcie);
2221
2222 switch (pcie->of_data->mode) {
2223 case DW_PCIE_RC_TYPE:
2224 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2225 IRQF_SHARED, "tegra-pcie-intr", pcie);
2226 if (ret) {
2227 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2228 ret);
2229 goto fail;
2230 }
2231
2232 ret = tegra_pcie_config_rp(pcie);
2233 if (ret && ret != -ENOMEDIUM)
2234 goto fail;
2235 else
2236 return 0;
2237 break;
2238
2239 case DW_PCIE_EP_TYPE:
2240 ret = devm_request_threaded_irq(dev, pp->irq,
2241 tegra_pcie_ep_hard_irq,
2242 tegra_pcie_ep_irq_thread,
2243 IRQF_SHARED | IRQF_ONESHOT,
2244 "tegra-pcie-ep-intr", pcie);
2245 if (ret) {
2246 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2247 ret);
2248 goto fail;
2249 }
2250
2251 ret = tegra_pcie_config_ep(pcie, pdev);
2252 if (ret < 0)
2253 goto fail;
2254 break;
2255
2256 default:
2257 dev_err(dev, "Invalid PCIe device type %d\n",
2258 pcie->of_data->mode);
2259 }
2260
2261fail:
2262 tegra_bpmp_put(pcie->bpmp);
2263 return ret;
2264}
2265
2266static int tegra_pcie_dw_remove(struct platform_device *pdev)
2267{
2268 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2269
2270 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2271 if (!pcie->link_state)
2272 return 0;
2273
2274 debugfs_remove_recursive(pcie->debugfs);
2275 tegra_pcie_deinit_controller(pcie);
2276 pm_runtime_put_sync(pcie->dev);
2277 } else {
2278 disable_irq(pcie->pex_rst_irq);
2279 pex_ep_event_pex_rst_assert(pcie);
2280 }
2281
2282 pm_runtime_disable(pcie->dev);
2283 tegra_bpmp_put(pcie->bpmp);
2284 if (pcie->pex_refclk_sel_gpiod)
2285 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2286
2287 return 0;
2288}
2289
2290static int tegra_pcie_dw_suspend_late(struct device *dev)
2291{
2292 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2293 u32 val;
2294
2295 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2296 dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n");
2297 return -EPERM;
2298 }
2299
2300 if (!pcie->link_state)
2301 return 0;
2302
2303 /* Enable HW_HOT_RST mode */
2304 if (!pcie->of_data->has_sbr_reset_fix) {
2305 val = appl_readl(pcie, APPL_CTRL);
2306 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2307 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2308 val |= APPL_CTRL_HW_HOT_RST_EN;
2309 appl_writel(pcie, val, APPL_CTRL);
2310 }
2311
2312 return 0;
2313}
2314
2315static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2316{
2317 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2318
2319 if (!pcie->link_state)
2320 return 0;
2321
2322 tegra_pcie_downstream_dev_to_D0(pcie);
2323 tegra_pcie_dw_pme_turnoff(pcie);
2324 tegra_pcie_unconfig_controller(pcie);
2325
2326 return 0;
2327}
2328
2329static int tegra_pcie_dw_resume_noirq(struct device *dev)
2330{
2331 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2332 int ret;
2333
2334 if (!pcie->link_state)
2335 return 0;
2336
2337 ret = tegra_pcie_config_controller(pcie, true);
2338 if (ret < 0)
2339 return ret;
2340
2341 ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2342 if (ret < 0) {
2343 dev_err(dev, "Failed to init host: %d\n", ret);
2344 goto fail_host_init;
2345 }
2346
2347 dw_pcie_setup_rc(&pcie->pci.pp);
2348
2349 ret = tegra_pcie_dw_start_link(&pcie->pci);
2350 if (ret < 0)
2351 goto fail_host_init;
2352
2353 return 0;
2354
2355fail_host_init:
2356 tegra_pcie_unconfig_controller(pcie);
2357 return ret;
2358}
2359
2360static int tegra_pcie_dw_resume_early(struct device *dev)
2361{
2362 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2363 u32 val;
2364
2365 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2366 dev_err(dev, "Suspend is not supported in EP mode");
2367 return -ENOTSUPP;
2368 }
2369
2370 if (!pcie->link_state)
2371 return 0;
2372
2373 /* Disable HW_HOT_RST mode */
2374 if (!pcie->of_data->has_sbr_reset_fix) {
2375 val = appl_readl(pcie, APPL_CTRL);
2376 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2377 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2378 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2379 APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2380 val &= ~APPL_CTRL_HW_HOT_RST_EN;
2381 appl_writel(pcie, val, APPL_CTRL);
2382 }
2383
2384 return 0;
2385}
2386
2387static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2388{
2389 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2390
2391 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2392 if (!pcie->link_state)
2393 return;
2394
2395 debugfs_remove_recursive(pcie->debugfs);
2396 tegra_pcie_downstream_dev_to_D0(pcie);
2397
2398 disable_irq(pcie->pci.pp.irq);
2399 if (IS_ENABLED(CONFIG_PCI_MSI))
2400 disable_irq(pcie->pci.pp.msi_irq[0]);
2401
2402 tegra_pcie_dw_pme_turnoff(pcie);
2403 tegra_pcie_unconfig_controller(pcie);
2404 pm_runtime_put_sync(pcie->dev);
2405 } else {
2406 disable_irq(pcie->pex_rst_irq);
2407 pex_ep_event_pex_rst_assert(pcie);
2408 }
2409}
2410
2411static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = {
2412 .version = TEGRA194_DWC_IP_VER,
2413 .mode = DW_PCIE_RC_TYPE,
2414 .cdm_chk_int_en_bit = BIT(19),
2415 /* Gen4 - 5, 6, 8 and 9 presets enabled */
2416 .gen4_preset_vec = 0x360,
2417 .n_fts = { 52, 52 },
2418};
2419
2420static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = {
2421 .version = TEGRA194_DWC_IP_VER,
2422 .mode = DW_PCIE_EP_TYPE,
2423 .cdm_chk_int_en_bit = BIT(19),
2424 /* Gen4 - 5, 6, 8 and 9 presets enabled */
2425 .gen4_preset_vec = 0x360,
2426 .n_fts = { 52, 52 },
2427};
2428
2429static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = {
2430 .version = TEGRA234_DWC_IP_VER,
2431 .mode = DW_PCIE_RC_TYPE,
2432 .has_msix_doorbell_access_fix = true,
2433 .has_sbr_reset_fix = true,
2434 .has_l1ss_exit_fix = true,
2435 .cdm_chk_int_en_bit = BIT(18),
2436 /* Gen4 - 6, 8 and 9 presets enabled */
2437 .gen4_preset_vec = 0x340,
2438 .n_fts = { 52, 80 },
2439};
2440
2441static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = {
2442 .version = TEGRA234_DWC_IP_VER,
2443 .mode = DW_PCIE_EP_TYPE,
2444 .has_l1ss_exit_fix = true,
2445 .has_ltr_req_fix = true,
2446 .cdm_chk_int_en_bit = BIT(18),
2447 /* Gen4 - 6, 8 and 9 presets enabled */
2448 .gen4_preset_vec = 0x340,
2449 .n_fts = { 52, 80 },
2450};
2451
2452static const struct of_device_id tegra_pcie_dw_of_match[] = {
2453 {
2454 .compatible = "nvidia,tegra194-pcie",
2455 .data = &tegra194_pcie_dw_rc_of_data,
2456 },
2457 {
2458 .compatible = "nvidia,tegra194-pcie-ep",
2459 .data = &tegra194_pcie_dw_ep_of_data,
2460 },
2461 {
2462 .compatible = "nvidia,tegra234-pcie",
2463 .data = &tegra234_pcie_dw_rc_of_data,
2464 },
2465 {
2466 .compatible = "nvidia,tegra234-pcie-ep",
2467 .data = &tegra234_pcie_dw_ep_of_data,
2468 },
2469 {}
2470};
2471
2472static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2473 .suspend_late = tegra_pcie_dw_suspend_late,
2474 .suspend_noirq = tegra_pcie_dw_suspend_noirq,
2475 .resume_noirq = tegra_pcie_dw_resume_noirq,
2476 .resume_early = tegra_pcie_dw_resume_early,
2477};
2478
2479static struct platform_driver tegra_pcie_dw_driver = {
2480 .probe = tegra_pcie_dw_probe,
2481 .remove = tegra_pcie_dw_remove,
2482 .shutdown = tegra_pcie_dw_shutdown,
2483 .driver = {
2484 .name = "tegra194-pcie",
2485 .pm = &tegra_pcie_dw_pm_ops,
2486 .of_match_table = tegra_pcie_dw_of_match,
2487 },
2488};
2489module_platform_driver(tegra_pcie_dw_driver);
2490
2491MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2492
2493MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2494MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2495MODULE_LICENSE("GPL v2");