1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * PCIe host controller driver for Tegra194 SoC
   4 *
   5 * Copyright (C) 2019 NVIDIA Corporation.
   6 *
   7 * Author: Vidya Sagar <vidyas@nvidia.com>
   8 */
   9
  10#include <linux/clk.h>
  11#include <linux/debugfs.h>
  12#include <linux/delay.h>
  13#include <linux/gpio.h>
  14#include <linux/gpio/consumer.h>
  15#include <linux/interrupt.h>
  16#include <linux/iopoll.h>
  17#include <linux/kernel.h>
  18#include <linux/module.h>
  19#include <linux/of.h>
  20#include <linux/of_device.h>
  21#include <linux/of_gpio.h>
  22#include <linux/of_irq.h>
  23#include <linux/of_pci.h>
  24#include <linux/pci.h>
  25#include <linux/phy/phy.h>
  26#include <linux/pinctrl/consumer.h>
  27#include <linux/platform_device.h>
  28#include <linux/pm_runtime.h>
  29#include <linux/random.h>
  30#include <linux/reset.h>
  31#include <linux/resource.h>
  32#include <linux/types.h>
  33#include "pcie-designware.h"
  34#include <soc/tegra/bpmp.h>
  35#include <soc/tegra/bpmp-abi.h>
  36#include "../../pci.h"
  37
  38#define APPL_PINMUX				0x0
  39#define APPL_PINMUX_PEX_RST			BIT(0)
  40#define APPL_PINMUX_CLKREQ_OVERRIDE_EN		BIT(2)
  41#define APPL_PINMUX_CLKREQ_OVERRIDE		BIT(3)
  42#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN	BIT(4)
  43#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE	BIT(5)
 
 
  44
  45#define APPL_CTRL				0x4
  46#define APPL_CTRL_SYS_PRE_DET_STATE		BIT(6)
  47#define APPL_CTRL_LTSSM_EN			BIT(7)
  48#define APPL_CTRL_HW_HOT_RST_EN			BIT(20)
  49#define APPL_CTRL_HW_HOT_RST_MODE_MASK		GENMASK(1, 0)
  50#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT		22
  51#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST	0x1
  52
  53#define APPL_INTR_EN_L0_0			0x8
  54#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN	BIT(0)
  55#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN	BIT(4)
  56#define APPL_INTR_EN_L0_0_INT_INT_EN		BIT(8)
  57#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN	BIT(15)
  58#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN	BIT(19)
  59#define APPL_INTR_EN_L0_0_SYS_INTR_EN		BIT(30)
  60#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN	BIT(31)
  61
  62#define APPL_INTR_STATUS_L0			0xC
  63#define APPL_INTR_STATUS_L0_LINK_STATE_INT	BIT(0)
  64#define APPL_INTR_STATUS_L0_INT_INT		BIT(8)
  65#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT	BIT(15)
  66#define APPL_INTR_STATUS_L0_PEX_RST_INT		BIT(16)
  67#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT	BIT(18)
  68
  69#define APPL_INTR_EN_L1_0_0				0x1C
  70#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN	BIT(1)
  71#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN		BIT(3)
  72#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN	BIT(30)
  73
  74#define APPL_INTR_STATUS_L1_0_0				0x20
  75#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED	BIT(1)
  76#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED	BIT(3)
  77#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE		BIT(30)
  78
  79#define APPL_INTR_STATUS_L1_1			0x2C
  80#define APPL_INTR_STATUS_L1_2			0x30
  81#define APPL_INTR_STATUS_L1_3			0x34
  82#define APPL_INTR_STATUS_L1_6			0x3C
  83#define APPL_INTR_STATUS_L1_7			0x40
  84#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED	BIT(1)
  85
  86#define APPL_INTR_EN_L1_8_0			0x44
  87#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN		BIT(2)
  88#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN	BIT(3)
  89#define APPL_INTR_EN_L1_8_INTX_EN		BIT(11)
  90#define APPL_INTR_EN_L1_8_AER_INT_EN		BIT(15)
  91
  92#define APPL_INTR_STATUS_L1_8_0			0x4C
  93#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK	GENMASK(11, 6)
  94#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS	BIT(2)
  95#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS	BIT(3)
  96
  97#define APPL_INTR_STATUS_L1_9			0x54
  98#define APPL_INTR_STATUS_L1_10			0x58
  99#define APPL_INTR_STATUS_L1_11			0x64
 100#define APPL_INTR_STATUS_L1_13			0x74
 101#define APPL_INTR_STATUS_L1_14			0x78
 102#define APPL_INTR_STATUS_L1_15			0x7C
 103#define APPL_INTR_STATUS_L1_17			0x88
 104
 105#define APPL_INTR_EN_L1_18				0x90
 106#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT		BIT(2)
 107#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR		BIT(1)
 108#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
 109
 110#define APPL_INTR_STATUS_L1_18				0x94
 111#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT	BIT(2)
 112#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR	BIT(1)
 113#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
 114
 115#define APPL_MSI_CTRL_1				0xAC
 116
 117#define APPL_MSI_CTRL_2				0xB0
 118
 119#define APPL_LEGACY_INTX			0xB8
 120
 121#define APPL_LTR_MSG_1				0xC4
 122#define LTR_MSG_REQ				BIT(15)
 123#define LTR_MST_NO_SNOOP_SHIFT			16
 124
 125#define APPL_LTR_MSG_2				0xC8
 126#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE	BIT(3)
 127
 128#define APPL_LINK_STATUS			0xCC
 129#define APPL_LINK_STATUS_RDLH_LINK_UP		BIT(0)
 130
 131#define APPL_DEBUG				0xD0
 132#define APPL_DEBUG_PM_LINKST_IN_L2_LAT		BIT(21)
 133#define APPL_DEBUG_PM_LINKST_IN_L0		0x11
 134#define APPL_DEBUG_LTSSM_STATE_MASK		GENMASK(8, 3)
 135#define APPL_DEBUG_LTSSM_STATE_SHIFT		3
 136#define LTSSM_STATE_PRE_DETECT			5
 137
 138#define APPL_RADM_STATUS			0xE4
 139#define APPL_PM_XMT_TURNOFF_STATE		BIT(0)
 140
 141#define APPL_DM_TYPE				0x100
 142#define APPL_DM_TYPE_MASK			GENMASK(3, 0)
 143#define APPL_DM_TYPE_RP				0x4
 144#define APPL_DM_TYPE_EP				0x0
 145
 146#define APPL_CFG_BASE_ADDR			0x104
 147#define APPL_CFG_BASE_ADDR_MASK			GENMASK(31, 12)
 148
 149#define APPL_CFG_IATU_DMA_BASE_ADDR		0x108
 150#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK	GENMASK(31, 18)
 151
 152#define APPL_CFG_MISC				0x110
 153#define APPL_CFG_MISC_SLV_EP_MODE		BIT(14)
 154#define APPL_CFG_MISC_ARCACHE_MASK		GENMASK(13, 10)
 155#define APPL_CFG_MISC_ARCACHE_SHIFT		10
 156#define APPL_CFG_MISC_ARCACHE_VAL		3
 157
 158#define APPL_CFG_SLCG_OVERRIDE			0x114
 159#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER	BIT(0)
 160
 161#define APPL_CAR_RESET_OVRD				0x12C
 162#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N	BIT(0)
 163
 164#define IO_BASE_IO_DECODE				BIT(0)
 165#define IO_BASE_IO_DECODE_BIT8				BIT(8)
 166
 167#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE		BIT(0)
 168#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE	BIT(16)
 169
 170#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF	0x718
 171#define CFG_TIMER_CTRL_ACK_NAK_SHIFT	(19)
 172
 173#define EVENT_COUNTER_ALL_CLEAR		0x3
 174#define EVENT_COUNTER_ENABLE_ALL	0x7
 175#define EVENT_COUNTER_ENABLE_SHIFT	2
 176#define EVENT_COUNTER_EVENT_SEL_MASK	GENMASK(7, 0)
 177#define EVENT_COUNTER_EVENT_SEL_SHIFT	16
 178#define EVENT_COUNTER_EVENT_Tx_L0S	0x2
 179#define EVENT_COUNTER_EVENT_Rx_L0S	0x3
 180#define EVENT_COUNTER_EVENT_L1		0x5
 181#define EVENT_COUNTER_EVENT_L1_1	0x7
 182#define EVENT_COUNTER_EVENT_L1_2	0x8
 183#define EVENT_COUNTER_GROUP_SEL_SHIFT	24
 184#define EVENT_COUNTER_GROUP_5		0x5
 185
 186#define PORT_LOGIC_ACK_F_ASPM_CTRL			0x70C
 187#define ENTER_ASPM					BIT(30)
 188#define L0S_ENTRANCE_LAT_SHIFT				24
 189#define L0S_ENTRANCE_LAT_MASK				GENMASK(26, 24)
 190#define L1_ENTRANCE_LAT_SHIFT				27
 191#define L1_ENTRANCE_LAT_MASK				GENMASK(29, 27)
 192#define N_FTS_SHIFT					8
 193#define N_FTS_MASK					GENMASK(7, 0)
 194#define N_FTS_VAL					52
 195
 196#define PORT_LOGIC_GEN2_CTRL				0x80C
 197#define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE	BIT(17)
 198#define FTS_MASK					GENMASK(7, 0)
 199#define FTS_VAL						52
 200
 201#define PORT_LOGIC_MSI_CTRL_INT_0_EN		0x828
 202
 203#define GEN3_EQ_CONTROL_OFF			0x8a8
 204#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT	8
 205#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK	GENMASK(23, 8)
 206#define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK	GENMASK(3, 0)
 207
 208#define GEN3_RELATED_OFF			0x890
 209#define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL	BIT(0)
 210#define GEN3_RELATED_OFF_GEN3_EQ_DISABLE	BIT(16)
 211#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT	24
 212#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK	GENMASK(25, 24)
 213
 214#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT	0x8D0
 215#define AMBA_ERROR_RESPONSE_CRS_SHIFT		3
 216#define AMBA_ERROR_RESPONSE_CRS_MASK		GENMASK(1, 0)
 217#define AMBA_ERROR_RESPONSE_CRS_OKAY		0
 218#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF	1
 219#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001	2
 220
 221#define MSIX_ADDR_MATCH_LOW_OFF			0x940
 222#define MSIX_ADDR_MATCH_LOW_OFF_EN		BIT(0)
 223#define MSIX_ADDR_MATCH_LOW_OFF_MASK		GENMASK(31, 2)
 224
 225#define MSIX_ADDR_MATCH_HIGH_OFF		0x944
 226#define MSIX_ADDR_MATCH_HIGH_OFF_MASK		GENMASK(31, 0)
 227
 228#define PORT_LOGIC_MSIX_DOORBELL			0x948
 229
 230#define CAP_SPCIE_CAP_OFF			0x154
 231#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK	GENMASK(3, 0)
 232#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK	GENMASK(11, 8)
 233#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT	8
 234
 235#define PME_ACK_TIMEOUT 10000
 236
 237#define LTSSM_TIMEOUT 50000	/* 50ms */
 238
 239#define GEN3_GEN4_EQ_PRESET_INIT	5
 240
 241#define GEN1_CORE_CLK_FREQ	62500000
 242#define GEN2_CORE_CLK_FREQ	125000000
 243#define GEN3_CORE_CLK_FREQ	250000000
 244#define GEN4_CORE_CLK_FREQ	500000000
 245
 246#define LTR_MSG_TIMEOUT		(100 * 1000)
 247
 248#define PERST_DEBOUNCE_TIME	(5 * 1000)
 249
 250#define EP_STATE_DISABLED	0
 251#define EP_STATE_ENABLED	1
 252
 253static const unsigned int pcie_gen_freq[] = {
 254	GEN1_CORE_CLK_FREQ,
 255	GEN2_CORE_CLK_FREQ,
 256	GEN3_CORE_CLK_FREQ,
 257	GEN4_CORE_CLK_FREQ
 258};
 259
 260static const u32 event_cntr_ctrl_offset[] = {
 261	0x1d8,
 262	0x1a8,
 263	0x1a8,
 264	0x1a8,
 265	0x1c4,
 266	0x1d8
 267};
 268
 269static const u32 event_cntr_data_offset[] = {
 270	0x1dc,
 271	0x1ac,
 272	0x1ac,
 273	0x1ac,
 274	0x1c8,
 275	0x1dc
 276};
 277
 278struct tegra_pcie_dw {
 279	struct device *dev;
 280	struct resource *appl_res;
 281	struct resource *dbi_res;
 282	struct resource *atu_dma_res;
 283	void __iomem *appl_base;
 284	struct clk *core_clk;
 285	struct reset_control *core_apb_rst;
 286	struct reset_control *core_rst;
 287	struct dw_pcie pci;
 288	struct tegra_bpmp *bpmp;
 289
 290	enum dw_pcie_device_mode mode;
 291
 292	bool supports_clkreq;
 293	bool enable_cdm_check;
 294	bool link_state;
 295	bool update_fc_fixup;
 296	u8 init_link_width;
 297	u32 msi_ctrl_int;
 298	u32 num_lanes;
 299	u32 max_speed;
 300	u32 cid;
 301	u32 cfg_link_cap_l1sub;
 302	u32 pcie_cap_base;
 303	u32 aspm_cmrt;
 304	u32 aspm_pwr_on_t;
 305	u32 aspm_l0s_enter_lat;
 306
 307	struct regulator *pex_ctl_supply;
 308	struct regulator *slot_ctl_3v3;
 309	struct regulator *slot_ctl_12v;
 310
 311	unsigned int phy_count;
 312	struct phy **phys;
 313
 314	struct dentry *debugfs;
 315
 316	/* Endpoint mode specific */
 317	struct gpio_desc *pex_rst_gpiod;
 318	struct gpio_desc *pex_refclk_sel_gpiod;
 319	unsigned int pex_rst_irq;
 320	int ep_state;
 321};
 322
 323struct tegra_pcie_dw_of_data {
 324	enum dw_pcie_device_mode mode;
 325};
 326
 327static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
 328{
 329	return container_of(pci, struct tegra_pcie_dw, pci);
 330}
 331
 332static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
 333			       const u32 reg)
 334{
 335	writel_relaxed(value, pcie->appl_base + reg);
 336}
 337
 338static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
 339{
 340	return readl_relaxed(pcie->appl_base + reg);
 341}
 342
 343struct tegra_pcie_soc {
 344	enum dw_pcie_device_mode mode;
 345};
 346
 347static void apply_bad_link_workaround(struct pcie_port *pp)
 348{
 349	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 350	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 351	u32 current_link_width;
 352	u16 val;
 353
 354	/*
 355	 * NOTE:- Since this scenario is uncommon and link as such is not
 356	 * stable anyway, not waiting to confirm if link is really
 357	 * transitioning to Gen-2 speed
 358	 */
 359	val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
 360	if (val & PCI_EXP_LNKSTA_LBMS) {
 361		current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
 362				     PCI_EXP_LNKSTA_NLW_SHIFT;
 363		if (pcie->init_link_width > current_link_width) {
 364			dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
 365			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
 366						PCI_EXP_LNKCTL2);
 367			val &= ~PCI_EXP_LNKCTL2_TLS;
 368			val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
 369			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
 370					   PCI_EXP_LNKCTL2, val);
 371
 372			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
 373						PCI_EXP_LNKCTL);
 374			val |= PCI_EXP_LNKCTL_RL;
 375			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
 376					   PCI_EXP_LNKCTL, val);
 377		}
 378	}
 379}
 380
 381static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
 382{
 383	struct tegra_pcie_dw *pcie = arg;
 384	struct dw_pcie *pci = &pcie->pci;
 385	struct pcie_port *pp = &pci->pp;
 386	u32 val, tmp;
 387	u16 val_w;
 388
 389	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
 390	if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
 391		val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
 392		if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
 393			appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
 394
 395			/* SBR & Surprise Link Down WAR */
 396			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
 397			val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
 398			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
 399			udelay(1);
 400			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
 401			val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
 402			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
 403
 404			val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
 405			val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE;
 406			dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
 407		}
 408	}
 409
 410	if (val & APPL_INTR_STATUS_L0_INT_INT) {
 411		val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
 412		if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
 413			appl_writel(pcie,
 414				    APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
 415				    APPL_INTR_STATUS_L1_8_0);
 416			apply_bad_link_workaround(pp);
 417		}
 418		if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
 419			appl_writel(pcie,
 420				    APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
 421				    APPL_INTR_STATUS_L1_8_0);
 422
 423			val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
 424						  PCI_EXP_LNKSTA);
 425			dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
 426				PCI_EXP_LNKSTA_CLS);
 427		}
 428	}
 429
 430	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
 431	if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
 432		val = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
 433		tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
 434		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
 435			dev_info(pci->dev, "CDM check complete\n");
 436			tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
 437		}
 438		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
 439			dev_err(pci->dev, "CDM comparison mismatch\n");
 440			tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
 441		}
 442		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
 443			dev_err(pci->dev, "CDM Logic error\n");
 444			tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
 445		}
 446		dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp);
 447		tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
 448		dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp);
 449	}
 450
 451	return IRQ_HANDLED;
 452}
 453
 454static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
 455{
 456	u32 val;
 457
 458	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
 459	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
 460	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
 461	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
 462	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
 463	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
 464	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
 465	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
 466	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
 467	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
 468	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
 469	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
 470	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
 471	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
 472	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
 473	appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
 474
 475	val = appl_readl(pcie, APPL_CTRL);
 476	val |= APPL_CTRL_LTSSM_EN;
 477	appl_writel(pcie, val, APPL_CTRL);
 478}
 479
 480static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
 481{
 482	struct tegra_pcie_dw *pcie = arg;
 483	struct dw_pcie *pci = &pcie->pci;
 484	u32 val, speed;
 485
 486	speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
 487		PCI_EXP_LNKSTA_CLS;
 488	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
 489
 490	/* If EP doesn't advertise L1SS, just return */
 491	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
 492	if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
 493		return IRQ_HANDLED;
 494
 495	/* Check if BME is set to '1' */
 496	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
 497	if (val & PCI_COMMAND_MASTER) {
 498		ktime_t timeout;
 499
 500		/* 110us for both snoop and no-snoop */
 501		val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ;
 502		val |= (val << LTR_MST_NO_SNOOP_SHIFT);
 503		appl_writel(pcie, val, APPL_LTR_MSG_1);
 504
 505		/* Send LTR upstream */
 506		val = appl_readl(pcie, APPL_LTR_MSG_2);
 507		val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
 508		appl_writel(pcie, val, APPL_LTR_MSG_2);
 509
 510		timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
 511		for (;;) {
 512			val = appl_readl(pcie, APPL_LTR_MSG_2);
 513			if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
 514				break;
 515			if (ktime_after(ktime_get(), timeout))
 516				break;
 517			usleep_range(1000, 1100);
 518		}
 519		if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
 520			dev_err(pcie->dev, "Failed to send LTR message\n");
 521	}
 522
 523	return IRQ_HANDLED;
 524}
 525
 526static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
 527{
 528	struct tegra_pcie_dw *pcie = arg;
 529	struct dw_pcie_ep *ep = &pcie->pci.ep;
 530	int spurious = 1;
 531	u32 val, tmp;
 532
 533	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
 534	if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
 535		val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
 536		appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
 537
 538		if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
 539			pex_ep_event_hot_rst_done(pcie);
 540
 541		if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
 542			tmp = appl_readl(pcie, APPL_LINK_STATUS);
 543			if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) {
 544				dev_dbg(pcie->dev, "Link is up with Host\n");
 545				dw_pcie_ep_linkup(ep);
 546			}
 547		}
 548
 549		spurious = 0;
 550	}
 551
 552	if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
 553		val = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
 554		appl_writel(pcie, val, APPL_INTR_STATUS_L1_15);
 555
 556		if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
 557			return IRQ_WAKE_THREAD;
 558
 559		spurious = 0;
 560	}
 561
 562	if (spurious) {
 563		dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
 564			 val);
 565		appl_writel(pcie, val, APPL_INTR_STATUS_L0);
 566	}
 567
 568	return IRQ_HANDLED;
 569}
 570
 571static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size,
 572				     u32 *val)
 573{
 574	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 575
 576	/*
 577	 * This is an endpoint mode specific register happen to appear even
 578	 * when controller is operating in root port mode and system hangs
 579	 * when it is accessed with link being in ASPM-L1 state.
 580	 * So skip accessing it altogether
 581	 */
 582	if (where == PORT_LOGIC_MSIX_DOORBELL) {
 583		*val = 0x00000000;
 584		return PCIBIOS_SUCCESSFUL;
 585	}
 586
 587	return dw_pcie_read(pci->dbi_base + where, size, val);
 588}
 589
 590static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size,
 591				     u32 val)
 592{
 593	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 594
 595	/*
 596	 * This is an endpoint mode specific register happen to appear even
 597	 * when controller is operating in root port mode and system hangs
 598	 * when it is accessed with link being in ASPM-L1 state.
 599	 * So skip accessing it altogether
 600	 */
 601	if (where == PORT_LOGIC_MSIX_DOORBELL)
 602		return PCIBIOS_SUCCESSFUL;
 603
 604	return dw_pcie_write(pci->dbi_base + where, size, val);
 605}
 606
 607#if defined(CONFIG_PCIEASPM)
 608static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
 609{
 610	u32 val;
 611
 612	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
 613	val &= ~PCI_L1SS_CAP_ASPM_L1_1;
 614	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
 615}
 616
 617static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
 618{
 619	u32 val;
 620
 621	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
 622	val &= ~PCI_L1SS_CAP_ASPM_L1_2;
 623	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
 624}
 625
 626static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
 627{
 628	u32 val;
 629
 630	val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]);
 631	val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
 632	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
 633	val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
 634	val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
 635	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
 636	val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]);
 637
 638	return val;
 639}
 640
 641static int aspm_state_cnt(struct seq_file *s, void *data)
 642{
 643	struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
 644				     dev_get_drvdata(s->private);
 645	u32 val;
 646
 647	seq_printf(s, "Tx L0s entry count : %u\n",
 648		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
 649
 650	seq_printf(s, "Rx L0s entry count : %u\n",
 651		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
 652
 653	seq_printf(s, "Link L1 entry count : %u\n",
 654		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
 655
 656	seq_printf(s, "Link L1.1 entry count : %u\n",
 657		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
 658
 659	seq_printf(s, "Link L1.2 entry count : %u\n",
 660		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
 661
 662	/* Clear all counters */
 663	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid],
 664			   EVENT_COUNTER_ALL_CLEAR);
 665
 666	/* Re-enable counting */
 667	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
 668	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
 669	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
 670
 671	return 0;
 672}
 673
 674static void init_host_aspm(struct tegra_pcie_dw *pcie)
 675{
 676	struct dw_pcie *pci = &pcie->pci;
 677	u32 val;
 678
 679	val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
 680	pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
 681
 682	/* Enable ASPM counters */
 683	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
 684	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
 685	dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val);
 686
 687	/* Program T_cmrt and T_pwr_on values */
 688	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
 689	val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
 690	val |= (pcie->aspm_cmrt << 8);
 691	val |= (pcie->aspm_pwr_on_t << 19);
 692	dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
 693
 694	/* Program L0s and L1 entrance latencies */
 695	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
 696	val &= ~L0S_ENTRANCE_LAT_MASK;
 697	val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT);
 698	val |= ENTER_ASPM;
 699	dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
 700}
 701
 702static int init_debugfs(struct tegra_pcie_dw *pcie)
 703{
 704	struct dentry *d;
 705
 706	d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt",
 707					pcie->debugfs, aspm_state_cnt);
 708	if (IS_ERR_OR_NULL(d))
 709		dev_err(pcie->dev,
 710			"Failed to create debugfs file \"aspm_state_cnt\"\n");
 711
 712	return 0;
 713}
 714#else
 715static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
 716static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
 717static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
 718static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; }
 719#endif
 720
 721static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp)
 722{
 723	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 724	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 725	u32 val;
 726	u16 val_w;
 727
 728	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
 729	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
 730	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
 731
 732	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
 733	val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
 734	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
 735
 736	if (pcie->enable_cdm_check) {
 737		val = appl_readl(pcie, APPL_INTR_EN_L0_0);
 738		val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN;
 739		appl_writel(pcie, val, APPL_INTR_EN_L0_0);
 740
 741		val = appl_readl(pcie, APPL_INTR_EN_L1_18);
 742		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
 743		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
 744		appl_writel(pcie, val, APPL_INTR_EN_L1_18);
 745	}
 746
 747	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
 748				  PCI_EXP_LNKSTA);
 749	pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
 750				PCI_EXP_LNKSTA_NLW_SHIFT;
 751
 752	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
 753				  PCI_EXP_LNKCTL);
 754	val_w |= PCI_EXP_LNKCTL_LBMIE;
 755	dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
 756			   val_w);
 757}
 758
 759static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp)
 760{
 761	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 762	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 763	u32 val;
 764
 765	/* Enable legacy interrupt generation */
 766	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
 767	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
 768	val |= APPL_INTR_EN_L0_0_INT_INT_EN;
 769	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
 770
 771	val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
 772	val |= APPL_INTR_EN_L1_8_INTX_EN;
 773	val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
 774	val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
 775	if (IS_ENABLED(CONFIG_PCIEAER))
 776		val |= APPL_INTR_EN_L1_8_AER_INT_EN;
 777	appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
 778}
 779
 780static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp)
 781{
 782	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 783	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 784	u32 val;
 785
 786	dw_pcie_msi_init(pp);
 787
 788	/* Enable MSI interrupt generation */
 789	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
 790	val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
 791	val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
 792	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
 793}
 794
 795static void tegra_pcie_enable_interrupts(struct pcie_port *pp)
 796{
 797	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 798	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 799
 800	/* Clear interrupt statuses before enabling interrupts */
 801	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
 802	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
 803	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
 804	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
 805	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
 806	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
 807	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
 808	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
 809	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
 810	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
 811	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
 812	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
 813	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
 814	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
 815	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
 816
 817	tegra_pcie_enable_system_interrupts(pp);
 818	tegra_pcie_enable_legacy_interrupts(pp);
 819	if (IS_ENABLED(CONFIG_PCI_MSI))
 820		tegra_pcie_enable_msi_interrupts(pp);
 821}
 822
 823static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
 824{
 825	struct dw_pcie *pci = &pcie->pci;
 826	u32 val, offset, i;
 827
 828	/* Program init preset */
 829	for (i = 0; i < pcie->num_lanes; i++) {
 830		dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF
 831				 + (i * 2), 2, &val);
 832		val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
 833		val |= GEN3_GEN4_EQ_PRESET_INIT;
 834		val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
 835		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
 836			   CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
 837		dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF
 838				 + (i * 2), 2, val);
 839
 840		offset = dw_pcie_find_ext_capability(pci,
 841						     PCI_EXT_CAP_ID_PL_16GT) +
 842				PCI_PL_16GT_LE_CTRL;
 843		dw_pcie_read(pci->dbi_base + offset + i, 1, &val);
 844		val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
 845		val |= GEN3_GEN4_EQ_PRESET_INIT;
 846		val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
 847		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
 848			PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
 849		dw_pcie_write(pci->dbi_base + offset + i, 1, val);
 850	}
 851
 852	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
 853	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
 854	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 855
 856	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
 857	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
 858	val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
 859	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
 860	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
 861
 862	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
 863	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
 864	val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
 865	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 866
 867	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
 868	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
 869	val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
 870	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
 871	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
 872
 873	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
 874	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
 875	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 876}
 877
 878static void tegra_pcie_prepare_host(struct pcie_port *pp)
 879{
 880	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 881	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 882	u32 val;
 883
 884	val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
 885	val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
 886	dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
 887
 888	val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
 889	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
 890	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
 891	dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
 892
 893	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
 894
 895	/* Configure FTS */
 896	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
 897	val &= ~(N_FTS_MASK << N_FTS_SHIFT);
 898	val |= N_FTS_VAL << N_FTS_SHIFT;
 899	dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
 900
 901	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
 902	val &= ~FTS_MASK;
 903	val |= FTS_VAL;
 904	dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
 905
 906	/* Enable as 0xFFFF0001 response for CRS */
 907	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
 908	val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
 909	val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
 910		AMBA_ERROR_RESPONSE_CRS_SHIFT);
 911	dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
 912
 913	/* Configure Max Speed from DT */
 914	if (pcie->max_speed && pcie->max_speed != -EINVAL) {
 915		val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base +
 916					PCI_EXP_LNKCAP);
 917		val &= ~PCI_EXP_LNKCAP_SLS;
 918		val |= pcie->max_speed;
 919		dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP,
 920				   val);
 921	}
 922
 923	/* Configure Max lane width from DT */
 924	val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
 925	val &= ~PCI_EXP_LNKCAP_MLW;
 926	val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
 927	dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
 928
 929	config_gen3_gen4_eq_presets(pcie);
 930
 931	init_host_aspm(pcie);
 932
 933	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
 934	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
 935	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 936
 937	if (pcie->update_fc_fixup) {
 938		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
 939		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
 940		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
 941	}
 942
 943	dw_pcie_setup_rc(pp);
 944
 945	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
 946
 947	/* Assert RST */
 948	val = appl_readl(pcie, APPL_PINMUX);
 949	val &= ~APPL_PINMUX_PEX_RST;
 950	appl_writel(pcie, val, APPL_PINMUX);
 951
 952	usleep_range(100, 200);
 953
 954	/* Enable LTSSM */
 955	val = appl_readl(pcie, APPL_CTRL);
 956	val |= APPL_CTRL_LTSSM_EN;
 957	appl_writel(pcie, val, APPL_CTRL);
 958
 959	/* De-assert RST */
 960	val = appl_readl(pcie, APPL_PINMUX);
 961	val |= APPL_PINMUX_PEX_RST;
 962	appl_writel(pcie, val, APPL_PINMUX);
 963
 964	msleep(100);
 965}
 966
 967static int tegra_pcie_dw_host_init(struct pcie_port *pp)
 968{
 969	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 970	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 971	u32 val, tmp, offset, speed;
 972
 973	tegra_pcie_prepare_host(pp);
 974
 975	if (dw_pcie_wait_for_link(pci)) {
 976		/*
 977		 * There are some endpoints which can't get the link up if
 978		 * root port has Data Link Feature (DLF) enabled.
 979		 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
 980		 * on Scaled Flow Control and DLF.
 981		 * So, need to confirm that is indeed the case here and attempt
 982		 * link up once again with DLF disabled.
 983		 */
 984		val = appl_readl(pcie, APPL_DEBUG);
 985		val &= APPL_DEBUG_LTSSM_STATE_MASK;
 986		val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
 987		tmp = appl_readl(pcie, APPL_LINK_STATUS);
 988		tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
 989		if (!(val == 0x11 && !tmp)) {
 990			/* Link is down for all good reasons */
 991			return 0;
 992		}
 993
 994		dev_info(pci->dev, "Link is down in DLL");
 995		dev_info(pci->dev, "Trying again with DLFE disabled\n");
 996		/* Disable LTSSM */
 997		val = appl_readl(pcie, APPL_CTRL);
 998		val &= ~APPL_CTRL_LTSSM_EN;
 999		appl_writel(pcie, val, APPL_CTRL);
1000
1001		reset_control_assert(pcie->core_rst);
1002		reset_control_deassert(pcie->core_rst);
1003
1004		offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1005		val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1006		val &= ~PCI_DLF_EXCHANGE_ENABLE;
1007		dw_pcie_writel_dbi(pci, offset, val);
1008
1009		tegra_pcie_prepare_host(pp);
1010
1011		if (dw_pcie_wait_for_link(pci))
1012			return 0;
1013	}
1014
1015	speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
1016		PCI_EXP_LNKSTA_CLS;
1017	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
1018
1019	tegra_pcie_enable_interrupts(pp);
1020
1021	return 0;
1022}
1023
1024static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1025{
1026	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1027	u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1028
1029	return !!(val & PCI_EXP_LNKSTA_DLLLA);
1030}
1031
1032static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp)
1033{
1034	pp->num_vectors = MAX_MSI_IRQS;
1035}
1036
1037static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
1038{
1039	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1040
1041	enable_irq(pcie->pex_rst_irq);
1042
1043	return 0;
1044}
1045
1046static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1047{
1048	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1049
1050	disable_irq(pcie->pex_rst_irq);
1051}
1052
1053static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1054	.link_up = tegra_pcie_dw_link_up,
1055	.start_link = tegra_pcie_dw_start_link,
1056	.stop_link = tegra_pcie_dw_stop_link,
1057};
1058
1059static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1060	.rd_own_conf = tegra_pcie_dw_rd_own_conf,
1061	.wr_own_conf = tegra_pcie_dw_wr_own_conf,
1062	.host_init = tegra_pcie_dw_host_init,
1063	.set_num_vectors = tegra_pcie_set_msi_vec_num,
1064};
1065
1066static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1067{
1068	unsigned int phy_count = pcie->phy_count;
1069
1070	while (phy_count--) {
1071		phy_power_off(pcie->phys[phy_count]);
1072		phy_exit(pcie->phys[phy_count]);
1073	}
1074}
1075
1076static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1077{
1078	unsigned int i;
1079	int ret;
1080
1081	for (i = 0; i < pcie->phy_count; i++) {
1082		ret = phy_init(pcie->phys[i]);
1083		if (ret < 0)
1084			goto phy_power_off;
1085
1086		ret = phy_power_on(pcie->phys[i]);
1087		if (ret < 0)
1088			goto phy_exit;
1089	}
1090
1091	return 0;
1092
1093phy_power_off:
1094	while (i--) {
1095		phy_power_off(pcie->phys[i]);
1096phy_exit:
1097		phy_exit(pcie->phys[i]);
1098	}
1099
1100	return ret;
1101}
1102
1103static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1104{
1105	struct device_node *np = pcie->dev->of_node;
1106	int ret;
1107
1108	ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1109	if (ret < 0) {
1110		dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1111		return ret;
1112	}
1113
1114	ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1115				   &pcie->aspm_pwr_on_t);
1116	if (ret < 0)
1117		dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1118			 ret);
1119
1120	ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1121				   &pcie->aspm_l0s_enter_lat);
1122	if (ret < 0)
1123		dev_info(pcie->dev,
1124			 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1125
1126	ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1127	if (ret < 0) {
1128		dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1129		return ret;
1130	}
1131
1132	pcie->max_speed = of_pci_get_max_link_speed(np);
1133
1134	ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1135	if (ret) {
1136		dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1137		return ret;
1138	}
1139
1140	ret = of_property_count_strings(np, "phy-names");
1141	if (ret < 0) {
1142		dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1143			ret);
1144		return ret;
1145	}
1146	pcie->phy_count = ret;
1147
1148	if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1149		pcie->update_fc_fixup = true;
1150
1151	pcie->supports_clkreq =
1152		of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1153
1154	pcie->enable_cdm_check =
1155		of_property_read_bool(np, "snps,enable-cdm-check");
1156
1157	if (pcie->mode == DW_PCIE_RC_TYPE)
1158		return 0;
1159
1160	/* Endpoint mode specific DT entries */
1161	pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1162	if (IS_ERR(pcie->pex_rst_gpiod)) {
1163		int err = PTR_ERR(pcie->pex_rst_gpiod);
1164		const char *level = KERN_ERR;
1165
1166		if (err == -EPROBE_DEFER)
1167			level = KERN_DEBUG;
1168
1169		dev_printk(level, pcie->dev,
1170			   dev_fmt("Failed to get PERST GPIO: %d\n"),
1171			   err);
1172		return err;
1173	}
1174
1175	pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1176						    "nvidia,refclk-select",
1177						    GPIOD_OUT_HIGH);
1178	if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1179		int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1180		const char *level = KERN_ERR;
1181
1182		if (err == -EPROBE_DEFER)
1183			level = KERN_DEBUG;
1184
1185		dev_printk(level, pcie->dev,
1186			   dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1187			   err);
1188		pcie->pex_refclk_sel_gpiod = NULL;
1189	}
1190
1191	return 0;
1192}
1193
1194static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1195					  bool enable)
1196{
1197	struct mrq_uphy_response resp;
1198	struct tegra_bpmp_message msg;
1199	struct mrq_uphy_request req;
1200
1201	/* Controller-5 doesn't need to have its state set by BPMP-FW */
1202	if (pcie->cid == 5)
1203		return 0;
1204
1205	memset(&req, 0, sizeof(req));
1206	memset(&resp, 0, sizeof(resp));
1207
1208	req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1209	req.controller_state.pcie_controller = pcie->cid;
1210	req.controller_state.enable = enable;
1211
1212	memset(&msg, 0, sizeof(msg));
1213	msg.mrq = MRQ_UPHY;
1214	msg.tx.data = &req;
1215	msg.tx.size = sizeof(req);
1216	msg.rx.data = &resp;
1217	msg.rx.size = sizeof(resp);
1218
1219	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1220}
1221
1222static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1223					 bool enable)
1224{
1225	struct mrq_uphy_response resp;
1226	struct tegra_bpmp_message msg;
1227	struct mrq_uphy_request req;
1228
1229	memset(&req, 0, sizeof(req));
1230	memset(&resp, 0, sizeof(resp));
1231
1232	if (enable) {
1233		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1234		req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1235	} else {
1236		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1237		req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1238	}
1239
1240	memset(&msg, 0, sizeof(msg));
1241	msg.mrq = MRQ_UPHY;
1242	msg.tx.data = &req;
1243	msg.tx.size = sizeof(req);
1244	msg.rx.data = &resp;
1245	msg.rx.size = sizeof(resp);
1246
1247	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1248}
1249
1250static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1251{
1252	struct pcie_port *pp = &pcie->pci.pp;
1253	struct pci_bus *child, *root_bus = NULL;
1254	struct pci_dev *pdev;
1255
1256	/*
1257	 * link doesn't go into L2 state with some of the endpoints with Tegra
1258	 * if they are not in D0 state. So, need to make sure that immediate
1259	 * downstream devices are in D0 state before sending PME_TurnOff to put
1260	 * link into L2 state.
1261	 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1262	 * 5.2 Link State Power Management (Page #428).
1263	 */
1264
1265	list_for_each_entry(child, &pp->root_bus->children, node) {
1266		/* Bring downstream devices to D0 if they are not already in */
1267		if (child->parent == pp->root_bus) {
1268			root_bus = child;
1269			break;
1270		}
1271	}
1272
1273	if (!root_bus) {
1274		dev_err(pcie->dev, "Failed to find downstream devices\n");
1275		return;
1276	}
1277
1278	list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1279		if (PCI_SLOT(pdev->devfn) == 0) {
1280			if (pci_set_power_state(pdev, PCI_D0))
1281				dev_err(pcie->dev,
1282					"Failed to transition %s to D0 state\n",
1283					dev_name(&pdev->dev));
1284		}
1285	}
1286}
1287
1288static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1289{
1290	pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1291	if (IS_ERR(pcie->slot_ctl_3v3)) {
1292		if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1293			return PTR_ERR(pcie->slot_ctl_3v3);
1294
1295		pcie->slot_ctl_3v3 = NULL;
1296	}
1297
1298	pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1299	if (IS_ERR(pcie->slot_ctl_12v)) {
1300		if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1301			return PTR_ERR(pcie->slot_ctl_12v);
1302
1303		pcie->slot_ctl_12v = NULL;
1304	}
1305
1306	return 0;
1307}
1308
1309static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1310{
1311	int ret;
1312
1313	if (pcie->slot_ctl_3v3) {
1314		ret = regulator_enable(pcie->slot_ctl_3v3);
1315		if (ret < 0) {
1316			dev_err(pcie->dev,
1317				"Failed to enable 3.3V slot supply: %d\n", ret);
1318			return ret;
1319		}
1320	}
1321
1322	if (pcie->slot_ctl_12v) {
1323		ret = regulator_enable(pcie->slot_ctl_12v);
1324		if (ret < 0) {
1325			dev_err(pcie->dev,
1326				"Failed to enable 12V slot supply: %d\n", ret);
1327			goto fail_12v_enable;
1328		}
1329	}
1330
1331	/*
1332	 * According to PCI Express Card Electromechanical Specification
1333	 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1334	 * should be a minimum of 100ms.
1335	 */
1336	if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1337		msleep(100);
1338
1339	return 0;
1340
1341fail_12v_enable:
1342	if (pcie->slot_ctl_3v3)
1343		regulator_disable(pcie->slot_ctl_3v3);
1344	return ret;
1345}
1346
1347static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1348{
1349	if (pcie->slot_ctl_12v)
1350		regulator_disable(pcie->slot_ctl_12v);
1351	if (pcie->slot_ctl_3v3)
1352		regulator_disable(pcie->slot_ctl_3v3);
1353}
1354
1355static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1356					bool en_hw_hot_rst)
1357{
1358	int ret;
1359	u32 val;
1360
1361	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1362	if (ret) {
1363		dev_err(pcie->dev,
1364			"Failed to enable controller %u: %d\n", pcie->cid, ret);
1365		return ret;
1366	}
1367
1368	ret = tegra_pcie_enable_slot_regulators(pcie);
1369	if (ret < 0)
1370		goto fail_slot_reg_en;
1371
1372	ret = regulator_enable(pcie->pex_ctl_supply);
1373	if (ret < 0) {
1374		dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1375		goto fail_reg_en;
1376	}
1377
1378	ret = clk_prepare_enable(pcie->core_clk);
1379	if (ret) {
1380		dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1381		goto fail_core_clk;
1382	}
1383
1384	ret = reset_control_deassert(pcie->core_apb_rst);
1385	if (ret) {
1386		dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1387			ret);
1388		goto fail_core_apb_rst;
1389	}
1390
1391	if (en_hw_hot_rst) {
1392		/* Enable HW_HOT_RST mode */
1393		val = appl_readl(pcie, APPL_CTRL);
1394		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1395			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1396		val |= APPL_CTRL_HW_HOT_RST_EN;
1397		appl_writel(pcie, val, APPL_CTRL);
1398	}
1399
1400	ret = tegra_pcie_enable_phy(pcie);
1401	if (ret) {
1402		dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1403		goto fail_phy;
1404	}
1405
1406	/* Update CFG base address */
1407	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1408		    APPL_CFG_BASE_ADDR);
1409
1410	/* Configure this core for RP mode operation */
1411	appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1412
1413	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1414
1415	val = appl_readl(pcie, APPL_CTRL);
1416	appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1417
1418	val = appl_readl(pcie, APPL_CFG_MISC);
1419	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1420	appl_writel(pcie, val, APPL_CFG_MISC);
1421
1422	if (!pcie->supports_clkreq) {
1423		val = appl_readl(pcie, APPL_PINMUX);
1424		val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1425		val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1426		appl_writel(pcie, val, APPL_PINMUX);
1427	}
1428
1429	/* Update iATU_DMA base address */
1430	appl_writel(pcie,
1431		    pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1432		    APPL_CFG_IATU_DMA_BASE_ADDR);
1433
1434	reset_control_deassert(pcie->core_rst);
1435
1436	pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1437						      PCI_CAP_ID_EXP);
1438
1439	/* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */
1440	if (!pcie->supports_clkreq) {
1441		disable_aspm_l11(pcie);
1442		disable_aspm_l12(pcie);
1443	}
1444
1445	return ret;
1446
1447fail_phy:
1448	reset_control_assert(pcie->core_apb_rst);
1449fail_core_apb_rst:
1450	clk_disable_unprepare(pcie->core_clk);
1451fail_core_clk:
1452	regulator_disable(pcie->pex_ctl_supply);
1453fail_reg_en:
1454	tegra_pcie_disable_slot_regulators(pcie);
1455fail_slot_reg_en:
1456	tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1457
1458	return ret;
1459}
1460
1461static int __deinit_controller(struct tegra_pcie_dw *pcie)
1462{
1463	int ret;
1464
1465	ret = reset_control_assert(pcie->core_rst);
1466	if (ret) {
1467		dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n",
1468			ret);
1469		return ret;
1470	}
1471
1472	tegra_pcie_disable_phy(pcie);
1473
1474	ret = reset_control_assert(pcie->core_apb_rst);
1475	if (ret) {
1476		dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1477		return ret;
1478	}
1479
1480	clk_disable_unprepare(pcie->core_clk);
1481
1482	ret = regulator_disable(pcie->pex_ctl_supply);
1483	if (ret) {
1484		dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1485		return ret;
1486	}
1487
1488	tegra_pcie_disable_slot_regulators(pcie);
1489
1490	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1491	if (ret) {
1492		dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1493			pcie->cid, ret);
1494		return ret;
1495	}
1496
1497	return ret;
1498}
1499
1500static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1501{
1502	struct dw_pcie *pci = &pcie->pci;
1503	struct pcie_port *pp = &pci->pp;
1504	int ret;
1505
1506	ret = tegra_pcie_config_controller(pcie, false);
1507	if (ret < 0)
1508		return ret;
1509
1510	pp->ops = &tegra_pcie_dw_host_ops;
1511
1512	ret = dw_pcie_host_init(pp);
1513	if (ret < 0) {
1514		dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1515		goto fail_host_init;
1516	}
1517
1518	return 0;
1519
1520fail_host_init:
1521	return __deinit_controller(pcie);
1522}
1523
1524static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1525{
1526	u32 val;
1527
1528	if (!tegra_pcie_dw_link_up(&pcie->pci))
1529		return 0;
1530
1531	val = appl_readl(pcie, APPL_RADM_STATUS);
1532	val |= APPL_PM_XMT_TURNOFF_STATE;
1533	appl_writel(pcie, val, APPL_RADM_STATUS);
1534
1535	return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1536				 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1537				 1, PME_ACK_TIMEOUT);
1538}
1539
1540static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1541{
1542	u32 data;
1543	int err;
1544
1545	if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1546		dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1547		return;
1548	}
1549
1550	if (tegra_pcie_try_link_l2(pcie)) {
1551		dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1552		/*
1553		 * TX lane clock freq will reset to Gen1 only if link is in L2
1554		 * or detect state.
1555		 * So apply pex_rst to end point to force RP to go into detect
1556		 * state
1557		 */
1558		data = appl_readl(pcie, APPL_PINMUX);
1559		data &= ~APPL_PINMUX_PEX_RST;
1560		appl_writel(pcie, data, APPL_PINMUX);
1561
1562		err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1563						data,
1564						((data &
1565						APPL_DEBUG_LTSSM_STATE_MASK) >>
1566						APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1567						LTSSM_STATE_PRE_DETECT,
1568						1, LTSSM_TIMEOUT);
1569		if (err) {
1570			dev_info(pcie->dev, "Link didn't go to detect state\n");
1571		} else {
1572			/* Disable LTSSM after link is in detect state */
1573			data = appl_readl(pcie, APPL_CTRL);
1574			data &= ~APPL_CTRL_LTSSM_EN;
1575			appl_writel(pcie, data, APPL_CTRL);
1576		}
1577	}
1578	/*
1579	 * DBI registers may not be accessible after this as PLL-E would be
1580	 * down depending on how CLKREQ is pulled by end point
1581	 */
1582	data = appl_readl(pcie, APPL_PINMUX);
1583	data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1584	/* Cut REFCLK to slot */
1585	data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1586	data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1587	appl_writel(pcie, data, APPL_PINMUX);
1588}
1589
1590static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1591{
1592	tegra_pcie_downstream_dev_to_D0(pcie);
1593	dw_pcie_host_deinit(&pcie->pci.pp);
1594	tegra_pcie_dw_pme_turnoff(pcie);
1595
1596	return __deinit_controller(pcie);
1597}
1598
1599static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1600{
1601	struct pcie_port *pp = &pcie->pci.pp;
1602	struct device *dev = pcie->dev;
1603	char *name;
1604	int ret;
1605
1606	if (IS_ENABLED(CONFIG_PCI_MSI)) {
1607		pp->msi_irq = of_irq_get_byname(dev->of_node, "msi");
1608		if (!pp->msi_irq) {
1609			dev_err(dev, "Failed to get MSI interrupt\n");
1610			return -ENODEV;
1611		}
1612	}
1613
1614	pm_runtime_enable(dev);
1615
1616	ret = pm_runtime_get_sync(dev);
1617	if (ret < 0) {
1618		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1619			ret);
1620		goto fail_pm_get_sync;
1621	}
1622
1623	ret = pinctrl_pm_select_default_state(dev);
1624	if (ret < 0) {
1625		dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1626		goto fail_pm_get_sync;
1627	}
1628
1629	tegra_pcie_init_controller(pcie);
1630
1631	pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1632	if (!pcie->link_state) {
1633		ret = -ENOMEDIUM;
1634		goto fail_host_init;
1635	}
1636
1637	name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1638	if (!name) {
1639		ret = -ENOMEM;
1640		goto fail_host_init;
1641	}
1642
1643	pcie->debugfs = debugfs_create_dir(name, NULL);
1644	if (!pcie->debugfs)
1645		dev_err(dev, "Failed to create debugfs\n");
1646	else
1647		init_debugfs(pcie);
1648
1649	return ret;
1650
1651fail_host_init:
1652	tegra_pcie_deinit_controller(pcie);
1653fail_pm_get_sync:
1654	pm_runtime_put_sync(dev);
 
1655	pm_runtime_disable(dev);
1656	return ret;
1657}
1658
1659static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1660{
1661	u32 val;
1662	int ret;
1663
1664	if (pcie->ep_state == EP_STATE_DISABLED)
1665		return;
1666
1667	/* Disable LTSSM */
1668	val = appl_readl(pcie, APPL_CTRL);
1669	val &= ~APPL_CTRL_LTSSM_EN;
1670	appl_writel(pcie, val, APPL_CTRL);
1671
1672	ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1673				 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1674				 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1675				 LTSSM_STATE_PRE_DETECT,
1676				 1, LTSSM_TIMEOUT);
1677	if (ret)
1678		dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1679
1680	reset_control_assert(pcie->core_rst);
1681
1682	tegra_pcie_disable_phy(pcie);
1683
1684	reset_control_assert(pcie->core_apb_rst);
1685
1686	clk_disable_unprepare(pcie->core_clk);
1687
1688	pm_runtime_put_sync(pcie->dev);
1689
1690	ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1691	if (ret)
1692		dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1693
1694	pcie->ep_state = EP_STATE_DISABLED;
1695	dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1696}
1697
1698static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1699{
1700	struct dw_pcie *pci = &pcie->pci;
1701	struct dw_pcie_ep *ep = &pci->ep;
1702	struct device *dev = pcie->dev;
1703	u32 val;
1704	int ret;
1705
1706	if (pcie->ep_state == EP_STATE_ENABLED)
1707		return;
1708
1709	ret = pm_runtime_get_sync(dev);
1710	if (ret < 0) {
1711		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1712			ret);
1713		return;
1714	}
1715
1716	ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1717	if (ret) {
1718		dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret);
1719		goto fail_pll_init;
1720	}
1721
1722	ret = clk_prepare_enable(pcie->core_clk);
1723	if (ret) {
1724		dev_err(dev, "Failed to enable core clock: %d\n", ret);
1725		goto fail_core_clk_enable;
1726	}
1727
1728	ret = reset_control_deassert(pcie->core_apb_rst);
1729	if (ret) {
1730		dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1731		goto fail_core_apb_rst;
1732	}
1733
1734	ret = tegra_pcie_enable_phy(pcie);
1735	if (ret) {
1736		dev_err(dev, "Failed to enable PHY: %d\n", ret);
1737		goto fail_phy;
1738	}
1739
1740	/* Clear any stale interrupt statuses */
1741	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1742	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1743	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1744	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1745	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1746	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1747	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1748	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1749	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1750	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1751	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1752	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1753	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1754	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1755	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1756
1757	/* configure this core for EP mode operation */
1758	val = appl_readl(pcie, APPL_DM_TYPE);
1759	val &= ~APPL_DM_TYPE_MASK;
1760	val |= APPL_DM_TYPE_EP;
1761	appl_writel(pcie, val, APPL_DM_TYPE);
1762
1763	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1764
1765	val = appl_readl(pcie, APPL_CTRL);
1766	val |= APPL_CTRL_SYS_PRE_DET_STATE;
1767	val |= APPL_CTRL_HW_HOT_RST_EN;
1768	appl_writel(pcie, val, APPL_CTRL);
1769
1770	val = appl_readl(pcie, APPL_CFG_MISC);
1771	val |= APPL_CFG_MISC_SLV_EP_MODE;
1772	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1773	appl_writel(pcie, val, APPL_CFG_MISC);
1774
1775	val = appl_readl(pcie, APPL_PINMUX);
1776	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1777	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1778	appl_writel(pcie, val, APPL_PINMUX);
1779
1780	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1781		    APPL_CFG_BASE_ADDR);
1782
1783	appl_writel(pcie, pcie->atu_dma_res->start &
1784		    APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1785		    APPL_CFG_IATU_DMA_BASE_ADDR);
1786
1787	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1788	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1789	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1790	val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1791	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1792
1793	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1794	val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1795	val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1796	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1797
1798	reset_control_deassert(pcie->core_rst);
1799
1800	if (pcie->update_fc_fixup) {
1801		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1802		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1803		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1804	}
1805
1806	config_gen3_gen4_eq_presets(pcie);
1807
1808	init_host_aspm(pcie);
1809
1810	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1811	if (!pcie->supports_clkreq) {
1812		disable_aspm_l11(pcie);
1813		disable_aspm_l12(pcie);
1814	}
1815
1816	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1817	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1818	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1819
1820	/* Configure N_FTS & FTS */
1821	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
1822	val &= ~(N_FTS_MASK << N_FTS_SHIFT);
1823	val |= N_FTS_VAL << N_FTS_SHIFT;
1824	dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
1825
1826	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
1827	val &= ~FTS_MASK;
1828	val |= FTS_VAL;
1829	dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
1830
1831	/* Configure Max Speed from DT */
1832	if (pcie->max_speed && pcie->max_speed != -EINVAL) {
1833		val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base +
1834					PCI_EXP_LNKCAP);
1835		val &= ~PCI_EXP_LNKCAP_SLS;
1836		val |= pcie->max_speed;
1837		dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP,
1838				   val);
1839	}
1840
1841	pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1842						      PCI_CAP_ID_EXP);
1843	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1844
1845	val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1846	val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1847	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1848	val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1849	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1850
1851	ret = dw_pcie_ep_init_complete(ep);
1852	if (ret) {
1853		dev_err(dev, "Failed to complete initialization: %d\n", ret);
1854		goto fail_init_complete;
1855	}
1856
1857	dw_pcie_ep_init_notify(ep);
1858
1859	/* Enable LTSSM */
1860	val = appl_readl(pcie, APPL_CTRL);
1861	val |= APPL_CTRL_LTSSM_EN;
1862	appl_writel(pcie, val, APPL_CTRL);
1863
1864	pcie->ep_state = EP_STATE_ENABLED;
1865	dev_dbg(dev, "Initialization of endpoint is completed\n");
1866
1867	return;
1868
1869fail_init_complete:
1870	reset_control_assert(pcie->core_rst);
1871	tegra_pcie_disable_phy(pcie);
1872fail_phy:
1873	reset_control_assert(pcie->core_apb_rst);
1874fail_core_apb_rst:
1875	clk_disable_unprepare(pcie->core_clk);
1876fail_core_clk_enable:
1877	tegra_pcie_bpmp_set_pll_state(pcie, false);
1878fail_pll_init:
1879	pm_runtime_put_sync(dev);
1880}
1881
1882static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1883{
1884	struct tegra_pcie_dw *pcie = arg;
1885
1886	if (gpiod_get_value(pcie->pex_rst_gpiod))
1887		pex_ep_event_pex_rst_assert(pcie);
1888	else
1889		pex_ep_event_pex_rst_deassert(pcie);
1890
1891	return IRQ_HANDLED;
1892}
1893
1894static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1895{
1896	/* Tegra194 supports only INTA */
1897	if (irq > 1)
1898		return -EINVAL;
1899
1900	appl_writel(pcie, 1, APPL_LEGACY_INTX);
1901	usleep_range(1000, 2000);
1902	appl_writel(pcie, 0, APPL_LEGACY_INTX);
1903	return 0;
1904}
1905
1906static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1907{
1908	if (unlikely(irq > 31))
1909		return -EINVAL;
1910
1911	appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1);
1912
1913	return 0;
1914}
1915
1916static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1917{
1918	struct dw_pcie_ep *ep = &pcie->pci.ep;
1919
1920	writel(irq, ep->msi_mem);
1921
1922	return 0;
1923}
1924
1925static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1926				   enum pci_epc_irq_type type,
1927				   u16 interrupt_num)
1928{
1929	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1930	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1931
1932	switch (type) {
1933	case PCI_EPC_IRQ_LEGACY:
1934		return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
1935
1936	case PCI_EPC_IRQ_MSI:
1937		return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1938
1939	case PCI_EPC_IRQ_MSIX:
1940		return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1941
1942	default:
1943		dev_err(pci->dev, "Unknown IRQ type\n");
1944		return -EPERM;
1945	}
1946
1947	return 0;
1948}
1949
1950static const struct pci_epc_features tegra_pcie_epc_features = {
1951	.linkup_notifier = true,
1952	.core_init_notifier = true,
1953	.msi_capable = false,
1954	.msix_capable = false,
1955	.reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
1956	.bar_fixed_64bit = 1 << BAR_0,
1957	.bar_fixed_size[0] = SZ_1M,
1958};
1959
1960static const struct pci_epc_features*
1961tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
1962{
1963	return &tegra_pcie_epc_features;
1964}
1965
1966static struct dw_pcie_ep_ops pcie_ep_ops = {
1967	.raise_irq = tegra_pcie_ep_raise_irq,
1968	.get_features = tegra_pcie_ep_get_features,
1969};
1970
1971static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
1972				struct platform_device *pdev)
1973{
1974	struct dw_pcie *pci = &pcie->pci;
1975	struct device *dev = pcie->dev;
1976	struct dw_pcie_ep *ep;
1977	struct resource *res;
1978	char *name;
1979	int ret;
1980
1981	ep = &pci->ep;
1982	ep->ops = &pcie_ep_ops;
1983
1984	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
1985	if (!res)
1986		return -EINVAL;
1987
1988	ep->phys_base = res->start;
1989	ep->addr_size = resource_size(res);
1990	ep->page_size = SZ_64K;
1991
1992	ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
1993	if (ret < 0) {
1994		dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
1995			ret);
1996		return ret;
1997	}
1998
1999	ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2000	if (ret < 0) {
2001		dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2002		return ret;
2003	}
2004	pcie->pex_rst_irq = (unsigned int)ret;
2005
2006	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2007			      pcie->cid);
2008	if (!name) {
2009		dev_err(dev, "Failed to create PERST IRQ string\n");
2010		return -ENOMEM;
2011	}
2012
2013	irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2014
2015	pcie->ep_state = EP_STATE_DISABLED;
2016
2017	ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2018					tegra_pcie_ep_pex_rst_irq,
2019					IRQF_TRIGGER_RISING |
2020					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2021					name, (void *)pcie);
2022	if (ret < 0) {
2023		dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2024		return ret;
2025	}
2026
2027	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work",
2028			      pcie->cid);
2029	if (!name) {
2030		dev_err(dev, "Failed to create PCIe EP work thread string\n");
2031		return -ENOMEM;
2032	}
2033
2034	pm_runtime_enable(dev);
2035
2036	ret = dw_pcie_ep_init(ep);
2037	if (ret) {
2038		dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2039			ret);
2040		return ret;
2041	}
2042
2043	return 0;
2044}
2045
2046static int tegra_pcie_dw_probe(struct platform_device *pdev)
2047{
2048	const struct tegra_pcie_dw_of_data *data;
2049	struct device *dev = &pdev->dev;
2050	struct resource *atu_dma_res;
2051	struct tegra_pcie_dw *pcie;
2052	struct resource *dbi_res;
2053	struct pcie_port *pp;
2054	struct dw_pcie *pci;
2055	struct phy **phys;
2056	char *name;
2057	int ret;
2058	u32 i;
2059
2060	data = of_device_get_match_data(dev);
2061
2062	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2063	if (!pcie)
2064		return -ENOMEM;
2065
2066	pci = &pcie->pci;
2067	pci->dev = &pdev->dev;
2068	pci->ops = &tegra_dw_pcie_ops;
2069	pp = &pci->pp;
2070	pcie->dev = &pdev->dev;
2071	pcie->mode = (enum dw_pcie_device_mode)data->mode;
2072
2073	ret = tegra_pcie_dw_parse_dt(pcie);
2074	if (ret < 0) {
2075		const char *level = KERN_ERR;
2076
2077		if (ret == -EPROBE_DEFER)
2078			level = KERN_DEBUG;
2079
2080		dev_printk(level, dev,
2081			   dev_fmt("Failed to parse device tree: %d\n"),
2082			   ret);
2083		return ret;
2084	}
2085
2086	ret = tegra_pcie_get_slot_regulators(pcie);
2087	if (ret < 0) {
2088		const char *level = KERN_ERR;
2089
2090		if (ret == -EPROBE_DEFER)
2091			level = KERN_DEBUG;
2092
2093		dev_printk(level, dev,
2094			   dev_fmt("Failed to get slot regulators: %d\n"),
2095			   ret);
2096		return ret;
2097	}
2098
2099	if (pcie->pex_refclk_sel_gpiod)
2100		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2101
2102	pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2103	if (IS_ERR(pcie->pex_ctl_supply)) {
2104		ret = PTR_ERR(pcie->pex_ctl_supply);
2105		if (ret != -EPROBE_DEFER)
2106			dev_err(dev, "Failed to get regulator: %ld\n",
2107				PTR_ERR(pcie->pex_ctl_supply));
2108		return ret;
2109	}
2110
2111	pcie->core_clk = devm_clk_get(dev, "core");
2112	if (IS_ERR(pcie->core_clk)) {
2113		dev_err(dev, "Failed to get core clock: %ld\n",
2114			PTR_ERR(pcie->core_clk));
2115		return PTR_ERR(pcie->core_clk);
2116	}
2117
2118	pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2119						      "appl");
2120	if (!pcie->appl_res) {
2121		dev_err(dev, "Failed to find \"appl\" region\n");
2122		return -ENODEV;
2123	}
2124
2125	pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2126	if (IS_ERR(pcie->appl_base))
2127		return PTR_ERR(pcie->appl_base);
2128
2129	pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2130	if (IS_ERR(pcie->core_apb_rst)) {
2131		dev_err(dev, "Failed to get APB reset: %ld\n",
2132			PTR_ERR(pcie->core_apb_rst));
2133		return PTR_ERR(pcie->core_apb_rst);
2134	}
2135
2136	phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2137	if (!phys)
2138		return -ENOMEM;
2139
2140	for (i = 0; i < pcie->phy_count; i++) {
2141		name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2142		if (!name) {
2143			dev_err(dev, "Failed to create P2U string\n");
2144			return -ENOMEM;
2145		}
2146		phys[i] = devm_phy_get(dev, name);
2147		kfree(name);
2148		if (IS_ERR(phys[i])) {
2149			ret = PTR_ERR(phys[i]);
2150			if (ret != -EPROBE_DEFER)
2151				dev_err(dev, "Failed to get PHY: %d\n", ret);
2152			return ret;
2153		}
2154	}
2155
2156	pcie->phys = phys;
2157
2158	dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
2159	if (!dbi_res) {
2160		dev_err(dev, "Failed to find \"dbi\" region\n");
2161		return -ENODEV;
2162	}
2163	pcie->dbi_res = dbi_res;
2164
2165	pci->dbi_base = devm_ioremap_resource(dev, dbi_res);
2166	if (IS_ERR(pci->dbi_base))
2167		return PTR_ERR(pci->dbi_base);
2168
2169	/* Tegra HW locates DBI2 at a fixed offset from DBI */
2170	pci->dbi_base2 = pci->dbi_base + 0x1000;
2171
2172	atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2173						   "atu_dma");
2174	if (!atu_dma_res) {
2175		dev_err(dev, "Failed to find \"atu_dma\" region\n");
2176		return -ENODEV;
2177	}
2178	pcie->atu_dma_res = atu_dma_res;
2179
2180	pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2181	if (IS_ERR(pci->atu_base))
2182		return PTR_ERR(pci->atu_base);
2183
2184	pcie->core_rst = devm_reset_control_get(dev, "core");
2185	if (IS_ERR(pcie->core_rst)) {
2186		dev_err(dev, "Failed to get core reset: %ld\n",
2187			PTR_ERR(pcie->core_rst));
2188		return PTR_ERR(pcie->core_rst);
2189	}
2190
2191	pp->irq = platform_get_irq_byname(pdev, "intr");
2192	if (pp->irq < 0)
2193		return pp->irq;
 
 
 
 
 
 
 
 
 
2194
2195	pcie->bpmp = tegra_bpmp_get(dev);
2196	if (IS_ERR(pcie->bpmp))
2197		return PTR_ERR(pcie->bpmp);
2198
2199	platform_set_drvdata(pdev, pcie);
2200
2201	switch (pcie->mode) {
2202	case DW_PCIE_RC_TYPE:
2203		ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2204				       IRQF_SHARED, "tegra-pcie-intr", pcie);
2205		if (ret) {
2206			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2207				ret);
2208			goto fail;
2209		}
2210
2211		ret = tegra_pcie_config_rp(pcie);
2212		if (ret && ret != -ENOMEDIUM)
2213			goto fail;
2214		else
2215			return 0;
2216		break;
2217
2218	case DW_PCIE_EP_TYPE:
2219		ret = devm_request_threaded_irq(dev, pp->irq,
2220						tegra_pcie_ep_hard_irq,
2221						tegra_pcie_ep_irq_thread,
2222						IRQF_SHARED | IRQF_ONESHOT,
2223						"tegra-pcie-ep-intr", pcie);
2224		if (ret) {
2225			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2226				ret);
2227			goto fail;
2228		}
2229
2230		ret = tegra_pcie_config_ep(pcie, pdev);
2231		if (ret < 0)
2232			goto fail;
2233		break;
2234
2235	default:
2236		dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode);
2237	}
2238
2239fail:
2240	tegra_bpmp_put(pcie->bpmp);
2241	return ret;
2242}
2243
2244static int tegra_pcie_dw_remove(struct platform_device *pdev)
2245{
2246	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2247
2248	if (!pcie->link_state)
2249		return 0;
2250
2251	debugfs_remove_recursive(pcie->debugfs);
2252	tegra_pcie_deinit_controller(pcie);
2253	pm_runtime_put_sync(pcie->dev);
2254	pm_runtime_disable(pcie->dev);
2255	tegra_bpmp_put(pcie->bpmp);
2256	if (pcie->pex_refclk_sel_gpiod)
2257		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2258
2259	return 0;
2260}
2261
2262static int tegra_pcie_dw_suspend_late(struct device *dev)
2263{
2264	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2265	u32 val;
2266
2267	if (!pcie->link_state)
2268		return 0;
2269
2270	/* Enable HW_HOT_RST mode */
2271	val = appl_readl(pcie, APPL_CTRL);
2272	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2273		 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2274	val |= APPL_CTRL_HW_HOT_RST_EN;
2275	appl_writel(pcie, val, APPL_CTRL);
2276
2277	return 0;
2278}
2279
2280static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2281{
2282	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2283
2284	if (!pcie->link_state)
2285		return 0;
2286
2287	/* Save MSI interrupt vector */
2288	pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci,
2289					       PORT_LOGIC_MSI_CTRL_INT_0_EN);
2290	tegra_pcie_downstream_dev_to_D0(pcie);
2291	tegra_pcie_dw_pme_turnoff(pcie);
2292
2293	return __deinit_controller(pcie);
2294}
2295
2296static int tegra_pcie_dw_resume_noirq(struct device *dev)
2297{
2298	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2299	int ret;
2300
2301	if (!pcie->link_state)
2302		return 0;
2303
2304	ret = tegra_pcie_config_controller(pcie, true);
2305	if (ret < 0)
2306		return ret;
2307
2308	ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2309	if (ret < 0) {
2310		dev_err(dev, "Failed to init host: %d\n", ret);
2311		goto fail_host_init;
2312	}
2313
2314	/* Restore MSI interrupt vector */
2315	dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN,
2316			   pcie->msi_ctrl_int);
2317
2318	return 0;
2319
2320fail_host_init:
2321	return __deinit_controller(pcie);
2322}
2323
2324static int tegra_pcie_dw_resume_early(struct device *dev)
2325{
2326	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2327	u32 val;
2328
2329	if (!pcie->link_state)
2330		return 0;
2331
2332	/* Disable HW_HOT_RST mode */
2333	val = appl_readl(pcie, APPL_CTRL);
2334	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2335		 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2336	val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2337	       APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2338	val &= ~APPL_CTRL_HW_HOT_RST_EN;
2339	appl_writel(pcie, val, APPL_CTRL);
2340
2341	return 0;
2342}
2343
2344static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2345{
2346	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2347
2348	if (!pcie->link_state)
2349		return;
2350
2351	debugfs_remove_recursive(pcie->debugfs);
2352	tegra_pcie_downstream_dev_to_D0(pcie);
2353
2354	disable_irq(pcie->pci.pp.irq);
2355	if (IS_ENABLED(CONFIG_PCI_MSI))
2356		disable_irq(pcie->pci.pp.msi_irq);
2357
2358	tegra_pcie_dw_pme_turnoff(pcie);
2359	__deinit_controller(pcie);
2360}
2361
2362static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = {
2363	.mode = DW_PCIE_RC_TYPE,
2364};
2365
2366static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = {
2367	.mode = DW_PCIE_EP_TYPE,
2368};
2369
2370static const struct of_device_id tegra_pcie_dw_of_match[] = {
2371	{
2372		.compatible = "nvidia,tegra194-pcie",
2373		.data = &tegra_pcie_dw_rc_of_data,
2374	},
2375	{
2376		.compatible = "nvidia,tegra194-pcie-ep",
2377		.data = &tegra_pcie_dw_ep_of_data,
2378	},
2379	{},
2380};
2381
2382static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2383	.suspend_late = tegra_pcie_dw_suspend_late,
2384	.suspend_noirq = tegra_pcie_dw_suspend_noirq,
2385	.resume_noirq = tegra_pcie_dw_resume_noirq,
2386	.resume_early = tegra_pcie_dw_resume_early,
2387};
2388
2389static struct platform_driver tegra_pcie_dw_driver = {
2390	.probe = tegra_pcie_dw_probe,
2391	.remove = tegra_pcie_dw_remove,
2392	.shutdown = tegra_pcie_dw_shutdown,
2393	.driver = {
2394		.name	= "tegra194-pcie",
2395		.pm = &tegra_pcie_dw_pm_ops,
2396		.of_match_table = tegra_pcie_dw_of_match,
2397	},
2398};
2399module_platform_driver(tegra_pcie_dw_driver);
2400
2401MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2402
2403MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2404MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2405MODULE_LICENSE("GPL v2");