1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * PCIe host controller driver for Tegra194 SoC
 
 
   4 *
   5 * Copyright (C) 2019 NVIDIA Corporation.
   6 *
   7 * Author: Vidya Sagar <vidyas@nvidia.com>
   8 */
   9
  10#include <linux/clk.h>
  11#include <linux/debugfs.h>
  12#include <linux/delay.h>
  13#include <linux/gpio.h>
  14#include <linux/gpio/consumer.h>
  15#include <linux/interrupt.h>
  16#include <linux/iopoll.h>
  17#include <linux/kernel.h>
  18#include <linux/module.h>
  19#include <linux/of.h>
  20#include <linux/of_device.h>
  21#include <linux/of_gpio.h>
  22#include <linux/of_irq.h>
  23#include <linux/of_pci.h>
  24#include <linux/pci.h>
  25#include <linux/phy/phy.h>
  26#include <linux/pinctrl/consumer.h>
  27#include <linux/platform_device.h>
  28#include <linux/pm_runtime.h>
  29#include <linux/random.h>
  30#include <linux/reset.h>
  31#include <linux/resource.h>
  32#include <linux/types.h>
  33#include "pcie-designware.h"
  34#include <soc/tegra/bpmp.h>
  35#include <soc/tegra/bpmp-abi.h>
  36#include "../../pci.h"
  37
 
 
 
  38#define APPL_PINMUX				0x0
  39#define APPL_PINMUX_PEX_RST			BIT(0)
  40#define APPL_PINMUX_CLKREQ_OVERRIDE_EN		BIT(2)
  41#define APPL_PINMUX_CLKREQ_OVERRIDE		BIT(3)
  42#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN	BIT(4)
  43#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE	BIT(5)
  44
  45#define APPL_CTRL				0x4
  46#define APPL_CTRL_SYS_PRE_DET_STATE		BIT(6)
  47#define APPL_CTRL_LTSSM_EN			BIT(7)
  48#define APPL_CTRL_HW_HOT_RST_EN			BIT(20)
  49#define APPL_CTRL_HW_HOT_RST_MODE_MASK		GENMASK(1, 0)
  50#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT		22
  51#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST	0x1
 
  52
  53#define APPL_INTR_EN_L0_0			0x8
  54#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN	BIT(0)
  55#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN	BIT(4)
  56#define APPL_INTR_EN_L0_0_INT_INT_EN		BIT(8)
  57#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN	BIT(15)
  58#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN	BIT(19)
  59#define APPL_INTR_EN_L0_0_SYS_INTR_EN		BIT(30)
  60#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN	BIT(31)
  61
  62#define APPL_INTR_STATUS_L0			0xC
  63#define APPL_INTR_STATUS_L0_LINK_STATE_INT	BIT(0)
  64#define APPL_INTR_STATUS_L0_INT_INT		BIT(8)
  65#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT	BIT(15)
  66#define APPL_INTR_STATUS_L0_PEX_RST_INT		BIT(16)
  67#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT	BIT(18)
  68
  69#define APPL_INTR_EN_L1_0_0				0x1C
  70#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN	BIT(1)
  71#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN		BIT(3)
  72#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN	BIT(30)
  73
  74#define APPL_INTR_STATUS_L1_0_0				0x20
  75#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED	BIT(1)
  76#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED	BIT(3)
  77#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE		BIT(30)
  78
  79#define APPL_INTR_STATUS_L1_1			0x2C
  80#define APPL_INTR_STATUS_L1_2			0x30
  81#define APPL_INTR_STATUS_L1_3			0x34
  82#define APPL_INTR_STATUS_L1_6			0x3C
  83#define APPL_INTR_STATUS_L1_7			0x40
  84#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED	BIT(1)
  85
  86#define APPL_INTR_EN_L1_8_0			0x44
  87#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN		BIT(2)
  88#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN	BIT(3)
  89#define APPL_INTR_EN_L1_8_INTX_EN		BIT(11)
  90#define APPL_INTR_EN_L1_8_AER_INT_EN		BIT(15)
  91
  92#define APPL_INTR_STATUS_L1_8_0			0x4C
  93#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK	GENMASK(11, 6)
  94#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS	BIT(2)
  95#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS	BIT(3)
  96
  97#define APPL_INTR_STATUS_L1_9			0x54
  98#define APPL_INTR_STATUS_L1_10			0x58
  99#define APPL_INTR_STATUS_L1_11			0x64
 100#define APPL_INTR_STATUS_L1_13			0x74
 101#define APPL_INTR_STATUS_L1_14			0x78
 102#define APPL_INTR_STATUS_L1_15			0x7C
 103#define APPL_INTR_STATUS_L1_17			0x88
 104
 105#define APPL_INTR_EN_L1_18				0x90
 106#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT		BIT(2)
 107#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR		BIT(1)
 108#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
 109
 110#define APPL_INTR_STATUS_L1_18				0x94
 111#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT	BIT(2)
 112#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR	BIT(1)
 113#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
 114
 115#define APPL_MSI_CTRL_1				0xAC
 116
 117#define APPL_MSI_CTRL_2				0xB0
 118
 119#define APPL_LEGACY_INTX			0xB8
 120
 121#define APPL_LTR_MSG_1				0xC4
 122#define LTR_MSG_REQ				BIT(15)
 123#define LTR_MST_NO_SNOOP_SHIFT			16
 124
 125#define APPL_LTR_MSG_2				0xC8
 126#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE	BIT(3)
 127
 128#define APPL_LINK_STATUS			0xCC
 129#define APPL_LINK_STATUS_RDLH_LINK_UP		BIT(0)
 130
 131#define APPL_DEBUG				0xD0
 132#define APPL_DEBUG_PM_LINKST_IN_L2_LAT		BIT(21)
 133#define APPL_DEBUG_PM_LINKST_IN_L0		0x11
 134#define APPL_DEBUG_LTSSM_STATE_MASK		GENMASK(8, 3)
 135#define APPL_DEBUG_LTSSM_STATE_SHIFT		3
 136#define LTSSM_STATE_PRE_DETECT			5
 137
 138#define APPL_RADM_STATUS			0xE4
 139#define APPL_PM_XMT_TURNOFF_STATE		BIT(0)
 140
 141#define APPL_DM_TYPE				0x100
 142#define APPL_DM_TYPE_MASK			GENMASK(3, 0)
 143#define APPL_DM_TYPE_RP				0x4
 144#define APPL_DM_TYPE_EP				0x0
 145
 146#define APPL_CFG_BASE_ADDR			0x104
 147#define APPL_CFG_BASE_ADDR_MASK			GENMASK(31, 12)
 148
 149#define APPL_CFG_IATU_DMA_BASE_ADDR		0x108
 150#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK	GENMASK(31, 18)
 151
 152#define APPL_CFG_MISC				0x110
 153#define APPL_CFG_MISC_SLV_EP_MODE		BIT(14)
 154#define APPL_CFG_MISC_ARCACHE_MASK		GENMASK(13, 10)
 155#define APPL_CFG_MISC_ARCACHE_SHIFT		10
 156#define APPL_CFG_MISC_ARCACHE_VAL		3
 157
 158#define APPL_CFG_SLCG_OVERRIDE			0x114
 159#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER	BIT(0)
 160
 161#define APPL_CAR_RESET_OVRD				0x12C
 162#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N	BIT(0)
 163
 164#define IO_BASE_IO_DECODE				BIT(0)
 165#define IO_BASE_IO_DECODE_BIT8				BIT(8)
 166
 167#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE		BIT(0)
 168#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE	BIT(16)
 169
 170#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF	0x718
 171#define CFG_TIMER_CTRL_ACK_NAK_SHIFT	(19)
 172
 173#define EVENT_COUNTER_ALL_CLEAR		0x3
 174#define EVENT_COUNTER_ENABLE_ALL	0x7
 175#define EVENT_COUNTER_ENABLE_SHIFT	2
 176#define EVENT_COUNTER_EVENT_SEL_MASK	GENMASK(7, 0)
 177#define EVENT_COUNTER_EVENT_SEL_SHIFT	16
 178#define EVENT_COUNTER_EVENT_Tx_L0S	0x2
 179#define EVENT_COUNTER_EVENT_Rx_L0S	0x3
 180#define EVENT_COUNTER_EVENT_L1		0x5
 181#define EVENT_COUNTER_EVENT_L1_1	0x7
 182#define EVENT_COUNTER_EVENT_L1_2	0x8
 183#define EVENT_COUNTER_GROUP_SEL_SHIFT	24
 184#define EVENT_COUNTER_GROUP_5		0x5
 185
 186#define N_FTS_VAL					52
 187#define FTS_VAL						52
 188
 189#define PORT_LOGIC_MSI_CTRL_INT_0_EN		0x828
 190
 191#define GEN3_EQ_CONTROL_OFF			0x8a8
 192#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT	8
 193#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK	GENMASK(23, 8)
 194#define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK	GENMASK(3, 0)
 195
 196#define GEN3_RELATED_OFF			0x890
 197#define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL	BIT(0)
 198#define GEN3_RELATED_OFF_GEN3_EQ_DISABLE	BIT(16)
 199#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT	24
 200#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK	GENMASK(25, 24)
 201
 202#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT	0x8D0
 203#define AMBA_ERROR_RESPONSE_CRS_SHIFT		3
 204#define AMBA_ERROR_RESPONSE_CRS_MASK		GENMASK(1, 0)
 205#define AMBA_ERROR_RESPONSE_CRS_OKAY		0
 206#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF	1
 207#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001	2
 208
 209#define MSIX_ADDR_MATCH_LOW_OFF			0x940
 210#define MSIX_ADDR_MATCH_LOW_OFF_EN		BIT(0)
 211#define MSIX_ADDR_MATCH_LOW_OFF_MASK		GENMASK(31, 2)
 212
 213#define MSIX_ADDR_MATCH_HIGH_OFF		0x944
 214#define MSIX_ADDR_MATCH_HIGH_OFF_MASK		GENMASK(31, 0)
 215
 216#define PORT_LOGIC_MSIX_DOORBELL			0x948
 217
 218#define CAP_SPCIE_CAP_OFF			0x154
 219#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK	GENMASK(3, 0)
 220#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK	GENMASK(11, 8)
 221#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT	8
 222
 223#define PME_ACK_TIMEOUT 10000
 224
 225#define LTSSM_TIMEOUT 50000	/* 50ms */
 226
 227#define GEN3_GEN4_EQ_PRESET_INIT	5
 228
 229#define GEN1_CORE_CLK_FREQ	62500000
 230#define GEN2_CORE_CLK_FREQ	125000000
 231#define GEN3_CORE_CLK_FREQ	250000000
 232#define GEN4_CORE_CLK_FREQ	500000000
 233
 234#define LTR_MSG_TIMEOUT		(100 * 1000)
 235
 236#define PERST_DEBOUNCE_TIME	(5 * 1000)
 237
 238#define EP_STATE_DISABLED	0
 239#define EP_STATE_ENABLED	1
 240
 241static const unsigned int pcie_gen_freq[] = {
 242	GEN1_CORE_CLK_FREQ,
 243	GEN2_CORE_CLK_FREQ,
 244	GEN3_CORE_CLK_FREQ,
 245	GEN4_CORE_CLK_FREQ
 246};
 247
 
 
 
 
 
 
 
 
 
 
 
 
 248struct tegra_pcie_dw {
 249	struct device *dev;
 250	struct resource *appl_res;
 251	struct resource *dbi_res;
 252	struct resource *atu_dma_res;
 253	void __iomem *appl_base;
 254	struct clk *core_clk;
 255	struct reset_control *core_apb_rst;
 256	struct reset_control *core_rst;
 257	struct dw_pcie pci;
 258	struct tegra_bpmp *bpmp;
 259
 260	enum dw_pcie_device_mode mode;
 261
 262	bool supports_clkreq;
 263	bool enable_cdm_check;
 
 264	bool link_state;
 265	bool update_fc_fixup;
 
 266	u8 init_link_width;
 267	u32 msi_ctrl_int;
 268	u32 num_lanes;
 269	u32 cid;
 270	u32 cfg_link_cap_l1sub;
 
 271	u32 pcie_cap_base;
 272	u32 aspm_cmrt;
 273	u32 aspm_pwr_on_t;
 274	u32 aspm_l0s_enter_lat;
 275
 276	struct regulator *pex_ctl_supply;
 277	struct regulator *slot_ctl_3v3;
 278	struct regulator *slot_ctl_12v;
 279
 280	unsigned int phy_count;
 281	struct phy **phys;
 282
 283	struct dentry *debugfs;
 284
 285	/* Endpoint mode specific */
 286	struct gpio_desc *pex_rst_gpiod;
 287	struct gpio_desc *pex_refclk_sel_gpiod;
 288	unsigned int pex_rst_irq;
 289	int ep_state;
 290};
 291
 292struct tegra_pcie_dw_of_data {
 293	enum dw_pcie_device_mode mode;
 294};
 295
 296static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
 297{
 298	return container_of(pci, struct tegra_pcie_dw, pci);
 299}
 300
 301static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
 302			       const u32 reg)
 303{
 304	writel_relaxed(value, pcie->appl_base + reg);
 305}
 306
 307static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
 308{
 309	return readl_relaxed(pcie->appl_base + reg);
 310}
 311
 312struct tegra_pcie_soc {
 313	enum dw_pcie_device_mode mode;
 314};
 315
 316static void apply_bad_link_workaround(struct pcie_port *pp)
 317{
 318	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 319	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 320	u32 current_link_width;
 321	u16 val;
 322
 323	/*
 324	 * NOTE:- Since this scenario is uncommon and link as such is not
 325	 * stable anyway, not waiting to confirm if link is really
 326	 * transitioning to Gen-2 speed
 327	 */
 328	val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
 329	if (val & PCI_EXP_LNKSTA_LBMS) {
 330		current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
 331				     PCI_EXP_LNKSTA_NLW_SHIFT;
 332		if (pcie->init_link_width > current_link_width) {
 333			dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
 334			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
 335						PCI_EXP_LNKCTL2);
 336			val &= ~PCI_EXP_LNKCTL2_TLS;
 337			val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
 338			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
 339					   PCI_EXP_LNKCTL2, val);
 340
 341			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
 342						PCI_EXP_LNKCTL);
 343			val |= PCI_EXP_LNKCTL_RL;
 344			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
 345					   PCI_EXP_LNKCTL, val);
 346		}
 347	}
 348}
 349
 350static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
 351{
 352	struct tegra_pcie_dw *pcie = arg;
 353	struct dw_pcie *pci = &pcie->pci;
 354	struct pcie_port *pp = &pci->pp;
 355	u32 val, tmp;
 356	u16 val_w;
 357
 358	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
 359	if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
 360		val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
 361		if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
 362			appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
 363
 364			/* SBR & Surprise Link Down WAR */
 365			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
 366			val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
 367			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
 368			udelay(1);
 369			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
 370			val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
 371			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
 372
 373			val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
 374			val |= PORT_LOGIC_SPEED_CHANGE;
 375			dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
 376		}
 377	}
 378
 379	if (val & APPL_INTR_STATUS_L0_INT_INT) {
 380		val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
 381		if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
 382			appl_writel(pcie,
 383				    APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
 384				    APPL_INTR_STATUS_L1_8_0);
 385			apply_bad_link_workaround(pp);
 386		}
 387		if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
 
 
 
 
 
 
 388			appl_writel(pcie,
 389				    APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
 390				    APPL_INTR_STATUS_L1_8_0);
 391
 392			val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
 393						  PCI_EXP_LNKSTA);
 394			dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
 395				PCI_EXP_LNKSTA_CLS);
 396		}
 397	}
 398
 399	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
 400	if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
 401		val = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
 402		tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
 403		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
 404			dev_info(pci->dev, "CDM check complete\n");
 405			tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
 406		}
 407		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
 408			dev_err(pci->dev, "CDM comparison mismatch\n");
 409			tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
 410		}
 411		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
 412			dev_err(pci->dev, "CDM Logic error\n");
 413			tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
 414		}
 415		dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp);
 416		tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
 417		dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp);
 418	}
 419
 420	return IRQ_HANDLED;
 421}
 422
 423static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
 424{
 425	u32 val;
 426
 427	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
 428	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
 429	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
 430	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
 431	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
 432	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
 433	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
 434	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
 435	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
 436	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
 437	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
 438	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
 439	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
 440	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
 441	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
 442	appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
 443
 444	val = appl_readl(pcie, APPL_CTRL);
 445	val |= APPL_CTRL_LTSSM_EN;
 446	appl_writel(pcie, val, APPL_CTRL);
 447}
 448
 449static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
 450{
 451	struct tegra_pcie_dw *pcie = arg;
 452	struct dw_pcie *pci = &pcie->pci;
 453	u32 val, speed;
 454
 455	speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
 456		PCI_EXP_LNKSTA_CLS;
 457	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
 458
 
 
 
 459	/* If EP doesn't advertise L1SS, just return */
 460	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
 461	if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
 462		return IRQ_HANDLED;
 463
 464	/* Check if BME is set to '1' */
 465	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
 466	if (val & PCI_COMMAND_MASTER) {
 467		ktime_t timeout;
 468
 469		/* 110us for both snoop and no-snoop */
 470		val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ;
 471		val |= (val << LTR_MST_NO_SNOOP_SHIFT);
 472		appl_writel(pcie, val, APPL_LTR_MSG_1);
 473
 474		/* Send LTR upstream */
 475		val = appl_readl(pcie, APPL_LTR_MSG_2);
 476		val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
 477		appl_writel(pcie, val, APPL_LTR_MSG_2);
 478
 479		timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
 480		for (;;) {
 481			val = appl_readl(pcie, APPL_LTR_MSG_2);
 482			if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
 483				break;
 484			if (ktime_after(ktime_get(), timeout))
 485				break;
 486			usleep_range(1000, 1100);
 487		}
 488		if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
 489			dev_err(pcie->dev, "Failed to send LTR message\n");
 490	}
 491
 492	return IRQ_HANDLED;
 493}
 494
 495static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
 496{
 497	struct tegra_pcie_dw *pcie = arg;
 498	struct dw_pcie_ep *ep = &pcie->pci.ep;
 499	int spurious = 1;
 500	u32 status_l0, status_l1, link_status;
 501
 502	status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
 503	if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
 504		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
 505		appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
 506
 507		if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
 508			pex_ep_event_hot_rst_done(pcie);
 509
 510		if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
 511			link_status = appl_readl(pcie, APPL_LINK_STATUS);
 512			if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {
 513				dev_dbg(pcie->dev, "Link is up with Host\n");
 514				dw_pcie_ep_linkup(ep);
 515			}
 516		}
 517
 518		spurious = 0;
 519	}
 520
 521	if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
 522		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
 523		appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15);
 524
 525		if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
 526			return IRQ_WAKE_THREAD;
 527
 528		spurious = 0;
 529	}
 530
 531	if (spurious) {
 532		dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
 533			 status_l0);
 534		appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0);
 535	}
 536
 537	return IRQ_HANDLED;
 538}
 539
 540static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
 541				     int size, u32 *val)
 542{
 
 
 
 
 543	/*
 544	 * This is an endpoint mode specific register happen to appear even
 545	 * when controller is operating in root port mode and system hangs
 546	 * when it is accessed with link being in ASPM-L1 state.
 547	 * So skip accessing it altogether
 548	 */
 549	if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
 
 550		*val = 0x00000000;
 551		return PCIBIOS_SUCCESSFUL;
 552	}
 553
 554	return pci_generic_config_read(bus, devfn, where, size, val);
 555}
 556
 557static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
 558				     int size, u32 val)
 559{
 
 
 
 
 560	/*
 561	 * This is an endpoint mode specific register happen to appear even
 562	 * when controller is operating in root port mode and system hangs
 563	 * when it is accessed with link being in ASPM-L1 state.
 564	 * So skip accessing it altogether
 565	 */
 566	if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
 
 567		return PCIBIOS_SUCCESSFUL;
 568
 569	return pci_generic_config_write(bus, devfn, where, size, val);
 570}
 571
 572static struct pci_ops tegra_pci_ops = {
 573	.map_bus = dw_pcie_own_conf_map_bus,
 574	.read = tegra_pcie_dw_rd_own_conf,
 575	.write = tegra_pcie_dw_wr_own_conf,
 576};
 577
 578#if defined(CONFIG_PCIEASPM)
 579static const u32 event_cntr_ctrl_offset[] = {
 580	0x1d8,
 581	0x1a8,
 582	0x1a8,
 583	0x1a8,
 584	0x1c4,
 585	0x1d8
 586};
 587
 588static const u32 event_cntr_data_offset[] = {
 589	0x1dc,
 590	0x1ac,
 591	0x1ac,
 592	0x1ac,
 593	0x1c8,
 594	0x1dc
 595};
 596
 597static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
 598{
 599	u32 val;
 600
 601	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
 602	val &= ~PCI_L1SS_CAP_ASPM_L1_1;
 603	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
 604}
 605
 606static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
 607{
 608	u32 val;
 609
 610	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
 611	val &= ~PCI_L1SS_CAP_ASPM_L1_2;
 612	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
 613}
 614
 615static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
 616{
 617	u32 val;
 618
 619	val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]);
 
 620	val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
 621	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
 622	val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
 623	val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
 624	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
 625	val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]);
 
 
 626
 627	return val;
 628}
 629
 630static int aspm_state_cnt(struct seq_file *s, void *data)
 631{
 632	struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
 633				     dev_get_drvdata(s->private);
 634	u32 val;
 635
 636	seq_printf(s, "Tx L0s entry count : %u\n",
 637		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
 638
 639	seq_printf(s, "Rx L0s entry count : %u\n",
 640		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
 641
 642	seq_printf(s, "Link L1 entry count : %u\n",
 643		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
 644
 645	seq_printf(s, "Link L1.1 entry count : %u\n",
 646		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
 647
 648	seq_printf(s, "Link L1.2 entry count : %u\n",
 649		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
 650
 651	/* Clear all counters */
 652	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid],
 
 653			   EVENT_COUNTER_ALL_CLEAR);
 654
 655	/* Re-enable counting */
 656	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
 657	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
 658	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
 
 659
 660	return 0;
 661}
 662
 663static void init_host_aspm(struct tegra_pcie_dw *pcie)
 664{
 665	struct dw_pcie *pci = &pcie->pci;
 666	u32 val;
 667
 668	val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
 669	pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
 670
 
 
 
 671	/* Enable ASPM counters */
 672	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
 673	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
 674	dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val);
 
 675
 676	/* Program T_cmrt and T_pwr_on values */
 677	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
 678	val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
 679	val |= (pcie->aspm_cmrt << 8);
 680	val |= (pcie->aspm_pwr_on_t << 19);
 681	dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
 682
 683	/* Program L0s and L1 entrance latencies */
 684	val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
 685	val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
 686	val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
 687	val |= PORT_AFR_ENTER_ASPM;
 688	dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
 689}
 690
 691static void init_debugfs(struct tegra_pcie_dw *pcie)
 692{
 693	debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,
 694				    aspm_state_cnt);
 695}
 696#else
 697static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
 698static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
 699static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
 700static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
 701#endif
 702
 703static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp)
 704{
 705	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 706	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 707	u32 val;
 708	u16 val_w;
 709
 710	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
 711	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
 712	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
 713
 714	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
 715	val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
 716	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
 
 
 717
 718	if (pcie->enable_cdm_check) {
 719		val = appl_readl(pcie, APPL_INTR_EN_L0_0);
 720		val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN;
 721		appl_writel(pcie, val, APPL_INTR_EN_L0_0);
 722
 723		val = appl_readl(pcie, APPL_INTR_EN_L1_18);
 724		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
 725		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
 726		appl_writel(pcie, val, APPL_INTR_EN_L1_18);
 727	}
 728
 729	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
 730				  PCI_EXP_LNKSTA);
 731	pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
 732				PCI_EXP_LNKSTA_NLW_SHIFT;
 733
 734	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
 735				  PCI_EXP_LNKCTL);
 736	val_w |= PCI_EXP_LNKCTL_LBMIE;
 737	dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
 738			   val_w);
 739}
 740
 741static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp)
 742{
 743	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 744	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 745	u32 val;
 746
 747	/* Enable legacy interrupt generation */
 748	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
 749	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
 750	val |= APPL_INTR_EN_L0_0_INT_INT_EN;
 751	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
 752
 753	val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
 754	val |= APPL_INTR_EN_L1_8_INTX_EN;
 755	val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
 756	val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
 757	if (IS_ENABLED(CONFIG_PCIEAER))
 758		val |= APPL_INTR_EN_L1_8_AER_INT_EN;
 759	appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
 760}
 761
 762static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp)
 763{
 764	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 765	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 766	u32 val;
 767
 768	/* Enable MSI interrupt generation */
 769	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
 770	val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
 771	val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
 772	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
 773}
 774
 775static void tegra_pcie_enable_interrupts(struct pcie_port *pp)
 776{
 777	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 778	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 779
 780	/* Clear interrupt statuses before enabling interrupts */
 781	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
 782	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
 783	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
 784	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
 785	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
 786	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
 787	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
 788	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
 789	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
 790	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
 791	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
 792	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
 793	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
 794	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
 795	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
 796
 797	tegra_pcie_enable_system_interrupts(pp);
 798	tegra_pcie_enable_legacy_interrupts(pp);
 799	if (IS_ENABLED(CONFIG_PCI_MSI))
 800		tegra_pcie_enable_msi_interrupts(pp);
 801}
 802
 803static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
 804{
 805	struct dw_pcie *pci = &pcie->pci;
 806	u32 val, offset, i;
 807
 808	/* Program init preset */
 809	for (i = 0; i < pcie->num_lanes; i++) {
 810		val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
 811		val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
 812		val |= GEN3_GEN4_EQ_PRESET_INIT;
 813		val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
 814		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
 815			   CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
 816		dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
 817
 818		offset = dw_pcie_find_ext_capability(pci,
 819						     PCI_EXT_CAP_ID_PL_16GT) +
 820				PCI_PL_16GT_LE_CTRL;
 821		val = dw_pcie_readb_dbi(pci, offset + i);
 822		val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
 823		val |= GEN3_GEN4_EQ_PRESET_INIT;
 824		val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
 825		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
 826			PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
 827		dw_pcie_writeb_dbi(pci, offset + i, val);
 828	}
 829
 830	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
 831	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
 832	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 833
 834	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
 835	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
 836	val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
 837	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
 838	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
 839
 840	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
 841	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
 842	val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
 843	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 844
 845	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
 846	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
 847	val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
 
 848	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
 849	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
 850
 851	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
 852	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
 853	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 854}
 855
 856static int tegra_pcie_dw_host_init(struct pcie_port *pp)
 857{
 858	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 859	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 860	u32 val;
 
 861
 862	pp->bridge->ops = &tegra_pci_ops;
 863
 864	if (!pcie->pcie_cap_base)
 865		pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
 866							      PCI_CAP_ID_EXP);
 867
 
 
 
 
 
 868	val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
 869	val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
 870	dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
 871
 872	val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
 873	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
 874	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
 875	dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
 876
 877	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
 878
 879	/* Enable as 0xFFFF0001 response for CRS */
 880	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
 881	val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
 882	val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
 883		AMBA_ERROR_RESPONSE_CRS_SHIFT);
 884	dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
 885
 886	/* Configure Max lane width from DT */
 887	val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
 888	val &= ~PCI_EXP_LNKCAP_MLW;
 889	val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
 890	dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
 891
 
 
 
 
 
 
 
 
 
 892	config_gen3_gen4_eq_presets(pcie);
 893
 894	init_host_aspm(pcie);
 895
 896	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
 897	if (!pcie->supports_clkreq) {
 898		disable_aspm_l11(pcie);
 899		disable_aspm_l12(pcie);
 900	}
 901
 902	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
 903	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
 904	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 
 
 905
 906	if (pcie->update_fc_fixup) {
 907		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
 908		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
 909		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
 910	}
 911
 912	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
 913
 914	return 0;
 915}
 916
 917static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
 918{
 919	u32 val, offset, speed, tmp;
 920	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
 921	struct pcie_port *pp = &pci->pp;
 922	bool retry = true;
 923
 924	if (pcie->mode == DW_PCIE_EP_TYPE) {
 925		enable_irq(pcie->pex_rst_irq);
 926		return 0;
 927	}
 928
 929retry_link:
 930	/* Assert RST */
 931	val = appl_readl(pcie, APPL_PINMUX);
 932	val &= ~APPL_PINMUX_PEX_RST;
 933	appl_writel(pcie, val, APPL_PINMUX);
 934
 935	usleep_range(100, 200);
 936
 937	/* Enable LTSSM */
 938	val = appl_readl(pcie, APPL_CTRL);
 939	val |= APPL_CTRL_LTSSM_EN;
 940	appl_writel(pcie, val, APPL_CTRL);
 941
 942	/* De-assert RST */
 943	val = appl_readl(pcie, APPL_PINMUX);
 944	val |= APPL_PINMUX_PEX_RST;
 945	appl_writel(pcie, val, APPL_PINMUX);
 946
 947	msleep(100);
 948
 949	if (dw_pcie_wait_for_link(pci)) {
 950		if (!retry)
 951			return 0;
 952		/*
 953		 * There are some endpoints which can't get the link up if
 954		 * root port has Data Link Feature (DLF) enabled.
 955		 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
 956		 * on Scaled Flow Control and DLF.
 957		 * So, need to confirm that is indeed the case here and attempt
 958		 * link up once again with DLF disabled.
 959		 */
 960		val = appl_readl(pcie, APPL_DEBUG);
 961		val &= APPL_DEBUG_LTSSM_STATE_MASK;
 962		val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
 963		tmp = appl_readl(pcie, APPL_LINK_STATUS);
 964		tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
 965		if (!(val == 0x11 && !tmp)) {
 966			/* Link is down for all good reasons */
 967			return 0;
 968		}
 969
 970		dev_info(pci->dev, "Link is down in DLL");
 971		dev_info(pci->dev, "Trying again with DLFE disabled\n");
 972		/* Disable LTSSM */
 973		val = appl_readl(pcie, APPL_CTRL);
 974		val &= ~APPL_CTRL_LTSSM_EN;
 975		appl_writel(pcie, val, APPL_CTRL);
 976
 977		reset_control_assert(pcie->core_rst);
 978		reset_control_deassert(pcie->core_rst);
 979
 980		offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
 981		val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
 982		val &= ~PCI_DLF_EXCHANGE_ENABLE;
 983		dw_pcie_writel_dbi(pci, offset, val);
 984
 985		tegra_pcie_dw_host_init(pp);
 986		dw_pcie_setup_rc(pp);
 987
 988		retry = false;
 989		goto retry_link;
 990	}
 991
 992	speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
 993		PCI_EXP_LNKSTA_CLS;
 994	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
 995
 996	tegra_pcie_enable_interrupts(pp);
 997
 998	return 0;
 999}
1000
1001static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1002{
1003	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1004	u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1005
1006	return !!(val & PCI_EXP_LNKSTA_DLLLA);
1007}
1008
1009static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1010{
1011	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1012
1013	disable_irq(pcie->pex_rst_irq);
1014}
1015
1016static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1017	.link_up = tegra_pcie_dw_link_up,
1018	.start_link = tegra_pcie_dw_start_link,
1019	.stop_link = tegra_pcie_dw_stop_link,
1020};
1021
1022static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1023	.host_init = tegra_pcie_dw_host_init,
1024};
1025
1026static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1027{
1028	unsigned int phy_count = pcie->phy_count;
1029
1030	while (phy_count--) {
1031		phy_power_off(pcie->phys[phy_count]);
1032		phy_exit(pcie->phys[phy_count]);
1033	}
1034}
1035
1036static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1037{
1038	unsigned int i;
1039	int ret;
1040
1041	for (i = 0; i < pcie->phy_count; i++) {
1042		ret = phy_init(pcie->phys[i]);
1043		if (ret < 0)
1044			goto phy_power_off;
1045
1046		ret = phy_power_on(pcie->phys[i]);
1047		if (ret < 0)
1048			goto phy_exit;
1049	}
1050
1051	return 0;
1052
1053phy_power_off:
1054	while (i--) {
1055		phy_power_off(pcie->phys[i]);
1056phy_exit:
1057		phy_exit(pcie->phys[i]);
1058	}
1059
1060	return ret;
1061}
1062
1063static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1064{
1065	struct platform_device *pdev = to_platform_device(pcie->dev);
1066	struct device_node *np = pcie->dev->of_node;
1067	int ret;
1068
1069	pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1070	if (!pcie->dbi_res) {
1071		dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1072		return -ENODEV;
1073	}
1074
1075	ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1076	if (ret < 0) {
1077		dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1078		return ret;
1079	}
1080
1081	ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1082				   &pcie->aspm_pwr_on_t);
1083	if (ret < 0)
1084		dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1085			 ret);
1086
1087	ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1088				   &pcie->aspm_l0s_enter_lat);
1089	if (ret < 0)
1090		dev_info(pcie->dev,
1091			 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1092
1093	ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1094	if (ret < 0) {
1095		dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1096		return ret;
1097	}
1098
1099	ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1100	if (ret) {
1101		dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1102		return ret;
1103	}
1104
1105	ret = of_property_count_strings(np, "phy-names");
1106	if (ret < 0) {
1107		dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1108			ret);
1109		return ret;
1110	}
1111	pcie->phy_count = ret;
1112
1113	if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1114		pcie->update_fc_fixup = true;
1115
 
 
 
 
 
 
 
 
 
 
1116	pcie->supports_clkreq =
1117		of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1118
1119	pcie->enable_cdm_check =
1120		of_property_read_bool(np, "snps,enable-cdm-check");
1121
1122	if (pcie->mode == DW_PCIE_RC_TYPE)
 
 
 
 
1123		return 0;
1124
1125	/* Endpoint mode specific DT entries */
1126	pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1127	if (IS_ERR(pcie->pex_rst_gpiod)) {
1128		int err = PTR_ERR(pcie->pex_rst_gpiod);
1129		const char *level = KERN_ERR;
1130
1131		if (err == -EPROBE_DEFER)
1132			level = KERN_DEBUG;
1133
1134		dev_printk(level, pcie->dev,
1135			   dev_fmt("Failed to get PERST GPIO: %d\n"),
1136			   err);
1137		return err;
1138	}
1139
1140	pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1141						    "nvidia,refclk-select",
1142						    GPIOD_OUT_HIGH);
1143	if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1144		int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1145		const char *level = KERN_ERR;
1146
1147		if (err == -EPROBE_DEFER)
1148			level = KERN_DEBUG;
1149
1150		dev_printk(level, pcie->dev,
1151			   dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1152			   err);
1153		pcie->pex_refclk_sel_gpiod = NULL;
1154	}
1155
1156	return 0;
1157}
1158
1159static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1160					  bool enable)
1161{
1162	struct mrq_uphy_response resp;
1163	struct tegra_bpmp_message msg;
1164	struct mrq_uphy_request req;
1165
1166	/* Controller-5 doesn't need to have its state set by BPMP-FW */
1167	if (pcie->cid == 5)
 
 
 
1168		return 0;
1169
1170	memset(&req, 0, sizeof(req));
1171	memset(&resp, 0, sizeof(resp));
1172
1173	req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1174	req.controller_state.pcie_controller = pcie->cid;
1175	req.controller_state.enable = enable;
1176
1177	memset(&msg, 0, sizeof(msg));
1178	msg.mrq = MRQ_UPHY;
1179	msg.tx.data = &req;
1180	msg.tx.size = sizeof(req);
1181	msg.rx.data = &resp;
1182	msg.rx.size = sizeof(resp);
1183
1184	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1185}
1186
1187static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1188					 bool enable)
1189{
1190	struct mrq_uphy_response resp;
1191	struct tegra_bpmp_message msg;
1192	struct mrq_uphy_request req;
1193
1194	memset(&req, 0, sizeof(req));
1195	memset(&resp, 0, sizeof(resp));
1196
1197	if (enable) {
1198		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1199		req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1200	} else {
1201		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1202		req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1203	}
1204
1205	memset(&msg, 0, sizeof(msg));
1206	msg.mrq = MRQ_UPHY;
1207	msg.tx.data = &req;
1208	msg.tx.size = sizeof(req);
1209	msg.rx.data = &resp;
1210	msg.rx.size = sizeof(resp);
1211
1212	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1213}
1214
1215static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1216{
1217	struct pcie_port *pp = &pcie->pci.pp;
1218	struct pci_bus *child, *root_bus = NULL;
1219	struct pci_dev *pdev;
1220
1221	/*
1222	 * link doesn't go into L2 state with some of the endpoints with Tegra
1223	 * if they are not in D0 state. So, need to make sure that immediate
1224	 * downstream devices are in D0 state before sending PME_TurnOff to put
1225	 * link into L2 state.
1226	 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1227	 * 5.2 Link State Power Management (Page #428).
1228	 */
1229
1230	list_for_each_entry(child, &pp->bridge->bus->children, node) {
1231		/* Bring downstream devices to D0 if they are not already in */
1232		if (child->parent == pp->bridge->bus) {
1233			root_bus = child;
1234			break;
1235		}
1236	}
1237
1238	if (!root_bus) {
1239		dev_err(pcie->dev, "Failed to find downstream devices\n");
1240		return;
1241	}
1242
1243	list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1244		if (PCI_SLOT(pdev->devfn) == 0) {
1245			if (pci_set_power_state(pdev, PCI_D0))
1246				dev_err(pcie->dev,
1247					"Failed to transition %s to D0 state\n",
1248					dev_name(&pdev->dev));
1249		}
1250	}
1251}
1252
1253static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1254{
1255	pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1256	if (IS_ERR(pcie->slot_ctl_3v3)) {
1257		if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1258			return PTR_ERR(pcie->slot_ctl_3v3);
1259
1260		pcie->slot_ctl_3v3 = NULL;
1261	}
1262
1263	pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1264	if (IS_ERR(pcie->slot_ctl_12v)) {
1265		if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1266			return PTR_ERR(pcie->slot_ctl_12v);
1267
1268		pcie->slot_ctl_12v = NULL;
1269	}
1270
1271	return 0;
1272}
1273
1274static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1275{
1276	int ret;
1277
1278	if (pcie->slot_ctl_3v3) {
1279		ret = regulator_enable(pcie->slot_ctl_3v3);
1280		if (ret < 0) {
1281			dev_err(pcie->dev,
1282				"Failed to enable 3.3V slot supply: %d\n", ret);
1283			return ret;
1284		}
1285	}
1286
1287	if (pcie->slot_ctl_12v) {
1288		ret = regulator_enable(pcie->slot_ctl_12v);
1289		if (ret < 0) {
1290			dev_err(pcie->dev,
1291				"Failed to enable 12V slot supply: %d\n", ret);
1292			goto fail_12v_enable;
1293		}
1294	}
1295
1296	/*
1297	 * According to PCI Express Card Electromechanical Specification
1298	 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1299	 * should be a minimum of 100ms.
1300	 */
1301	if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1302		msleep(100);
1303
1304	return 0;
1305
1306fail_12v_enable:
1307	if (pcie->slot_ctl_3v3)
1308		regulator_disable(pcie->slot_ctl_3v3);
1309	return ret;
1310}
1311
1312static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1313{
1314	if (pcie->slot_ctl_12v)
1315		regulator_disable(pcie->slot_ctl_12v);
1316	if (pcie->slot_ctl_3v3)
1317		regulator_disable(pcie->slot_ctl_3v3);
1318}
1319
1320static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1321					bool en_hw_hot_rst)
1322{
1323	int ret;
1324	u32 val;
1325
1326	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1327	if (ret) {
1328		dev_err(pcie->dev,
1329			"Failed to enable controller %u: %d\n", pcie->cid, ret);
1330		return ret;
1331	}
1332
 
 
 
 
 
 
 
 
1333	ret = tegra_pcie_enable_slot_regulators(pcie);
1334	if (ret < 0)
1335		goto fail_slot_reg_en;
1336
1337	ret = regulator_enable(pcie->pex_ctl_supply);
1338	if (ret < 0) {
1339		dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1340		goto fail_reg_en;
1341	}
1342
1343	ret = clk_prepare_enable(pcie->core_clk);
1344	if (ret) {
1345		dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1346		goto fail_core_clk;
1347	}
1348
1349	ret = reset_control_deassert(pcie->core_apb_rst);
1350	if (ret) {
1351		dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1352			ret);
1353		goto fail_core_apb_rst;
1354	}
1355
1356	if (en_hw_hot_rst) {
1357		/* Enable HW_HOT_RST mode */
1358		val = appl_readl(pcie, APPL_CTRL);
1359		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1360			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
 
 
1361		val |= APPL_CTRL_HW_HOT_RST_EN;
1362		appl_writel(pcie, val, APPL_CTRL);
1363	}
1364
1365	ret = tegra_pcie_enable_phy(pcie);
1366	if (ret) {
1367		dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1368		goto fail_phy;
1369	}
1370
1371	/* Update CFG base address */
1372	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1373		    APPL_CFG_BASE_ADDR);
1374
1375	/* Configure this core for RP mode operation */
1376	appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1377
1378	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1379
1380	val = appl_readl(pcie, APPL_CTRL);
1381	appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1382
1383	val = appl_readl(pcie, APPL_CFG_MISC);
1384	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1385	appl_writel(pcie, val, APPL_CFG_MISC);
1386
 
 
 
 
 
 
 
 
 
 
 
 
 
1387	if (!pcie->supports_clkreq) {
1388		val = appl_readl(pcie, APPL_PINMUX);
1389		val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1390		val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1391		appl_writel(pcie, val, APPL_PINMUX);
1392	}
1393
1394	/* Update iATU_DMA base address */
1395	appl_writel(pcie,
1396		    pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1397		    APPL_CFG_IATU_DMA_BASE_ADDR);
1398
1399	reset_control_deassert(pcie->core_rst);
1400
1401	return ret;
1402
1403fail_phy:
1404	reset_control_assert(pcie->core_apb_rst);
1405fail_core_apb_rst:
1406	clk_disable_unprepare(pcie->core_clk);
1407fail_core_clk:
1408	regulator_disable(pcie->pex_ctl_supply);
1409fail_reg_en:
1410	tegra_pcie_disable_slot_regulators(pcie);
1411fail_slot_reg_en:
 
 
 
1412	tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1413
1414	return ret;
1415}
1416
1417static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1418{
1419	int ret;
1420
1421	ret = reset_control_assert(pcie->core_rst);
1422	if (ret)
1423		dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1424
1425	tegra_pcie_disable_phy(pcie);
1426
1427	ret = reset_control_assert(pcie->core_apb_rst);
1428	if (ret)
1429		dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1430
1431	clk_disable_unprepare(pcie->core_clk);
1432
1433	ret = regulator_disable(pcie->pex_ctl_supply);
1434	if (ret)
1435		dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1436
1437	tegra_pcie_disable_slot_regulators(pcie);
1438
 
 
 
 
 
 
1439	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1440	if (ret)
1441		dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1442			pcie->cid, ret);
1443}
1444
1445static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1446{
1447	struct dw_pcie *pci = &pcie->pci;
1448	struct pcie_port *pp = &pci->pp;
1449	int ret;
1450
1451	ret = tegra_pcie_config_controller(pcie, false);
1452	if (ret < 0)
1453		return ret;
1454
1455	pp->ops = &tegra_pcie_dw_host_ops;
1456
1457	ret = dw_pcie_host_init(pp);
1458	if (ret < 0) {
1459		dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1460		goto fail_host_init;
1461	}
1462
1463	return 0;
1464
1465fail_host_init:
1466	tegra_pcie_unconfig_controller(pcie);
1467	return ret;
1468}
1469
1470static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1471{
1472	u32 val;
1473
1474	if (!tegra_pcie_dw_link_up(&pcie->pci))
1475		return 0;
1476
1477	val = appl_readl(pcie, APPL_RADM_STATUS);
1478	val |= APPL_PM_XMT_TURNOFF_STATE;
1479	appl_writel(pcie, val, APPL_RADM_STATUS);
1480
1481	return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1482				 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1483				 1, PME_ACK_TIMEOUT);
1484}
1485
1486static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1487{
1488	u32 data;
1489	int err;
1490
1491	if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1492		dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1493		return;
1494	}
1495
 
 
 
 
 
 
 
 
 
 
1496	if (tegra_pcie_try_link_l2(pcie)) {
1497		dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1498		/*
1499		 * TX lane clock freq will reset to Gen1 only if link is in L2
1500		 * or detect state.
1501		 * So apply pex_rst to end point to force RP to go into detect
1502		 * state
1503		 */
1504		data = appl_readl(pcie, APPL_PINMUX);
1505		data &= ~APPL_PINMUX_PEX_RST;
1506		appl_writel(pcie, data, APPL_PINMUX);
1507
1508		/*
1509		 * Some cards do not go to detect state even after de-asserting
1510		 * PERST#. So, de-assert LTSSM to bring link to detect state.
1511		 */
1512		data = readl(pcie->appl_base + APPL_CTRL);
1513		data &= ~APPL_CTRL_LTSSM_EN;
1514		writel(data, pcie->appl_base + APPL_CTRL);
1515
1516		err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1517						data,
1518						((data &
1519						APPL_DEBUG_LTSSM_STATE_MASK) >>
1520						APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1521						LTSSM_STATE_PRE_DETECT,
1522						1, LTSSM_TIMEOUT);
1523		if (err)
1524			dev_info(pcie->dev, "Link didn't go to detect state\n");
1525	}
1526	/*
1527	 * DBI registers may not be accessible after this as PLL-E would be
1528	 * down depending on how CLKREQ is pulled by end point
1529	 */
1530	data = appl_readl(pcie, APPL_PINMUX);
1531	data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1532	/* Cut REFCLK to slot */
1533	data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1534	data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1535	appl_writel(pcie, data, APPL_PINMUX);
1536}
1537
1538static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1539{
1540	tegra_pcie_downstream_dev_to_D0(pcie);
1541	dw_pcie_host_deinit(&pcie->pci.pp);
1542	tegra_pcie_dw_pme_turnoff(pcie);
1543	tegra_pcie_unconfig_controller(pcie);
1544}
1545
1546static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1547{
1548	struct device *dev = pcie->dev;
1549	char *name;
1550	int ret;
1551
1552	pm_runtime_enable(dev);
1553
1554	ret = pm_runtime_get_sync(dev);
1555	if (ret < 0) {
1556		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1557			ret);
1558		goto fail_pm_get_sync;
1559	}
1560
1561	ret = pinctrl_pm_select_default_state(dev);
1562	if (ret < 0) {
1563		dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1564		goto fail_pm_get_sync;
1565	}
1566
1567	ret = tegra_pcie_init_controller(pcie);
1568	if (ret < 0) {
1569		dev_err(dev, "Failed to initialize controller: %d\n", ret);
1570		goto fail_pm_get_sync;
1571	}
1572
1573	pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1574	if (!pcie->link_state) {
1575		ret = -ENOMEDIUM;
1576		goto fail_host_init;
1577	}
1578
1579	name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1580	if (!name) {
1581		ret = -ENOMEM;
1582		goto fail_host_init;
1583	}
1584
1585	pcie->debugfs = debugfs_create_dir(name, NULL);
1586	init_debugfs(pcie);
1587
1588	return ret;
1589
1590fail_host_init:
1591	tegra_pcie_deinit_controller(pcie);
1592fail_pm_get_sync:
1593	pm_runtime_put_sync(dev);
1594	pm_runtime_disable(dev);
1595	return ret;
1596}
1597
1598static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1599{
1600	u32 val;
1601	int ret;
1602
1603	if (pcie->ep_state == EP_STATE_DISABLED)
1604		return;
1605
1606	/* Disable LTSSM */
1607	val = appl_readl(pcie, APPL_CTRL);
1608	val &= ~APPL_CTRL_LTSSM_EN;
1609	appl_writel(pcie, val, APPL_CTRL);
1610
1611	ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1612				 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1613				 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1614				 LTSSM_STATE_PRE_DETECT,
1615				 1, LTSSM_TIMEOUT);
1616	if (ret)
1617		dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1618
1619	reset_control_assert(pcie->core_rst);
1620
1621	tegra_pcie_disable_phy(pcie);
1622
1623	reset_control_assert(pcie->core_apb_rst);
1624
1625	clk_disable_unprepare(pcie->core_clk);
1626
1627	pm_runtime_put_sync(pcie->dev);
1628
 
 
 
 
 
 
 
1629	ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1630	if (ret)
1631		dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1632
1633	pcie->ep_state = EP_STATE_DISABLED;
1634	dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1635}
1636
1637static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1638{
1639	struct dw_pcie *pci = &pcie->pci;
1640	struct dw_pcie_ep *ep = &pci->ep;
1641	struct device *dev = pcie->dev;
1642	u32 val;
1643	int ret;
 
1644
1645	if (pcie->ep_state == EP_STATE_ENABLED)
1646		return;
1647
1648	ret = pm_runtime_resume_and_get(dev);
1649	if (ret < 0) {
1650		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1651			ret);
1652		return;
1653	}
1654
1655	ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1656	if (ret) {
1657		dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret);
1658		goto fail_pll_init;
 
 
 
 
 
 
 
 
 
 
1659	}
1660
1661	ret = clk_prepare_enable(pcie->core_clk);
1662	if (ret) {
1663		dev_err(dev, "Failed to enable core clock: %d\n", ret);
1664		goto fail_core_clk_enable;
1665	}
1666
1667	ret = reset_control_deassert(pcie->core_apb_rst);
1668	if (ret) {
1669		dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1670		goto fail_core_apb_rst;
1671	}
1672
1673	ret = tegra_pcie_enable_phy(pcie);
1674	if (ret) {
1675		dev_err(dev, "Failed to enable PHY: %d\n", ret);
1676		goto fail_phy;
1677	}
1678
1679	/* Clear any stale interrupt statuses */
1680	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1681	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1682	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1683	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1684	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1685	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1686	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1687	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1688	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1689	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1690	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1691	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1692	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1693	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1694	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1695
1696	/* configure this core for EP mode operation */
1697	val = appl_readl(pcie, APPL_DM_TYPE);
1698	val &= ~APPL_DM_TYPE_MASK;
1699	val |= APPL_DM_TYPE_EP;
1700	appl_writel(pcie, val, APPL_DM_TYPE);
1701
1702	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1703
1704	val = appl_readl(pcie, APPL_CTRL);
1705	val |= APPL_CTRL_SYS_PRE_DET_STATE;
1706	val |= APPL_CTRL_HW_HOT_RST_EN;
1707	appl_writel(pcie, val, APPL_CTRL);
1708
1709	val = appl_readl(pcie, APPL_CFG_MISC);
1710	val |= APPL_CFG_MISC_SLV_EP_MODE;
1711	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1712	appl_writel(pcie, val, APPL_CFG_MISC);
1713
1714	val = appl_readl(pcie, APPL_PINMUX);
1715	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1716	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1717	appl_writel(pcie, val, APPL_PINMUX);
1718
1719	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1720		    APPL_CFG_BASE_ADDR);
1721
1722	appl_writel(pcie, pcie->atu_dma_res->start &
1723		    APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1724		    APPL_CFG_IATU_DMA_BASE_ADDR);
1725
1726	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1727	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1728	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1729	val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1730	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1731
1732	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1733	val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1734	val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1735	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1736
1737	reset_control_deassert(pcie->core_rst);
1738
1739	if (pcie->update_fc_fixup) {
1740		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1741		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1742		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1743	}
1744
1745	config_gen3_gen4_eq_presets(pcie);
1746
1747	init_host_aspm(pcie);
1748
1749	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1750	if (!pcie->supports_clkreq) {
1751		disable_aspm_l11(pcie);
1752		disable_aspm_l12(pcie);
1753	}
1754
1755	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1756	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1757	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 
 
1758
1759	pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1760						      PCI_CAP_ID_EXP);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1761	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1762
1763	val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1764	val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1765	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1766	val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1767	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1768
1769	ret = dw_pcie_ep_init_complete(ep);
1770	if (ret) {
1771		dev_err(dev, "Failed to complete initialization: %d\n", ret);
1772		goto fail_init_complete;
1773	}
1774
1775	dw_pcie_ep_init_notify(ep);
1776
 
 
 
 
 
 
 
1777	/* Enable LTSSM */
1778	val = appl_readl(pcie, APPL_CTRL);
1779	val |= APPL_CTRL_LTSSM_EN;
1780	appl_writel(pcie, val, APPL_CTRL);
1781
1782	pcie->ep_state = EP_STATE_ENABLED;
1783	dev_dbg(dev, "Initialization of endpoint is completed\n");
1784
1785	return;
1786
1787fail_init_complete:
1788	reset_control_assert(pcie->core_rst);
1789	tegra_pcie_disable_phy(pcie);
1790fail_phy:
1791	reset_control_assert(pcie->core_apb_rst);
1792fail_core_apb_rst:
1793	clk_disable_unprepare(pcie->core_clk);
1794fail_core_clk_enable:
1795	tegra_pcie_bpmp_set_pll_state(pcie, false);
1796fail_pll_init:
 
 
1797	pm_runtime_put_sync(dev);
1798}
1799
1800static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1801{
1802	struct tegra_pcie_dw *pcie = arg;
1803
1804	if (gpiod_get_value(pcie->pex_rst_gpiod))
1805		pex_ep_event_pex_rst_assert(pcie);
1806	else
1807		pex_ep_event_pex_rst_deassert(pcie);
1808
1809	return IRQ_HANDLED;
1810}
1811
1812static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1813{
1814	/* Tegra194 supports only INTA */
1815	if (irq > 1)
1816		return -EINVAL;
1817
1818	appl_writel(pcie, 1, APPL_LEGACY_INTX);
1819	usleep_range(1000, 2000);
1820	appl_writel(pcie, 0, APPL_LEGACY_INTX);
1821	return 0;
1822}
1823
1824static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1825{
1826	if (unlikely(irq > 31))
1827		return -EINVAL;
1828
1829	appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1830
1831	return 0;
1832}
1833
1834static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1835{
1836	struct dw_pcie_ep *ep = &pcie->pci.ep;
1837
1838	writel(irq, ep->msi_mem);
1839
1840	return 0;
1841}
1842
1843static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1844				   enum pci_epc_irq_type type,
1845				   u16 interrupt_num)
1846{
1847	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1848	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1849
1850	switch (type) {
1851	case PCI_EPC_IRQ_LEGACY:
1852		return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
1853
1854	case PCI_EPC_IRQ_MSI:
1855		return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1856
1857	case PCI_EPC_IRQ_MSIX:
1858		return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1859
1860	default:
1861		dev_err(pci->dev, "Unknown IRQ type\n");
1862		return -EPERM;
1863	}
1864
1865	return 0;
1866}
1867
1868static const struct pci_epc_features tegra_pcie_epc_features = {
1869	.linkup_notifier = true,
1870	.core_init_notifier = true,
1871	.msi_capable = false,
1872	.msix_capable = false,
1873	.reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
1874	.bar_fixed_64bit = 1 << BAR_0,
1875	.bar_fixed_size[0] = SZ_1M,
1876};
1877
1878static const struct pci_epc_features*
1879tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
1880{
1881	return &tegra_pcie_epc_features;
1882}
1883
1884static const struct dw_pcie_ep_ops pcie_ep_ops = {
1885	.raise_irq = tegra_pcie_ep_raise_irq,
1886	.get_features = tegra_pcie_ep_get_features,
1887};
1888
1889static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
1890				struct platform_device *pdev)
1891{
1892	struct dw_pcie *pci = &pcie->pci;
1893	struct device *dev = pcie->dev;
1894	struct dw_pcie_ep *ep;
1895	char *name;
1896	int ret;
1897
1898	ep = &pci->ep;
1899	ep->ops = &pcie_ep_ops;
1900
1901	ep->page_size = SZ_64K;
1902
1903	ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
1904	if (ret < 0) {
1905		dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
1906			ret);
1907		return ret;
1908	}
1909
1910	ret = gpiod_to_irq(pcie->pex_rst_gpiod);
1911	if (ret < 0) {
1912		dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
1913		return ret;
1914	}
1915	pcie->pex_rst_irq = (unsigned int)ret;
1916
1917	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
1918			      pcie->cid);
1919	if (!name) {
1920		dev_err(dev, "Failed to create PERST IRQ string\n");
1921		return -ENOMEM;
1922	}
1923
1924	irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
1925
1926	pcie->ep_state = EP_STATE_DISABLED;
1927
1928	ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
1929					tegra_pcie_ep_pex_rst_irq,
1930					IRQF_TRIGGER_RISING |
1931					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
1932					name, (void *)pcie);
1933	if (ret < 0) {
1934		dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
1935		return ret;
1936	}
1937
1938	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work",
1939			      pcie->cid);
1940	if (!name) {
1941		dev_err(dev, "Failed to create PCIe EP work thread string\n");
1942		return -ENOMEM;
1943	}
1944
1945	pm_runtime_enable(dev);
1946
1947	ret = dw_pcie_ep_init(ep);
1948	if (ret) {
1949		dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
1950			ret);
 
1951		return ret;
1952	}
1953
1954	return 0;
1955}
1956
1957static int tegra_pcie_dw_probe(struct platform_device *pdev)
1958{
1959	const struct tegra_pcie_dw_of_data *data;
1960	struct device *dev = &pdev->dev;
1961	struct resource *atu_dma_res;
1962	struct tegra_pcie_dw *pcie;
1963	struct pcie_port *pp;
1964	struct dw_pcie *pci;
1965	struct phy **phys;
1966	char *name;
1967	int ret;
1968	u32 i;
1969
1970	data = of_device_get_match_data(dev);
1971
1972	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
1973	if (!pcie)
1974		return -ENOMEM;
1975
1976	pci = &pcie->pci;
1977	pci->dev = &pdev->dev;
1978	pci->ops = &tegra_dw_pcie_ops;
1979	pci->n_fts[0] = N_FTS_VAL;
1980	pci->n_fts[1] = FTS_VAL;
1981	pci->version = 0x490A;
1982
1983	pp = &pci->pp;
1984	pp->num_vectors = MAX_MSI_IRQS;
1985	pcie->dev = &pdev->dev;
1986	pcie->mode = (enum dw_pcie_device_mode)data->mode;
1987
1988	ret = tegra_pcie_dw_parse_dt(pcie);
1989	if (ret < 0) {
1990		const char *level = KERN_ERR;
1991
1992		if (ret == -EPROBE_DEFER)
1993			level = KERN_DEBUG;
1994
1995		dev_printk(level, dev,
1996			   dev_fmt("Failed to parse device tree: %d\n"),
1997			   ret);
1998		return ret;
1999	}
2000
2001	ret = tegra_pcie_get_slot_regulators(pcie);
2002	if (ret < 0) {
2003		const char *level = KERN_ERR;
2004
2005		if (ret == -EPROBE_DEFER)
2006			level = KERN_DEBUG;
2007
2008		dev_printk(level, dev,
2009			   dev_fmt("Failed to get slot regulators: %d\n"),
2010			   ret);
2011		return ret;
2012	}
2013
2014	if (pcie->pex_refclk_sel_gpiod)
2015		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2016
2017	pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2018	if (IS_ERR(pcie->pex_ctl_supply)) {
2019		ret = PTR_ERR(pcie->pex_ctl_supply);
2020		if (ret != -EPROBE_DEFER)
2021			dev_err(dev, "Failed to get regulator: %ld\n",
2022				PTR_ERR(pcie->pex_ctl_supply));
2023		return ret;
2024	}
2025
2026	pcie->core_clk = devm_clk_get(dev, "core");
2027	if (IS_ERR(pcie->core_clk)) {
2028		dev_err(dev, "Failed to get core clock: %ld\n",
2029			PTR_ERR(pcie->core_clk));
2030		return PTR_ERR(pcie->core_clk);
2031	}
2032
2033	pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2034						      "appl");
2035	if (!pcie->appl_res) {
2036		dev_err(dev, "Failed to find \"appl\" region\n");
2037		return -ENODEV;
2038	}
2039
2040	pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2041	if (IS_ERR(pcie->appl_base))
2042		return PTR_ERR(pcie->appl_base);
2043
2044	pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2045	if (IS_ERR(pcie->core_apb_rst)) {
2046		dev_err(dev, "Failed to get APB reset: %ld\n",
2047			PTR_ERR(pcie->core_apb_rst));
2048		return PTR_ERR(pcie->core_apb_rst);
2049	}
2050
2051	phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2052	if (!phys)
2053		return -ENOMEM;
2054
2055	for (i = 0; i < pcie->phy_count; i++) {
2056		name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2057		if (!name) {
2058			dev_err(dev, "Failed to create P2U string\n");
2059			return -ENOMEM;
2060		}
2061		phys[i] = devm_phy_get(dev, name);
2062		kfree(name);
2063		if (IS_ERR(phys[i])) {
2064			ret = PTR_ERR(phys[i]);
2065			if (ret != -EPROBE_DEFER)
2066				dev_err(dev, "Failed to get PHY: %d\n", ret);
2067			return ret;
2068		}
2069	}
2070
2071	pcie->phys = phys;
2072
2073	atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2074						   "atu_dma");
2075	if (!atu_dma_res) {
2076		dev_err(dev, "Failed to find \"atu_dma\" region\n");
2077		return -ENODEV;
2078	}
2079	pcie->atu_dma_res = atu_dma_res;
2080
2081	pci->atu_size = resource_size(atu_dma_res);
2082	pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2083	if (IS_ERR(pci->atu_base))
2084		return PTR_ERR(pci->atu_base);
2085
2086	pcie->core_rst = devm_reset_control_get(dev, "core");
2087	if (IS_ERR(pcie->core_rst)) {
2088		dev_err(dev, "Failed to get core reset: %ld\n",
2089			PTR_ERR(pcie->core_rst));
2090		return PTR_ERR(pcie->core_rst);
2091	}
2092
2093	pp->irq = platform_get_irq_byname(pdev, "intr");
2094	if (pp->irq < 0)
2095		return pp->irq;
2096
2097	pcie->bpmp = tegra_bpmp_get(dev);
2098	if (IS_ERR(pcie->bpmp))
2099		return PTR_ERR(pcie->bpmp);
2100
2101	platform_set_drvdata(pdev, pcie);
2102
2103	switch (pcie->mode) {
2104	case DW_PCIE_RC_TYPE:
2105		ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2106				       IRQF_SHARED, "tegra-pcie-intr", pcie);
2107		if (ret) {
2108			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2109				ret);
2110			goto fail;
2111		}
2112
2113		ret = tegra_pcie_config_rp(pcie);
2114		if (ret && ret != -ENOMEDIUM)
2115			goto fail;
2116		else
2117			return 0;
2118		break;
2119
2120	case DW_PCIE_EP_TYPE:
2121		ret = devm_request_threaded_irq(dev, pp->irq,
2122						tegra_pcie_ep_hard_irq,
2123						tegra_pcie_ep_irq_thread,
2124						IRQF_SHARED | IRQF_ONESHOT,
2125						"tegra-pcie-ep-intr", pcie);
2126		if (ret) {
2127			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2128				ret);
2129			goto fail;
2130		}
2131
2132		ret = tegra_pcie_config_ep(pcie, pdev);
2133		if (ret < 0)
2134			goto fail;
2135		break;
2136
2137	default:
2138		dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode);
 
2139	}
2140
2141fail:
2142	tegra_bpmp_put(pcie->bpmp);
2143	return ret;
2144}
2145
2146static int tegra_pcie_dw_remove(struct platform_device *pdev)
2147{
2148	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2149
2150	if (!pcie->link_state)
2151		return 0;
 
 
 
 
 
 
 
 
 
2152
2153	debugfs_remove_recursive(pcie->debugfs);
2154	tegra_pcie_deinit_controller(pcie);
2155	pm_runtime_put_sync(pcie->dev);
2156	pm_runtime_disable(pcie->dev);
2157	tegra_bpmp_put(pcie->bpmp);
2158	if (pcie->pex_refclk_sel_gpiod)
2159		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2160
2161	return 0;
2162}
2163
2164static int tegra_pcie_dw_suspend_late(struct device *dev)
2165{
2166	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2167	u32 val;
2168
 
 
 
 
 
2169	if (!pcie->link_state)
2170		return 0;
2171
2172	/* Enable HW_HOT_RST mode */
2173	val = appl_readl(pcie, APPL_CTRL);
2174	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2175		 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2176	val |= APPL_CTRL_HW_HOT_RST_EN;
2177	appl_writel(pcie, val, APPL_CTRL);
 
 
2178
2179	return 0;
2180}
2181
2182static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2183{
2184	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2185
2186	if (!pcie->link_state)
2187		return 0;
2188
2189	/* Save MSI interrupt vector */
2190	pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci,
2191					       PORT_LOGIC_MSI_CTRL_INT_0_EN);
2192	tegra_pcie_downstream_dev_to_D0(pcie);
2193	tegra_pcie_dw_pme_turnoff(pcie);
2194	tegra_pcie_unconfig_controller(pcie);
2195
2196	return 0;
2197}
2198
2199static int tegra_pcie_dw_resume_noirq(struct device *dev)
2200{
2201	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2202	int ret;
2203
2204	if (!pcie->link_state)
2205		return 0;
2206
2207	ret = tegra_pcie_config_controller(pcie, true);
2208	if (ret < 0)
2209		return ret;
2210
2211	ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2212	if (ret < 0) {
2213		dev_err(dev, "Failed to init host: %d\n", ret);
2214		goto fail_host_init;
2215	}
2216
2217	dw_pcie_setup_rc(&pcie->pci.pp);
2218
2219	ret = tegra_pcie_dw_start_link(&pcie->pci);
2220	if (ret < 0)
2221		goto fail_host_init;
2222
2223	/* Restore MSI interrupt vector */
2224	dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN,
2225			   pcie->msi_ctrl_int);
2226
2227	return 0;
2228
2229fail_host_init:
2230	tegra_pcie_unconfig_controller(pcie);
2231	return ret;
2232}
2233
2234static int tegra_pcie_dw_resume_early(struct device *dev)
2235{
2236	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2237	u32 val;
2238
 
 
 
 
 
2239	if (!pcie->link_state)
2240		return 0;
2241
2242	/* Disable HW_HOT_RST mode */
2243	val = appl_readl(pcie, APPL_CTRL);
2244	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2245		 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2246	val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2247	       APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2248	val &= ~APPL_CTRL_HW_HOT_RST_EN;
2249	appl_writel(pcie, val, APPL_CTRL);
 
 
2250
2251	return 0;
2252}
2253
2254static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2255{
2256	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2257
2258	if (!pcie->link_state)
2259		return;
2260
2261	debugfs_remove_recursive(pcie->debugfs);
2262	tegra_pcie_downstream_dev_to_D0(pcie);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2263
2264	disable_irq(pcie->pci.pp.irq);
2265	if (IS_ENABLED(CONFIG_PCI_MSI))
2266		disable_irq(pcie->pci.pp.msi_irq);
 
 
 
 
 
2267
2268	tegra_pcie_dw_pme_turnoff(pcie);
2269	tegra_pcie_unconfig_controller(pcie);
2270}
 
 
 
 
 
2271
2272static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = {
 
2273	.mode = DW_PCIE_RC_TYPE,
 
 
 
 
 
 
 
2274};
2275
2276static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = {
 
2277	.mode = DW_PCIE_EP_TYPE,
 
 
 
 
 
 
2278};
2279
2280static const struct of_device_id tegra_pcie_dw_of_match[] = {
2281	{
2282		.compatible = "nvidia,tegra194-pcie",
2283		.data = &tegra_pcie_dw_rc_of_data,
2284	},
2285	{
2286		.compatible = "nvidia,tegra194-pcie-ep",
2287		.data = &tegra_pcie_dw_ep_of_data,
 
 
 
 
 
 
 
 
2288	},
2289	{},
2290};
2291
2292static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2293	.suspend_late = tegra_pcie_dw_suspend_late,
2294	.suspend_noirq = tegra_pcie_dw_suspend_noirq,
2295	.resume_noirq = tegra_pcie_dw_resume_noirq,
2296	.resume_early = tegra_pcie_dw_resume_early,
2297};
2298
2299static struct platform_driver tegra_pcie_dw_driver = {
2300	.probe = tegra_pcie_dw_probe,
2301	.remove = tegra_pcie_dw_remove,
2302	.shutdown = tegra_pcie_dw_shutdown,
2303	.driver = {
2304		.name	= "tegra194-pcie",
2305		.pm = &tegra_pcie_dw_pm_ops,
2306		.of_match_table = tegra_pcie_dw_of_match,
2307	},
2308};
2309module_platform_driver(tegra_pcie_dw_driver);
2310
2311MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2312
2313MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2314MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2315MODULE_LICENSE("GPL v2");