1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Driver for the Aardvark PCIe controller, used on Marvell Armada
   4 * 3700.
   5 *
   6 * Copyright (C) 2016 Marvell
   7 *
   8 * Author: Hezi Shahmoon <hezi.shahmoon@marvell.com>
   9 */
  10
  11#include <linux/bitfield.h>
  12#include <linux/delay.h>
  13#include <linux/gpio/consumer.h>
  14#include <linux/interrupt.h>
  15#include <linux/irq.h>
  16#include <linux/irqdomain.h>
  17#include <linux/kernel.h>
  18#include <linux/module.h>
  19#include <linux/pci.h>
  20#include <linux/pci-ecam.h>
  21#include <linux/init.h>
  22#include <linux/phy/phy.h>
  23#include <linux/platform_device.h>
  24#include <linux/msi.h>
  25#include <linux/of_address.h>
  26#include <linux/of_gpio.h>
  27#include <linux/of_pci.h>
  28
  29#include "../pci.h"
  30#include "../pci-bridge-emul.h"
  31
  32/* PCIe core registers */
  33#define PCIE_CORE_DEV_ID_REG					0x0
  34#define PCIE_CORE_CMD_STATUS_REG				0x4
 
 
 
  35#define PCIE_CORE_DEV_REV_REG					0x8
  36#define PCIE_CORE_SSDEV_ID_REG					0x2c
  37#define PCIE_CORE_PCIEXP_CAP					0xc0
  38#define PCIE_CORE_PCIERR_CAP					0x100
  39#define PCIE_CORE_ERR_CAPCTL_REG				0x118
  40#define     PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX			BIT(5)
  41#define     PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN			BIT(6)
  42#define     PCIE_CORE_ERR_CAPCTL_ECRC_CHCK			BIT(7)
  43#define     PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV			BIT(8)
 
 
 
 
  44/* PIO registers base address and register offsets */
  45#define PIO_BASE_ADDR				0x4000
  46#define PIO_CTRL				(PIO_BASE_ADDR + 0x0)
  47#define   PIO_CTRL_TYPE_MASK			GENMASK(3, 0)
  48#define   PIO_CTRL_ADDR_WIN_DISABLE		BIT(24)
  49#define PIO_STAT				(PIO_BASE_ADDR + 0x4)
  50#define   PIO_COMPLETION_STATUS_SHIFT		7
  51#define   PIO_COMPLETION_STATUS_MASK		GENMASK(9, 7)
  52#define   PIO_COMPLETION_STATUS_OK		0
  53#define   PIO_COMPLETION_STATUS_UR		1
  54#define   PIO_COMPLETION_STATUS_CRS		2
  55#define   PIO_COMPLETION_STATUS_CA		4
  56#define   PIO_NON_POSTED_REQ			BIT(10)
  57#define   PIO_ERR_STATUS			BIT(11)
  58#define PIO_ADDR_LS				(PIO_BASE_ADDR + 0x8)
  59#define PIO_ADDR_MS				(PIO_BASE_ADDR + 0xc)
  60#define PIO_WR_DATA				(PIO_BASE_ADDR + 0x10)
  61#define PIO_WR_DATA_STRB			(PIO_BASE_ADDR + 0x14)
  62#define PIO_RD_DATA				(PIO_BASE_ADDR + 0x18)
  63#define PIO_START				(PIO_BASE_ADDR + 0x1c)
  64#define PIO_ISR					(PIO_BASE_ADDR + 0x20)
  65#define PIO_ISRM				(PIO_BASE_ADDR + 0x24)
  66
  67/* Aardvark Control registers */
  68#define CONTROL_BASE_ADDR			0x4800
  69#define PCIE_CORE_CTRL0_REG			(CONTROL_BASE_ADDR + 0x0)
  70#define     PCIE_GEN_SEL_MSK			0x3
  71#define     PCIE_GEN_SEL_SHIFT			0x0
  72#define     SPEED_GEN_1				0
  73#define     SPEED_GEN_2				1
  74#define     SPEED_GEN_3				2
  75#define     IS_RC_MSK				1
  76#define     IS_RC_SHIFT				2
  77#define     LANE_CNT_MSK			0x18
  78#define     LANE_CNT_SHIFT			0x3
  79#define     LANE_COUNT_1			(0 << LANE_CNT_SHIFT)
  80#define     LANE_COUNT_2			(1 << LANE_CNT_SHIFT)
  81#define     LANE_COUNT_4			(2 << LANE_CNT_SHIFT)
  82#define     LANE_COUNT_8			(3 << LANE_CNT_SHIFT)
  83#define     LINK_TRAINING_EN			BIT(6)
  84#define     LEGACY_INTA				BIT(28)
  85#define     LEGACY_INTB				BIT(29)
  86#define     LEGACY_INTC				BIT(30)
  87#define     LEGACY_INTD				BIT(31)
  88#define PCIE_CORE_CTRL1_REG			(CONTROL_BASE_ADDR + 0x4)
  89#define     HOT_RESET_GEN			BIT(0)
  90#define PCIE_CORE_CTRL2_REG			(CONTROL_BASE_ADDR + 0x8)
  91#define     PCIE_CORE_CTRL2_RESERVED		0x7
  92#define     PCIE_CORE_CTRL2_TD_ENABLE		BIT(4)
  93#define     PCIE_CORE_CTRL2_STRICT_ORDER_ENABLE	BIT(5)
  94#define     PCIE_CORE_CTRL2_OB_WIN_ENABLE	BIT(6)
  95#define     PCIE_CORE_CTRL2_MSI_ENABLE		BIT(10)
  96#define PCIE_CORE_REF_CLK_REG			(CONTROL_BASE_ADDR + 0x14)
  97#define     PCIE_CORE_REF_CLK_TX_ENABLE		BIT(1)
  98#define     PCIE_CORE_REF_CLK_RX_ENABLE		BIT(2)
  99#define PCIE_MSG_LOG_REG			(CONTROL_BASE_ADDR + 0x30)
 100#define PCIE_ISR0_REG				(CONTROL_BASE_ADDR + 0x40)
 101#define PCIE_MSG_PM_PME_MASK			BIT(7)
 102#define PCIE_ISR0_MASK_REG			(CONTROL_BASE_ADDR + 0x44)
 103#define     PCIE_ISR0_MSI_INT_PENDING		BIT(24)
 104#define     PCIE_ISR0_CORR_ERR			BIT(11)
 105#define     PCIE_ISR0_NFAT_ERR			BIT(12)
 106#define     PCIE_ISR0_FAT_ERR			BIT(13)
 107#define     PCIE_ISR0_ERR_MASK			GENMASK(13, 11)
 108#define     PCIE_ISR0_INTX_ASSERT(val)		BIT(16 + (val))
 109#define     PCIE_ISR0_INTX_DEASSERT(val)	BIT(20 + (val))
 110#define     PCIE_ISR0_ALL_MASK			GENMASK(31, 0)
 111#define PCIE_ISR1_REG				(CONTROL_BASE_ADDR + 0x48)
 112#define PCIE_ISR1_MASK_REG			(CONTROL_BASE_ADDR + 0x4C)
 113#define     PCIE_ISR1_POWER_STATE_CHANGE	BIT(4)
 114#define     PCIE_ISR1_FLUSH			BIT(5)
 115#define     PCIE_ISR1_INTX_ASSERT(val)		BIT(8 + (val))
 116#define     PCIE_ISR1_ALL_MASK			GENMASK(31, 0)
 117#define PCIE_MSI_ADDR_LOW_REG			(CONTROL_BASE_ADDR + 0x50)
 118#define PCIE_MSI_ADDR_HIGH_REG			(CONTROL_BASE_ADDR + 0x54)
 119#define PCIE_MSI_STATUS_REG			(CONTROL_BASE_ADDR + 0x58)
 120#define PCIE_MSI_MASK_REG			(CONTROL_BASE_ADDR + 0x5C)
 121#define     PCIE_MSI_ALL_MASK			GENMASK(31, 0)
 122#define PCIE_MSI_PAYLOAD_REG			(CONTROL_BASE_ADDR + 0x9C)
 123#define     PCIE_MSI_DATA_MASK			GENMASK(15, 0)
 124
 125/* PCIe window configuration */
 126#define OB_WIN_BASE_ADDR			0x4c00
 127#define OB_WIN_BLOCK_SIZE			0x20
 128#define OB_WIN_COUNT				8
 129#define OB_WIN_REG_ADDR(win, offset)		(OB_WIN_BASE_ADDR + \
 130						 OB_WIN_BLOCK_SIZE * (win) + \
 131						 (offset))
 132#define OB_WIN_MATCH_LS(win)			OB_WIN_REG_ADDR(win, 0x00)
 133#define     OB_WIN_ENABLE			BIT(0)
 134#define OB_WIN_MATCH_MS(win)			OB_WIN_REG_ADDR(win, 0x04)
 135#define OB_WIN_REMAP_LS(win)			OB_WIN_REG_ADDR(win, 0x08)
 136#define OB_WIN_REMAP_MS(win)			OB_WIN_REG_ADDR(win, 0x0c)
 137#define OB_WIN_MASK_LS(win)			OB_WIN_REG_ADDR(win, 0x10)
 138#define OB_WIN_MASK_MS(win)			OB_WIN_REG_ADDR(win, 0x14)
 139#define OB_WIN_ACTIONS(win)			OB_WIN_REG_ADDR(win, 0x18)
 140#define OB_WIN_DEFAULT_ACTIONS			(OB_WIN_ACTIONS(OB_WIN_COUNT-1) + 0x4)
 141#define     OB_WIN_FUNC_NUM_MASK		GENMASK(31, 24)
 142#define     OB_WIN_FUNC_NUM_SHIFT		24
 143#define     OB_WIN_FUNC_NUM_ENABLE		BIT(23)
 144#define     OB_WIN_BUS_NUM_BITS_MASK		GENMASK(22, 20)
 145#define     OB_WIN_BUS_NUM_BITS_SHIFT		20
 146#define     OB_WIN_MSG_CODE_ENABLE		BIT(22)
 147#define     OB_WIN_MSG_CODE_MASK		GENMASK(21, 14)
 148#define     OB_WIN_MSG_CODE_SHIFT		14
 149#define     OB_WIN_MSG_PAYLOAD_LEN		BIT(12)
 150#define     OB_WIN_ATTR_ENABLE			BIT(11)
 151#define     OB_WIN_ATTR_TC_MASK			GENMASK(10, 8)
 152#define     OB_WIN_ATTR_TC_SHIFT		8
 153#define     OB_WIN_ATTR_RELAXED			BIT(7)
 154#define     OB_WIN_ATTR_NOSNOOP			BIT(6)
 155#define     OB_WIN_ATTR_POISON			BIT(5)
 156#define     OB_WIN_ATTR_IDO			BIT(4)
 157#define     OB_WIN_TYPE_MASK			GENMASK(3, 0)
 158#define     OB_WIN_TYPE_SHIFT			0
 159#define     OB_WIN_TYPE_MEM			0x0
 160#define     OB_WIN_TYPE_IO			0x4
 161#define     OB_WIN_TYPE_CONFIG_TYPE0		0x8
 162#define     OB_WIN_TYPE_CONFIG_TYPE1		0x9
 163#define     OB_WIN_TYPE_MSG			0xc
 164
 165/* LMI registers base address and register offsets */
 166#define LMI_BASE_ADDR				0x6000
 167#define CFG_REG					(LMI_BASE_ADDR + 0x0)
 168#define     LTSSM_SHIFT				24
 169#define     LTSSM_MASK				0x3f
 
 170#define     RC_BAR_CONFIG			0x300
 171
 172/* LTSSM values in CFG_REG */
 173enum {
 174	LTSSM_DETECT_QUIET			= 0x0,
 175	LTSSM_DETECT_ACTIVE			= 0x1,
 176	LTSSM_POLLING_ACTIVE			= 0x2,
 177	LTSSM_POLLING_COMPLIANCE		= 0x3,
 178	LTSSM_POLLING_CONFIGURATION		= 0x4,
 179	LTSSM_CONFIG_LINKWIDTH_START		= 0x5,
 180	LTSSM_CONFIG_LINKWIDTH_ACCEPT		= 0x6,
 181	LTSSM_CONFIG_LANENUM_ACCEPT		= 0x7,
 182	LTSSM_CONFIG_LANENUM_WAIT		= 0x8,
 183	LTSSM_CONFIG_COMPLETE			= 0x9,
 184	LTSSM_CONFIG_IDLE			= 0xa,
 185	LTSSM_RECOVERY_RCVR_LOCK		= 0xb,
 186	LTSSM_RECOVERY_SPEED			= 0xc,
 187	LTSSM_RECOVERY_RCVR_CFG			= 0xd,
 188	LTSSM_RECOVERY_IDLE			= 0xe,
 189	LTSSM_L0				= 0x10,
 190	LTSSM_RX_L0S_ENTRY			= 0x11,
 191	LTSSM_RX_L0S_IDLE			= 0x12,
 192	LTSSM_RX_L0S_FTS			= 0x13,
 193	LTSSM_TX_L0S_ENTRY			= 0x14,
 194	LTSSM_TX_L0S_IDLE			= 0x15,
 195	LTSSM_TX_L0S_FTS			= 0x16,
 196	LTSSM_L1_ENTRY				= 0x17,
 197	LTSSM_L1_IDLE				= 0x18,
 198	LTSSM_L2_IDLE				= 0x19,
 199	LTSSM_L2_TRANSMIT_WAKE			= 0x1a,
 200	LTSSM_DISABLED				= 0x20,
 201	LTSSM_LOOPBACK_ENTRY_MASTER		= 0x21,
 202	LTSSM_LOOPBACK_ACTIVE_MASTER		= 0x22,
 203	LTSSM_LOOPBACK_EXIT_MASTER		= 0x23,
 204	LTSSM_LOOPBACK_ENTRY_SLAVE		= 0x24,
 205	LTSSM_LOOPBACK_ACTIVE_SLAVE		= 0x25,
 206	LTSSM_LOOPBACK_EXIT_SLAVE		= 0x26,
 207	LTSSM_HOT_RESET				= 0x27,
 208	LTSSM_RECOVERY_EQUALIZATION_PHASE0	= 0x28,
 209	LTSSM_RECOVERY_EQUALIZATION_PHASE1	= 0x29,
 210	LTSSM_RECOVERY_EQUALIZATION_PHASE2	= 0x2a,
 211	LTSSM_RECOVERY_EQUALIZATION_PHASE3	= 0x2b,
 212};
 213
 214#define VENDOR_ID_REG				(LMI_BASE_ADDR + 0x44)
 215
 216/* PCIe core controller registers */
 217#define CTRL_CORE_BASE_ADDR			0x18000
 218#define CTRL_CONFIG_REG				(CTRL_CORE_BASE_ADDR + 0x0)
 219#define     CTRL_MODE_SHIFT			0x0
 220#define     CTRL_MODE_MASK			0x1
 221#define     PCIE_CORE_MODE_DIRECT		0x0
 222#define     PCIE_CORE_MODE_COMMAND		0x1
 223
 224/* PCIe Central Interrupts Registers */
 225#define CENTRAL_INT_BASE_ADDR			0x1b000
 226#define HOST_CTRL_INT_STATUS_REG		(CENTRAL_INT_BASE_ADDR + 0x0)
 227#define HOST_CTRL_INT_MASK_REG			(CENTRAL_INT_BASE_ADDR + 0x4)
 228#define     PCIE_IRQ_CMDQ_INT			BIT(0)
 229#define     PCIE_IRQ_MSI_STATUS_INT		BIT(1)
 230#define     PCIE_IRQ_CMD_SENT_DONE		BIT(3)
 231#define     PCIE_IRQ_DMA_INT			BIT(4)
 232#define     PCIE_IRQ_IB_DXFERDONE		BIT(5)
 233#define     PCIE_IRQ_OB_DXFERDONE		BIT(6)
 234#define     PCIE_IRQ_OB_RXFERDONE		BIT(7)
 235#define     PCIE_IRQ_COMPQ_INT			BIT(12)
 236#define     PCIE_IRQ_DIR_RD_DDR_DET		BIT(13)
 237#define     PCIE_IRQ_DIR_WR_DDR_DET		BIT(14)
 238#define     PCIE_IRQ_CORE_INT			BIT(16)
 239#define     PCIE_IRQ_CORE_INT_PIO		BIT(17)
 240#define     PCIE_IRQ_DPMU_INT			BIT(18)
 241#define     PCIE_IRQ_PCIE_MIS_INT		BIT(19)
 242#define     PCIE_IRQ_MSI_INT1_DET		BIT(20)
 243#define     PCIE_IRQ_MSI_INT2_DET		BIT(21)
 244#define     PCIE_IRQ_RC_DBELL_DET		BIT(22)
 245#define     PCIE_IRQ_EP_STATUS			BIT(23)
 246#define     PCIE_IRQ_ALL_MASK			GENMASK(31, 0)
 247#define     PCIE_IRQ_ENABLE_INTS_MASK		PCIE_IRQ_CORE_INT
 248
 249/* Transaction types */
 250#define PCIE_CONFIG_RD_TYPE0			0x8
 251#define PCIE_CONFIG_RD_TYPE1			0x9
 252#define PCIE_CONFIG_WR_TYPE0			0xa
 253#define PCIE_CONFIG_WR_TYPE1			0xb
 254
 255#define PIO_RETRY_CNT			750000 /* 1.5 s */
 256#define PIO_RETRY_DELAY			2 /* 2 us*/
 257
 258#define LINK_WAIT_MAX_RETRIES		10
 259#define LINK_WAIT_USLEEP_MIN		90000
 260#define LINK_WAIT_USLEEP_MAX		100000
 261#define RETRAIN_WAIT_MAX_RETRIES	10
 262#define RETRAIN_WAIT_USLEEP_US		2000
 263
 264#define MSI_IRQ_NUM			32
 265
 266#define CFG_RD_CRS_VAL			0xffff0001
 267
 268struct advk_pcie {
 269	struct platform_device *pdev;
 270	void __iomem *base;
 271	struct {
 272		phys_addr_t match;
 273		phys_addr_t remap;
 274		phys_addr_t mask;
 275		u32 actions;
 276	} wins[OB_WIN_COUNT];
 277	u8 wins_count;
 278	struct irq_domain *rp_irq_domain;
 279	struct irq_domain *irq_domain;
 280	struct irq_chip irq_chip;
 281	raw_spinlock_t irq_lock;
 282	struct irq_domain *msi_domain;
 283	struct irq_domain *msi_inner_domain;
 284	raw_spinlock_t msi_irq_lock;
 
 
 285	DECLARE_BITMAP(msi_used, MSI_IRQ_NUM);
 286	struct mutex msi_used_lock;
 
 287	int link_gen;
 288	struct pci_bridge_emul bridge;
 289	struct gpio_desc *reset_gpio;
 290	struct phy *phy;
 291};
 292
 293static inline void advk_writel(struct advk_pcie *pcie, u32 val, u64 reg)
 294{
 295	writel(val, pcie->base + reg);
 296}
 297
 298static inline u32 advk_readl(struct advk_pcie *pcie, u64 reg)
 299{
 300	return readl(pcie->base + reg);
 301}
 302
 303static u8 advk_pcie_ltssm_state(struct advk_pcie *pcie)
 304{
 305	u32 val;
 306	u8 ltssm_state;
 307
 308	val = advk_readl(pcie, CFG_REG);
 309	ltssm_state = (val >> LTSSM_SHIFT) & LTSSM_MASK;
 310	return ltssm_state;
 311}
 312
 313static inline bool advk_pcie_link_up(struct advk_pcie *pcie)
 314{
 315	/* check if LTSSM is in normal operation - some L* state */
 316	u8 ltssm_state = advk_pcie_ltssm_state(pcie);
 317	return ltssm_state >= LTSSM_L0 && ltssm_state < LTSSM_DISABLED;
 318}
 319
 320static inline bool advk_pcie_link_active(struct advk_pcie *pcie)
 321{
 322	/*
 323	 * According to PCIe Base specification 3.0, Table 4-14: Link
 324	 * Status Mapped to the LTSSM, and 4.2.6.3.6 Configuration.Idle
 325	 * is Link Up mapped to LTSSM Configuration.Idle, Recovery, L0,
 326	 * L0s, L1 and L2 states. And according to 3.2.1. Data Link
 327	 * Control and Management State Machine Rules is DL Up status
 328	 * reported in DL Active state.
 329	 */
 330	u8 ltssm_state = advk_pcie_ltssm_state(pcie);
 331	return ltssm_state >= LTSSM_CONFIG_IDLE && ltssm_state < LTSSM_DISABLED;
 332}
 333
 334static inline bool advk_pcie_link_training(struct advk_pcie *pcie)
 335{
 336	/*
 337	 * According to PCIe Base specification 3.0, Table 4-14: Link
 338	 * Status Mapped to the LTSSM is Link Training mapped to LTSSM
 339	 * Configuration and Recovery states.
 340	 */
 341	u8 ltssm_state = advk_pcie_ltssm_state(pcie);
 342	return ((ltssm_state >= LTSSM_CONFIG_LINKWIDTH_START &&
 343		 ltssm_state < LTSSM_L0) ||
 344		(ltssm_state >= LTSSM_RECOVERY_EQUALIZATION_PHASE0 &&
 345		 ltssm_state <= LTSSM_RECOVERY_EQUALIZATION_PHASE3));
 346}
 347
 348static int advk_pcie_wait_for_link(struct advk_pcie *pcie)
 349{
 350	int retries;
 351
 352	/* check if the link is up or not */
 353	for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
 354		if (advk_pcie_link_up(pcie))
 355			return 0;
 356
 357		usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
 358	}
 359
 360	return -ETIMEDOUT;
 361}
 362
 363static void advk_pcie_wait_for_retrain(struct advk_pcie *pcie)
 364{
 365	size_t retries;
 366
 367	for (retries = 0; retries < RETRAIN_WAIT_MAX_RETRIES; ++retries) {
 368		if (advk_pcie_link_training(pcie))
 369			break;
 370		udelay(RETRAIN_WAIT_USLEEP_US);
 371	}
 372}
 373
 374static void advk_pcie_issue_perst(struct advk_pcie *pcie)
 375{
 
 
 376	if (!pcie->reset_gpio)
 377		return;
 378
 
 
 
 
 
 
 
 
 
 
 
 
 379	/* 10ms delay is needed for some cards */
 380	dev_info(&pcie->pdev->dev, "issuing PERST via reset GPIO for 10ms\n");
 381	gpiod_set_value_cansleep(pcie->reset_gpio, 1);
 382	usleep_range(10000, 11000);
 383	gpiod_set_value_cansleep(pcie->reset_gpio, 0);
 384}
 385
 386static void advk_pcie_train_link(struct advk_pcie *pcie)
 387{
 388	struct device *dev = &pcie->pdev->dev;
 389	u32 reg;
 390	int ret;
 391
 392	/*
 393	 * Setup PCIe rev / gen compliance based on device tree property
 394	 * 'max-link-speed' which also forces maximal link speed.
 395	 */
 396	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
 397	reg &= ~PCIE_GEN_SEL_MSK;
 398	if (pcie->link_gen == 3)
 399		reg |= SPEED_GEN_3;
 400	else if (pcie->link_gen == 2)
 401		reg |= SPEED_GEN_2;
 402	else
 403		reg |= SPEED_GEN_1;
 404	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
 405
 406	/*
 407	 * Set maximal link speed value also into PCIe Link Control 2 register.
 408	 * Armada 3700 Functional Specification says that default value is based
 409	 * on SPEED_GEN but tests showed that default value is always 8.0 GT/s.
 410	 */
 411	reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2);
 412	reg &= ~PCI_EXP_LNKCTL2_TLS;
 413	if (pcie->link_gen == 3)
 414		reg |= PCI_EXP_LNKCTL2_TLS_8_0GT;
 415	else if (pcie->link_gen == 2)
 416		reg |= PCI_EXP_LNKCTL2_TLS_5_0GT;
 417	else
 418		reg |= PCI_EXP_LNKCTL2_TLS_2_5GT;
 419	advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2);
 420
 421	/* Enable link training after selecting PCIe generation */
 422	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
 423	reg |= LINK_TRAINING_EN;
 424	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
 425
 426	/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 427	 * Reset PCIe card via PERST# signal. Some cards are not detected
 428	 * during link training when they are in some non-initial state.
 429	 */
 430	advk_pcie_issue_perst(pcie);
 431
 432	/*
 433	 * PERST# signal could have been asserted by pinctrl subsystem before
 434	 * probe() callback has been called or issued explicitly by reset gpio
 435	 * function advk_pcie_issue_perst(), making the endpoint going into
 436	 * fundamental reset. As required by PCI Express spec (PCI Express
 437	 * Base Specification, REV. 4.0 PCI Express, February 19 2014, 6.6.1
 438	 * Conventional Reset) a delay for at least 100ms after such a reset
 439	 * before sending a Configuration Request to the device is needed.
 440	 * So wait until PCIe link is up. Function advk_pcie_wait_for_link()
 441	 * waits for link at least 900ms.
 442	 */
 443	ret = advk_pcie_wait_for_link(pcie);
 444	if (ret < 0)
 445		dev_err(dev, "link never came up\n");
 446	else
 447		dev_info(dev, "link up\n");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 448}
 449
 450/*
 451 * Set PCIe address window register which could be used for memory
 452 * mapping.
 453 */
 454static void advk_pcie_set_ob_win(struct advk_pcie *pcie, u8 win_num,
 455				 phys_addr_t match, phys_addr_t remap,
 456				 phys_addr_t mask, u32 actions)
 457{
 458	advk_writel(pcie, OB_WIN_ENABLE |
 459			  lower_32_bits(match), OB_WIN_MATCH_LS(win_num));
 460	advk_writel(pcie, upper_32_bits(match), OB_WIN_MATCH_MS(win_num));
 461	advk_writel(pcie, lower_32_bits(remap), OB_WIN_REMAP_LS(win_num));
 462	advk_writel(pcie, upper_32_bits(remap), OB_WIN_REMAP_MS(win_num));
 463	advk_writel(pcie, lower_32_bits(mask), OB_WIN_MASK_LS(win_num));
 464	advk_writel(pcie, upper_32_bits(mask), OB_WIN_MASK_MS(win_num));
 465	advk_writel(pcie, actions, OB_WIN_ACTIONS(win_num));
 466}
 467
 468static void advk_pcie_disable_ob_win(struct advk_pcie *pcie, u8 win_num)
 469{
 470	advk_writel(pcie, 0, OB_WIN_MATCH_LS(win_num));
 471	advk_writel(pcie, 0, OB_WIN_MATCH_MS(win_num));
 472	advk_writel(pcie, 0, OB_WIN_REMAP_LS(win_num));
 473	advk_writel(pcie, 0, OB_WIN_REMAP_MS(win_num));
 474	advk_writel(pcie, 0, OB_WIN_MASK_LS(win_num));
 475	advk_writel(pcie, 0, OB_WIN_MASK_MS(win_num));
 476	advk_writel(pcie, 0, OB_WIN_ACTIONS(win_num));
 477}
 478
 479static void advk_pcie_setup_hw(struct advk_pcie *pcie)
 480{
 481	phys_addr_t msi_addr;
 482	u32 reg;
 483	int i;
 484
 485	/*
 486	 * Configure PCIe Reference clock. Direction is from the PCIe
 487	 * controller to the endpoint card, so enable transmitting of
 488	 * Reference clock differential signal off-chip and disable
 489	 * receiving off-chip differential signal.
 490	 */
 491	reg = advk_readl(pcie, PCIE_CORE_REF_CLK_REG);
 492	reg |= PCIE_CORE_REF_CLK_TX_ENABLE;
 493	reg &= ~PCIE_CORE_REF_CLK_RX_ENABLE;
 494	advk_writel(pcie, reg, PCIE_CORE_REF_CLK_REG);
 495
 496	/* Set to Direct mode */
 497	reg = advk_readl(pcie, CTRL_CONFIG_REG);
 498	reg &= ~(CTRL_MODE_MASK << CTRL_MODE_SHIFT);
 499	reg |= ((PCIE_CORE_MODE_DIRECT & CTRL_MODE_MASK) << CTRL_MODE_SHIFT);
 500	advk_writel(pcie, reg, CTRL_CONFIG_REG);
 501
 502	/* Set PCI global control register to RC mode */
 503	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
 504	reg |= (IS_RC_MSK << IS_RC_SHIFT);
 505	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
 506
 507	/*
 508	 * Replace incorrect PCI vendor id value 0x1b4b by correct value 0x11ab.
 509	 * VENDOR_ID_REG contains vendor id in low 16 bits and subsystem vendor
 510	 * id in high 16 bits. Updating this register changes readback value of
 511	 * read-only vendor id bits in PCIE_CORE_DEV_ID_REG register. Workaround
 512	 * for erratum 4.1: "The value of device and vendor ID is incorrect".
 513	 */
 514	reg = (PCI_VENDOR_ID_MARVELL << 16) | PCI_VENDOR_ID_MARVELL;
 515	advk_writel(pcie, reg, VENDOR_ID_REG);
 516
 517	/*
 518	 * Change Class Code of PCI Bridge device to PCI Bridge (0x600400),
 519	 * because the default value is Mass storage controller (0x010400).
 520	 *
 521	 * Note that this Aardvark PCI Bridge does not have compliant Type 1
 522	 * Configuration Space and it even cannot be accessed via Aardvark's
 523	 * PCI config space access method. Something like config space is
 524	 * available in internal Aardvark registers starting at offset 0x0
 525	 * and is reported as Type 0. In range 0x10 - 0x34 it has totally
 526	 * different registers.
 527	 *
 528	 * Therefore driver uses emulation of PCI Bridge which emulates
 529	 * access to configuration space via internal Aardvark registers or
 530	 * emulated configuration buffer.
 531	 */
 532	reg = advk_readl(pcie, PCIE_CORE_DEV_REV_REG);
 533	reg &= ~0xffffff00;
 534	reg |= PCI_CLASS_BRIDGE_PCI_NORMAL << 8;
 535	advk_writel(pcie, reg, PCIE_CORE_DEV_REV_REG);
 536
 537	/* Disable Root Bridge I/O space, memory space and bus mastering */
 538	reg = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
 539	reg &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
 540	advk_writel(pcie, reg, PCIE_CORE_CMD_STATUS_REG);
 541
 542	/* Set Advanced Error Capabilities and Control PF0 register */
 543	reg = PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX |
 544		PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN |
 545		PCIE_CORE_ERR_CAPCTL_ECRC_CHCK |
 546		PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV;
 547	advk_writel(pcie, reg, PCIE_CORE_ERR_CAPCTL_REG);
 548
 549	/* Set PCIe Device Control register */
 550	reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL);
 551	reg &= ~PCI_EXP_DEVCTL_RELAX_EN;
 552	reg &= ~PCI_EXP_DEVCTL_NOSNOOP_EN;
 553	reg &= ~PCI_EXP_DEVCTL_PAYLOAD;
 554	reg &= ~PCI_EXP_DEVCTL_READRQ;
 555	reg |= PCI_EXP_DEVCTL_PAYLOAD_512B;
 556	reg |= PCI_EXP_DEVCTL_READRQ_512B;
 557	advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL);
 558
 559	/* Program PCIe Control 2 to disable strict ordering */
 560	reg = PCIE_CORE_CTRL2_RESERVED |
 561		PCIE_CORE_CTRL2_TD_ENABLE;
 562	advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
 563
 564	/* Set lane X1 */
 565	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
 566	reg &= ~LANE_CNT_MSK;
 567	reg |= LANE_COUNT_1;
 568	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
 569
 570	/* Set MSI address */
 571	msi_addr = virt_to_phys(pcie);
 572	advk_writel(pcie, lower_32_bits(msi_addr), PCIE_MSI_ADDR_LOW_REG);
 573	advk_writel(pcie, upper_32_bits(msi_addr), PCIE_MSI_ADDR_HIGH_REG);
 574
 575	/* Enable MSI */
 576	reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
 577	reg |= PCIE_CORE_CTRL2_MSI_ENABLE;
 578	advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
 579
 580	/* Clear all interrupts */
 581	advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_STATUS_REG);
 582	advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_REG);
 583	advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_REG);
 584	advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_STATUS_REG);
 585
 586	/* Disable All ISR0/1 and MSI Sources */
 587	advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_MASK_REG);
 588	advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG);
 589	advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_MASK_REG);
 590
 591	/* Unmask summary MSI interrupt */
 592	reg = advk_readl(pcie, PCIE_ISR0_MASK_REG);
 593	reg &= ~PCIE_ISR0_MSI_INT_PENDING;
 594	advk_writel(pcie, reg, PCIE_ISR0_MASK_REG);
 595
 596	/* Unmask PME interrupt for processing of PME requester */
 597	reg = advk_readl(pcie, PCIE_ISR0_MASK_REG);
 598	reg &= ~PCIE_MSG_PM_PME_MASK;
 599	advk_writel(pcie, reg, PCIE_ISR0_MASK_REG);
 600
 601	/* Enable summary interrupt for GIC SPI source */
 602	reg = PCIE_IRQ_ALL_MASK & (~PCIE_IRQ_ENABLE_INTS_MASK);
 603	advk_writel(pcie, reg, HOST_CTRL_INT_MASK_REG);
 604
 605	/*
 606	 * Enable AXI address window location generation:
 607	 * When it is enabled, the default outbound window
 608	 * configurations (Default User Field: 0xD0074CFC)
 609	 * are used to transparent address translation for
 610	 * the outbound transactions. Thus, PCIe address
 611	 * windows are not required for transparent memory
 612	 * access when default outbound window configuration
 613	 * is set for memory access.
 614	 */
 615	reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
 616	reg |= PCIE_CORE_CTRL2_OB_WIN_ENABLE;
 617	advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
 618
 619	/*
 620	 * Set memory access in Default User Field so it
 621	 * is not required to configure PCIe address for
 622	 * transparent memory access.
 623	 */
 624	advk_writel(pcie, OB_WIN_TYPE_MEM, OB_WIN_DEFAULT_ACTIONS);
 625
 626	/*
 627	 * Bypass the address window mapping for PIO:
 628	 * Since PIO access already contains all required
 629	 * info over AXI interface by PIO registers, the
 630	 * address window is not required.
 631	 */
 632	reg = advk_readl(pcie, PIO_CTRL);
 633	reg |= PIO_CTRL_ADDR_WIN_DISABLE;
 634	advk_writel(pcie, reg, PIO_CTRL);
 635
 636	/*
 637	 * Configure PCIe address windows for non-memory or
 638	 * non-transparent access as by default PCIe uses
 639	 * transparent memory access.
 640	 */
 641	for (i = 0; i < pcie->wins_count; i++)
 642		advk_pcie_set_ob_win(pcie, i,
 643				     pcie->wins[i].match, pcie->wins[i].remap,
 644				     pcie->wins[i].mask, pcie->wins[i].actions);
 645
 646	/* Disable remaining PCIe outbound windows */
 647	for (i = pcie->wins_count; i < OB_WIN_COUNT; i++)
 648		advk_pcie_disable_ob_win(pcie, i);
 649
 650	advk_pcie_train_link(pcie);
 
 
 
 
 
 
 
 
 
 
 
 
 
 651}
 652
 653static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u32 *val)
 654{
 655	struct device *dev = &pcie->pdev->dev;
 656	u32 reg;
 657	unsigned int status;
 658	char *strcomp_status, *str_posted;
 659	int ret;
 660
 661	reg = advk_readl(pcie, PIO_STAT);
 662	status = (reg & PIO_COMPLETION_STATUS_MASK) >>
 663		PIO_COMPLETION_STATUS_SHIFT;
 664
 665	/*
 666	 * According to HW spec, the PIO status check sequence as below:
 667	 * 1) even if COMPLETION_STATUS(bit9:7) indicates successful,
 668	 *    it still needs to check Error Status(bit11), only when this bit
 669	 *    indicates no error happen, the operation is successful.
 670	 * 2) value Unsupported Request(1) of COMPLETION_STATUS(bit9:7) only
 671	 *    means a PIO write error, and for PIO read it is successful with
 672	 *    a read value of 0xFFFFFFFF.
 673	 * 3) value Completion Retry Status(CRS) of COMPLETION_STATUS(bit9:7)
 674	 *    only means a PIO write error, and for PIO read it is successful
 675	 *    with a read value of 0xFFFF0001.
 676	 * 4) value Completer Abort (CA) of COMPLETION_STATUS(bit9:7) means
 677	 *    error for both PIO read and PIO write operation.
 678	 * 5) other errors are indicated as 'unknown'.
 679	 */
 680	switch (status) {
 681	case PIO_COMPLETION_STATUS_OK:
 682		if (reg & PIO_ERR_STATUS) {
 683			strcomp_status = "COMP_ERR";
 684			ret = -EFAULT;
 685			break;
 686		}
 687		/* Get the read result */
 688		if (val)
 689			*val = advk_readl(pcie, PIO_RD_DATA);
 690		/* No error */
 691		strcomp_status = NULL;
 692		ret = 0;
 693		break;
 694	case PIO_COMPLETION_STATUS_UR:
 695		strcomp_status = "UR";
 696		ret = -EOPNOTSUPP;
 697		break;
 698	case PIO_COMPLETION_STATUS_CRS:
 699		if (allow_crs && val) {
 700			/* PCIe r4.0, sec 2.3.2, says:
 701			 * If CRS Software Visibility is enabled:
 702			 * For a Configuration Read Request that includes both
 703			 * bytes of the Vendor ID field of a device Function's
 704			 * Configuration Space Header, the Root Complex must
 705			 * complete the Request to the host by returning a
 706			 * read-data value of 0001h for the Vendor ID field and
 707			 * all '1's for any additional bytes included in the
 708			 * request.
 709			 *
 710			 * So CRS in this case is not an error status.
 711			 */
 712			*val = CFG_RD_CRS_VAL;
 713			strcomp_status = NULL;
 714			ret = 0;
 715			break;
 716		}
 717		/* PCIe r4.0, sec 2.3.2, says:
 718		 * If CRS Software Visibility is not enabled, the Root Complex
 719		 * must re-issue the Configuration Request as a new Request.
 720		 * If CRS Software Visibility is enabled: For a Configuration
 721		 * Write Request or for any other Configuration Read Request,
 722		 * the Root Complex must re-issue the Configuration Request as
 723		 * a new Request.
 724		 * A Root Complex implementation may choose to limit the number
 725		 * of Configuration Request/CRS Completion Status loops before
 726		 * determining that something is wrong with the target of the
 727		 * Request and taking appropriate action, e.g., complete the
 728		 * Request to the host as a failed transaction.
 729		 *
 730		 * So return -EAGAIN and caller (pci-aardvark.c driver) will
 731		 * re-issue request again up to the PIO_RETRY_CNT retries.
 732		 */
 733		strcomp_status = "CRS";
 734		ret = -EAGAIN;
 735		break;
 736	case PIO_COMPLETION_STATUS_CA:
 737		strcomp_status = "CA";
 738		ret = -ECANCELED;
 739		break;
 740	default:
 741		strcomp_status = "Unknown";
 742		ret = -EINVAL;
 743		break;
 744	}
 745
 746	if (!strcomp_status)
 747		return ret;
 748
 749	if (reg & PIO_NON_POSTED_REQ)
 750		str_posted = "Non-posted";
 751	else
 752		str_posted = "Posted";
 753
 754	dev_dbg(dev, "%s PIO Response Status: %s, %#x @ %#x\n",
 755		str_posted, strcomp_status, reg, advk_readl(pcie, PIO_ADDR_LS));
 756
 757	return ret;
 758}
 759
 760static int advk_pcie_wait_pio(struct advk_pcie *pcie)
 761{
 762	struct device *dev = &pcie->pdev->dev;
 763	int i;
 764
 765	for (i = 1; i <= PIO_RETRY_CNT; i++) {
 766		u32 start, isr;
 767
 768		start = advk_readl(pcie, PIO_START);
 769		isr = advk_readl(pcie, PIO_ISR);
 770		if (!start && isr)
 771			return i;
 772		udelay(PIO_RETRY_DELAY);
 773	}
 774
 775	dev_err(dev, "PIO read/write transfer time out\n");
 776	return -ETIMEDOUT;
 777}
 778
 
 779static pci_bridge_emul_read_status_t
 780advk_pci_bridge_emul_base_conf_read(struct pci_bridge_emul *bridge,
 781				    int reg, u32 *value)
 782{
 783	struct advk_pcie *pcie = bridge->data;
 784
 
 785	switch (reg) {
 786	case PCI_COMMAND:
 787		*value = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
 788		return PCI_BRIDGE_EMUL_HANDLED;
 789
 790	case PCI_INTERRUPT_LINE: {
 791		/*
 792		 * From the whole 32bit register we support reading from HW only
 793		 * two bits: PCI_BRIDGE_CTL_BUS_RESET and PCI_BRIDGE_CTL_SERR.
 794		 * Other bits are retrieved only from emulated config buffer.
 795		 */
 796		__le32 *cfgspace = (__le32 *)&bridge->conf;
 797		u32 val = le32_to_cpu(cfgspace[PCI_INTERRUPT_LINE / 4]);
 798		if (advk_readl(pcie, PCIE_ISR0_MASK_REG) & PCIE_ISR0_ERR_MASK)
 799			val &= ~(PCI_BRIDGE_CTL_SERR << 16);
 800		else
 801			val |= PCI_BRIDGE_CTL_SERR << 16;
 802		if (advk_readl(pcie, PCIE_CORE_CTRL1_REG) & HOT_RESET_GEN)
 803			val |= PCI_BRIDGE_CTL_BUS_RESET << 16;
 804		else
 805			val &= ~(PCI_BRIDGE_CTL_BUS_RESET << 16);
 806		*value = val;
 807		return PCI_BRIDGE_EMUL_HANDLED;
 808	}
 809
 810	default:
 811		return PCI_BRIDGE_EMUL_NOT_HANDLED;
 812	}
 813}
 814
 815static void
 816advk_pci_bridge_emul_base_conf_write(struct pci_bridge_emul *bridge,
 817				     int reg, u32 old, u32 new, u32 mask)
 818{
 819	struct advk_pcie *pcie = bridge->data;
 820
 821	switch (reg) {
 822	case PCI_COMMAND:
 823		advk_writel(pcie, new, PCIE_CORE_CMD_STATUS_REG);
 824		break;
 825
 826	case PCI_INTERRUPT_LINE:
 827		/*
 828		 * According to Figure 6-3: Pseudo Logic Diagram for Error
 829		 * Message Controls in PCIe base specification, SERR# Enable bit
 830		 * in Bridge Control register enable receiving of ERR_* messages
 831		 */
 832		if (mask & (PCI_BRIDGE_CTL_SERR << 16)) {
 833			u32 val = advk_readl(pcie, PCIE_ISR0_MASK_REG);
 834			if (new & (PCI_BRIDGE_CTL_SERR << 16))
 835				val &= ~PCIE_ISR0_ERR_MASK;
 836			else
 837				val |= PCIE_ISR0_ERR_MASK;
 838			advk_writel(pcie, val, PCIE_ISR0_MASK_REG);
 839		}
 840		if (mask & (PCI_BRIDGE_CTL_BUS_RESET << 16)) {
 841			u32 val = advk_readl(pcie, PCIE_CORE_CTRL1_REG);
 842			if (new & (PCI_BRIDGE_CTL_BUS_RESET << 16))
 843				val |= HOT_RESET_GEN;
 844			else
 845				val &= ~HOT_RESET_GEN;
 846			advk_writel(pcie, val, PCIE_CORE_CTRL1_REG);
 847		}
 848		break;
 849
 850	default:
 851		break;
 852	}
 853}
 854
 855static pci_bridge_emul_read_status_t
 856advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge,
 857				    int reg, u32 *value)
 858{
 859	struct advk_pcie *pcie = bridge->data;
 860
 861
 862	switch (reg) {
 863	/*
 864	 * PCI_EXP_SLTCAP, PCI_EXP_SLTCTL, PCI_EXP_RTCTL and PCI_EXP_RTSTA are
 865	 * also supported, but do not need to be handled here, because their
 866	 * values are stored in emulated config space buffer, and we read them
 867	 * from there when needed.
 868	 */
 869
 870	case PCI_EXP_LNKCAP: {
 871		u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg);
 872		/*
 873		 * PCI_EXP_LNKCAP_DLLLARC bit is hardwired in aardvark HW to 0.
 874		 * But support for PCI_EXP_LNKSTA_DLLLA is emulated via ltssm
 875		 * state so explicitly enable PCI_EXP_LNKCAP_DLLLARC flag.
 876		 */
 877		val |= PCI_EXP_LNKCAP_DLLLARC;
 878		*value = val;
 879		return PCI_BRIDGE_EMUL_HANDLED;
 880	}
 881
 882	case PCI_EXP_LNKCTL: {
 883		/* u32 contains both PCI_EXP_LNKCTL and PCI_EXP_LNKSTA */
 884		u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg) &
 885			~(PCI_EXP_LNKSTA_LT << 16);
 886		if (advk_pcie_link_training(pcie))
 887			val |= (PCI_EXP_LNKSTA_LT << 16);
 888		if (advk_pcie_link_active(pcie))
 889			val |= (PCI_EXP_LNKSTA_DLLLA << 16);
 890		*value = val;
 891		return PCI_BRIDGE_EMUL_HANDLED;
 892	}
 893
 
 894	case PCI_EXP_DEVCAP:
 895	case PCI_EXP_DEVCTL:
 896	case PCI_EXP_DEVCAP2:
 897	case PCI_EXP_DEVCTL2:
 898	case PCI_EXP_LNKCAP2:
 899	case PCI_EXP_LNKCTL2:
 900		*value = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg);
 901		return PCI_BRIDGE_EMUL_HANDLED;
 902
 903	default:
 904		return PCI_BRIDGE_EMUL_NOT_HANDLED;
 905	}
 906
 907}
 908
 909static void
 910advk_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge,
 911				     int reg, u32 old, u32 new, u32 mask)
 912{
 913	struct advk_pcie *pcie = bridge->data;
 914
 915	switch (reg) {
 
 
 
 
 916	case PCI_EXP_LNKCTL:
 917		advk_writel(pcie, new, PCIE_CORE_PCIEXP_CAP + reg);
 918		if (new & PCI_EXP_LNKCTL_RL)
 919			advk_pcie_wait_for_retrain(pcie);
 920		break;
 921
 922	case PCI_EXP_RTCTL: {
 923		u16 rootctl = le16_to_cpu(bridge->pcie_conf.rootctl);
 924		/* Only emulation of PMEIE and CRSSVE bits is provided */
 925		rootctl &= PCI_EXP_RTCTL_PMEIE | PCI_EXP_RTCTL_CRSSVE;
 926		bridge->pcie_conf.rootctl = cpu_to_le16(rootctl);
 
 
 927		break;
 928	}
 929
 930	/*
 931	 * PCI_EXP_RTSTA is also supported, but does not need to be handled
 932	 * here, because its value is stored in emulated config space buffer,
 933	 * and we write it there when needed.
 934	 */
 935
 936	case PCI_EXP_DEVCTL:
 937	case PCI_EXP_DEVCTL2:
 938	case PCI_EXP_LNKCTL2:
 939		advk_writel(pcie, new, PCIE_CORE_PCIEXP_CAP + reg);
 940		break;
 941
 942	default:
 943		break;
 944	}
 945}
 946
 947static pci_bridge_emul_read_status_t
 948advk_pci_bridge_emul_ext_conf_read(struct pci_bridge_emul *bridge,
 949				   int reg, u32 *value)
 950{
 951	struct advk_pcie *pcie = bridge->data;
 952
 953	switch (reg) {
 954	case 0:
 955		*value = advk_readl(pcie, PCIE_CORE_PCIERR_CAP + reg);
 956
 957		/*
 958		 * PCI_EXT_CAP_NEXT bits are set to offset 0x150, but Armada
 959		 * 3700 Functional Specification does not document registers
 960		 * at those addresses.
 961		 *
 962		 * Thus we clear PCI_EXT_CAP_NEXT bits to make Advanced Error
 963		 * Reporting Capability header the last Extended Capability.
 964		 * If we obtain documentation for those registers in the
 965		 * future, this can be changed.
 966		 */
 967		*value &= 0x000fffff;
 968		return PCI_BRIDGE_EMUL_HANDLED;
 969
 970	case PCI_ERR_UNCOR_STATUS:
 971	case PCI_ERR_UNCOR_MASK:
 972	case PCI_ERR_UNCOR_SEVER:
 973	case PCI_ERR_COR_STATUS:
 974	case PCI_ERR_COR_MASK:
 975	case PCI_ERR_CAP:
 976	case PCI_ERR_HEADER_LOG + 0:
 977	case PCI_ERR_HEADER_LOG + 4:
 978	case PCI_ERR_HEADER_LOG + 8:
 979	case PCI_ERR_HEADER_LOG + 12:
 980	case PCI_ERR_ROOT_COMMAND:
 981	case PCI_ERR_ROOT_STATUS:
 982	case PCI_ERR_ROOT_ERR_SRC:
 983		*value = advk_readl(pcie, PCIE_CORE_PCIERR_CAP + reg);
 984		return PCI_BRIDGE_EMUL_HANDLED;
 985
 986	default:
 987		return PCI_BRIDGE_EMUL_NOT_HANDLED;
 988	}
 989}
 990
 991static void
 992advk_pci_bridge_emul_ext_conf_write(struct pci_bridge_emul *bridge,
 993				    int reg, u32 old, u32 new, u32 mask)
 994{
 995	struct advk_pcie *pcie = bridge->data;
 996
 997	switch (reg) {
 998	/* These are W1C registers, so clear other bits */
 999	case PCI_ERR_UNCOR_STATUS:
1000	case PCI_ERR_COR_STATUS:
1001	case PCI_ERR_ROOT_STATUS:
1002		new &= mask;
1003		fallthrough;
1004
1005	case PCI_ERR_UNCOR_MASK:
1006	case PCI_ERR_UNCOR_SEVER:
1007	case PCI_ERR_COR_MASK:
1008	case PCI_ERR_CAP:
1009	case PCI_ERR_HEADER_LOG + 0:
1010	case PCI_ERR_HEADER_LOG + 4:
1011	case PCI_ERR_HEADER_LOG + 8:
1012	case PCI_ERR_HEADER_LOG + 12:
1013	case PCI_ERR_ROOT_COMMAND:
1014	case PCI_ERR_ROOT_ERR_SRC:
1015		advk_writel(pcie, new, PCIE_CORE_PCIERR_CAP + reg);
1016		break;
1017
1018	default:
1019		break;
1020	}
1021}
1022
1023static const struct pci_bridge_emul_ops advk_pci_bridge_emul_ops = {
1024	.read_base = advk_pci_bridge_emul_base_conf_read,
1025	.write_base = advk_pci_bridge_emul_base_conf_write,
1026	.read_pcie = advk_pci_bridge_emul_pcie_conf_read,
1027	.write_pcie = advk_pci_bridge_emul_pcie_conf_write,
1028	.read_ext = advk_pci_bridge_emul_ext_conf_read,
1029	.write_ext = advk_pci_bridge_emul_ext_conf_write,
1030};
1031
1032/*
1033 * Initialize the configuration space of the PCI-to-PCI bridge
1034 * associated with the given PCIe interface.
1035 */
1036static int advk_sw_pci_bridge_init(struct advk_pcie *pcie)
1037{
1038	struct pci_bridge_emul *bridge = &pcie->bridge;
 
1039
1040	bridge->conf.vendor =
1041		cpu_to_le16(advk_readl(pcie, PCIE_CORE_DEV_ID_REG) & 0xffff);
1042	bridge->conf.device =
1043		cpu_to_le16(advk_readl(pcie, PCIE_CORE_DEV_ID_REG) >> 16);
1044	bridge->conf.class_revision =
1045		cpu_to_le32(advk_readl(pcie, PCIE_CORE_DEV_REV_REG) & 0xff);
1046
1047	/* Support 32 bits I/O addressing */
1048	bridge->conf.iobase = PCI_IO_RANGE_TYPE_32;
1049	bridge->conf.iolimit = PCI_IO_RANGE_TYPE_32;
1050
1051	/* Support 64 bits memory pref */
1052	bridge->conf.pref_mem_base = cpu_to_le16(PCI_PREF_RANGE_TYPE_64);
1053	bridge->conf.pref_mem_limit = cpu_to_le16(PCI_PREF_RANGE_TYPE_64);
1054
1055	/* Support interrupt A for MSI feature */
1056	bridge->conf.intpin = PCI_INTERRUPT_INTA;
1057
1058	/*
1059	 * Aardvark HW provides PCIe Capability structure in version 2 and
1060	 * indicate slot support, which is emulated.
1061	 */
1062	bridge->pcie_conf.cap = cpu_to_le16(2 | PCI_EXP_FLAGS_SLOT);
1063
1064	/*
1065	 * Set Presence Detect State bit permanently since there is no support
1066	 * for unplugging the card nor detecting whether it is plugged. (If a
1067	 * platform exists in the future that supports it, via a GPIO for
1068	 * example, it should be implemented via this bit.)
1069	 *
1070	 * Set physical slot number to 1 since there is only one port and zero
1071	 * value is reserved for ports within the same silicon as Root Port
1072	 * which is not our case.
1073	 */
1074	bridge->pcie_conf.slotcap = cpu_to_le32(FIELD_PREP(PCI_EXP_SLTCAP_PSN,
1075							   1));
1076	bridge->pcie_conf.slotsta = cpu_to_le16(PCI_EXP_SLTSTA_PDS);
1077
1078	/* Indicates supports for Completion Retry Status */
1079	bridge->pcie_conf.rootcap = cpu_to_le16(PCI_EXP_RTCAP_CRSVIS);
1080
1081	bridge->subsystem_vendor_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) & 0xffff;
1082	bridge->subsystem_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) >> 16;
1083	bridge->has_pcie = true;
1084	bridge->pcie_start = PCIE_CORE_PCIEXP_CAP;
1085	bridge->data = pcie;
1086	bridge->ops = &advk_pci_bridge_emul_ops;
1087
1088	return pci_bridge_emul_init(bridge, 0);
1089}
1090
1091static bool advk_pcie_valid_device(struct advk_pcie *pcie, struct pci_bus *bus,
1092				  int devfn)
1093{
1094	if (pci_is_root_bus(bus) && PCI_SLOT(devfn) != 0)
1095		return false;
1096
1097	/*
1098	 * If the link goes down after we check for link-up, we have a problem:
1099	 * if a PIO request is executed while link-down, the whole controller
1100	 * gets stuck in a non-functional state, and even after link comes up
1101	 * again, PIO requests won't work anymore, and a reset of the whole PCIe
1102	 * controller is needed. Therefore we need to prevent sending PIO
1103	 * requests while the link is down.
1104	 */
1105	if (!pci_is_root_bus(bus) && !advk_pcie_link_up(pcie))
1106		return false;
1107
1108	return true;
1109}
1110
1111static bool advk_pcie_pio_is_running(struct advk_pcie *pcie)
1112{
1113	struct device *dev = &pcie->pdev->dev;
1114
1115	/*
1116	 * Trying to start a new PIO transfer when previous has not completed
1117	 * cause External Abort on CPU which results in kernel panic:
1118	 *
1119	 *     SError Interrupt on CPU0, code 0xbf000002 -- SError
1120	 *     Kernel panic - not syncing: Asynchronous SError Interrupt
1121	 *
1122	 * Functions advk_pcie_rd_conf() and advk_pcie_wr_conf() are protected
1123	 * by raw_spin_lock_irqsave() at pci_lock_config() level to prevent
1124	 * concurrent calls at the same time. But because PIO transfer may take
1125	 * about 1.5s when link is down or card is disconnected, it means that
1126	 * advk_pcie_wait_pio() does not always have to wait for completion.
1127	 *
1128	 * Some versions of ARM Trusted Firmware handles this External Abort at
1129	 * EL3 level and mask it to prevent kernel panic. Relevant TF-A commit:
1130	 * https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git/commit/?id=3c7dcdac5c50
1131	 */
1132	if (advk_readl(pcie, PIO_START)) {
1133		dev_err(dev, "Previous PIO read/write transfer is still running\n");
1134		return true;
1135	}
1136
1137	return false;
1138}
1139
1140static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn,
1141			     int where, int size, u32 *val)
1142{
1143	struct advk_pcie *pcie = bus->sysdata;
1144	int retry_count;
1145	bool allow_crs;
1146	u32 reg;
1147	int ret;
1148
1149	if (!advk_pcie_valid_device(pcie, bus, devfn))
 
1150		return PCIBIOS_DEVICE_NOT_FOUND;
 
1151
1152	if (pci_is_root_bus(bus))
1153		return pci_bridge_emul_conf_read(&pcie->bridge, where,
1154						 size, val);
1155
1156	/*
1157	 * Completion Retry Status is possible to return only when reading all
1158	 * 4 bytes from PCI_VENDOR_ID and PCI_DEVICE_ID registers at once and
1159	 * CRSSVE flag on Root Bridge is enabled.
1160	 */
1161	allow_crs = (where == PCI_VENDOR_ID) && (size == 4) &&
1162		    (le16_to_cpu(pcie->bridge.pcie_conf.rootctl) &
1163		     PCI_EXP_RTCTL_CRSSVE);
1164
1165	if (advk_pcie_pio_is_running(pcie))
1166		goto try_crs;
 
 
 
 
 
 
 
 
 
 
1167
1168	/* Program the control register */
1169	reg = advk_readl(pcie, PIO_CTRL);
1170	reg &= ~PIO_CTRL_TYPE_MASK;
1171	if (pci_is_root_bus(bus->parent))
1172		reg |= PCIE_CONFIG_RD_TYPE0;
1173	else
1174		reg |= PCIE_CONFIG_RD_TYPE1;
1175	advk_writel(pcie, reg, PIO_CTRL);
1176
1177	/* Program the address registers */
1178	reg = ALIGN_DOWN(PCIE_ECAM_OFFSET(bus->number, devfn, where), 4);
1179	advk_writel(pcie, reg, PIO_ADDR_LS);
1180	advk_writel(pcie, 0, PIO_ADDR_MS);
1181
1182	/* Program the data strobe */
1183	advk_writel(pcie, 0xf, PIO_WR_DATA_STRB);
1184
1185	retry_count = 0;
1186	do {
1187		/* Clear PIO DONE ISR and start the transfer */
1188		advk_writel(pcie, 1, PIO_ISR);
1189		advk_writel(pcie, 1, PIO_START);
1190
1191		ret = advk_pcie_wait_pio(pcie);
1192		if (ret < 0)
1193			goto try_crs;
1194
1195		retry_count += ret;
1196
1197		/* Check PIO status and get the read result */
1198		ret = advk_pcie_check_pio_status(pcie, allow_crs, val);
1199	} while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT);
1200
1201	if (ret < 0)
1202		goto fail;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1203
1204	if (size == 1)
1205		*val = (*val >> (8 * (where & 3))) & 0xff;
1206	else if (size == 2)
1207		*val = (*val >> (8 * (where & 3))) & 0xffff;
1208
1209	return PCIBIOS_SUCCESSFUL;
1210
1211try_crs:
1212	/*
1213	 * If it is possible, return Completion Retry Status so that caller
1214	 * tries to issue the request again instead of failing.
1215	 */
1216	if (allow_crs) {
1217		*val = CFG_RD_CRS_VAL;
1218		return PCIBIOS_SUCCESSFUL;
1219	}
1220
1221fail:
1222	*val = 0xffffffff;
1223	return PCIBIOS_SET_FAILED;
1224}
1225
1226static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
1227				int where, int size, u32 val)
1228{
1229	struct advk_pcie *pcie = bus->sysdata;
1230	u32 reg;
1231	u32 data_strobe = 0x0;
1232	int retry_count;
1233	int offset;
1234	int ret;
1235
1236	if (!advk_pcie_valid_device(pcie, bus, devfn))
1237		return PCIBIOS_DEVICE_NOT_FOUND;
1238
1239	if (pci_is_root_bus(bus))
1240		return pci_bridge_emul_conf_write(&pcie->bridge, where,
1241						  size, val);
1242
1243	if (where % size)
1244		return PCIBIOS_SET_FAILED;
1245
1246	if (advk_pcie_pio_is_running(pcie))
1247		return PCIBIOS_SET_FAILED;
1248
1249	/* Program the control register */
1250	reg = advk_readl(pcie, PIO_CTRL);
1251	reg &= ~PIO_CTRL_TYPE_MASK;
1252	if (pci_is_root_bus(bus->parent))
1253		reg |= PCIE_CONFIG_WR_TYPE0;
1254	else
1255		reg |= PCIE_CONFIG_WR_TYPE1;
1256	advk_writel(pcie, reg, PIO_CTRL);
1257
1258	/* Program the address registers */
1259	reg = ALIGN_DOWN(PCIE_ECAM_OFFSET(bus->number, devfn, where), 4);
1260	advk_writel(pcie, reg, PIO_ADDR_LS);
1261	advk_writel(pcie, 0, PIO_ADDR_MS);
1262
1263	/* Calculate the write strobe */
1264	offset      = where & 0x3;
1265	reg         = val << (8 * offset);
1266	data_strobe = GENMASK(size - 1, 0) << offset;
1267
1268	/* Program the data register */
1269	advk_writel(pcie, reg, PIO_WR_DATA);
1270
1271	/* Program the data strobe */
1272	advk_writel(pcie, data_strobe, PIO_WR_DATA_STRB);
1273
1274	retry_count = 0;
1275	do {
1276		/* Clear PIO DONE ISR and start the transfer */
1277		advk_writel(pcie, 1, PIO_ISR);
1278		advk_writel(pcie, 1, PIO_START);
1279
1280		ret = advk_pcie_wait_pio(pcie);
1281		if (ret < 0)
1282			return PCIBIOS_SET_FAILED;
1283
1284		retry_count += ret;
 
 
1285
1286		ret = advk_pcie_check_pio_status(pcie, false, NULL);
1287	} while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT);
 
1288
1289	return ret < 0 ? PCIBIOS_SET_FAILED : PCIBIOS_SUCCESSFUL;
1290}
1291
1292static struct pci_ops advk_pcie_ops = {
1293	.read = advk_pcie_rd_conf,
1294	.write = advk_pcie_wr_conf,
1295};
1296
1297static void advk_msi_irq_compose_msi_msg(struct irq_data *data,
1298					 struct msi_msg *msg)
1299{
1300	struct advk_pcie *pcie = irq_data_get_irq_chip_data(data);
1301	phys_addr_t msi_addr = virt_to_phys(pcie);
1302
1303	msg->address_lo = lower_32_bits(msi_addr);
1304	msg->address_hi = upper_32_bits(msi_addr);
1305	msg->data = data->hwirq;
1306}
1307
1308static int advk_msi_set_affinity(struct irq_data *irq_data,
1309				 const struct cpumask *mask, bool force)
1310{
1311	return -EINVAL;
1312}
1313
1314static void advk_msi_irq_mask(struct irq_data *d)
1315{
1316	struct advk_pcie *pcie = d->domain->host_data;
1317	irq_hw_number_t hwirq = irqd_to_hwirq(d);
1318	unsigned long flags;
1319	u32 mask;
1320
1321	raw_spin_lock_irqsave(&pcie->msi_irq_lock, flags);
1322	mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
1323	mask |= BIT(hwirq);
1324	advk_writel(pcie, mask, PCIE_MSI_MASK_REG);
1325	raw_spin_unlock_irqrestore(&pcie->msi_irq_lock, flags);
1326}
1327
1328static void advk_msi_irq_unmask(struct irq_data *d)
1329{
1330	struct advk_pcie *pcie = d->domain->host_data;
1331	irq_hw_number_t hwirq = irqd_to_hwirq(d);
1332	unsigned long flags;
1333	u32 mask;
1334
1335	raw_spin_lock_irqsave(&pcie->msi_irq_lock, flags);
1336	mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
1337	mask &= ~BIT(hwirq);
1338	advk_writel(pcie, mask, PCIE_MSI_MASK_REG);
1339	raw_spin_unlock_irqrestore(&pcie->msi_irq_lock, flags);
1340}
1341
1342static void advk_msi_top_irq_mask(struct irq_data *d)
1343{
1344	pci_msi_mask_irq(d);
1345	irq_chip_mask_parent(d);
1346}
1347
1348static void advk_msi_top_irq_unmask(struct irq_data *d)
1349{
1350	pci_msi_unmask_irq(d);
1351	irq_chip_unmask_parent(d);
1352}
1353
1354static struct irq_chip advk_msi_bottom_irq_chip = {
1355	.name			= "MSI",
1356	.irq_compose_msi_msg	= advk_msi_irq_compose_msi_msg,
1357	.irq_set_affinity	= advk_msi_set_affinity,
1358	.irq_mask		= advk_msi_irq_mask,
1359	.irq_unmask		= advk_msi_irq_unmask,
1360};
1361
1362static int advk_msi_irq_domain_alloc(struct irq_domain *domain,
1363				     unsigned int virq,
1364				     unsigned int nr_irqs, void *args)
1365{
1366	struct advk_pcie *pcie = domain->host_data;
1367	int hwirq, i;
1368
1369	mutex_lock(&pcie->msi_used_lock);
1370	hwirq = bitmap_find_free_region(pcie->msi_used, MSI_IRQ_NUM,
1371					order_base_2(nr_irqs));
1372	mutex_unlock(&pcie->msi_used_lock);
1373	if (hwirq < 0)
1374		return -ENOSPC;
 
 
 
 
1375
1376	for (i = 0; i < nr_irqs; i++)
1377		irq_domain_set_info(domain, virq + i, hwirq + i,
1378				    &advk_msi_bottom_irq_chip,
1379				    domain->host_data, handle_simple_irq,
1380				    NULL, NULL);
1381
1382	return 0;
1383}
1384
1385static void advk_msi_irq_domain_free(struct irq_domain *domain,
1386				     unsigned int virq, unsigned int nr_irqs)
1387{
1388	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
1389	struct advk_pcie *pcie = domain->host_data;
1390
1391	mutex_lock(&pcie->msi_used_lock);
1392	bitmap_release_region(pcie->msi_used, d->hwirq, order_base_2(nr_irqs));
1393	mutex_unlock(&pcie->msi_used_lock);
1394}
1395
1396static const struct irq_domain_ops advk_msi_domain_ops = {
1397	.alloc = advk_msi_irq_domain_alloc,
1398	.free = advk_msi_irq_domain_free,
1399};
1400
1401static void advk_pcie_irq_mask(struct irq_data *d)
1402{
1403	struct advk_pcie *pcie = d->domain->host_data;
1404	irq_hw_number_t hwirq = irqd_to_hwirq(d);
1405	unsigned long flags;
1406	u32 mask;
1407
1408	raw_spin_lock_irqsave(&pcie->irq_lock, flags);
1409	mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
1410	mask |= PCIE_ISR1_INTX_ASSERT(hwirq);
1411	advk_writel(pcie, mask, PCIE_ISR1_MASK_REG);
1412	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);
1413}
1414
1415static void advk_pcie_irq_unmask(struct irq_data *d)
1416{
1417	struct advk_pcie *pcie = d->domain->host_data;
1418	irq_hw_number_t hwirq = irqd_to_hwirq(d);
1419	unsigned long flags;
1420	u32 mask;
1421
1422	raw_spin_lock_irqsave(&pcie->irq_lock, flags);
1423	mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
1424	mask &= ~PCIE_ISR1_INTX_ASSERT(hwirq);
1425	advk_writel(pcie, mask, PCIE_ISR1_MASK_REG);
1426	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);
1427}
1428
1429static int advk_pcie_irq_map(struct irq_domain *h,
1430			     unsigned int virq, irq_hw_number_t hwirq)
1431{
1432	struct advk_pcie *pcie = h->host_data;
1433
 
1434	irq_set_status_flags(virq, IRQ_LEVEL);
1435	irq_set_chip_and_handler(virq, &pcie->irq_chip,
1436				 handle_level_irq);
1437	irq_set_chip_data(virq, pcie);
1438
1439	return 0;
1440}
1441
1442static const struct irq_domain_ops advk_pcie_irq_domain_ops = {
1443	.map = advk_pcie_irq_map,
1444	.xlate = irq_domain_xlate_onecell,
1445};
1446
1447static struct irq_chip advk_msi_irq_chip = {
1448	.name		= "advk-MSI",
1449	.irq_mask	= advk_msi_top_irq_mask,
1450	.irq_unmask	= advk_msi_top_irq_unmask,
1451};
1452
1453static struct msi_domain_info advk_msi_domain_info = {
1454	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
1455		  MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX,
1456	.chip	= &advk_msi_irq_chip,
1457};
1458
1459static int advk_pcie_init_msi_irq_domain(struct advk_pcie *pcie)
1460{
1461	struct device *dev = &pcie->pdev->dev;
 
 
 
 
1462
1463	raw_spin_lock_init(&pcie->msi_irq_lock);
1464	mutex_init(&pcie->msi_used_lock);
1465
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1466	pcie->msi_inner_domain =
1467		irq_domain_add_linear(NULL, MSI_IRQ_NUM,
1468				      &advk_msi_domain_ops, pcie);
1469	if (!pcie->msi_inner_domain)
1470		return -ENOMEM;
1471
1472	pcie->msi_domain =
1473		pci_msi_create_irq_domain(dev_fwnode(dev),
1474					  &advk_msi_domain_info,
1475					  pcie->msi_inner_domain);
1476	if (!pcie->msi_domain) {
1477		irq_domain_remove(pcie->msi_inner_domain);
1478		return -ENOMEM;
1479	}
1480
1481	return 0;
1482}
1483
1484static void advk_pcie_remove_msi_irq_domain(struct advk_pcie *pcie)
1485{
1486	irq_domain_remove(pcie->msi_domain);
1487	irq_domain_remove(pcie->msi_inner_domain);
1488}
1489
1490static int advk_pcie_init_irq_domain(struct advk_pcie *pcie)
1491{
1492	struct device *dev = &pcie->pdev->dev;
1493	struct device_node *node = dev->of_node;
1494	struct device_node *pcie_intc_node;
1495	struct irq_chip *irq_chip;
1496	int ret = 0;
1497
1498	raw_spin_lock_init(&pcie->irq_lock);
1499
1500	pcie_intc_node =  of_get_next_child(node, NULL);
1501	if (!pcie_intc_node) {
1502		dev_err(dev, "No PCIe Intc node found\n");
1503		return -ENODEV;
1504	}
1505
1506	irq_chip = &pcie->irq_chip;
1507
1508	irq_chip->name = devm_kasprintf(dev, GFP_KERNEL, "%s-irq",
1509					dev_name(dev));
1510	if (!irq_chip->name) {
1511		ret = -ENOMEM;
1512		goto out_put_node;
1513	}
1514
1515	irq_chip->irq_mask = advk_pcie_irq_mask;
 
1516	irq_chip->irq_unmask = advk_pcie_irq_unmask;
1517
1518	pcie->irq_domain =
1519		irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
1520				      &advk_pcie_irq_domain_ops, pcie);
1521	if (!pcie->irq_domain) {
1522		dev_err(dev, "Failed to get a INTx IRQ domain\n");
1523		ret = -ENOMEM;
1524		goto out_put_node;
1525	}
1526
1527out_put_node:
1528	of_node_put(pcie_intc_node);
1529	return ret;
1530}
1531
1532static void advk_pcie_remove_irq_domain(struct advk_pcie *pcie)
1533{
1534	irq_domain_remove(pcie->irq_domain);
1535}
1536
1537static struct irq_chip advk_rp_irq_chip = {
1538	.name = "advk-RP",
1539};
1540
1541static int advk_pcie_rp_irq_map(struct irq_domain *h,
1542				unsigned int virq, irq_hw_number_t hwirq)
1543{
1544	struct advk_pcie *pcie = h->host_data;
1545
1546	irq_set_chip_and_handler(virq, &advk_rp_irq_chip, handle_simple_irq);
1547	irq_set_chip_data(virq, pcie);
1548
1549	return 0;
1550}
1551
1552static const struct irq_domain_ops advk_pcie_rp_irq_domain_ops = {
1553	.map = advk_pcie_rp_irq_map,
1554	.xlate = irq_domain_xlate_onecell,
1555};
1556
1557static int advk_pcie_init_rp_irq_domain(struct advk_pcie *pcie)
1558{
1559	pcie->rp_irq_domain = irq_domain_add_linear(NULL, 1,
1560						    &advk_pcie_rp_irq_domain_ops,
1561						    pcie);
1562	if (!pcie->rp_irq_domain) {
1563		dev_err(&pcie->pdev->dev, "Failed to add Root Port IRQ domain\n");
1564		return -ENOMEM;
1565	}
1566
1567	return 0;
1568}
1569
1570static void advk_pcie_remove_rp_irq_domain(struct advk_pcie *pcie)
1571{
1572	irq_domain_remove(pcie->rp_irq_domain);
1573}
1574
1575static void advk_pcie_handle_pme(struct advk_pcie *pcie)
1576{
1577	u32 requester = advk_readl(pcie, PCIE_MSG_LOG_REG) >> 16;
1578
1579	advk_writel(pcie, PCIE_MSG_PM_PME_MASK, PCIE_ISR0_REG);
1580
1581	/*
1582	 * PCIE_MSG_LOG_REG contains the last inbound message, so store
1583	 * the requester ID only when PME was not asserted yet.
1584	 * Also do not trigger PME interrupt when PME is still asserted.
1585	 */
1586	if (!(le32_to_cpu(pcie->bridge.pcie_conf.rootsta) & PCI_EXP_RTSTA_PME)) {
1587		pcie->bridge.pcie_conf.rootsta = cpu_to_le32(requester | PCI_EXP_RTSTA_PME);
1588
1589		/*
1590		 * Trigger PME interrupt only if PMEIE bit in Root Control is set.
1591		 * Aardvark HW returns zero for PCI_EXP_FLAGS_IRQ, so use PCIe interrupt 0.
1592		 */
1593		if (!(le16_to_cpu(pcie->bridge.pcie_conf.rootctl) & PCI_EXP_RTCTL_PMEIE))
1594			return;
1595
1596		if (generic_handle_domain_irq(pcie->rp_irq_domain, 0) == -EINVAL)
1597			dev_err_ratelimited(&pcie->pdev->dev, "unhandled PME IRQ\n");
1598	}
1599}
1600
1601static void advk_pcie_handle_msi(struct advk_pcie *pcie)
1602{
1603	u32 msi_val, msi_mask, msi_status, msi_idx;
 
1604
1605	msi_mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
1606	msi_val = advk_readl(pcie, PCIE_MSI_STATUS_REG);
1607	msi_status = msi_val & ((~msi_mask) & PCIE_MSI_ALL_MASK);
1608
1609	for (msi_idx = 0; msi_idx < MSI_IRQ_NUM; msi_idx++) {
1610		if (!(BIT(msi_idx) & msi_status))
1611			continue;
1612
1613		advk_writel(pcie, BIT(msi_idx), PCIE_MSI_STATUS_REG);
1614		if (generic_handle_domain_irq(pcie->msi_inner_domain, msi_idx) == -EINVAL)
1615			dev_err_ratelimited(&pcie->pdev->dev, "unexpected MSI 0x%02x\n", msi_idx);
1616	}
1617
1618	advk_writel(pcie, PCIE_ISR0_MSI_INT_PENDING,
1619		    PCIE_ISR0_REG);
1620}
1621
1622static void advk_pcie_handle_int(struct advk_pcie *pcie)
1623{
1624	u32 isr0_val, isr0_mask, isr0_status;
1625	u32 isr1_val, isr1_mask, isr1_status;
1626	int i;
1627
1628	isr0_val = advk_readl(pcie, PCIE_ISR0_REG);
1629	isr0_mask = advk_readl(pcie, PCIE_ISR0_MASK_REG);
1630	isr0_status = isr0_val & ((~isr0_mask) & PCIE_ISR0_ALL_MASK);
1631
1632	isr1_val = advk_readl(pcie, PCIE_ISR1_REG);
1633	isr1_mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
1634	isr1_status = isr1_val & ((~isr1_mask) & PCIE_ISR1_ALL_MASK);
1635
1636	/* Process PME interrupt as the first one to do not miss PME requester id */
1637	if (isr0_status & PCIE_MSG_PM_PME_MASK)
1638		advk_pcie_handle_pme(pcie);
1639
1640	/* Process ERR interrupt */
1641	if (isr0_status & PCIE_ISR0_ERR_MASK) {
1642		advk_writel(pcie, PCIE_ISR0_ERR_MASK, PCIE_ISR0_REG);
1643
1644		/*
1645		 * Aardvark HW returns zero for PCI_ERR_ROOT_AER_IRQ, so use
1646		 * PCIe interrupt 0
1647		 */
1648		if (generic_handle_domain_irq(pcie->rp_irq_domain, 0) == -EINVAL)
1649			dev_err_ratelimited(&pcie->pdev->dev, "unhandled ERR IRQ\n");
1650	}
1651
1652	/* Process MSI interrupts */
1653	if (isr0_status & PCIE_ISR0_MSI_INT_PENDING)
1654		advk_pcie_handle_msi(pcie);
1655
1656	/* Process legacy interrupts */
1657	for (i = 0; i < PCI_NUM_INTX; i++) {
1658		if (!(isr1_status & PCIE_ISR1_INTX_ASSERT(i)))
1659			continue;
1660
1661		advk_writel(pcie, PCIE_ISR1_INTX_ASSERT(i),
1662			    PCIE_ISR1_REG);
1663
1664		if (generic_handle_domain_irq(pcie->irq_domain, i) == -EINVAL)
1665			dev_err_ratelimited(&pcie->pdev->dev, "unexpected INT%c IRQ\n",
1666					    (char)i + 'A');
1667	}
1668}
1669
1670static irqreturn_t advk_pcie_irq_handler(int irq, void *arg)
1671{
1672	struct advk_pcie *pcie = arg;
1673	u32 status;
1674
1675	status = advk_readl(pcie, HOST_CTRL_INT_STATUS_REG);
1676	if (!(status & PCIE_IRQ_CORE_INT))
1677		return IRQ_NONE;
1678
1679	advk_pcie_handle_int(pcie);
1680
1681	/* Clear interrupt */
1682	advk_writel(pcie, PCIE_IRQ_CORE_INT, HOST_CTRL_INT_STATUS_REG);
1683
1684	return IRQ_HANDLED;
1685}
1686
1687static int advk_pcie_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
1688{
1689	struct advk_pcie *pcie = dev->bus->sysdata;
1690
1691	/*
1692	 * Emulated root bridge has its own emulated irq chip and irq domain.
1693	 * Argument pin is the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD) and
1694	 * hwirq for irq_create_mapping() is indexed from zero.
1695	 */
1696	if (pci_is_root_bus(dev->bus))
1697		return irq_create_mapping(pcie->rp_irq_domain, pin - 1);
1698	else
1699		return of_irq_parse_and_map_pci(dev, slot, pin);
1700}
1701
1702static void advk_pcie_disable_phy(struct advk_pcie *pcie)
1703{
1704	phy_power_off(pcie->phy);
1705	phy_exit(pcie->phy);
1706}
1707
1708static int advk_pcie_enable_phy(struct advk_pcie *pcie)
1709{
1710	int ret;
1711
1712	if (!pcie->phy)
1713		return 0;
1714
1715	ret = phy_init(pcie->phy);
1716	if (ret)
1717		return ret;
1718
1719	ret = phy_set_mode(pcie->phy, PHY_MODE_PCIE);
1720	if (ret) {
1721		phy_exit(pcie->phy);
1722		return ret;
1723	}
1724
1725	ret = phy_power_on(pcie->phy);
1726	if (ret) {
 
 
1727		phy_exit(pcie->phy);
1728		return ret;
1729	}
1730
1731	return 0;
1732}
1733
1734static int advk_pcie_setup_phy(struct advk_pcie *pcie)
1735{
1736	struct device *dev = &pcie->pdev->dev;
1737	struct device_node *node = dev->of_node;
1738	int ret = 0;
1739
1740	pcie->phy = devm_of_phy_get(dev, node, NULL);
1741	if (IS_ERR(pcie->phy) && (PTR_ERR(pcie->phy) == -EPROBE_DEFER))
1742		return PTR_ERR(pcie->phy);
1743
1744	/* Old bindings miss the PHY handle */
1745	if (IS_ERR(pcie->phy)) {
1746		dev_warn(dev, "PHY unavailable (%ld)\n", PTR_ERR(pcie->phy));
1747		pcie->phy = NULL;
1748		return 0;
1749	}
1750
1751	ret = advk_pcie_enable_phy(pcie);
1752	if (ret)
1753		dev_err(dev, "Failed to initialize PHY (%d)\n", ret);
1754
1755	return ret;
1756}
1757
1758static int advk_pcie_probe(struct platform_device *pdev)
1759{
1760	struct device *dev = &pdev->dev;
1761	struct advk_pcie *pcie;
1762	struct pci_host_bridge *bridge;
1763	struct resource_entry *entry;
1764	int ret, irq;
1765
1766	bridge = devm_pci_alloc_host_bridge(dev, sizeof(struct advk_pcie));
1767	if (!bridge)
1768		return -ENOMEM;
1769
1770	pcie = pci_host_bridge_priv(bridge);
1771	pcie->pdev = pdev;
1772	platform_set_drvdata(pdev, pcie);
1773
1774	resource_list_for_each_entry(entry, &bridge->windows) {
1775		resource_size_t start = entry->res->start;
1776		resource_size_t size = resource_size(entry->res);
1777		unsigned long type = resource_type(entry->res);
1778		u64 win_size;
1779
1780		/*
1781		 * Aardvark hardware allows to configure also PCIe window
1782		 * for config type 0 and type 1 mapping, but driver uses
1783		 * only PIO for issuing configuration transfers which does
1784		 * not use PCIe window configuration.
1785		 */
1786		if (type != IORESOURCE_MEM && type != IORESOURCE_IO)
 
1787			continue;
1788
1789		/*
1790		 * Skip transparent memory resources. Default outbound access
1791		 * configuration is set to transparent memory access so it
1792		 * does not need window configuration.
1793		 */
1794		if (type == IORESOURCE_MEM && entry->offset == 0)
 
1795			continue;
1796
1797		/*
1798		 * The n-th PCIe window is configured by tuple (match, remap, mask)
1799		 * and an access to address A uses this window if A matches the
1800		 * match with given mask.
1801		 * So every PCIe window size must be a power of two and every start
1802		 * address must be aligned to window size. Minimal size is 64 KiB
1803		 * because lower 16 bits of mask must be zero. Remapped address
1804		 * may have set only bits from the mask.
1805		 */
1806		while (pcie->wins_count < OB_WIN_COUNT && size > 0) {
1807			/* Calculate the largest aligned window size */
1808			win_size = (1ULL << (fls64(size)-1)) |
1809				   (start ? (1ULL << __ffs64(start)) : 0);
1810			win_size = 1ULL << __ffs64(win_size);
1811			if (win_size < 0x10000)
1812				break;
1813
1814			dev_dbg(dev,
1815				"Configuring PCIe window %d: [0x%llx-0x%llx] as %lu\n",
1816				pcie->wins_count, (unsigned long long)start,
1817				(unsigned long long)start + win_size, type);
1818
1819			if (type == IORESOURCE_IO) {
1820				pcie->wins[pcie->wins_count].actions = OB_WIN_TYPE_IO;
1821				pcie->wins[pcie->wins_count].match = pci_pio_to_address(start);
1822			} else {
1823				pcie->wins[pcie->wins_count].actions = OB_WIN_TYPE_MEM;
1824				pcie->wins[pcie->wins_count].match = start;
1825			}
1826			pcie->wins[pcie->wins_count].remap = start - entry->offset;
1827			pcie->wins[pcie->wins_count].mask = ~(win_size - 1);
1828
1829			if (pcie->wins[pcie->wins_count].remap & (win_size - 1))
1830				break;
1831
1832			start += win_size;
1833			size -= win_size;
1834			pcie->wins_count++;
1835		}
1836
1837		if (size > 0) {
1838			dev_err(&pcie->pdev->dev,
1839				"Invalid PCIe region [0x%llx-0x%llx]\n",
1840				(unsigned long long)entry->res->start,
1841				(unsigned long long)entry->res->end + 1);
1842			return -EINVAL;
1843		}
1844	}
1845
1846	pcie->base = devm_platform_ioremap_resource(pdev, 0);
1847	if (IS_ERR(pcie->base))
1848		return PTR_ERR(pcie->base);
1849
1850	irq = platform_get_irq(pdev, 0);
1851	if (irq < 0)
1852		return irq;
1853
1854	ret = devm_request_irq(dev, irq, advk_pcie_irq_handler,
1855			       IRQF_SHARED | IRQF_NO_THREAD, "advk-pcie",
1856			       pcie);
1857	if (ret) {
1858		dev_err(dev, "Failed to register interrupt\n");
1859		return ret;
1860	}
1861
1862	pcie->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
 
 
 
1863	ret = PTR_ERR_OR_ZERO(pcie->reset_gpio);
1864	if (ret) {
1865		if (ret != -EPROBE_DEFER)
1866			dev_err(dev, "Failed to get reset-gpio: %i\n", ret);
1867		return ret;
1868	}
1869
1870	ret = gpiod_set_consumer_name(pcie->reset_gpio, "pcie1-reset");
1871	if (ret) {
1872		dev_err(dev, "Failed to set reset gpio name: %d\n", ret);
1873		return ret;
1874	}
1875
1876	ret = of_pci_get_max_link_speed(dev->of_node);
1877	if (ret <= 0 || ret > 3)
1878		pcie->link_gen = 3;
1879	else
1880		pcie->link_gen = ret;
1881
1882	ret = advk_pcie_setup_phy(pcie);
1883	if (ret)
1884		return ret;
1885
1886	advk_pcie_setup_hw(pcie);
1887
1888	ret = advk_sw_pci_bridge_init(pcie);
1889	if (ret) {
1890		dev_err(dev, "Failed to register emulated root PCI bridge\n");
1891		return ret;
1892	}
1893
1894	ret = advk_pcie_init_irq_domain(pcie);
1895	if (ret) {
1896		dev_err(dev, "Failed to initialize irq\n");
1897		return ret;
1898	}
1899
1900	ret = advk_pcie_init_msi_irq_domain(pcie);
1901	if (ret) {
1902		dev_err(dev, "Failed to initialize irq\n");
1903		advk_pcie_remove_irq_domain(pcie);
1904		return ret;
1905	}
1906
1907	ret = advk_pcie_init_rp_irq_domain(pcie);
1908	if (ret) {
1909		dev_err(dev, "Failed to initialize irq\n");
1910		advk_pcie_remove_msi_irq_domain(pcie);
1911		advk_pcie_remove_irq_domain(pcie);
1912		return ret;
1913	}
1914
1915	bridge->sysdata = pcie;
1916	bridge->ops = &advk_pcie_ops;
1917	bridge->map_irq = advk_pcie_map_irq;
1918
1919	ret = pci_host_probe(bridge);
1920	if (ret < 0) {
1921		advk_pcie_remove_rp_irq_domain(pcie);
1922		advk_pcie_remove_msi_irq_domain(pcie);
1923		advk_pcie_remove_irq_domain(pcie);
1924		return ret;
1925	}
1926
1927	return 0;
1928}
1929
1930static void advk_pcie_remove(struct platform_device *pdev)
1931{
1932	struct advk_pcie *pcie = platform_get_drvdata(pdev);
1933	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1934	u32 val;
1935	int i;
1936
1937	/* Remove PCI bus with all devices */
1938	pci_lock_rescan_remove();
1939	pci_stop_root_bus(bridge->bus);
1940	pci_remove_root_bus(bridge->bus);
1941	pci_unlock_rescan_remove();
1942
1943	/* Disable Root Bridge I/O space, memory space and bus mastering */
1944	val = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
1945	val &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
1946	advk_writel(pcie, val, PCIE_CORE_CMD_STATUS_REG);
1947
1948	/* Disable MSI */
1949	val = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
1950	val &= ~PCIE_CORE_CTRL2_MSI_ENABLE;
1951	advk_writel(pcie, val, PCIE_CORE_CTRL2_REG);
1952
1953	/* Clear MSI address */
1954	advk_writel(pcie, 0, PCIE_MSI_ADDR_LOW_REG);
1955	advk_writel(pcie, 0, PCIE_MSI_ADDR_HIGH_REG);
1956
1957	/* Mask all interrupts */
1958	advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_MASK_REG);
1959	advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_MASK_REG);
1960	advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG);
1961	advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_MASK_REG);
1962
1963	/* Clear all interrupts */
1964	advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_STATUS_REG);
1965	advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_REG);
1966	advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_REG);
1967	advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_STATUS_REG);
1968
1969	/* Remove IRQ domains */
1970	advk_pcie_remove_rp_irq_domain(pcie);
1971	advk_pcie_remove_msi_irq_domain(pcie);
1972	advk_pcie_remove_irq_domain(pcie);
1973
1974	/* Free config space for emulated root bridge */
1975	pci_bridge_emul_cleanup(&pcie->bridge);
1976
1977	/* Assert PERST# signal which prepares PCIe card for power down */
1978	if (pcie->reset_gpio)
1979		gpiod_set_value_cansleep(pcie->reset_gpio, 1);
1980
1981	/* Disable link training */
1982	val = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
1983	val &= ~LINK_TRAINING_EN;
1984	advk_writel(pcie, val, PCIE_CORE_CTRL0_REG);
1985
1986	/* Disable outbound address windows mapping */
1987	for (i = 0; i < OB_WIN_COUNT; i++)
1988		advk_pcie_disable_ob_win(pcie, i);
1989
1990	/* Disable phy */
1991	advk_pcie_disable_phy(pcie);
1992}
1993
1994static const struct of_device_id advk_pcie_of_match_table[] = {
1995	{ .compatible = "marvell,armada-3700-pcie", },
1996	{},
1997};
1998MODULE_DEVICE_TABLE(of, advk_pcie_of_match_table);
1999
2000static struct platform_driver advk_pcie_driver = {
2001	.driver = {
2002		.name = "advk-pcie",
2003		.of_match_table = advk_pcie_of_match_table,
2004	},
2005	.probe = advk_pcie_probe,
2006	.remove_new = advk_pcie_remove,
2007};
2008module_platform_driver(advk_pcie_driver);
2009
2010MODULE_DESCRIPTION("Aardvark PCIe controller");
2011MODULE_LICENSE("GPL v2");