1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2014 Hauke Mehrtens <hauke@hauke-m.de>
   4 * Copyright (C) 2015 Broadcom Corporation
   5 */
   6
   7#include <linux/kernel.h>
   8#include <linux/pci.h>
   9#include <linux/pci-ecam.h>
  10#include <linux/msi.h>
  11#include <linux/clk.h>
  12#include <linux/module.h>
  13#include <linux/mbus.h>
  14#include <linux/slab.h>
  15#include <linux/delay.h>
  16#include <linux/interrupt.h>
  17#include <linux/irqchip/arm-gic-v3.h>
  18#include <linux/platform_device.h>
  19#include <linux/of_address.h>
  20#include <linux/of_pci.h>
  21#include <linux/of_platform.h>
  22#include <linux/phy/phy.h>
  23
  24#include "pcie-iproc.h"
  25
  26#define EP_PERST_SOURCE_SELECT_SHIFT	2
  27#define EP_PERST_SOURCE_SELECT		BIT(EP_PERST_SOURCE_SELECT_SHIFT)
  28#define EP_MODE_SURVIVE_PERST_SHIFT	1
  29#define EP_MODE_SURVIVE_PERST		BIT(EP_MODE_SURVIVE_PERST_SHIFT)
  30#define RC_PCIE_RST_OUTPUT_SHIFT	0
  31#define RC_PCIE_RST_OUTPUT		BIT(RC_PCIE_RST_OUTPUT_SHIFT)
  32#define PAXC_RESET_MASK			0x7f
  33
  34#define GIC_V3_CFG_SHIFT		0
  35#define GIC_V3_CFG			BIT(GIC_V3_CFG_SHIFT)
  36
  37#define MSI_ENABLE_CFG_SHIFT		0
  38#define MSI_ENABLE_CFG			BIT(MSI_ENABLE_CFG_SHIFT)
  39
  40#define CFG_IND_ADDR_MASK		0x00001ffc
  41
  42#define CFG_ADDR_REG_NUM_MASK		0x00000ffc
  43#define CFG_ADDR_CFG_TYPE_1		1
  44
  45#define SYS_RC_INTX_MASK		0xf
  46
  47#define PCIE_PHYLINKUP_SHIFT		3
  48#define PCIE_PHYLINKUP			BIT(PCIE_PHYLINKUP_SHIFT)
  49#define PCIE_DL_ACTIVE_SHIFT		2
  50#define PCIE_DL_ACTIVE			BIT(PCIE_DL_ACTIVE_SHIFT)
  51
  52#define APB_ERR_EN_SHIFT		0
  53#define APB_ERR_EN			BIT(APB_ERR_EN_SHIFT)
  54
  55#define CFG_RD_SUCCESS			0
  56#define CFG_RD_UR			1
  57#define CFG_RD_RRS			2
  58#define CFG_RD_CA			3
  59#define CFG_RETRY_STATUS		0xffff0001
  60#define CFG_RETRY_STATUS_TIMEOUT_US	500000 /* 500 milliseconds */
  61
  62/* derive the enum index of the outbound/inbound mapping registers */
  63#define MAP_REG(base_reg, index)	((base_reg) + (index) * 2)
  64
  65/*
  66 * Maximum number of outbound mapping window sizes that can be supported by any
  67 * OARR/OMAP mapping pair
  68 */
  69#define MAX_NUM_OB_WINDOW_SIZES		4
  70
  71#define OARR_VALID_SHIFT		0
  72#define OARR_VALID			BIT(OARR_VALID_SHIFT)
  73#define OARR_SIZE_CFG_SHIFT		1
  74
  75/*
  76 * Maximum number of inbound mapping region sizes that can be supported by an
  77 * IARR
  78 */
  79#define MAX_NUM_IB_REGION_SIZES		9
  80
  81#define IMAP_VALID_SHIFT		0
  82#define IMAP_VALID			BIT(IMAP_VALID_SHIFT)
  83
  84#define IPROC_PCI_PM_CAP		0x48
  85#define IPROC_PCI_PM_CAP_MASK		0xffff
  86#define IPROC_PCI_EXP_CAP		0xac
  87
  88#define IPROC_PCIE_REG_INVALID		0xffff
  89
  90/**
  91 * struct iproc_pcie_ob_map - iProc PCIe outbound mapping controller-specific
  92 * parameters
  93 * @window_sizes: list of supported outbound mapping window sizes in MB
  94 * @nr_sizes: number of supported outbound mapping window sizes
  95 */
  96struct iproc_pcie_ob_map {
  97	resource_size_t window_sizes[MAX_NUM_OB_WINDOW_SIZES];
  98	unsigned int nr_sizes;
  99};
 100
 101static const struct iproc_pcie_ob_map paxb_ob_map[] = {
 102	{
 103		/* OARR0/OMAP0 */
 104		.window_sizes = { 128, 256 },
 105		.nr_sizes = 2,
 106	},
 107	{
 108		/* OARR1/OMAP1 */
 109		.window_sizes = { 128, 256 },
 110		.nr_sizes = 2,
 111	},
 112};
 113
 114static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = {
 115	{
 116		/* OARR0/OMAP0 */
 117		.window_sizes = { 128, 256 },
 118		.nr_sizes = 2,
 119	},
 120	{
 121		/* OARR1/OMAP1 */
 122		.window_sizes = { 128, 256 },
 123		.nr_sizes = 2,
 124	},
 125	{
 126		/* OARR2/OMAP2 */
 127		.window_sizes = { 128, 256, 512, 1024 },
 128		.nr_sizes = 4,
 129	},
 130	{
 131		/* OARR3/OMAP3 */
 132		.window_sizes = { 128, 256, 512, 1024 },
 133		.nr_sizes = 4,
 134	},
 135};
 136
 137/**
 138 * enum iproc_pcie_ib_map_type - iProc PCIe inbound mapping type
 139 * @IPROC_PCIE_IB_MAP_MEM: DDR memory
 140 * @IPROC_PCIE_IB_MAP_IO: device I/O memory
 141 * @IPROC_PCIE_IB_MAP_INVALID: invalid or unused
 142 */
 143enum iproc_pcie_ib_map_type {
 144	IPROC_PCIE_IB_MAP_MEM = 0,
 145	IPROC_PCIE_IB_MAP_IO,
 146	IPROC_PCIE_IB_MAP_INVALID
 147};
 148
 149/**
 150 * struct iproc_pcie_ib_map - iProc PCIe inbound mapping controller-specific
 151 * parameters
 152 * @type: inbound mapping region type
 153 * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or
 154 * SZ_1G
 155 * @region_sizes: list of supported inbound mapping region sizes in KB, MB, or
 156 * GB, depending on the size unit
 157 * @nr_sizes: number of supported inbound mapping region sizes
 158 * @nr_windows: number of supported inbound mapping windows for the region
 159 * @imap_addr_offset: register offset between the upper and lower 32-bit
 160 * IMAP address registers
 161 * @imap_window_offset: register offset between each IMAP window
 162 */
 163struct iproc_pcie_ib_map {
 164	enum iproc_pcie_ib_map_type type;
 165	unsigned int size_unit;
 166	resource_size_t region_sizes[MAX_NUM_IB_REGION_SIZES];
 167	unsigned int nr_sizes;
 168	unsigned int nr_windows;
 169	u16 imap_addr_offset;
 170	u16 imap_window_offset;
 171};
 172
 173static const struct iproc_pcie_ib_map paxb_v2_ib_map[] = {
 174	{
 175		/* IARR0/IMAP0 */
 176		.type = IPROC_PCIE_IB_MAP_IO,
 177		.size_unit = SZ_1K,
 178		.region_sizes = { 32 },
 179		.nr_sizes = 1,
 180		.nr_windows = 8,
 181		.imap_addr_offset = 0x40,
 182		.imap_window_offset = 0x4,
 183	},
 184	{
 185		/* IARR1/IMAP1 */
 186		.type = IPROC_PCIE_IB_MAP_MEM,
 187		.size_unit = SZ_1M,
 188		.region_sizes = { 8 },
 189		.nr_sizes = 1,
 190		.nr_windows = 8,
 191		.imap_addr_offset = 0x4,
 192		.imap_window_offset = 0x8,
 193
 194	},
 195	{
 196		/* IARR2/IMAP2 */
 197		.type = IPROC_PCIE_IB_MAP_MEM,
 198		.size_unit = SZ_1M,
 199		.region_sizes = { 64, 128, 256, 512, 1024, 2048, 4096, 8192,
 200				  16384 },
 201		.nr_sizes = 9,
 202		.nr_windows = 1,
 203		.imap_addr_offset = 0x4,
 204		.imap_window_offset = 0x8,
 205	},
 206	{
 207		/* IARR3/IMAP3 */
 208		.type = IPROC_PCIE_IB_MAP_MEM,
 209		.size_unit = SZ_1G,
 210		.region_sizes = { 1, 2, 4, 8, 16, 32 },
 211		.nr_sizes = 6,
 212		.nr_windows = 8,
 213		.imap_addr_offset = 0x4,
 214		.imap_window_offset = 0x8,
 215	},
 216	{
 217		/* IARR4/IMAP4 */
 218		.type = IPROC_PCIE_IB_MAP_MEM,
 219		.size_unit = SZ_1G,
 220		.region_sizes = { 32, 64, 128, 256, 512 },
 221		.nr_sizes = 5,
 222		.nr_windows = 8,
 223		.imap_addr_offset = 0x4,
 224		.imap_window_offset = 0x8,
 225	},
 226};
 227
 228/*
 229 * iProc PCIe host registers
 230 */
 231enum iproc_pcie_reg {
 232	/* clock/reset signal control */
 233	IPROC_PCIE_CLK_CTRL = 0,
 234
 235	/*
 236	 * To allow MSI to be steered to an external MSI controller (e.g., ARM
 237	 * GICv3 ITS)
 238	 */
 239	IPROC_PCIE_MSI_GIC_MODE,
 240
 241	/*
 242	 * IPROC_PCIE_MSI_BASE_ADDR and IPROC_PCIE_MSI_WINDOW_SIZE define the
 243	 * window where the MSI posted writes are written, for the writes to be
 244	 * interpreted as MSI writes.
 245	 */
 246	IPROC_PCIE_MSI_BASE_ADDR,
 247	IPROC_PCIE_MSI_WINDOW_SIZE,
 248
 249	/*
 250	 * To hold the address of the register where the MSI writes are
 251	 * programmed.  When ARM GICv3 ITS is used, this should be programmed
 252	 * with the address of the GITS_TRANSLATER register.
 253	 */
 254	IPROC_PCIE_MSI_ADDR_LO,
 255	IPROC_PCIE_MSI_ADDR_HI,
 256
 257	/* enable MSI */
 258	IPROC_PCIE_MSI_EN_CFG,
 259
 260	/* allow access to root complex configuration space */
 261	IPROC_PCIE_CFG_IND_ADDR,
 262	IPROC_PCIE_CFG_IND_DATA,
 263
 264	/* allow access to device configuration space */
 265	IPROC_PCIE_CFG_ADDR,
 266	IPROC_PCIE_CFG_DATA,
 267
 268	/* enable INTx */
 269	IPROC_PCIE_INTX_EN,
 270
 271	/* outbound address mapping */
 272	IPROC_PCIE_OARR0,
 273	IPROC_PCIE_OMAP0,
 274	IPROC_PCIE_OARR1,
 275	IPROC_PCIE_OMAP1,
 276	IPROC_PCIE_OARR2,
 277	IPROC_PCIE_OMAP2,
 278	IPROC_PCIE_OARR3,
 279	IPROC_PCIE_OMAP3,
 280
 281	/* inbound address mapping */
 282	IPROC_PCIE_IARR0,
 283	IPROC_PCIE_IMAP0,
 284	IPROC_PCIE_IARR1,
 285	IPROC_PCIE_IMAP1,
 286	IPROC_PCIE_IARR2,
 287	IPROC_PCIE_IMAP2,
 288	IPROC_PCIE_IARR3,
 289	IPROC_PCIE_IMAP3,
 290	IPROC_PCIE_IARR4,
 291	IPROC_PCIE_IMAP4,
 292
 293	/* config read status */
 294	IPROC_PCIE_CFG_RD_STATUS,
 295
 296	/* link status */
 297	IPROC_PCIE_LINK_STATUS,
 298
 299	/* enable APB error for unsupported requests */
 300	IPROC_PCIE_APB_ERR_EN,
 301
 302	/* total number of core registers */
 303	IPROC_PCIE_MAX_NUM_REG,
 304};
 305
 306/* iProc PCIe PAXB BCMA registers */
 307static const u16 iproc_pcie_reg_paxb_bcma[IPROC_PCIE_MAX_NUM_REG] = {
 308	[IPROC_PCIE_CLK_CTRL]		= 0x000,
 309	[IPROC_PCIE_CFG_IND_ADDR]	= 0x120,
 310	[IPROC_PCIE_CFG_IND_DATA]	= 0x124,
 311	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
 312	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
 313	[IPROC_PCIE_INTX_EN]		= 0x330,
 314	[IPROC_PCIE_LINK_STATUS]	= 0xf0c,
 315};
 316
 317/* iProc PCIe PAXB registers */
 318static const u16 iproc_pcie_reg_paxb[IPROC_PCIE_MAX_NUM_REG] = {
 319	[IPROC_PCIE_CLK_CTRL]		= 0x000,
 320	[IPROC_PCIE_CFG_IND_ADDR]	= 0x120,
 321	[IPROC_PCIE_CFG_IND_DATA]	= 0x124,
 322	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
 323	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
 324	[IPROC_PCIE_INTX_EN]		= 0x330,
 325	[IPROC_PCIE_OARR0]		= 0xd20,
 326	[IPROC_PCIE_OMAP0]		= 0xd40,
 327	[IPROC_PCIE_OARR1]		= 0xd28,
 328	[IPROC_PCIE_OMAP1]		= 0xd48,
 329	[IPROC_PCIE_LINK_STATUS]	= 0xf0c,
 330	[IPROC_PCIE_APB_ERR_EN]		= 0xf40,
 331};
 332
 333/* iProc PCIe PAXB v2 registers */
 334static const u16 iproc_pcie_reg_paxb_v2[IPROC_PCIE_MAX_NUM_REG] = {
 335	[IPROC_PCIE_CLK_CTRL]		= 0x000,
 336	[IPROC_PCIE_CFG_IND_ADDR]	= 0x120,
 337	[IPROC_PCIE_CFG_IND_DATA]	= 0x124,
 338	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
 339	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
 340	[IPROC_PCIE_INTX_EN]		= 0x330,
 341	[IPROC_PCIE_OARR0]		= 0xd20,
 342	[IPROC_PCIE_OMAP0]		= 0xd40,
 343	[IPROC_PCIE_OARR1]		= 0xd28,
 344	[IPROC_PCIE_OMAP1]		= 0xd48,
 345	[IPROC_PCIE_OARR2]		= 0xd60,
 346	[IPROC_PCIE_OMAP2]		= 0xd68,
 347	[IPROC_PCIE_OARR3]		= 0xdf0,
 348	[IPROC_PCIE_OMAP3]		= 0xdf8,
 349	[IPROC_PCIE_IARR0]		= 0xd00,
 350	[IPROC_PCIE_IMAP0]		= 0xc00,
 351	[IPROC_PCIE_IARR1]		= 0xd08,
 352	[IPROC_PCIE_IMAP1]		= 0xd70,
 353	[IPROC_PCIE_IARR2]		= 0xd10,
 354	[IPROC_PCIE_IMAP2]		= 0xcc0,
 355	[IPROC_PCIE_IARR3]		= 0xe00,
 356	[IPROC_PCIE_IMAP3]		= 0xe08,
 357	[IPROC_PCIE_IARR4]		= 0xe68,
 358	[IPROC_PCIE_IMAP4]		= 0xe70,
 359	[IPROC_PCIE_CFG_RD_STATUS]	= 0xee0,
 360	[IPROC_PCIE_LINK_STATUS]	= 0xf0c,
 361	[IPROC_PCIE_APB_ERR_EN]		= 0xf40,
 362};
 363
 364/* iProc PCIe PAXC v1 registers */
 365static const u16 iproc_pcie_reg_paxc[IPROC_PCIE_MAX_NUM_REG] = {
 366	[IPROC_PCIE_CLK_CTRL]		= 0x000,
 367	[IPROC_PCIE_CFG_IND_ADDR]	= 0x1f0,
 368	[IPROC_PCIE_CFG_IND_DATA]	= 0x1f4,
 369	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
 370	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
 371};
 372
 373/* iProc PCIe PAXC v2 registers */
 374static const u16 iproc_pcie_reg_paxc_v2[IPROC_PCIE_MAX_NUM_REG] = {
 375	[IPROC_PCIE_MSI_GIC_MODE]	= 0x050,
 376	[IPROC_PCIE_MSI_BASE_ADDR]	= 0x074,
 377	[IPROC_PCIE_MSI_WINDOW_SIZE]	= 0x078,
 378	[IPROC_PCIE_MSI_ADDR_LO]	= 0x07c,
 379	[IPROC_PCIE_MSI_ADDR_HI]	= 0x080,
 380	[IPROC_PCIE_MSI_EN_CFG]		= 0x09c,
 381	[IPROC_PCIE_CFG_IND_ADDR]	= 0x1f0,
 382	[IPROC_PCIE_CFG_IND_DATA]	= 0x1f4,
 383	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
 384	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
 385};
 386
 387/*
 388 * List of device IDs of controllers that have corrupted capability list that
 389 * require SW fixup
 390 */
 391static const u16 iproc_pcie_corrupt_cap_did[] = {
 392	0x16cd,
 393	0x16f0,
 394	0xd802,
 395	0xd804
 396};
 397
 398static inline struct iproc_pcie *iproc_data(struct pci_bus *bus)
 399{
 400	struct iproc_pcie *pcie = bus->sysdata;
 401	return pcie;
 402}
 403
 404static inline bool iproc_pcie_reg_is_invalid(u16 reg_offset)
 405{
 406	return !!(reg_offset == IPROC_PCIE_REG_INVALID);
 407}
 408
 409static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie,
 410					enum iproc_pcie_reg reg)
 411{
 412	return pcie->reg_offsets[reg];
 413}
 414
 415static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie,
 416				      enum iproc_pcie_reg reg)
 417{
 418	u16 offset = iproc_pcie_reg_offset(pcie, reg);
 419
 420	if (iproc_pcie_reg_is_invalid(offset))
 421		return 0;
 422
 423	return readl(pcie->base + offset);
 424}
 425
 426static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie,
 427					enum iproc_pcie_reg reg, u32 val)
 428{
 429	u16 offset = iproc_pcie_reg_offset(pcie, reg);
 430
 431	if (iproc_pcie_reg_is_invalid(offset))
 432		return;
 433
 434	writel(val, pcie->base + offset);
 435}
 436
 437/*
 438 * APB error forwarding can be disabled during access of configuration
 439 * registers of the endpoint device, to prevent unsupported requests
 440 * (typically seen during enumeration with multi-function devices) from
 441 * triggering a system exception.
 442 */
 443static inline void iproc_pcie_apb_err_disable(struct pci_bus *bus,
 444					      bool disable)
 445{
 446	struct iproc_pcie *pcie = iproc_data(bus);
 447	u32 val;
 448
 449	if (bus->number && pcie->has_apb_err_disable) {
 450		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN);
 451		if (disable)
 452			val &= ~APB_ERR_EN;
 453		else
 454			val |= APB_ERR_EN;
 455		iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val);
 456	}
 457}
 458
 459static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie,
 460					       unsigned int busno,
 461					       unsigned int devfn,
 462					       int where)
 463{
 464	u16 offset;
 465	u32 val;
 466
 467	/* EP device access */
 468	val = ALIGN_DOWN(PCIE_ECAM_OFFSET(busno, devfn, where), 4) |
 469		CFG_ADDR_CFG_TYPE_1;
 470
 471	iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val);
 472	offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA);
 473
 474	if (iproc_pcie_reg_is_invalid(offset))
 475		return NULL;
 476
 477	return (pcie->base + offset);
 478}
 479
 480static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie,
 481					 void __iomem *cfg_data_p)
 482{
 483	int timeout = CFG_RETRY_STATUS_TIMEOUT_US;
 484	unsigned int data;
 485	u32 status;
 486
 487	/*
 488	 * As per PCIe r6.0, sec 2.3.2, Config RRS Software Visibility only
 489	 * affects config reads of the Vendor ID.  For config writes or any
 490	 * other config reads, the Root may automatically reissue the
 491	 * configuration request again as a new request.
 492	 *
 493	 * For config reads, this hardware returns CFG_RETRY_STATUS data
 494	 * when it receives a RRS completion, regardless of the address of
 495	 * the read or the RRS Software Visibility Enable bit.  As a
 496	 * partial workaround for this, we retry in software any read that
 497	 * returns CFG_RETRY_STATUS.
 498	 *
 499	 * Note that a non-Vendor ID config register may have a value of
 500	 * CFG_RETRY_STATUS.  If we read that, we can't distinguish it from
 501	 * a RRS completion, so we will incorrectly retry the read and
 502	 * eventually return the wrong data (0xffffffff).
 503	 */
 504	data = readl(cfg_data_p);
 505	while (data == CFG_RETRY_STATUS && timeout--) {
 506		/*
 507		 * RRS state is set in CFG_RD status register
 508		 * This will handle the case where CFG_RETRY_STATUS is
 509		 * valid config data.
 510		 */
 511		status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS);
 512		if (status != CFG_RD_RRS)
 513			return data;
 514
 515		udelay(1);
 516		data = readl(cfg_data_p);
 517	}
 518
 519	if (data == CFG_RETRY_STATUS)
 520		data = 0xffffffff;
 521
 522	return data;
 523}
 524
 525static void iproc_pcie_fix_cap(struct iproc_pcie *pcie, int where, u32 *val)
 526{
 527	u32 i, dev_id;
 528
 529	switch (where & ~0x3) {
 530	case PCI_VENDOR_ID:
 531		dev_id = *val >> 16;
 532
 533		/*
 534		 * Activate fixup for those controllers that have corrupted
 535		 * capability list registers
 536		 */
 537		for (i = 0; i < ARRAY_SIZE(iproc_pcie_corrupt_cap_did); i++)
 538			if (dev_id == iproc_pcie_corrupt_cap_did[i])
 539				pcie->fix_paxc_cap = true;
 540		break;
 541
 542	case IPROC_PCI_PM_CAP:
 543		if (pcie->fix_paxc_cap) {
 544			/* advertise PM, force next capability to PCIe */
 545			*val &= ~IPROC_PCI_PM_CAP_MASK;
 546			*val |= IPROC_PCI_EXP_CAP << 8 | PCI_CAP_ID_PM;
 547		}
 548		break;
 549
 550	case IPROC_PCI_EXP_CAP:
 551		if (pcie->fix_paxc_cap) {
 552			/* advertise root port, version 2, terminate here */
 553			*val = (PCI_EXP_TYPE_ROOT_PORT << 4 | 2) << 16 |
 554				PCI_CAP_ID_EXP;
 555		}
 556		break;
 557
 558	case IPROC_PCI_EXP_CAP + PCI_EXP_RTCTL:
 559		/* Don't advertise RRS SV support */
 560		*val &= ~(PCI_EXP_RTCAP_RRS_SV << 16);
 561		break;
 562
 563	default:
 564		break;
 565	}
 566}
 567
 568static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
 569				  int where, int size, u32 *val)
 570{
 571	struct iproc_pcie *pcie = iproc_data(bus);
 572	unsigned int busno = bus->number;
 573	void __iomem *cfg_data_p;
 574	unsigned int data;
 575	int ret;
 576
 577	/* root complex access */
 578	if (busno == 0) {
 579		ret = pci_generic_config_read32(bus, devfn, where, size, val);
 580		if (ret == PCIBIOS_SUCCESSFUL)
 581			iproc_pcie_fix_cap(pcie, where, val);
 582
 583		return ret;
 584	}
 585
 586	cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where);
 587
 588	if (!cfg_data_p)
 589		return PCIBIOS_DEVICE_NOT_FOUND;
 590
 591	data = iproc_pcie_cfg_retry(pcie, cfg_data_p);
 592
 593	*val = data;
 594	if (size <= 2)
 595		*val = (data >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
 596
 597	/*
 598	 * For PAXC and PAXCv2, the total number of PFs that one can enumerate
 599	 * depends on the firmware configuration. Unfortunately, due to an ASIC
 600	 * bug, unconfigured PFs cannot be properly hidden from the root
 601	 * complex. As a result, write access to these PFs will cause bus lock
 602	 * up on the embedded processor
 603	 *
 604	 * Since all unconfigured PFs are left with an incorrect, staled device
 605	 * ID of 0x168e (PCI_DEVICE_ID_NX2_57810), we try to catch those access
 606	 * early here and reject them all
 607	 */
 608#define DEVICE_ID_MASK     0xffff0000
 609#define DEVICE_ID_SHIFT    16
 610	if (pcie->rej_unconfig_pf &&
 611	    (where & CFG_ADDR_REG_NUM_MASK) == PCI_VENDOR_ID)
 612		if ((*val & DEVICE_ID_MASK) ==
 613		    (PCI_DEVICE_ID_NX2_57810 << DEVICE_ID_SHIFT))
 614			return PCIBIOS_FUNC_NOT_SUPPORTED;
 615
 616	return PCIBIOS_SUCCESSFUL;
 617}
 618
 619/*
 620 * Note access to the configuration registers are protected at the higher layer
 621 * by 'pci_lock' in drivers/pci/access.c
 622 */
 623static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie,
 624					    int busno, unsigned int devfn,
 625					    int where)
 626{
 627	u16 offset;
 628
 629	/* root complex access */
 630	if (busno == 0) {
 631		if (PCIE_ECAM_DEVFN(devfn) > 0)
 632			return NULL;
 633
 634		iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR,
 635				     where & CFG_IND_ADDR_MASK);
 636		offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA);
 637		if (iproc_pcie_reg_is_invalid(offset))
 638			return NULL;
 639		else
 640			return (pcie->base + offset);
 641	}
 642
 643	return iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where);
 644}
 645
 646static void __iomem *iproc_pcie_bus_map_cfg_bus(struct pci_bus *bus,
 647						unsigned int devfn,
 648						int where)
 649{
 650	return iproc_pcie_map_cfg_bus(iproc_data(bus), bus->number, devfn,
 651				      where);
 652}
 653
 654static int iproc_pci_raw_config_read32(struct iproc_pcie *pcie,
 655				       unsigned int devfn, int where,
 656				       int size, u32 *val)
 657{
 658	void __iomem *addr;
 659
 660	addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);
 661	if (!addr)
 662		return PCIBIOS_DEVICE_NOT_FOUND;
 663
 664	*val = readl(addr);
 665
 666	if (size <= 2)
 667		*val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
 668
 669	return PCIBIOS_SUCCESSFUL;
 670}
 671
 672static int iproc_pci_raw_config_write32(struct iproc_pcie *pcie,
 673					unsigned int devfn, int where,
 674					int size, u32 val)
 675{
 676	void __iomem *addr;
 677	u32 mask, tmp;
 678
 679	addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);
 680	if (!addr)
 681		return PCIBIOS_DEVICE_NOT_FOUND;
 682
 683	if (size == 4) {
 684		writel(val, addr);
 685		return PCIBIOS_SUCCESSFUL;
 686	}
 687
 688	mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
 689	tmp = readl(addr) & mask;
 690	tmp |= val << ((where & 0x3) * 8);
 691	writel(tmp, addr);
 692
 693	return PCIBIOS_SUCCESSFUL;
 694}
 695
 696static int iproc_pcie_config_read32(struct pci_bus *bus, unsigned int devfn,
 697				    int where, int size, u32 *val)
 698{
 699	int ret;
 700	struct iproc_pcie *pcie = iproc_data(bus);
 701
 702	iproc_pcie_apb_err_disable(bus, true);
 703	if (pcie->iproc_cfg_read)
 704		ret = iproc_pcie_config_read(bus, devfn, where, size, val);
 705	else
 706		ret = pci_generic_config_read32(bus, devfn, where, size, val);
 707	iproc_pcie_apb_err_disable(bus, false);
 708
 709	return ret;
 710}
 711
 712static int iproc_pcie_config_write32(struct pci_bus *bus, unsigned int devfn,
 713				     int where, int size, u32 val)
 714{
 715	int ret;
 716
 717	iproc_pcie_apb_err_disable(bus, true);
 718	ret = pci_generic_config_write32(bus, devfn, where, size, val);
 719	iproc_pcie_apb_err_disable(bus, false);
 720
 721	return ret;
 722}
 723
 724static struct pci_ops iproc_pcie_ops = {
 725	.map_bus = iproc_pcie_bus_map_cfg_bus,
 726	.read = iproc_pcie_config_read32,
 727	.write = iproc_pcie_config_write32,
 728};
 729
 730static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert)
 731{
 732	u32 val;
 733
 734	/*
 735	 * PAXC and the internal emulated endpoint device downstream should not
 736	 * be reset.  If firmware has been loaded on the endpoint device at an
 737	 * earlier boot stage, reset here causes issues.
 738	 */
 739	if (pcie->ep_is_internal)
 740		return;
 741
 742	if (assert) {
 743		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
 744		val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST &
 745			~RC_PCIE_RST_OUTPUT;
 746		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
 747		udelay(250);
 748	} else {
 749		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
 750		val |= RC_PCIE_RST_OUTPUT;
 751		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
 752		msleep(100);
 753	}
 754}
 755
 756int iproc_pcie_shutdown(struct iproc_pcie *pcie)
 757{
 758	iproc_pcie_perst_ctrl(pcie, true);
 759	msleep(500);
 760
 761	return 0;
 762}
 763EXPORT_SYMBOL_GPL(iproc_pcie_shutdown);
 764
 765static int iproc_pcie_check_link(struct iproc_pcie *pcie)
 766{
 767	struct device *dev = pcie->dev;
 768	u32 hdr_type, link_ctrl, link_status, class, val;
 769	bool link_is_active = false;
 770
 771	/*
 772	 * PAXC connects to emulated endpoint devices directly and does not
 773	 * have a Serdes.  Therefore skip the link detection logic here.
 774	 */
 775	if (pcie->ep_is_internal)
 776		return 0;
 777
 778	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS);
 779	if (!(val & PCIE_PHYLINKUP) || !(val & PCIE_DL_ACTIVE)) {
 780		dev_err(dev, "PHY or data link is INACTIVE!\n");
 781		return -ENODEV;
 782	}
 783
 784	/* make sure we are not in EP mode */
 785	iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type);
 786	if ((hdr_type & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_BRIDGE) {
 787		dev_err(dev, "in EP mode, hdr=%#02x\n", hdr_type);
 788		return -EFAULT;
 789	}
 790
 791	/* force class to PCI_CLASS_BRIDGE_PCI_NORMAL (0x060400) */
 792#define PCI_BRIDGE_CTRL_REG_OFFSET	0x43c
 793#define PCI_BRIDGE_CTRL_REG_CLASS_MASK	0xffffff
 794	iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,
 795				    4, &class);
 796	class &= ~PCI_BRIDGE_CTRL_REG_CLASS_MASK;
 797	class |= PCI_CLASS_BRIDGE_PCI_NORMAL;
 798	iproc_pci_raw_config_write32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,
 799				     4, class);
 800
 801	/* check link status to see if link is active */
 802	iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,
 803				    2, &link_status);
 804	if (link_status & PCI_EXP_LNKSTA_NLW)
 805		link_is_active = true;
 806
 807	if (!link_is_active) {
 808		/* try GEN 1 link speed */
 809#define PCI_TARGET_LINK_SPEED_MASK	0xf
 810#define PCI_TARGET_LINK_SPEED_GEN2	0x2
 811#define PCI_TARGET_LINK_SPEED_GEN1	0x1
 812		iproc_pci_raw_config_read32(pcie, 0,
 813					    IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2,
 814					    4, &link_ctrl);
 815		if ((link_ctrl & PCI_TARGET_LINK_SPEED_MASK) ==
 816		    PCI_TARGET_LINK_SPEED_GEN2) {
 817			link_ctrl &= ~PCI_TARGET_LINK_SPEED_MASK;
 818			link_ctrl |= PCI_TARGET_LINK_SPEED_GEN1;
 819			iproc_pci_raw_config_write32(pcie, 0,
 820					IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2,
 821					4, link_ctrl);
 822			msleep(100);
 823
 824			iproc_pci_raw_config_read32(pcie, 0,
 825					IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,
 826					2, &link_status);
 827			if (link_status & PCI_EXP_LNKSTA_NLW)
 828				link_is_active = true;
 829		}
 830	}
 831
 832	dev_info(dev, "link: %s\n", link_is_active ? "UP" : "DOWN");
 833
 834	return link_is_active ? 0 : -ENODEV;
 835}
 836
 837static void iproc_pcie_enable(struct iproc_pcie *pcie)
 838{
 839	iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK);
 840}
 841
 842static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie,
 843					  int window_idx)
 844{
 845	u32 val;
 846
 847	val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx));
 848
 849	return !!(val & OARR_VALID);
 850}
 851
 852static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx,
 853				      int size_idx, u64 axi_addr, u64 pci_addr)
 854{
 855	struct device *dev = pcie->dev;
 856	u16 oarr_offset, omap_offset;
 857
 858	/*
 859	 * Derive the OARR/OMAP offset from the first pair (OARR0/OMAP0) based
 860	 * on window index.
 861	 */
 862	oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0,
 863							  window_idx));
 864	omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0,
 865							  window_idx));
 866	if (iproc_pcie_reg_is_invalid(oarr_offset) ||
 867	    iproc_pcie_reg_is_invalid(omap_offset))
 868		return -EINVAL;
 869
 870	/*
 871	 * Program the OARR registers.  The upper 32-bit OARR register is
 872	 * always right after the lower 32-bit OARR register.
 873	 */
 874	writel(lower_32_bits(axi_addr) | (size_idx << OARR_SIZE_CFG_SHIFT) |
 875	       OARR_VALID, pcie->base + oarr_offset);
 876	writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4);
 877
 878	/* now program the OMAP registers */
 879	writel(lower_32_bits(pci_addr), pcie->base + omap_offset);
 880	writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4);
 881
 882	dev_dbg(dev, "ob window [%d]: offset 0x%x axi %pap pci %pap\n",
 883		window_idx, oarr_offset, &axi_addr, &pci_addr);
 884	dev_dbg(dev, "oarr lo 0x%x oarr hi 0x%x\n",
 885		readl(pcie->base + oarr_offset),
 886		readl(pcie->base + oarr_offset + 4));
 887	dev_dbg(dev, "omap lo 0x%x omap hi 0x%x\n",
 888		readl(pcie->base + omap_offset),
 889		readl(pcie->base + omap_offset + 4));
 890
 891	return 0;
 892}
 893
 894/*
 895 * Some iProc SoCs require the SW to configure the outbound address mapping
 896 *
 897 * Outbound address translation:
 898 *
 899 * iproc_pcie_address = axi_address - axi_offset
 900 * OARR = iproc_pcie_address
 901 * OMAP = pci_addr
 902 *
 903 * axi_addr -> iproc_pcie_address -> OARR -> OMAP -> pci_address
 904 */
 905static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr,
 906			       u64 pci_addr, resource_size_t size)
 907{
 908	struct iproc_pcie_ob *ob = &pcie->ob;
 909	struct device *dev = pcie->dev;
 910	int ret = -EINVAL, window_idx, size_idx;
 911
 912	if (axi_addr < ob->axi_offset) {
 913		dev_err(dev, "axi address %pap less than offset %pap\n",
 914			&axi_addr, &ob->axi_offset);
 915		return -EINVAL;
 916	}
 917
 918	/*
 919	 * Translate the AXI address to the internal address used by the iProc
 920	 * PCIe core before programming the OARR
 921	 */
 922	axi_addr -= ob->axi_offset;
 923
 924	/* iterate through all OARR/OMAP mapping windows */
 925	for (window_idx = ob->nr_windows - 1; window_idx >= 0; window_idx--) {
 926		const struct iproc_pcie_ob_map *ob_map =
 927			&pcie->ob_map[window_idx];
 928
 929		/*
 930		 * If current outbound window is already in use, move on to the
 931		 * next one.
 932		 */
 933		if (iproc_pcie_ob_is_valid(pcie, window_idx))
 934			continue;
 935
 936		/*
 937		 * Iterate through all supported window sizes within the
 938		 * OARR/OMAP pair to find a match.  Go through the window sizes
 939		 * in a descending order.
 940		 */
 941		for (size_idx = ob_map->nr_sizes - 1; size_idx >= 0;
 942		     size_idx--) {
 943			resource_size_t window_size =
 944				ob_map->window_sizes[size_idx] * SZ_1M;
 945
 946			/*
 947			 * Keep iterating until we reach the last window and
 948			 * with the minimal window size at index zero. In this
 949			 * case, we take a compromise by mapping it using the
 950			 * minimum window size that can be supported
 951			 */
 952			if (size < window_size) {
 953				if (size_idx > 0 || window_idx > 0)
 954					continue;
 955
 956				/*
 957				 * For the corner case of reaching the minimal
 958				 * window size that can be supported on the
 959				 * last window
 960				 */
 961				axi_addr = ALIGN_DOWN(axi_addr, window_size);
 962				pci_addr = ALIGN_DOWN(pci_addr, window_size);
 963				size = window_size;
 964			}
 965
 966			if (!IS_ALIGNED(axi_addr, window_size) ||
 967			    !IS_ALIGNED(pci_addr, window_size)) {
 968				dev_err(dev,
 969					"axi %pap or pci %pap not aligned\n",
 970					&axi_addr, &pci_addr);
 971				return -EINVAL;
 972			}
 973
 974			/*
 975			 * Match found!  Program both OARR and OMAP and mark
 976			 * them as a valid entry.
 977			 */
 978			ret = iproc_pcie_ob_write(pcie, window_idx, size_idx,
 979						  axi_addr, pci_addr);
 980			if (ret)
 981				goto err_ob;
 982
 983			size -= window_size;
 984			if (size == 0)
 985				return 0;
 986
 987			/*
 988			 * If we are here, we are done with the current window,
 989			 * but not yet finished all mappings.  Need to move on
 990			 * to the next window.
 991			 */
 992			axi_addr += window_size;
 993			pci_addr += window_size;
 994			break;
 995		}
 996	}
 997
 998err_ob:
 999	dev_err(dev, "unable to configure outbound mapping\n");
1000	dev_err(dev,
1001		"axi %pap, axi offset %pap, pci %pap, res size %pap\n",
1002		&axi_addr, &ob->axi_offset, &pci_addr, &size);
1003
1004	return ret;
1005}
1006
1007static int iproc_pcie_map_ranges(struct iproc_pcie *pcie,
1008				 struct list_head *resources)
1009{
1010	struct device *dev = pcie->dev;
1011	struct resource_entry *window;
1012	int ret;
1013
1014	resource_list_for_each_entry(window, resources) {
1015		struct resource *res = window->res;
1016		u64 res_type = resource_type(res);
1017
1018		switch (res_type) {
1019		case IORESOURCE_IO:
1020		case IORESOURCE_BUS:
1021			break;
1022		case IORESOURCE_MEM:
1023			ret = iproc_pcie_setup_ob(pcie, res->start,
1024						  res->start - window->offset,
1025						  resource_size(res));
1026			if (ret)
1027				return ret;
1028			break;
1029		default:
1030			dev_err(dev, "invalid resource %pR\n", res);
1031			return -EINVAL;
1032		}
1033	}
1034
1035	return 0;
1036}
1037
1038static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie,
1039					   int region_idx)
1040{
1041	const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];
1042	u32 val;
1043
1044	val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx));
1045
1046	return !!(val & (BIT(ib_map->nr_sizes) - 1));
1047}
1048
1049static inline bool iproc_pcie_ib_check_type(const struct iproc_pcie_ib_map *ib_map,
1050					    enum iproc_pcie_ib_map_type type)
1051{
1052	return !!(ib_map->type == type);
1053}
1054
1055static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx,
1056			       int size_idx, int nr_windows, u64 axi_addr,
1057			       u64 pci_addr, resource_size_t size)
1058{
1059	struct device *dev = pcie->dev;
1060	const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];
1061	u16 iarr_offset, imap_offset;
1062	u32 val;
1063	int window_idx;
1064
1065	iarr_offset = iproc_pcie_reg_offset(pcie,
1066				MAP_REG(IPROC_PCIE_IARR0, region_idx));
1067	imap_offset = iproc_pcie_reg_offset(pcie,
1068				MAP_REG(IPROC_PCIE_IMAP0, region_idx));
1069	if (iproc_pcie_reg_is_invalid(iarr_offset) ||
1070	    iproc_pcie_reg_is_invalid(imap_offset))
1071		return -EINVAL;
1072
1073	dev_dbg(dev, "ib region [%d]: offset 0x%x axi %pap pci %pap\n",
1074		region_idx, iarr_offset, &axi_addr, &pci_addr);
1075
1076	/*
1077	 * Program the IARR registers.  The upper 32-bit IARR register is
1078	 * always right after the lower 32-bit IARR register.
1079	 */
1080	writel(lower_32_bits(pci_addr) | BIT(size_idx),
1081	       pcie->base + iarr_offset);
1082	writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4);
1083
1084	dev_dbg(dev, "iarr lo 0x%x iarr hi 0x%x\n",
1085		readl(pcie->base + iarr_offset),
1086		readl(pcie->base + iarr_offset + 4));
1087
1088	/*
1089	 * Now program the IMAP registers.  Each IARR region may have one or
1090	 * more IMAP windows.
1091	 */
1092	size >>= ilog2(nr_windows);
1093	for (window_idx = 0; window_idx < nr_windows; window_idx++) {
1094		val = readl(pcie->base + imap_offset);
1095		val |= lower_32_bits(axi_addr) | IMAP_VALID;
1096		writel(val, pcie->base + imap_offset);
1097		writel(upper_32_bits(axi_addr),
1098		       pcie->base + imap_offset + ib_map->imap_addr_offset);
1099
1100		dev_dbg(dev, "imap window [%d] lo 0x%x hi 0x%x\n",
1101			window_idx, readl(pcie->base + imap_offset),
1102			readl(pcie->base + imap_offset +
1103			      ib_map->imap_addr_offset));
1104
1105		imap_offset += ib_map->imap_window_offset;
1106		axi_addr += size;
1107	}
1108
1109	return 0;
1110}
1111
1112static int iproc_pcie_setup_ib(struct iproc_pcie *pcie,
1113			       struct resource_entry *entry,
1114			       enum iproc_pcie_ib_map_type type)
1115{
1116	struct device *dev = pcie->dev;
1117	struct iproc_pcie_ib *ib = &pcie->ib;
1118	int ret;
1119	unsigned int region_idx, size_idx;
1120	u64 axi_addr = entry->res->start;
1121	u64 pci_addr = entry->res->start - entry->offset;
1122	resource_size_t size = resource_size(entry->res);
1123
1124	/* iterate through all IARR mapping regions */
1125	for (region_idx = 0; region_idx < ib->nr_regions; region_idx++) {
1126		const struct iproc_pcie_ib_map *ib_map =
1127			&pcie->ib_map[region_idx];
1128
1129		/*
1130		 * If current inbound region is already in use or not a
1131		 * compatible type, move on to the next.
1132		 */
1133		if (iproc_pcie_ib_is_in_use(pcie, region_idx) ||
1134		    !iproc_pcie_ib_check_type(ib_map, type))
1135			continue;
1136
1137		/* iterate through all supported region sizes to find a match */
1138		for (size_idx = 0; size_idx < ib_map->nr_sizes; size_idx++) {
1139			resource_size_t region_size =
1140			ib_map->region_sizes[size_idx] * ib_map->size_unit;
1141
1142			if (size != region_size)
1143				continue;
1144
1145			if (!IS_ALIGNED(axi_addr, region_size) ||
1146			    !IS_ALIGNED(pci_addr, region_size)) {
1147				dev_err(dev,
1148					"axi %pap or pci %pap not aligned\n",
1149					&axi_addr, &pci_addr);
1150				return -EINVAL;
1151			}
1152
1153			/* Match found!  Program IARR and all IMAP windows. */
1154			ret = iproc_pcie_ib_write(pcie, region_idx, size_idx,
1155						  ib_map->nr_windows, axi_addr,
1156						  pci_addr, size);
1157			if (ret)
1158				goto err_ib;
1159			else
1160				return 0;
1161
1162		}
1163	}
1164	ret = -EINVAL;
1165
1166err_ib:
1167	dev_err(dev, "unable to configure inbound mapping\n");
1168	dev_err(dev, "axi %pap, pci %pap, res size %pap\n",
1169		&axi_addr, &pci_addr, &size);
1170
1171	return ret;
1172}
1173
1174static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie)
1175{
1176	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
1177	struct resource_entry *entry;
1178	int ret = 0;
1179
1180	resource_list_for_each_entry(entry, &host->dma_ranges) {
1181		/* Each range entry corresponds to an inbound mapping region */
1182		ret = iproc_pcie_setup_ib(pcie, entry, IPROC_PCIE_IB_MAP_MEM);
1183		if (ret)
1184			break;
1185	}
1186
1187	return ret;
1188}
1189
1190static void iproc_pcie_invalidate_mapping(struct iproc_pcie *pcie)
1191{
1192	struct iproc_pcie_ib *ib = &pcie->ib;
1193	struct iproc_pcie_ob *ob = &pcie->ob;
1194	int idx;
1195
1196	if (pcie->ep_is_internal)
1197		return;
1198
1199	if (pcie->need_ob_cfg) {
1200		/* iterate through all OARR mapping regions */
1201		for (idx = ob->nr_windows - 1; idx >= 0; idx--) {
1202			iproc_pcie_write_reg(pcie,
1203					     MAP_REG(IPROC_PCIE_OARR0, idx), 0);
1204		}
1205	}
1206
1207	if (pcie->need_ib_cfg) {
1208		/* iterate through all IARR mapping regions */
1209		for (idx = 0; idx < ib->nr_regions; idx++) {
1210			iproc_pcie_write_reg(pcie,
1211					     MAP_REG(IPROC_PCIE_IARR0, idx), 0);
1212		}
1213	}
1214}
1215
1216static int iproce_pcie_get_msi(struct iproc_pcie *pcie,
1217			       struct device_node *msi_node,
1218			       u64 *msi_addr)
1219{
1220	struct device *dev = pcie->dev;
1221	int ret;
1222	struct resource res;
1223
1224	/*
1225	 * Check if 'msi-map' points to ARM GICv3 ITS, which is the only
1226	 * supported external MSI controller that requires steering.
1227	 */
1228	if (!of_device_is_compatible(msi_node, "arm,gic-v3-its")) {
1229		dev_err(dev, "unable to find compatible MSI controller\n");
1230		return -ENODEV;
1231	}
1232
1233	/* derive GITS_TRANSLATER address from GICv3 */
1234	ret = of_address_to_resource(msi_node, 0, &res);
1235	if (ret < 0) {
1236		dev_err(dev, "unable to obtain MSI controller resources\n");
1237		return ret;
1238	}
1239
1240	*msi_addr = res.start + GITS_TRANSLATER;
1241	return 0;
1242}
1243
1244static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr)
1245{
1246	int ret;
1247	struct resource_entry entry;
1248
1249	memset(&entry, 0, sizeof(entry));
1250	entry.res = &entry.__res;
1251
1252	msi_addr &= ~(SZ_32K - 1);
1253	entry.res->start = msi_addr;
1254	entry.res->end = msi_addr + SZ_32K - 1;
1255
1256	ret = iproc_pcie_setup_ib(pcie, &entry, IPROC_PCIE_IB_MAP_IO);
1257	return ret;
1258}
1259
1260static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr,
1261					 bool enable)
1262{
1263	u32 val;
1264
1265	if (!enable) {
1266		/*
1267		 * Disable PAXC MSI steering. All write transfers will be
1268		 * treated as non-MSI transfers
1269		 */
1270		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);
1271		val &= ~MSI_ENABLE_CFG;
1272		iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);
1273		return;
1274	}
1275
1276	/*
1277	 * Program bits [43:13] of address of GITS_TRANSLATER register into
1278	 * bits [30:0] of the MSI base address register.  In fact, in all iProc
1279	 * based SoCs, all I/O register bases are well below the 32-bit
1280	 * boundary, so we can safely assume bits [43:32] are always zeros.
1281	 */
1282	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR,
1283			     (u32)(msi_addr >> 13));
1284
1285	/* use a default 8K window size */
1286	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0);
1287
1288	/* steering MSI to GICv3 ITS */
1289	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE);
1290	val |= GIC_V3_CFG;
1291	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val);
1292
1293	/*
1294	 * Program bits [43:2] of address of GITS_TRANSLATER register into the
1295	 * iProc MSI address registers.
1296	 */
1297	msi_addr >>= 2;
1298	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI,
1299			     upper_32_bits(msi_addr));
1300	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO,
1301			     lower_32_bits(msi_addr));
1302
1303	/* enable MSI */
1304	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);
1305	val |= MSI_ENABLE_CFG;
1306	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);
1307}
1308
1309static int iproc_pcie_msi_steer(struct iproc_pcie *pcie,
1310				struct device_node *msi_node)
1311{
1312	struct device *dev = pcie->dev;
1313	int ret;
1314	u64 msi_addr;
1315
1316	ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr);
1317	if (ret < 0) {
1318		dev_err(dev, "msi steering failed\n");
1319		return ret;
1320	}
1321
1322	switch (pcie->type) {
1323	case IPROC_PCIE_PAXB_V2:
1324		ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr);
1325		if (ret)
1326			return ret;
1327		break;
1328	case IPROC_PCIE_PAXC_V2:
1329		iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr, true);
1330		break;
1331	default:
1332		return -EINVAL;
1333	}
1334
1335	return 0;
1336}
1337
1338static int iproc_pcie_msi_enable(struct iproc_pcie *pcie)
1339{
1340	struct device_node *msi_node;
1341	int ret;
1342
1343	/*
1344	 * Either the "msi-parent" or the "msi-map" phandle needs to exist
1345	 * for us to obtain the MSI node.
1346	 */
1347
1348	msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0);
1349	if (!msi_node) {
1350		const __be32 *msi_map = NULL;
1351		int len;
1352		u32 phandle;
1353
1354		msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len);
1355		if (!msi_map)
1356			return -ENODEV;
1357
1358		phandle = be32_to_cpup(msi_map + 1);
1359		msi_node = of_find_node_by_phandle(phandle);
1360		if (!msi_node)
1361			return -ENODEV;
1362	}
1363
1364	/*
1365	 * Certain revisions of the iProc PCIe controller require additional
1366	 * configurations to steer the MSI writes towards an external MSI
1367	 * controller.
1368	 */
1369	if (pcie->need_msi_steer) {
1370		ret = iproc_pcie_msi_steer(pcie, msi_node);
1371		if (ret)
1372			goto out_put_node;
1373	}
1374
1375	/*
1376	 * If another MSI controller is being used, the call below should fail
1377	 * but that is okay
1378	 */
1379	ret = iproc_msi_init(pcie, msi_node);
1380
1381out_put_node:
1382	of_node_put(msi_node);
1383	return ret;
1384}
1385
1386static void iproc_pcie_msi_disable(struct iproc_pcie *pcie)
1387{
1388	iproc_msi_exit(pcie);
1389}
1390
1391static int iproc_pcie_rev_init(struct iproc_pcie *pcie)
1392{
1393	struct device *dev = pcie->dev;
1394	unsigned int reg_idx;
1395	const u16 *regs;
1396
1397	switch (pcie->type) {
1398	case IPROC_PCIE_PAXB_BCMA:
1399		regs = iproc_pcie_reg_paxb_bcma;
1400		break;
1401	case IPROC_PCIE_PAXB:
1402		regs = iproc_pcie_reg_paxb;
1403		pcie->has_apb_err_disable = true;
1404		if (pcie->need_ob_cfg) {
1405			pcie->ob_map = paxb_ob_map;
1406			pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map);
1407		}
1408		break;
1409	case IPROC_PCIE_PAXB_V2:
1410		regs = iproc_pcie_reg_paxb_v2;
1411		pcie->iproc_cfg_read = true;
1412		pcie->has_apb_err_disable = true;
1413		if (pcie->need_ob_cfg) {
1414			pcie->ob_map = paxb_v2_ob_map;
1415			pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map);
1416		}
1417		pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map);
1418		pcie->ib_map = paxb_v2_ib_map;
1419		pcie->need_msi_steer = true;
1420		dev_warn(dev, "reads of config registers that contain %#x return incorrect data\n",
1421			 CFG_RETRY_STATUS);
1422		break;
1423	case IPROC_PCIE_PAXC:
1424		regs = iproc_pcie_reg_paxc;
1425		pcie->ep_is_internal = true;
1426		pcie->iproc_cfg_read = true;
1427		pcie->rej_unconfig_pf = true;
1428		break;
1429	case IPROC_PCIE_PAXC_V2:
1430		regs = iproc_pcie_reg_paxc_v2;
1431		pcie->ep_is_internal = true;
1432		pcie->iproc_cfg_read = true;
1433		pcie->rej_unconfig_pf = true;
1434		pcie->need_msi_steer = true;
1435		break;
1436	default:
1437		dev_err(dev, "incompatible iProc PCIe interface\n");
1438		return -EINVAL;
1439	}
1440
1441	pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG,
1442					 sizeof(*pcie->reg_offsets),
1443					 GFP_KERNEL);
1444	if (!pcie->reg_offsets)
1445		return -ENOMEM;
1446
1447	/* go through the register table and populate all valid registers */
1448	pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ?
1449		IPROC_PCIE_REG_INVALID : regs[0];
1450	for (reg_idx = 1; reg_idx < IPROC_PCIE_MAX_NUM_REG; reg_idx++)
1451		pcie->reg_offsets[reg_idx] = regs[reg_idx] ?
1452			regs[reg_idx] : IPROC_PCIE_REG_INVALID;
1453
1454	return 0;
1455}
1456
1457int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
1458{
1459	struct device *dev;
1460	int ret;
1461	struct pci_dev *pdev;
1462	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
1463
1464	dev = pcie->dev;
1465
1466	ret = iproc_pcie_rev_init(pcie);
1467	if (ret) {
1468		dev_err(dev, "unable to initialize controller parameters\n");
1469		return ret;
1470	}
1471
1472	ret = phy_init(pcie->phy);
1473	if (ret) {
1474		dev_err(dev, "unable to initialize PCIe PHY\n");
1475		return ret;
1476	}
1477
1478	ret = phy_power_on(pcie->phy);
1479	if (ret) {
1480		dev_err(dev, "unable to power on PCIe PHY\n");
1481		goto err_exit_phy;
1482	}
1483
1484	iproc_pcie_perst_ctrl(pcie, true);
1485	iproc_pcie_perst_ctrl(pcie, false);
1486
1487	iproc_pcie_invalidate_mapping(pcie);
1488
1489	if (pcie->need_ob_cfg) {
1490		ret = iproc_pcie_map_ranges(pcie, res);
1491		if (ret) {
1492			dev_err(dev, "map failed\n");
1493			goto err_power_off_phy;
1494		}
1495	}
1496
1497	if (pcie->need_ib_cfg) {
1498		ret = iproc_pcie_map_dma_ranges(pcie);
1499		if (ret && ret != -ENOENT)
1500			goto err_power_off_phy;
1501	}
1502
1503	ret = iproc_pcie_check_link(pcie);
1504	if (ret) {
1505		dev_err(dev, "no PCIe EP device detected\n");
1506		goto err_power_off_phy;
1507	}
1508
1509	iproc_pcie_enable(pcie);
1510
1511	if (IS_ENABLED(CONFIG_PCI_MSI))
1512		if (iproc_pcie_msi_enable(pcie))
1513			dev_info(dev, "not using iProc MSI\n");
1514
1515	host->ops = &iproc_pcie_ops;
1516	host->sysdata = pcie;
1517	host->map_irq = pcie->map_irq;
1518
1519	ret = pci_host_probe(host);
1520	if (ret < 0) {
1521		dev_err(dev, "failed to scan host: %d\n", ret);
1522		goto err_power_off_phy;
1523	}
1524
1525	for_each_pci_bridge(pdev, host->bus) {
1526		if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT)
1527			pcie_print_link_status(pdev);
1528	}
1529
1530	return 0;
1531
1532err_power_off_phy:
1533	phy_power_off(pcie->phy);
1534err_exit_phy:
1535	phy_exit(pcie->phy);
1536	return ret;
1537}
1538EXPORT_SYMBOL(iproc_pcie_setup);
1539
1540void iproc_pcie_remove(struct iproc_pcie *pcie)
1541{
1542	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
1543
1544	pci_stop_root_bus(host->bus);
1545	pci_remove_root_bus(host->bus);
1546
1547	iproc_pcie_msi_disable(pcie);
1548
1549	phy_power_off(pcie->phy);
1550	phy_exit(pcie->phy);
 
 
1551}
1552EXPORT_SYMBOL(iproc_pcie_remove);
1553
1554/*
1555 * The MSI parsing logic in certain revisions of Broadcom PAXC based root
1556 * complex does not work and needs to be disabled
1557 */
1558static void quirk_paxc_disable_msi_parsing(struct pci_dev *pdev)
1559{
1560	struct iproc_pcie *pcie = iproc_data(pdev->bus);
1561
1562	if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
1563		iproc_pcie_paxc_v2_msi_steer(pcie, 0, false);
1564}
1565DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0,
1566			quirk_paxc_disable_msi_parsing);
1567DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802,
1568			quirk_paxc_disable_msi_parsing);
1569DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804,
1570			quirk_paxc_disable_msi_parsing);
1571
1572static void quirk_paxc_bridge(struct pci_dev *pdev)
1573{
1574	/*
1575	 * The PCI config space is shared with the PAXC root port and the first
1576	 * Ethernet device.  So, we need to workaround this by telling the PCI
1577	 * code that the bridge is not an Ethernet device.
1578	 */
1579	if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
1580		pdev->class = PCI_CLASS_BRIDGE_PCI_NORMAL;
1581
1582	/*
1583	 * MPSS is not being set properly (as it is currently 0).  This is
1584	 * because that area of the PCI config space is hard coded to zero, and
1585	 * is not modifiable by firmware.  Set this to 2 (e.g., 512 byte MPS)
1586	 * so that the MPS can be set to the real max value.
1587	 */
1588	pdev->pcie_mpss = 2;
1589}
1590DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16cd, quirk_paxc_bridge);
1591DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, quirk_paxc_bridge);
1592DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd750, quirk_paxc_bridge);
1593DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, quirk_paxc_bridge);
1594DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, quirk_paxc_bridge);
1595
1596MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>");
1597MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver");
1598MODULE_LICENSE("GPL v2");