Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Driver for the Aardvark PCIe controller, used on Marvell Armada
4 * 3700.
5 *
6 * Copyright (C) 2016 Marvell
7 *
8 * Author: Hezi Shahmoon <hezi.shahmoon@marvell.com>
9 */
10
11#include <linux/bitfield.h>
12#include <linux/delay.h>
13#include <linux/gpio/consumer.h>
14#include <linux/interrupt.h>
15#include <linux/irq.h>
16#include <linux/irqdomain.h>
17#include <linux/kernel.h>
18#include <linux/module.h>
19#include <linux/pci.h>
20#include <linux/pci-ecam.h>
21#include <linux/init.h>
22#include <linux/phy/phy.h>
23#include <linux/platform_device.h>
24#include <linux/msi.h>
25#include <linux/of_address.h>
26#include <linux/of_gpio.h>
27#include <linux/of_pci.h>
28
29#include "../pci.h"
30#include "../pci-bridge-emul.h"
31
32/* PCIe core registers */
33#define PCIE_CORE_DEV_ID_REG 0x0
34#define PCIE_CORE_CMD_STATUS_REG 0x4
35#define PCIE_CORE_DEV_REV_REG 0x8
36#define PCIE_CORE_SSDEV_ID_REG 0x2c
37#define PCIE_CORE_PCIEXP_CAP 0xc0
38#define PCIE_CORE_PCIERR_CAP 0x100
39#define PCIE_CORE_ERR_CAPCTL_REG 0x118
40#define PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX BIT(5)
41#define PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN BIT(6)
42#define PCIE_CORE_ERR_CAPCTL_ECRC_CHCK BIT(7)
43#define PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV BIT(8)
44/* PIO registers base address and register offsets */
45#define PIO_BASE_ADDR 0x4000
46#define PIO_CTRL (PIO_BASE_ADDR + 0x0)
47#define PIO_CTRL_TYPE_MASK GENMASK(3, 0)
48#define PIO_CTRL_ADDR_WIN_DISABLE BIT(24)
49#define PIO_STAT (PIO_BASE_ADDR + 0x4)
50#define PIO_COMPLETION_STATUS_SHIFT 7
51#define PIO_COMPLETION_STATUS_MASK GENMASK(9, 7)
52#define PIO_COMPLETION_STATUS_OK 0
53#define PIO_COMPLETION_STATUS_UR 1
54#define PIO_COMPLETION_STATUS_CRS 2
55#define PIO_COMPLETION_STATUS_CA 4
56#define PIO_NON_POSTED_REQ BIT(10)
57#define PIO_ERR_STATUS BIT(11)
58#define PIO_ADDR_LS (PIO_BASE_ADDR + 0x8)
59#define PIO_ADDR_MS (PIO_BASE_ADDR + 0xc)
60#define PIO_WR_DATA (PIO_BASE_ADDR + 0x10)
61#define PIO_WR_DATA_STRB (PIO_BASE_ADDR + 0x14)
62#define PIO_RD_DATA (PIO_BASE_ADDR + 0x18)
63#define PIO_START (PIO_BASE_ADDR + 0x1c)
64#define PIO_ISR (PIO_BASE_ADDR + 0x20)
65#define PIO_ISRM (PIO_BASE_ADDR + 0x24)
66
67/* Aardvark Control registers */
68#define CONTROL_BASE_ADDR 0x4800
69#define PCIE_CORE_CTRL0_REG (CONTROL_BASE_ADDR + 0x0)
70#define PCIE_GEN_SEL_MSK 0x3
71#define PCIE_GEN_SEL_SHIFT 0x0
72#define SPEED_GEN_1 0
73#define SPEED_GEN_2 1
74#define SPEED_GEN_3 2
75#define IS_RC_MSK 1
76#define IS_RC_SHIFT 2
77#define LANE_CNT_MSK 0x18
78#define LANE_CNT_SHIFT 0x3
79#define LANE_COUNT_1 (0 << LANE_CNT_SHIFT)
80#define LANE_COUNT_2 (1 << LANE_CNT_SHIFT)
81#define LANE_COUNT_4 (2 << LANE_CNT_SHIFT)
82#define LANE_COUNT_8 (3 << LANE_CNT_SHIFT)
83#define LINK_TRAINING_EN BIT(6)
84#define LEGACY_INTA BIT(28)
85#define LEGACY_INTB BIT(29)
86#define LEGACY_INTC BIT(30)
87#define LEGACY_INTD BIT(31)
88#define PCIE_CORE_CTRL1_REG (CONTROL_BASE_ADDR + 0x4)
89#define HOT_RESET_GEN BIT(0)
90#define PCIE_CORE_CTRL2_REG (CONTROL_BASE_ADDR + 0x8)
91#define PCIE_CORE_CTRL2_RESERVED 0x7
92#define PCIE_CORE_CTRL2_TD_ENABLE BIT(4)
93#define PCIE_CORE_CTRL2_STRICT_ORDER_ENABLE BIT(5)
94#define PCIE_CORE_CTRL2_OB_WIN_ENABLE BIT(6)
95#define PCIE_CORE_CTRL2_MSI_ENABLE BIT(10)
96#define PCIE_CORE_REF_CLK_REG (CONTROL_BASE_ADDR + 0x14)
97#define PCIE_CORE_REF_CLK_TX_ENABLE BIT(1)
98#define PCIE_CORE_REF_CLK_RX_ENABLE BIT(2)
99#define PCIE_MSG_LOG_REG (CONTROL_BASE_ADDR + 0x30)
100#define PCIE_ISR0_REG (CONTROL_BASE_ADDR + 0x40)
101#define PCIE_MSG_PM_PME_MASK BIT(7)
102#define PCIE_ISR0_MASK_REG (CONTROL_BASE_ADDR + 0x44)
103#define PCIE_ISR0_MSI_INT_PENDING BIT(24)
104#define PCIE_ISR0_CORR_ERR BIT(11)
105#define PCIE_ISR0_NFAT_ERR BIT(12)
106#define PCIE_ISR0_FAT_ERR BIT(13)
107#define PCIE_ISR0_ERR_MASK GENMASK(13, 11)
108#define PCIE_ISR0_INTX_ASSERT(val) BIT(16 + (val))
109#define PCIE_ISR0_INTX_DEASSERT(val) BIT(20 + (val))
110#define PCIE_ISR0_ALL_MASK GENMASK(31, 0)
111#define PCIE_ISR1_REG (CONTROL_BASE_ADDR + 0x48)
112#define PCIE_ISR1_MASK_REG (CONTROL_BASE_ADDR + 0x4C)
113#define PCIE_ISR1_POWER_STATE_CHANGE BIT(4)
114#define PCIE_ISR1_FLUSH BIT(5)
115#define PCIE_ISR1_INTX_ASSERT(val) BIT(8 + (val))
116#define PCIE_ISR1_ALL_MASK GENMASK(31, 0)
117#define PCIE_MSI_ADDR_LOW_REG (CONTROL_BASE_ADDR + 0x50)
118#define PCIE_MSI_ADDR_HIGH_REG (CONTROL_BASE_ADDR + 0x54)
119#define PCIE_MSI_STATUS_REG (CONTROL_BASE_ADDR + 0x58)
120#define PCIE_MSI_MASK_REG (CONTROL_BASE_ADDR + 0x5C)
121#define PCIE_MSI_ALL_MASK GENMASK(31, 0)
122#define PCIE_MSI_PAYLOAD_REG (CONTROL_BASE_ADDR + 0x9C)
123#define PCIE_MSI_DATA_MASK GENMASK(15, 0)
124
125/* PCIe window configuration */
126#define OB_WIN_BASE_ADDR 0x4c00
127#define OB_WIN_BLOCK_SIZE 0x20
128#define OB_WIN_COUNT 8
129#define OB_WIN_REG_ADDR(win, offset) (OB_WIN_BASE_ADDR + \
130 OB_WIN_BLOCK_SIZE * (win) + \
131 (offset))
132#define OB_WIN_MATCH_LS(win) OB_WIN_REG_ADDR(win, 0x00)
133#define OB_WIN_ENABLE BIT(0)
134#define OB_WIN_MATCH_MS(win) OB_WIN_REG_ADDR(win, 0x04)
135#define OB_WIN_REMAP_LS(win) OB_WIN_REG_ADDR(win, 0x08)
136#define OB_WIN_REMAP_MS(win) OB_WIN_REG_ADDR(win, 0x0c)
137#define OB_WIN_MASK_LS(win) OB_WIN_REG_ADDR(win, 0x10)
138#define OB_WIN_MASK_MS(win) OB_WIN_REG_ADDR(win, 0x14)
139#define OB_WIN_ACTIONS(win) OB_WIN_REG_ADDR(win, 0x18)
140#define OB_WIN_DEFAULT_ACTIONS (OB_WIN_ACTIONS(OB_WIN_COUNT-1) + 0x4)
141#define OB_WIN_FUNC_NUM_MASK GENMASK(31, 24)
142#define OB_WIN_FUNC_NUM_SHIFT 24
143#define OB_WIN_FUNC_NUM_ENABLE BIT(23)
144#define OB_WIN_BUS_NUM_BITS_MASK GENMASK(22, 20)
145#define OB_WIN_BUS_NUM_BITS_SHIFT 20
146#define OB_WIN_MSG_CODE_ENABLE BIT(22)
147#define OB_WIN_MSG_CODE_MASK GENMASK(21, 14)
148#define OB_WIN_MSG_CODE_SHIFT 14
149#define OB_WIN_MSG_PAYLOAD_LEN BIT(12)
150#define OB_WIN_ATTR_ENABLE BIT(11)
151#define OB_WIN_ATTR_TC_MASK GENMASK(10, 8)
152#define OB_WIN_ATTR_TC_SHIFT 8
153#define OB_WIN_ATTR_RELAXED BIT(7)
154#define OB_WIN_ATTR_NOSNOOP BIT(6)
155#define OB_WIN_ATTR_POISON BIT(5)
156#define OB_WIN_ATTR_IDO BIT(4)
157#define OB_WIN_TYPE_MASK GENMASK(3, 0)
158#define OB_WIN_TYPE_SHIFT 0
159#define OB_WIN_TYPE_MEM 0x0
160#define OB_WIN_TYPE_IO 0x4
161#define OB_WIN_TYPE_CONFIG_TYPE0 0x8
162#define OB_WIN_TYPE_CONFIG_TYPE1 0x9
163#define OB_WIN_TYPE_MSG 0xc
164
165/* LMI registers base address and register offsets */
166#define LMI_BASE_ADDR 0x6000
167#define CFG_REG (LMI_BASE_ADDR + 0x0)
168#define LTSSM_SHIFT 24
169#define LTSSM_MASK 0x3f
170#define RC_BAR_CONFIG 0x300
171
172/* LTSSM values in CFG_REG */
173enum {
174 LTSSM_DETECT_QUIET = 0x0,
175 LTSSM_DETECT_ACTIVE = 0x1,
176 LTSSM_POLLING_ACTIVE = 0x2,
177 LTSSM_POLLING_COMPLIANCE = 0x3,
178 LTSSM_POLLING_CONFIGURATION = 0x4,
179 LTSSM_CONFIG_LINKWIDTH_START = 0x5,
180 LTSSM_CONFIG_LINKWIDTH_ACCEPT = 0x6,
181 LTSSM_CONFIG_LANENUM_ACCEPT = 0x7,
182 LTSSM_CONFIG_LANENUM_WAIT = 0x8,
183 LTSSM_CONFIG_COMPLETE = 0x9,
184 LTSSM_CONFIG_IDLE = 0xa,
185 LTSSM_RECOVERY_RCVR_LOCK = 0xb,
186 LTSSM_RECOVERY_SPEED = 0xc,
187 LTSSM_RECOVERY_RCVR_CFG = 0xd,
188 LTSSM_RECOVERY_IDLE = 0xe,
189 LTSSM_L0 = 0x10,
190 LTSSM_RX_L0S_ENTRY = 0x11,
191 LTSSM_RX_L0S_IDLE = 0x12,
192 LTSSM_RX_L0S_FTS = 0x13,
193 LTSSM_TX_L0S_ENTRY = 0x14,
194 LTSSM_TX_L0S_IDLE = 0x15,
195 LTSSM_TX_L0S_FTS = 0x16,
196 LTSSM_L1_ENTRY = 0x17,
197 LTSSM_L1_IDLE = 0x18,
198 LTSSM_L2_IDLE = 0x19,
199 LTSSM_L2_TRANSMIT_WAKE = 0x1a,
200 LTSSM_DISABLED = 0x20,
201 LTSSM_LOOPBACK_ENTRY_MASTER = 0x21,
202 LTSSM_LOOPBACK_ACTIVE_MASTER = 0x22,
203 LTSSM_LOOPBACK_EXIT_MASTER = 0x23,
204 LTSSM_LOOPBACK_ENTRY_SLAVE = 0x24,
205 LTSSM_LOOPBACK_ACTIVE_SLAVE = 0x25,
206 LTSSM_LOOPBACK_EXIT_SLAVE = 0x26,
207 LTSSM_HOT_RESET = 0x27,
208 LTSSM_RECOVERY_EQUALIZATION_PHASE0 = 0x28,
209 LTSSM_RECOVERY_EQUALIZATION_PHASE1 = 0x29,
210 LTSSM_RECOVERY_EQUALIZATION_PHASE2 = 0x2a,
211 LTSSM_RECOVERY_EQUALIZATION_PHASE3 = 0x2b,
212};
213
214#define VENDOR_ID_REG (LMI_BASE_ADDR + 0x44)
215
216/* PCIe core controller registers */
217#define CTRL_CORE_BASE_ADDR 0x18000
218#define CTRL_CONFIG_REG (CTRL_CORE_BASE_ADDR + 0x0)
219#define CTRL_MODE_SHIFT 0x0
220#define CTRL_MODE_MASK 0x1
221#define PCIE_CORE_MODE_DIRECT 0x0
222#define PCIE_CORE_MODE_COMMAND 0x1
223
224/* PCIe Central Interrupts Registers */
225#define CENTRAL_INT_BASE_ADDR 0x1b000
226#define HOST_CTRL_INT_STATUS_REG (CENTRAL_INT_BASE_ADDR + 0x0)
227#define HOST_CTRL_INT_MASK_REG (CENTRAL_INT_BASE_ADDR + 0x4)
228#define PCIE_IRQ_CMDQ_INT BIT(0)
229#define PCIE_IRQ_MSI_STATUS_INT BIT(1)
230#define PCIE_IRQ_CMD_SENT_DONE BIT(3)
231#define PCIE_IRQ_DMA_INT BIT(4)
232#define PCIE_IRQ_IB_DXFERDONE BIT(5)
233#define PCIE_IRQ_OB_DXFERDONE BIT(6)
234#define PCIE_IRQ_OB_RXFERDONE BIT(7)
235#define PCIE_IRQ_COMPQ_INT BIT(12)
236#define PCIE_IRQ_DIR_RD_DDR_DET BIT(13)
237#define PCIE_IRQ_DIR_WR_DDR_DET BIT(14)
238#define PCIE_IRQ_CORE_INT BIT(16)
239#define PCIE_IRQ_CORE_INT_PIO BIT(17)
240#define PCIE_IRQ_DPMU_INT BIT(18)
241#define PCIE_IRQ_PCIE_MIS_INT BIT(19)
242#define PCIE_IRQ_MSI_INT1_DET BIT(20)
243#define PCIE_IRQ_MSI_INT2_DET BIT(21)
244#define PCIE_IRQ_RC_DBELL_DET BIT(22)
245#define PCIE_IRQ_EP_STATUS BIT(23)
246#define PCIE_IRQ_ALL_MASK GENMASK(31, 0)
247#define PCIE_IRQ_ENABLE_INTS_MASK PCIE_IRQ_CORE_INT
248
249/* Transaction types */
250#define PCIE_CONFIG_RD_TYPE0 0x8
251#define PCIE_CONFIG_RD_TYPE1 0x9
252#define PCIE_CONFIG_WR_TYPE0 0xa
253#define PCIE_CONFIG_WR_TYPE1 0xb
254
255#define PIO_RETRY_CNT 750000 /* 1.5 s */
256#define PIO_RETRY_DELAY 2 /* 2 us*/
257
258#define LINK_WAIT_MAX_RETRIES 10
259#define LINK_WAIT_USLEEP_MIN 90000
260#define LINK_WAIT_USLEEP_MAX 100000
261#define RETRAIN_WAIT_MAX_RETRIES 10
262#define RETRAIN_WAIT_USLEEP_US 2000
263
264#define MSI_IRQ_NUM 32
265
266#define CFG_RD_CRS_VAL 0xffff0001
267
268struct advk_pcie {
269 struct platform_device *pdev;
270 void __iomem *base;
271 struct {
272 phys_addr_t match;
273 phys_addr_t remap;
274 phys_addr_t mask;
275 u32 actions;
276 } wins[OB_WIN_COUNT];
277 u8 wins_count;
278 struct irq_domain *rp_irq_domain;
279 struct irq_domain *irq_domain;
280 struct irq_chip irq_chip;
281 raw_spinlock_t irq_lock;
282 struct irq_domain *msi_domain;
283 struct irq_domain *msi_inner_domain;
284 raw_spinlock_t msi_irq_lock;
285 DECLARE_BITMAP(msi_used, MSI_IRQ_NUM);
286 struct mutex msi_used_lock;
287 int link_gen;
288 struct pci_bridge_emul bridge;
289 struct gpio_desc *reset_gpio;
290 struct phy *phy;
291};
292
293static inline void advk_writel(struct advk_pcie *pcie, u32 val, u64 reg)
294{
295 writel(val, pcie->base + reg);
296}
297
298static inline u32 advk_readl(struct advk_pcie *pcie, u64 reg)
299{
300 return readl(pcie->base + reg);
301}
302
303static u8 advk_pcie_ltssm_state(struct advk_pcie *pcie)
304{
305 u32 val;
306 u8 ltssm_state;
307
308 val = advk_readl(pcie, CFG_REG);
309 ltssm_state = (val >> LTSSM_SHIFT) & LTSSM_MASK;
310 return ltssm_state;
311}
312
313static inline bool advk_pcie_link_up(struct advk_pcie *pcie)
314{
315 /* check if LTSSM is in normal operation - some L* state */
316 u8 ltssm_state = advk_pcie_ltssm_state(pcie);
317 return ltssm_state >= LTSSM_L0 && ltssm_state < LTSSM_DISABLED;
318}
319
320static inline bool advk_pcie_link_active(struct advk_pcie *pcie)
321{
322 /*
323 * According to PCIe Base specification 3.0, Table 4-14: Link
324 * Status Mapped to the LTSSM, and 4.2.6.3.6 Configuration.Idle
325 * is Link Up mapped to LTSSM Configuration.Idle, Recovery, L0,
326 * L0s, L1 and L2 states. And according to 3.2.1. Data Link
327 * Control and Management State Machine Rules is DL Up status
328 * reported in DL Active state.
329 */
330 u8 ltssm_state = advk_pcie_ltssm_state(pcie);
331 return ltssm_state >= LTSSM_CONFIG_IDLE && ltssm_state < LTSSM_DISABLED;
332}
333
334static inline bool advk_pcie_link_training(struct advk_pcie *pcie)
335{
336 /*
337 * According to PCIe Base specification 3.0, Table 4-14: Link
338 * Status Mapped to the LTSSM is Link Training mapped to LTSSM
339 * Configuration and Recovery states.
340 */
341 u8 ltssm_state = advk_pcie_ltssm_state(pcie);
342 return ((ltssm_state >= LTSSM_CONFIG_LINKWIDTH_START &&
343 ltssm_state < LTSSM_L0) ||
344 (ltssm_state >= LTSSM_RECOVERY_EQUALIZATION_PHASE0 &&
345 ltssm_state <= LTSSM_RECOVERY_EQUALIZATION_PHASE3));
346}
347
348static int advk_pcie_wait_for_link(struct advk_pcie *pcie)
349{
350 int retries;
351
352 /* check if the link is up or not */
353 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
354 if (advk_pcie_link_up(pcie))
355 return 0;
356
357 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
358 }
359
360 return -ETIMEDOUT;
361}
362
363static void advk_pcie_wait_for_retrain(struct advk_pcie *pcie)
364{
365 size_t retries;
366
367 for (retries = 0; retries < RETRAIN_WAIT_MAX_RETRIES; ++retries) {
368 if (advk_pcie_link_training(pcie))
369 break;
370 udelay(RETRAIN_WAIT_USLEEP_US);
371 }
372}
373
374static void advk_pcie_issue_perst(struct advk_pcie *pcie)
375{
376 if (!pcie->reset_gpio)
377 return;
378
379 /* 10ms delay is needed for some cards */
380 dev_info(&pcie->pdev->dev, "issuing PERST via reset GPIO for 10ms\n");
381 gpiod_set_value_cansleep(pcie->reset_gpio, 1);
382 usleep_range(10000, 11000);
383 gpiod_set_value_cansleep(pcie->reset_gpio, 0);
384}
385
386static void advk_pcie_train_link(struct advk_pcie *pcie)
387{
388 struct device *dev = &pcie->pdev->dev;
389 u32 reg;
390 int ret;
391
392 /*
393 * Setup PCIe rev / gen compliance based on device tree property
394 * 'max-link-speed' which also forces maximal link speed.
395 */
396 reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
397 reg &= ~PCIE_GEN_SEL_MSK;
398 if (pcie->link_gen == 3)
399 reg |= SPEED_GEN_3;
400 else if (pcie->link_gen == 2)
401 reg |= SPEED_GEN_2;
402 else
403 reg |= SPEED_GEN_1;
404 advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
405
406 /*
407 * Set maximal link speed value also into PCIe Link Control 2 register.
408 * Armada 3700 Functional Specification says that default value is based
409 * on SPEED_GEN but tests showed that default value is always 8.0 GT/s.
410 */
411 reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2);
412 reg &= ~PCI_EXP_LNKCTL2_TLS;
413 if (pcie->link_gen == 3)
414 reg |= PCI_EXP_LNKCTL2_TLS_8_0GT;
415 else if (pcie->link_gen == 2)
416 reg |= PCI_EXP_LNKCTL2_TLS_5_0GT;
417 else
418 reg |= PCI_EXP_LNKCTL2_TLS_2_5GT;
419 advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2);
420
421 /* Enable link training after selecting PCIe generation */
422 reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
423 reg |= LINK_TRAINING_EN;
424 advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
425
426 /*
427 * Reset PCIe card via PERST# signal. Some cards are not detected
428 * during link training when they are in some non-initial state.
429 */
430 advk_pcie_issue_perst(pcie);
431
432 /*
433 * PERST# signal could have been asserted by pinctrl subsystem before
434 * probe() callback has been called or issued explicitly by reset gpio
435 * function advk_pcie_issue_perst(), making the endpoint going into
436 * fundamental reset. As required by PCI Express spec (PCI Express
437 * Base Specification, REV. 4.0 PCI Express, February 19 2014, 6.6.1
438 * Conventional Reset) a delay for at least 100ms after such a reset
439 * before sending a Configuration Request to the device is needed.
440 * So wait until PCIe link is up. Function advk_pcie_wait_for_link()
441 * waits for link at least 900ms.
442 */
443 ret = advk_pcie_wait_for_link(pcie);
444 if (ret < 0)
445 dev_err(dev, "link never came up\n");
446 else
447 dev_info(dev, "link up\n");
448}
449
450/*
451 * Set PCIe address window register which could be used for memory
452 * mapping.
453 */
454static void advk_pcie_set_ob_win(struct advk_pcie *pcie, u8 win_num,
455 phys_addr_t match, phys_addr_t remap,
456 phys_addr_t mask, u32 actions)
457{
458 advk_writel(pcie, OB_WIN_ENABLE |
459 lower_32_bits(match), OB_WIN_MATCH_LS(win_num));
460 advk_writel(pcie, upper_32_bits(match), OB_WIN_MATCH_MS(win_num));
461 advk_writel(pcie, lower_32_bits(remap), OB_WIN_REMAP_LS(win_num));
462 advk_writel(pcie, upper_32_bits(remap), OB_WIN_REMAP_MS(win_num));
463 advk_writel(pcie, lower_32_bits(mask), OB_WIN_MASK_LS(win_num));
464 advk_writel(pcie, upper_32_bits(mask), OB_WIN_MASK_MS(win_num));
465 advk_writel(pcie, actions, OB_WIN_ACTIONS(win_num));
466}
467
468static void advk_pcie_disable_ob_win(struct advk_pcie *pcie, u8 win_num)
469{
470 advk_writel(pcie, 0, OB_WIN_MATCH_LS(win_num));
471 advk_writel(pcie, 0, OB_WIN_MATCH_MS(win_num));
472 advk_writel(pcie, 0, OB_WIN_REMAP_LS(win_num));
473 advk_writel(pcie, 0, OB_WIN_REMAP_MS(win_num));
474 advk_writel(pcie, 0, OB_WIN_MASK_LS(win_num));
475 advk_writel(pcie, 0, OB_WIN_MASK_MS(win_num));
476 advk_writel(pcie, 0, OB_WIN_ACTIONS(win_num));
477}
478
479static void advk_pcie_setup_hw(struct advk_pcie *pcie)
480{
481 phys_addr_t msi_addr;
482 u32 reg;
483 int i;
484
485 /*
486 * Configure PCIe Reference clock. Direction is from the PCIe
487 * controller to the endpoint card, so enable transmitting of
488 * Reference clock differential signal off-chip and disable
489 * receiving off-chip differential signal.
490 */
491 reg = advk_readl(pcie, PCIE_CORE_REF_CLK_REG);
492 reg |= PCIE_CORE_REF_CLK_TX_ENABLE;
493 reg &= ~PCIE_CORE_REF_CLK_RX_ENABLE;
494 advk_writel(pcie, reg, PCIE_CORE_REF_CLK_REG);
495
496 /* Set to Direct mode */
497 reg = advk_readl(pcie, CTRL_CONFIG_REG);
498 reg &= ~(CTRL_MODE_MASK << CTRL_MODE_SHIFT);
499 reg |= ((PCIE_CORE_MODE_DIRECT & CTRL_MODE_MASK) << CTRL_MODE_SHIFT);
500 advk_writel(pcie, reg, CTRL_CONFIG_REG);
501
502 /* Set PCI global control register to RC mode */
503 reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
504 reg |= (IS_RC_MSK << IS_RC_SHIFT);
505 advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
506
507 /*
508 * Replace incorrect PCI vendor id value 0x1b4b by correct value 0x11ab.
509 * VENDOR_ID_REG contains vendor id in low 16 bits and subsystem vendor
510 * id in high 16 bits. Updating this register changes readback value of
511 * read-only vendor id bits in PCIE_CORE_DEV_ID_REG register. Workaround
512 * for erratum 4.1: "The value of device and vendor ID is incorrect".
513 */
514 reg = (PCI_VENDOR_ID_MARVELL << 16) | PCI_VENDOR_ID_MARVELL;
515 advk_writel(pcie, reg, VENDOR_ID_REG);
516
517 /*
518 * Change Class Code of PCI Bridge device to PCI Bridge (0x600400),
519 * because the default value is Mass storage controller (0x010400).
520 *
521 * Note that this Aardvark PCI Bridge does not have compliant Type 1
522 * Configuration Space and it even cannot be accessed via Aardvark's
523 * PCI config space access method. Something like config space is
524 * available in internal Aardvark registers starting at offset 0x0
525 * and is reported as Type 0. In range 0x10 - 0x34 it has totally
526 * different registers.
527 *
528 * Therefore driver uses emulation of PCI Bridge which emulates
529 * access to configuration space via internal Aardvark registers or
530 * emulated configuration buffer.
531 */
532 reg = advk_readl(pcie, PCIE_CORE_DEV_REV_REG);
533 reg &= ~0xffffff00;
534 reg |= PCI_CLASS_BRIDGE_PCI_NORMAL << 8;
535 advk_writel(pcie, reg, PCIE_CORE_DEV_REV_REG);
536
537 /* Disable Root Bridge I/O space, memory space and bus mastering */
538 reg = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
539 reg &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
540 advk_writel(pcie, reg, PCIE_CORE_CMD_STATUS_REG);
541
542 /* Set Advanced Error Capabilities and Control PF0 register */
543 reg = PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX |
544 PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN |
545 PCIE_CORE_ERR_CAPCTL_ECRC_CHCK |
546 PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV;
547 advk_writel(pcie, reg, PCIE_CORE_ERR_CAPCTL_REG);
548
549 /* Set PCIe Device Control register */
550 reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL);
551 reg &= ~PCI_EXP_DEVCTL_RELAX_EN;
552 reg &= ~PCI_EXP_DEVCTL_NOSNOOP_EN;
553 reg &= ~PCI_EXP_DEVCTL_PAYLOAD;
554 reg &= ~PCI_EXP_DEVCTL_READRQ;
555 reg |= PCI_EXP_DEVCTL_PAYLOAD_512B;
556 reg |= PCI_EXP_DEVCTL_READRQ_512B;
557 advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL);
558
559 /* Program PCIe Control 2 to disable strict ordering */
560 reg = PCIE_CORE_CTRL2_RESERVED |
561 PCIE_CORE_CTRL2_TD_ENABLE;
562 advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
563
564 /* Set lane X1 */
565 reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
566 reg &= ~LANE_CNT_MSK;
567 reg |= LANE_COUNT_1;
568 advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
569
570 /* Set MSI address */
571 msi_addr = virt_to_phys(pcie);
572 advk_writel(pcie, lower_32_bits(msi_addr), PCIE_MSI_ADDR_LOW_REG);
573 advk_writel(pcie, upper_32_bits(msi_addr), PCIE_MSI_ADDR_HIGH_REG);
574
575 /* Enable MSI */
576 reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
577 reg |= PCIE_CORE_CTRL2_MSI_ENABLE;
578 advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
579
580 /* Clear all interrupts */
581 advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_STATUS_REG);
582 advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_REG);
583 advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_REG);
584 advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_STATUS_REG);
585
586 /* Disable All ISR0/1 and MSI Sources */
587 advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_MASK_REG);
588 advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG);
589 advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_MASK_REG);
590
591 /* Unmask summary MSI interrupt */
592 reg = advk_readl(pcie, PCIE_ISR0_MASK_REG);
593 reg &= ~PCIE_ISR0_MSI_INT_PENDING;
594 advk_writel(pcie, reg, PCIE_ISR0_MASK_REG);
595
596 /* Unmask PME interrupt for processing of PME requester */
597 reg = advk_readl(pcie, PCIE_ISR0_MASK_REG);
598 reg &= ~PCIE_MSG_PM_PME_MASK;
599 advk_writel(pcie, reg, PCIE_ISR0_MASK_REG);
600
601 /* Enable summary interrupt for GIC SPI source */
602 reg = PCIE_IRQ_ALL_MASK & (~PCIE_IRQ_ENABLE_INTS_MASK);
603 advk_writel(pcie, reg, HOST_CTRL_INT_MASK_REG);
604
605 /*
606 * Enable AXI address window location generation:
607 * When it is enabled, the default outbound window
608 * configurations (Default User Field: 0xD0074CFC)
609 * are used to transparent address translation for
610 * the outbound transactions. Thus, PCIe address
611 * windows are not required for transparent memory
612 * access when default outbound window configuration
613 * is set for memory access.
614 */
615 reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
616 reg |= PCIE_CORE_CTRL2_OB_WIN_ENABLE;
617 advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
618
619 /*
620 * Set memory access in Default User Field so it
621 * is not required to configure PCIe address for
622 * transparent memory access.
623 */
624 advk_writel(pcie, OB_WIN_TYPE_MEM, OB_WIN_DEFAULT_ACTIONS);
625
626 /*
627 * Bypass the address window mapping for PIO:
628 * Since PIO access already contains all required
629 * info over AXI interface by PIO registers, the
630 * address window is not required.
631 */
632 reg = advk_readl(pcie, PIO_CTRL);
633 reg |= PIO_CTRL_ADDR_WIN_DISABLE;
634 advk_writel(pcie, reg, PIO_CTRL);
635
636 /*
637 * Configure PCIe address windows for non-memory or
638 * non-transparent access as by default PCIe uses
639 * transparent memory access.
640 */
641 for (i = 0; i < pcie->wins_count; i++)
642 advk_pcie_set_ob_win(pcie, i,
643 pcie->wins[i].match, pcie->wins[i].remap,
644 pcie->wins[i].mask, pcie->wins[i].actions);
645
646 /* Disable remaining PCIe outbound windows */
647 for (i = pcie->wins_count; i < OB_WIN_COUNT; i++)
648 advk_pcie_disable_ob_win(pcie, i);
649
650 advk_pcie_train_link(pcie);
651}
652
653static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u32 *val)
654{
655 struct device *dev = &pcie->pdev->dev;
656 u32 reg;
657 unsigned int status;
658 char *strcomp_status, *str_posted;
659 int ret;
660
661 reg = advk_readl(pcie, PIO_STAT);
662 status = (reg & PIO_COMPLETION_STATUS_MASK) >>
663 PIO_COMPLETION_STATUS_SHIFT;
664
665 /*
666 * According to HW spec, the PIO status check sequence as below:
667 * 1) even if COMPLETION_STATUS(bit9:7) indicates successful,
668 * it still needs to check Error Status(bit11), only when this bit
669 * indicates no error happen, the operation is successful.
670 * 2) value Unsupported Request(1) of COMPLETION_STATUS(bit9:7) only
671 * means a PIO write error, and for PIO read it is successful with
672 * a read value of 0xFFFFFFFF.
673 * 3) value Completion Retry Status(CRS) of COMPLETION_STATUS(bit9:7)
674 * only means a PIO write error, and for PIO read it is successful
675 * with a read value of 0xFFFF0001.
676 * 4) value Completer Abort (CA) of COMPLETION_STATUS(bit9:7) means
677 * error for both PIO read and PIO write operation.
678 * 5) other errors are indicated as 'unknown'.
679 */
680 switch (status) {
681 case PIO_COMPLETION_STATUS_OK:
682 if (reg & PIO_ERR_STATUS) {
683 strcomp_status = "COMP_ERR";
684 ret = -EFAULT;
685 break;
686 }
687 /* Get the read result */
688 if (val)
689 *val = advk_readl(pcie, PIO_RD_DATA);
690 /* No error */
691 strcomp_status = NULL;
692 ret = 0;
693 break;
694 case PIO_COMPLETION_STATUS_UR:
695 strcomp_status = "UR";
696 ret = -EOPNOTSUPP;
697 break;
698 case PIO_COMPLETION_STATUS_CRS:
699 if (allow_crs && val) {
700 /* PCIe r4.0, sec 2.3.2, says:
701 * If CRS Software Visibility is enabled:
702 * For a Configuration Read Request that includes both
703 * bytes of the Vendor ID field of a device Function's
704 * Configuration Space Header, the Root Complex must
705 * complete the Request to the host by returning a
706 * read-data value of 0001h for the Vendor ID field and
707 * all '1's for any additional bytes included in the
708 * request.
709 *
710 * So CRS in this case is not an error status.
711 */
712 *val = CFG_RD_CRS_VAL;
713 strcomp_status = NULL;
714 ret = 0;
715 break;
716 }
717 /* PCIe r4.0, sec 2.3.2, says:
718 * If CRS Software Visibility is not enabled, the Root Complex
719 * must re-issue the Configuration Request as a new Request.
720 * If CRS Software Visibility is enabled: For a Configuration
721 * Write Request or for any other Configuration Read Request,
722 * the Root Complex must re-issue the Configuration Request as
723 * a new Request.
724 * A Root Complex implementation may choose to limit the number
725 * of Configuration Request/CRS Completion Status loops before
726 * determining that something is wrong with the target of the
727 * Request and taking appropriate action, e.g., complete the
728 * Request to the host as a failed transaction.
729 *
730 * So return -EAGAIN and caller (pci-aardvark.c driver) will
731 * re-issue request again up to the PIO_RETRY_CNT retries.
732 */
733 strcomp_status = "CRS";
734 ret = -EAGAIN;
735 break;
736 case PIO_COMPLETION_STATUS_CA:
737 strcomp_status = "CA";
738 ret = -ECANCELED;
739 break;
740 default:
741 strcomp_status = "Unknown";
742 ret = -EINVAL;
743 break;
744 }
745
746 if (!strcomp_status)
747 return ret;
748
749 if (reg & PIO_NON_POSTED_REQ)
750 str_posted = "Non-posted";
751 else
752 str_posted = "Posted";
753
754 dev_dbg(dev, "%s PIO Response Status: %s, %#x @ %#x\n",
755 str_posted, strcomp_status, reg, advk_readl(pcie, PIO_ADDR_LS));
756
757 return ret;
758}
759
760static int advk_pcie_wait_pio(struct advk_pcie *pcie)
761{
762 struct device *dev = &pcie->pdev->dev;
763 int i;
764
765 for (i = 1; i <= PIO_RETRY_CNT; i++) {
766 u32 start, isr;
767
768 start = advk_readl(pcie, PIO_START);
769 isr = advk_readl(pcie, PIO_ISR);
770 if (!start && isr)
771 return i;
772 udelay(PIO_RETRY_DELAY);
773 }
774
775 dev_err(dev, "PIO read/write transfer time out\n");
776 return -ETIMEDOUT;
777}
778
779static pci_bridge_emul_read_status_t
780advk_pci_bridge_emul_base_conf_read(struct pci_bridge_emul *bridge,
781 int reg, u32 *value)
782{
783 struct advk_pcie *pcie = bridge->data;
784
785 switch (reg) {
786 case PCI_COMMAND:
787 *value = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
788 return PCI_BRIDGE_EMUL_HANDLED;
789
790 case PCI_INTERRUPT_LINE: {
791 /*
792 * From the whole 32bit register we support reading from HW only
793 * two bits: PCI_BRIDGE_CTL_BUS_RESET and PCI_BRIDGE_CTL_SERR.
794 * Other bits are retrieved only from emulated config buffer.
795 */
796 __le32 *cfgspace = (__le32 *)&bridge->conf;
797 u32 val = le32_to_cpu(cfgspace[PCI_INTERRUPT_LINE / 4]);
798 if (advk_readl(pcie, PCIE_ISR0_MASK_REG) & PCIE_ISR0_ERR_MASK)
799 val &= ~(PCI_BRIDGE_CTL_SERR << 16);
800 else
801 val |= PCI_BRIDGE_CTL_SERR << 16;
802 if (advk_readl(pcie, PCIE_CORE_CTRL1_REG) & HOT_RESET_GEN)
803 val |= PCI_BRIDGE_CTL_BUS_RESET << 16;
804 else
805 val &= ~(PCI_BRIDGE_CTL_BUS_RESET << 16);
806 *value = val;
807 return PCI_BRIDGE_EMUL_HANDLED;
808 }
809
810 default:
811 return PCI_BRIDGE_EMUL_NOT_HANDLED;
812 }
813}
814
815static void
816advk_pci_bridge_emul_base_conf_write(struct pci_bridge_emul *bridge,
817 int reg, u32 old, u32 new, u32 mask)
818{
819 struct advk_pcie *pcie = bridge->data;
820
821 switch (reg) {
822 case PCI_COMMAND:
823 advk_writel(pcie, new, PCIE_CORE_CMD_STATUS_REG);
824 break;
825
826 case PCI_INTERRUPT_LINE:
827 /*
828 * According to Figure 6-3: Pseudo Logic Diagram for Error
829 * Message Controls in PCIe base specification, SERR# Enable bit
830 * in Bridge Control register enable receiving of ERR_* messages
831 */
832 if (mask & (PCI_BRIDGE_CTL_SERR << 16)) {
833 u32 val = advk_readl(pcie, PCIE_ISR0_MASK_REG);
834 if (new & (PCI_BRIDGE_CTL_SERR << 16))
835 val &= ~PCIE_ISR0_ERR_MASK;
836 else
837 val |= PCIE_ISR0_ERR_MASK;
838 advk_writel(pcie, val, PCIE_ISR0_MASK_REG);
839 }
840 if (mask & (PCI_BRIDGE_CTL_BUS_RESET << 16)) {
841 u32 val = advk_readl(pcie, PCIE_CORE_CTRL1_REG);
842 if (new & (PCI_BRIDGE_CTL_BUS_RESET << 16))
843 val |= HOT_RESET_GEN;
844 else
845 val &= ~HOT_RESET_GEN;
846 advk_writel(pcie, val, PCIE_CORE_CTRL1_REG);
847 }
848 break;
849
850 default:
851 break;
852 }
853}
854
855static pci_bridge_emul_read_status_t
856advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge,
857 int reg, u32 *value)
858{
859 struct advk_pcie *pcie = bridge->data;
860
861
862 switch (reg) {
863 /*
864 * PCI_EXP_SLTCAP, PCI_EXP_SLTCTL, PCI_EXP_RTCTL and PCI_EXP_RTSTA are
865 * also supported, but do not need to be handled here, because their
866 * values are stored in emulated config space buffer, and we read them
867 * from there when needed.
868 */
869
870 case PCI_EXP_LNKCAP: {
871 u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg);
872 /*
873 * PCI_EXP_LNKCAP_DLLLARC bit is hardwired in aardvark HW to 0.
874 * But support for PCI_EXP_LNKSTA_DLLLA is emulated via ltssm
875 * state so explicitly enable PCI_EXP_LNKCAP_DLLLARC flag.
876 */
877 val |= PCI_EXP_LNKCAP_DLLLARC;
878 *value = val;
879 return PCI_BRIDGE_EMUL_HANDLED;
880 }
881
882 case PCI_EXP_LNKCTL: {
883 /* u32 contains both PCI_EXP_LNKCTL and PCI_EXP_LNKSTA */
884 u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg) &
885 ~(PCI_EXP_LNKSTA_LT << 16);
886 if (advk_pcie_link_training(pcie))
887 val |= (PCI_EXP_LNKSTA_LT << 16);
888 if (advk_pcie_link_active(pcie))
889 val |= (PCI_EXP_LNKSTA_DLLLA << 16);
890 *value = val;
891 return PCI_BRIDGE_EMUL_HANDLED;
892 }
893
894 case PCI_EXP_DEVCAP:
895 case PCI_EXP_DEVCTL:
896 case PCI_EXP_DEVCAP2:
897 case PCI_EXP_DEVCTL2:
898 case PCI_EXP_LNKCAP2:
899 case PCI_EXP_LNKCTL2:
900 *value = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg);
901 return PCI_BRIDGE_EMUL_HANDLED;
902
903 default:
904 return PCI_BRIDGE_EMUL_NOT_HANDLED;
905 }
906
907}
908
909static void
910advk_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge,
911 int reg, u32 old, u32 new, u32 mask)
912{
913 struct advk_pcie *pcie = bridge->data;
914
915 switch (reg) {
916 case PCI_EXP_LNKCTL:
917 advk_writel(pcie, new, PCIE_CORE_PCIEXP_CAP + reg);
918 if (new & PCI_EXP_LNKCTL_RL)
919 advk_pcie_wait_for_retrain(pcie);
920 break;
921
922 case PCI_EXP_RTCTL: {
923 u16 rootctl = le16_to_cpu(bridge->pcie_conf.rootctl);
924 /* Only emulation of PMEIE and CRSSVE bits is provided */
925 rootctl &= PCI_EXP_RTCTL_PMEIE | PCI_EXP_RTCTL_CRSSVE;
926 bridge->pcie_conf.rootctl = cpu_to_le16(rootctl);
927 break;
928 }
929
930 /*
931 * PCI_EXP_RTSTA is also supported, but does not need to be handled
932 * here, because its value is stored in emulated config space buffer,
933 * and we write it there when needed.
934 */
935
936 case PCI_EXP_DEVCTL:
937 case PCI_EXP_DEVCTL2:
938 case PCI_EXP_LNKCTL2:
939 advk_writel(pcie, new, PCIE_CORE_PCIEXP_CAP + reg);
940 break;
941
942 default:
943 break;
944 }
945}
946
947static pci_bridge_emul_read_status_t
948advk_pci_bridge_emul_ext_conf_read(struct pci_bridge_emul *bridge,
949 int reg, u32 *value)
950{
951 struct advk_pcie *pcie = bridge->data;
952
953 switch (reg) {
954 case 0:
955 *value = advk_readl(pcie, PCIE_CORE_PCIERR_CAP + reg);
956
957 /*
958 * PCI_EXT_CAP_NEXT bits are set to offset 0x150, but Armada
959 * 3700 Functional Specification does not document registers
960 * at those addresses.
961 *
962 * Thus we clear PCI_EXT_CAP_NEXT bits to make Advanced Error
963 * Reporting Capability header the last Extended Capability.
964 * If we obtain documentation for those registers in the
965 * future, this can be changed.
966 */
967 *value &= 0x000fffff;
968 return PCI_BRIDGE_EMUL_HANDLED;
969
970 case PCI_ERR_UNCOR_STATUS:
971 case PCI_ERR_UNCOR_MASK:
972 case PCI_ERR_UNCOR_SEVER:
973 case PCI_ERR_COR_STATUS:
974 case PCI_ERR_COR_MASK:
975 case PCI_ERR_CAP:
976 case PCI_ERR_HEADER_LOG + 0:
977 case PCI_ERR_HEADER_LOG + 4:
978 case PCI_ERR_HEADER_LOG + 8:
979 case PCI_ERR_HEADER_LOG + 12:
980 case PCI_ERR_ROOT_COMMAND:
981 case PCI_ERR_ROOT_STATUS:
982 case PCI_ERR_ROOT_ERR_SRC:
983 *value = advk_readl(pcie, PCIE_CORE_PCIERR_CAP + reg);
984 return PCI_BRIDGE_EMUL_HANDLED;
985
986 default:
987 return PCI_BRIDGE_EMUL_NOT_HANDLED;
988 }
989}
990
991static void
992advk_pci_bridge_emul_ext_conf_write(struct pci_bridge_emul *bridge,
993 int reg, u32 old, u32 new, u32 mask)
994{
995 struct advk_pcie *pcie = bridge->data;
996
997 switch (reg) {
998 /* These are W1C registers, so clear other bits */
999 case PCI_ERR_UNCOR_STATUS:
1000 case PCI_ERR_COR_STATUS:
1001 case PCI_ERR_ROOT_STATUS:
1002 new &= mask;
1003 fallthrough;
1004
1005 case PCI_ERR_UNCOR_MASK:
1006 case PCI_ERR_UNCOR_SEVER:
1007 case PCI_ERR_COR_MASK:
1008 case PCI_ERR_CAP:
1009 case PCI_ERR_HEADER_LOG + 0:
1010 case PCI_ERR_HEADER_LOG + 4:
1011 case PCI_ERR_HEADER_LOG + 8:
1012 case PCI_ERR_HEADER_LOG + 12:
1013 case PCI_ERR_ROOT_COMMAND:
1014 case PCI_ERR_ROOT_ERR_SRC:
1015 advk_writel(pcie, new, PCIE_CORE_PCIERR_CAP + reg);
1016 break;
1017
1018 default:
1019 break;
1020 }
1021}
1022
1023static const struct pci_bridge_emul_ops advk_pci_bridge_emul_ops = {
1024 .read_base = advk_pci_bridge_emul_base_conf_read,
1025 .write_base = advk_pci_bridge_emul_base_conf_write,
1026 .read_pcie = advk_pci_bridge_emul_pcie_conf_read,
1027 .write_pcie = advk_pci_bridge_emul_pcie_conf_write,
1028 .read_ext = advk_pci_bridge_emul_ext_conf_read,
1029 .write_ext = advk_pci_bridge_emul_ext_conf_write,
1030};
1031
1032/*
1033 * Initialize the configuration space of the PCI-to-PCI bridge
1034 * associated with the given PCIe interface.
1035 */
1036static int advk_sw_pci_bridge_init(struct advk_pcie *pcie)
1037{
1038 struct pci_bridge_emul *bridge = &pcie->bridge;
1039
1040 bridge->conf.vendor =
1041 cpu_to_le16(advk_readl(pcie, PCIE_CORE_DEV_ID_REG) & 0xffff);
1042 bridge->conf.device =
1043 cpu_to_le16(advk_readl(pcie, PCIE_CORE_DEV_ID_REG) >> 16);
1044 bridge->conf.class_revision =
1045 cpu_to_le32(advk_readl(pcie, PCIE_CORE_DEV_REV_REG) & 0xff);
1046
1047 /* Support 32 bits I/O addressing */
1048 bridge->conf.iobase = PCI_IO_RANGE_TYPE_32;
1049 bridge->conf.iolimit = PCI_IO_RANGE_TYPE_32;
1050
1051 /* Support 64 bits memory pref */
1052 bridge->conf.pref_mem_base = cpu_to_le16(PCI_PREF_RANGE_TYPE_64);
1053 bridge->conf.pref_mem_limit = cpu_to_le16(PCI_PREF_RANGE_TYPE_64);
1054
1055 /* Support interrupt A for MSI feature */
1056 bridge->conf.intpin = PCI_INTERRUPT_INTA;
1057
1058 /*
1059 * Aardvark HW provides PCIe Capability structure in version 2 and
1060 * indicate slot support, which is emulated.
1061 */
1062 bridge->pcie_conf.cap = cpu_to_le16(2 | PCI_EXP_FLAGS_SLOT);
1063
1064 /*
1065 * Set Presence Detect State bit permanently since there is no support
1066 * for unplugging the card nor detecting whether it is plugged. (If a
1067 * platform exists in the future that supports it, via a GPIO for
1068 * example, it should be implemented via this bit.)
1069 *
1070 * Set physical slot number to 1 since there is only one port and zero
1071 * value is reserved for ports within the same silicon as Root Port
1072 * which is not our case.
1073 */
1074 bridge->pcie_conf.slotcap = cpu_to_le32(FIELD_PREP(PCI_EXP_SLTCAP_PSN,
1075 1));
1076 bridge->pcie_conf.slotsta = cpu_to_le16(PCI_EXP_SLTSTA_PDS);
1077
1078 /* Indicates supports for Completion Retry Status */
1079 bridge->pcie_conf.rootcap = cpu_to_le16(PCI_EXP_RTCAP_CRSVIS);
1080
1081 bridge->subsystem_vendor_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) & 0xffff;
1082 bridge->subsystem_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) >> 16;
1083 bridge->has_pcie = true;
1084 bridge->pcie_start = PCIE_CORE_PCIEXP_CAP;
1085 bridge->data = pcie;
1086 bridge->ops = &advk_pci_bridge_emul_ops;
1087
1088 return pci_bridge_emul_init(bridge, 0);
1089}
1090
1091static bool advk_pcie_valid_device(struct advk_pcie *pcie, struct pci_bus *bus,
1092 int devfn)
1093{
1094 if (pci_is_root_bus(bus) && PCI_SLOT(devfn) != 0)
1095 return false;
1096
1097 /*
1098 * If the link goes down after we check for link-up, we have a problem:
1099 * if a PIO request is executed while link-down, the whole controller
1100 * gets stuck in a non-functional state, and even after link comes up
1101 * again, PIO requests won't work anymore, and a reset of the whole PCIe
1102 * controller is needed. Therefore we need to prevent sending PIO
1103 * requests while the link is down.
1104 */
1105 if (!pci_is_root_bus(bus) && !advk_pcie_link_up(pcie))
1106 return false;
1107
1108 return true;
1109}
1110
1111static bool advk_pcie_pio_is_running(struct advk_pcie *pcie)
1112{
1113 struct device *dev = &pcie->pdev->dev;
1114
1115 /*
1116 * Trying to start a new PIO transfer when previous has not completed
1117 * cause External Abort on CPU which results in kernel panic:
1118 *
1119 * SError Interrupt on CPU0, code 0xbf000002 -- SError
1120 * Kernel panic - not syncing: Asynchronous SError Interrupt
1121 *
1122 * Functions advk_pcie_rd_conf() and advk_pcie_wr_conf() are protected
1123 * by raw_spin_lock_irqsave() at pci_lock_config() level to prevent
1124 * concurrent calls at the same time. But because PIO transfer may take
1125 * about 1.5s when link is down or card is disconnected, it means that
1126 * advk_pcie_wait_pio() does not always have to wait for completion.
1127 *
1128 * Some versions of ARM Trusted Firmware handles this External Abort at
1129 * EL3 level and mask it to prevent kernel panic. Relevant TF-A commit:
1130 * https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git/commit/?id=3c7dcdac5c50
1131 */
1132 if (advk_readl(pcie, PIO_START)) {
1133 dev_err(dev, "Previous PIO read/write transfer is still running\n");
1134 return true;
1135 }
1136
1137 return false;
1138}
1139
1140static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn,
1141 int where, int size, u32 *val)
1142{
1143 struct advk_pcie *pcie = bus->sysdata;
1144 int retry_count;
1145 bool allow_crs;
1146 u32 reg;
1147 int ret;
1148
1149 if (!advk_pcie_valid_device(pcie, bus, devfn))
1150 return PCIBIOS_DEVICE_NOT_FOUND;
1151
1152 if (pci_is_root_bus(bus))
1153 return pci_bridge_emul_conf_read(&pcie->bridge, where,
1154 size, val);
1155
1156 /*
1157 * Completion Retry Status is possible to return only when reading all
1158 * 4 bytes from PCI_VENDOR_ID and PCI_DEVICE_ID registers at once and
1159 * CRSSVE flag on Root Bridge is enabled.
1160 */
1161 allow_crs = (where == PCI_VENDOR_ID) && (size == 4) &&
1162 (le16_to_cpu(pcie->bridge.pcie_conf.rootctl) &
1163 PCI_EXP_RTCTL_CRSSVE);
1164
1165 if (advk_pcie_pio_is_running(pcie))
1166 goto try_crs;
1167
1168 /* Program the control register */
1169 reg = advk_readl(pcie, PIO_CTRL);
1170 reg &= ~PIO_CTRL_TYPE_MASK;
1171 if (pci_is_root_bus(bus->parent))
1172 reg |= PCIE_CONFIG_RD_TYPE0;
1173 else
1174 reg |= PCIE_CONFIG_RD_TYPE1;
1175 advk_writel(pcie, reg, PIO_CTRL);
1176
1177 /* Program the address registers */
1178 reg = ALIGN_DOWN(PCIE_ECAM_OFFSET(bus->number, devfn, where), 4);
1179 advk_writel(pcie, reg, PIO_ADDR_LS);
1180 advk_writel(pcie, 0, PIO_ADDR_MS);
1181
1182 /* Program the data strobe */
1183 advk_writel(pcie, 0xf, PIO_WR_DATA_STRB);
1184
1185 retry_count = 0;
1186 do {
1187 /* Clear PIO DONE ISR and start the transfer */
1188 advk_writel(pcie, 1, PIO_ISR);
1189 advk_writel(pcie, 1, PIO_START);
1190
1191 ret = advk_pcie_wait_pio(pcie);
1192 if (ret < 0)
1193 goto try_crs;
1194
1195 retry_count += ret;
1196
1197 /* Check PIO status and get the read result */
1198 ret = advk_pcie_check_pio_status(pcie, allow_crs, val);
1199 } while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT);
1200
1201 if (ret < 0)
1202 goto fail;
1203
1204 if (size == 1)
1205 *val = (*val >> (8 * (where & 3))) & 0xff;
1206 else if (size == 2)
1207 *val = (*val >> (8 * (where & 3))) & 0xffff;
1208
1209 return PCIBIOS_SUCCESSFUL;
1210
1211try_crs:
1212 /*
1213 * If it is possible, return Completion Retry Status so that caller
1214 * tries to issue the request again instead of failing.
1215 */
1216 if (allow_crs) {
1217 *val = CFG_RD_CRS_VAL;
1218 return PCIBIOS_SUCCESSFUL;
1219 }
1220
1221fail:
1222 *val = 0xffffffff;
1223 return PCIBIOS_SET_FAILED;
1224}
1225
1226static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
1227 int where, int size, u32 val)
1228{
1229 struct advk_pcie *pcie = bus->sysdata;
1230 u32 reg;
1231 u32 data_strobe = 0x0;
1232 int retry_count;
1233 int offset;
1234 int ret;
1235
1236 if (!advk_pcie_valid_device(pcie, bus, devfn))
1237 return PCIBIOS_DEVICE_NOT_FOUND;
1238
1239 if (pci_is_root_bus(bus))
1240 return pci_bridge_emul_conf_write(&pcie->bridge, where,
1241 size, val);
1242
1243 if (where % size)
1244 return PCIBIOS_SET_FAILED;
1245
1246 if (advk_pcie_pio_is_running(pcie))
1247 return PCIBIOS_SET_FAILED;
1248
1249 /* Program the control register */
1250 reg = advk_readl(pcie, PIO_CTRL);
1251 reg &= ~PIO_CTRL_TYPE_MASK;
1252 if (pci_is_root_bus(bus->parent))
1253 reg |= PCIE_CONFIG_WR_TYPE0;
1254 else
1255 reg |= PCIE_CONFIG_WR_TYPE1;
1256 advk_writel(pcie, reg, PIO_CTRL);
1257
1258 /* Program the address registers */
1259 reg = ALIGN_DOWN(PCIE_ECAM_OFFSET(bus->number, devfn, where), 4);
1260 advk_writel(pcie, reg, PIO_ADDR_LS);
1261 advk_writel(pcie, 0, PIO_ADDR_MS);
1262
1263 /* Calculate the write strobe */
1264 offset = where & 0x3;
1265 reg = val << (8 * offset);
1266 data_strobe = GENMASK(size - 1, 0) << offset;
1267
1268 /* Program the data register */
1269 advk_writel(pcie, reg, PIO_WR_DATA);
1270
1271 /* Program the data strobe */
1272 advk_writel(pcie, data_strobe, PIO_WR_DATA_STRB);
1273
1274 retry_count = 0;
1275 do {
1276 /* Clear PIO DONE ISR and start the transfer */
1277 advk_writel(pcie, 1, PIO_ISR);
1278 advk_writel(pcie, 1, PIO_START);
1279
1280 ret = advk_pcie_wait_pio(pcie);
1281 if (ret < 0)
1282 return PCIBIOS_SET_FAILED;
1283
1284 retry_count += ret;
1285
1286 ret = advk_pcie_check_pio_status(pcie, false, NULL);
1287 } while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT);
1288
1289 return ret < 0 ? PCIBIOS_SET_FAILED : PCIBIOS_SUCCESSFUL;
1290}
1291
1292static struct pci_ops advk_pcie_ops = {
1293 .read = advk_pcie_rd_conf,
1294 .write = advk_pcie_wr_conf,
1295};
1296
1297static void advk_msi_irq_compose_msi_msg(struct irq_data *data,
1298 struct msi_msg *msg)
1299{
1300 struct advk_pcie *pcie = irq_data_get_irq_chip_data(data);
1301 phys_addr_t msi_addr = virt_to_phys(pcie);
1302
1303 msg->address_lo = lower_32_bits(msi_addr);
1304 msg->address_hi = upper_32_bits(msi_addr);
1305 msg->data = data->hwirq;
1306}
1307
1308static int advk_msi_set_affinity(struct irq_data *irq_data,
1309 const struct cpumask *mask, bool force)
1310{
1311 return -EINVAL;
1312}
1313
1314static void advk_msi_irq_mask(struct irq_data *d)
1315{
1316 struct advk_pcie *pcie = d->domain->host_data;
1317 irq_hw_number_t hwirq = irqd_to_hwirq(d);
1318 unsigned long flags;
1319 u32 mask;
1320
1321 raw_spin_lock_irqsave(&pcie->msi_irq_lock, flags);
1322 mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
1323 mask |= BIT(hwirq);
1324 advk_writel(pcie, mask, PCIE_MSI_MASK_REG);
1325 raw_spin_unlock_irqrestore(&pcie->msi_irq_lock, flags);
1326}
1327
1328static void advk_msi_irq_unmask(struct irq_data *d)
1329{
1330 struct advk_pcie *pcie = d->domain->host_data;
1331 irq_hw_number_t hwirq = irqd_to_hwirq(d);
1332 unsigned long flags;
1333 u32 mask;
1334
1335 raw_spin_lock_irqsave(&pcie->msi_irq_lock, flags);
1336 mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
1337 mask &= ~BIT(hwirq);
1338 advk_writel(pcie, mask, PCIE_MSI_MASK_REG);
1339 raw_spin_unlock_irqrestore(&pcie->msi_irq_lock, flags);
1340}
1341
1342static void advk_msi_top_irq_mask(struct irq_data *d)
1343{
1344 pci_msi_mask_irq(d);
1345 irq_chip_mask_parent(d);
1346}
1347
1348static void advk_msi_top_irq_unmask(struct irq_data *d)
1349{
1350 pci_msi_unmask_irq(d);
1351 irq_chip_unmask_parent(d);
1352}
1353
1354static struct irq_chip advk_msi_bottom_irq_chip = {
1355 .name = "MSI",
1356 .irq_compose_msi_msg = advk_msi_irq_compose_msi_msg,
1357 .irq_set_affinity = advk_msi_set_affinity,
1358 .irq_mask = advk_msi_irq_mask,
1359 .irq_unmask = advk_msi_irq_unmask,
1360};
1361
1362static int advk_msi_irq_domain_alloc(struct irq_domain *domain,
1363 unsigned int virq,
1364 unsigned int nr_irqs, void *args)
1365{
1366 struct advk_pcie *pcie = domain->host_data;
1367 int hwirq, i;
1368
1369 mutex_lock(&pcie->msi_used_lock);
1370 hwirq = bitmap_find_free_region(pcie->msi_used, MSI_IRQ_NUM,
1371 order_base_2(nr_irqs));
1372 mutex_unlock(&pcie->msi_used_lock);
1373 if (hwirq < 0)
1374 return -ENOSPC;
1375
1376 for (i = 0; i < nr_irqs; i++)
1377 irq_domain_set_info(domain, virq + i, hwirq + i,
1378 &advk_msi_bottom_irq_chip,
1379 domain->host_data, handle_simple_irq,
1380 NULL, NULL);
1381
1382 return 0;
1383}
1384
1385static void advk_msi_irq_domain_free(struct irq_domain *domain,
1386 unsigned int virq, unsigned int nr_irqs)
1387{
1388 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
1389 struct advk_pcie *pcie = domain->host_data;
1390
1391 mutex_lock(&pcie->msi_used_lock);
1392 bitmap_release_region(pcie->msi_used, d->hwirq, order_base_2(nr_irqs));
1393 mutex_unlock(&pcie->msi_used_lock);
1394}
1395
1396static const struct irq_domain_ops advk_msi_domain_ops = {
1397 .alloc = advk_msi_irq_domain_alloc,
1398 .free = advk_msi_irq_domain_free,
1399};
1400
1401static void advk_pcie_irq_mask(struct irq_data *d)
1402{
1403 struct advk_pcie *pcie = d->domain->host_data;
1404 irq_hw_number_t hwirq = irqd_to_hwirq(d);
1405 unsigned long flags;
1406 u32 mask;
1407
1408 raw_spin_lock_irqsave(&pcie->irq_lock, flags);
1409 mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
1410 mask |= PCIE_ISR1_INTX_ASSERT(hwirq);
1411 advk_writel(pcie, mask, PCIE_ISR1_MASK_REG);
1412 raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);
1413}
1414
1415static void advk_pcie_irq_unmask(struct irq_data *d)
1416{
1417 struct advk_pcie *pcie = d->domain->host_data;
1418 irq_hw_number_t hwirq = irqd_to_hwirq(d);
1419 unsigned long flags;
1420 u32 mask;
1421
1422 raw_spin_lock_irqsave(&pcie->irq_lock, flags);
1423 mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
1424 mask &= ~PCIE_ISR1_INTX_ASSERT(hwirq);
1425 advk_writel(pcie, mask, PCIE_ISR1_MASK_REG);
1426 raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);
1427}
1428
1429static int advk_pcie_irq_map(struct irq_domain *h,
1430 unsigned int virq, irq_hw_number_t hwirq)
1431{
1432 struct advk_pcie *pcie = h->host_data;
1433
1434 irq_set_status_flags(virq, IRQ_LEVEL);
1435 irq_set_chip_and_handler(virq, &pcie->irq_chip,
1436 handle_level_irq);
1437 irq_set_chip_data(virq, pcie);
1438
1439 return 0;
1440}
1441
1442static const struct irq_domain_ops advk_pcie_irq_domain_ops = {
1443 .map = advk_pcie_irq_map,
1444 .xlate = irq_domain_xlate_onecell,
1445};
1446
1447static struct irq_chip advk_msi_irq_chip = {
1448 .name = "advk-MSI",
1449 .irq_mask = advk_msi_top_irq_mask,
1450 .irq_unmask = advk_msi_top_irq_unmask,
1451};
1452
1453static struct msi_domain_info advk_msi_domain_info = {
1454 .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
1455 MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX,
1456 .chip = &advk_msi_irq_chip,
1457};
1458
1459static int advk_pcie_init_msi_irq_domain(struct advk_pcie *pcie)
1460{
1461 struct device *dev = &pcie->pdev->dev;
1462
1463 raw_spin_lock_init(&pcie->msi_irq_lock);
1464 mutex_init(&pcie->msi_used_lock);
1465
1466 pcie->msi_inner_domain =
1467 irq_domain_add_linear(NULL, MSI_IRQ_NUM,
1468 &advk_msi_domain_ops, pcie);
1469 if (!pcie->msi_inner_domain)
1470 return -ENOMEM;
1471
1472 pcie->msi_domain =
1473 pci_msi_create_irq_domain(dev_fwnode(dev),
1474 &advk_msi_domain_info,
1475 pcie->msi_inner_domain);
1476 if (!pcie->msi_domain) {
1477 irq_domain_remove(pcie->msi_inner_domain);
1478 return -ENOMEM;
1479 }
1480
1481 return 0;
1482}
1483
1484static void advk_pcie_remove_msi_irq_domain(struct advk_pcie *pcie)
1485{
1486 irq_domain_remove(pcie->msi_domain);
1487 irq_domain_remove(pcie->msi_inner_domain);
1488}
1489
1490static int advk_pcie_init_irq_domain(struct advk_pcie *pcie)
1491{
1492 struct device *dev = &pcie->pdev->dev;
1493 struct device_node *node = dev->of_node;
1494 struct device_node *pcie_intc_node;
1495 struct irq_chip *irq_chip;
1496 int ret = 0;
1497
1498 raw_spin_lock_init(&pcie->irq_lock);
1499
1500 pcie_intc_node = of_get_next_child(node, NULL);
1501 if (!pcie_intc_node) {
1502 dev_err(dev, "No PCIe Intc node found\n");
1503 return -ENODEV;
1504 }
1505
1506 irq_chip = &pcie->irq_chip;
1507
1508 irq_chip->name = devm_kasprintf(dev, GFP_KERNEL, "%s-irq",
1509 dev_name(dev));
1510 if (!irq_chip->name) {
1511 ret = -ENOMEM;
1512 goto out_put_node;
1513 }
1514
1515 irq_chip->irq_mask = advk_pcie_irq_mask;
1516 irq_chip->irq_unmask = advk_pcie_irq_unmask;
1517
1518 pcie->irq_domain =
1519 irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
1520 &advk_pcie_irq_domain_ops, pcie);
1521 if (!pcie->irq_domain) {
1522 dev_err(dev, "Failed to get a INTx IRQ domain\n");
1523 ret = -ENOMEM;
1524 goto out_put_node;
1525 }
1526
1527out_put_node:
1528 of_node_put(pcie_intc_node);
1529 return ret;
1530}
1531
1532static void advk_pcie_remove_irq_domain(struct advk_pcie *pcie)
1533{
1534 irq_domain_remove(pcie->irq_domain);
1535}
1536
1537static struct irq_chip advk_rp_irq_chip = {
1538 .name = "advk-RP",
1539};
1540
1541static int advk_pcie_rp_irq_map(struct irq_domain *h,
1542 unsigned int virq, irq_hw_number_t hwirq)
1543{
1544 struct advk_pcie *pcie = h->host_data;
1545
1546 irq_set_chip_and_handler(virq, &advk_rp_irq_chip, handle_simple_irq);
1547 irq_set_chip_data(virq, pcie);
1548
1549 return 0;
1550}
1551
1552static const struct irq_domain_ops advk_pcie_rp_irq_domain_ops = {
1553 .map = advk_pcie_rp_irq_map,
1554 .xlate = irq_domain_xlate_onecell,
1555};
1556
1557static int advk_pcie_init_rp_irq_domain(struct advk_pcie *pcie)
1558{
1559 pcie->rp_irq_domain = irq_domain_add_linear(NULL, 1,
1560 &advk_pcie_rp_irq_domain_ops,
1561 pcie);
1562 if (!pcie->rp_irq_domain) {
1563 dev_err(&pcie->pdev->dev, "Failed to add Root Port IRQ domain\n");
1564 return -ENOMEM;
1565 }
1566
1567 return 0;
1568}
1569
1570static void advk_pcie_remove_rp_irq_domain(struct advk_pcie *pcie)
1571{
1572 irq_domain_remove(pcie->rp_irq_domain);
1573}
1574
1575static void advk_pcie_handle_pme(struct advk_pcie *pcie)
1576{
1577 u32 requester = advk_readl(pcie, PCIE_MSG_LOG_REG) >> 16;
1578
1579 advk_writel(pcie, PCIE_MSG_PM_PME_MASK, PCIE_ISR0_REG);
1580
1581 /*
1582 * PCIE_MSG_LOG_REG contains the last inbound message, so store
1583 * the requester ID only when PME was not asserted yet.
1584 * Also do not trigger PME interrupt when PME is still asserted.
1585 */
1586 if (!(le32_to_cpu(pcie->bridge.pcie_conf.rootsta) & PCI_EXP_RTSTA_PME)) {
1587 pcie->bridge.pcie_conf.rootsta = cpu_to_le32(requester | PCI_EXP_RTSTA_PME);
1588
1589 /*
1590 * Trigger PME interrupt only if PMEIE bit in Root Control is set.
1591 * Aardvark HW returns zero for PCI_EXP_FLAGS_IRQ, so use PCIe interrupt 0.
1592 */
1593 if (!(le16_to_cpu(pcie->bridge.pcie_conf.rootctl) & PCI_EXP_RTCTL_PMEIE))
1594 return;
1595
1596 if (generic_handle_domain_irq(pcie->rp_irq_domain, 0) == -EINVAL)
1597 dev_err_ratelimited(&pcie->pdev->dev, "unhandled PME IRQ\n");
1598 }
1599}
1600
1601static void advk_pcie_handle_msi(struct advk_pcie *pcie)
1602{
1603 u32 msi_val, msi_mask, msi_status, msi_idx;
1604
1605 msi_mask = advk_readl(pcie, PCIE_MSI_MASK_REG);
1606 msi_val = advk_readl(pcie, PCIE_MSI_STATUS_REG);
1607 msi_status = msi_val & ((~msi_mask) & PCIE_MSI_ALL_MASK);
1608
1609 for (msi_idx = 0; msi_idx < MSI_IRQ_NUM; msi_idx++) {
1610 if (!(BIT(msi_idx) & msi_status))
1611 continue;
1612
1613 advk_writel(pcie, BIT(msi_idx), PCIE_MSI_STATUS_REG);
1614 if (generic_handle_domain_irq(pcie->msi_inner_domain, msi_idx) == -EINVAL)
1615 dev_err_ratelimited(&pcie->pdev->dev, "unexpected MSI 0x%02x\n", msi_idx);
1616 }
1617
1618 advk_writel(pcie, PCIE_ISR0_MSI_INT_PENDING,
1619 PCIE_ISR0_REG);
1620}
1621
1622static void advk_pcie_handle_int(struct advk_pcie *pcie)
1623{
1624 u32 isr0_val, isr0_mask, isr0_status;
1625 u32 isr1_val, isr1_mask, isr1_status;
1626 int i;
1627
1628 isr0_val = advk_readl(pcie, PCIE_ISR0_REG);
1629 isr0_mask = advk_readl(pcie, PCIE_ISR0_MASK_REG);
1630 isr0_status = isr0_val & ((~isr0_mask) & PCIE_ISR0_ALL_MASK);
1631
1632 isr1_val = advk_readl(pcie, PCIE_ISR1_REG);
1633 isr1_mask = advk_readl(pcie, PCIE_ISR1_MASK_REG);
1634 isr1_status = isr1_val & ((~isr1_mask) & PCIE_ISR1_ALL_MASK);
1635
1636 /* Process PME interrupt as the first one to do not miss PME requester id */
1637 if (isr0_status & PCIE_MSG_PM_PME_MASK)
1638 advk_pcie_handle_pme(pcie);
1639
1640 /* Process ERR interrupt */
1641 if (isr0_status & PCIE_ISR0_ERR_MASK) {
1642 advk_writel(pcie, PCIE_ISR0_ERR_MASK, PCIE_ISR0_REG);
1643
1644 /*
1645 * Aardvark HW returns zero for PCI_ERR_ROOT_AER_IRQ, so use
1646 * PCIe interrupt 0
1647 */
1648 if (generic_handle_domain_irq(pcie->rp_irq_domain, 0) == -EINVAL)
1649 dev_err_ratelimited(&pcie->pdev->dev, "unhandled ERR IRQ\n");
1650 }
1651
1652 /* Process MSI interrupts */
1653 if (isr0_status & PCIE_ISR0_MSI_INT_PENDING)
1654 advk_pcie_handle_msi(pcie);
1655
1656 /* Process legacy interrupts */
1657 for (i = 0; i < PCI_NUM_INTX; i++) {
1658 if (!(isr1_status & PCIE_ISR1_INTX_ASSERT(i)))
1659 continue;
1660
1661 advk_writel(pcie, PCIE_ISR1_INTX_ASSERT(i),
1662 PCIE_ISR1_REG);
1663
1664 if (generic_handle_domain_irq(pcie->irq_domain, i) == -EINVAL)
1665 dev_err_ratelimited(&pcie->pdev->dev, "unexpected INT%c IRQ\n",
1666 (char)i + 'A');
1667 }
1668}
1669
1670static irqreturn_t advk_pcie_irq_handler(int irq, void *arg)
1671{
1672 struct advk_pcie *pcie = arg;
1673 u32 status;
1674
1675 status = advk_readl(pcie, HOST_CTRL_INT_STATUS_REG);
1676 if (!(status & PCIE_IRQ_CORE_INT))
1677 return IRQ_NONE;
1678
1679 advk_pcie_handle_int(pcie);
1680
1681 /* Clear interrupt */
1682 advk_writel(pcie, PCIE_IRQ_CORE_INT, HOST_CTRL_INT_STATUS_REG);
1683
1684 return IRQ_HANDLED;
1685}
1686
1687static int advk_pcie_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
1688{
1689 struct advk_pcie *pcie = dev->bus->sysdata;
1690
1691 /*
1692 * Emulated root bridge has its own emulated irq chip and irq domain.
1693 * Argument pin is the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD) and
1694 * hwirq for irq_create_mapping() is indexed from zero.
1695 */
1696 if (pci_is_root_bus(dev->bus))
1697 return irq_create_mapping(pcie->rp_irq_domain, pin - 1);
1698 else
1699 return of_irq_parse_and_map_pci(dev, slot, pin);
1700}
1701
1702static void advk_pcie_disable_phy(struct advk_pcie *pcie)
1703{
1704 phy_power_off(pcie->phy);
1705 phy_exit(pcie->phy);
1706}
1707
1708static int advk_pcie_enable_phy(struct advk_pcie *pcie)
1709{
1710 int ret;
1711
1712 if (!pcie->phy)
1713 return 0;
1714
1715 ret = phy_init(pcie->phy);
1716 if (ret)
1717 return ret;
1718
1719 ret = phy_set_mode(pcie->phy, PHY_MODE_PCIE);
1720 if (ret) {
1721 phy_exit(pcie->phy);
1722 return ret;
1723 }
1724
1725 ret = phy_power_on(pcie->phy);
1726 if (ret) {
1727 phy_exit(pcie->phy);
1728 return ret;
1729 }
1730
1731 return 0;
1732}
1733
1734static int advk_pcie_setup_phy(struct advk_pcie *pcie)
1735{
1736 struct device *dev = &pcie->pdev->dev;
1737 struct device_node *node = dev->of_node;
1738 int ret = 0;
1739
1740 pcie->phy = devm_of_phy_get(dev, node, NULL);
1741 if (IS_ERR(pcie->phy) && (PTR_ERR(pcie->phy) == -EPROBE_DEFER))
1742 return PTR_ERR(pcie->phy);
1743
1744 /* Old bindings miss the PHY handle */
1745 if (IS_ERR(pcie->phy)) {
1746 dev_warn(dev, "PHY unavailable (%ld)\n", PTR_ERR(pcie->phy));
1747 pcie->phy = NULL;
1748 return 0;
1749 }
1750
1751 ret = advk_pcie_enable_phy(pcie);
1752 if (ret)
1753 dev_err(dev, "Failed to initialize PHY (%d)\n", ret);
1754
1755 return ret;
1756}
1757
1758static int advk_pcie_probe(struct platform_device *pdev)
1759{
1760 struct device *dev = &pdev->dev;
1761 struct advk_pcie *pcie;
1762 struct pci_host_bridge *bridge;
1763 struct resource_entry *entry;
1764 int ret, irq;
1765
1766 bridge = devm_pci_alloc_host_bridge(dev, sizeof(struct advk_pcie));
1767 if (!bridge)
1768 return -ENOMEM;
1769
1770 pcie = pci_host_bridge_priv(bridge);
1771 pcie->pdev = pdev;
1772 platform_set_drvdata(pdev, pcie);
1773
1774 resource_list_for_each_entry(entry, &bridge->windows) {
1775 resource_size_t start = entry->res->start;
1776 resource_size_t size = resource_size(entry->res);
1777 unsigned long type = resource_type(entry->res);
1778 u64 win_size;
1779
1780 /*
1781 * Aardvark hardware allows to configure also PCIe window
1782 * for config type 0 and type 1 mapping, but driver uses
1783 * only PIO for issuing configuration transfers which does
1784 * not use PCIe window configuration.
1785 */
1786 if (type != IORESOURCE_MEM && type != IORESOURCE_IO)
1787 continue;
1788
1789 /*
1790 * Skip transparent memory resources. Default outbound access
1791 * configuration is set to transparent memory access so it
1792 * does not need window configuration.
1793 */
1794 if (type == IORESOURCE_MEM && entry->offset == 0)
1795 continue;
1796
1797 /*
1798 * The n-th PCIe window is configured by tuple (match, remap, mask)
1799 * and an access to address A uses this window if A matches the
1800 * match with given mask.
1801 * So every PCIe window size must be a power of two and every start
1802 * address must be aligned to window size. Minimal size is 64 KiB
1803 * because lower 16 bits of mask must be zero. Remapped address
1804 * may have set only bits from the mask.
1805 */
1806 while (pcie->wins_count < OB_WIN_COUNT && size > 0) {
1807 /* Calculate the largest aligned window size */
1808 win_size = (1ULL << (fls64(size)-1)) |
1809 (start ? (1ULL << __ffs64(start)) : 0);
1810 win_size = 1ULL << __ffs64(win_size);
1811 if (win_size < 0x10000)
1812 break;
1813
1814 dev_dbg(dev,
1815 "Configuring PCIe window %d: [0x%llx-0x%llx] as %lu\n",
1816 pcie->wins_count, (unsigned long long)start,
1817 (unsigned long long)start + win_size, type);
1818
1819 if (type == IORESOURCE_IO) {
1820 pcie->wins[pcie->wins_count].actions = OB_WIN_TYPE_IO;
1821 pcie->wins[pcie->wins_count].match = pci_pio_to_address(start);
1822 } else {
1823 pcie->wins[pcie->wins_count].actions = OB_WIN_TYPE_MEM;
1824 pcie->wins[pcie->wins_count].match = start;
1825 }
1826 pcie->wins[pcie->wins_count].remap = start - entry->offset;
1827 pcie->wins[pcie->wins_count].mask = ~(win_size - 1);
1828
1829 if (pcie->wins[pcie->wins_count].remap & (win_size - 1))
1830 break;
1831
1832 start += win_size;
1833 size -= win_size;
1834 pcie->wins_count++;
1835 }
1836
1837 if (size > 0) {
1838 dev_err(&pcie->pdev->dev,
1839 "Invalid PCIe region [0x%llx-0x%llx]\n",
1840 (unsigned long long)entry->res->start,
1841 (unsigned long long)entry->res->end + 1);
1842 return -EINVAL;
1843 }
1844 }
1845
1846 pcie->base = devm_platform_ioremap_resource(pdev, 0);
1847 if (IS_ERR(pcie->base))
1848 return PTR_ERR(pcie->base);
1849
1850 irq = platform_get_irq(pdev, 0);
1851 if (irq < 0)
1852 return irq;
1853
1854 ret = devm_request_irq(dev, irq, advk_pcie_irq_handler,
1855 IRQF_SHARED | IRQF_NO_THREAD, "advk-pcie",
1856 pcie);
1857 if (ret) {
1858 dev_err(dev, "Failed to register interrupt\n");
1859 return ret;
1860 }
1861
1862 pcie->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
1863 ret = PTR_ERR_OR_ZERO(pcie->reset_gpio);
1864 if (ret) {
1865 if (ret != -EPROBE_DEFER)
1866 dev_err(dev, "Failed to get reset-gpio: %i\n", ret);
1867 return ret;
1868 }
1869
1870 ret = gpiod_set_consumer_name(pcie->reset_gpio, "pcie1-reset");
1871 if (ret) {
1872 dev_err(dev, "Failed to set reset gpio name: %d\n", ret);
1873 return ret;
1874 }
1875
1876 ret = of_pci_get_max_link_speed(dev->of_node);
1877 if (ret <= 0 || ret > 3)
1878 pcie->link_gen = 3;
1879 else
1880 pcie->link_gen = ret;
1881
1882 ret = advk_pcie_setup_phy(pcie);
1883 if (ret)
1884 return ret;
1885
1886 advk_pcie_setup_hw(pcie);
1887
1888 ret = advk_sw_pci_bridge_init(pcie);
1889 if (ret) {
1890 dev_err(dev, "Failed to register emulated root PCI bridge\n");
1891 return ret;
1892 }
1893
1894 ret = advk_pcie_init_irq_domain(pcie);
1895 if (ret) {
1896 dev_err(dev, "Failed to initialize irq\n");
1897 return ret;
1898 }
1899
1900 ret = advk_pcie_init_msi_irq_domain(pcie);
1901 if (ret) {
1902 dev_err(dev, "Failed to initialize irq\n");
1903 advk_pcie_remove_irq_domain(pcie);
1904 return ret;
1905 }
1906
1907 ret = advk_pcie_init_rp_irq_domain(pcie);
1908 if (ret) {
1909 dev_err(dev, "Failed to initialize irq\n");
1910 advk_pcie_remove_msi_irq_domain(pcie);
1911 advk_pcie_remove_irq_domain(pcie);
1912 return ret;
1913 }
1914
1915 bridge->sysdata = pcie;
1916 bridge->ops = &advk_pcie_ops;
1917 bridge->map_irq = advk_pcie_map_irq;
1918
1919 ret = pci_host_probe(bridge);
1920 if (ret < 0) {
1921 advk_pcie_remove_rp_irq_domain(pcie);
1922 advk_pcie_remove_msi_irq_domain(pcie);
1923 advk_pcie_remove_irq_domain(pcie);
1924 return ret;
1925 }
1926
1927 return 0;
1928}
1929
1930static void advk_pcie_remove(struct platform_device *pdev)
1931{
1932 struct advk_pcie *pcie = platform_get_drvdata(pdev);
1933 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1934 u32 val;
1935 int i;
1936
1937 /* Remove PCI bus with all devices */
1938 pci_lock_rescan_remove();
1939 pci_stop_root_bus(bridge->bus);
1940 pci_remove_root_bus(bridge->bus);
1941 pci_unlock_rescan_remove();
1942
1943 /* Disable Root Bridge I/O space, memory space and bus mastering */
1944 val = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
1945 val &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
1946 advk_writel(pcie, val, PCIE_CORE_CMD_STATUS_REG);
1947
1948 /* Disable MSI */
1949 val = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
1950 val &= ~PCIE_CORE_CTRL2_MSI_ENABLE;
1951 advk_writel(pcie, val, PCIE_CORE_CTRL2_REG);
1952
1953 /* Clear MSI address */
1954 advk_writel(pcie, 0, PCIE_MSI_ADDR_LOW_REG);
1955 advk_writel(pcie, 0, PCIE_MSI_ADDR_HIGH_REG);
1956
1957 /* Mask all interrupts */
1958 advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_MASK_REG);
1959 advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_MASK_REG);
1960 advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG);
1961 advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_MASK_REG);
1962
1963 /* Clear all interrupts */
1964 advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_STATUS_REG);
1965 advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_REG);
1966 advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_REG);
1967 advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_STATUS_REG);
1968
1969 /* Remove IRQ domains */
1970 advk_pcie_remove_rp_irq_domain(pcie);
1971 advk_pcie_remove_msi_irq_domain(pcie);
1972 advk_pcie_remove_irq_domain(pcie);
1973
1974 /* Free config space for emulated root bridge */
1975 pci_bridge_emul_cleanup(&pcie->bridge);
1976
1977 /* Assert PERST# signal which prepares PCIe card for power down */
1978 if (pcie->reset_gpio)
1979 gpiod_set_value_cansleep(pcie->reset_gpio, 1);
1980
1981 /* Disable link training */
1982 val = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
1983 val &= ~LINK_TRAINING_EN;
1984 advk_writel(pcie, val, PCIE_CORE_CTRL0_REG);
1985
1986 /* Disable outbound address windows mapping */
1987 for (i = 0; i < OB_WIN_COUNT; i++)
1988 advk_pcie_disable_ob_win(pcie, i);
1989
1990 /* Disable phy */
1991 advk_pcie_disable_phy(pcie);
1992}
1993
1994static const struct of_device_id advk_pcie_of_match_table[] = {
1995 { .compatible = "marvell,armada-3700-pcie", },
1996 {},
1997};
1998MODULE_DEVICE_TABLE(of, advk_pcie_of_match_table);
1999
2000static struct platform_driver advk_pcie_driver = {
2001 .driver = {
2002 .name = "advk-pcie",
2003 .of_match_table = advk_pcie_of_match_table,
2004 },
2005 .probe = advk_pcie_probe,
2006 .remove_new = advk_pcie_remove,
2007};
2008module_platform_driver(advk_pcie_driver);
2009
2010MODULE_DESCRIPTION("Aardvark PCIe controller");
2011MODULE_LICENSE("GPL v2");