Loading...
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017 Cadence
3// Cadence PCIe host controller driver.
4// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
5
6#include <linux/delay.h>
7#include <linux/kernel.h>
8#include <linux/list_sort.h>
9#include <linux/of_address.h>
10#include <linux/of_pci.h>
11#include <linux/platform_device.h>
12
13#include "pcie-cadence.h"
14
15#define LINK_RETRAIN_TIMEOUT HZ
16
17static u64 bar_max_size[] = {
18 [RP_BAR0] = _ULL(128 * SZ_2G),
19 [RP_BAR1] = SZ_2G,
20 [RP_NO_BAR] = _BITULL(63),
21};
22
23static u8 bar_aperture_mask[] = {
24 [RP_BAR0] = 0x1F,
25 [RP_BAR1] = 0xF,
26};
27
28void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
29 int where)
30{
31 struct pci_host_bridge *bridge = pci_find_host_bridge(bus);
32 struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);
33 struct cdns_pcie *pcie = &rc->pcie;
34 unsigned int busn = bus->number;
35 u32 addr0, desc0;
36
37 if (pci_is_root_bus(bus)) {
38 /*
39 * Only the root port (devfn == 0) is connected to this bus.
40 * All other PCI devices are behind some bridge hence on another
41 * bus.
42 */
43 if (devfn)
44 return NULL;
45
46 return pcie->reg_base + (where & 0xfff);
47 }
48 /* Check that the link is up */
49 if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1))
50 return NULL;
51 /* Clear AXI link-down status */
52 cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0);
53
54 /* Update Output registers for AXI region 0. */
55 addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) |
56 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) |
57 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn);
58 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0);
59
60 /* Configuration Type 0 or Type 1 access. */
61 desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID |
62 CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0);
63 /*
64 * The bus number was already set once for all in desc1 by
65 * cdns_pcie_host_init_address_translation().
66 */
67 if (busn == bridge->busnr + 1)
68 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0;
69 else
70 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1;
71 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0);
72
73 return rc->cfg_base + (where & 0xfff);
74}
75
76static struct pci_ops cdns_pcie_host_ops = {
77 .map_bus = cdns_pci_map_bus,
78 .read = pci_generic_config_read,
79 .write = pci_generic_config_write,
80};
81
82static int cdns_pcie_host_training_complete(struct cdns_pcie *pcie)
83{
84 u32 pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;
85 unsigned long end_jiffies;
86 u16 lnk_stat;
87
88 /* Wait for link training to complete. Exit after timeout. */
89 end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT;
90 do {
91 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);
92 if (!(lnk_stat & PCI_EXP_LNKSTA_LT))
93 break;
94 usleep_range(0, 1000);
95 } while (time_before(jiffies, end_jiffies));
96
97 if (!(lnk_stat & PCI_EXP_LNKSTA_LT))
98 return 0;
99
100 return -ETIMEDOUT;
101}
102
103static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie)
104{
105 struct device *dev = pcie->dev;
106 int retries;
107
108 /* Check if the link is up or not */
109 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
110 if (cdns_pcie_link_up(pcie)) {
111 dev_info(dev, "Link up\n");
112 return 0;
113 }
114 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
115 }
116
117 return -ETIMEDOUT;
118}
119
120static int cdns_pcie_retrain(struct cdns_pcie *pcie)
121{
122 u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;
123 u16 lnk_stat, lnk_ctl;
124 int ret = 0;
125
126 /*
127 * Set retrain bit if current speed is 2.5 GB/s,
128 * but the PCIe root port support is > 2.5 GB/s.
129 */
130
131 lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off +
132 PCI_EXP_LNKCAP));
133 if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
134 return ret;
135
136 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);
137 if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {
138 lnk_ctl = cdns_pcie_rp_readw(pcie,
139 pcie_cap_off + PCI_EXP_LNKCTL);
140 lnk_ctl |= PCI_EXP_LNKCTL_RL;
141 cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL,
142 lnk_ctl);
143
144 ret = cdns_pcie_host_training_complete(pcie);
145 if (ret)
146 return ret;
147
148 ret = cdns_pcie_host_wait_for_link(pcie);
149 }
150 return ret;
151}
152
153static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie)
154{
155 u32 val;
156
157 val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL);
158 cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN);
159}
160
161static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc)
162{
163 struct cdns_pcie *pcie = &rc->pcie;
164 int ret;
165
166 ret = cdns_pcie_host_wait_for_link(pcie);
167
168 /*
169 * Retrain link for Gen2 training defect
170 * if quirk flag is set.
171 */
172 if (!ret && rc->quirk_retrain_flag)
173 ret = cdns_pcie_retrain(pcie);
174
175 return ret;
176}
177
178static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc)
179{
180 struct cdns_pcie *pcie = &rc->pcie;
181 u32 value, ctrl;
182 u32 id;
183
184 /*
185 * Set the root complex BAR configuration register:
186 * - disable both BAR0 and BAR1.
187 * - enable Prefetchable Memory Base and Limit registers in type 1
188 * config space (64 bits).
189 * - enable IO Base and Limit registers in type 1 config
190 * space (32 bits).
191 */
192 ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
193 value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) |
194 CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) |
195 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE |
196 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS |
197 CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE |
198 CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS;
199 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
200
201 /* Set root port configuration space */
202 if (rc->vendor_id != 0xffff) {
203 id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |
204 CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);
205 cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);
206 }
207
208 if (rc->device_id != 0xffff)
209 cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);
210
211 cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0);
212 cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0);
213 cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
214
215 return 0;
216}
217
218static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc,
219 enum cdns_pcie_rp_bar bar,
220 u64 cpu_addr, u64 size,
221 unsigned long flags)
222{
223 struct cdns_pcie *pcie = &rc->pcie;
224 u32 addr0, addr1, aperture, value;
225
226 if (!rc->avail_ib_bar[bar])
227 return -EBUSY;
228
229 rc->avail_ib_bar[bar] = false;
230
231 aperture = ilog2(size);
232 addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) |
233 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
234 addr1 = upper_32_bits(cpu_addr);
235 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0);
236 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1);
237
238 if (bar == RP_NO_BAR)
239 return 0;
240
241 value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG);
242 value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) |
243 LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) |
244 LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) |
245 LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) |
246 LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2));
247 if (size + cpu_addr >= SZ_4G) {
248 if (!(flags & IORESOURCE_PREFETCH))
249 value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar);
250 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar);
251 } else {
252 if (!(flags & IORESOURCE_PREFETCH))
253 value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar);
254 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar);
255 }
256
257 value |= LM_RC_BAR_CFG_APERTURE(bar, aperture);
258 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
259
260 return 0;
261}
262
263static enum cdns_pcie_rp_bar
264cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size)
265{
266 enum cdns_pcie_rp_bar bar, sel_bar;
267
268 sel_bar = RP_BAR_UNDEFINED;
269 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
270 if (!rc->avail_ib_bar[bar])
271 continue;
272
273 if (size <= bar_max_size[bar]) {
274 if (sel_bar == RP_BAR_UNDEFINED) {
275 sel_bar = bar;
276 continue;
277 }
278
279 if (bar_max_size[bar] < bar_max_size[sel_bar])
280 sel_bar = bar;
281 }
282 }
283
284 return sel_bar;
285}
286
287static enum cdns_pcie_rp_bar
288cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size)
289{
290 enum cdns_pcie_rp_bar bar, sel_bar;
291
292 sel_bar = RP_BAR_UNDEFINED;
293 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
294 if (!rc->avail_ib_bar[bar])
295 continue;
296
297 if (size >= bar_max_size[bar]) {
298 if (sel_bar == RP_BAR_UNDEFINED) {
299 sel_bar = bar;
300 continue;
301 }
302
303 if (bar_max_size[bar] > bar_max_size[sel_bar])
304 sel_bar = bar;
305 }
306 }
307
308 return sel_bar;
309}
310
311static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc,
312 struct resource_entry *entry)
313{
314 u64 cpu_addr, pci_addr, size, winsize;
315 struct cdns_pcie *pcie = &rc->pcie;
316 struct device *dev = pcie->dev;
317 enum cdns_pcie_rp_bar bar;
318 unsigned long flags;
319 int ret;
320
321 cpu_addr = entry->res->start;
322 pci_addr = entry->res->start - entry->offset;
323 flags = entry->res->flags;
324 size = resource_size(entry->res);
325
326 if (entry->offset) {
327 dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n",
328 pci_addr, cpu_addr);
329 return -EINVAL;
330 }
331
332 while (size > 0) {
333 /*
334 * Try to find a minimum BAR whose size is greater than
335 * or equal to the remaining resource_entry size. This will
336 * fail if the size of each of the available BARs is less than
337 * the remaining resource_entry size.
338 * If a minimum BAR is found, IB ATU will be configured and
339 * exited.
340 */
341 bar = cdns_pcie_host_find_min_bar(rc, size);
342 if (bar != RP_BAR_UNDEFINED) {
343 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr,
344 size, flags);
345 if (ret)
346 dev_err(dev, "IB BAR: %d config failed\n", bar);
347 return ret;
348 }
349
350 /*
351 * If the control reaches here, it would mean the remaining
352 * resource_entry size cannot be fitted in a single BAR. So we
353 * find a maximum BAR whose size is less than or equal to the
354 * remaining resource_entry size and split the resource entry
355 * so that part of resource entry is fitted inside the maximum
356 * BAR. The remaining size would be fitted during the next
357 * iteration of the loop.
358 * If a maximum BAR is not found, there is no way we can fit
359 * this resource_entry, so we error out.
360 */
361 bar = cdns_pcie_host_find_max_bar(rc, size);
362 if (bar == RP_BAR_UNDEFINED) {
363 dev_err(dev, "No free BAR to map cpu_addr %llx\n",
364 cpu_addr);
365 return -EINVAL;
366 }
367
368 winsize = bar_max_size[bar];
369 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize,
370 flags);
371 if (ret) {
372 dev_err(dev, "IB BAR: %d config failed\n", bar);
373 return ret;
374 }
375
376 size -= winsize;
377 cpu_addr += winsize;
378 }
379
380 return 0;
381}
382
383static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a,
384 const struct list_head *b)
385{
386 struct resource_entry *entry1, *entry2;
387
388 entry1 = container_of(a, struct resource_entry, node);
389 entry2 = container_of(b, struct resource_entry, node);
390
391 return resource_size(entry2->res) - resource_size(entry1->res);
392}
393
394static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc)
395{
396 struct cdns_pcie *pcie = &rc->pcie;
397 struct device *dev = pcie->dev;
398 struct device_node *np = dev->of_node;
399 struct pci_host_bridge *bridge;
400 struct resource_entry *entry;
401 u32 no_bar_nbits = 32;
402 int err;
403
404 bridge = pci_host_bridge_from_priv(rc);
405 if (!bridge)
406 return -ENOMEM;
407
408 if (list_empty(&bridge->dma_ranges)) {
409 of_property_read_u32(np, "cdns,no-bar-match-nbits",
410 &no_bar_nbits);
411 err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0,
412 (u64)1 << no_bar_nbits, 0);
413 if (err)
414 dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR);
415 return err;
416 }
417
418 list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp);
419
420 resource_list_for_each_entry(entry, &bridge->dma_ranges) {
421 err = cdns_pcie_host_bar_config(rc, entry);
422 if (err) {
423 dev_err(dev, "Fail to configure IB using dma-ranges\n");
424 return err;
425 }
426 }
427
428 return 0;
429}
430
431static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc)
432{
433 struct cdns_pcie *pcie = &rc->pcie;
434 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc);
435 struct resource *cfg_res = rc->cfg_res;
436 struct resource_entry *entry;
437 u64 cpu_addr = cfg_res->start;
438 u32 addr0, addr1, desc1;
439 int r, busnr = 0;
440
441 entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS);
442 if (entry)
443 busnr = entry->res->start;
444
445 /*
446 * Reserve region 0 for PCI configure space accesses:
447 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by
448 * cdns_pci_map_bus(), other region registers are set here once for all.
449 */
450 addr1 = 0; /* Should be programmed to zero. */
451 desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr);
452 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);
453 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);
454
455 if (pcie->ops->cpu_addr_fixup)
456 cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr);
457
458 addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) |
459 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
460 addr1 = upper_32_bits(cpu_addr);
461 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0);
462 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1);
463
464 r = 1;
465 resource_list_for_each_entry(entry, &bridge->windows) {
466 struct resource *res = entry->res;
467 u64 pci_addr = res->start - entry->offset;
468
469 if (resource_type(res) == IORESOURCE_IO)
470 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
471 true,
472 pci_pio_to_address(res->start),
473 pci_addr,
474 resource_size(res));
475 else
476 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
477 false,
478 res->start,
479 pci_addr,
480 resource_size(res));
481
482 r++;
483 }
484
485 return cdns_pcie_host_map_dma_ranges(rc);
486}
487
488int cdns_pcie_host_init(struct cdns_pcie_rc *rc)
489{
490 int err;
491
492 err = cdns_pcie_host_init_root_port(rc);
493 if (err)
494 return err;
495
496 return cdns_pcie_host_init_address_translation(rc);
497}
498
499int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc)
500{
501 struct cdns_pcie *pcie = &rc->pcie;
502 struct device *dev = rc->pcie.dev;
503 int ret;
504
505 if (rc->quirk_detect_quiet_flag)
506 cdns_pcie_detect_quiet_min_delay_set(&rc->pcie);
507
508 cdns_pcie_host_enable_ptm_response(pcie);
509
510 ret = cdns_pcie_start_link(pcie);
511 if (ret) {
512 dev_err(dev, "Failed to start link\n");
513 return ret;
514 }
515
516 ret = cdns_pcie_host_start_link(rc);
517 if (ret)
518 dev_dbg(dev, "PCIe link never came up\n");
519
520 return 0;
521}
522
523int cdns_pcie_host_setup(struct cdns_pcie_rc *rc)
524{
525 struct device *dev = rc->pcie.dev;
526 struct platform_device *pdev = to_platform_device(dev);
527 struct device_node *np = dev->of_node;
528 struct pci_host_bridge *bridge;
529 enum cdns_pcie_rp_bar bar;
530 struct cdns_pcie *pcie;
531 struct resource *res;
532 int ret;
533
534 bridge = pci_host_bridge_from_priv(rc);
535 if (!bridge)
536 return -ENOMEM;
537
538 pcie = &rc->pcie;
539 pcie->is_rc = true;
540
541 rc->vendor_id = 0xffff;
542 of_property_read_u32(np, "vendor-id", &rc->vendor_id);
543
544 rc->device_id = 0xffff;
545 of_property_read_u32(np, "device-id", &rc->device_id);
546
547 pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");
548 if (IS_ERR(pcie->reg_base)) {
549 dev_err(dev, "missing \"reg\"\n");
550 return PTR_ERR(pcie->reg_base);
551 }
552
553 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");
554 rc->cfg_base = devm_pci_remap_cfg_resource(dev, res);
555 if (IS_ERR(rc->cfg_base))
556 return PTR_ERR(rc->cfg_base);
557 rc->cfg_res = res;
558
559 ret = cdns_pcie_host_link_setup(rc);
560 if (ret)
561 return ret;
562
563 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++)
564 rc->avail_ib_bar[bar] = true;
565
566 ret = cdns_pcie_host_init(rc);
567 if (ret)
568 return ret;
569
570 if (!bridge->ops)
571 bridge->ops = &cdns_pcie_host_ops;
572
573 ret = pci_host_probe(bridge);
574 if (ret < 0)
575 goto err_init;
576
577 return 0;
578
579 err_init:
580 pm_runtime_put_sync(dev);
581
582 return ret;
583}
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017 Cadence
3// Cadence PCIe host controller driver.
4// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
5
6#include <linux/delay.h>
7#include <linux/kernel.h>
8#include <linux/list_sort.h>
9#include <linux/of_address.h>
10#include <linux/of_pci.h>
11#include <linux/platform_device.h>
12
13#include "pcie-cadence.h"
14
15static u64 bar_max_size[] = {
16 [RP_BAR0] = _ULL(128 * SZ_2G),
17 [RP_BAR1] = SZ_2G,
18 [RP_NO_BAR] = _BITULL(63),
19};
20
21static u8 bar_aperture_mask[] = {
22 [RP_BAR0] = 0x1F,
23 [RP_BAR1] = 0xF,
24};
25
26void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
27 int where)
28{
29 struct pci_host_bridge *bridge = pci_find_host_bridge(bus);
30 struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);
31 struct cdns_pcie *pcie = &rc->pcie;
32 unsigned int busn = bus->number;
33 u32 addr0, desc0;
34
35 if (pci_is_root_bus(bus)) {
36 /*
37 * Only the root port (devfn == 0) is connected to this bus.
38 * All other PCI devices are behind some bridge hence on another
39 * bus.
40 */
41 if (devfn)
42 return NULL;
43
44 return pcie->reg_base + (where & 0xfff);
45 }
46 /* Check that the link is up */
47 if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1))
48 return NULL;
49 /* Clear AXI link-down status */
50 cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0);
51
52 /* Update Output registers for AXI region 0. */
53 addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) |
54 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) |
55 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn);
56 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0);
57
58 /* Configuration Type 0 or Type 1 access. */
59 desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID |
60 CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0);
61 /*
62 * The bus number was already set once for all in desc1 by
63 * cdns_pcie_host_init_address_translation().
64 */
65 if (busn == bridge->busnr + 1)
66 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0;
67 else
68 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1;
69 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0);
70
71 return rc->cfg_base + (where & 0xfff);
72}
73
74static struct pci_ops cdns_pcie_host_ops = {
75 .map_bus = cdns_pci_map_bus,
76 .read = pci_generic_config_read,
77 .write = pci_generic_config_write,
78};
79
80static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie)
81{
82 struct device *dev = pcie->dev;
83 int retries;
84
85 /* Check if the link is up or not */
86 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
87 if (cdns_pcie_link_up(pcie)) {
88 dev_info(dev, "Link up\n");
89 return 0;
90 }
91 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
92 }
93
94 return -ETIMEDOUT;
95}
96
97static int cdns_pcie_retrain(struct cdns_pcie *pcie)
98{
99 u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;
100 u16 lnk_stat, lnk_ctl;
101 int ret = 0;
102
103 /*
104 * Set retrain bit if current speed is 2.5 GB/s,
105 * but the PCIe root port support is > 2.5 GB/s.
106 */
107
108 lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off +
109 PCI_EXP_LNKCAP));
110 if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
111 return ret;
112
113 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);
114 if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {
115 lnk_ctl = cdns_pcie_rp_readw(pcie,
116 pcie_cap_off + PCI_EXP_LNKCTL);
117 lnk_ctl |= PCI_EXP_LNKCTL_RL;
118 cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL,
119 lnk_ctl);
120
121 ret = cdns_pcie_host_wait_for_link(pcie);
122 }
123 return ret;
124}
125
126static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc)
127{
128 struct cdns_pcie *pcie = &rc->pcie;
129 int ret;
130
131 ret = cdns_pcie_host_wait_for_link(pcie);
132
133 /*
134 * Retrain link for Gen2 training defect
135 * if quirk flag is set.
136 */
137 if (!ret && rc->quirk_retrain_flag)
138 ret = cdns_pcie_retrain(pcie);
139
140 return ret;
141}
142
143static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc)
144{
145 struct cdns_pcie *pcie = &rc->pcie;
146 u32 value, ctrl;
147 u32 id;
148
149 /*
150 * Set the root complex BAR configuration register:
151 * - disable both BAR0 and BAR1.
152 * - enable Prefetchable Memory Base and Limit registers in type 1
153 * config space (64 bits).
154 * - enable IO Base and Limit registers in type 1 config
155 * space (32 bits).
156 */
157 ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
158 value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) |
159 CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) |
160 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE |
161 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS |
162 CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE |
163 CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS;
164 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
165
166 /* Set root port configuration space */
167 if (rc->vendor_id != 0xffff) {
168 id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |
169 CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);
170 cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);
171 }
172
173 if (rc->device_id != 0xffff)
174 cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);
175
176 cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0);
177 cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0);
178 cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
179
180 return 0;
181}
182
183static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc,
184 enum cdns_pcie_rp_bar bar,
185 u64 cpu_addr, u64 size,
186 unsigned long flags)
187{
188 struct cdns_pcie *pcie = &rc->pcie;
189 u32 addr0, addr1, aperture, value;
190
191 if (!rc->avail_ib_bar[bar])
192 return -EBUSY;
193
194 rc->avail_ib_bar[bar] = false;
195
196 aperture = ilog2(size);
197 addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) |
198 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
199 addr1 = upper_32_bits(cpu_addr);
200 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0);
201 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1);
202
203 if (bar == RP_NO_BAR)
204 return 0;
205
206 value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG);
207 value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) |
208 LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) |
209 LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) |
210 LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) |
211 LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2));
212 if (size + cpu_addr >= SZ_4G) {
213 if (!(flags & IORESOURCE_PREFETCH))
214 value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar);
215 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar);
216 } else {
217 if (!(flags & IORESOURCE_PREFETCH))
218 value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar);
219 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar);
220 }
221
222 value |= LM_RC_BAR_CFG_APERTURE(bar, aperture);
223 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
224
225 return 0;
226}
227
228static enum cdns_pcie_rp_bar
229cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size)
230{
231 enum cdns_pcie_rp_bar bar, sel_bar;
232
233 sel_bar = RP_BAR_UNDEFINED;
234 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
235 if (!rc->avail_ib_bar[bar])
236 continue;
237
238 if (size <= bar_max_size[bar]) {
239 if (sel_bar == RP_BAR_UNDEFINED) {
240 sel_bar = bar;
241 continue;
242 }
243
244 if (bar_max_size[bar] < bar_max_size[sel_bar])
245 sel_bar = bar;
246 }
247 }
248
249 return sel_bar;
250}
251
252static enum cdns_pcie_rp_bar
253cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size)
254{
255 enum cdns_pcie_rp_bar bar, sel_bar;
256
257 sel_bar = RP_BAR_UNDEFINED;
258 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
259 if (!rc->avail_ib_bar[bar])
260 continue;
261
262 if (size >= bar_max_size[bar]) {
263 if (sel_bar == RP_BAR_UNDEFINED) {
264 sel_bar = bar;
265 continue;
266 }
267
268 if (bar_max_size[bar] > bar_max_size[sel_bar])
269 sel_bar = bar;
270 }
271 }
272
273 return sel_bar;
274}
275
276static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc,
277 struct resource_entry *entry)
278{
279 u64 cpu_addr, pci_addr, size, winsize;
280 struct cdns_pcie *pcie = &rc->pcie;
281 struct device *dev = pcie->dev;
282 enum cdns_pcie_rp_bar bar;
283 unsigned long flags;
284 int ret;
285
286 cpu_addr = entry->res->start;
287 pci_addr = entry->res->start - entry->offset;
288 flags = entry->res->flags;
289 size = resource_size(entry->res);
290
291 if (entry->offset) {
292 dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n",
293 pci_addr, cpu_addr);
294 return -EINVAL;
295 }
296
297 while (size > 0) {
298 /*
299 * Try to find a minimum BAR whose size is greater than
300 * or equal to the remaining resource_entry size. This will
301 * fail if the size of each of the available BARs is less than
302 * the remaining resource_entry size.
303 * If a minimum BAR is found, IB ATU will be configured and
304 * exited.
305 */
306 bar = cdns_pcie_host_find_min_bar(rc, size);
307 if (bar != RP_BAR_UNDEFINED) {
308 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr,
309 size, flags);
310 if (ret)
311 dev_err(dev, "IB BAR: %d config failed\n", bar);
312 return ret;
313 }
314
315 /*
316 * If the control reaches here, it would mean the remaining
317 * resource_entry size cannot be fitted in a single BAR. So we
318 * find a maximum BAR whose size is less than or equal to the
319 * remaining resource_entry size and split the resource entry
320 * so that part of resource entry is fitted inside the maximum
321 * BAR. The remaining size would be fitted during the next
322 * iteration of the loop.
323 * If a maximum BAR is not found, there is no way we can fit
324 * this resource_entry, so we error out.
325 */
326 bar = cdns_pcie_host_find_max_bar(rc, size);
327 if (bar == RP_BAR_UNDEFINED) {
328 dev_err(dev, "No free BAR to map cpu_addr %llx\n",
329 cpu_addr);
330 return -EINVAL;
331 }
332
333 winsize = bar_max_size[bar];
334 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize,
335 flags);
336 if (ret) {
337 dev_err(dev, "IB BAR: %d config failed\n", bar);
338 return ret;
339 }
340
341 size -= winsize;
342 cpu_addr += winsize;
343 }
344
345 return 0;
346}
347
348static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a,
349 const struct list_head *b)
350{
351 struct resource_entry *entry1, *entry2;
352
353 entry1 = container_of(a, struct resource_entry, node);
354 entry2 = container_of(b, struct resource_entry, node);
355
356 return resource_size(entry2->res) - resource_size(entry1->res);
357}
358
359static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc)
360{
361 struct cdns_pcie *pcie = &rc->pcie;
362 struct device *dev = pcie->dev;
363 struct device_node *np = dev->of_node;
364 struct pci_host_bridge *bridge;
365 struct resource_entry *entry;
366 u32 no_bar_nbits = 32;
367 int err;
368
369 bridge = pci_host_bridge_from_priv(rc);
370 if (!bridge)
371 return -ENOMEM;
372
373 if (list_empty(&bridge->dma_ranges)) {
374 of_property_read_u32(np, "cdns,no-bar-match-nbits",
375 &no_bar_nbits);
376 err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0,
377 (u64)1 << no_bar_nbits, 0);
378 if (err)
379 dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR);
380 return err;
381 }
382
383 list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp);
384
385 resource_list_for_each_entry(entry, &bridge->dma_ranges) {
386 err = cdns_pcie_host_bar_config(rc, entry);
387 if (err) {
388 dev_err(dev, "Fail to configure IB using dma-ranges\n");
389 return err;
390 }
391 }
392
393 return 0;
394}
395
396static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc)
397{
398 struct cdns_pcie *pcie = &rc->pcie;
399 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc);
400 struct resource *cfg_res = rc->cfg_res;
401 struct resource_entry *entry;
402 u64 cpu_addr = cfg_res->start;
403 u32 addr0, addr1, desc1;
404 int r, busnr = 0;
405
406 entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS);
407 if (entry)
408 busnr = entry->res->start;
409
410 /*
411 * Reserve region 0 for PCI configure space accesses:
412 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by
413 * cdns_pci_map_bus(), other region registers are set here once for all.
414 */
415 addr1 = 0; /* Should be programmed to zero. */
416 desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr);
417 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);
418 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);
419
420 if (pcie->ops->cpu_addr_fixup)
421 cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr);
422
423 addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) |
424 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
425 addr1 = upper_32_bits(cpu_addr);
426 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0);
427 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1);
428
429 r = 1;
430 resource_list_for_each_entry(entry, &bridge->windows) {
431 struct resource *res = entry->res;
432 u64 pci_addr = res->start - entry->offset;
433
434 if (resource_type(res) == IORESOURCE_IO)
435 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
436 true,
437 pci_pio_to_address(res->start),
438 pci_addr,
439 resource_size(res));
440 else
441 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
442 false,
443 res->start,
444 pci_addr,
445 resource_size(res));
446
447 r++;
448 }
449
450 return cdns_pcie_host_map_dma_ranges(rc);
451}
452
453static int cdns_pcie_host_init(struct device *dev,
454 struct cdns_pcie_rc *rc)
455{
456 int err;
457
458 err = cdns_pcie_host_init_root_port(rc);
459 if (err)
460 return err;
461
462 return cdns_pcie_host_init_address_translation(rc);
463}
464
465int cdns_pcie_host_setup(struct cdns_pcie_rc *rc)
466{
467 struct device *dev = rc->pcie.dev;
468 struct platform_device *pdev = to_platform_device(dev);
469 struct device_node *np = dev->of_node;
470 struct pci_host_bridge *bridge;
471 enum cdns_pcie_rp_bar bar;
472 struct cdns_pcie *pcie;
473 struct resource *res;
474 int ret;
475
476 bridge = pci_host_bridge_from_priv(rc);
477 if (!bridge)
478 return -ENOMEM;
479
480 pcie = &rc->pcie;
481 pcie->is_rc = true;
482
483 rc->vendor_id = 0xffff;
484 of_property_read_u32(np, "vendor-id", &rc->vendor_id);
485
486 rc->device_id = 0xffff;
487 of_property_read_u32(np, "device-id", &rc->device_id);
488
489 pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");
490 if (IS_ERR(pcie->reg_base)) {
491 dev_err(dev, "missing \"reg\"\n");
492 return PTR_ERR(pcie->reg_base);
493 }
494
495 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");
496 rc->cfg_base = devm_pci_remap_cfg_resource(dev, res);
497 if (IS_ERR(rc->cfg_base))
498 return PTR_ERR(rc->cfg_base);
499 rc->cfg_res = res;
500
501 if (rc->quirk_detect_quiet_flag)
502 cdns_pcie_detect_quiet_min_delay_set(&rc->pcie);
503
504 ret = cdns_pcie_start_link(pcie);
505 if (ret) {
506 dev_err(dev, "Failed to start link\n");
507 return ret;
508 }
509
510 ret = cdns_pcie_host_start_link(rc);
511 if (ret)
512 dev_dbg(dev, "PCIe link never came up\n");
513
514 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++)
515 rc->avail_ib_bar[bar] = true;
516
517 ret = cdns_pcie_host_init(dev, rc);
518 if (ret)
519 return ret;
520
521 if (!bridge->ops)
522 bridge->ops = &cdns_pcie_host_ops;
523
524 ret = pci_host_probe(bridge);
525 if (ret < 0)
526 goto err_init;
527
528 return 0;
529
530 err_init:
531 pm_runtime_put_sync(dev);
532
533 return ret;
534}