Loading...
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017 Cadence
3// Cadence PCIe host controller driver.
4// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
5
6#include <linux/delay.h>
7#include <linux/kernel.h>
8#include <linux/list_sort.h>
9#include <linux/of_address.h>
10#include <linux/of_pci.h>
11#include <linux/platform_device.h>
12
13#include "pcie-cadence.h"
14
15static u64 bar_max_size[] = {
16 [RP_BAR0] = _ULL(128 * SZ_2G),
17 [RP_BAR1] = SZ_2G,
18 [RP_NO_BAR] = _BITULL(63),
19};
20
21static u8 bar_aperture_mask[] = {
22 [RP_BAR0] = 0x1F,
23 [RP_BAR1] = 0xF,
24};
25
26void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
27 int where)
28{
29 struct pci_host_bridge *bridge = pci_find_host_bridge(bus);
30 struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);
31 struct cdns_pcie *pcie = &rc->pcie;
32 unsigned int busn = bus->number;
33 u32 addr0, desc0;
34
35 if (pci_is_root_bus(bus)) {
36 /*
37 * Only the root port (devfn == 0) is connected to this bus.
38 * All other PCI devices are behind some bridge hence on another
39 * bus.
40 */
41 if (devfn)
42 return NULL;
43
44 return pcie->reg_base + (where & 0xfff);
45 }
46 /* Check that the link is up */
47 if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1))
48 return NULL;
49 /* Clear AXI link-down status */
50 cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0);
51
52 /* Update Output registers for AXI region 0. */
53 addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) |
54 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) |
55 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn);
56 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0);
57
58 /* Configuration Type 0 or Type 1 access. */
59 desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID |
60 CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0);
61 /*
62 * The bus number was already set once for all in desc1 by
63 * cdns_pcie_host_init_address_translation().
64 */
65 if (busn == bridge->busnr + 1)
66 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0;
67 else
68 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1;
69 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0);
70
71 return rc->cfg_base + (where & 0xfff);
72}
73
74static struct pci_ops cdns_pcie_host_ops = {
75 .map_bus = cdns_pci_map_bus,
76 .read = pci_generic_config_read,
77 .write = pci_generic_config_write,
78};
79
80
81static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc)
82{
83 struct cdns_pcie *pcie = &rc->pcie;
84 u32 value, ctrl;
85 u32 id;
86
87 /*
88 * Set the root complex BAR configuration register:
89 * - disable both BAR0 and BAR1.
90 * - enable Prefetchable Memory Base and Limit registers in type 1
91 * config space (64 bits).
92 * - enable IO Base and Limit registers in type 1 config
93 * space (32 bits).
94 */
95 ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
96 value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) |
97 CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) |
98 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE |
99 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS |
100 CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE |
101 CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS;
102 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
103
104 /* Set root port configuration space */
105 if (rc->vendor_id != 0xffff) {
106 id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |
107 CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);
108 cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);
109 }
110
111 if (rc->device_id != 0xffff)
112 cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);
113
114 cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0);
115 cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0);
116 cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
117
118 return 0;
119}
120
121static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc,
122 enum cdns_pcie_rp_bar bar,
123 u64 cpu_addr, u64 size,
124 unsigned long flags)
125{
126 struct cdns_pcie *pcie = &rc->pcie;
127 u32 addr0, addr1, aperture, value;
128
129 if (!rc->avail_ib_bar[bar])
130 return -EBUSY;
131
132 rc->avail_ib_bar[bar] = false;
133
134 aperture = ilog2(size);
135 addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) |
136 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
137 addr1 = upper_32_bits(cpu_addr);
138 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0);
139 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1);
140
141 if (bar == RP_NO_BAR)
142 return 0;
143
144 value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG);
145 value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) |
146 LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) |
147 LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) |
148 LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) |
149 LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2));
150 if (size + cpu_addr >= SZ_4G) {
151 if (!(flags & IORESOURCE_PREFETCH))
152 value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar);
153 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar);
154 } else {
155 if (!(flags & IORESOURCE_PREFETCH))
156 value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar);
157 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar);
158 }
159
160 value |= LM_RC_BAR_CFG_APERTURE(bar, aperture);
161 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
162
163 return 0;
164}
165
166static enum cdns_pcie_rp_bar
167cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size)
168{
169 enum cdns_pcie_rp_bar bar, sel_bar;
170
171 sel_bar = RP_BAR_UNDEFINED;
172 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
173 if (!rc->avail_ib_bar[bar])
174 continue;
175
176 if (size <= bar_max_size[bar]) {
177 if (sel_bar == RP_BAR_UNDEFINED) {
178 sel_bar = bar;
179 continue;
180 }
181
182 if (bar_max_size[bar] < bar_max_size[sel_bar])
183 sel_bar = bar;
184 }
185 }
186
187 return sel_bar;
188}
189
190static enum cdns_pcie_rp_bar
191cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size)
192{
193 enum cdns_pcie_rp_bar bar, sel_bar;
194
195 sel_bar = RP_BAR_UNDEFINED;
196 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
197 if (!rc->avail_ib_bar[bar])
198 continue;
199
200 if (size >= bar_max_size[bar]) {
201 if (sel_bar == RP_BAR_UNDEFINED) {
202 sel_bar = bar;
203 continue;
204 }
205
206 if (bar_max_size[bar] > bar_max_size[sel_bar])
207 sel_bar = bar;
208 }
209 }
210
211 return sel_bar;
212}
213
214static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc,
215 struct resource_entry *entry)
216{
217 u64 cpu_addr, pci_addr, size, winsize;
218 struct cdns_pcie *pcie = &rc->pcie;
219 struct device *dev = pcie->dev;
220 enum cdns_pcie_rp_bar bar;
221 unsigned long flags;
222 int ret;
223
224 cpu_addr = entry->res->start;
225 pci_addr = entry->res->start - entry->offset;
226 flags = entry->res->flags;
227 size = resource_size(entry->res);
228
229 if (entry->offset) {
230 dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n",
231 pci_addr, cpu_addr);
232 return -EINVAL;
233 }
234
235 while (size > 0) {
236 /*
237 * Try to find a minimum BAR whose size is greater than
238 * or equal to the remaining resource_entry size. This will
239 * fail if the size of each of the available BARs is less than
240 * the remaining resource_entry size.
241 * If a minimum BAR is found, IB ATU will be configured and
242 * exited.
243 */
244 bar = cdns_pcie_host_find_min_bar(rc, size);
245 if (bar != RP_BAR_UNDEFINED) {
246 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr,
247 size, flags);
248 if (ret)
249 dev_err(dev, "IB BAR: %d config failed\n", bar);
250 return ret;
251 }
252
253 /*
254 * If the control reaches here, it would mean the remaining
255 * resource_entry size cannot be fitted in a single BAR. So we
256 * find a maximum BAR whose size is less than or equal to the
257 * remaining resource_entry size and split the resource entry
258 * so that part of resource entry is fitted inside the maximum
259 * BAR. The remaining size would be fitted during the next
260 * iteration of the loop.
261 * If a maximum BAR is not found, there is no way we can fit
262 * this resource_entry, so we error out.
263 */
264 bar = cdns_pcie_host_find_max_bar(rc, size);
265 if (bar == RP_BAR_UNDEFINED) {
266 dev_err(dev, "No free BAR to map cpu_addr %llx\n",
267 cpu_addr);
268 return -EINVAL;
269 }
270
271 winsize = bar_max_size[bar];
272 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize,
273 flags);
274 if (ret) {
275 dev_err(dev, "IB BAR: %d config failed\n", bar);
276 return ret;
277 }
278
279 size -= winsize;
280 cpu_addr += winsize;
281 }
282
283 return 0;
284}
285
286static int cdns_pcie_host_dma_ranges_cmp(void *priv, struct list_head *a, struct list_head *b)
287{
288 struct resource_entry *entry1, *entry2;
289
290 entry1 = container_of(a, struct resource_entry, node);
291 entry2 = container_of(b, struct resource_entry, node);
292
293 return resource_size(entry2->res) - resource_size(entry1->res);
294}
295
296static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc)
297{
298 struct cdns_pcie *pcie = &rc->pcie;
299 struct device *dev = pcie->dev;
300 struct device_node *np = dev->of_node;
301 struct pci_host_bridge *bridge;
302 struct resource_entry *entry;
303 u32 no_bar_nbits = 32;
304 int err;
305
306 bridge = pci_host_bridge_from_priv(rc);
307 if (!bridge)
308 return -ENOMEM;
309
310 if (list_empty(&bridge->dma_ranges)) {
311 of_property_read_u32(np, "cdns,no-bar-match-nbits",
312 &no_bar_nbits);
313 err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0,
314 (u64)1 << no_bar_nbits, 0);
315 if (err)
316 dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR);
317 return err;
318 }
319
320 list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp);
321
322 resource_list_for_each_entry(entry, &bridge->dma_ranges) {
323 err = cdns_pcie_host_bar_config(rc, entry);
324 if (err)
325 dev_err(dev, "Fail to configure IB using dma-ranges\n");
326 return err;
327 }
328
329 return 0;
330}
331
332static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc)
333{
334 struct cdns_pcie *pcie = &rc->pcie;
335 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc);
336 struct resource *cfg_res = rc->cfg_res;
337 struct resource_entry *entry;
338 u64 cpu_addr = cfg_res->start;
339 u32 addr0, addr1, desc1;
340 int r, err, busnr = 0;
341
342 entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS);
343 if (entry)
344 busnr = entry->res->start;
345
346 /*
347 * Reserve region 0 for PCI configure space accesses:
348 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by
349 * cdns_pci_map_bus(), other region registers are set here once for all.
350 */
351 addr1 = 0; /* Should be programmed to zero. */
352 desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr);
353 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);
354 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);
355
356 if (pcie->ops->cpu_addr_fixup)
357 cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr);
358
359 addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) |
360 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
361 addr1 = upper_32_bits(cpu_addr);
362 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0);
363 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1);
364
365 r = 1;
366 resource_list_for_each_entry(entry, &bridge->windows) {
367 struct resource *res = entry->res;
368 u64 pci_addr = res->start - entry->offset;
369
370 if (resource_type(res) == IORESOURCE_IO)
371 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
372 true,
373 pci_pio_to_address(res->start),
374 pci_addr,
375 resource_size(res));
376 else
377 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
378 false,
379 res->start,
380 pci_addr,
381 resource_size(res));
382
383 r++;
384 }
385
386 err = cdns_pcie_host_map_dma_ranges(rc);
387 if (err)
388 return err;
389
390 return 0;
391}
392
393static int cdns_pcie_host_init(struct device *dev,
394 struct cdns_pcie_rc *rc)
395{
396 int err;
397
398 err = cdns_pcie_host_init_root_port(rc);
399 if (err)
400 return err;
401
402 return cdns_pcie_host_init_address_translation(rc);
403}
404
405static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie)
406{
407 struct device *dev = pcie->dev;
408 int retries;
409
410 /* Check if the link is up or not */
411 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
412 if (cdns_pcie_link_up(pcie)) {
413 dev_info(dev, "Link up\n");
414 return 0;
415 }
416 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
417 }
418
419 return -ETIMEDOUT;
420}
421
422int cdns_pcie_host_setup(struct cdns_pcie_rc *rc)
423{
424 struct device *dev = rc->pcie.dev;
425 struct platform_device *pdev = to_platform_device(dev);
426 struct device_node *np = dev->of_node;
427 struct pci_host_bridge *bridge;
428 enum cdns_pcie_rp_bar bar;
429 struct cdns_pcie *pcie;
430 struct resource *res;
431 int ret;
432
433 bridge = pci_host_bridge_from_priv(rc);
434 if (!bridge)
435 return -ENOMEM;
436
437 pcie = &rc->pcie;
438 pcie->is_rc = true;
439
440 rc->vendor_id = 0xffff;
441 of_property_read_u32(np, "vendor-id", &rc->vendor_id);
442
443 rc->device_id = 0xffff;
444 of_property_read_u32(np, "device-id", &rc->device_id);
445
446 pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");
447 if (IS_ERR(pcie->reg_base)) {
448 dev_err(dev, "missing \"reg\"\n");
449 return PTR_ERR(pcie->reg_base);
450 }
451
452 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");
453 rc->cfg_base = devm_pci_remap_cfg_resource(dev, res);
454 if (IS_ERR(rc->cfg_base))
455 return PTR_ERR(rc->cfg_base);
456 rc->cfg_res = res;
457
458 ret = cdns_pcie_start_link(pcie);
459 if (ret) {
460 dev_err(dev, "Failed to start link\n");
461 return ret;
462 }
463
464 ret = cdns_pcie_host_wait_for_link(pcie);
465 if (ret)
466 dev_dbg(dev, "PCIe link never came up\n");
467
468 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++)
469 rc->avail_ib_bar[bar] = true;
470
471 ret = cdns_pcie_host_init(dev, rc);
472 if (ret)
473 return ret;
474
475 if (!bridge->ops)
476 bridge->ops = &cdns_pcie_host_ops;
477
478 ret = pci_host_probe(bridge);
479 if (ret < 0)
480 goto err_init;
481
482 return 0;
483
484 err_init:
485 pm_runtime_put_sync(dev);
486
487 return ret;
488}
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017 Cadence
3// Cadence PCIe host controller driver.
4// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
5
6#include <linux/delay.h>
7#include <linux/kernel.h>
8#include <linux/list_sort.h>
9#include <linux/of_address.h>
10#include <linux/of_pci.h>
11#include <linux/platform_device.h>
12
13#include "pcie-cadence.h"
14
15static u64 bar_max_size[] = {
16 [RP_BAR0] = _ULL(128 * SZ_2G),
17 [RP_BAR1] = SZ_2G,
18 [RP_NO_BAR] = _BITULL(63),
19};
20
21static u8 bar_aperture_mask[] = {
22 [RP_BAR0] = 0x1F,
23 [RP_BAR1] = 0xF,
24};
25
26void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
27 int where)
28{
29 struct pci_host_bridge *bridge = pci_find_host_bridge(bus);
30 struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);
31 struct cdns_pcie *pcie = &rc->pcie;
32 unsigned int busn = bus->number;
33 u32 addr0, desc0;
34
35 if (pci_is_root_bus(bus)) {
36 /*
37 * Only the root port (devfn == 0) is connected to this bus.
38 * All other PCI devices are behind some bridge hence on another
39 * bus.
40 */
41 if (devfn)
42 return NULL;
43
44 return pcie->reg_base + (where & 0xfff);
45 }
46 /* Check that the link is up */
47 if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1))
48 return NULL;
49 /* Clear AXI link-down status */
50 cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0);
51
52 /* Update Output registers for AXI region 0. */
53 addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) |
54 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) |
55 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn);
56 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0);
57
58 /* Configuration Type 0 or Type 1 access. */
59 desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID |
60 CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0);
61 /*
62 * The bus number was already set once for all in desc1 by
63 * cdns_pcie_host_init_address_translation().
64 */
65 if (busn == bridge->busnr + 1)
66 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0;
67 else
68 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1;
69 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0);
70
71 return rc->cfg_base + (where & 0xfff);
72}
73
74static struct pci_ops cdns_pcie_host_ops = {
75 .map_bus = cdns_pci_map_bus,
76 .read = pci_generic_config_read,
77 .write = pci_generic_config_write,
78};
79
80static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie)
81{
82 struct device *dev = pcie->dev;
83 int retries;
84
85 /* Check if the link is up or not */
86 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
87 if (cdns_pcie_link_up(pcie)) {
88 dev_info(dev, "Link up\n");
89 return 0;
90 }
91 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
92 }
93
94 return -ETIMEDOUT;
95}
96
97static int cdns_pcie_retrain(struct cdns_pcie *pcie)
98{
99 u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;
100 u16 lnk_stat, lnk_ctl;
101 int ret = 0;
102
103 /*
104 * Set retrain bit if current speed is 2.5 GB/s,
105 * but the PCIe root port support is > 2.5 GB/s.
106 */
107
108 lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off +
109 PCI_EXP_LNKCAP));
110 if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
111 return ret;
112
113 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);
114 if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {
115 lnk_ctl = cdns_pcie_rp_readw(pcie,
116 pcie_cap_off + PCI_EXP_LNKCTL);
117 lnk_ctl |= PCI_EXP_LNKCTL_RL;
118 cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL,
119 lnk_ctl);
120
121 ret = cdns_pcie_host_wait_for_link(pcie);
122 }
123 return ret;
124}
125
126static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie)
127{
128 u32 val;
129
130 val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL);
131 cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN);
132}
133
134static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc)
135{
136 struct cdns_pcie *pcie = &rc->pcie;
137 int ret;
138
139 ret = cdns_pcie_host_wait_for_link(pcie);
140
141 /*
142 * Retrain link for Gen2 training defect
143 * if quirk flag is set.
144 */
145 if (!ret && rc->quirk_retrain_flag)
146 ret = cdns_pcie_retrain(pcie);
147
148 return ret;
149}
150
151static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc)
152{
153 struct cdns_pcie *pcie = &rc->pcie;
154 u32 value, ctrl;
155 u32 id;
156
157 /*
158 * Set the root complex BAR configuration register:
159 * - disable both BAR0 and BAR1.
160 * - enable Prefetchable Memory Base and Limit registers in type 1
161 * config space (64 bits).
162 * - enable IO Base and Limit registers in type 1 config
163 * space (32 bits).
164 */
165 ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
166 value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) |
167 CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) |
168 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE |
169 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS |
170 CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE |
171 CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS;
172 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
173
174 /* Set root port configuration space */
175 if (rc->vendor_id != 0xffff) {
176 id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |
177 CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);
178 cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);
179 }
180
181 if (rc->device_id != 0xffff)
182 cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);
183
184 cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0);
185 cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0);
186 cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
187
188 return 0;
189}
190
191static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc,
192 enum cdns_pcie_rp_bar bar,
193 u64 cpu_addr, u64 size,
194 unsigned long flags)
195{
196 struct cdns_pcie *pcie = &rc->pcie;
197 u32 addr0, addr1, aperture, value;
198
199 if (!rc->avail_ib_bar[bar])
200 return -EBUSY;
201
202 rc->avail_ib_bar[bar] = false;
203
204 aperture = ilog2(size);
205 addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) |
206 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
207 addr1 = upper_32_bits(cpu_addr);
208 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0);
209 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1);
210
211 if (bar == RP_NO_BAR)
212 return 0;
213
214 value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG);
215 value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) |
216 LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) |
217 LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) |
218 LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) |
219 LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2));
220 if (size + cpu_addr >= SZ_4G) {
221 if (!(flags & IORESOURCE_PREFETCH))
222 value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar);
223 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar);
224 } else {
225 if (!(flags & IORESOURCE_PREFETCH))
226 value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar);
227 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar);
228 }
229
230 value |= LM_RC_BAR_CFG_APERTURE(bar, aperture);
231 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
232
233 return 0;
234}
235
236static enum cdns_pcie_rp_bar
237cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size)
238{
239 enum cdns_pcie_rp_bar bar, sel_bar;
240
241 sel_bar = RP_BAR_UNDEFINED;
242 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
243 if (!rc->avail_ib_bar[bar])
244 continue;
245
246 if (size <= bar_max_size[bar]) {
247 if (sel_bar == RP_BAR_UNDEFINED) {
248 sel_bar = bar;
249 continue;
250 }
251
252 if (bar_max_size[bar] < bar_max_size[sel_bar])
253 sel_bar = bar;
254 }
255 }
256
257 return sel_bar;
258}
259
260static enum cdns_pcie_rp_bar
261cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size)
262{
263 enum cdns_pcie_rp_bar bar, sel_bar;
264
265 sel_bar = RP_BAR_UNDEFINED;
266 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
267 if (!rc->avail_ib_bar[bar])
268 continue;
269
270 if (size >= bar_max_size[bar]) {
271 if (sel_bar == RP_BAR_UNDEFINED) {
272 sel_bar = bar;
273 continue;
274 }
275
276 if (bar_max_size[bar] > bar_max_size[sel_bar])
277 sel_bar = bar;
278 }
279 }
280
281 return sel_bar;
282}
283
284static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc,
285 struct resource_entry *entry)
286{
287 u64 cpu_addr, pci_addr, size, winsize;
288 struct cdns_pcie *pcie = &rc->pcie;
289 struct device *dev = pcie->dev;
290 enum cdns_pcie_rp_bar bar;
291 unsigned long flags;
292 int ret;
293
294 cpu_addr = entry->res->start;
295 pci_addr = entry->res->start - entry->offset;
296 flags = entry->res->flags;
297 size = resource_size(entry->res);
298
299 if (entry->offset) {
300 dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n",
301 pci_addr, cpu_addr);
302 return -EINVAL;
303 }
304
305 while (size > 0) {
306 /*
307 * Try to find a minimum BAR whose size is greater than
308 * or equal to the remaining resource_entry size. This will
309 * fail if the size of each of the available BARs is less than
310 * the remaining resource_entry size.
311 * If a minimum BAR is found, IB ATU will be configured and
312 * exited.
313 */
314 bar = cdns_pcie_host_find_min_bar(rc, size);
315 if (bar != RP_BAR_UNDEFINED) {
316 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr,
317 size, flags);
318 if (ret)
319 dev_err(dev, "IB BAR: %d config failed\n", bar);
320 return ret;
321 }
322
323 /*
324 * If the control reaches here, it would mean the remaining
325 * resource_entry size cannot be fitted in a single BAR. So we
326 * find a maximum BAR whose size is less than or equal to the
327 * remaining resource_entry size and split the resource entry
328 * so that part of resource entry is fitted inside the maximum
329 * BAR. The remaining size would be fitted during the next
330 * iteration of the loop.
331 * If a maximum BAR is not found, there is no way we can fit
332 * this resource_entry, so we error out.
333 */
334 bar = cdns_pcie_host_find_max_bar(rc, size);
335 if (bar == RP_BAR_UNDEFINED) {
336 dev_err(dev, "No free BAR to map cpu_addr %llx\n",
337 cpu_addr);
338 return -EINVAL;
339 }
340
341 winsize = bar_max_size[bar];
342 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize,
343 flags);
344 if (ret) {
345 dev_err(dev, "IB BAR: %d config failed\n", bar);
346 return ret;
347 }
348
349 size -= winsize;
350 cpu_addr += winsize;
351 }
352
353 return 0;
354}
355
356static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a,
357 const struct list_head *b)
358{
359 struct resource_entry *entry1, *entry2;
360
361 entry1 = container_of(a, struct resource_entry, node);
362 entry2 = container_of(b, struct resource_entry, node);
363
364 return resource_size(entry2->res) - resource_size(entry1->res);
365}
366
367static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc)
368{
369 struct cdns_pcie *pcie = &rc->pcie;
370 struct device *dev = pcie->dev;
371 struct device_node *np = dev->of_node;
372 struct pci_host_bridge *bridge;
373 struct resource_entry *entry;
374 u32 no_bar_nbits = 32;
375 int err;
376
377 bridge = pci_host_bridge_from_priv(rc);
378 if (!bridge)
379 return -ENOMEM;
380
381 if (list_empty(&bridge->dma_ranges)) {
382 of_property_read_u32(np, "cdns,no-bar-match-nbits",
383 &no_bar_nbits);
384 err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0,
385 (u64)1 << no_bar_nbits, 0);
386 if (err)
387 dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR);
388 return err;
389 }
390
391 list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp);
392
393 resource_list_for_each_entry(entry, &bridge->dma_ranges) {
394 err = cdns_pcie_host_bar_config(rc, entry);
395 if (err) {
396 dev_err(dev, "Fail to configure IB using dma-ranges\n");
397 return err;
398 }
399 }
400
401 return 0;
402}
403
404static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc)
405{
406 struct cdns_pcie *pcie = &rc->pcie;
407 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc);
408 struct resource *cfg_res = rc->cfg_res;
409 struct resource_entry *entry;
410 u64 cpu_addr = cfg_res->start;
411 u32 addr0, addr1, desc1;
412 int r, busnr = 0;
413
414 entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS);
415 if (entry)
416 busnr = entry->res->start;
417
418 /*
419 * Reserve region 0 for PCI configure space accesses:
420 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by
421 * cdns_pci_map_bus(), other region registers are set here once for all.
422 */
423 addr1 = 0; /* Should be programmed to zero. */
424 desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr);
425 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);
426 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);
427
428 if (pcie->ops->cpu_addr_fixup)
429 cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr);
430
431 addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) |
432 (lower_32_bits(cpu_addr) & GENMASK(31, 8));
433 addr1 = upper_32_bits(cpu_addr);
434 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0);
435 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1);
436
437 r = 1;
438 resource_list_for_each_entry(entry, &bridge->windows) {
439 struct resource *res = entry->res;
440 u64 pci_addr = res->start - entry->offset;
441
442 if (resource_type(res) == IORESOURCE_IO)
443 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
444 true,
445 pci_pio_to_address(res->start),
446 pci_addr,
447 resource_size(res));
448 else
449 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
450 false,
451 res->start,
452 pci_addr,
453 resource_size(res));
454
455 r++;
456 }
457
458 return cdns_pcie_host_map_dma_ranges(rc);
459}
460
461static int cdns_pcie_host_init(struct device *dev,
462 struct cdns_pcie_rc *rc)
463{
464 int err;
465
466 err = cdns_pcie_host_init_root_port(rc);
467 if (err)
468 return err;
469
470 return cdns_pcie_host_init_address_translation(rc);
471}
472
473int cdns_pcie_host_setup(struct cdns_pcie_rc *rc)
474{
475 struct device *dev = rc->pcie.dev;
476 struct platform_device *pdev = to_platform_device(dev);
477 struct device_node *np = dev->of_node;
478 struct pci_host_bridge *bridge;
479 enum cdns_pcie_rp_bar bar;
480 struct cdns_pcie *pcie;
481 struct resource *res;
482 int ret;
483
484 bridge = pci_host_bridge_from_priv(rc);
485 if (!bridge)
486 return -ENOMEM;
487
488 pcie = &rc->pcie;
489 pcie->is_rc = true;
490
491 rc->vendor_id = 0xffff;
492 of_property_read_u32(np, "vendor-id", &rc->vendor_id);
493
494 rc->device_id = 0xffff;
495 of_property_read_u32(np, "device-id", &rc->device_id);
496
497 pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");
498 if (IS_ERR(pcie->reg_base)) {
499 dev_err(dev, "missing \"reg\"\n");
500 return PTR_ERR(pcie->reg_base);
501 }
502
503 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");
504 rc->cfg_base = devm_pci_remap_cfg_resource(dev, res);
505 if (IS_ERR(rc->cfg_base))
506 return PTR_ERR(rc->cfg_base);
507 rc->cfg_res = res;
508
509 if (rc->quirk_detect_quiet_flag)
510 cdns_pcie_detect_quiet_min_delay_set(&rc->pcie);
511
512 cdns_pcie_host_enable_ptm_response(pcie);
513
514 ret = cdns_pcie_start_link(pcie);
515 if (ret) {
516 dev_err(dev, "Failed to start link\n");
517 return ret;
518 }
519
520 ret = cdns_pcie_host_start_link(rc);
521 if (ret)
522 dev_dbg(dev, "PCIe link never came up\n");
523
524 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++)
525 rc->avail_ib_bar[bar] = true;
526
527 ret = cdns_pcie_host_init(dev, rc);
528 if (ret)
529 return ret;
530
531 if (!bridge->ops)
532 bridge->ops = &cdns_pcie_host_ops;
533
534 ret = pci_host_probe(bridge);
535 if (ret < 0)
536 goto err_init;
537
538 return 0;
539
540 err_init:
541 pm_runtime_put_sync(dev);
542
543 return ret;
544}