Linux Audio

Check our new training course

Loading...
v5.14.15
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Synopsys DesignWare PCIe host controller driver
  4 *
  5 * Copyright (C) 2013 Samsung Electronics Co., Ltd.
  6 *		https://www.samsung.com
  7 *
  8 * Author: Jingoo Han <jg1.han@samsung.com>
  9 */
 10
 11#include <linux/irqchip/chained_irq.h>
 12#include <linux/irqdomain.h>
 13#include <linux/msi.h>
 14#include <linux/of_address.h>
 15#include <linux/of_pci.h>
 16#include <linux/pci_regs.h>
 17#include <linux/platform_device.h>
 18
 19#include "../../pci.h"
 20#include "pcie-designware.h"
 21
 22static struct pci_ops dw_pcie_ops;
 23static struct pci_ops dw_child_pcie_ops;
 24
 25static void dw_msi_ack_irq(struct irq_data *d)
 26{
 27	irq_chip_ack_parent(d);
 28}
 29
 30static void dw_msi_mask_irq(struct irq_data *d)
 31{
 32	pci_msi_mask_irq(d);
 33	irq_chip_mask_parent(d);
 34}
 35
 36static void dw_msi_unmask_irq(struct irq_data *d)
 37{
 38	pci_msi_unmask_irq(d);
 39	irq_chip_unmask_parent(d);
 40}
 41
 42static struct irq_chip dw_pcie_msi_irq_chip = {
 43	.name = "PCI-MSI",
 44	.irq_ack = dw_msi_ack_irq,
 45	.irq_mask = dw_msi_mask_irq,
 46	.irq_unmask = dw_msi_unmask_irq,
 47};
 48
 49static struct msi_domain_info dw_pcie_msi_domain_info = {
 50	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
 51		   MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI),
 52	.chip	= &dw_pcie_msi_irq_chip,
 53};
 54
 55/* MSI int handler */
 56irqreturn_t dw_handle_msi_irq(struct pcie_port *pp)
 57{
 58	int i, pos, irq;
 59	unsigned long val;
 60	u32 status, num_ctrls;
 61	irqreturn_t ret = IRQ_NONE;
 62	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 63
 64	num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
 65
 66	for (i = 0; i < num_ctrls; i++) {
 67		status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS +
 68					   (i * MSI_REG_CTRL_BLOCK_SIZE));
 69		if (!status)
 70			continue;
 71
 72		ret = IRQ_HANDLED;
 73		val = status;
 74		pos = 0;
 75		while ((pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL,
 76					    pos)) != MAX_MSI_IRQS_PER_CTRL) {
 77			irq = irq_find_mapping(pp->irq_domain,
 78					       (i * MAX_MSI_IRQS_PER_CTRL) +
 79					       pos);
 80			generic_handle_irq(irq);
 81			pos++;
 82		}
 83	}
 84
 85	return ret;
 86}
 87
 88/* Chained MSI interrupt service routine */
 89static void dw_chained_msi_isr(struct irq_desc *desc)
 90{
 91	struct irq_chip *chip = irq_desc_get_chip(desc);
 92	struct pcie_port *pp;
 93
 94	chained_irq_enter(chip, desc);
 95
 96	pp = irq_desc_get_handler_data(desc);
 97	dw_handle_msi_irq(pp);
 98
 99	chained_irq_exit(chip, desc);
100}
101
102static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg)
103{
104	struct pcie_port *pp = irq_data_get_irq_chip_data(d);
105	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
106	u64 msi_target;
107
108	msi_target = (u64)pp->msi_data;
109
110	msg->address_lo = lower_32_bits(msi_target);
111	msg->address_hi = upper_32_bits(msi_target);
112
113	msg->data = d->hwirq;
114
115	dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n",
116		(int)d->hwirq, msg->address_hi, msg->address_lo);
117}
118
119static int dw_pci_msi_set_affinity(struct irq_data *d,
120				   const struct cpumask *mask, bool force)
121{
122	return -EINVAL;
123}
124
125static void dw_pci_bottom_mask(struct irq_data *d)
126{
127	struct pcie_port *pp = irq_data_get_irq_chip_data(d);
128	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
129	unsigned int res, bit, ctrl;
130	unsigned long flags;
131
132	raw_spin_lock_irqsave(&pp->lock, flags);
133
134	ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
135	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
136	bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
137
138	pp->irq_mask[ctrl] |= BIT(bit);
139	dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
140
141	raw_spin_unlock_irqrestore(&pp->lock, flags);
142}
143
144static void dw_pci_bottom_unmask(struct irq_data *d)
145{
146	struct pcie_port *pp = irq_data_get_irq_chip_data(d);
147	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
148	unsigned int res, bit, ctrl;
149	unsigned long flags;
150
151	raw_spin_lock_irqsave(&pp->lock, flags);
152
153	ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
154	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
155	bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
156
157	pp->irq_mask[ctrl] &= ~BIT(bit);
158	dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
159
160	raw_spin_unlock_irqrestore(&pp->lock, flags);
161}
162
163static void dw_pci_bottom_ack(struct irq_data *d)
164{
165	struct pcie_port *pp  = irq_data_get_irq_chip_data(d);
166	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
167	unsigned int res, bit, ctrl;
168
169	ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
170	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
171	bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
172
173	dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_STATUS + res, BIT(bit));
174}
175
176static struct irq_chip dw_pci_msi_bottom_irq_chip = {
177	.name = "DWPCI-MSI",
178	.irq_ack = dw_pci_bottom_ack,
179	.irq_compose_msi_msg = dw_pci_setup_msi_msg,
180	.irq_set_affinity = dw_pci_msi_set_affinity,
181	.irq_mask = dw_pci_bottom_mask,
182	.irq_unmask = dw_pci_bottom_unmask,
183};
184
185static int dw_pcie_irq_domain_alloc(struct irq_domain *domain,
186				    unsigned int virq, unsigned int nr_irqs,
187				    void *args)
188{
189	struct pcie_port *pp = domain->host_data;
190	unsigned long flags;
191	u32 i;
192	int bit;
193
194	raw_spin_lock_irqsave(&pp->lock, flags);
195
196	bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors,
197				      order_base_2(nr_irqs));
198
199	raw_spin_unlock_irqrestore(&pp->lock, flags);
200
201	if (bit < 0)
202		return -ENOSPC;
203
204	for (i = 0; i < nr_irqs; i++)
205		irq_domain_set_info(domain, virq + i, bit + i,
206				    pp->msi_irq_chip,
207				    pp, handle_edge_irq,
208				    NULL, NULL);
209
210	return 0;
211}
212
213static void dw_pcie_irq_domain_free(struct irq_domain *domain,
214				    unsigned int virq, unsigned int nr_irqs)
215{
216	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
217	struct pcie_port *pp = domain->host_data;
218	unsigned long flags;
219
220	raw_spin_lock_irqsave(&pp->lock, flags);
221
222	bitmap_release_region(pp->msi_irq_in_use, d->hwirq,
223			      order_base_2(nr_irqs));
224
225	raw_spin_unlock_irqrestore(&pp->lock, flags);
226}
227
228static const struct irq_domain_ops dw_pcie_msi_domain_ops = {
229	.alloc	= dw_pcie_irq_domain_alloc,
230	.free	= dw_pcie_irq_domain_free,
231};
232
233int dw_pcie_allocate_domains(struct pcie_port *pp)
234{
235	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
236	struct fwnode_handle *fwnode = of_node_to_fwnode(pci->dev->of_node);
237
238	pp->irq_domain = irq_domain_create_linear(fwnode, pp->num_vectors,
239					       &dw_pcie_msi_domain_ops, pp);
240	if (!pp->irq_domain) {
241		dev_err(pci->dev, "Failed to create IRQ domain\n");
242		return -ENOMEM;
243	}
244
245	irq_domain_update_bus_token(pp->irq_domain, DOMAIN_BUS_NEXUS);
246
247	pp->msi_domain = pci_msi_create_irq_domain(fwnode,
248						   &dw_pcie_msi_domain_info,
249						   pp->irq_domain);
250	if (!pp->msi_domain) {
251		dev_err(pci->dev, "Failed to create MSI domain\n");
252		irq_domain_remove(pp->irq_domain);
253		return -ENOMEM;
254	}
255
256	return 0;
257}
258
259static void dw_pcie_free_msi(struct pcie_port *pp)
260{
261	if (pp->msi_irq)
262		irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL);
 
 
 
 
 
263
264	irq_domain_remove(pp->msi_domain);
265	irq_domain_remove(pp->irq_domain);
266
267	if (pp->msi_data) {
268		struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
269		struct device *dev = pci->dev;
270
271		dma_unmap_single_attrs(dev, pp->msi_data, sizeof(pp->msi_msg),
272				       DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
273	}
274}
275
276static void dw_pcie_msi_init(struct pcie_port *pp)
277{
278	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
279	u64 msi_target = (u64)pp->msi_data;
280
281	if (!pci_msi_enabled() || !pp->has_msi_ctrl)
282		return;
283
284	/* Program the msi_data */
285	dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_LO, lower_32_bits(msi_target));
286	dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_HI, upper_32_bits(msi_target));
287}
288
289int dw_pcie_host_init(struct pcie_port *pp)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290{
291	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
292	struct device *dev = pci->dev;
293	struct device_node *np = dev->of_node;
294	struct platform_device *pdev = to_platform_device(dev);
295	struct resource_entry *win;
296	struct pci_host_bridge *bridge;
297	struct resource *cfg_res;
298	int ret;
299
300	raw_spin_lock_init(&pci->pp.lock);
301
302	cfg_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
303	if (cfg_res) {
304		pp->cfg0_size = resource_size(cfg_res);
305		pp->cfg0_base = cfg_res->start;
 
 
 
 
306
307		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, cfg_res);
308		if (IS_ERR(pp->va_cfg0_base))
309			return PTR_ERR(pp->va_cfg0_base);
310	} else {
311		dev_err(dev, "Missing *config* reg space\n");
312		return -ENODEV;
313	}
314
315	if (!pci->dbi_base) {
316		struct resource *dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
317		pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_res);
318		if (IS_ERR(pci->dbi_base))
319			return PTR_ERR(pci->dbi_base);
320	}
321
322	bridge = devm_pci_alloc_host_bridge(dev, 0);
323	if (!bridge)
324		return -ENOMEM;
325
326	pp->bridge = bridge;
327
328	/* Get the I/O range from DT */
329	win = resource_list_first_type(&bridge->windows, IORESOURCE_IO);
330	if (win) {
331		pp->io_size = resource_size(win->res);
332		pp->io_bus_addr = win->res->start - win->offset;
333		pp->io_base = pci_pio_to_address(win->res->start);
334	}
335
336	if (pci->link_gen < 1)
337		pci->link_gen = of_pci_get_max_link_speed(np);
 
 
 
 
 
 
 
338
339	if (pci_msi_enabled()) {
340		pp->has_msi_ctrl = !(pp->ops->msi_host_init ||
341				     of_property_read_bool(np, "msi-parent") ||
342				     of_property_read_bool(np, "msi-map"));
343
344		if (!pp->num_vectors) {
 
 
 
 
345			pp->num_vectors = MSI_DEF_NUM_VECTORS;
346		} else if (pp->num_vectors > MAX_MSI_IRQS) {
347			dev_err(dev, "Invalid number of vectors\n");
348			return -EINVAL;
 
349		}
350
351		if (pp->ops->msi_host_init) {
352			ret = pp->ops->msi_host_init(pp);
353			if (ret < 0)
354				return ret;
355		} else if (pp->has_msi_ctrl) {
356			if (!pp->msi_irq) {
357				pp->msi_irq = platform_get_irq_byname_optional(pdev, "msi");
358				if (pp->msi_irq < 0) {
359					pp->msi_irq = platform_get_irq(pdev, 0);
360					if (pp->msi_irq < 0)
361						return pp->msi_irq;
362				}
363			}
364
365			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
366
367			ret = dw_pcie_allocate_domains(pp);
368			if (ret)
369				return ret;
370
371			if (pp->msi_irq > 0)
372				irq_set_chained_handler_and_data(pp->msi_irq,
373							    dw_chained_msi_isr,
374							    pp);
375
376			ret = dma_set_mask(pci->dev, DMA_BIT_MASK(32));
377			if (ret)
378				dev_warn(pci->dev, "Failed to set DMA mask to 32-bit. Devices with only 32-bit MSI support may not work properly\n");
379
380			pp->msi_data = dma_map_single_attrs(pci->dev, &pp->msi_msg,
381						      sizeof(pp->msi_msg),
382						      DMA_FROM_DEVICE,
383						      DMA_ATTR_SKIP_CPU_SYNC);
384			if (dma_mapping_error(pci->dev, pp->msi_data)) {
385				dev_err(pci->dev, "Failed to map MSI data\n");
386				pp->msi_data = 0;
387				goto err_free_msi;
388			}
389		}
390	}
391
392	/* Set default bus ops */
393	bridge->ops = &dw_pcie_ops;
394	bridge->child_ops = &dw_child_pcie_ops;
395
396	if (pp->ops->host_init) {
397		ret = pp->ops->host_init(pp);
398		if (ret)
399			goto err_free_msi;
400	}
401	dw_pcie_iatu_detect(pci);
402
403	dw_pcie_setup_rc(pp);
 
 
404
405	if (!dw_pcie_link_up(pci) && pci->ops && pci->ops->start_link) {
406		ret = pci->ops->start_link(pci);
407		if (ret)
408			goto err_free_msi;
409	}
410
411	/* Ignore errors, the link may come up later */
412	dw_pcie_wait_for_link(pci);
413
414	bridge->sysdata = pp;
415
416	ret = pci_host_probe(bridge);
417	if (!ret)
418		return 0;
 
 
 
 
 
419
420err_free_msi:
421	if (pp->has_msi_ctrl)
422		dw_pcie_free_msi(pp);
 
 
 
 
 
423	return ret;
424}
425EXPORT_SYMBOL_GPL(dw_pcie_host_init);
426
427void dw_pcie_host_deinit(struct pcie_port *pp)
428{
 
 
429	pci_stop_root_bus(pp->bridge->bus);
430	pci_remove_root_bus(pp->bridge->bus);
 
 
 
431	if (pp->has_msi_ctrl)
432		dw_pcie_free_msi(pp);
 
 
 
433}
434EXPORT_SYMBOL_GPL(dw_pcie_host_deinit);
435
436static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus,
437						unsigned int devfn, int where)
438{
439	int type;
440	u32 busdev;
441	struct pcie_port *pp = bus->sysdata;
442	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 
 
443
444	/*
445	 * Checking whether the link is up here is a last line of defense
446	 * against platforms that forward errors on the system bus as
447	 * SError upon PCI configuration transactions issued when the link
448	 * is down. This check is racy by definition and does not stop
449	 * the system from triggering an SError if the link goes down
450	 * after this check is performed.
451	 */
452	if (!dw_pcie_link_up(pci))
453		return NULL;
454
455	busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) |
456		 PCIE_ATU_FUNC(PCI_FUNC(devfn));
457
458	if (pci_is_root_bus(bus->parent))
459		type = PCIE_ATU_TYPE_CFG0;
460	else
461		type = PCIE_ATU_TYPE_CFG1;
462
463
464	dw_pcie_prog_outbound_atu(pci, 0, type, pp->cfg0_base, busdev, pp->cfg0_size);
 
 
465
466	return pp->va_cfg0_base + where;
467}
468
469static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn,
470				 int where, int size, u32 *val)
471{
472	int ret;
473	struct pcie_port *pp = bus->sysdata;
474	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 
475
476	ret = pci_generic_config_read(bus, devfn, where, size, val);
 
 
477
478	if (!ret && pci->io_cfg_atu_shared)
479		dw_pcie_prog_outbound_atu(pci, 0, PCIE_ATU_TYPE_IO, pp->io_base,
480					  pp->io_bus_addr, pp->io_size);
 
 
 
 
481
482	return ret;
483}
484
485static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn,
486				 int where, int size, u32 val)
487{
488	int ret;
489	struct pcie_port *pp = bus->sysdata;
490	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 
491
492	ret = pci_generic_config_write(bus, devfn, where, size, val);
 
 
493
494	if (!ret && pci->io_cfg_atu_shared)
495		dw_pcie_prog_outbound_atu(pci, 0, PCIE_ATU_TYPE_IO, pp->io_base,
496					  pp->io_bus_addr, pp->io_size);
 
 
 
 
497
498	return ret;
499}
500
501static struct pci_ops dw_child_pcie_ops = {
502	.map_bus = dw_pcie_other_conf_map_bus,
503	.read = dw_pcie_rd_other_conf,
504	.write = dw_pcie_wr_other_conf,
505};
506
507void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where)
508{
509	struct pcie_port *pp = bus->sysdata;
510	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
511
512	if (PCI_SLOT(devfn) > 0)
513		return NULL;
514
515	return pci->dbi_base + where;
516}
517EXPORT_SYMBOL_GPL(dw_pcie_own_conf_map_bus);
518
519static struct pci_ops dw_pcie_ops = {
520	.map_bus = dw_pcie_own_conf_map_bus,
521	.read = pci_generic_config_read,
522	.write = pci_generic_config_write,
523};
524
525void dw_pcie_setup_rc(struct pcie_port *pp)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
526{
527	int i;
528	u32 val, ctrl, num_ctrls;
529	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 
 
530
531	/*
532	 * Enable DBI read-only registers for writing/updating configuration.
533	 * Write permission gets disabled towards the end of this function.
534	 */
535	dw_pcie_dbi_ro_wr_en(pci);
536
537	dw_pcie_setup(pci);
538
539	if (pp->has_msi_ctrl) {
540		num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
541
542		/* Initialize IRQ Status array */
543		for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
544			pp->irq_mask[ctrl] = ~0;
545			dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK +
546					    (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
547					    pp->irq_mask[ctrl]);
548			dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_ENABLE +
549					    (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
550					    ~0);
551		}
552	}
553
554	dw_pcie_msi_init(pp);
555
556	/* Setup RC BARs */
557	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0x00000004);
558	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000);
559
560	/* Setup interrupt pins */
561	val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE);
562	val &= 0xffff00ff;
563	val |= 0x00000100;
564	dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val);
565
566	/* Setup bus numbers */
567	val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS);
568	val &= 0xff000000;
569	val |= 0x00ff0100;
570	dw_pcie_writel_dbi(pci, PCI_PRIMARY_BUS, val);
571
572	/* Setup command register */
573	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
574	val &= 0xffff0000;
575	val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
576		PCI_COMMAND_MASTER | PCI_COMMAND_SERR;
577	dw_pcie_writel_dbi(pci, PCI_COMMAND, val);
578
579	/* Ensure all outbound windows are disabled so there are multiple matches */
580	for (i = 0; i < pci->num_ob_windows; i++)
581		dw_pcie_disable_atu(pci, i, DW_PCIE_REGION_OUTBOUND);
582
583	/*
584	 * If the platform provides its own child bus config accesses, it means
585	 * the platform uses its own address translation component rather than
586	 * ATU, so we should not program the ATU here.
587	 */
588	if (pp->bridge->child_ops == &dw_child_pcie_ops) {
589		int atu_idx = 0;
590		struct resource_entry *entry;
591
592		/* Get last memory resource entry */
593		resource_list_for_each_entry(entry, &pp->bridge->windows) {
594			if (resource_type(entry->res) != IORESOURCE_MEM)
595				continue;
596
597			if (pci->num_ob_windows <= ++atu_idx)
598				break;
599
600			dw_pcie_prog_outbound_atu(pci, atu_idx,
601						  PCIE_ATU_TYPE_MEM, entry->res->start,
602						  entry->res->start - entry->offset,
603						  resource_size(entry->res));
604		}
605
606		if (pp->io_size) {
607			if (pci->num_ob_windows > ++atu_idx)
608				dw_pcie_prog_outbound_atu(pci, atu_idx,
609							  PCIE_ATU_TYPE_IO, pp->io_base,
610							  pp->io_bus_addr, pp->io_size);
611			else
612				pci->io_cfg_atu_shared = true;
613		}
614
615		if (pci->num_ob_windows <= atu_idx)
616			dev_warn(pci->dev, "Resources exceed number of ATU entries (%d)",
617				 pci->num_ob_windows);
618	}
619
620	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
621
622	/* Program correct class for RC */
623	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
624
625	val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
626	val |= PORT_LOGIC_SPEED_CHANGE;
627	dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
628
629	dw_pcie_dbi_ro_wr_dis(pci);
 
 
630}
631EXPORT_SYMBOL_GPL(dw_pcie_setup_rc);
v6.2
  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Synopsys DesignWare PCIe host controller driver
  4 *
  5 * Copyright (C) 2013 Samsung Electronics Co., Ltd.
  6 *		https://www.samsung.com
  7 *
  8 * Author: Jingoo Han <jg1.han@samsung.com>
  9 */
 10
 11#include <linux/irqchip/chained_irq.h>
 12#include <linux/irqdomain.h>
 13#include <linux/msi.h>
 14#include <linux/of_address.h>
 15#include <linux/of_pci.h>
 16#include <linux/pci_regs.h>
 17#include <linux/platform_device.h>
 18
 
 19#include "pcie-designware.h"
 20
 21static struct pci_ops dw_pcie_ops;
 22static struct pci_ops dw_child_pcie_ops;
 23
 24static void dw_msi_ack_irq(struct irq_data *d)
 25{
 26	irq_chip_ack_parent(d);
 27}
 28
 29static void dw_msi_mask_irq(struct irq_data *d)
 30{
 31	pci_msi_mask_irq(d);
 32	irq_chip_mask_parent(d);
 33}
 34
 35static void dw_msi_unmask_irq(struct irq_data *d)
 36{
 37	pci_msi_unmask_irq(d);
 38	irq_chip_unmask_parent(d);
 39}
 40
 41static struct irq_chip dw_pcie_msi_irq_chip = {
 42	.name = "PCI-MSI",
 43	.irq_ack = dw_msi_ack_irq,
 44	.irq_mask = dw_msi_mask_irq,
 45	.irq_unmask = dw_msi_unmask_irq,
 46};
 47
 48static struct msi_domain_info dw_pcie_msi_domain_info = {
 49	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
 50		   MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI),
 51	.chip	= &dw_pcie_msi_irq_chip,
 52};
 53
 54/* MSI int handler */
 55irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp)
 56{
 57	int i, pos;
 58	unsigned long val;
 59	u32 status, num_ctrls;
 60	irqreturn_t ret = IRQ_NONE;
 61	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 62
 63	num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
 64
 65	for (i = 0; i < num_ctrls; i++) {
 66		status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS +
 67					   (i * MSI_REG_CTRL_BLOCK_SIZE));
 68		if (!status)
 69			continue;
 70
 71		ret = IRQ_HANDLED;
 72		val = status;
 73		pos = 0;
 74		while ((pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL,
 75					    pos)) != MAX_MSI_IRQS_PER_CTRL) {
 76			generic_handle_domain_irq(pp->irq_domain,
 77						  (i * MAX_MSI_IRQS_PER_CTRL) +
 78						  pos);
 
 79			pos++;
 80		}
 81	}
 82
 83	return ret;
 84}
 85
 86/* Chained MSI interrupt service routine */
 87static void dw_chained_msi_isr(struct irq_desc *desc)
 88{
 89	struct irq_chip *chip = irq_desc_get_chip(desc);
 90	struct dw_pcie_rp *pp;
 91
 92	chained_irq_enter(chip, desc);
 93
 94	pp = irq_desc_get_handler_data(desc);
 95	dw_handle_msi_irq(pp);
 96
 97	chained_irq_exit(chip, desc);
 98}
 99
100static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg)
101{
102	struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
103	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
104	u64 msi_target;
105
106	msi_target = (u64)pp->msi_data;
107
108	msg->address_lo = lower_32_bits(msi_target);
109	msg->address_hi = upper_32_bits(msi_target);
110
111	msg->data = d->hwirq;
112
113	dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n",
114		(int)d->hwirq, msg->address_hi, msg->address_lo);
115}
116
117static int dw_pci_msi_set_affinity(struct irq_data *d,
118				   const struct cpumask *mask, bool force)
119{
120	return -EINVAL;
121}
122
123static void dw_pci_bottom_mask(struct irq_data *d)
124{
125	struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
126	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
127	unsigned int res, bit, ctrl;
128	unsigned long flags;
129
130	raw_spin_lock_irqsave(&pp->lock, flags);
131
132	ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
133	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
134	bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
135
136	pp->irq_mask[ctrl] |= BIT(bit);
137	dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
138
139	raw_spin_unlock_irqrestore(&pp->lock, flags);
140}
141
142static void dw_pci_bottom_unmask(struct irq_data *d)
143{
144	struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
145	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
146	unsigned int res, bit, ctrl;
147	unsigned long flags;
148
149	raw_spin_lock_irqsave(&pp->lock, flags);
150
151	ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
152	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
153	bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
154
155	pp->irq_mask[ctrl] &= ~BIT(bit);
156	dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
157
158	raw_spin_unlock_irqrestore(&pp->lock, flags);
159}
160
161static void dw_pci_bottom_ack(struct irq_data *d)
162{
163	struct dw_pcie_rp *pp  = irq_data_get_irq_chip_data(d);
164	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
165	unsigned int res, bit, ctrl;
166
167	ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
168	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
169	bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
170
171	dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_STATUS + res, BIT(bit));
172}
173
174static struct irq_chip dw_pci_msi_bottom_irq_chip = {
175	.name = "DWPCI-MSI",
176	.irq_ack = dw_pci_bottom_ack,
177	.irq_compose_msi_msg = dw_pci_setup_msi_msg,
178	.irq_set_affinity = dw_pci_msi_set_affinity,
179	.irq_mask = dw_pci_bottom_mask,
180	.irq_unmask = dw_pci_bottom_unmask,
181};
182
183static int dw_pcie_irq_domain_alloc(struct irq_domain *domain,
184				    unsigned int virq, unsigned int nr_irqs,
185				    void *args)
186{
187	struct dw_pcie_rp *pp = domain->host_data;
188	unsigned long flags;
189	u32 i;
190	int bit;
191
192	raw_spin_lock_irqsave(&pp->lock, flags);
193
194	bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors,
195				      order_base_2(nr_irqs));
196
197	raw_spin_unlock_irqrestore(&pp->lock, flags);
198
199	if (bit < 0)
200		return -ENOSPC;
201
202	for (i = 0; i < nr_irqs; i++)
203		irq_domain_set_info(domain, virq + i, bit + i,
204				    pp->msi_irq_chip,
205				    pp, handle_edge_irq,
206				    NULL, NULL);
207
208	return 0;
209}
210
211static void dw_pcie_irq_domain_free(struct irq_domain *domain,
212				    unsigned int virq, unsigned int nr_irqs)
213{
214	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
215	struct dw_pcie_rp *pp = domain->host_data;
216	unsigned long flags;
217
218	raw_spin_lock_irqsave(&pp->lock, flags);
219
220	bitmap_release_region(pp->msi_irq_in_use, d->hwirq,
221			      order_base_2(nr_irqs));
222
223	raw_spin_unlock_irqrestore(&pp->lock, flags);
224}
225
226static const struct irq_domain_ops dw_pcie_msi_domain_ops = {
227	.alloc	= dw_pcie_irq_domain_alloc,
228	.free	= dw_pcie_irq_domain_free,
229};
230
231int dw_pcie_allocate_domains(struct dw_pcie_rp *pp)
232{
233	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
234	struct fwnode_handle *fwnode = of_node_to_fwnode(pci->dev->of_node);
235
236	pp->irq_domain = irq_domain_create_linear(fwnode, pp->num_vectors,
237					       &dw_pcie_msi_domain_ops, pp);
238	if (!pp->irq_domain) {
239		dev_err(pci->dev, "Failed to create IRQ domain\n");
240		return -ENOMEM;
241	}
242
243	irq_domain_update_bus_token(pp->irq_domain, DOMAIN_BUS_NEXUS);
244
245	pp->msi_domain = pci_msi_create_irq_domain(fwnode,
246						   &dw_pcie_msi_domain_info,
247						   pp->irq_domain);
248	if (!pp->msi_domain) {
249		dev_err(pci->dev, "Failed to create MSI domain\n");
250		irq_domain_remove(pp->irq_domain);
251		return -ENOMEM;
252	}
253
254	return 0;
255}
256
257static void dw_pcie_free_msi(struct dw_pcie_rp *pp)
258{
259	u32 ctrl;
260
261	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) {
262		if (pp->msi_irq[ctrl] > 0)
263			irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
264							 NULL, NULL);
265	}
266
267	irq_domain_remove(pp->msi_domain);
268	irq_domain_remove(pp->irq_domain);
 
 
 
 
 
 
 
 
269}
270
271static void dw_pcie_msi_init(struct dw_pcie_rp *pp)
272{
273	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
274	u64 msi_target = (u64)pp->msi_data;
275
276	if (!pci_msi_enabled() || !pp->has_msi_ctrl)
277		return;
278
279	/* Program the msi_data */
280	dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_LO, lower_32_bits(msi_target));
281	dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_HI, upper_32_bits(msi_target));
282}
283
284static int dw_pcie_parse_split_msi_irq(struct dw_pcie_rp *pp)
285{
286	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
287	struct device *dev = pci->dev;
288	struct platform_device *pdev = to_platform_device(dev);
289	u32 ctrl, max_vectors;
290	int irq;
291
292	/* Parse any "msiX" IRQs described in the devicetree */
293	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) {
294		char msi_name[] = "msiX";
295
296		msi_name[3] = '0' + ctrl;
297		irq = platform_get_irq_byname_optional(pdev, msi_name);
298		if (irq == -ENXIO)
299			break;
300		if (irq < 0)
301			return dev_err_probe(dev, irq,
302					     "Failed to parse MSI IRQ '%s'\n",
303					     msi_name);
304
305		pp->msi_irq[ctrl] = irq;
306	}
307
308	/* If no "msiX" IRQs, caller should fallback to "msi" IRQ */
309	if (ctrl == 0)
310		return -ENXIO;
311
312	max_vectors = ctrl * MAX_MSI_IRQS_PER_CTRL;
313	if (pp->num_vectors > max_vectors) {
314		dev_warn(dev, "Exceeding number of MSI vectors, limiting to %u\n",
315			 max_vectors);
316		pp->num_vectors = max_vectors;
317	}
318	if (!pp->num_vectors)
319		pp->num_vectors = max_vectors;
320
321	return 0;
322}
323
324static int dw_pcie_msi_host_init(struct dw_pcie_rp *pp)
325{
326	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
327	struct device *dev = pci->dev;
328	struct platform_device *pdev = to_platform_device(dev);
329	u64 *msi_vaddr;
330	int ret;
331	u32 ctrl, num_ctrls;
332
333	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
334		pp->irq_mask[ctrl] = ~0;
335
336	if (!pp->msi_irq[0]) {
337		ret = dw_pcie_parse_split_msi_irq(pp);
338		if (ret < 0 && ret != -ENXIO)
339			return ret;
340	}
341
342	if (!pp->num_vectors)
343		pp->num_vectors = MSI_DEF_NUM_VECTORS;
344	num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
345
346	if (!pp->msi_irq[0]) {
347		pp->msi_irq[0] = platform_get_irq_byname_optional(pdev, "msi");
348		if (pp->msi_irq[0] < 0) {
349			pp->msi_irq[0] = platform_get_irq(pdev, 0);
350			if (pp->msi_irq[0] < 0)
351				return pp->msi_irq[0];
352		}
353	}
354
355	dev_dbg(dev, "Using %d MSI vectors\n", pp->num_vectors);
356
357	pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
358
359	ret = dw_pcie_allocate_domains(pp);
360	if (ret)
361		return ret;
362
363	for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
364		if (pp->msi_irq[ctrl] > 0)
365			irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
366						    dw_chained_msi_isr, pp);
367	}
368
369	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
370	if (ret)
371		dev_warn(dev, "Failed to set DMA mask to 32-bit. Devices with only 32-bit MSI support may not work properly\n");
372
373	msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data,
374					GFP_KERNEL);
375	if (!msi_vaddr) {
376		dev_err(dev, "Failed to alloc and map MSI data\n");
377		dw_pcie_free_msi(pp);
378		return -ENOMEM;
379	}
380
381	return 0;
382}
383
384int dw_pcie_host_init(struct dw_pcie_rp *pp)
385{
386	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
387	struct device *dev = pci->dev;
388	struct device_node *np = dev->of_node;
389	struct platform_device *pdev = to_platform_device(dev);
390	struct resource_entry *win;
391	struct pci_host_bridge *bridge;
392	struct resource *res;
393	int ret;
394
395	raw_spin_lock_init(&pp->lock);
396
397	ret = dw_pcie_get_resources(pci);
398	if (ret)
399		return ret;
400
401	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
402	if (res) {
403		pp->cfg0_size = resource_size(res);
404		pp->cfg0_base = res->start;
405
406		pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
407		if (IS_ERR(pp->va_cfg0_base))
408			return PTR_ERR(pp->va_cfg0_base);
409	} else {
410		dev_err(dev, "Missing *config* reg space\n");
411		return -ENODEV;
412	}
413
 
 
 
 
 
 
 
414	bridge = devm_pci_alloc_host_bridge(dev, 0);
415	if (!bridge)
416		return -ENOMEM;
417
418	pp->bridge = bridge;
419
420	/* Get the I/O range from DT */
421	win = resource_list_first_type(&bridge->windows, IORESOURCE_IO);
422	if (win) {
423		pp->io_size = resource_size(win->res);
424		pp->io_bus_addr = win->res->start - win->offset;
425		pp->io_base = pci_pio_to_address(win->res->start);
426	}
427
428	/* Set default bus ops */
429	bridge->ops = &dw_pcie_ops;
430	bridge->child_ops = &dw_child_pcie_ops;
431
432	if (pp->ops->host_init) {
433		ret = pp->ops->host_init(pp);
434		if (ret)
435			return ret;
436	}
437
438	if (pci_msi_enabled()) {
439		pp->has_msi_ctrl = !(pp->ops->msi_host_init ||
440				     of_property_read_bool(np, "msi-parent") ||
441				     of_property_read_bool(np, "msi-map"));
442
443		/*
444		 * For the has_msi_ctrl case the default assignment is handled
445		 * in the dw_pcie_msi_host_init().
446		 */
447		if (!pp->has_msi_ctrl && !pp->num_vectors) {
448			pp->num_vectors = MSI_DEF_NUM_VECTORS;
449		} else if (pp->num_vectors > MAX_MSI_IRQS) {
450			dev_err(dev, "Invalid number of vectors\n");
451			ret = -EINVAL;
452			goto err_deinit_host;
453		}
454
455		if (pp->ops->msi_host_init) {
456			ret = pp->ops->msi_host_init(pp);
457			if (ret < 0)
458				goto err_deinit_host;
459		} else if (pp->has_msi_ctrl) {
460			ret = dw_pcie_msi_host_init(pp);
461			if (ret < 0)
462				goto err_deinit_host;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
463		}
464	}
465
466	dw_pcie_version_detect(pci);
 
 
467
 
 
 
 
 
468	dw_pcie_iatu_detect(pci);
469
470	ret = dw_pcie_setup_rc(pp);
471	if (ret)
472		goto err_free_msi;
473
474	if (!dw_pcie_link_up(pci)) {
475		ret = dw_pcie_start_link(pci);
476		if (ret)
477			goto err_free_msi;
478	}
479
480	/* Ignore errors, the link may come up later */
481	dw_pcie_wait_for_link(pci);
482
483	bridge->sysdata = pp;
484
485	ret = pci_host_probe(bridge);
486	if (ret)
487		goto err_stop_link;
488
489	return 0;
490
491err_stop_link:
492	dw_pcie_stop_link(pci);
493
494err_free_msi:
495	if (pp->has_msi_ctrl)
496		dw_pcie_free_msi(pp);
497
498err_deinit_host:
499	if (pp->ops->host_deinit)
500		pp->ops->host_deinit(pp);
501
502	return ret;
503}
504EXPORT_SYMBOL_GPL(dw_pcie_host_init);
505
506void dw_pcie_host_deinit(struct dw_pcie_rp *pp)
507{
508	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
509
510	pci_stop_root_bus(pp->bridge->bus);
511	pci_remove_root_bus(pp->bridge->bus);
512
513	dw_pcie_stop_link(pci);
514
515	if (pp->has_msi_ctrl)
516		dw_pcie_free_msi(pp);
517
518	if (pp->ops->host_deinit)
519		pp->ops->host_deinit(pp);
520}
521EXPORT_SYMBOL_GPL(dw_pcie_host_deinit);
522
523static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus,
524						unsigned int devfn, int where)
525{
526	struct dw_pcie_rp *pp = bus->sysdata;
 
 
527	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
528	int type, ret;
529	u32 busdev;
530
531	/*
532	 * Checking whether the link is up here is a last line of defense
533	 * against platforms that forward errors on the system bus as
534	 * SError upon PCI configuration transactions issued when the link
535	 * is down. This check is racy by definition and does not stop
536	 * the system from triggering an SError if the link goes down
537	 * after this check is performed.
538	 */
539	if (!dw_pcie_link_up(pci))
540		return NULL;
541
542	busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) |
543		 PCIE_ATU_FUNC(PCI_FUNC(devfn));
544
545	if (pci_is_root_bus(bus->parent))
546		type = PCIE_ATU_TYPE_CFG0;
547	else
548		type = PCIE_ATU_TYPE_CFG1;
549
550	ret = dw_pcie_prog_outbound_atu(pci, 0, type, pp->cfg0_base, busdev,
551					pp->cfg0_size);
552	if (ret)
553		return NULL;
554
555	return pp->va_cfg0_base + where;
556}
557
558static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn,
559				 int where, int size, u32 *val)
560{
561	struct dw_pcie_rp *pp = bus->sysdata;
 
562	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
563	int ret;
564
565	ret = pci_generic_config_read(bus, devfn, where, size, val);
566	if (ret != PCIBIOS_SUCCESSFUL)
567		return ret;
568
569	if (pp->cfg0_io_shared) {
570		ret = dw_pcie_prog_outbound_atu(pci, 0, PCIE_ATU_TYPE_IO,
571						pp->io_base, pp->io_bus_addr,
572						pp->io_size);
573		if (ret)
574			return PCIBIOS_SET_FAILED;
575	}
576
577	return PCIBIOS_SUCCESSFUL;
578}
579
580static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn,
581				 int where, int size, u32 val)
582{
583	struct dw_pcie_rp *pp = bus->sysdata;
 
584	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
585	int ret;
586
587	ret = pci_generic_config_write(bus, devfn, where, size, val);
588	if (ret != PCIBIOS_SUCCESSFUL)
589		return ret;
590
591	if (pp->cfg0_io_shared) {
592		ret = dw_pcie_prog_outbound_atu(pci, 0, PCIE_ATU_TYPE_IO,
593						pp->io_base, pp->io_bus_addr,
594						pp->io_size);
595		if (ret)
596			return PCIBIOS_SET_FAILED;
597	}
598
599	return PCIBIOS_SUCCESSFUL;
600}
601
602static struct pci_ops dw_child_pcie_ops = {
603	.map_bus = dw_pcie_other_conf_map_bus,
604	.read = dw_pcie_rd_other_conf,
605	.write = dw_pcie_wr_other_conf,
606};
607
608void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where)
609{
610	struct dw_pcie_rp *pp = bus->sysdata;
611	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
612
613	if (PCI_SLOT(devfn) > 0)
614		return NULL;
615
616	return pci->dbi_base + where;
617}
618EXPORT_SYMBOL_GPL(dw_pcie_own_conf_map_bus);
619
620static struct pci_ops dw_pcie_ops = {
621	.map_bus = dw_pcie_own_conf_map_bus,
622	.read = pci_generic_config_read,
623	.write = pci_generic_config_write,
624};
625
626static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp)
627{
628	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
629	struct resource_entry *entry;
630	int i, ret;
631
632	/* Note the very first outbound ATU is used for CFG IOs */
633	if (!pci->num_ob_windows) {
634		dev_err(pci->dev, "No outbound iATU found\n");
635		return -EINVAL;
636	}
637
638	/*
639	 * Ensure all out/inbound windows are disabled before proceeding with
640	 * the MEM/IO (dma-)ranges setups.
641	 */
642	for (i = 0; i < pci->num_ob_windows; i++)
643		dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, i);
644
645	for (i = 0; i < pci->num_ib_windows; i++)
646		dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, i);
647
648	i = 0;
649	resource_list_for_each_entry(entry, &pp->bridge->windows) {
650		if (resource_type(entry->res) != IORESOURCE_MEM)
651			continue;
652
653		if (pci->num_ob_windows <= ++i)
654			break;
655
656		ret = dw_pcie_prog_outbound_atu(pci, i, PCIE_ATU_TYPE_MEM,
657						entry->res->start,
658						entry->res->start - entry->offset,
659						resource_size(entry->res));
660		if (ret) {
661			dev_err(pci->dev, "Failed to set MEM range %pr\n",
662				entry->res);
663			return ret;
664		}
665	}
666
667	if (pp->io_size) {
668		if (pci->num_ob_windows > ++i) {
669			ret = dw_pcie_prog_outbound_atu(pci, i, PCIE_ATU_TYPE_IO,
670							pp->io_base,
671							pp->io_bus_addr,
672							pp->io_size);
673			if (ret) {
674				dev_err(pci->dev, "Failed to set IO range %pr\n",
675					entry->res);
676				return ret;
677			}
678		} else {
679			pp->cfg0_io_shared = true;
680		}
681	}
682
683	if (pci->num_ob_windows <= i)
684		dev_warn(pci->dev, "Ranges exceed outbound iATU size (%d)\n",
685			 pci->num_ob_windows);
686
687	i = 0;
688	resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) {
689		if (resource_type(entry->res) != IORESOURCE_MEM)
690			continue;
691
692		if (pci->num_ib_windows <= i)
693			break;
694
695		ret = dw_pcie_prog_inbound_atu(pci, i++, PCIE_ATU_TYPE_MEM,
696					       entry->res->start,
697					       entry->res->start - entry->offset,
698					       resource_size(entry->res));
699		if (ret) {
700			dev_err(pci->dev, "Failed to set DMA range %pr\n",
701				entry->res);
702			return ret;
703		}
704	}
705
706	if (pci->num_ib_windows <= i)
707		dev_warn(pci->dev, "Dma-ranges exceed inbound iATU size (%u)\n",
708			 pci->num_ib_windows);
709
710	return 0;
711}
712
713int dw_pcie_setup_rc(struct dw_pcie_rp *pp)
714{
 
 
715	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
716	u32 val, ctrl, num_ctrls;
717	int ret;
718
719	/*
720	 * Enable DBI read-only registers for writing/updating configuration.
721	 * Write permission gets disabled towards the end of this function.
722	 */
723	dw_pcie_dbi_ro_wr_en(pci);
724
725	dw_pcie_setup(pci);
726
727	if (pp->has_msi_ctrl) {
728		num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
729
730		/* Initialize IRQ Status array */
731		for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
 
732			dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK +
733					    (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
734					    pp->irq_mask[ctrl]);
735			dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_ENABLE +
736					    (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
737					    ~0);
738		}
739	}
740
741	dw_pcie_msi_init(pp);
742
743	/* Setup RC BARs */
744	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0x00000004);
745	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000);
746
747	/* Setup interrupt pins */
748	val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE);
749	val &= 0xffff00ff;
750	val |= 0x00000100;
751	dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val);
752
753	/* Setup bus numbers */
754	val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS);
755	val &= 0xff000000;
756	val |= 0x00ff0100;
757	dw_pcie_writel_dbi(pci, PCI_PRIMARY_BUS, val);
758
759	/* Setup command register */
760	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
761	val &= 0xffff0000;
762	val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
763		PCI_COMMAND_MASTER | PCI_COMMAND_SERR;
764	dw_pcie_writel_dbi(pci, PCI_COMMAND, val);
765
 
 
 
 
766	/*
767	 * If the platform provides its own child bus config accesses, it means
768	 * the platform uses its own address translation component rather than
769	 * ATU, so we should not program the ATU here.
770	 */
771	if (pp->bridge->child_ops == &dw_child_pcie_ops) {
772		ret = dw_pcie_iatu_setup(pp);
773		if (ret)
774			return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
775	}
776
777	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
778
779	/* Program correct class for RC */
780	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
781
782	val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
783	val |= PORT_LOGIC_SPEED_CHANGE;
784	dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
785
786	dw_pcie_dbi_ro_wr_dis(pci);
787
788	return 0;
789}
790EXPORT_SYMBOL_GPL(dw_pcie_setup_rc);