Loading...
1/*
2 * probe.c - PCI detection and setup code
3 */
4
5#include <linux/kernel.h>
6#include <linux/delay.h>
7#include <linux/init.h>
8#include <linux/pci.h>
9#include <linux/slab.h>
10#include <linux/module.h>
11#include <linux/cpumask.h>
12#include <linux/pci-aspm.h>
13#include "pci.h"
14
15#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
16#define CARDBUS_RESERVE_BUSNR 3
17
18/* Ugh. Need to stop exporting this to modules. */
19LIST_HEAD(pci_root_buses);
20EXPORT_SYMBOL(pci_root_buses);
21
22
23static int find_anything(struct device *dev, void *data)
24{
25 return 1;
26}
27
28/*
29 * Some device drivers need know if pci is initiated.
30 * Basically, we think pci is not initiated when there
31 * is no device to be found on the pci_bus_type.
32 */
33int no_pci_devices(void)
34{
35 struct device *dev;
36 int no_devices;
37
38 dev = bus_find_device(&pci_bus_type, NULL, NULL, find_anything);
39 no_devices = (dev == NULL);
40 put_device(dev);
41 return no_devices;
42}
43EXPORT_SYMBOL(no_pci_devices);
44
45/*
46 * PCI Bus Class
47 */
48static void release_pcibus_dev(struct device *dev)
49{
50 struct pci_bus *pci_bus = to_pci_bus(dev);
51
52 if (pci_bus->bridge)
53 put_device(pci_bus->bridge);
54 pci_bus_remove_resources(pci_bus);
55 pci_release_bus_of_node(pci_bus);
56 kfree(pci_bus);
57}
58
59static struct class pcibus_class = {
60 .name = "pci_bus",
61 .dev_release = &release_pcibus_dev,
62 .dev_attrs = pcibus_dev_attrs,
63};
64
65static int __init pcibus_class_init(void)
66{
67 return class_register(&pcibus_class);
68}
69postcore_initcall(pcibus_class_init);
70
71static u64 pci_size(u64 base, u64 maxbase, u64 mask)
72{
73 u64 size = mask & maxbase; /* Find the significant bits */
74 if (!size)
75 return 0;
76
77 /* Get the lowest of them to find the decode size, and
78 from that the extent. */
79 size = (size & ~(size-1)) - 1;
80
81 /* base == maxbase can be valid only if the BAR has
82 already been programmed with all 1s. */
83 if (base == maxbase && ((base | size) & mask) != mask)
84 return 0;
85
86 return size;
87}
88
89static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar)
90{
91 u32 mem_type;
92 unsigned long flags;
93
94 if ((bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
95 flags = bar & ~PCI_BASE_ADDRESS_IO_MASK;
96 flags |= IORESOURCE_IO;
97 return flags;
98 }
99
100 flags = bar & ~PCI_BASE_ADDRESS_MEM_MASK;
101 flags |= IORESOURCE_MEM;
102 if (flags & PCI_BASE_ADDRESS_MEM_PREFETCH)
103 flags |= IORESOURCE_PREFETCH;
104
105 mem_type = bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK;
106 switch (mem_type) {
107 case PCI_BASE_ADDRESS_MEM_TYPE_32:
108 break;
109 case PCI_BASE_ADDRESS_MEM_TYPE_1M:
110 dev_info(&dev->dev, "1M mem BAR treated as 32-bit BAR\n");
111 break;
112 case PCI_BASE_ADDRESS_MEM_TYPE_64:
113 flags |= IORESOURCE_MEM_64;
114 break;
115 default:
116 dev_warn(&dev->dev,
117 "mem unknown type %x treated as 32-bit BAR\n",
118 mem_type);
119 break;
120 }
121 return flags;
122}
123
124/**
125 * pci_read_base - read a PCI BAR
126 * @dev: the PCI device
127 * @type: type of the BAR
128 * @res: resource buffer to be filled in
129 * @pos: BAR position in the config space
130 *
131 * Returns 1 if the BAR is 64-bit, or 0 if 32-bit.
132 */
133int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
134 struct resource *res, unsigned int pos)
135{
136 u32 l, sz, mask;
137 u16 orig_cmd;
138
139 mask = type ? PCI_ROM_ADDRESS_MASK : ~0;
140
141 if (!dev->mmio_always_on) {
142 pci_read_config_word(dev, PCI_COMMAND, &orig_cmd);
143 pci_write_config_word(dev, PCI_COMMAND,
144 orig_cmd & ~(PCI_COMMAND_MEMORY | PCI_COMMAND_IO));
145 }
146
147 res->name = pci_name(dev);
148
149 pci_read_config_dword(dev, pos, &l);
150 pci_write_config_dword(dev, pos, l | mask);
151 pci_read_config_dword(dev, pos, &sz);
152 pci_write_config_dword(dev, pos, l);
153
154 if (!dev->mmio_always_on)
155 pci_write_config_word(dev, PCI_COMMAND, orig_cmd);
156
157 /*
158 * All bits set in sz means the device isn't working properly.
159 * If the BAR isn't implemented, all bits must be 0. If it's a
160 * memory BAR or a ROM, bit 0 must be clear; if it's an io BAR, bit
161 * 1 must be clear.
162 */
163 if (!sz || sz == 0xffffffff)
164 goto fail;
165
166 /*
167 * I don't know how l can have all bits set. Copied from old code.
168 * Maybe it fixes a bug on some ancient platform.
169 */
170 if (l == 0xffffffff)
171 l = 0;
172
173 if (type == pci_bar_unknown) {
174 res->flags = decode_bar(dev, l);
175 res->flags |= IORESOURCE_SIZEALIGN;
176 if (res->flags & IORESOURCE_IO) {
177 l &= PCI_BASE_ADDRESS_IO_MASK;
178 mask = PCI_BASE_ADDRESS_IO_MASK & (u32) IO_SPACE_LIMIT;
179 } else {
180 l &= PCI_BASE_ADDRESS_MEM_MASK;
181 mask = (u32)PCI_BASE_ADDRESS_MEM_MASK;
182 }
183 } else {
184 res->flags |= (l & IORESOURCE_ROM_ENABLE);
185 l &= PCI_ROM_ADDRESS_MASK;
186 mask = (u32)PCI_ROM_ADDRESS_MASK;
187 }
188
189 if (res->flags & IORESOURCE_MEM_64) {
190 u64 l64 = l;
191 u64 sz64 = sz;
192 u64 mask64 = mask | (u64)~0 << 32;
193
194 pci_read_config_dword(dev, pos + 4, &l);
195 pci_write_config_dword(dev, pos + 4, ~0);
196 pci_read_config_dword(dev, pos + 4, &sz);
197 pci_write_config_dword(dev, pos + 4, l);
198
199 l64 |= ((u64)l << 32);
200 sz64 |= ((u64)sz << 32);
201
202 sz64 = pci_size(l64, sz64, mask64);
203
204 if (!sz64)
205 goto fail;
206
207 if ((sizeof(resource_size_t) < 8) && (sz64 > 0x100000000ULL)) {
208 dev_err(&dev->dev, "reg %x: can't handle 64-bit BAR\n",
209 pos);
210 goto fail;
211 }
212
213 if ((sizeof(resource_size_t) < 8) && l) {
214 /* Address above 32-bit boundary; disable the BAR */
215 pci_write_config_dword(dev, pos, 0);
216 pci_write_config_dword(dev, pos + 4, 0);
217 res->start = 0;
218 res->end = sz64;
219 } else {
220 res->start = l64;
221 res->end = l64 + sz64;
222 dev_printk(KERN_DEBUG, &dev->dev, "reg %x: %pR\n",
223 pos, res);
224 }
225 } else {
226 sz = pci_size(l, sz, mask);
227
228 if (!sz)
229 goto fail;
230
231 res->start = l;
232 res->end = l + sz;
233
234 dev_printk(KERN_DEBUG, &dev->dev, "reg %x: %pR\n", pos, res);
235 }
236
237 out:
238 return (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
239 fail:
240 res->flags = 0;
241 goto out;
242}
243
244static void pci_read_bases(struct pci_dev *dev, unsigned int howmany, int rom)
245{
246 unsigned int pos, reg;
247
248 for (pos = 0; pos < howmany; pos++) {
249 struct resource *res = &dev->resource[pos];
250 reg = PCI_BASE_ADDRESS_0 + (pos << 2);
251 pos += __pci_read_base(dev, pci_bar_unknown, res, reg);
252 }
253
254 if (rom) {
255 struct resource *res = &dev->resource[PCI_ROM_RESOURCE];
256 dev->rom_base_reg = rom;
257 res->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH |
258 IORESOURCE_READONLY | IORESOURCE_CACHEABLE |
259 IORESOURCE_SIZEALIGN;
260 __pci_read_base(dev, pci_bar_mem32, res, rom);
261 }
262}
263
264static void __devinit pci_read_bridge_io(struct pci_bus *child)
265{
266 struct pci_dev *dev = child->self;
267 u8 io_base_lo, io_limit_lo;
268 unsigned long base, limit;
269 struct resource *res;
270
271 res = child->resource[0];
272 pci_read_config_byte(dev, PCI_IO_BASE, &io_base_lo);
273 pci_read_config_byte(dev, PCI_IO_LIMIT, &io_limit_lo);
274 base = (io_base_lo & PCI_IO_RANGE_MASK) << 8;
275 limit = (io_limit_lo & PCI_IO_RANGE_MASK) << 8;
276
277 if ((io_base_lo & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) {
278 u16 io_base_hi, io_limit_hi;
279 pci_read_config_word(dev, PCI_IO_BASE_UPPER16, &io_base_hi);
280 pci_read_config_word(dev, PCI_IO_LIMIT_UPPER16, &io_limit_hi);
281 base |= (io_base_hi << 16);
282 limit |= (io_limit_hi << 16);
283 }
284
285 if (base && base <= limit) {
286 res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO;
287 if (!res->start)
288 res->start = base;
289 if (!res->end)
290 res->end = limit + 0xfff;
291 dev_printk(KERN_DEBUG, &dev->dev, " bridge window %pR\n", res);
292 }
293}
294
295static void __devinit pci_read_bridge_mmio(struct pci_bus *child)
296{
297 struct pci_dev *dev = child->self;
298 u16 mem_base_lo, mem_limit_lo;
299 unsigned long base, limit;
300 struct resource *res;
301
302 res = child->resource[1];
303 pci_read_config_word(dev, PCI_MEMORY_BASE, &mem_base_lo);
304 pci_read_config_word(dev, PCI_MEMORY_LIMIT, &mem_limit_lo);
305 base = (mem_base_lo & PCI_MEMORY_RANGE_MASK) << 16;
306 limit = (mem_limit_lo & PCI_MEMORY_RANGE_MASK) << 16;
307 if (base && base <= limit) {
308 res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | IORESOURCE_MEM;
309 res->start = base;
310 res->end = limit + 0xfffff;
311 dev_printk(KERN_DEBUG, &dev->dev, " bridge window %pR\n", res);
312 }
313}
314
315static void __devinit pci_read_bridge_mmio_pref(struct pci_bus *child)
316{
317 struct pci_dev *dev = child->self;
318 u16 mem_base_lo, mem_limit_lo;
319 unsigned long base, limit;
320 struct resource *res;
321
322 res = child->resource[2];
323 pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, &mem_base_lo);
324 pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, &mem_limit_lo);
325 base = (mem_base_lo & PCI_PREF_RANGE_MASK) << 16;
326 limit = (mem_limit_lo & PCI_PREF_RANGE_MASK) << 16;
327
328 if ((mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
329 u32 mem_base_hi, mem_limit_hi;
330 pci_read_config_dword(dev, PCI_PREF_BASE_UPPER32, &mem_base_hi);
331 pci_read_config_dword(dev, PCI_PREF_LIMIT_UPPER32, &mem_limit_hi);
332
333 /*
334 * Some bridges set the base > limit by default, and some
335 * (broken) BIOSes do not initialize them. If we find
336 * this, just assume they are not being used.
337 */
338 if (mem_base_hi <= mem_limit_hi) {
339#if BITS_PER_LONG == 64
340 base |= ((long) mem_base_hi) << 32;
341 limit |= ((long) mem_limit_hi) << 32;
342#else
343 if (mem_base_hi || mem_limit_hi) {
344 dev_err(&dev->dev, "can't handle 64-bit "
345 "address space for bridge\n");
346 return;
347 }
348#endif
349 }
350 }
351 if (base && base <= limit) {
352 res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) |
353 IORESOURCE_MEM | IORESOURCE_PREFETCH;
354 if (res->flags & PCI_PREF_RANGE_TYPE_64)
355 res->flags |= IORESOURCE_MEM_64;
356 res->start = base;
357 res->end = limit + 0xfffff;
358 dev_printk(KERN_DEBUG, &dev->dev, " bridge window %pR\n", res);
359 }
360}
361
362void __devinit pci_read_bridge_bases(struct pci_bus *child)
363{
364 struct pci_dev *dev = child->self;
365 struct resource *res;
366 int i;
367
368 if (pci_is_root_bus(child)) /* It's a host bus, nothing to read */
369 return;
370
371 dev_info(&dev->dev, "PCI bridge to [bus %02x-%02x]%s\n",
372 child->secondary, child->subordinate,
373 dev->transparent ? " (subtractive decode)" : "");
374
375 pci_bus_remove_resources(child);
376 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
377 child->resource[i] = &dev->resource[PCI_BRIDGE_RESOURCES+i];
378
379 pci_read_bridge_io(child);
380 pci_read_bridge_mmio(child);
381 pci_read_bridge_mmio_pref(child);
382
383 if (dev->transparent) {
384 pci_bus_for_each_resource(child->parent, res, i) {
385 if (res) {
386 pci_bus_add_resource(child, res,
387 PCI_SUBTRACTIVE_DECODE);
388 dev_printk(KERN_DEBUG, &dev->dev,
389 " bridge window %pR (subtractive decode)\n",
390 res);
391 }
392 }
393 }
394}
395
396static struct pci_bus * pci_alloc_bus(void)
397{
398 struct pci_bus *b;
399
400 b = kzalloc(sizeof(*b), GFP_KERNEL);
401 if (b) {
402 INIT_LIST_HEAD(&b->node);
403 INIT_LIST_HEAD(&b->children);
404 INIT_LIST_HEAD(&b->devices);
405 INIT_LIST_HEAD(&b->slots);
406 INIT_LIST_HEAD(&b->resources);
407 b->max_bus_speed = PCI_SPEED_UNKNOWN;
408 b->cur_bus_speed = PCI_SPEED_UNKNOWN;
409 }
410 return b;
411}
412
413static unsigned char pcix_bus_speed[] = {
414 PCI_SPEED_UNKNOWN, /* 0 */
415 PCI_SPEED_66MHz_PCIX, /* 1 */
416 PCI_SPEED_100MHz_PCIX, /* 2 */
417 PCI_SPEED_133MHz_PCIX, /* 3 */
418 PCI_SPEED_UNKNOWN, /* 4 */
419 PCI_SPEED_66MHz_PCIX_ECC, /* 5 */
420 PCI_SPEED_100MHz_PCIX_ECC, /* 6 */
421 PCI_SPEED_133MHz_PCIX_ECC, /* 7 */
422 PCI_SPEED_UNKNOWN, /* 8 */
423 PCI_SPEED_66MHz_PCIX_266, /* 9 */
424 PCI_SPEED_100MHz_PCIX_266, /* A */
425 PCI_SPEED_133MHz_PCIX_266, /* B */
426 PCI_SPEED_UNKNOWN, /* C */
427 PCI_SPEED_66MHz_PCIX_533, /* D */
428 PCI_SPEED_100MHz_PCIX_533, /* E */
429 PCI_SPEED_133MHz_PCIX_533 /* F */
430};
431
432static unsigned char pcie_link_speed[] = {
433 PCI_SPEED_UNKNOWN, /* 0 */
434 PCIE_SPEED_2_5GT, /* 1 */
435 PCIE_SPEED_5_0GT, /* 2 */
436 PCIE_SPEED_8_0GT, /* 3 */
437 PCI_SPEED_UNKNOWN, /* 4 */
438 PCI_SPEED_UNKNOWN, /* 5 */
439 PCI_SPEED_UNKNOWN, /* 6 */
440 PCI_SPEED_UNKNOWN, /* 7 */
441 PCI_SPEED_UNKNOWN, /* 8 */
442 PCI_SPEED_UNKNOWN, /* 9 */
443 PCI_SPEED_UNKNOWN, /* A */
444 PCI_SPEED_UNKNOWN, /* B */
445 PCI_SPEED_UNKNOWN, /* C */
446 PCI_SPEED_UNKNOWN, /* D */
447 PCI_SPEED_UNKNOWN, /* E */
448 PCI_SPEED_UNKNOWN /* F */
449};
450
451void pcie_update_link_speed(struct pci_bus *bus, u16 linksta)
452{
453 bus->cur_bus_speed = pcie_link_speed[linksta & 0xf];
454}
455EXPORT_SYMBOL_GPL(pcie_update_link_speed);
456
457static unsigned char agp_speeds[] = {
458 AGP_UNKNOWN,
459 AGP_1X,
460 AGP_2X,
461 AGP_4X,
462 AGP_8X
463};
464
465static enum pci_bus_speed agp_speed(int agp3, int agpstat)
466{
467 int index = 0;
468
469 if (agpstat & 4)
470 index = 3;
471 else if (agpstat & 2)
472 index = 2;
473 else if (agpstat & 1)
474 index = 1;
475 else
476 goto out;
477
478 if (agp3) {
479 index += 2;
480 if (index == 5)
481 index = 0;
482 }
483
484 out:
485 return agp_speeds[index];
486}
487
488
489static void pci_set_bus_speed(struct pci_bus *bus)
490{
491 struct pci_dev *bridge = bus->self;
492 int pos;
493
494 pos = pci_find_capability(bridge, PCI_CAP_ID_AGP);
495 if (!pos)
496 pos = pci_find_capability(bridge, PCI_CAP_ID_AGP3);
497 if (pos) {
498 u32 agpstat, agpcmd;
499
500 pci_read_config_dword(bridge, pos + PCI_AGP_STATUS, &agpstat);
501 bus->max_bus_speed = agp_speed(agpstat & 8, agpstat & 7);
502
503 pci_read_config_dword(bridge, pos + PCI_AGP_COMMAND, &agpcmd);
504 bus->cur_bus_speed = agp_speed(agpstat & 8, agpcmd & 7);
505 }
506
507 pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
508 if (pos) {
509 u16 status;
510 enum pci_bus_speed max;
511 pci_read_config_word(bridge, pos + 2, &status);
512
513 if (status & 0x8000) {
514 max = PCI_SPEED_133MHz_PCIX_533;
515 } else if (status & 0x4000) {
516 max = PCI_SPEED_133MHz_PCIX_266;
517 } else if (status & 0x0002) {
518 if (((status >> 12) & 0x3) == 2) {
519 max = PCI_SPEED_133MHz_PCIX_ECC;
520 } else {
521 max = PCI_SPEED_133MHz_PCIX;
522 }
523 } else {
524 max = PCI_SPEED_66MHz_PCIX;
525 }
526
527 bus->max_bus_speed = max;
528 bus->cur_bus_speed = pcix_bus_speed[(status >> 6) & 0xf];
529
530 return;
531 }
532
533 pos = pci_find_capability(bridge, PCI_CAP_ID_EXP);
534 if (pos) {
535 u32 linkcap;
536 u16 linksta;
537
538 pci_read_config_dword(bridge, pos + PCI_EXP_LNKCAP, &linkcap);
539 bus->max_bus_speed = pcie_link_speed[linkcap & 0xf];
540
541 pci_read_config_word(bridge, pos + PCI_EXP_LNKSTA, &linksta);
542 pcie_update_link_speed(bus, linksta);
543 }
544}
545
546
547static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
548 struct pci_dev *bridge, int busnr)
549{
550 struct pci_bus *child;
551 int i;
552
553 /*
554 * Allocate a new bus, and inherit stuff from the parent..
555 */
556 child = pci_alloc_bus();
557 if (!child)
558 return NULL;
559
560 child->parent = parent;
561 child->ops = parent->ops;
562 child->sysdata = parent->sysdata;
563 child->bus_flags = parent->bus_flags;
564
565 /* initialize some portions of the bus device, but don't register it
566 * now as the parent is not properly set up yet. This device will get
567 * registered later in pci_bus_add_devices()
568 */
569 child->dev.class = &pcibus_class;
570 dev_set_name(&child->dev, "%04x:%02x", pci_domain_nr(child), busnr);
571
572 /*
573 * Set up the primary, secondary and subordinate
574 * bus numbers.
575 */
576 child->number = child->secondary = busnr;
577 child->primary = parent->secondary;
578 child->subordinate = 0xff;
579
580 if (!bridge)
581 return child;
582
583 child->self = bridge;
584 child->bridge = get_device(&bridge->dev);
585 pci_set_bus_of_node(child);
586 pci_set_bus_speed(child);
587
588 /* Set up default resource pointers and names.. */
589 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
590 child->resource[i] = &bridge->resource[PCI_BRIDGE_RESOURCES+i];
591 child->resource[i]->name = child->name;
592 }
593 bridge->subordinate = child;
594
595 return child;
596}
597
598struct pci_bus *__ref pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev, int busnr)
599{
600 struct pci_bus *child;
601
602 child = pci_alloc_child_bus(parent, dev, busnr);
603 if (child) {
604 down_write(&pci_bus_sem);
605 list_add_tail(&child->node, &parent->children);
606 up_write(&pci_bus_sem);
607 }
608 return child;
609}
610
611static void pci_fixup_parent_subordinate_busnr(struct pci_bus *child, int max)
612{
613 struct pci_bus *parent = child->parent;
614
615 /* Attempts to fix that up are really dangerous unless
616 we're going to re-assign all bus numbers. */
617 if (!pcibios_assign_all_busses())
618 return;
619
620 while (parent->parent && parent->subordinate < max) {
621 parent->subordinate = max;
622 pci_write_config_byte(parent->self, PCI_SUBORDINATE_BUS, max);
623 parent = parent->parent;
624 }
625}
626
627/*
628 * If it's a bridge, configure it and scan the bus behind it.
629 * For CardBus bridges, we don't scan behind as the devices will
630 * be handled by the bridge driver itself.
631 *
632 * We need to process bridges in two passes -- first we scan those
633 * already configured by the BIOS and after we are done with all of
634 * them, we proceed to assigning numbers to the remaining buses in
635 * order to avoid overlaps between old and new bus numbers.
636 */
637int __devinit pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, int pass)
638{
639 struct pci_bus *child;
640 int is_cardbus = (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS);
641 u32 buses, i, j = 0;
642 u16 bctl;
643 u8 primary, secondary, subordinate;
644 int broken = 0;
645
646 pci_read_config_dword(dev, PCI_PRIMARY_BUS, &buses);
647 primary = buses & 0xFF;
648 secondary = (buses >> 8) & 0xFF;
649 subordinate = (buses >> 16) & 0xFF;
650
651 dev_dbg(&dev->dev, "scanning [bus %02x-%02x] behind bridge, pass %d\n",
652 secondary, subordinate, pass);
653
654 /* Check if setup is sensible at all */
655 if (!pass &&
656 (primary != bus->number || secondary <= bus->number)) {
657 dev_dbg(&dev->dev, "bus configuration invalid, reconfiguring\n");
658 broken = 1;
659 }
660
661 /* Disable MasterAbortMode during probing to avoid reporting
662 of bus errors (in some architectures) */
663 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &bctl);
664 pci_write_config_word(dev, PCI_BRIDGE_CONTROL,
665 bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT);
666
667 if ((secondary || subordinate) && !pcibios_assign_all_busses() &&
668 !is_cardbus && !broken) {
669 unsigned int cmax;
670 /*
671 * Bus already configured by firmware, process it in the first
672 * pass and just note the configuration.
673 */
674 if (pass)
675 goto out;
676
677 /*
678 * If we already got to this bus through a different bridge,
679 * don't re-add it. This can happen with the i450NX chipset.
680 *
681 * However, we continue to descend down the hierarchy and
682 * scan remaining child buses.
683 */
684 child = pci_find_bus(pci_domain_nr(bus), secondary);
685 if (!child) {
686 child = pci_add_new_bus(bus, dev, secondary);
687 if (!child)
688 goto out;
689 child->primary = primary;
690 child->subordinate = subordinate;
691 child->bridge_ctl = bctl;
692 }
693
694 cmax = pci_scan_child_bus(child);
695 if (cmax > max)
696 max = cmax;
697 if (child->subordinate > max)
698 max = child->subordinate;
699 } else {
700 /*
701 * We need to assign a number to this bus which we always
702 * do in the second pass.
703 */
704 if (!pass) {
705 if (pcibios_assign_all_busses() || broken)
706 /* Temporarily disable forwarding of the
707 configuration cycles on all bridges in
708 this bus segment to avoid possible
709 conflicts in the second pass between two
710 bridges programmed with overlapping
711 bus ranges. */
712 pci_write_config_dword(dev, PCI_PRIMARY_BUS,
713 buses & ~0xffffff);
714 goto out;
715 }
716
717 /* Clear errors */
718 pci_write_config_word(dev, PCI_STATUS, 0xffff);
719
720 /* Prevent assigning a bus number that already exists.
721 * This can happen when a bridge is hot-plugged, so in
722 * this case we only re-scan this bus. */
723 child = pci_find_bus(pci_domain_nr(bus), max+1);
724 if (!child) {
725 child = pci_add_new_bus(bus, dev, ++max);
726 if (!child)
727 goto out;
728 }
729 buses = (buses & 0xff000000)
730 | ((unsigned int)(child->primary) << 0)
731 | ((unsigned int)(child->secondary) << 8)
732 | ((unsigned int)(child->subordinate) << 16);
733
734 /*
735 * yenta.c forces a secondary latency timer of 176.
736 * Copy that behaviour here.
737 */
738 if (is_cardbus) {
739 buses &= ~0xff000000;
740 buses |= CARDBUS_LATENCY_TIMER << 24;
741 }
742
743 /*
744 * We need to blast all three values with a single write.
745 */
746 pci_write_config_dword(dev, PCI_PRIMARY_BUS, buses);
747
748 if (!is_cardbus) {
749 child->bridge_ctl = bctl;
750 /*
751 * Adjust subordinate busnr in parent buses.
752 * We do this before scanning for children because
753 * some devices may not be detected if the bios
754 * was lazy.
755 */
756 pci_fixup_parent_subordinate_busnr(child, max);
757 /* Now we can scan all subordinate buses... */
758 max = pci_scan_child_bus(child);
759 /*
760 * now fix it up again since we have found
761 * the real value of max.
762 */
763 pci_fixup_parent_subordinate_busnr(child, max);
764 } else {
765 /*
766 * For CardBus bridges, we leave 4 bus numbers
767 * as cards with a PCI-to-PCI bridge can be
768 * inserted later.
769 */
770 for (i=0; i<CARDBUS_RESERVE_BUSNR; i++) {
771 struct pci_bus *parent = bus;
772 if (pci_find_bus(pci_domain_nr(bus),
773 max+i+1))
774 break;
775 while (parent->parent) {
776 if ((!pcibios_assign_all_busses()) &&
777 (parent->subordinate > max) &&
778 (parent->subordinate <= max+i)) {
779 j = 1;
780 }
781 parent = parent->parent;
782 }
783 if (j) {
784 /*
785 * Often, there are two cardbus bridges
786 * -- try to leave one valid bus number
787 * for each one.
788 */
789 i /= 2;
790 break;
791 }
792 }
793 max += i;
794 pci_fixup_parent_subordinate_busnr(child, max);
795 }
796 /*
797 * Set the subordinate bus number to its real value.
798 */
799 child->subordinate = max;
800 pci_write_config_byte(dev, PCI_SUBORDINATE_BUS, max);
801 }
802
803 sprintf(child->name,
804 (is_cardbus ? "PCI CardBus %04x:%02x" : "PCI Bus %04x:%02x"),
805 pci_domain_nr(bus), child->number);
806
807 /* Has only triggered on CardBus, fixup is in yenta_socket */
808 while (bus->parent) {
809 if ((child->subordinate > bus->subordinate) ||
810 (child->number > bus->subordinate) ||
811 (child->number < bus->number) ||
812 (child->subordinate < bus->number)) {
813 dev_info(&child->dev, "[bus %02x-%02x] %s "
814 "hidden behind%s bridge %s [bus %02x-%02x]\n",
815 child->number, child->subordinate,
816 (bus->number > child->subordinate &&
817 bus->subordinate < child->number) ?
818 "wholly" : "partially",
819 bus->self->transparent ? " transparent" : "",
820 dev_name(&bus->dev),
821 bus->number, bus->subordinate);
822 }
823 bus = bus->parent;
824 }
825
826out:
827 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl);
828
829 return max;
830}
831
832/*
833 * Read interrupt line and base address registers.
834 * The architecture-dependent code can tweak these, of course.
835 */
836static void pci_read_irq(struct pci_dev *dev)
837{
838 unsigned char irq;
839
840 pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &irq);
841 dev->pin = irq;
842 if (irq)
843 pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &irq);
844 dev->irq = irq;
845}
846
847void set_pcie_port_type(struct pci_dev *pdev)
848{
849 int pos;
850 u16 reg16;
851
852 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
853 if (!pos)
854 return;
855 pdev->is_pcie = 1;
856 pdev->pcie_cap = pos;
857 pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16);
858 pdev->pcie_type = (reg16 & PCI_EXP_FLAGS_TYPE) >> 4;
859 pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, ®16);
860 pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD;
861}
862
863void set_pcie_hotplug_bridge(struct pci_dev *pdev)
864{
865 int pos;
866 u16 reg16;
867 u32 reg32;
868
869 pos = pci_pcie_cap(pdev);
870 if (!pos)
871 return;
872 pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16);
873 if (!(reg16 & PCI_EXP_FLAGS_SLOT))
874 return;
875 pci_read_config_dword(pdev, pos + PCI_EXP_SLTCAP, ®32);
876 if (reg32 & PCI_EXP_SLTCAP_HPC)
877 pdev->is_hotplug_bridge = 1;
878}
879
880#define LEGACY_IO_RESOURCE (IORESOURCE_IO | IORESOURCE_PCI_FIXED)
881
882/**
883 * pci_setup_device - fill in class and map information of a device
884 * @dev: the device structure to fill
885 *
886 * Initialize the device structure with information about the device's
887 * vendor,class,memory and IO-space addresses,IRQ lines etc.
888 * Called at initialisation of the PCI subsystem and by CardBus services.
889 * Returns 0 on success and negative if unknown type of device (not normal,
890 * bridge or CardBus).
891 */
892int pci_setup_device(struct pci_dev *dev)
893{
894 u32 class;
895 u8 hdr_type;
896 struct pci_slot *slot;
897 int pos = 0;
898
899 if (pci_read_config_byte(dev, PCI_HEADER_TYPE, &hdr_type))
900 return -EIO;
901
902 dev->sysdata = dev->bus->sysdata;
903 dev->dev.parent = dev->bus->bridge;
904 dev->dev.bus = &pci_bus_type;
905 dev->hdr_type = hdr_type & 0x7f;
906 dev->multifunction = !!(hdr_type & 0x80);
907 dev->error_state = pci_channel_io_normal;
908 set_pcie_port_type(dev);
909
910 list_for_each_entry(slot, &dev->bus->slots, list)
911 if (PCI_SLOT(dev->devfn) == slot->number)
912 dev->slot = slot;
913
914 /* Assume 32-bit PCI; let 64-bit PCI cards (which are far rarer)
915 set this higher, assuming the system even supports it. */
916 dev->dma_mask = 0xffffffff;
917
918 dev_set_name(&dev->dev, "%04x:%02x:%02x.%d", pci_domain_nr(dev->bus),
919 dev->bus->number, PCI_SLOT(dev->devfn),
920 PCI_FUNC(dev->devfn));
921
922 pci_read_config_dword(dev, PCI_CLASS_REVISION, &class);
923 dev->revision = class & 0xff;
924 class >>= 8; /* upper 3 bytes */
925 dev->class = class;
926 class >>= 8;
927
928 dev_printk(KERN_DEBUG, &dev->dev, "[%04x:%04x] type %d class %#08x\n",
929 dev->vendor, dev->device, dev->hdr_type, class);
930
931 /* need to have dev->class ready */
932 dev->cfg_size = pci_cfg_space_size(dev);
933
934 /* "Unknown power state" */
935 dev->current_state = PCI_UNKNOWN;
936
937 /* Early fixups, before probing the BARs */
938 pci_fixup_device(pci_fixup_early, dev);
939 /* device class may be changed after fixup */
940 class = dev->class >> 8;
941
942 switch (dev->hdr_type) { /* header type */
943 case PCI_HEADER_TYPE_NORMAL: /* standard header */
944 if (class == PCI_CLASS_BRIDGE_PCI)
945 goto bad;
946 pci_read_irq(dev);
947 pci_read_bases(dev, 6, PCI_ROM_ADDRESS);
948 pci_read_config_word(dev, PCI_SUBSYSTEM_VENDOR_ID, &dev->subsystem_vendor);
949 pci_read_config_word(dev, PCI_SUBSYSTEM_ID, &dev->subsystem_device);
950
951 /*
952 * Do the ugly legacy mode stuff here rather than broken chip
953 * quirk code. Legacy mode ATA controllers have fixed
954 * addresses. These are not always echoed in BAR0-3, and
955 * BAR0-3 in a few cases contain junk!
956 */
957 if (class == PCI_CLASS_STORAGE_IDE) {
958 u8 progif;
959 pci_read_config_byte(dev, PCI_CLASS_PROG, &progif);
960 if ((progif & 1) == 0) {
961 dev->resource[0].start = 0x1F0;
962 dev->resource[0].end = 0x1F7;
963 dev->resource[0].flags = LEGACY_IO_RESOURCE;
964 dev->resource[1].start = 0x3F6;
965 dev->resource[1].end = 0x3F6;
966 dev->resource[1].flags = LEGACY_IO_RESOURCE;
967 }
968 if ((progif & 4) == 0) {
969 dev->resource[2].start = 0x170;
970 dev->resource[2].end = 0x177;
971 dev->resource[2].flags = LEGACY_IO_RESOURCE;
972 dev->resource[3].start = 0x376;
973 dev->resource[3].end = 0x376;
974 dev->resource[3].flags = LEGACY_IO_RESOURCE;
975 }
976 }
977 break;
978
979 case PCI_HEADER_TYPE_BRIDGE: /* bridge header */
980 if (class != PCI_CLASS_BRIDGE_PCI)
981 goto bad;
982 /* The PCI-to-PCI bridge spec requires that subtractive
983 decoding (i.e. transparent) bridge must have programming
984 interface code of 0x01. */
985 pci_read_irq(dev);
986 dev->transparent = ((dev->class & 0xff) == 1);
987 pci_read_bases(dev, 2, PCI_ROM_ADDRESS1);
988 set_pcie_hotplug_bridge(dev);
989 pos = pci_find_capability(dev, PCI_CAP_ID_SSVID);
990 if (pos) {
991 pci_read_config_word(dev, pos + PCI_SSVID_VENDOR_ID, &dev->subsystem_vendor);
992 pci_read_config_word(dev, pos + PCI_SSVID_DEVICE_ID, &dev->subsystem_device);
993 }
994 break;
995
996 case PCI_HEADER_TYPE_CARDBUS: /* CardBus bridge header */
997 if (class != PCI_CLASS_BRIDGE_CARDBUS)
998 goto bad;
999 pci_read_irq(dev);
1000 pci_read_bases(dev, 1, 0);
1001 pci_read_config_word(dev, PCI_CB_SUBSYSTEM_VENDOR_ID, &dev->subsystem_vendor);
1002 pci_read_config_word(dev, PCI_CB_SUBSYSTEM_ID, &dev->subsystem_device);
1003 break;
1004
1005 default: /* unknown header */
1006 dev_err(&dev->dev, "unknown header type %02x, "
1007 "ignoring device\n", dev->hdr_type);
1008 return -EIO;
1009
1010 bad:
1011 dev_err(&dev->dev, "ignoring class %02x (doesn't match header "
1012 "type %02x)\n", class, dev->hdr_type);
1013 dev->class = PCI_CLASS_NOT_DEFINED;
1014 }
1015
1016 /* We found a fine healthy device, go go go... */
1017 return 0;
1018}
1019
1020static void pci_release_capabilities(struct pci_dev *dev)
1021{
1022 pci_vpd_release(dev);
1023 pci_iov_release(dev);
1024}
1025
1026/**
1027 * pci_release_dev - free a pci device structure when all users of it are finished.
1028 * @dev: device that's been disconnected
1029 *
1030 * Will be called only by the device core when all users of this pci device are
1031 * done.
1032 */
1033static void pci_release_dev(struct device *dev)
1034{
1035 struct pci_dev *pci_dev;
1036
1037 pci_dev = to_pci_dev(dev);
1038 pci_release_capabilities(pci_dev);
1039 pci_release_of_node(pci_dev);
1040 kfree(pci_dev);
1041}
1042
1043/**
1044 * pci_cfg_space_size - get the configuration space size of the PCI device.
1045 * @dev: PCI device
1046 *
1047 * Regular PCI devices have 256 bytes, but PCI-X 2 and PCI Express devices
1048 * have 4096 bytes. Even if the device is capable, that doesn't mean we can
1049 * access it. Maybe we don't have a way to generate extended config space
1050 * accesses, or the device is behind a reverse Express bridge. So we try
1051 * reading the dword at 0x100 which must either be 0 or a valid extended
1052 * capability header.
1053 */
1054int pci_cfg_space_size_ext(struct pci_dev *dev)
1055{
1056 u32 status;
1057 int pos = PCI_CFG_SPACE_SIZE;
1058
1059 if (pci_read_config_dword(dev, pos, &status) != PCIBIOS_SUCCESSFUL)
1060 goto fail;
1061 if (status == 0xffffffff)
1062 goto fail;
1063
1064 return PCI_CFG_SPACE_EXP_SIZE;
1065
1066 fail:
1067 return PCI_CFG_SPACE_SIZE;
1068}
1069
1070int pci_cfg_space_size(struct pci_dev *dev)
1071{
1072 int pos;
1073 u32 status;
1074 u16 class;
1075
1076 class = dev->class >> 8;
1077 if (class == PCI_CLASS_BRIDGE_HOST)
1078 return pci_cfg_space_size_ext(dev);
1079
1080 pos = pci_pcie_cap(dev);
1081 if (!pos) {
1082 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1083 if (!pos)
1084 goto fail;
1085
1086 pci_read_config_dword(dev, pos + PCI_X_STATUS, &status);
1087 if (!(status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ)))
1088 goto fail;
1089 }
1090
1091 return pci_cfg_space_size_ext(dev);
1092
1093 fail:
1094 return PCI_CFG_SPACE_SIZE;
1095}
1096
1097static void pci_release_bus_bridge_dev(struct device *dev)
1098{
1099 kfree(dev);
1100}
1101
1102struct pci_dev *alloc_pci_dev(void)
1103{
1104 struct pci_dev *dev;
1105
1106 dev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL);
1107 if (!dev)
1108 return NULL;
1109
1110 INIT_LIST_HEAD(&dev->bus_list);
1111
1112 return dev;
1113}
1114EXPORT_SYMBOL(alloc_pci_dev);
1115
1116/*
1117 * Read the config data for a PCI device, sanity-check it
1118 * and fill in the dev structure...
1119 */
1120static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
1121{
1122 struct pci_dev *dev;
1123 u32 l;
1124 int delay = 1;
1125
1126 if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, &l))
1127 return NULL;
1128
1129 /* some broken boards return 0 or ~0 if a slot is empty: */
1130 if (l == 0xffffffff || l == 0x00000000 ||
1131 l == 0x0000ffff || l == 0xffff0000)
1132 return NULL;
1133
1134 /* Configuration request Retry Status */
1135 while (l == 0xffff0001) {
1136 msleep(delay);
1137 delay *= 2;
1138 if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, &l))
1139 return NULL;
1140 /* Card hasn't responded in 60 seconds? Must be stuck. */
1141 if (delay > 60 * 1000) {
1142 printk(KERN_WARNING "pci %04x:%02x:%02x.%d: not "
1143 "responding\n", pci_domain_nr(bus),
1144 bus->number, PCI_SLOT(devfn),
1145 PCI_FUNC(devfn));
1146 return NULL;
1147 }
1148 }
1149
1150 dev = alloc_pci_dev();
1151 if (!dev)
1152 return NULL;
1153
1154 dev->bus = bus;
1155 dev->devfn = devfn;
1156 dev->vendor = l & 0xffff;
1157 dev->device = (l >> 16) & 0xffff;
1158
1159 pci_set_of_node(dev);
1160
1161 if (pci_setup_device(dev)) {
1162 kfree(dev);
1163 return NULL;
1164 }
1165
1166 return dev;
1167}
1168
1169static void pci_init_capabilities(struct pci_dev *dev)
1170{
1171 /* MSI/MSI-X list */
1172 pci_msi_init_pci_dev(dev);
1173
1174 /* Buffers for saving PCIe and PCI-X capabilities */
1175 pci_allocate_cap_save_buffers(dev);
1176
1177 /* Power Management */
1178 pci_pm_init(dev);
1179 platform_pci_wakeup_init(dev);
1180
1181 /* Vital Product Data */
1182 pci_vpd_pci22_init(dev);
1183
1184 /* Alternative Routing-ID Forwarding */
1185 pci_enable_ari(dev);
1186
1187 /* Single Root I/O Virtualization */
1188 pci_iov_init(dev);
1189
1190 /* Enable ACS P2P upstream forwarding */
1191 pci_enable_acs(dev);
1192}
1193
1194void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
1195{
1196 device_initialize(&dev->dev);
1197 dev->dev.release = pci_release_dev;
1198 pci_dev_get(dev);
1199
1200 dev->dev.dma_mask = &dev->dma_mask;
1201 dev->dev.dma_parms = &dev->dma_parms;
1202 dev->dev.coherent_dma_mask = 0xffffffffull;
1203
1204 pci_set_dma_max_seg_size(dev, 65536);
1205 pci_set_dma_seg_boundary(dev, 0xffffffff);
1206
1207 /* Fix up broken headers */
1208 pci_fixup_device(pci_fixup_header, dev);
1209
1210 /* Clear the state_saved flag. */
1211 dev->state_saved = false;
1212
1213 /* Initialize various capabilities */
1214 pci_init_capabilities(dev);
1215
1216 /*
1217 * Add the device to our list of discovered devices
1218 * and the bus list for fixup functions, etc.
1219 */
1220 down_write(&pci_bus_sem);
1221 list_add_tail(&dev->bus_list, &bus->devices);
1222 up_write(&pci_bus_sem);
1223}
1224
1225struct pci_dev *__ref pci_scan_single_device(struct pci_bus *bus, int devfn)
1226{
1227 struct pci_dev *dev;
1228
1229 dev = pci_get_slot(bus, devfn);
1230 if (dev) {
1231 pci_dev_put(dev);
1232 return dev;
1233 }
1234
1235 dev = pci_scan_device(bus, devfn);
1236 if (!dev)
1237 return NULL;
1238
1239 pci_device_add(dev, bus);
1240
1241 return dev;
1242}
1243EXPORT_SYMBOL(pci_scan_single_device);
1244
1245static unsigned next_ari_fn(struct pci_dev *dev, unsigned fn)
1246{
1247 u16 cap;
1248 unsigned pos, next_fn;
1249
1250 if (!dev)
1251 return 0;
1252
1253 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI);
1254 if (!pos)
1255 return 0;
1256 pci_read_config_word(dev, pos + 4, &cap);
1257 next_fn = cap >> 8;
1258 if (next_fn <= fn)
1259 return 0;
1260 return next_fn;
1261}
1262
1263static unsigned next_trad_fn(struct pci_dev *dev, unsigned fn)
1264{
1265 return (fn + 1) % 8;
1266}
1267
1268static unsigned no_next_fn(struct pci_dev *dev, unsigned fn)
1269{
1270 return 0;
1271}
1272
1273static int only_one_child(struct pci_bus *bus)
1274{
1275 struct pci_dev *parent = bus->self;
1276 if (!parent || !pci_is_pcie(parent))
1277 return 0;
1278 if (parent->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
1279 parent->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
1280 return 1;
1281 return 0;
1282}
1283
1284/**
1285 * pci_scan_slot - scan a PCI slot on a bus for devices.
1286 * @bus: PCI bus to scan
1287 * @devfn: slot number to scan (must have zero function.)
1288 *
1289 * Scan a PCI slot on the specified PCI bus for devices, adding
1290 * discovered devices to the @bus->devices list. New devices
1291 * will not have is_added set.
1292 *
1293 * Returns the number of new devices found.
1294 */
1295int pci_scan_slot(struct pci_bus *bus, int devfn)
1296{
1297 unsigned fn, nr = 0;
1298 struct pci_dev *dev;
1299 unsigned (*next_fn)(struct pci_dev *, unsigned) = no_next_fn;
1300
1301 if (only_one_child(bus) && (devfn > 0))
1302 return 0; /* Already scanned the entire slot */
1303
1304 dev = pci_scan_single_device(bus, devfn);
1305 if (!dev)
1306 return 0;
1307 if (!dev->is_added)
1308 nr++;
1309
1310 if (pci_ari_enabled(bus))
1311 next_fn = next_ari_fn;
1312 else if (dev->multifunction)
1313 next_fn = next_trad_fn;
1314
1315 for (fn = next_fn(dev, 0); fn > 0; fn = next_fn(dev, fn)) {
1316 dev = pci_scan_single_device(bus, devfn + fn);
1317 if (dev) {
1318 if (!dev->is_added)
1319 nr++;
1320 dev->multifunction = 1;
1321 }
1322 }
1323
1324 /* only one slot has pcie device */
1325 if (bus->self && nr)
1326 pcie_aspm_init_link_state(bus->self);
1327
1328 return nr;
1329}
1330
1331static int pcie_find_smpss(struct pci_dev *dev, void *data)
1332{
1333 u8 *smpss = data;
1334
1335 if (!pci_is_pcie(dev))
1336 return 0;
1337
1338 /* For PCIE hotplug enabled slots not connected directly to a
1339 * PCI-E root port, there can be problems when hotplugging
1340 * devices. This is due to the possibility of hotplugging a
1341 * device into the fabric with a smaller MPS that the devices
1342 * currently running have configured. Modifying the MPS on the
1343 * running devices could cause a fatal bus error due to an
1344 * incoming frame being larger than the newly configured MPS.
1345 * To work around this, the MPS for the entire fabric must be
1346 * set to the minimum size. Any devices hotplugged into this
1347 * fabric will have the minimum MPS set. If the PCI hotplug
1348 * slot is directly connected to the root port and there are not
1349 * other devices on the fabric (which seems to be the most
1350 * common case), then this is not an issue and MPS discovery
1351 * will occur as normal.
1352 */
1353 if (dev->is_hotplug_bridge && (!list_is_singular(&dev->bus->devices) ||
1354 (dev->bus->self &&
1355 dev->bus->self->pcie_type != PCI_EXP_TYPE_ROOT_PORT)))
1356 *smpss = 0;
1357
1358 if (*smpss > dev->pcie_mpss)
1359 *smpss = dev->pcie_mpss;
1360
1361 return 0;
1362}
1363
1364static void pcie_write_mps(struct pci_dev *dev, int mps)
1365{
1366 int rc, dev_mpss;
1367
1368 dev_mpss = 128 << dev->pcie_mpss;
1369
1370 if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
1371 if (dev->bus->self) {
1372 dev_dbg(&dev->bus->dev, "Bus MPSS %d\n",
1373 128 << dev->bus->self->pcie_mpss);
1374
1375 /* For "MPS Force Max", the assumption is made that
1376 * downstream communication will never be larger than
1377 * the MRRS. So, the MPS only needs to be configured
1378 * for the upstream communication. This being the case,
1379 * walk from the top down and set the MPS of the child
1380 * to that of the parent bus.
1381 */
1382 mps = 128 << dev->bus->self->pcie_mpss;
1383 if (mps > dev_mpss)
1384 dev_warn(&dev->dev, "MPS configured higher than"
1385 " maximum supported by the device. If"
1386 " a bus issue occurs, try running with"
1387 " pci=pcie_bus_safe.\n");
1388 }
1389
1390 dev->pcie_mpss = ffs(mps) - 8;
1391 }
1392
1393 rc = pcie_set_mps(dev, mps);
1394 if (rc)
1395 dev_err(&dev->dev, "Failed attempting to set the MPS\n");
1396}
1397
1398static void pcie_write_mrrs(struct pci_dev *dev, int mps)
1399{
1400 int rc, mrrs, dev_mpss;
1401
1402 /* In the "safe" case, do not configure the MRRS. There appear to be
1403 * issues with setting MRRS to 0 on a number of devices.
1404 */
1405
1406 if (pcie_bus_config != PCIE_BUS_PERFORMANCE)
1407 return;
1408
1409 dev_mpss = 128 << dev->pcie_mpss;
1410
1411 /* For Max performance, the MRRS must be set to the largest supported
1412 * value. However, it cannot be configured larger than the MPS the
1413 * device or the bus can support. This assumes that the largest MRRS
1414 * available on the device cannot be smaller than the device MPSS.
1415 */
1416 mrrs = min(mps, dev_mpss);
1417
1418 /* MRRS is a R/W register. Invalid values can be written, but a
1419 * subsequent read will verify if the value is acceptable or not.
1420 * If the MRRS value provided is not acceptable (e.g., too large),
1421 * shrink the value until it is acceptable to the HW.
1422 */
1423 while (mrrs != pcie_get_readrq(dev) && mrrs >= 128) {
1424 dev_warn(&dev->dev, "Attempting to modify the PCI-E MRRS value"
1425 " to %d. If any issues are encountered, please try "
1426 "running with pci=pcie_bus_safe\n", mrrs);
1427 rc = pcie_set_readrq(dev, mrrs);
1428 if (rc)
1429 dev_err(&dev->dev,
1430 "Failed attempting to set the MRRS\n");
1431
1432 mrrs /= 2;
1433 }
1434}
1435
1436static int pcie_bus_configure_set(struct pci_dev *dev, void *data)
1437{
1438 int mps = 128 << *(u8 *)data;
1439
1440 if (!pci_is_pcie(dev))
1441 return 0;
1442
1443 dev_dbg(&dev->dev, "Dev MPS %d MPSS %d MRRS %d\n",
1444 pcie_get_mps(dev), 128<<dev->pcie_mpss, pcie_get_readrq(dev));
1445
1446 pcie_write_mps(dev, mps);
1447 pcie_write_mrrs(dev, mps);
1448
1449 dev_dbg(&dev->dev, "Dev MPS %d MPSS %d MRRS %d\n",
1450 pcie_get_mps(dev), 128<<dev->pcie_mpss, pcie_get_readrq(dev));
1451
1452 return 0;
1453}
1454
1455/* pcie_bus_configure_mps requires that pci_walk_bus work in a top-down,
1456 * parents then children fashion. If this changes, then this code will not
1457 * work as designed.
1458 */
1459void pcie_bus_configure_settings(struct pci_bus *bus, u8 mpss)
1460{
1461 u8 smpss;
1462
1463 if (!pci_is_pcie(bus->self))
1464 return;
1465
1466 if (pcie_bus_config == PCIE_BUS_TUNE_OFF)
1467 return;
1468
1469 /* FIXME - Peer to peer DMA is possible, though the endpoint would need
1470 * to be aware to the MPS of the destination. To work around this,
1471 * simply force the MPS of the entire system to the smallest possible.
1472 */
1473 if (pcie_bus_config == PCIE_BUS_PEER2PEER)
1474 smpss = 0;
1475
1476 if (pcie_bus_config == PCIE_BUS_SAFE) {
1477 smpss = mpss;
1478
1479 pcie_find_smpss(bus->self, &smpss);
1480 pci_walk_bus(bus, pcie_find_smpss, &smpss);
1481 }
1482
1483 pcie_bus_configure_set(bus->self, &smpss);
1484 pci_walk_bus(bus, pcie_bus_configure_set, &smpss);
1485}
1486EXPORT_SYMBOL_GPL(pcie_bus_configure_settings);
1487
1488unsigned int __devinit pci_scan_child_bus(struct pci_bus *bus)
1489{
1490 unsigned int devfn, pass, max = bus->secondary;
1491 struct pci_dev *dev;
1492
1493 dev_dbg(&bus->dev, "scanning bus\n");
1494
1495 /* Go find them, Rover! */
1496 for (devfn = 0; devfn < 0x100; devfn += 8)
1497 pci_scan_slot(bus, devfn);
1498
1499 /* Reserve buses for SR-IOV capability. */
1500 max += pci_iov_bus_range(bus);
1501
1502 /*
1503 * After performing arch-dependent fixup of the bus, look behind
1504 * all PCI-to-PCI bridges on this bus.
1505 */
1506 if (!bus->is_added) {
1507 dev_dbg(&bus->dev, "fixups for bus\n");
1508 pcibios_fixup_bus(bus);
1509 if (pci_is_root_bus(bus))
1510 bus->is_added = 1;
1511 }
1512
1513 for (pass=0; pass < 2; pass++)
1514 list_for_each_entry(dev, &bus->devices, bus_list) {
1515 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE ||
1516 dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)
1517 max = pci_scan_bridge(bus, dev, max, pass);
1518 }
1519
1520 /*
1521 * We've scanned the bus and so we know all about what's on
1522 * the other side of any bridges that may be on this bus plus
1523 * any devices.
1524 *
1525 * Return how far we've got finding sub-buses.
1526 */
1527 dev_dbg(&bus->dev, "bus scan returning with max=%02x\n", max);
1528 return max;
1529}
1530
1531struct pci_bus * pci_create_bus(struct device *parent,
1532 int bus, struct pci_ops *ops, void *sysdata)
1533{
1534 int error;
1535 struct pci_bus *b, *b2;
1536 struct device *dev;
1537
1538 b = pci_alloc_bus();
1539 if (!b)
1540 return NULL;
1541
1542 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1543 if (!dev){
1544 kfree(b);
1545 return NULL;
1546 }
1547
1548 b->sysdata = sysdata;
1549 b->ops = ops;
1550
1551 b2 = pci_find_bus(pci_domain_nr(b), bus);
1552 if (b2) {
1553 /* If we already got to this bus through a different bridge, ignore it */
1554 dev_dbg(&b2->dev, "bus already known\n");
1555 goto err_out;
1556 }
1557
1558 down_write(&pci_bus_sem);
1559 list_add_tail(&b->node, &pci_root_buses);
1560 up_write(&pci_bus_sem);
1561
1562 dev->parent = parent;
1563 dev->release = pci_release_bus_bridge_dev;
1564 dev_set_name(dev, "pci%04x:%02x", pci_domain_nr(b), bus);
1565 error = device_register(dev);
1566 if (error)
1567 goto dev_reg_err;
1568 b->bridge = get_device(dev);
1569 device_enable_async_suspend(b->bridge);
1570 pci_set_bus_of_node(b);
1571
1572 if (!parent)
1573 set_dev_node(b->bridge, pcibus_to_node(b));
1574
1575 b->dev.class = &pcibus_class;
1576 b->dev.parent = b->bridge;
1577 dev_set_name(&b->dev, "%04x:%02x", pci_domain_nr(b), bus);
1578 error = device_register(&b->dev);
1579 if (error)
1580 goto class_dev_reg_err;
1581
1582 /* Create legacy_io and legacy_mem files for this bus */
1583 pci_create_legacy_files(b);
1584
1585 b->number = b->secondary = bus;
1586 b->resource[0] = &ioport_resource;
1587 b->resource[1] = &iomem_resource;
1588
1589 return b;
1590
1591class_dev_reg_err:
1592 device_unregister(dev);
1593dev_reg_err:
1594 down_write(&pci_bus_sem);
1595 list_del(&b->node);
1596 up_write(&pci_bus_sem);
1597err_out:
1598 kfree(dev);
1599 kfree(b);
1600 return NULL;
1601}
1602
1603struct pci_bus * __devinit pci_scan_bus_parented(struct device *parent,
1604 int bus, struct pci_ops *ops, void *sysdata)
1605{
1606 struct pci_bus *b;
1607
1608 b = pci_create_bus(parent, bus, ops, sysdata);
1609 if (b)
1610 b->subordinate = pci_scan_child_bus(b);
1611 return b;
1612}
1613EXPORT_SYMBOL(pci_scan_bus_parented);
1614
1615#ifdef CONFIG_HOTPLUG
1616/**
1617 * pci_rescan_bus - scan a PCI bus for devices.
1618 * @bus: PCI bus to scan
1619 *
1620 * Scan a PCI bus and child buses for new devices, adds them,
1621 * and enables them.
1622 *
1623 * Returns the max number of subordinate bus discovered.
1624 */
1625unsigned int __ref pci_rescan_bus(struct pci_bus *bus)
1626{
1627 unsigned int max;
1628 struct pci_dev *dev;
1629
1630 max = pci_scan_child_bus(bus);
1631
1632 down_read(&pci_bus_sem);
1633 list_for_each_entry(dev, &bus->devices, bus_list)
1634 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE ||
1635 dev->hdr_type == PCI_HEADER_TYPE_CARDBUS)
1636 if (dev->subordinate)
1637 pci_bus_size_bridges(dev->subordinate);
1638 up_read(&pci_bus_sem);
1639
1640 pci_bus_assign_resources(bus);
1641 pci_enable_bridges(bus);
1642 pci_bus_add_devices(bus);
1643
1644 return max;
1645}
1646EXPORT_SYMBOL_GPL(pci_rescan_bus);
1647
1648EXPORT_SYMBOL(pci_add_new_bus);
1649EXPORT_SYMBOL(pci_scan_slot);
1650EXPORT_SYMBOL(pci_scan_bridge);
1651EXPORT_SYMBOL_GPL(pci_scan_child_bus);
1652#endif
1653
1654static int __init pci_sort_bf_cmp(const struct device *d_a, const struct device *d_b)
1655{
1656 const struct pci_dev *a = to_pci_dev(d_a);
1657 const struct pci_dev *b = to_pci_dev(d_b);
1658
1659 if (pci_domain_nr(a->bus) < pci_domain_nr(b->bus)) return -1;
1660 else if (pci_domain_nr(a->bus) > pci_domain_nr(b->bus)) return 1;
1661
1662 if (a->bus->number < b->bus->number) return -1;
1663 else if (a->bus->number > b->bus->number) return 1;
1664
1665 if (a->devfn < b->devfn) return -1;
1666 else if (a->devfn > b->devfn) return 1;
1667
1668 return 0;
1669}
1670
1671void __init pci_sort_breadthfirst(void)
1672{
1673 bus_sort_breadthfirst(&pci_bus_type, &pci_sort_bf_cmp);
1674}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * PCI detection and setup code
4 */
5
6#include <linux/kernel.h>
7#include <linux/delay.h>
8#include <linux/init.h>
9#include <linux/pci.h>
10#include <linux/msi.h>
11#include <linux/of_pci.h>
12#include <linux/pci_hotplug.h>
13#include <linux/slab.h>
14#include <linux/module.h>
15#include <linux/cpumask.h>
16#include <linux/aer.h>
17#include <linux/acpi.h>
18#include <linux/hypervisor.h>
19#include <linux/irqdomain.h>
20#include <linux/pm_runtime.h>
21#include <linux/bitfield.h>
22#include "pci.h"
23
24#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
25#define CARDBUS_RESERVE_BUSNR 3
26
27static struct resource busn_resource = {
28 .name = "PCI busn",
29 .start = 0,
30 .end = 255,
31 .flags = IORESOURCE_BUS,
32};
33
34/* Ugh. Need to stop exporting this to modules. */
35LIST_HEAD(pci_root_buses);
36EXPORT_SYMBOL(pci_root_buses);
37
38static LIST_HEAD(pci_domain_busn_res_list);
39
40struct pci_domain_busn_res {
41 struct list_head list;
42 struct resource res;
43 int domain_nr;
44};
45
46static struct resource *get_pci_domain_busn_res(int domain_nr)
47{
48 struct pci_domain_busn_res *r;
49
50 list_for_each_entry(r, &pci_domain_busn_res_list, list)
51 if (r->domain_nr == domain_nr)
52 return &r->res;
53
54 r = kzalloc(sizeof(*r), GFP_KERNEL);
55 if (!r)
56 return NULL;
57
58 r->domain_nr = domain_nr;
59 r->res.start = 0;
60 r->res.end = 0xff;
61 r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
62
63 list_add_tail(&r->list, &pci_domain_busn_res_list);
64
65 return &r->res;
66}
67
68/*
69 * Some device drivers need know if PCI is initiated.
70 * Basically, we think PCI is not initiated when there
71 * is no device to be found on the pci_bus_type.
72 */
73int no_pci_devices(void)
74{
75 struct device *dev;
76 int no_devices;
77
78 dev = bus_find_next_device(&pci_bus_type, NULL);
79 no_devices = (dev == NULL);
80 put_device(dev);
81 return no_devices;
82}
83EXPORT_SYMBOL(no_pci_devices);
84
85/*
86 * PCI Bus Class
87 */
88static void release_pcibus_dev(struct device *dev)
89{
90 struct pci_bus *pci_bus = to_pci_bus(dev);
91
92 put_device(pci_bus->bridge);
93 pci_bus_remove_resources(pci_bus);
94 pci_release_bus_of_node(pci_bus);
95 kfree(pci_bus);
96}
97
98static struct class pcibus_class = {
99 .name = "pci_bus",
100 .dev_release = &release_pcibus_dev,
101 .dev_groups = pcibus_groups,
102};
103
104static int __init pcibus_class_init(void)
105{
106 return class_register(&pcibus_class);
107}
108postcore_initcall(pcibus_class_init);
109
110static u64 pci_size(u64 base, u64 maxbase, u64 mask)
111{
112 u64 size = mask & maxbase; /* Find the significant bits */
113 if (!size)
114 return 0;
115
116 /*
117 * Get the lowest of them to find the decode size, and from that
118 * the extent.
119 */
120 size = size & ~(size-1);
121
122 /*
123 * base == maxbase can be valid only if the BAR has already been
124 * programmed with all 1s.
125 */
126 if (base == maxbase && ((base | (size - 1)) & mask) != mask)
127 return 0;
128
129 return size;
130}
131
132static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar)
133{
134 u32 mem_type;
135 unsigned long flags;
136
137 if ((bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
138 flags = bar & ~PCI_BASE_ADDRESS_IO_MASK;
139 flags |= IORESOURCE_IO;
140 return flags;
141 }
142
143 flags = bar & ~PCI_BASE_ADDRESS_MEM_MASK;
144 flags |= IORESOURCE_MEM;
145 if (flags & PCI_BASE_ADDRESS_MEM_PREFETCH)
146 flags |= IORESOURCE_PREFETCH;
147
148 mem_type = bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK;
149 switch (mem_type) {
150 case PCI_BASE_ADDRESS_MEM_TYPE_32:
151 break;
152 case PCI_BASE_ADDRESS_MEM_TYPE_1M:
153 /* 1M mem BAR treated as 32-bit BAR */
154 break;
155 case PCI_BASE_ADDRESS_MEM_TYPE_64:
156 flags |= IORESOURCE_MEM_64;
157 break;
158 default:
159 /* mem unknown type treated as 32-bit BAR */
160 break;
161 }
162 return flags;
163}
164
165#define PCI_COMMAND_DECODE_ENABLE (PCI_COMMAND_MEMORY | PCI_COMMAND_IO)
166
167/**
168 * __pci_read_base - Read a PCI BAR
169 * @dev: the PCI device
170 * @type: type of the BAR
171 * @res: resource buffer to be filled in
172 * @pos: BAR position in the config space
173 *
174 * Returns 1 if the BAR is 64-bit, or 0 if 32-bit.
175 */
176int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
177 struct resource *res, unsigned int pos)
178{
179 u32 l = 0, sz = 0, mask;
180 u64 l64, sz64, mask64;
181 u16 orig_cmd;
182 struct pci_bus_region region, inverted_region;
183 const char *res_name = pci_resource_name(dev, res - dev->resource);
184
185 mask = type ? PCI_ROM_ADDRESS_MASK : ~0;
186
187 /* No printks while decoding is disabled! */
188 if (!dev->mmio_always_on) {
189 pci_read_config_word(dev, PCI_COMMAND, &orig_cmd);
190 if (orig_cmd & PCI_COMMAND_DECODE_ENABLE) {
191 pci_write_config_word(dev, PCI_COMMAND,
192 orig_cmd & ~PCI_COMMAND_DECODE_ENABLE);
193 }
194 }
195
196 res->name = pci_name(dev);
197
198 pci_read_config_dword(dev, pos, &l);
199 pci_write_config_dword(dev, pos, l | mask);
200 pci_read_config_dword(dev, pos, &sz);
201 pci_write_config_dword(dev, pos, l);
202
203 /*
204 * All bits set in sz means the device isn't working properly.
205 * If the BAR isn't implemented, all bits must be 0. If it's a
206 * memory BAR or a ROM, bit 0 must be clear; if it's an io BAR, bit
207 * 1 must be clear.
208 */
209 if (PCI_POSSIBLE_ERROR(sz))
210 sz = 0;
211
212 /*
213 * I don't know how l can have all bits set. Copied from old code.
214 * Maybe it fixes a bug on some ancient platform.
215 */
216 if (PCI_POSSIBLE_ERROR(l))
217 l = 0;
218
219 if (type == pci_bar_unknown) {
220 res->flags = decode_bar(dev, l);
221 res->flags |= IORESOURCE_SIZEALIGN;
222 if (res->flags & IORESOURCE_IO) {
223 l64 = l & PCI_BASE_ADDRESS_IO_MASK;
224 sz64 = sz & PCI_BASE_ADDRESS_IO_MASK;
225 mask64 = PCI_BASE_ADDRESS_IO_MASK & (u32)IO_SPACE_LIMIT;
226 } else {
227 l64 = l & PCI_BASE_ADDRESS_MEM_MASK;
228 sz64 = sz & PCI_BASE_ADDRESS_MEM_MASK;
229 mask64 = (u32)PCI_BASE_ADDRESS_MEM_MASK;
230 }
231 } else {
232 if (l & PCI_ROM_ADDRESS_ENABLE)
233 res->flags |= IORESOURCE_ROM_ENABLE;
234 l64 = l & PCI_ROM_ADDRESS_MASK;
235 sz64 = sz & PCI_ROM_ADDRESS_MASK;
236 mask64 = PCI_ROM_ADDRESS_MASK;
237 }
238
239 if (res->flags & IORESOURCE_MEM_64) {
240 pci_read_config_dword(dev, pos + 4, &l);
241 pci_write_config_dword(dev, pos + 4, ~0);
242 pci_read_config_dword(dev, pos + 4, &sz);
243 pci_write_config_dword(dev, pos + 4, l);
244
245 l64 |= ((u64)l << 32);
246 sz64 |= ((u64)sz << 32);
247 mask64 |= ((u64)~0 << 32);
248 }
249
250 if (!dev->mmio_always_on && (orig_cmd & PCI_COMMAND_DECODE_ENABLE))
251 pci_write_config_word(dev, PCI_COMMAND, orig_cmd);
252
253 if (!sz64)
254 goto fail;
255
256 sz64 = pci_size(l64, sz64, mask64);
257 if (!sz64) {
258 pci_info(dev, FW_BUG "%s: invalid; can't size\n", res_name);
259 goto fail;
260 }
261
262 if (res->flags & IORESOURCE_MEM_64) {
263 if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8)
264 && sz64 > 0x100000000ULL) {
265 res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
266 res->start = 0;
267 res->end = 0;
268 pci_err(dev, "%s: can't handle BAR larger than 4GB (size %#010llx)\n",
269 res_name, (unsigned long long)sz64);
270 goto out;
271 }
272
273 if ((sizeof(pci_bus_addr_t) < 8) && l) {
274 /* Above 32-bit boundary; try to reallocate */
275 res->flags |= IORESOURCE_UNSET;
276 res->start = 0;
277 res->end = sz64 - 1;
278 pci_info(dev, "%s: can't handle BAR above 4GB (bus address %#010llx)\n",
279 res_name, (unsigned long long)l64);
280 goto out;
281 }
282 }
283
284 region.start = l64;
285 region.end = l64 + sz64 - 1;
286
287 pcibios_bus_to_resource(dev->bus, res, ®ion);
288 pcibios_resource_to_bus(dev->bus, &inverted_region, res);
289
290 /*
291 * If "A" is a BAR value (a bus address), "bus_to_resource(A)" is
292 * the corresponding resource address (the physical address used by
293 * the CPU. Converting that resource address back to a bus address
294 * should yield the original BAR value:
295 *
296 * resource_to_bus(bus_to_resource(A)) == A
297 *
298 * If it doesn't, CPU accesses to "bus_to_resource(A)" will not
299 * be claimed by the device.
300 */
301 if (inverted_region.start != region.start) {
302 res->flags |= IORESOURCE_UNSET;
303 res->start = 0;
304 res->end = region.end - region.start;
305 pci_info(dev, "%s: initial BAR value %#010llx invalid\n",
306 res_name, (unsigned long long)region.start);
307 }
308
309 goto out;
310
311
312fail:
313 res->flags = 0;
314out:
315 if (res->flags)
316 pci_info(dev, "%s %pR\n", res_name, res);
317
318 return (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
319}
320
321static void pci_read_bases(struct pci_dev *dev, unsigned int howmany, int rom)
322{
323 unsigned int pos, reg;
324
325 if (dev->non_compliant_bars)
326 return;
327
328 /* Per PCIe r4.0, sec 9.3.4.1.11, the VF BARs are all RO Zero */
329 if (dev->is_virtfn)
330 return;
331
332 for (pos = 0; pos < howmany; pos++) {
333 struct resource *res = &dev->resource[pos];
334 reg = PCI_BASE_ADDRESS_0 + (pos << 2);
335 pos += __pci_read_base(dev, pci_bar_unknown, res, reg);
336 }
337
338 if (rom) {
339 struct resource *res = &dev->resource[PCI_ROM_RESOURCE];
340 dev->rom_base_reg = rom;
341 res->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH |
342 IORESOURCE_READONLY | IORESOURCE_SIZEALIGN;
343 __pci_read_base(dev, pci_bar_mem32, res, rom);
344 }
345}
346
347static void pci_read_bridge_io(struct pci_dev *dev, struct resource *res,
348 bool log)
349{
350 u8 io_base_lo, io_limit_lo;
351 unsigned long io_mask, io_granularity, base, limit;
352 struct pci_bus_region region;
353
354 io_mask = PCI_IO_RANGE_MASK;
355 io_granularity = 0x1000;
356 if (dev->io_window_1k) {
357 /* Support 1K I/O space granularity */
358 io_mask = PCI_IO_1K_RANGE_MASK;
359 io_granularity = 0x400;
360 }
361
362 pci_read_config_byte(dev, PCI_IO_BASE, &io_base_lo);
363 pci_read_config_byte(dev, PCI_IO_LIMIT, &io_limit_lo);
364 base = (io_base_lo & io_mask) << 8;
365 limit = (io_limit_lo & io_mask) << 8;
366
367 if ((io_base_lo & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) {
368 u16 io_base_hi, io_limit_hi;
369
370 pci_read_config_word(dev, PCI_IO_BASE_UPPER16, &io_base_hi);
371 pci_read_config_word(dev, PCI_IO_LIMIT_UPPER16, &io_limit_hi);
372 base |= ((unsigned long) io_base_hi << 16);
373 limit |= ((unsigned long) io_limit_hi << 16);
374 }
375
376 if (base <= limit) {
377 res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO;
378 region.start = base;
379 region.end = limit + io_granularity - 1;
380 pcibios_bus_to_resource(dev->bus, res, ®ion);
381 if (log)
382 pci_info(dev, " bridge window %pR\n", res);
383 }
384}
385
386static void pci_read_bridge_mmio(struct pci_dev *dev, struct resource *res,
387 bool log)
388{
389 u16 mem_base_lo, mem_limit_lo;
390 unsigned long base, limit;
391 struct pci_bus_region region;
392
393 pci_read_config_word(dev, PCI_MEMORY_BASE, &mem_base_lo);
394 pci_read_config_word(dev, PCI_MEMORY_LIMIT, &mem_limit_lo);
395 base = ((unsigned long) mem_base_lo & PCI_MEMORY_RANGE_MASK) << 16;
396 limit = ((unsigned long) mem_limit_lo & PCI_MEMORY_RANGE_MASK) << 16;
397 if (base <= limit) {
398 res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | IORESOURCE_MEM;
399 region.start = base;
400 region.end = limit + 0xfffff;
401 pcibios_bus_to_resource(dev->bus, res, ®ion);
402 if (log)
403 pci_info(dev, " bridge window %pR\n", res);
404 }
405}
406
407static void pci_read_bridge_mmio_pref(struct pci_dev *dev, struct resource *res,
408 bool log)
409{
410 u16 mem_base_lo, mem_limit_lo;
411 u64 base64, limit64;
412 pci_bus_addr_t base, limit;
413 struct pci_bus_region region;
414
415 pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, &mem_base_lo);
416 pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, &mem_limit_lo);
417 base64 = (mem_base_lo & PCI_PREF_RANGE_MASK) << 16;
418 limit64 = (mem_limit_lo & PCI_PREF_RANGE_MASK) << 16;
419
420 if ((mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
421 u32 mem_base_hi, mem_limit_hi;
422
423 pci_read_config_dword(dev, PCI_PREF_BASE_UPPER32, &mem_base_hi);
424 pci_read_config_dword(dev, PCI_PREF_LIMIT_UPPER32, &mem_limit_hi);
425
426 /*
427 * Some bridges set the base > limit by default, and some
428 * (broken) BIOSes do not initialize them. If we find
429 * this, just assume they are not being used.
430 */
431 if (mem_base_hi <= mem_limit_hi) {
432 base64 |= (u64) mem_base_hi << 32;
433 limit64 |= (u64) mem_limit_hi << 32;
434 }
435 }
436
437 base = (pci_bus_addr_t) base64;
438 limit = (pci_bus_addr_t) limit64;
439
440 if (base != base64) {
441 pci_err(dev, "can't handle bridge window above 4GB (bus address %#010llx)\n",
442 (unsigned long long) base64);
443 return;
444 }
445
446 if (base <= limit) {
447 res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) |
448 IORESOURCE_MEM | IORESOURCE_PREFETCH;
449 if (res->flags & PCI_PREF_RANGE_TYPE_64)
450 res->flags |= IORESOURCE_MEM_64;
451 region.start = base;
452 region.end = limit + 0xfffff;
453 pcibios_bus_to_resource(dev->bus, res, ®ion);
454 if (log)
455 pci_info(dev, " bridge window %pR\n", res);
456 }
457}
458
459static void pci_read_bridge_windows(struct pci_dev *bridge)
460{
461 u32 buses;
462 u16 io;
463 u32 pmem, tmp;
464 struct resource res;
465
466 pci_read_config_dword(bridge, PCI_PRIMARY_BUS, &buses);
467 res.flags = IORESOURCE_BUS;
468 res.start = (buses >> 8) & 0xff;
469 res.end = (buses >> 16) & 0xff;
470 pci_info(bridge, "PCI bridge to %pR%s\n", &res,
471 bridge->transparent ? " (subtractive decode)" : "");
472
473 pci_read_config_word(bridge, PCI_IO_BASE, &io);
474 if (!io) {
475 pci_write_config_word(bridge, PCI_IO_BASE, 0xe0f0);
476 pci_read_config_word(bridge, PCI_IO_BASE, &io);
477 pci_write_config_word(bridge, PCI_IO_BASE, 0x0);
478 }
479 if (io) {
480 bridge->io_window = 1;
481 pci_read_bridge_io(bridge, &res, true);
482 }
483
484 pci_read_bridge_mmio(bridge, &res, true);
485
486 /*
487 * DECchip 21050 pass 2 errata: the bridge may miss an address
488 * disconnect boundary by one PCI data phase. Workaround: do not
489 * use prefetching on this device.
490 */
491 if (bridge->vendor == PCI_VENDOR_ID_DEC && bridge->device == 0x0001)
492 return;
493
494 pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem);
495 if (!pmem) {
496 pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE,
497 0xffe0fff0);
498 pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem);
499 pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE, 0x0);
500 }
501 if (!pmem)
502 return;
503
504 bridge->pref_window = 1;
505
506 if ((pmem & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
507
508 /*
509 * Bridge claims to have a 64-bit prefetchable memory
510 * window; verify that the upper bits are actually
511 * writable.
512 */
513 pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &pmem);
514 pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32,
515 0xffffffff);
516 pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &tmp);
517 pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, pmem);
518 if (tmp)
519 bridge->pref_64_window = 1;
520 }
521
522 pci_read_bridge_mmio_pref(bridge, &res, true);
523}
524
525void pci_read_bridge_bases(struct pci_bus *child)
526{
527 struct pci_dev *dev = child->self;
528 struct resource *res;
529 int i;
530
531 if (pci_is_root_bus(child)) /* It's a host bus, nothing to read */
532 return;
533
534 pci_info(dev, "PCI bridge to %pR%s\n",
535 &child->busn_res,
536 dev->transparent ? " (subtractive decode)" : "");
537
538 pci_bus_remove_resources(child);
539 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
540 child->resource[i] = &dev->resource[PCI_BRIDGE_RESOURCES+i];
541
542 pci_read_bridge_io(child->self, child->resource[0], false);
543 pci_read_bridge_mmio(child->self, child->resource[1], false);
544 pci_read_bridge_mmio_pref(child->self, child->resource[2], false);
545
546 if (dev->transparent) {
547 pci_bus_for_each_resource(child->parent, res) {
548 if (res && res->flags) {
549 pci_bus_add_resource(child, res,
550 PCI_SUBTRACTIVE_DECODE);
551 pci_info(dev, " bridge window %pR (subtractive decode)\n",
552 res);
553 }
554 }
555 }
556}
557
558static struct pci_bus *pci_alloc_bus(struct pci_bus *parent)
559{
560 struct pci_bus *b;
561
562 b = kzalloc(sizeof(*b), GFP_KERNEL);
563 if (!b)
564 return NULL;
565
566 INIT_LIST_HEAD(&b->node);
567 INIT_LIST_HEAD(&b->children);
568 INIT_LIST_HEAD(&b->devices);
569 INIT_LIST_HEAD(&b->slots);
570 INIT_LIST_HEAD(&b->resources);
571 b->max_bus_speed = PCI_SPEED_UNKNOWN;
572 b->cur_bus_speed = PCI_SPEED_UNKNOWN;
573#ifdef CONFIG_PCI_DOMAINS_GENERIC
574 if (parent)
575 b->domain_nr = parent->domain_nr;
576#endif
577 return b;
578}
579
580static void pci_release_host_bridge_dev(struct device *dev)
581{
582 struct pci_host_bridge *bridge = to_pci_host_bridge(dev);
583
584 if (bridge->release_fn)
585 bridge->release_fn(bridge);
586
587 pci_free_resource_list(&bridge->windows);
588 pci_free_resource_list(&bridge->dma_ranges);
589 kfree(bridge);
590}
591
592static void pci_init_host_bridge(struct pci_host_bridge *bridge)
593{
594 INIT_LIST_HEAD(&bridge->windows);
595 INIT_LIST_HEAD(&bridge->dma_ranges);
596
597 /*
598 * We assume we can manage these PCIe features. Some systems may
599 * reserve these for use by the platform itself, e.g., an ACPI BIOS
600 * may implement its own AER handling and use _OSC to prevent the
601 * OS from interfering.
602 */
603 bridge->native_aer = 1;
604 bridge->native_pcie_hotplug = 1;
605 bridge->native_shpc_hotplug = 1;
606 bridge->native_pme = 1;
607 bridge->native_ltr = 1;
608 bridge->native_dpc = 1;
609 bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET;
610 bridge->native_cxl_error = 1;
611
612 device_initialize(&bridge->dev);
613}
614
615struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
616{
617 struct pci_host_bridge *bridge;
618
619 bridge = kzalloc(sizeof(*bridge) + priv, GFP_KERNEL);
620 if (!bridge)
621 return NULL;
622
623 pci_init_host_bridge(bridge);
624 bridge->dev.release = pci_release_host_bridge_dev;
625
626 return bridge;
627}
628EXPORT_SYMBOL(pci_alloc_host_bridge);
629
630static void devm_pci_alloc_host_bridge_release(void *data)
631{
632 pci_free_host_bridge(data);
633}
634
635struct pci_host_bridge *devm_pci_alloc_host_bridge(struct device *dev,
636 size_t priv)
637{
638 int ret;
639 struct pci_host_bridge *bridge;
640
641 bridge = pci_alloc_host_bridge(priv);
642 if (!bridge)
643 return NULL;
644
645 bridge->dev.parent = dev;
646
647 ret = devm_add_action_or_reset(dev, devm_pci_alloc_host_bridge_release,
648 bridge);
649 if (ret)
650 return NULL;
651
652 ret = devm_of_pci_bridge_init(dev, bridge);
653 if (ret)
654 return NULL;
655
656 return bridge;
657}
658EXPORT_SYMBOL(devm_pci_alloc_host_bridge);
659
660void pci_free_host_bridge(struct pci_host_bridge *bridge)
661{
662 put_device(&bridge->dev);
663}
664EXPORT_SYMBOL(pci_free_host_bridge);
665
666/* Indexed by PCI_X_SSTATUS_FREQ (secondary bus mode and frequency) */
667static const unsigned char pcix_bus_speed[] = {
668 PCI_SPEED_UNKNOWN, /* 0 */
669 PCI_SPEED_66MHz_PCIX, /* 1 */
670 PCI_SPEED_100MHz_PCIX, /* 2 */
671 PCI_SPEED_133MHz_PCIX, /* 3 */
672 PCI_SPEED_UNKNOWN, /* 4 */
673 PCI_SPEED_66MHz_PCIX_ECC, /* 5 */
674 PCI_SPEED_100MHz_PCIX_ECC, /* 6 */
675 PCI_SPEED_133MHz_PCIX_ECC, /* 7 */
676 PCI_SPEED_UNKNOWN, /* 8 */
677 PCI_SPEED_66MHz_PCIX_266, /* 9 */
678 PCI_SPEED_100MHz_PCIX_266, /* A */
679 PCI_SPEED_133MHz_PCIX_266, /* B */
680 PCI_SPEED_UNKNOWN, /* C */
681 PCI_SPEED_66MHz_PCIX_533, /* D */
682 PCI_SPEED_100MHz_PCIX_533, /* E */
683 PCI_SPEED_133MHz_PCIX_533 /* F */
684};
685
686/* Indexed by PCI_EXP_LNKCAP_SLS, PCI_EXP_LNKSTA_CLS */
687const unsigned char pcie_link_speed[] = {
688 PCI_SPEED_UNKNOWN, /* 0 */
689 PCIE_SPEED_2_5GT, /* 1 */
690 PCIE_SPEED_5_0GT, /* 2 */
691 PCIE_SPEED_8_0GT, /* 3 */
692 PCIE_SPEED_16_0GT, /* 4 */
693 PCIE_SPEED_32_0GT, /* 5 */
694 PCIE_SPEED_64_0GT, /* 6 */
695 PCI_SPEED_UNKNOWN, /* 7 */
696 PCI_SPEED_UNKNOWN, /* 8 */
697 PCI_SPEED_UNKNOWN, /* 9 */
698 PCI_SPEED_UNKNOWN, /* A */
699 PCI_SPEED_UNKNOWN, /* B */
700 PCI_SPEED_UNKNOWN, /* C */
701 PCI_SPEED_UNKNOWN, /* D */
702 PCI_SPEED_UNKNOWN, /* E */
703 PCI_SPEED_UNKNOWN /* F */
704};
705EXPORT_SYMBOL_GPL(pcie_link_speed);
706
707const char *pci_speed_string(enum pci_bus_speed speed)
708{
709 /* Indexed by the pci_bus_speed enum */
710 static const char *speed_strings[] = {
711 "33 MHz PCI", /* 0x00 */
712 "66 MHz PCI", /* 0x01 */
713 "66 MHz PCI-X", /* 0x02 */
714 "100 MHz PCI-X", /* 0x03 */
715 "133 MHz PCI-X", /* 0x04 */
716 NULL, /* 0x05 */
717 NULL, /* 0x06 */
718 NULL, /* 0x07 */
719 NULL, /* 0x08 */
720 "66 MHz PCI-X 266", /* 0x09 */
721 "100 MHz PCI-X 266", /* 0x0a */
722 "133 MHz PCI-X 266", /* 0x0b */
723 "Unknown AGP", /* 0x0c */
724 "1x AGP", /* 0x0d */
725 "2x AGP", /* 0x0e */
726 "4x AGP", /* 0x0f */
727 "8x AGP", /* 0x10 */
728 "66 MHz PCI-X 533", /* 0x11 */
729 "100 MHz PCI-X 533", /* 0x12 */
730 "133 MHz PCI-X 533", /* 0x13 */
731 "2.5 GT/s PCIe", /* 0x14 */
732 "5.0 GT/s PCIe", /* 0x15 */
733 "8.0 GT/s PCIe", /* 0x16 */
734 "16.0 GT/s PCIe", /* 0x17 */
735 "32.0 GT/s PCIe", /* 0x18 */
736 "64.0 GT/s PCIe", /* 0x19 */
737 };
738
739 if (speed < ARRAY_SIZE(speed_strings))
740 return speed_strings[speed];
741 return "Unknown";
742}
743EXPORT_SYMBOL_GPL(pci_speed_string);
744
745void pcie_update_link_speed(struct pci_bus *bus, u16 linksta)
746{
747 bus->cur_bus_speed = pcie_link_speed[linksta & PCI_EXP_LNKSTA_CLS];
748}
749EXPORT_SYMBOL_GPL(pcie_update_link_speed);
750
751static unsigned char agp_speeds[] = {
752 AGP_UNKNOWN,
753 AGP_1X,
754 AGP_2X,
755 AGP_4X,
756 AGP_8X
757};
758
759static enum pci_bus_speed agp_speed(int agp3, int agpstat)
760{
761 int index = 0;
762
763 if (agpstat & 4)
764 index = 3;
765 else if (agpstat & 2)
766 index = 2;
767 else if (agpstat & 1)
768 index = 1;
769 else
770 goto out;
771
772 if (agp3) {
773 index += 2;
774 if (index == 5)
775 index = 0;
776 }
777
778 out:
779 return agp_speeds[index];
780}
781
782static void pci_set_bus_speed(struct pci_bus *bus)
783{
784 struct pci_dev *bridge = bus->self;
785 int pos;
786
787 pos = pci_find_capability(bridge, PCI_CAP_ID_AGP);
788 if (!pos)
789 pos = pci_find_capability(bridge, PCI_CAP_ID_AGP3);
790 if (pos) {
791 u32 agpstat, agpcmd;
792
793 pci_read_config_dword(bridge, pos + PCI_AGP_STATUS, &agpstat);
794 bus->max_bus_speed = agp_speed(agpstat & 8, agpstat & 7);
795
796 pci_read_config_dword(bridge, pos + PCI_AGP_COMMAND, &agpcmd);
797 bus->cur_bus_speed = agp_speed(agpstat & 8, agpcmd & 7);
798 }
799
800 pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
801 if (pos) {
802 u16 status;
803 enum pci_bus_speed max;
804
805 pci_read_config_word(bridge, pos + PCI_X_BRIDGE_SSTATUS,
806 &status);
807
808 if (status & PCI_X_SSTATUS_533MHZ) {
809 max = PCI_SPEED_133MHz_PCIX_533;
810 } else if (status & PCI_X_SSTATUS_266MHZ) {
811 max = PCI_SPEED_133MHz_PCIX_266;
812 } else if (status & PCI_X_SSTATUS_133MHZ) {
813 if ((status & PCI_X_SSTATUS_VERS) == PCI_X_SSTATUS_V2)
814 max = PCI_SPEED_133MHz_PCIX_ECC;
815 else
816 max = PCI_SPEED_133MHz_PCIX;
817 } else {
818 max = PCI_SPEED_66MHz_PCIX;
819 }
820
821 bus->max_bus_speed = max;
822 bus->cur_bus_speed =
823 pcix_bus_speed[FIELD_GET(PCI_X_SSTATUS_FREQ, status)];
824
825 return;
826 }
827
828 if (pci_is_pcie(bridge)) {
829 u32 linkcap;
830 u16 linksta;
831
832 pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &linkcap);
833 bus->max_bus_speed = pcie_link_speed[linkcap & PCI_EXP_LNKCAP_SLS];
834
835 pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &linksta);
836 pcie_update_link_speed(bus, linksta);
837 }
838}
839
840static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus)
841{
842 struct irq_domain *d;
843
844 /* If the host bridge driver sets a MSI domain of the bridge, use it */
845 d = dev_get_msi_domain(bus->bridge);
846
847 /*
848 * Any firmware interface that can resolve the msi_domain
849 * should be called from here.
850 */
851 if (!d)
852 d = pci_host_bridge_of_msi_domain(bus);
853 if (!d)
854 d = pci_host_bridge_acpi_msi_domain(bus);
855
856 /*
857 * If no IRQ domain was found via the OF tree, try looking it up
858 * directly through the fwnode_handle.
859 */
860 if (!d) {
861 struct fwnode_handle *fwnode = pci_root_bus_fwnode(bus);
862
863 if (fwnode)
864 d = irq_find_matching_fwnode(fwnode,
865 DOMAIN_BUS_PCI_MSI);
866 }
867
868 return d;
869}
870
871static void pci_set_bus_msi_domain(struct pci_bus *bus)
872{
873 struct irq_domain *d;
874 struct pci_bus *b;
875
876 /*
877 * The bus can be a root bus, a subordinate bus, or a virtual bus
878 * created by an SR-IOV device. Walk up to the first bridge device
879 * found or derive the domain from the host bridge.
880 */
881 for (b = bus, d = NULL; !d && !pci_is_root_bus(b); b = b->parent) {
882 if (b->self)
883 d = dev_get_msi_domain(&b->self->dev);
884 }
885
886 if (!d)
887 d = pci_host_bridge_msi_domain(b);
888
889 dev_set_msi_domain(&bus->dev, d);
890}
891
892static int pci_register_host_bridge(struct pci_host_bridge *bridge)
893{
894 struct device *parent = bridge->dev.parent;
895 struct resource_entry *window, *next, *n;
896 struct pci_bus *bus, *b;
897 resource_size_t offset, next_offset;
898 LIST_HEAD(resources);
899 struct resource *res, *next_res;
900 char addr[64], *fmt;
901 const char *name;
902 int err;
903
904 bus = pci_alloc_bus(NULL);
905 if (!bus)
906 return -ENOMEM;
907
908 bridge->bus = bus;
909
910 bus->sysdata = bridge->sysdata;
911 bus->ops = bridge->ops;
912 bus->number = bus->busn_res.start = bridge->busnr;
913#ifdef CONFIG_PCI_DOMAINS_GENERIC
914 if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)
915 bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
916 else
917 bus->domain_nr = bridge->domain_nr;
918 if (bus->domain_nr < 0) {
919 err = bus->domain_nr;
920 goto free;
921 }
922#endif
923
924 b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
925 if (b) {
926 /* Ignore it if we already got here via a different bridge */
927 dev_dbg(&b->dev, "bus already known\n");
928 err = -EEXIST;
929 goto free;
930 }
931
932 dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus),
933 bridge->busnr);
934
935 err = pcibios_root_bridge_prepare(bridge);
936 if (err)
937 goto free;
938
939 /* Temporarily move resources off the list */
940 list_splice_init(&bridge->windows, &resources);
941 err = device_add(&bridge->dev);
942 if (err) {
943 put_device(&bridge->dev);
944 goto free;
945 }
946 bus->bridge = get_device(&bridge->dev);
947 device_enable_async_suspend(bus->bridge);
948 pci_set_bus_of_node(bus);
949 pci_set_bus_msi_domain(bus);
950 if (bridge->msi_domain && !dev_get_msi_domain(&bus->dev) &&
951 !pci_host_of_has_msi_map(parent))
952 bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI;
953
954 if (!parent)
955 set_dev_node(bus->bridge, pcibus_to_node(bus));
956
957 bus->dev.class = &pcibus_class;
958 bus->dev.parent = bus->bridge;
959
960 dev_set_name(&bus->dev, "%04x:%02x", pci_domain_nr(bus), bus->number);
961 name = dev_name(&bus->dev);
962
963 err = device_register(&bus->dev);
964 if (err)
965 goto unregister;
966
967 pcibios_add_bus(bus);
968
969 if (bus->ops->add_bus) {
970 err = bus->ops->add_bus(bus);
971 if (WARN_ON(err < 0))
972 dev_err(&bus->dev, "failed to add bus: %d\n", err);
973 }
974
975 /* Create legacy_io and legacy_mem files for this bus */
976 pci_create_legacy_files(bus);
977
978 if (parent)
979 dev_info(parent, "PCI host bridge to bus %s\n", name);
980 else
981 pr_info("PCI host bridge to bus %s\n", name);
982
983 if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE)
984 dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n");
985
986 /* Coalesce contiguous windows */
987 resource_list_for_each_entry_safe(window, n, &resources) {
988 if (list_is_last(&window->node, &resources))
989 break;
990
991 next = list_next_entry(window, node);
992 offset = window->offset;
993 res = window->res;
994 next_offset = next->offset;
995 next_res = next->res;
996
997 if (res->flags != next_res->flags || offset != next_offset)
998 continue;
999
1000 if (res->end + 1 == next_res->start) {
1001 next_res->start = res->start;
1002 res->flags = res->start = res->end = 0;
1003 }
1004 }
1005
1006 /* Add initial resources to the bus */
1007 resource_list_for_each_entry_safe(window, n, &resources) {
1008 offset = window->offset;
1009 res = window->res;
1010 if (!res->flags && !res->start && !res->end) {
1011 release_resource(res);
1012 resource_list_destroy_entry(window);
1013 continue;
1014 }
1015
1016 list_move_tail(&window->node, &bridge->windows);
1017
1018 if (res->flags & IORESOURCE_BUS)
1019 pci_bus_insert_busn_res(bus, bus->number, res->end);
1020 else
1021 pci_bus_add_resource(bus, res, 0);
1022
1023 if (offset) {
1024 if (resource_type(res) == IORESOURCE_IO)
1025 fmt = " (bus address [%#06llx-%#06llx])";
1026 else
1027 fmt = " (bus address [%#010llx-%#010llx])";
1028
1029 snprintf(addr, sizeof(addr), fmt,
1030 (unsigned long long)(res->start - offset),
1031 (unsigned long long)(res->end - offset));
1032 } else
1033 addr[0] = '\0';
1034
1035 dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
1036 }
1037
1038 down_write(&pci_bus_sem);
1039 list_add_tail(&bus->node, &pci_root_buses);
1040 up_write(&pci_bus_sem);
1041
1042 return 0;
1043
1044unregister:
1045 put_device(&bridge->dev);
1046 device_del(&bridge->dev);
1047
1048free:
1049#ifdef CONFIG_PCI_DOMAINS_GENERIC
1050 pci_bus_release_domain_nr(bus, parent);
1051#endif
1052 kfree(bus);
1053 return err;
1054}
1055
1056static bool pci_bridge_child_ext_cfg_accessible(struct pci_dev *bridge)
1057{
1058 int pos;
1059 u32 status;
1060
1061 /*
1062 * If extended config space isn't accessible on a bridge's primary
1063 * bus, we certainly can't access it on the secondary bus.
1064 */
1065 if (bridge->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1066 return false;
1067
1068 /*
1069 * PCIe Root Ports and switch ports are PCIe on both sides, so if
1070 * extended config space is accessible on the primary, it's also
1071 * accessible on the secondary.
1072 */
1073 if (pci_is_pcie(bridge) &&
1074 (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT ||
1075 pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM ||
1076 pci_pcie_type(bridge) == PCI_EXP_TYPE_DOWNSTREAM))
1077 return true;
1078
1079 /*
1080 * For the other bridge types:
1081 * - PCI-to-PCI bridges
1082 * - PCIe-to-PCI/PCI-X forward bridges
1083 * - PCI/PCI-X-to-PCIe reverse bridges
1084 * extended config space on the secondary side is only accessible
1085 * if the bridge supports PCI-X Mode 2.
1086 */
1087 pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
1088 if (!pos)
1089 return false;
1090
1091 pci_read_config_dword(bridge, pos + PCI_X_STATUS, &status);
1092 return status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ);
1093}
1094
1095static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
1096 struct pci_dev *bridge, int busnr)
1097{
1098 struct pci_bus *child;
1099 struct pci_host_bridge *host;
1100 int i;
1101 int ret;
1102
1103 /* Allocate a new bus and inherit stuff from the parent */
1104 child = pci_alloc_bus(parent);
1105 if (!child)
1106 return NULL;
1107
1108 child->parent = parent;
1109 child->sysdata = parent->sysdata;
1110 child->bus_flags = parent->bus_flags;
1111
1112 host = pci_find_host_bridge(parent);
1113 if (host->child_ops)
1114 child->ops = host->child_ops;
1115 else
1116 child->ops = parent->ops;
1117
1118 /*
1119 * Initialize some portions of the bus device, but don't register
1120 * it now as the parent is not properly set up yet.
1121 */
1122 child->dev.class = &pcibus_class;
1123 dev_set_name(&child->dev, "%04x:%02x", pci_domain_nr(child), busnr);
1124
1125 /* Set up the primary, secondary and subordinate bus numbers */
1126 child->number = child->busn_res.start = busnr;
1127 child->primary = parent->busn_res.start;
1128 child->busn_res.end = 0xff;
1129
1130 if (!bridge) {
1131 child->dev.parent = parent->bridge;
1132 goto add_dev;
1133 }
1134
1135 child->self = bridge;
1136 child->bridge = get_device(&bridge->dev);
1137 child->dev.parent = child->bridge;
1138 pci_set_bus_of_node(child);
1139 pci_set_bus_speed(child);
1140
1141 /*
1142 * Check whether extended config space is accessible on the child
1143 * bus. Note that we currently assume it is always accessible on
1144 * the root bus.
1145 */
1146 if (!pci_bridge_child_ext_cfg_accessible(bridge)) {
1147 child->bus_flags |= PCI_BUS_FLAGS_NO_EXTCFG;
1148 pci_info(child, "extended config space not accessible\n");
1149 }
1150
1151 /* Set up default resource pointers and names */
1152 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
1153 child->resource[i] = &bridge->resource[PCI_BRIDGE_RESOURCES+i];
1154 child->resource[i]->name = child->name;
1155 }
1156 bridge->subordinate = child;
1157
1158add_dev:
1159 pci_set_bus_msi_domain(child);
1160 ret = device_register(&child->dev);
1161 WARN_ON(ret < 0);
1162
1163 pcibios_add_bus(child);
1164
1165 if (child->ops->add_bus) {
1166 ret = child->ops->add_bus(child);
1167 if (WARN_ON(ret < 0))
1168 dev_err(&child->dev, "failed to add bus: %d\n", ret);
1169 }
1170
1171 /* Create legacy_io and legacy_mem files for this bus */
1172 pci_create_legacy_files(child);
1173
1174 return child;
1175}
1176
1177struct pci_bus *pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev,
1178 int busnr)
1179{
1180 struct pci_bus *child;
1181
1182 child = pci_alloc_child_bus(parent, dev, busnr);
1183 if (child) {
1184 down_write(&pci_bus_sem);
1185 list_add_tail(&child->node, &parent->children);
1186 up_write(&pci_bus_sem);
1187 }
1188 return child;
1189}
1190EXPORT_SYMBOL(pci_add_new_bus);
1191
1192static void pci_enable_crs(struct pci_dev *pdev)
1193{
1194 u16 root_cap = 0;
1195
1196 /* Enable CRS Software Visibility if supported */
1197 pcie_capability_read_word(pdev, PCI_EXP_RTCAP, &root_cap);
1198 if (root_cap & PCI_EXP_RTCAP_CRSVIS)
1199 pcie_capability_set_word(pdev, PCI_EXP_RTCTL,
1200 PCI_EXP_RTCTL_CRSSVE);
1201}
1202
1203static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
1204 unsigned int available_buses);
1205/**
1206 * pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus
1207 * numbers from EA capability.
1208 * @dev: Bridge
1209 * @sec: updated with secondary bus number from EA
1210 * @sub: updated with subordinate bus number from EA
1211 *
1212 * If @dev is a bridge with EA capability that specifies valid secondary
1213 * and subordinate bus numbers, return true with the bus numbers in @sec
1214 * and @sub. Otherwise return false.
1215 */
1216static bool pci_ea_fixed_busnrs(struct pci_dev *dev, u8 *sec, u8 *sub)
1217{
1218 int ea, offset;
1219 u32 dw;
1220 u8 ea_sec, ea_sub;
1221
1222 if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
1223 return false;
1224
1225 /* find PCI EA capability in list */
1226 ea = pci_find_capability(dev, PCI_CAP_ID_EA);
1227 if (!ea)
1228 return false;
1229
1230 offset = ea + PCI_EA_FIRST_ENT;
1231 pci_read_config_dword(dev, offset, &dw);
1232 ea_sec = FIELD_GET(PCI_EA_SEC_BUS_MASK, dw);
1233 ea_sub = FIELD_GET(PCI_EA_SUB_BUS_MASK, dw);
1234 if (ea_sec == 0 || ea_sub < ea_sec)
1235 return false;
1236
1237 *sec = ea_sec;
1238 *sub = ea_sub;
1239 return true;
1240}
1241
1242/*
1243 * pci_scan_bridge_extend() - Scan buses behind a bridge
1244 * @bus: Parent bus the bridge is on
1245 * @dev: Bridge itself
1246 * @max: Starting subordinate number of buses behind this bridge
1247 * @available_buses: Total number of buses available for this bridge and
1248 * the devices below. After the minimal bus space has
1249 * been allocated the remaining buses will be
1250 * distributed equally between hotplug-capable bridges.
1251 * @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1252 * that need to be reconfigured.
1253 *
1254 * If it's a bridge, configure it and scan the bus behind it.
1255 * For CardBus bridges, we don't scan behind as the devices will
1256 * be handled by the bridge driver itself.
1257 *
1258 * We need to process bridges in two passes -- first we scan those
1259 * already configured by the BIOS and after we are done with all of
1260 * them, we proceed to assigning numbers to the remaining buses in
1261 * order to avoid overlaps between old and new bus numbers.
1262 *
1263 * Return: New subordinate number covering all buses behind this bridge.
1264 */
1265static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
1266 int max, unsigned int available_buses,
1267 int pass)
1268{
1269 struct pci_bus *child;
1270 int is_cardbus = (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS);
1271 u32 buses, i, j = 0;
1272 u16 bctl;
1273 u8 primary, secondary, subordinate;
1274 int broken = 0;
1275 bool fixed_buses;
1276 u8 fixed_sec, fixed_sub;
1277 int next_busnr;
1278
1279 /*
1280 * Make sure the bridge is powered on to be able to access config
1281 * space of devices below it.
1282 */
1283 pm_runtime_get_sync(&dev->dev);
1284
1285 pci_read_config_dword(dev, PCI_PRIMARY_BUS, &buses);
1286 primary = buses & 0xFF;
1287 secondary = (buses >> 8) & 0xFF;
1288 subordinate = (buses >> 16) & 0xFF;
1289
1290 pci_dbg(dev, "scanning [bus %02x-%02x] behind bridge, pass %d\n",
1291 secondary, subordinate, pass);
1292
1293 if (!primary && (primary != bus->number) && secondary && subordinate) {
1294 pci_warn(dev, "Primary bus is hard wired to 0\n");
1295 primary = bus->number;
1296 }
1297
1298 /* Check if setup is sensible at all */
1299 if (!pass &&
1300 (primary != bus->number || secondary <= bus->number ||
1301 secondary > subordinate)) {
1302 pci_info(dev, "bridge configuration invalid ([bus %02x-%02x]), reconfiguring\n",
1303 secondary, subordinate);
1304 broken = 1;
1305 }
1306
1307 /*
1308 * Disable Master-Abort Mode during probing to avoid reporting of
1309 * bus errors in some architectures.
1310 */
1311 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &bctl);
1312 pci_write_config_word(dev, PCI_BRIDGE_CONTROL,
1313 bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT);
1314
1315 pci_enable_crs(dev);
1316
1317 if ((secondary || subordinate) && !pcibios_assign_all_busses() &&
1318 !is_cardbus && !broken) {
1319 unsigned int cmax, buses;
1320
1321 /*
1322 * Bus already configured by firmware, process it in the
1323 * first pass and just note the configuration.
1324 */
1325 if (pass)
1326 goto out;
1327
1328 /*
1329 * The bus might already exist for two reasons: Either we
1330 * are rescanning the bus or the bus is reachable through
1331 * more than one bridge. The second case can happen with
1332 * the i450NX chipset.
1333 */
1334 child = pci_find_bus(pci_domain_nr(bus), secondary);
1335 if (!child) {
1336 child = pci_add_new_bus(bus, dev, secondary);
1337 if (!child)
1338 goto out;
1339 child->primary = primary;
1340 pci_bus_insert_busn_res(child, secondary, subordinate);
1341 child->bridge_ctl = bctl;
1342 }
1343
1344 buses = subordinate - secondary;
1345 cmax = pci_scan_child_bus_extend(child, buses);
1346 if (cmax > subordinate)
1347 pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n",
1348 subordinate, cmax);
1349
1350 /* Subordinate should equal child->busn_res.end */
1351 if (subordinate > max)
1352 max = subordinate;
1353 } else {
1354
1355 /*
1356 * We need to assign a number to this bus which we always
1357 * do in the second pass.
1358 */
1359 if (!pass) {
1360 if (pcibios_assign_all_busses() || broken || is_cardbus)
1361
1362 /*
1363 * Temporarily disable forwarding of the
1364 * configuration cycles on all bridges in
1365 * this bus segment to avoid possible
1366 * conflicts in the second pass between two
1367 * bridges programmed with overlapping bus
1368 * ranges.
1369 */
1370 pci_write_config_dword(dev, PCI_PRIMARY_BUS,
1371 buses & ~0xffffff);
1372 goto out;
1373 }
1374
1375 /* Clear errors */
1376 pci_write_config_word(dev, PCI_STATUS, 0xffff);
1377
1378 /* Read bus numbers from EA Capability (if present) */
1379 fixed_buses = pci_ea_fixed_busnrs(dev, &fixed_sec, &fixed_sub);
1380 if (fixed_buses)
1381 next_busnr = fixed_sec;
1382 else
1383 next_busnr = max + 1;
1384
1385 /*
1386 * Prevent assigning a bus number that already exists.
1387 * This can happen when a bridge is hot-plugged, so in this
1388 * case we only re-scan this bus.
1389 */
1390 child = pci_find_bus(pci_domain_nr(bus), next_busnr);
1391 if (!child) {
1392 child = pci_add_new_bus(bus, dev, next_busnr);
1393 if (!child)
1394 goto out;
1395 pci_bus_insert_busn_res(child, next_busnr,
1396 bus->busn_res.end);
1397 }
1398 max++;
1399 if (available_buses)
1400 available_buses--;
1401
1402 buses = (buses & 0xff000000)
1403 | ((unsigned int)(child->primary) << 0)
1404 | ((unsigned int)(child->busn_res.start) << 8)
1405 | ((unsigned int)(child->busn_res.end) << 16);
1406
1407 /*
1408 * yenta.c forces a secondary latency timer of 176.
1409 * Copy that behaviour here.
1410 */
1411 if (is_cardbus) {
1412 buses &= ~0xff000000;
1413 buses |= CARDBUS_LATENCY_TIMER << 24;
1414 }
1415
1416 /* We need to blast all three values with a single write */
1417 pci_write_config_dword(dev, PCI_PRIMARY_BUS, buses);
1418
1419 if (!is_cardbus) {
1420 child->bridge_ctl = bctl;
1421 max = pci_scan_child_bus_extend(child, available_buses);
1422 } else {
1423
1424 /*
1425 * For CardBus bridges, we leave 4 bus numbers as
1426 * cards with a PCI-to-PCI bridge can be inserted
1427 * later.
1428 */
1429 for (i = 0; i < CARDBUS_RESERVE_BUSNR; i++) {
1430 struct pci_bus *parent = bus;
1431 if (pci_find_bus(pci_domain_nr(bus),
1432 max+i+1))
1433 break;
1434 while (parent->parent) {
1435 if ((!pcibios_assign_all_busses()) &&
1436 (parent->busn_res.end > max) &&
1437 (parent->busn_res.end <= max+i)) {
1438 j = 1;
1439 }
1440 parent = parent->parent;
1441 }
1442 if (j) {
1443
1444 /*
1445 * Often, there are two CardBus
1446 * bridges -- try to leave one
1447 * valid bus number for each one.
1448 */
1449 i /= 2;
1450 break;
1451 }
1452 }
1453 max += i;
1454 }
1455
1456 /*
1457 * Set subordinate bus number to its real value.
1458 * If fixed subordinate bus number exists from EA
1459 * capability then use it.
1460 */
1461 if (fixed_buses)
1462 max = fixed_sub;
1463 pci_bus_update_busn_res_end(child, max);
1464 pci_write_config_byte(dev, PCI_SUBORDINATE_BUS, max);
1465 }
1466
1467 sprintf(child->name,
1468 (is_cardbus ? "PCI CardBus %04x:%02x" : "PCI Bus %04x:%02x"),
1469 pci_domain_nr(bus), child->number);
1470
1471 /* Check that all devices are accessible */
1472 while (bus->parent) {
1473 if ((child->busn_res.end > bus->busn_res.end) ||
1474 (child->number > bus->busn_res.end) ||
1475 (child->number < bus->number) ||
1476 (child->busn_res.end < bus->number)) {
1477 dev_info(&dev->dev, "devices behind bridge are unusable because %pR cannot be assigned for them\n",
1478 &child->busn_res);
1479 break;
1480 }
1481 bus = bus->parent;
1482 }
1483
1484out:
1485 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl);
1486
1487 pm_runtime_put(&dev->dev);
1488
1489 return max;
1490}
1491
1492/*
1493 * pci_scan_bridge() - Scan buses behind a bridge
1494 * @bus: Parent bus the bridge is on
1495 * @dev: Bridge itself
1496 * @max: Starting subordinate number of buses behind this bridge
1497 * @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1498 * that need to be reconfigured.
1499 *
1500 * If it's a bridge, configure it and scan the bus behind it.
1501 * For CardBus bridges, we don't scan behind as the devices will
1502 * be handled by the bridge driver itself.
1503 *
1504 * We need to process bridges in two passes -- first we scan those
1505 * already configured by the BIOS and after we are done with all of
1506 * them, we proceed to assigning numbers to the remaining buses in
1507 * order to avoid overlaps between old and new bus numbers.
1508 *
1509 * Return: New subordinate number covering all buses behind this bridge.
1510 */
1511int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, int pass)
1512{
1513 return pci_scan_bridge_extend(bus, dev, max, 0, pass);
1514}
1515EXPORT_SYMBOL(pci_scan_bridge);
1516
1517/*
1518 * Read interrupt line and base address registers.
1519 * The architecture-dependent code can tweak these, of course.
1520 */
1521static void pci_read_irq(struct pci_dev *dev)
1522{
1523 unsigned char irq;
1524
1525 /* VFs are not allowed to use INTx, so skip the config reads */
1526 if (dev->is_virtfn) {
1527 dev->pin = 0;
1528 dev->irq = 0;
1529 return;
1530 }
1531
1532 pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &irq);
1533 dev->pin = irq;
1534 if (irq)
1535 pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &irq);
1536 dev->irq = irq;
1537}
1538
1539void set_pcie_port_type(struct pci_dev *pdev)
1540{
1541 int pos;
1542 u16 reg16;
1543 u32 reg32;
1544 int type;
1545 struct pci_dev *parent;
1546
1547 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
1548 if (!pos)
1549 return;
1550
1551 pdev->pcie_cap = pos;
1552 pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16);
1553 pdev->pcie_flags_reg = reg16;
1554 pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, &pdev->devcap);
1555 pdev->pcie_mpss = FIELD_GET(PCI_EXP_DEVCAP_PAYLOAD, pdev->devcap);
1556
1557 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32);
1558 if (reg32 & PCI_EXP_LNKCAP_DLLLARC)
1559 pdev->link_active_reporting = 1;
1560
1561 parent = pci_upstream_bridge(pdev);
1562 if (!parent)
1563 return;
1564
1565 /*
1566 * Some systems do not identify their upstream/downstream ports
1567 * correctly so detect impossible configurations here and correct
1568 * the port type accordingly.
1569 */
1570 type = pci_pcie_type(pdev);
1571 if (type == PCI_EXP_TYPE_DOWNSTREAM) {
1572 /*
1573 * If pdev claims to be downstream port but the parent
1574 * device is also downstream port assume pdev is actually
1575 * upstream port.
1576 */
1577 if (pcie_downstream_port(parent)) {
1578 pci_info(pdev, "claims to be downstream port but is acting as upstream port, correcting type\n");
1579 pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1580 pdev->pcie_flags_reg |= PCI_EXP_TYPE_UPSTREAM;
1581 }
1582 } else if (type == PCI_EXP_TYPE_UPSTREAM) {
1583 /*
1584 * If pdev claims to be upstream port but the parent
1585 * device is also upstream port assume pdev is actually
1586 * downstream port.
1587 */
1588 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM) {
1589 pci_info(pdev, "claims to be upstream port but is acting as downstream port, correcting type\n");
1590 pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1591 pdev->pcie_flags_reg |= PCI_EXP_TYPE_DOWNSTREAM;
1592 }
1593 }
1594}
1595
1596void set_pcie_hotplug_bridge(struct pci_dev *pdev)
1597{
1598 u32 reg32;
1599
1600 pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, ®32);
1601 if (reg32 & PCI_EXP_SLTCAP_HPC)
1602 pdev->is_hotplug_bridge = 1;
1603}
1604
1605static void set_pcie_thunderbolt(struct pci_dev *dev)
1606{
1607 u16 vsec;
1608
1609 /* Is the device part of a Thunderbolt controller? */
1610 vsec = pci_find_vsec_capability(dev, PCI_VENDOR_ID_INTEL, PCI_VSEC_ID_INTEL_TBT);
1611 if (vsec)
1612 dev->is_thunderbolt = 1;
1613}
1614
1615static void set_pcie_untrusted(struct pci_dev *dev)
1616{
1617 struct pci_dev *parent;
1618
1619 /*
1620 * If the upstream bridge is untrusted we treat this device
1621 * untrusted as well.
1622 */
1623 parent = pci_upstream_bridge(dev);
1624 if (parent && (parent->untrusted || parent->external_facing))
1625 dev->untrusted = true;
1626}
1627
1628static void pci_set_removable(struct pci_dev *dev)
1629{
1630 struct pci_dev *parent = pci_upstream_bridge(dev);
1631
1632 /*
1633 * We (only) consider everything downstream from an external_facing
1634 * device to be removable by the user. We're mainly concerned with
1635 * consumer platforms with user accessible thunderbolt ports that are
1636 * vulnerable to DMA attacks, and we expect those ports to be marked by
1637 * the firmware as external_facing. Devices in traditional hotplug
1638 * slots can technically be removed, but the expectation is that unless
1639 * the port is marked with external_facing, such devices are less
1640 * accessible to user / may not be removed by end user, and thus not
1641 * exposed as "removable" to userspace.
1642 */
1643 if (parent &&
1644 (parent->external_facing || dev_is_removable(&parent->dev)))
1645 dev_set_removable(&dev->dev, DEVICE_REMOVABLE);
1646}
1647
1648/**
1649 * pci_ext_cfg_is_aliased - Is ext config space just an alias of std config?
1650 * @dev: PCI device
1651 *
1652 * PCI Express to PCI/PCI-X Bridge Specification, rev 1.0, 4.1.4 says that
1653 * when forwarding a type1 configuration request the bridge must check that
1654 * the extended register address field is zero. The bridge is not permitted
1655 * to forward the transactions and must handle it as an Unsupported Request.
1656 * Some bridges do not follow this rule and simply drop the extended register
1657 * bits, resulting in the standard config space being aliased, every 256
1658 * bytes across the entire configuration space. Test for this condition by
1659 * comparing the first dword of each potential alias to the vendor/device ID.
1660 * Known offenders:
1661 * ASM1083/1085 PCIe-to-PCI Reversible Bridge (1b21:1080, rev 01 & 03)
1662 * AMD/ATI SBx00 PCI to PCI Bridge (1002:4384, rev 40)
1663 */
1664static bool pci_ext_cfg_is_aliased(struct pci_dev *dev)
1665{
1666#ifdef CONFIG_PCI_QUIRKS
1667 int pos, ret;
1668 u32 header, tmp;
1669
1670 pci_read_config_dword(dev, PCI_VENDOR_ID, &header);
1671
1672 for (pos = PCI_CFG_SPACE_SIZE;
1673 pos < PCI_CFG_SPACE_EXP_SIZE; pos += PCI_CFG_SPACE_SIZE) {
1674 ret = pci_read_config_dword(dev, pos, &tmp);
1675 if ((ret != PCIBIOS_SUCCESSFUL) || (header != tmp))
1676 return false;
1677 }
1678
1679 return true;
1680#else
1681 return false;
1682#endif
1683}
1684
1685/**
1686 * pci_cfg_space_size_ext - Get the configuration space size of the PCI device
1687 * @dev: PCI device
1688 *
1689 * Regular PCI devices have 256 bytes, but PCI-X 2 and PCI Express devices
1690 * have 4096 bytes. Even if the device is capable, that doesn't mean we can
1691 * access it. Maybe we don't have a way to generate extended config space
1692 * accesses, or the device is behind a reverse Express bridge. So we try
1693 * reading the dword at 0x100 which must either be 0 or a valid extended
1694 * capability header.
1695 */
1696static int pci_cfg_space_size_ext(struct pci_dev *dev)
1697{
1698 u32 status;
1699 int pos = PCI_CFG_SPACE_SIZE;
1700
1701 if (pci_read_config_dword(dev, pos, &status) != PCIBIOS_SUCCESSFUL)
1702 return PCI_CFG_SPACE_SIZE;
1703 if (PCI_POSSIBLE_ERROR(status) || pci_ext_cfg_is_aliased(dev))
1704 return PCI_CFG_SPACE_SIZE;
1705
1706 return PCI_CFG_SPACE_EXP_SIZE;
1707}
1708
1709int pci_cfg_space_size(struct pci_dev *dev)
1710{
1711 int pos;
1712 u32 status;
1713 u16 class;
1714
1715#ifdef CONFIG_PCI_IOV
1716 /*
1717 * Per the SR-IOV specification (rev 1.1, sec 3.5), VFs are required to
1718 * implement a PCIe capability and therefore must implement extended
1719 * config space. We can skip the NO_EXTCFG test below and the
1720 * reachability/aliasing test in pci_cfg_space_size_ext() by virtue of
1721 * the fact that the SR-IOV capability on the PF resides in extended
1722 * config space and must be accessible and non-aliased to have enabled
1723 * support for this VF. This is a micro performance optimization for
1724 * systems supporting many VFs.
1725 */
1726 if (dev->is_virtfn)
1727 return PCI_CFG_SPACE_EXP_SIZE;
1728#endif
1729
1730 if (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1731 return PCI_CFG_SPACE_SIZE;
1732
1733 class = dev->class >> 8;
1734 if (class == PCI_CLASS_BRIDGE_HOST)
1735 return pci_cfg_space_size_ext(dev);
1736
1737 if (pci_is_pcie(dev))
1738 return pci_cfg_space_size_ext(dev);
1739
1740 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1741 if (!pos)
1742 return PCI_CFG_SPACE_SIZE;
1743
1744 pci_read_config_dword(dev, pos + PCI_X_STATUS, &status);
1745 if (status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ))
1746 return pci_cfg_space_size_ext(dev);
1747
1748 return PCI_CFG_SPACE_SIZE;
1749}
1750
1751static u32 pci_class(struct pci_dev *dev)
1752{
1753 u32 class;
1754
1755#ifdef CONFIG_PCI_IOV
1756 if (dev->is_virtfn)
1757 return dev->physfn->sriov->class;
1758#endif
1759 pci_read_config_dword(dev, PCI_CLASS_REVISION, &class);
1760 return class;
1761}
1762
1763static void pci_subsystem_ids(struct pci_dev *dev, u16 *vendor, u16 *device)
1764{
1765#ifdef CONFIG_PCI_IOV
1766 if (dev->is_virtfn) {
1767 *vendor = dev->physfn->sriov->subsystem_vendor;
1768 *device = dev->physfn->sriov->subsystem_device;
1769 return;
1770 }
1771#endif
1772 pci_read_config_word(dev, PCI_SUBSYSTEM_VENDOR_ID, vendor);
1773 pci_read_config_word(dev, PCI_SUBSYSTEM_ID, device);
1774}
1775
1776static u8 pci_hdr_type(struct pci_dev *dev)
1777{
1778 u8 hdr_type;
1779
1780#ifdef CONFIG_PCI_IOV
1781 if (dev->is_virtfn)
1782 return dev->physfn->sriov->hdr_type;
1783#endif
1784 pci_read_config_byte(dev, PCI_HEADER_TYPE, &hdr_type);
1785 return hdr_type;
1786}
1787
1788#define LEGACY_IO_RESOURCE (IORESOURCE_IO | IORESOURCE_PCI_FIXED)
1789
1790/**
1791 * pci_intx_mask_broken - Test PCI_COMMAND_INTX_DISABLE writability
1792 * @dev: PCI device
1793 *
1794 * Test whether PCI_COMMAND_INTX_DISABLE is writable for @dev. Check this
1795 * at enumeration-time to avoid modifying PCI_COMMAND at run-time.
1796 */
1797static int pci_intx_mask_broken(struct pci_dev *dev)
1798{
1799 u16 orig, toggle, new;
1800
1801 pci_read_config_word(dev, PCI_COMMAND, &orig);
1802 toggle = orig ^ PCI_COMMAND_INTX_DISABLE;
1803 pci_write_config_word(dev, PCI_COMMAND, toggle);
1804 pci_read_config_word(dev, PCI_COMMAND, &new);
1805
1806 pci_write_config_word(dev, PCI_COMMAND, orig);
1807
1808 /*
1809 * PCI_COMMAND_INTX_DISABLE was reserved and read-only prior to PCI
1810 * r2.3, so strictly speaking, a device is not *broken* if it's not
1811 * writable. But we'll live with the misnomer for now.
1812 */
1813 if (new != toggle)
1814 return 1;
1815 return 0;
1816}
1817
1818static void early_dump_pci_device(struct pci_dev *pdev)
1819{
1820 u32 value[256 / 4];
1821 int i;
1822
1823 pci_info(pdev, "config space:\n");
1824
1825 for (i = 0; i < 256; i += 4)
1826 pci_read_config_dword(pdev, i, &value[i / 4]);
1827
1828 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
1829 value, 256, false);
1830}
1831
1832static const char *pci_type_str(struct pci_dev *dev)
1833{
1834 static const char * const str[] = {
1835 "PCIe Endpoint",
1836 "PCIe Legacy Endpoint",
1837 "PCIe unknown",
1838 "PCIe unknown",
1839 "PCIe Root Port",
1840 "PCIe Switch Upstream Port",
1841 "PCIe Switch Downstream Port",
1842 "PCIe to PCI/PCI-X bridge",
1843 "PCI/PCI-X to PCIe bridge",
1844 "PCIe Root Complex Integrated Endpoint",
1845 "PCIe Root Complex Event Collector",
1846 };
1847 int type;
1848
1849 if (pci_is_pcie(dev)) {
1850 type = pci_pcie_type(dev);
1851 if (type < ARRAY_SIZE(str))
1852 return str[type];
1853
1854 return "PCIe unknown";
1855 }
1856
1857 switch (dev->hdr_type) {
1858 case PCI_HEADER_TYPE_NORMAL:
1859 return "conventional PCI endpoint";
1860 case PCI_HEADER_TYPE_BRIDGE:
1861 return "conventional PCI bridge";
1862 case PCI_HEADER_TYPE_CARDBUS:
1863 return "CardBus bridge";
1864 default:
1865 return "conventional PCI";
1866 }
1867}
1868
1869/**
1870 * pci_setup_device - Fill in class and map information of a device
1871 * @dev: the device structure to fill
1872 *
1873 * Initialize the device structure with information about the device's
1874 * vendor,class,memory and IO-space addresses, IRQ lines etc.
1875 * Called at initialisation of the PCI subsystem and by CardBus services.
1876 * Returns 0 on success and negative if unknown type of device (not normal,
1877 * bridge or CardBus).
1878 */
1879int pci_setup_device(struct pci_dev *dev)
1880{
1881 u32 class;
1882 u16 cmd;
1883 u8 hdr_type;
1884 int err, pos = 0;
1885 struct pci_bus_region region;
1886 struct resource *res;
1887
1888 hdr_type = pci_hdr_type(dev);
1889
1890 dev->sysdata = dev->bus->sysdata;
1891 dev->dev.parent = dev->bus->bridge;
1892 dev->dev.bus = &pci_bus_type;
1893 dev->hdr_type = hdr_type & 0x7f;
1894 dev->multifunction = !!(hdr_type & 0x80);
1895 dev->error_state = pci_channel_io_normal;
1896 set_pcie_port_type(dev);
1897
1898 err = pci_set_of_node(dev);
1899 if (err)
1900 return err;
1901 pci_set_acpi_fwnode(dev);
1902
1903 pci_dev_assign_slot(dev);
1904
1905 /*
1906 * Assume 32-bit PCI; let 64-bit PCI cards (which are far rarer)
1907 * set this higher, assuming the system even supports it.
1908 */
1909 dev->dma_mask = 0xffffffff;
1910
1911 dev_set_name(&dev->dev, "%04x:%02x:%02x.%d", pci_domain_nr(dev->bus),
1912 dev->bus->number, PCI_SLOT(dev->devfn),
1913 PCI_FUNC(dev->devfn));
1914
1915 class = pci_class(dev);
1916
1917 dev->revision = class & 0xff;
1918 dev->class = class >> 8; /* upper 3 bytes */
1919
1920 if (pci_early_dump)
1921 early_dump_pci_device(dev);
1922
1923 /* Need to have dev->class ready */
1924 dev->cfg_size = pci_cfg_space_size(dev);
1925
1926 /* Need to have dev->cfg_size ready */
1927 set_pcie_thunderbolt(dev);
1928
1929 set_pcie_untrusted(dev);
1930
1931 /* "Unknown power state" */
1932 dev->current_state = PCI_UNKNOWN;
1933
1934 /* Early fixups, before probing the BARs */
1935 pci_fixup_device(pci_fixup_early, dev);
1936
1937 pci_set_removable(dev);
1938
1939 pci_info(dev, "[%04x:%04x] type %02x class %#08x %s\n",
1940 dev->vendor, dev->device, dev->hdr_type, dev->class,
1941 pci_type_str(dev));
1942
1943 /* Device class may be changed after fixup */
1944 class = dev->class >> 8;
1945
1946 if (dev->non_compliant_bars && !dev->mmio_always_on) {
1947 pci_read_config_word(dev, PCI_COMMAND, &cmd);
1948 if (cmd & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY)) {
1949 pci_info(dev, "device has non-compliant BARs; disabling IO/MEM decoding\n");
1950 cmd &= ~PCI_COMMAND_IO;
1951 cmd &= ~PCI_COMMAND_MEMORY;
1952 pci_write_config_word(dev, PCI_COMMAND, cmd);
1953 }
1954 }
1955
1956 dev->broken_intx_masking = pci_intx_mask_broken(dev);
1957
1958 switch (dev->hdr_type) { /* header type */
1959 case PCI_HEADER_TYPE_NORMAL: /* standard header */
1960 if (class == PCI_CLASS_BRIDGE_PCI)
1961 goto bad;
1962 pci_read_irq(dev);
1963 pci_read_bases(dev, 6, PCI_ROM_ADDRESS);
1964
1965 pci_subsystem_ids(dev, &dev->subsystem_vendor, &dev->subsystem_device);
1966
1967 /*
1968 * Do the ugly legacy mode stuff here rather than broken chip
1969 * quirk code. Legacy mode ATA controllers have fixed
1970 * addresses. These are not always echoed in BAR0-3, and
1971 * BAR0-3 in a few cases contain junk!
1972 */
1973 if (class == PCI_CLASS_STORAGE_IDE) {
1974 u8 progif;
1975 pci_read_config_byte(dev, PCI_CLASS_PROG, &progif);
1976 if ((progif & 1) == 0) {
1977 region.start = 0x1F0;
1978 region.end = 0x1F7;
1979 res = &dev->resource[0];
1980 res->flags = LEGACY_IO_RESOURCE;
1981 pcibios_bus_to_resource(dev->bus, res, ®ion);
1982 pci_info(dev, "BAR 0 %pR: legacy IDE quirk\n",
1983 res);
1984 region.start = 0x3F6;
1985 region.end = 0x3F6;
1986 res = &dev->resource[1];
1987 res->flags = LEGACY_IO_RESOURCE;
1988 pcibios_bus_to_resource(dev->bus, res, ®ion);
1989 pci_info(dev, "BAR 1 %pR: legacy IDE quirk\n",
1990 res);
1991 }
1992 if ((progif & 4) == 0) {
1993 region.start = 0x170;
1994 region.end = 0x177;
1995 res = &dev->resource[2];
1996 res->flags = LEGACY_IO_RESOURCE;
1997 pcibios_bus_to_resource(dev->bus, res, ®ion);
1998 pci_info(dev, "BAR 2 %pR: legacy IDE quirk\n",
1999 res);
2000 region.start = 0x376;
2001 region.end = 0x376;
2002 res = &dev->resource[3];
2003 res->flags = LEGACY_IO_RESOURCE;
2004 pcibios_bus_to_resource(dev->bus, res, ®ion);
2005 pci_info(dev, "BAR 3 %pR: legacy IDE quirk\n",
2006 res);
2007 }
2008 }
2009 break;
2010
2011 case PCI_HEADER_TYPE_BRIDGE: /* bridge header */
2012 /*
2013 * The PCI-to-PCI bridge spec requires that subtractive
2014 * decoding (i.e. transparent) bridge must have programming
2015 * interface code of 0x01.
2016 */
2017 pci_read_irq(dev);
2018 dev->transparent = ((dev->class & 0xff) == 1);
2019 pci_read_bases(dev, 2, PCI_ROM_ADDRESS1);
2020 pci_read_bridge_windows(dev);
2021 set_pcie_hotplug_bridge(dev);
2022 pos = pci_find_capability(dev, PCI_CAP_ID_SSVID);
2023 if (pos) {
2024 pci_read_config_word(dev, pos + PCI_SSVID_VENDOR_ID, &dev->subsystem_vendor);
2025 pci_read_config_word(dev, pos + PCI_SSVID_DEVICE_ID, &dev->subsystem_device);
2026 }
2027 break;
2028
2029 case PCI_HEADER_TYPE_CARDBUS: /* CardBus bridge header */
2030 if (class != PCI_CLASS_BRIDGE_CARDBUS)
2031 goto bad;
2032 pci_read_irq(dev);
2033 pci_read_bases(dev, 1, 0);
2034 pci_read_config_word(dev, PCI_CB_SUBSYSTEM_VENDOR_ID, &dev->subsystem_vendor);
2035 pci_read_config_word(dev, PCI_CB_SUBSYSTEM_ID, &dev->subsystem_device);
2036 break;
2037
2038 default: /* unknown header */
2039 pci_err(dev, "unknown header type %02x, ignoring device\n",
2040 dev->hdr_type);
2041 pci_release_of_node(dev);
2042 return -EIO;
2043
2044 bad:
2045 pci_err(dev, "ignoring class %#08x (doesn't match header type %02x)\n",
2046 dev->class, dev->hdr_type);
2047 dev->class = PCI_CLASS_NOT_DEFINED << 8;
2048 }
2049
2050 /* We found a fine healthy device, go go go... */
2051 return 0;
2052}
2053
2054static void pci_configure_mps(struct pci_dev *dev)
2055{
2056 struct pci_dev *bridge = pci_upstream_bridge(dev);
2057 int mps, mpss, p_mps, rc;
2058
2059 if (!pci_is_pcie(dev))
2060 return;
2061
2062 /* MPS and MRRS fields are of type 'RsvdP' for VFs, short-circuit out */
2063 if (dev->is_virtfn)
2064 return;
2065
2066 /*
2067 * For Root Complex Integrated Endpoints, program the maximum
2068 * supported value unless limited by the PCIE_BUS_PEER2PEER case.
2069 */
2070 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
2071 if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2072 mps = 128;
2073 else
2074 mps = 128 << dev->pcie_mpss;
2075 rc = pcie_set_mps(dev, mps);
2076 if (rc) {
2077 pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2078 mps);
2079 }
2080 return;
2081 }
2082
2083 if (!bridge || !pci_is_pcie(bridge))
2084 return;
2085
2086 mps = pcie_get_mps(dev);
2087 p_mps = pcie_get_mps(bridge);
2088
2089 if (mps == p_mps)
2090 return;
2091
2092 if (pcie_bus_config == PCIE_BUS_TUNE_OFF) {
2093 pci_warn(dev, "Max Payload Size %d, but upstream %s set to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2094 mps, pci_name(bridge), p_mps);
2095 return;
2096 }
2097
2098 /*
2099 * Fancier MPS configuration is done later by
2100 * pcie_bus_configure_settings()
2101 */
2102 if (pcie_bus_config != PCIE_BUS_DEFAULT)
2103 return;
2104
2105 mpss = 128 << dev->pcie_mpss;
2106 if (mpss < p_mps && pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
2107 pcie_set_mps(bridge, mpss);
2108 pci_info(dev, "Upstream bridge's Max Payload Size set to %d (was %d, max %d)\n",
2109 mpss, p_mps, 128 << bridge->pcie_mpss);
2110 p_mps = pcie_get_mps(bridge);
2111 }
2112
2113 rc = pcie_set_mps(dev, p_mps);
2114 if (rc) {
2115 pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2116 p_mps);
2117 return;
2118 }
2119
2120 pci_info(dev, "Max Payload Size set to %d (was %d, max %d)\n",
2121 p_mps, mps, mpss);
2122}
2123
2124int pci_configure_extended_tags(struct pci_dev *dev, void *ign)
2125{
2126 struct pci_host_bridge *host;
2127 u32 cap;
2128 u16 ctl;
2129 int ret;
2130
2131 if (!pci_is_pcie(dev))
2132 return 0;
2133
2134 ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
2135 if (ret)
2136 return 0;
2137
2138 if (!(cap & PCI_EXP_DEVCAP_EXT_TAG))
2139 return 0;
2140
2141 ret = pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
2142 if (ret)
2143 return 0;
2144
2145 host = pci_find_host_bridge(dev->bus);
2146 if (!host)
2147 return 0;
2148
2149 /*
2150 * If some device in the hierarchy doesn't handle Extended Tags
2151 * correctly, make sure they're disabled.
2152 */
2153 if (host->no_ext_tags) {
2154 if (ctl & PCI_EXP_DEVCTL_EXT_TAG) {
2155 pci_info(dev, "disabling Extended Tags\n");
2156 pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2157 PCI_EXP_DEVCTL_EXT_TAG);
2158 }
2159 return 0;
2160 }
2161
2162 if (!(ctl & PCI_EXP_DEVCTL_EXT_TAG)) {
2163 pci_info(dev, "enabling Extended Tags\n");
2164 pcie_capability_set_word(dev, PCI_EXP_DEVCTL,
2165 PCI_EXP_DEVCTL_EXT_TAG);
2166 }
2167 return 0;
2168}
2169
2170/**
2171 * pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable
2172 * @dev: PCI device to query
2173 *
2174 * Returns true if the device has enabled relaxed ordering attribute.
2175 */
2176bool pcie_relaxed_ordering_enabled(struct pci_dev *dev)
2177{
2178 u16 v;
2179
2180 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &v);
2181
2182 return !!(v & PCI_EXP_DEVCTL_RELAX_EN);
2183}
2184EXPORT_SYMBOL(pcie_relaxed_ordering_enabled);
2185
2186static void pci_configure_relaxed_ordering(struct pci_dev *dev)
2187{
2188 struct pci_dev *root;
2189
2190 /* PCI_EXP_DEVCTL_RELAX_EN is RsvdP in VFs */
2191 if (dev->is_virtfn)
2192 return;
2193
2194 if (!pcie_relaxed_ordering_enabled(dev))
2195 return;
2196
2197 /*
2198 * For now, we only deal with Relaxed Ordering issues with Root
2199 * Ports. Peer-to-Peer DMA is another can of worms.
2200 */
2201 root = pcie_find_root_port(dev);
2202 if (!root)
2203 return;
2204
2205 if (root->dev_flags & PCI_DEV_FLAGS_NO_RELAXED_ORDERING) {
2206 pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2207 PCI_EXP_DEVCTL_RELAX_EN);
2208 pci_info(dev, "Relaxed Ordering disabled because the Root Port didn't support it\n");
2209 }
2210}
2211
2212static void pci_configure_eetlp_prefix(struct pci_dev *dev)
2213{
2214#ifdef CONFIG_PCI_PASID
2215 struct pci_dev *bridge;
2216 int pcie_type;
2217 u32 cap;
2218
2219 if (!pci_is_pcie(dev))
2220 return;
2221
2222 pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &cap);
2223 if (!(cap & PCI_EXP_DEVCAP2_EE_PREFIX))
2224 return;
2225
2226 pcie_type = pci_pcie_type(dev);
2227 if (pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
2228 pcie_type == PCI_EXP_TYPE_RC_END)
2229 dev->eetlp_prefix_path = 1;
2230 else {
2231 bridge = pci_upstream_bridge(dev);
2232 if (bridge && bridge->eetlp_prefix_path)
2233 dev->eetlp_prefix_path = 1;
2234 }
2235#endif
2236}
2237
2238static void pci_configure_serr(struct pci_dev *dev)
2239{
2240 u16 control;
2241
2242 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
2243
2244 /*
2245 * A bridge will not forward ERR_ messages coming from an
2246 * endpoint unless SERR# forwarding is enabled.
2247 */
2248 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &control);
2249 if (!(control & PCI_BRIDGE_CTL_SERR)) {
2250 control |= PCI_BRIDGE_CTL_SERR;
2251 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, control);
2252 }
2253 }
2254}
2255
2256static void pci_configure_device(struct pci_dev *dev)
2257{
2258 pci_configure_mps(dev);
2259 pci_configure_extended_tags(dev, NULL);
2260 pci_configure_relaxed_ordering(dev);
2261 pci_configure_ltr(dev);
2262 pci_configure_aspm_l1ss(dev);
2263 pci_configure_eetlp_prefix(dev);
2264 pci_configure_serr(dev);
2265
2266 pci_acpi_program_hp_params(dev);
2267}
2268
2269static void pci_release_capabilities(struct pci_dev *dev)
2270{
2271 pci_aer_exit(dev);
2272 pci_rcec_exit(dev);
2273 pci_iov_release(dev);
2274 pci_free_cap_save_buffers(dev);
2275}
2276
2277/**
2278 * pci_release_dev - Free a PCI device structure when all users of it are
2279 * finished
2280 * @dev: device that's been disconnected
2281 *
2282 * Will be called only by the device core when all users of this PCI device are
2283 * done.
2284 */
2285static void pci_release_dev(struct device *dev)
2286{
2287 struct pci_dev *pci_dev;
2288
2289 pci_dev = to_pci_dev(dev);
2290 pci_release_capabilities(pci_dev);
2291 pci_release_of_node(pci_dev);
2292 pcibios_release_device(pci_dev);
2293 pci_bus_put(pci_dev->bus);
2294 kfree(pci_dev->driver_override);
2295 bitmap_free(pci_dev->dma_alias_mask);
2296 dev_dbg(dev, "device released\n");
2297 kfree(pci_dev);
2298}
2299
2300static const struct device_type pci_dev_type = {
2301 .groups = pci_dev_attr_groups,
2302};
2303
2304struct pci_dev *pci_alloc_dev(struct pci_bus *bus)
2305{
2306 struct pci_dev *dev;
2307
2308 dev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL);
2309 if (!dev)
2310 return NULL;
2311
2312 INIT_LIST_HEAD(&dev->bus_list);
2313 dev->dev.type = &pci_dev_type;
2314 dev->bus = pci_bus_get(bus);
2315 dev->driver_exclusive_resource = (struct resource) {
2316 .name = "PCI Exclusive",
2317 .start = 0,
2318 .end = -1,
2319 };
2320
2321 spin_lock_init(&dev->pcie_cap_lock);
2322#ifdef CONFIG_PCI_MSI
2323 raw_spin_lock_init(&dev->msi_lock);
2324#endif
2325 return dev;
2326}
2327EXPORT_SYMBOL(pci_alloc_dev);
2328
2329static bool pci_bus_crs_vendor_id(u32 l)
2330{
2331 return (l & 0xffff) == PCI_VENDOR_ID_PCI_SIG;
2332}
2333
2334static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
2335 int timeout)
2336{
2337 int delay = 1;
2338
2339 if (!pci_bus_crs_vendor_id(*l))
2340 return true; /* not a CRS completion */
2341
2342 if (!timeout)
2343 return false; /* CRS, but caller doesn't want to wait */
2344
2345 /*
2346 * We got the reserved Vendor ID that indicates a completion with
2347 * Configuration Request Retry Status (CRS). Retry until we get a
2348 * valid Vendor ID or we time out.
2349 */
2350 while (pci_bus_crs_vendor_id(*l)) {
2351 if (delay > timeout) {
2352 pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n",
2353 pci_domain_nr(bus), bus->number,
2354 PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2355
2356 return false;
2357 }
2358 if (delay >= 1000)
2359 pr_info("pci %04x:%02x:%02x.%d: not ready after %dms; waiting\n",
2360 pci_domain_nr(bus), bus->number,
2361 PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2362
2363 msleep(delay);
2364 delay *= 2;
2365
2366 if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
2367 return false;
2368 }
2369
2370 if (delay >= 1000)
2371 pr_info("pci %04x:%02x:%02x.%d: ready after %dms\n",
2372 pci_domain_nr(bus), bus->number,
2373 PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2374
2375 return true;
2376}
2377
2378bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2379 int timeout)
2380{
2381 if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
2382 return false;
2383
2384 /* Some broken boards return 0 or ~0 (PCI_ERROR_RESPONSE) if a slot is empty: */
2385 if (PCI_POSSIBLE_ERROR(*l) || *l == 0x00000000 ||
2386 *l == 0x0000ffff || *l == 0xffff0000)
2387 return false;
2388
2389 if (pci_bus_crs_vendor_id(*l))
2390 return pci_bus_wait_crs(bus, devfn, l, timeout);
2391
2392 return true;
2393}
2394
2395bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2396 int timeout)
2397{
2398#ifdef CONFIG_PCI_QUIRKS
2399 struct pci_dev *bridge = bus->self;
2400
2401 /*
2402 * Certain IDT switches have an issue where they improperly trigger
2403 * ACS Source Validation errors on completions for config reads.
2404 */
2405 if (bridge && bridge->vendor == PCI_VENDOR_ID_IDT &&
2406 bridge->device == 0x80b5)
2407 return pci_idt_bus_quirk(bus, devfn, l, timeout);
2408#endif
2409
2410 return pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
2411}
2412EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
2413
2414/*
2415 * Read the config data for a PCI device, sanity-check it,
2416 * and fill in the dev structure.
2417 */
2418static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
2419{
2420 struct pci_dev *dev;
2421 u32 l;
2422
2423 if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000))
2424 return NULL;
2425
2426 dev = pci_alloc_dev(bus);
2427 if (!dev)
2428 return NULL;
2429
2430 dev->devfn = devfn;
2431 dev->vendor = l & 0xffff;
2432 dev->device = (l >> 16) & 0xffff;
2433
2434 if (pci_setup_device(dev)) {
2435 pci_bus_put(dev->bus);
2436 kfree(dev);
2437 return NULL;
2438 }
2439
2440 return dev;
2441}
2442
2443void pcie_report_downtraining(struct pci_dev *dev)
2444{
2445 if (!pci_is_pcie(dev))
2446 return;
2447
2448 /* Look from the device up to avoid downstream ports with no devices */
2449 if ((pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT) &&
2450 (pci_pcie_type(dev) != PCI_EXP_TYPE_LEG_END) &&
2451 (pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM))
2452 return;
2453
2454 /* Multi-function PCIe devices share the same link/status */
2455 if (PCI_FUNC(dev->devfn) != 0 || dev->is_virtfn)
2456 return;
2457
2458 /* Print link status only if the device is constrained by the fabric */
2459 __pcie_print_link_status(dev, false);
2460}
2461
2462static void pci_init_capabilities(struct pci_dev *dev)
2463{
2464 pci_ea_init(dev); /* Enhanced Allocation */
2465 pci_msi_init(dev); /* Disable MSI */
2466 pci_msix_init(dev); /* Disable MSI-X */
2467
2468 /* Buffers for saving PCIe and PCI-X capabilities */
2469 pci_allocate_cap_save_buffers(dev);
2470
2471 pci_pm_init(dev); /* Power Management */
2472 pci_vpd_init(dev); /* Vital Product Data */
2473 pci_configure_ari(dev); /* Alternative Routing-ID Forwarding */
2474 pci_iov_init(dev); /* Single Root I/O Virtualization */
2475 pci_ats_init(dev); /* Address Translation Services */
2476 pci_pri_init(dev); /* Page Request Interface */
2477 pci_pasid_init(dev); /* Process Address Space ID */
2478 pci_acs_init(dev); /* Access Control Services */
2479 pci_ptm_init(dev); /* Precision Time Measurement */
2480 pci_aer_init(dev); /* Advanced Error Reporting */
2481 pci_dpc_init(dev); /* Downstream Port Containment */
2482 pci_rcec_init(dev); /* Root Complex Event Collector */
2483 pci_doe_init(dev); /* Data Object Exchange */
2484
2485 pcie_report_downtraining(dev);
2486 pci_init_reset_methods(dev);
2487}
2488
2489/*
2490 * This is the equivalent of pci_host_bridge_msi_domain() that acts on
2491 * devices. Firmware interfaces that can select the MSI domain on a
2492 * per-device basis should be called from here.
2493 */
2494static struct irq_domain *pci_dev_msi_domain(struct pci_dev *dev)
2495{
2496 struct irq_domain *d;
2497
2498 /*
2499 * If a domain has been set through the pcibios_device_add()
2500 * callback, then this is the one (platform code knows best).
2501 */
2502 d = dev_get_msi_domain(&dev->dev);
2503 if (d)
2504 return d;
2505
2506 /*
2507 * Let's see if we have a firmware interface able to provide
2508 * the domain.
2509 */
2510 d = pci_msi_get_device_domain(dev);
2511 if (d)
2512 return d;
2513
2514 return NULL;
2515}
2516
2517static void pci_set_msi_domain(struct pci_dev *dev)
2518{
2519 struct irq_domain *d;
2520
2521 /*
2522 * If the platform or firmware interfaces cannot supply a
2523 * device-specific MSI domain, then inherit the default domain
2524 * from the host bridge itself.
2525 */
2526 d = pci_dev_msi_domain(dev);
2527 if (!d)
2528 d = dev_get_msi_domain(&dev->bus->dev);
2529
2530 dev_set_msi_domain(&dev->dev, d);
2531}
2532
2533void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
2534{
2535 int ret;
2536
2537 pci_configure_device(dev);
2538
2539 device_initialize(&dev->dev);
2540 dev->dev.release = pci_release_dev;
2541
2542 set_dev_node(&dev->dev, pcibus_to_node(bus));
2543 dev->dev.dma_mask = &dev->dma_mask;
2544 dev->dev.dma_parms = &dev->dma_parms;
2545 dev->dev.coherent_dma_mask = 0xffffffffull;
2546
2547 dma_set_max_seg_size(&dev->dev, 65536);
2548 dma_set_seg_boundary(&dev->dev, 0xffffffff);
2549
2550 pcie_failed_link_retrain(dev);
2551
2552 /* Fix up broken headers */
2553 pci_fixup_device(pci_fixup_header, dev);
2554
2555 pci_reassigndev_resource_alignment(dev);
2556
2557 dev->state_saved = false;
2558
2559 pci_init_capabilities(dev);
2560
2561 /*
2562 * Add the device to our list of discovered devices
2563 * and the bus list for fixup functions, etc.
2564 */
2565 down_write(&pci_bus_sem);
2566 list_add_tail(&dev->bus_list, &bus->devices);
2567 up_write(&pci_bus_sem);
2568
2569 ret = pcibios_device_add(dev);
2570 WARN_ON(ret < 0);
2571
2572 /* Set up MSI IRQ domain */
2573 pci_set_msi_domain(dev);
2574
2575 /* Notifier could use PCI capabilities */
2576 dev->match_driver = false;
2577 ret = device_add(&dev->dev);
2578 WARN_ON(ret < 0);
2579}
2580
2581struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn)
2582{
2583 struct pci_dev *dev;
2584
2585 dev = pci_get_slot(bus, devfn);
2586 if (dev) {
2587 pci_dev_put(dev);
2588 return dev;
2589 }
2590
2591 dev = pci_scan_device(bus, devfn);
2592 if (!dev)
2593 return NULL;
2594
2595 pci_device_add(dev, bus);
2596
2597 return dev;
2598}
2599EXPORT_SYMBOL(pci_scan_single_device);
2600
2601static int next_ari_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2602{
2603 int pos;
2604 u16 cap = 0;
2605 unsigned int next_fn;
2606
2607 if (!dev)
2608 return -ENODEV;
2609
2610 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI);
2611 if (!pos)
2612 return -ENODEV;
2613
2614 pci_read_config_word(dev, pos + PCI_ARI_CAP, &cap);
2615 next_fn = PCI_ARI_CAP_NFN(cap);
2616 if (next_fn <= fn)
2617 return -ENODEV; /* protect against malformed list */
2618
2619 return next_fn;
2620}
2621
2622static int next_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2623{
2624 if (pci_ari_enabled(bus))
2625 return next_ari_fn(bus, dev, fn);
2626
2627 if (fn >= 7)
2628 return -ENODEV;
2629 /* only multifunction devices may have more functions */
2630 if (dev && !dev->multifunction)
2631 return -ENODEV;
2632
2633 return fn + 1;
2634}
2635
2636static int only_one_child(struct pci_bus *bus)
2637{
2638 struct pci_dev *bridge = bus->self;
2639
2640 /*
2641 * Systems with unusual topologies set PCI_SCAN_ALL_PCIE_DEVS so
2642 * we scan for all possible devices, not just Device 0.
2643 */
2644 if (pci_has_flag(PCI_SCAN_ALL_PCIE_DEVS))
2645 return 0;
2646
2647 /*
2648 * A PCIe Downstream Port normally leads to a Link with only Device
2649 * 0 on it (PCIe spec r3.1, sec 7.3.1). As an optimization, scan
2650 * only for Device 0 in that situation.
2651 */
2652 if (bridge && pci_is_pcie(bridge) && pcie_downstream_port(bridge))
2653 return 1;
2654
2655 return 0;
2656}
2657
2658/**
2659 * pci_scan_slot - Scan a PCI slot on a bus for devices
2660 * @bus: PCI bus to scan
2661 * @devfn: slot number to scan (must have zero function)
2662 *
2663 * Scan a PCI slot on the specified PCI bus for devices, adding
2664 * discovered devices to the @bus->devices list. New devices
2665 * will not have is_added set.
2666 *
2667 * Returns the number of new devices found.
2668 */
2669int pci_scan_slot(struct pci_bus *bus, int devfn)
2670{
2671 struct pci_dev *dev;
2672 int fn = 0, nr = 0;
2673
2674 if (only_one_child(bus) && (devfn > 0))
2675 return 0; /* Already scanned the entire slot */
2676
2677 do {
2678 dev = pci_scan_single_device(bus, devfn + fn);
2679 if (dev) {
2680 if (!pci_dev_is_added(dev))
2681 nr++;
2682 if (fn > 0)
2683 dev->multifunction = 1;
2684 } else if (fn == 0) {
2685 /*
2686 * Function 0 is required unless we are running on
2687 * a hypervisor that passes through individual PCI
2688 * functions.
2689 */
2690 if (!hypervisor_isolated_pci_functions())
2691 break;
2692 }
2693 fn = next_fn(bus, dev, fn);
2694 } while (fn >= 0);
2695
2696 /* Only one slot has PCIe device */
2697 if (bus->self && nr)
2698 pcie_aspm_init_link_state(bus->self);
2699
2700 return nr;
2701}
2702EXPORT_SYMBOL(pci_scan_slot);
2703
2704static int pcie_find_smpss(struct pci_dev *dev, void *data)
2705{
2706 u8 *smpss = data;
2707
2708 if (!pci_is_pcie(dev))
2709 return 0;
2710
2711 /*
2712 * We don't have a way to change MPS settings on devices that have
2713 * drivers attached. A hot-added device might support only the minimum
2714 * MPS setting (MPS=128). Therefore, if the fabric contains a bridge
2715 * where devices may be hot-added, we limit the fabric MPS to 128 so
2716 * hot-added devices will work correctly.
2717 *
2718 * However, if we hot-add a device to a slot directly below a Root
2719 * Port, it's impossible for there to be other existing devices below
2720 * the port. We don't limit the MPS in this case because we can
2721 * reconfigure MPS on both the Root Port and the hot-added device,
2722 * and there are no other devices involved.
2723 *
2724 * Note that this PCIE_BUS_SAFE path assumes no peer-to-peer DMA.
2725 */
2726 if (dev->is_hotplug_bridge &&
2727 pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT)
2728 *smpss = 0;
2729
2730 if (*smpss > dev->pcie_mpss)
2731 *smpss = dev->pcie_mpss;
2732
2733 return 0;
2734}
2735
2736static void pcie_write_mps(struct pci_dev *dev, int mps)
2737{
2738 int rc;
2739
2740 if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
2741 mps = 128 << dev->pcie_mpss;
2742
2743 if (pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT &&
2744 dev->bus->self)
2745
2746 /*
2747 * For "Performance", the assumption is made that
2748 * downstream communication will never be larger than
2749 * the MRRS. So, the MPS only needs to be configured
2750 * for the upstream communication. This being the case,
2751 * walk from the top down and set the MPS of the child
2752 * to that of the parent bus.
2753 *
2754 * Configure the device MPS with the smaller of the
2755 * device MPSS or the bridge MPS (which is assumed to be
2756 * properly configured at this point to the largest
2757 * allowable MPS based on its parent bus).
2758 */
2759 mps = min(mps, pcie_get_mps(dev->bus->self));
2760 }
2761
2762 rc = pcie_set_mps(dev, mps);
2763 if (rc)
2764 pci_err(dev, "Failed attempting to set the MPS\n");
2765}
2766
2767static void pcie_write_mrrs(struct pci_dev *dev)
2768{
2769 int rc, mrrs;
2770
2771 /*
2772 * In the "safe" case, do not configure the MRRS. There appear to be
2773 * issues with setting MRRS to 0 on a number of devices.
2774 */
2775 if (pcie_bus_config != PCIE_BUS_PERFORMANCE)
2776 return;
2777
2778 /*
2779 * For max performance, the MRRS must be set to the largest supported
2780 * value. However, it cannot be configured larger than the MPS the
2781 * device or the bus can support. This should already be properly
2782 * configured by a prior call to pcie_write_mps().
2783 */
2784 mrrs = pcie_get_mps(dev);
2785
2786 /*
2787 * MRRS is a R/W register. Invalid values can be written, but a
2788 * subsequent read will verify if the value is acceptable or not.
2789 * If the MRRS value provided is not acceptable (e.g., too large),
2790 * shrink the value until it is acceptable to the HW.
2791 */
2792 while (mrrs != pcie_get_readrq(dev) && mrrs >= 128) {
2793 rc = pcie_set_readrq(dev, mrrs);
2794 if (!rc)
2795 break;
2796
2797 pci_warn(dev, "Failed attempting to set the MRRS\n");
2798 mrrs /= 2;
2799 }
2800
2801 if (mrrs < 128)
2802 pci_err(dev, "MRRS was unable to be configured with a safe value. If problems are experienced, try running with pci=pcie_bus_safe\n");
2803}
2804
2805static int pcie_bus_configure_set(struct pci_dev *dev, void *data)
2806{
2807 int mps, orig_mps;
2808
2809 if (!pci_is_pcie(dev))
2810 return 0;
2811
2812 if (pcie_bus_config == PCIE_BUS_TUNE_OFF ||
2813 pcie_bus_config == PCIE_BUS_DEFAULT)
2814 return 0;
2815
2816 mps = 128 << *(u8 *)data;
2817 orig_mps = pcie_get_mps(dev);
2818
2819 pcie_write_mps(dev, mps);
2820 pcie_write_mrrs(dev);
2821
2822 pci_info(dev, "Max Payload Size set to %4d/%4d (was %4d), Max Read Rq %4d\n",
2823 pcie_get_mps(dev), 128 << dev->pcie_mpss,
2824 orig_mps, pcie_get_readrq(dev));
2825
2826 return 0;
2827}
2828
2829/*
2830 * pcie_bus_configure_settings() requires that pci_walk_bus work in a top-down,
2831 * parents then children fashion. If this changes, then this code will not
2832 * work as designed.
2833 */
2834void pcie_bus_configure_settings(struct pci_bus *bus)
2835{
2836 u8 smpss = 0;
2837
2838 if (!bus->self)
2839 return;
2840
2841 if (!pci_is_pcie(bus->self))
2842 return;
2843
2844 /*
2845 * FIXME - Peer to peer DMA is possible, though the endpoint would need
2846 * to be aware of the MPS of the destination. To work around this,
2847 * simply force the MPS of the entire system to the smallest possible.
2848 */
2849 if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2850 smpss = 0;
2851
2852 if (pcie_bus_config == PCIE_BUS_SAFE) {
2853 smpss = bus->self->pcie_mpss;
2854
2855 pcie_find_smpss(bus->self, &smpss);
2856 pci_walk_bus(bus, pcie_find_smpss, &smpss);
2857 }
2858
2859 pcie_bus_configure_set(bus->self, &smpss);
2860 pci_walk_bus(bus, pcie_bus_configure_set, &smpss);
2861}
2862EXPORT_SYMBOL_GPL(pcie_bus_configure_settings);
2863
2864/*
2865 * Called after each bus is probed, but before its children are examined. This
2866 * is marked as __weak because multiple architectures define it.
2867 */
2868void __weak pcibios_fixup_bus(struct pci_bus *bus)
2869{
2870 /* nothing to do, expected to be removed in the future */
2871}
2872
2873/**
2874 * pci_scan_child_bus_extend() - Scan devices below a bus
2875 * @bus: Bus to scan for devices
2876 * @available_buses: Total number of buses available (%0 does not try to
2877 * extend beyond the minimal)
2878 *
2879 * Scans devices below @bus including subordinate buses. Returns new
2880 * subordinate number including all the found devices. Passing
2881 * @available_buses causes the remaining bus space to be distributed
2882 * equally between hotplug-capable bridges to allow future extension of the
2883 * hierarchy.
2884 */
2885static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
2886 unsigned int available_buses)
2887{
2888 unsigned int used_buses, normal_bridges = 0, hotplug_bridges = 0;
2889 unsigned int start = bus->busn_res.start;
2890 unsigned int devfn, cmax, max = start;
2891 struct pci_dev *dev;
2892
2893 dev_dbg(&bus->dev, "scanning bus\n");
2894
2895 /* Go find them, Rover! */
2896 for (devfn = 0; devfn < 256; devfn += 8)
2897 pci_scan_slot(bus, devfn);
2898
2899 /* Reserve buses for SR-IOV capability */
2900 used_buses = pci_iov_bus_range(bus);
2901 max += used_buses;
2902
2903 /*
2904 * After performing arch-dependent fixup of the bus, look behind
2905 * all PCI-to-PCI bridges on this bus.
2906 */
2907 if (!bus->is_added) {
2908 dev_dbg(&bus->dev, "fixups for bus\n");
2909 pcibios_fixup_bus(bus);
2910 bus->is_added = 1;
2911 }
2912
2913 /*
2914 * Calculate how many hotplug bridges and normal bridges there
2915 * are on this bus. We will distribute the additional available
2916 * buses between hotplug bridges.
2917 */
2918 for_each_pci_bridge(dev, bus) {
2919 if (dev->is_hotplug_bridge)
2920 hotplug_bridges++;
2921 else
2922 normal_bridges++;
2923 }
2924
2925 /*
2926 * Scan bridges that are already configured. We don't touch them
2927 * unless they are misconfigured (which will be done in the second
2928 * scan below).
2929 */
2930 for_each_pci_bridge(dev, bus) {
2931 cmax = max;
2932 max = pci_scan_bridge_extend(bus, dev, max, 0, 0);
2933
2934 /*
2935 * Reserve one bus for each bridge now to avoid extending
2936 * hotplug bridges too much during the second scan below.
2937 */
2938 used_buses++;
2939 if (max - cmax > 1)
2940 used_buses += max - cmax - 1;
2941 }
2942
2943 /* Scan bridges that need to be reconfigured */
2944 for_each_pci_bridge(dev, bus) {
2945 unsigned int buses = 0;
2946
2947 if (!hotplug_bridges && normal_bridges == 1) {
2948 /*
2949 * There is only one bridge on the bus (upstream
2950 * port) so it gets all available buses which it
2951 * can then distribute to the possible hotplug
2952 * bridges below.
2953 */
2954 buses = available_buses;
2955 } else if (dev->is_hotplug_bridge) {
2956 /*
2957 * Distribute the extra buses between hotplug
2958 * bridges if any.
2959 */
2960 buses = available_buses / hotplug_bridges;
2961 buses = min(buses, available_buses - used_buses + 1);
2962 }
2963
2964 cmax = max;
2965 max = pci_scan_bridge_extend(bus, dev, cmax, buses, 1);
2966 /* One bus is already accounted so don't add it again */
2967 if (max - cmax > 1)
2968 used_buses += max - cmax - 1;
2969 }
2970
2971 /*
2972 * Make sure a hotplug bridge has at least the minimum requested
2973 * number of buses but allow it to grow up to the maximum available
2974 * bus number if there is room.
2975 */
2976 if (bus->self && bus->self->is_hotplug_bridge) {
2977 used_buses = max_t(unsigned int, available_buses,
2978 pci_hotplug_bus_size - 1);
2979 if (max - start < used_buses) {
2980 max = start + used_buses;
2981
2982 /* Do not allocate more buses than we have room left */
2983 if (max > bus->busn_res.end)
2984 max = bus->busn_res.end;
2985
2986 dev_dbg(&bus->dev, "%pR extended by %#02x\n",
2987 &bus->busn_res, max - start);
2988 }
2989 }
2990
2991 /*
2992 * We've scanned the bus and so we know all about what's on
2993 * the other side of any bridges that may be on this bus plus
2994 * any devices.
2995 *
2996 * Return how far we've got finding sub-buses.
2997 */
2998 dev_dbg(&bus->dev, "bus scan returning with max=%02x\n", max);
2999 return max;
3000}
3001
3002/**
3003 * pci_scan_child_bus() - Scan devices below a bus
3004 * @bus: Bus to scan for devices
3005 *
3006 * Scans devices below @bus including subordinate buses. Returns new
3007 * subordinate number including all the found devices.
3008 */
3009unsigned int pci_scan_child_bus(struct pci_bus *bus)
3010{
3011 return pci_scan_child_bus_extend(bus, 0);
3012}
3013EXPORT_SYMBOL_GPL(pci_scan_child_bus);
3014
3015/**
3016 * pcibios_root_bridge_prepare - Platform-specific host bridge setup
3017 * @bridge: Host bridge to set up
3018 *
3019 * Default empty implementation. Replace with an architecture-specific setup
3020 * routine, if necessary.
3021 */
3022int __weak pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
3023{
3024 return 0;
3025}
3026
3027void __weak pcibios_add_bus(struct pci_bus *bus)
3028{
3029}
3030
3031void __weak pcibios_remove_bus(struct pci_bus *bus)
3032{
3033}
3034
3035struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
3036 struct pci_ops *ops, void *sysdata, struct list_head *resources)
3037{
3038 int error;
3039 struct pci_host_bridge *bridge;
3040
3041 bridge = pci_alloc_host_bridge(0);
3042 if (!bridge)
3043 return NULL;
3044
3045 bridge->dev.parent = parent;
3046
3047 list_splice_init(resources, &bridge->windows);
3048 bridge->sysdata = sysdata;
3049 bridge->busnr = bus;
3050 bridge->ops = ops;
3051
3052 error = pci_register_host_bridge(bridge);
3053 if (error < 0)
3054 goto err_out;
3055
3056 return bridge->bus;
3057
3058err_out:
3059 put_device(&bridge->dev);
3060 return NULL;
3061}
3062EXPORT_SYMBOL_GPL(pci_create_root_bus);
3063
3064int pci_host_probe(struct pci_host_bridge *bridge)
3065{
3066 struct pci_bus *bus, *child;
3067 int ret;
3068
3069 ret = pci_scan_root_bus_bridge(bridge);
3070 if (ret < 0) {
3071 dev_err(bridge->dev.parent, "Scanning root bridge failed");
3072 return ret;
3073 }
3074
3075 bus = bridge->bus;
3076
3077 /*
3078 * We insert PCI resources into the iomem_resource and
3079 * ioport_resource trees in either pci_bus_claim_resources()
3080 * or pci_bus_assign_resources().
3081 */
3082 if (pci_has_flag(PCI_PROBE_ONLY)) {
3083 pci_bus_claim_resources(bus);
3084 } else {
3085 pci_bus_size_bridges(bus);
3086 pci_bus_assign_resources(bus);
3087
3088 list_for_each_entry(child, &bus->children, node)
3089 pcie_bus_configure_settings(child);
3090 }
3091
3092 pci_bus_add_devices(bus);
3093 return 0;
3094}
3095EXPORT_SYMBOL_GPL(pci_host_probe);
3096
3097int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int bus_max)
3098{
3099 struct resource *res = &b->busn_res;
3100 struct resource *parent_res, *conflict;
3101
3102 res->start = bus;
3103 res->end = bus_max;
3104 res->flags = IORESOURCE_BUS;
3105
3106 if (!pci_is_root_bus(b))
3107 parent_res = &b->parent->busn_res;
3108 else {
3109 parent_res = get_pci_domain_busn_res(pci_domain_nr(b));
3110 res->flags |= IORESOURCE_PCI_FIXED;
3111 }
3112
3113 conflict = request_resource_conflict(parent_res, res);
3114
3115 if (conflict)
3116 dev_info(&b->dev,
3117 "busn_res: can not insert %pR under %s%pR (conflicts with %s %pR)\n",
3118 res, pci_is_root_bus(b) ? "domain " : "",
3119 parent_res, conflict->name, conflict);
3120
3121 return conflict == NULL;
3122}
3123
3124int pci_bus_update_busn_res_end(struct pci_bus *b, int bus_max)
3125{
3126 struct resource *res = &b->busn_res;
3127 struct resource old_res = *res;
3128 resource_size_t size;
3129 int ret;
3130
3131 if (res->start > bus_max)
3132 return -EINVAL;
3133
3134 size = bus_max - res->start + 1;
3135 ret = adjust_resource(res, res->start, size);
3136 dev_info(&b->dev, "busn_res: %pR end %s updated to %02x\n",
3137 &old_res, ret ? "can not be" : "is", bus_max);
3138
3139 if (!ret && !res->parent)
3140 pci_bus_insert_busn_res(b, res->start, res->end);
3141
3142 return ret;
3143}
3144
3145void pci_bus_release_busn_res(struct pci_bus *b)
3146{
3147 struct resource *res = &b->busn_res;
3148 int ret;
3149
3150 if (!res->flags || !res->parent)
3151 return;
3152
3153 ret = release_resource(res);
3154 dev_info(&b->dev, "busn_res: %pR %s released\n",
3155 res, ret ? "can not be" : "is");
3156}
3157
3158int pci_scan_root_bus_bridge(struct pci_host_bridge *bridge)
3159{
3160 struct resource_entry *window;
3161 bool found = false;
3162 struct pci_bus *b;
3163 int max, bus, ret;
3164
3165 if (!bridge)
3166 return -EINVAL;
3167
3168 resource_list_for_each_entry(window, &bridge->windows)
3169 if (window->res->flags & IORESOURCE_BUS) {
3170 bridge->busnr = window->res->start;
3171 found = true;
3172 break;
3173 }
3174
3175 ret = pci_register_host_bridge(bridge);
3176 if (ret < 0)
3177 return ret;
3178
3179 b = bridge->bus;
3180 bus = bridge->busnr;
3181
3182 if (!found) {
3183 dev_info(&b->dev,
3184 "No busn resource found for root bus, will use [bus %02x-ff]\n",
3185 bus);
3186 pci_bus_insert_busn_res(b, bus, 255);
3187 }
3188
3189 max = pci_scan_child_bus(b);
3190
3191 if (!found)
3192 pci_bus_update_busn_res_end(b, max);
3193
3194 return 0;
3195}
3196EXPORT_SYMBOL(pci_scan_root_bus_bridge);
3197
3198struct pci_bus *pci_scan_root_bus(struct device *parent, int bus,
3199 struct pci_ops *ops, void *sysdata, struct list_head *resources)
3200{
3201 struct resource_entry *window;
3202 bool found = false;
3203 struct pci_bus *b;
3204 int max;
3205
3206 resource_list_for_each_entry(window, resources)
3207 if (window->res->flags & IORESOURCE_BUS) {
3208 found = true;
3209 break;
3210 }
3211
3212 b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
3213 if (!b)
3214 return NULL;
3215
3216 if (!found) {
3217 dev_info(&b->dev,
3218 "No busn resource found for root bus, will use [bus %02x-ff]\n",
3219 bus);
3220 pci_bus_insert_busn_res(b, bus, 255);
3221 }
3222
3223 max = pci_scan_child_bus(b);
3224
3225 if (!found)
3226 pci_bus_update_busn_res_end(b, max);
3227
3228 return b;
3229}
3230EXPORT_SYMBOL(pci_scan_root_bus);
3231
3232struct pci_bus *pci_scan_bus(int bus, struct pci_ops *ops,
3233 void *sysdata)
3234{
3235 LIST_HEAD(resources);
3236 struct pci_bus *b;
3237
3238 pci_add_resource(&resources, &ioport_resource);
3239 pci_add_resource(&resources, &iomem_resource);
3240 pci_add_resource(&resources, &busn_resource);
3241 b = pci_create_root_bus(NULL, bus, ops, sysdata, &resources);
3242 if (b) {
3243 pci_scan_child_bus(b);
3244 } else {
3245 pci_free_resource_list(&resources);
3246 }
3247 return b;
3248}
3249EXPORT_SYMBOL(pci_scan_bus);
3250
3251/**
3252 * pci_rescan_bus_bridge_resize - Scan a PCI bus for devices
3253 * @bridge: PCI bridge for the bus to scan
3254 *
3255 * Scan a PCI bus and child buses for new devices, add them,
3256 * and enable them, resizing bridge mmio/io resource if necessary
3257 * and possible. The caller must ensure the child devices are already
3258 * removed for resizing to occur.
3259 *
3260 * Returns the max number of subordinate bus discovered.
3261 */
3262unsigned int pci_rescan_bus_bridge_resize(struct pci_dev *bridge)
3263{
3264 unsigned int max;
3265 struct pci_bus *bus = bridge->subordinate;
3266
3267 max = pci_scan_child_bus(bus);
3268
3269 pci_assign_unassigned_bridge_resources(bridge);
3270
3271 pci_bus_add_devices(bus);
3272
3273 return max;
3274}
3275
3276/**
3277 * pci_rescan_bus - Scan a PCI bus for devices
3278 * @bus: PCI bus to scan
3279 *
3280 * Scan a PCI bus and child buses for new devices, add them,
3281 * and enable them.
3282 *
3283 * Returns the max number of subordinate bus discovered.
3284 */
3285unsigned int pci_rescan_bus(struct pci_bus *bus)
3286{
3287 unsigned int max;
3288
3289 max = pci_scan_child_bus(bus);
3290 pci_assign_unassigned_bus_resources(bus);
3291 pci_bus_add_devices(bus);
3292
3293 return max;
3294}
3295EXPORT_SYMBOL_GPL(pci_rescan_bus);
3296
3297/*
3298 * pci_rescan_bus(), pci_rescan_bus_bridge_resize() and PCI device removal
3299 * routines should always be executed under this mutex.
3300 */
3301static DEFINE_MUTEX(pci_rescan_remove_lock);
3302
3303void pci_lock_rescan_remove(void)
3304{
3305 mutex_lock(&pci_rescan_remove_lock);
3306}
3307EXPORT_SYMBOL_GPL(pci_lock_rescan_remove);
3308
3309void pci_unlock_rescan_remove(void)
3310{
3311 mutex_unlock(&pci_rescan_remove_lock);
3312}
3313EXPORT_SYMBOL_GPL(pci_unlock_rescan_remove);
3314
3315static int __init pci_sort_bf_cmp(const struct device *d_a,
3316 const struct device *d_b)
3317{
3318 const struct pci_dev *a = to_pci_dev(d_a);
3319 const struct pci_dev *b = to_pci_dev(d_b);
3320
3321 if (pci_domain_nr(a->bus) < pci_domain_nr(b->bus)) return -1;
3322 else if (pci_domain_nr(a->bus) > pci_domain_nr(b->bus)) return 1;
3323
3324 if (a->bus->number < b->bus->number) return -1;
3325 else if (a->bus->number > b->bus->number) return 1;
3326
3327 if (a->devfn < b->devfn) return -1;
3328 else if (a->devfn > b->devfn) return 1;
3329
3330 return 0;
3331}
3332
3333void __init pci_sort_breadthfirst(void)
3334{
3335 bus_sort_breadthfirst(&pci_bus_type, &pci_sort_bf_cmp);
3336}
3337
3338int pci_hp_add_bridge(struct pci_dev *dev)
3339{
3340 struct pci_bus *parent = dev->bus;
3341 int busnr, start = parent->busn_res.start;
3342 unsigned int available_buses = 0;
3343 int end = parent->busn_res.end;
3344
3345 for (busnr = start; busnr <= end; busnr++) {
3346 if (!pci_find_bus(pci_domain_nr(parent), busnr))
3347 break;
3348 }
3349 if (busnr-- > end) {
3350 pci_err(dev, "No bus number available for hot-added bridge\n");
3351 return -1;
3352 }
3353
3354 /* Scan bridges that are already configured */
3355 busnr = pci_scan_bridge(parent, dev, busnr, 0);
3356
3357 /*
3358 * Distribute the available bus numbers between hotplug-capable
3359 * bridges to make extending the chain later possible.
3360 */
3361 available_buses = end - busnr;
3362
3363 /* Scan bridges that need to be reconfigured */
3364 pci_scan_bridge_extend(parent, dev, busnr, available_buses, 1);
3365
3366 if (!dev->subordinate)
3367 return -1;
3368
3369 return 0;
3370}
3371EXPORT_SYMBOL_GPL(pci_hp_add_bridge);