Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * PCI Message Signaled Interrupt (MSI)
4 *
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 * Copyright (C) 2016 Christoph Hellwig.
8 */
9#include <linux/bitfield.h>
10#include <linux/err.h>
11#include <linux/export.h>
12#include <linux/irq.h>
13
14#include "../pci.h"
15#include "msi.h"
16
17int pci_msi_enable = 1;
18int pci_msi_ignore_mask;
19
20/**
21 * pci_msi_supported - check whether MSI may be enabled on a device
22 * @dev: pointer to the pci_dev data structure of MSI device function
23 * @nvec: how many MSIs have been requested?
24 *
25 * Look at global flags, the device itself, and its parent buses
26 * to determine if MSI/-X are supported for the device. If MSI/-X is
27 * supported return 1, else return 0.
28 **/
29static int pci_msi_supported(struct pci_dev *dev, int nvec)
30{
31 struct pci_bus *bus;
32
33 /* MSI must be globally enabled and supported by the device */
34 if (!pci_msi_enable)
35 return 0;
36
37 if (!dev || dev->no_msi)
38 return 0;
39
40 /*
41 * You can't ask to have 0 or less MSIs configured.
42 * a) it's stupid ..
43 * b) the list manipulation code assumes nvec >= 1.
44 */
45 if (nvec < 1)
46 return 0;
47
48 /*
49 * Any bridge which does NOT route MSI transactions from its
50 * secondary bus to its primary bus must set NO_MSI flag on
51 * the secondary pci_bus.
52 *
53 * The NO_MSI flag can either be set directly by:
54 * - arch-specific PCI host bus controller drivers (deprecated)
55 * - quirks for specific PCI bridges
56 *
57 * or indirectly by platform-specific PCI host bridge drivers by
58 * advertising the 'msi_domain' property, which results in
59 * the NO_MSI flag when no MSI domain is found for this bridge
60 * at probe time.
61 */
62 for (bus = dev->bus; bus; bus = bus->parent)
63 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
64 return 0;
65
66 return 1;
67}
68
69static void pcim_msi_release(void *pcidev)
70{
71 struct pci_dev *dev = pcidev;
72
73 dev->is_msi_managed = false;
74 pci_free_irq_vectors(dev);
75}
76
77/*
78 * Needs to be separate from pcim_release to prevent an ordering problem
79 * vs. msi_device_data_release() in the MSI core code.
80 */
81static int pcim_setup_msi_release(struct pci_dev *dev)
82{
83 int ret;
84
85 if (!pci_is_managed(dev) || dev->is_msi_managed)
86 return 0;
87
88 ret = devm_add_action(&dev->dev, pcim_msi_release, dev);
89 if (ret)
90 return ret;
91
92 dev->is_msi_managed = true;
93 return 0;
94}
95
96/*
97 * Ordering vs. devres: msi device data has to be installed first so that
98 * pcim_msi_release() is invoked before it on device release.
99 */
100static int pci_setup_msi_context(struct pci_dev *dev)
101{
102 int ret = msi_setup_device_data(&dev->dev);
103
104 if (ret)
105 return ret;
106
107 return pcim_setup_msi_release(dev);
108}
109
110/*
111 * Helper functions for mask/unmask and MSI message handling
112 */
113
114void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set)
115{
116 raw_spinlock_t *lock = &to_pci_dev(desc->dev)->msi_lock;
117 unsigned long flags;
118
119 if (!desc->pci.msi_attrib.can_mask)
120 return;
121
122 raw_spin_lock_irqsave(lock, flags);
123 desc->pci.msi_mask &= ~clear;
124 desc->pci.msi_mask |= set;
125 pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->pci.mask_pos,
126 desc->pci.msi_mask);
127 raw_spin_unlock_irqrestore(lock, flags);
128}
129
130/**
131 * pci_msi_mask_irq - Generic IRQ chip callback to mask PCI/MSI interrupts
132 * @data: pointer to irqdata associated to that interrupt
133 */
134void pci_msi_mask_irq(struct irq_data *data)
135{
136 struct msi_desc *desc = irq_data_get_msi_desc(data);
137
138 __pci_msi_mask_desc(desc, BIT(data->irq - desc->irq));
139}
140EXPORT_SYMBOL_GPL(pci_msi_mask_irq);
141
142/**
143 * pci_msi_unmask_irq - Generic IRQ chip callback to unmask PCI/MSI interrupts
144 * @data: pointer to irqdata associated to that interrupt
145 */
146void pci_msi_unmask_irq(struct irq_data *data)
147{
148 struct msi_desc *desc = irq_data_get_msi_desc(data);
149
150 __pci_msi_unmask_desc(desc, BIT(data->irq - desc->irq));
151}
152EXPORT_SYMBOL_GPL(pci_msi_unmask_irq);
153
154void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
155{
156 struct pci_dev *dev = msi_desc_to_pci_dev(entry);
157
158 BUG_ON(dev->current_state != PCI_D0);
159
160 if (entry->pci.msi_attrib.is_msix) {
161 void __iomem *base = pci_msix_desc_addr(entry);
162
163 if (WARN_ON_ONCE(entry->pci.msi_attrib.is_virtual))
164 return;
165
166 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR);
167 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR);
168 msg->data = readl(base + PCI_MSIX_ENTRY_DATA);
169 } else {
170 int pos = dev->msi_cap;
171 u16 data;
172
173 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,
174 &msg->address_lo);
175 if (entry->pci.msi_attrib.is_64) {
176 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,
177 &msg->address_hi);
178 pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data);
179 } else {
180 msg->address_hi = 0;
181 pci_read_config_word(dev, pos + PCI_MSI_DATA_32, &data);
182 }
183 msg->data = data;
184 }
185}
186
187static inline void pci_write_msg_msi(struct pci_dev *dev, struct msi_desc *desc,
188 struct msi_msg *msg)
189{
190 int pos = dev->msi_cap;
191 u16 msgctl;
192
193 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
194 msgctl &= ~PCI_MSI_FLAGS_QSIZE;
195 msgctl |= FIELD_PREP(PCI_MSI_FLAGS_QSIZE, desc->pci.msi_attrib.multiple);
196 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl);
197
198 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, msg->address_lo);
199 if (desc->pci.msi_attrib.is_64) {
200 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, msg->address_hi);
201 pci_write_config_word(dev, pos + PCI_MSI_DATA_64, msg->data);
202 } else {
203 pci_write_config_word(dev, pos + PCI_MSI_DATA_32, msg->data);
204 }
205 /* Ensure that the writes are visible in the device */
206 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
207}
208
209static inline void pci_write_msg_msix(struct msi_desc *desc, struct msi_msg *msg)
210{
211 void __iomem *base = pci_msix_desc_addr(desc);
212 u32 ctrl = desc->pci.msix_ctrl;
213 bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT);
214
215 if (desc->pci.msi_attrib.is_virtual)
216 return;
217 /*
218 * The specification mandates that the entry is masked
219 * when the message is modified:
220 *
221 * "If software changes the Address or Data value of an
222 * entry while the entry is unmasked, the result is
223 * undefined."
224 */
225 if (unmasked)
226 pci_msix_write_vector_ctrl(desc, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT);
227
228 writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR);
229 writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR);
230 writel(msg->data, base + PCI_MSIX_ENTRY_DATA);
231
232 if (unmasked)
233 pci_msix_write_vector_ctrl(desc, ctrl);
234
235 /* Ensure that the writes are visible in the device */
236 readl(base + PCI_MSIX_ENTRY_DATA);
237}
238
239void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
240{
241 struct pci_dev *dev = msi_desc_to_pci_dev(entry);
242
243 if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) {
244 /* Don't touch the hardware now */
245 } else if (entry->pci.msi_attrib.is_msix) {
246 pci_write_msg_msix(entry, msg);
247 } else {
248 pci_write_msg_msi(dev, entry, msg);
249 }
250
251 entry->msg = *msg;
252
253 if (entry->write_msi_msg)
254 entry->write_msi_msg(entry, entry->write_msi_msg_data);
255}
256
257void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg)
258{
259 struct msi_desc *entry = irq_get_msi_desc(irq);
260
261 __pci_write_msi_msg(entry, msg);
262}
263EXPORT_SYMBOL_GPL(pci_write_msi_msg);
264
265
266/* PCI/MSI specific functionality */
267
268static void pci_intx_for_msi(struct pci_dev *dev, int enable)
269{
270 if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG))
271 pci_intx(dev, enable);
272}
273
274static void pci_msi_set_enable(struct pci_dev *dev, int enable)
275{
276 u16 control;
277
278 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
279 control &= ~PCI_MSI_FLAGS_ENABLE;
280 if (enable)
281 control |= PCI_MSI_FLAGS_ENABLE;
282 pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
283}
284
285static int msi_setup_msi_desc(struct pci_dev *dev, int nvec,
286 struct irq_affinity_desc *masks)
287{
288 struct msi_desc desc;
289 u16 control;
290
291 /* MSI Entry Initialization */
292 memset(&desc, 0, sizeof(desc));
293
294 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
295 /* Lies, damned lies, and MSIs */
296 if (dev->dev_flags & PCI_DEV_FLAGS_HAS_MSI_MASKING)
297 control |= PCI_MSI_FLAGS_MASKBIT;
298 /* Respect XEN's mask disabling */
299 if (pci_msi_ignore_mask)
300 control &= ~PCI_MSI_FLAGS_MASKBIT;
301
302 desc.nvec_used = nvec;
303 desc.pci.msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT);
304 desc.pci.msi_attrib.can_mask = !!(control & PCI_MSI_FLAGS_MASKBIT);
305 desc.pci.msi_attrib.default_irq = dev->irq;
306 desc.pci.msi_attrib.multi_cap = FIELD_GET(PCI_MSI_FLAGS_QMASK, control);
307 desc.pci.msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec));
308 desc.affinity = masks;
309
310 if (control & PCI_MSI_FLAGS_64BIT)
311 desc.pci.mask_pos = dev->msi_cap + PCI_MSI_MASK_64;
312 else
313 desc.pci.mask_pos = dev->msi_cap + PCI_MSI_MASK_32;
314
315 /* Save the initial mask status */
316 if (desc.pci.msi_attrib.can_mask)
317 pci_read_config_dword(dev, desc.pci.mask_pos, &desc.pci.msi_mask);
318
319 return msi_insert_msi_desc(&dev->dev, &desc);
320}
321
322static int msi_verify_entries(struct pci_dev *dev)
323{
324 struct msi_desc *entry;
325
326 if (!dev->no_64bit_msi)
327 return 0;
328
329 msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) {
330 if (entry->msg.address_hi) {
331 pci_err(dev, "arch assigned 64-bit MSI address %#x%08x but device only supports 32 bits\n",
332 entry->msg.address_hi, entry->msg.address_lo);
333 break;
334 }
335 }
336 return !entry ? 0 : -EIO;
337}
338
339/**
340 * msi_capability_init - configure device's MSI capability structure
341 * @dev: pointer to the pci_dev data structure of MSI device function
342 * @nvec: number of interrupts to allocate
343 * @affd: description of automatic IRQ affinity assignments (may be %NULL)
344 *
345 * Setup the MSI capability structure of the device with the requested
346 * number of interrupts. A return value of zero indicates the successful
347 * setup of an entry with the new MSI IRQ. A negative return value indicates
348 * an error, and a positive return value indicates the number of interrupts
349 * which could have been allocated.
350 */
351static int msi_capability_init(struct pci_dev *dev, int nvec,
352 struct irq_affinity *affd)
353{
354 struct irq_affinity_desc *masks = NULL;
355 struct msi_desc *entry, desc;
356 int ret;
357
358 /* Reject multi-MSI early on irq domain enabled architectures */
359 if (nvec > 1 && !pci_msi_domain_supports(dev, MSI_FLAG_MULTI_PCI_MSI, ALLOW_LEGACY))
360 return 1;
361
362 /*
363 * Disable MSI during setup in the hardware, but mark it enabled
364 * so that setup code can evaluate it.
365 */
366 pci_msi_set_enable(dev, 0);
367 dev->msi_enabled = 1;
368
369 if (affd)
370 masks = irq_create_affinity_masks(nvec, affd);
371
372 msi_lock_descs(&dev->dev);
373 ret = msi_setup_msi_desc(dev, nvec, masks);
374 if (ret)
375 goto fail;
376
377 /* All MSIs are unmasked by default; mask them all */
378 entry = msi_first_desc(&dev->dev, MSI_DESC_ALL);
379 pci_msi_mask(entry, msi_multi_mask(entry));
380 /*
381 * Copy the MSI descriptor for the error path because
382 * pci_msi_setup_msi_irqs() will free it for the hierarchical
383 * interrupt domain case.
384 */
385 memcpy(&desc, entry, sizeof(desc));
386
387 /* Configure MSI capability structure */
388 ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI);
389 if (ret)
390 goto err;
391
392 ret = msi_verify_entries(dev);
393 if (ret)
394 goto err;
395
396 /* Set MSI enabled bits */
397 pci_intx_for_msi(dev, 0);
398 pci_msi_set_enable(dev, 1);
399
400 pcibios_free_irq(dev);
401 dev->irq = entry->irq;
402 goto unlock;
403
404err:
405 pci_msi_unmask(&desc, msi_multi_mask(&desc));
406 pci_free_msi_irqs(dev);
407fail:
408 dev->msi_enabled = 0;
409unlock:
410 msi_unlock_descs(&dev->dev);
411 kfree(masks);
412 return ret;
413}
414
415int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
416 struct irq_affinity *affd)
417{
418 int nvec;
419 int rc;
420
421 if (!pci_msi_supported(dev, minvec) || dev->current_state != PCI_D0)
422 return -EINVAL;
423
424 /* Check whether driver already requested MSI-X IRQs */
425 if (dev->msix_enabled) {
426 pci_info(dev, "can't enable MSI (MSI-X already enabled)\n");
427 return -EINVAL;
428 }
429
430 if (maxvec < minvec)
431 return -ERANGE;
432
433 if (WARN_ON_ONCE(dev->msi_enabled))
434 return -EINVAL;
435
436 /* Test for the availability of MSI support */
437 if (!pci_msi_domain_supports(dev, 0, ALLOW_LEGACY))
438 return -ENOTSUPP;
439
440 nvec = pci_msi_vec_count(dev);
441 if (nvec < 0)
442 return nvec;
443 if (nvec < minvec)
444 return -ENOSPC;
445
446 if (nvec > maxvec)
447 nvec = maxvec;
448
449 rc = pci_setup_msi_context(dev);
450 if (rc)
451 return rc;
452
453 if (!pci_setup_msi_device_domain(dev))
454 return -ENODEV;
455
456 for (;;) {
457 if (affd) {
458 nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
459 if (nvec < minvec)
460 return -ENOSPC;
461 }
462
463 rc = msi_capability_init(dev, nvec, affd);
464 if (rc == 0)
465 return nvec;
466
467 if (rc < 0)
468 return rc;
469 if (rc < minvec)
470 return -ENOSPC;
471
472 nvec = rc;
473 }
474}
475
476/**
477 * pci_msi_vec_count - Return the number of MSI vectors a device can send
478 * @dev: device to report about
479 *
480 * This function returns the number of MSI vectors a device requested via
481 * Multiple Message Capable register. It returns a negative errno if the
482 * device is not capable sending MSI interrupts. Otherwise, the call succeeds
483 * and returns a power of two, up to a maximum of 2^5 (32), according to the
484 * MSI specification.
485 **/
486int pci_msi_vec_count(struct pci_dev *dev)
487{
488 int ret;
489 u16 msgctl;
490
491 if (!dev->msi_cap)
492 return -EINVAL;
493
494 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl);
495 ret = 1 << FIELD_GET(PCI_MSI_FLAGS_QMASK, msgctl);
496
497 return ret;
498}
499EXPORT_SYMBOL(pci_msi_vec_count);
500
501/*
502 * Architecture override returns true when the PCI MSI message should be
503 * written by the generic restore function.
504 */
505bool __weak arch_restore_msi_irqs(struct pci_dev *dev)
506{
507 return true;
508}
509
510void __pci_restore_msi_state(struct pci_dev *dev)
511{
512 struct msi_desc *entry;
513 u16 control;
514
515 if (!dev->msi_enabled)
516 return;
517
518 entry = irq_get_msi_desc(dev->irq);
519
520 pci_intx_for_msi(dev, 0);
521 pci_msi_set_enable(dev, 0);
522 if (arch_restore_msi_irqs(dev))
523 __pci_write_msi_msg(entry, &entry->msg);
524
525 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
526 pci_msi_update_mask(entry, 0, 0);
527 control &= ~PCI_MSI_FLAGS_QSIZE;
528 control |= PCI_MSI_FLAGS_ENABLE |
529 FIELD_PREP(PCI_MSI_FLAGS_QSIZE, entry->pci.msi_attrib.multiple);
530 pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
531}
532
533void pci_msi_shutdown(struct pci_dev *dev)
534{
535 struct msi_desc *desc;
536
537 if (!pci_msi_enable || !dev || !dev->msi_enabled)
538 return;
539
540 pci_msi_set_enable(dev, 0);
541 pci_intx_for_msi(dev, 1);
542 dev->msi_enabled = 0;
543
544 /* Return the device with MSI unmasked as initial states */
545 desc = msi_first_desc(&dev->dev, MSI_DESC_ALL);
546 if (!WARN_ON_ONCE(!desc))
547 pci_msi_unmask(desc, msi_multi_mask(desc));
548
549 /* Restore dev->irq to its default pin-assertion IRQ */
550 dev->irq = desc->pci.msi_attrib.default_irq;
551 pcibios_alloc_irq(dev);
552}
553
554/* PCI/MSI-X specific functionality */
555
556static void pci_msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set)
557{
558 u16 ctrl;
559
560 pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
561 ctrl &= ~clear;
562 ctrl |= set;
563 pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, ctrl);
564}
565
566static void __iomem *msix_map_region(struct pci_dev *dev,
567 unsigned int nr_entries)
568{
569 resource_size_t phys_addr;
570 u32 table_offset;
571 unsigned long flags;
572 u8 bir;
573
574 pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE,
575 &table_offset);
576 bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR);
577 flags = pci_resource_flags(dev, bir);
578 if (!flags || (flags & IORESOURCE_UNSET))
579 return NULL;
580
581 table_offset &= PCI_MSIX_TABLE_OFFSET;
582 phys_addr = pci_resource_start(dev, bir) + table_offset;
583
584 return ioremap(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
585}
586
587/**
588 * msix_prepare_msi_desc - Prepare a half initialized MSI descriptor for operation
589 * @dev: The PCI device for which the descriptor is prepared
590 * @desc: The MSI descriptor for preparation
591 *
592 * This is separate from msix_setup_msi_descs() below to handle dynamic
593 * allocations for MSI-X after initial enablement.
594 *
595 * Ideally the whole MSI-X setup would work that way, but there is no way to
596 * support this for the legacy arch_setup_msi_irqs() mechanism and for the
597 * fake irq domains like the x86 XEN one. Sigh...
598 *
599 * The descriptor is zeroed and only @desc::msi_index and @desc::affinity
600 * are set. When called from msix_setup_msi_descs() then the is_virtual
601 * attribute is initialized as well.
602 *
603 * Fill in the rest.
604 */
605void msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc)
606{
607 desc->nvec_used = 1;
608 desc->pci.msi_attrib.is_msix = 1;
609 desc->pci.msi_attrib.is_64 = 1;
610 desc->pci.msi_attrib.default_irq = dev->irq;
611 desc->pci.mask_base = dev->msix_base;
612 desc->pci.msi_attrib.can_mask = !pci_msi_ignore_mask &&
613 !desc->pci.msi_attrib.is_virtual;
614
615 if (desc->pci.msi_attrib.can_mask) {
616 void __iomem *addr = pci_msix_desc_addr(desc);
617
618 desc->pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
619 }
620}
621
622static int msix_setup_msi_descs(struct pci_dev *dev, struct msix_entry *entries,
623 int nvec, struct irq_affinity_desc *masks)
624{
625 int ret = 0, i, vec_count = pci_msix_vec_count(dev);
626 struct irq_affinity_desc *curmsk;
627 struct msi_desc desc;
628
629 memset(&desc, 0, sizeof(desc));
630
631 for (i = 0, curmsk = masks; i < nvec; i++, curmsk++) {
632 desc.msi_index = entries ? entries[i].entry : i;
633 desc.affinity = masks ? curmsk : NULL;
634 desc.pci.msi_attrib.is_virtual = desc.msi_index >= vec_count;
635
636 msix_prepare_msi_desc(dev, &desc);
637
638 ret = msi_insert_msi_desc(&dev->dev, &desc);
639 if (ret)
640 break;
641 }
642 return ret;
643}
644
645static void msix_update_entries(struct pci_dev *dev, struct msix_entry *entries)
646{
647 struct msi_desc *desc;
648
649 if (entries) {
650 msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) {
651 entries->vector = desc->irq;
652 entries++;
653 }
654 }
655}
656
657static void msix_mask_all(void __iomem *base, int tsize)
658{
659 u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT;
660 int i;
661
662 if (pci_msi_ignore_mask)
663 return;
664
665 for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE)
666 writel(ctrl, base + PCI_MSIX_ENTRY_VECTOR_CTRL);
667}
668
669static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries,
670 int nvec, struct irq_affinity *affd)
671{
672 struct irq_affinity_desc *masks = NULL;
673 int ret;
674
675 if (affd)
676 masks = irq_create_affinity_masks(nvec, affd);
677
678 msi_lock_descs(&dev->dev);
679 ret = msix_setup_msi_descs(dev, entries, nvec, masks);
680 if (ret)
681 goto out_free;
682
683 ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
684 if (ret)
685 goto out_free;
686
687 /* Check if all MSI entries honor device restrictions */
688 ret = msi_verify_entries(dev);
689 if (ret)
690 goto out_free;
691
692 msix_update_entries(dev, entries);
693 goto out_unlock;
694
695out_free:
696 pci_free_msi_irqs(dev);
697out_unlock:
698 msi_unlock_descs(&dev->dev);
699 kfree(masks);
700 return ret;
701}
702
703/**
704 * msix_capability_init - configure device's MSI-X capability
705 * @dev: pointer to the pci_dev data structure of MSI-X device function
706 * @entries: pointer to an array of struct msix_entry entries
707 * @nvec: number of @entries
708 * @affd: Optional pointer to enable automatic affinity assignment
709 *
710 * Setup the MSI-X capability structure of device function with a
711 * single MSI-X IRQ. A return of zero indicates the successful setup of
712 * requested MSI-X entries with allocated IRQs or non-zero for otherwise.
713 **/
714static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
715 int nvec, struct irq_affinity *affd)
716{
717 int ret, tsize;
718 u16 control;
719
720 /*
721 * Some devices require MSI-X to be enabled before the MSI-X
722 * registers can be accessed. Mask all the vectors to prevent
723 * interrupts coming in before they're fully set up.
724 */
725 pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_MASKALL |
726 PCI_MSIX_FLAGS_ENABLE);
727
728 /* Mark it enabled so setup functions can query it */
729 dev->msix_enabled = 1;
730
731 pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
732 /* Request & Map MSI-X table region */
733 tsize = msix_table_size(control);
734 dev->msix_base = msix_map_region(dev, tsize);
735 if (!dev->msix_base) {
736 ret = -ENOMEM;
737 goto out_disable;
738 }
739
740 ret = msix_setup_interrupts(dev, entries, nvec, affd);
741 if (ret)
742 goto out_disable;
743
744 /* Disable INTX */
745 pci_intx_for_msi(dev, 0);
746
747 /*
748 * Ensure that all table entries are masked to prevent
749 * stale entries from firing in a crash kernel.
750 *
751 * Done late to deal with a broken Marvell NVME device
752 * which takes the MSI-X mask bits into account even
753 * when MSI-X is disabled, which prevents MSI delivery.
754 */
755 msix_mask_all(dev->msix_base, tsize);
756 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
757
758 pcibios_free_irq(dev);
759 return 0;
760
761out_disable:
762 dev->msix_enabled = 0;
763 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0);
764
765 return ret;
766}
767
768static bool pci_msix_validate_entries(struct pci_dev *dev, struct msix_entry *entries, int nvec)
769{
770 bool nogap;
771 int i, j;
772
773 if (!entries)
774 return true;
775
776 nogap = pci_msi_domain_supports(dev, MSI_FLAG_MSIX_CONTIGUOUS, DENY_LEGACY);
777
778 for (i = 0; i < nvec; i++) {
779 /* Check for duplicate entries */
780 for (j = i + 1; j < nvec; j++) {
781 if (entries[i].entry == entries[j].entry)
782 return false;
783 }
784 /* Check for unsupported gaps */
785 if (nogap && entries[i].entry != i)
786 return false;
787 }
788 return true;
789}
790
791int __pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, int minvec,
792 int maxvec, struct irq_affinity *affd, int flags)
793{
794 int hwsize, rc, nvec = maxvec;
795
796 if (maxvec < minvec)
797 return -ERANGE;
798
799 if (dev->msi_enabled) {
800 pci_info(dev, "can't enable MSI-X (MSI already enabled)\n");
801 return -EINVAL;
802 }
803
804 if (WARN_ON_ONCE(dev->msix_enabled))
805 return -EINVAL;
806
807 /* Check MSI-X early on irq domain enabled architectures */
808 if (!pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX, ALLOW_LEGACY))
809 return -ENOTSUPP;
810
811 if (!pci_msi_supported(dev, nvec) || dev->current_state != PCI_D0)
812 return -EINVAL;
813
814 hwsize = pci_msix_vec_count(dev);
815 if (hwsize < 0)
816 return hwsize;
817
818 if (!pci_msix_validate_entries(dev, entries, nvec))
819 return -EINVAL;
820
821 if (hwsize < nvec) {
822 /* Keep the IRQ virtual hackery working */
823 if (flags & PCI_IRQ_VIRTUAL)
824 hwsize = nvec;
825 else
826 nvec = hwsize;
827 }
828
829 if (nvec < minvec)
830 return -ENOSPC;
831
832 rc = pci_setup_msi_context(dev);
833 if (rc)
834 return rc;
835
836 if (!pci_setup_msix_device_domain(dev, hwsize))
837 return -ENODEV;
838
839 for (;;) {
840 if (affd) {
841 nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
842 if (nvec < minvec)
843 return -ENOSPC;
844 }
845
846 rc = msix_capability_init(dev, entries, nvec, affd);
847 if (rc == 0)
848 return nvec;
849
850 if (rc < 0)
851 return rc;
852 if (rc < minvec)
853 return -ENOSPC;
854
855 nvec = rc;
856 }
857}
858
859void __pci_restore_msix_state(struct pci_dev *dev)
860{
861 struct msi_desc *entry;
862 bool write_msg;
863
864 if (!dev->msix_enabled)
865 return;
866
867 /* route the table */
868 pci_intx_for_msi(dev, 0);
869 pci_msix_clear_and_set_ctrl(dev, 0,
870 PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL);
871
872 write_msg = arch_restore_msi_irqs(dev);
873
874 msi_lock_descs(&dev->dev);
875 msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) {
876 if (write_msg)
877 __pci_write_msi_msg(entry, &entry->msg);
878 pci_msix_write_vector_ctrl(entry, entry->pci.msix_ctrl);
879 }
880 msi_unlock_descs(&dev->dev);
881
882 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
883}
884
885void pci_msix_shutdown(struct pci_dev *dev)
886{
887 struct msi_desc *desc;
888
889 if (!pci_msi_enable || !dev || !dev->msix_enabled)
890 return;
891
892 if (pci_dev_is_disconnected(dev)) {
893 dev->msix_enabled = 0;
894 return;
895 }
896
897 /* Return the device with MSI-X masked as initial states */
898 msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL)
899 pci_msix_mask(desc);
900
901 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
902 pci_intx_for_msi(dev, 1);
903 dev->msix_enabled = 0;
904 pcibios_alloc_irq(dev);
905}
906
907/* Common interfaces */
908
909void pci_free_msi_irqs(struct pci_dev *dev)
910{
911 pci_msi_teardown_msi_irqs(dev);
912
913 if (dev->msix_base) {
914 iounmap(dev->msix_base);
915 dev->msix_base = NULL;
916 }
917}
918
919/* Misc. infrastructure */
920
921struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc)
922{
923 return to_pci_dev(desc->dev);
924}
925EXPORT_SYMBOL(msi_desc_to_pci_dev);
926
927void pci_no_msi(void)
928{
929 pci_msi_enable = 0;
930}