1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Support of MSI, HPET and DMAR interrupts.
  4 *
  5 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
  6 *	Moved from arch/x86/kernel/apic/io_apic.c.
  7 * Jiang Liu <jiang.liu@linux.intel.com>
  8 *	Convert to hierarchical irqdomain
  9 */
 10#include <linux/mm.h>
 11#include <linux/interrupt.h>
 12#include <linux/irq.h>
 13#include <linux/pci.h>
 14#include <linux/dmar.h>
 15#include <linux/hpet.h>
 16#include <linux/msi.h>
 17#include <asm/irqdomain.h>
 18#include <asm/msidef.h>
 19#include <asm/hpet.h>
 20#include <asm/hw_irq.h>
 21#include <asm/apic.h>
 22#include <asm/irq_remapping.h>
 23
 24static struct irq_domain *msi_default_domain;
 25
 26static void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg)
 27{
 28	msg->address_hi = MSI_ADDR_BASE_HI;
 29
 30	if (x2apic_enabled())
 31		msg->address_hi |= MSI_ADDR_EXT_DEST_ID(cfg->dest_apicid);
 32
 33	msg->address_lo =
 34		MSI_ADDR_BASE_LO |
 35		((apic->irq_dest_mode == 0) ?
 36			MSI_ADDR_DEST_MODE_PHYSICAL :
 37			MSI_ADDR_DEST_MODE_LOGICAL) |
 38		MSI_ADDR_REDIRECTION_CPU |
 39		MSI_ADDR_DEST_ID(cfg->dest_apicid);
 40
 41	msg->data =
 42		MSI_DATA_TRIGGER_EDGE |
 43		MSI_DATA_LEVEL_ASSERT |
 44		MSI_DATA_DELIVERY_FIXED |
 45		MSI_DATA_VECTOR(cfg->vector);
 46}
 47
 48static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg)
 49{
 50	__irq_msi_compose_msg(irqd_cfg(data), msg);
 51}
 52
 53static void irq_msi_update_msg(struct irq_data *irqd, struct irq_cfg *cfg)
 54{
 55	struct msi_msg msg[2] = { [1] = { }, };
 56
 57	__irq_msi_compose_msg(cfg, msg);
 58	irq_data_get_irq_chip(irqd)->irq_write_msi_msg(irqd, msg);
 59}
 60
 61static int
 62msi_set_affinity(struct irq_data *irqd, const struct cpumask *mask, bool force)
 63{
 64	struct irq_cfg old_cfg, *cfg = irqd_cfg(irqd);
 65	struct irq_data *parent = irqd->parent_data;
 66	unsigned int cpu;
 67	int ret;
 68
 69	/* Save the current configuration */
 70	cpu = cpumask_first(irq_data_get_effective_affinity_mask(irqd));
 71	old_cfg = *cfg;
 72
 73	/* Allocate a new target vector */
 74	ret = parent->chip->irq_set_affinity(parent, mask, force);
 75	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
 76		return ret;
 77
 78	/*
 79	 * For non-maskable and non-remapped MSI interrupts the migration
 80	 * to a different destination CPU and a different vector has to be
 81	 * done careful to handle the possible stray interrupt which can be
 82	 * caused by the non-atomic update of the address/data pair.
 83	 *
 84	 * Direct update is possible when:
 85	 * - The MSI is maskable (remapped MSI does not use this code path)).
 86	 *   The quirk bit is not set in this case.
 87	 * - The new vector is the same as the old vector
 88	 * - The old vector is MANAGED_IRQ_SHUTDOWN_VECTOR (interrupt starts up)
 89	 * - The new destination CPU is the same as the old destination CPU
 90	 */
 91	if (!irqd_msi_nomask_quirk(irqd) ||
 92	    cfg->vector == old_cfg.vector ||
 93	    old_cfg.vector == MANAGED_IRQ_SHUTDOWN_VECTOR ||
 94	    cfg->dest_apicid == old_cfg.dest_apicid) {
 95		irq_msi_update_msg(irqd, cfg);
 96		return ret;
 97	}
 98
 99	/*
100	 * Paranoia: Validate that the interrupt target is the local
101	 * CPU.
102	 */
103	if (WARN_ON_ONCE(cpu != smp_processor_id())) {
104		irq_msi_update_msg(irqd, cfg);
105		return ret;
106	}
107
108	/*
109	 * Redirect the interrupt to the new vector on the current CPU
110	 * first. This might cause a spurious interrupt on this vector if
111	 * the device raises an interrupt right between this update and the
112	 * update to the final destination CPU.
113	 *
114	 * If the vector is in use then the installed device handler will
115	 * denote it as spurious which is no harm as this is a rare event
116	 * and interrupt handlers have to cope with spurious interrupts
117	 * anyway. If the vector is unused, then it is marked so it won't
118	 * trigger the 'No irq handler for vector' warning in
119	 * common_interrupt().
120	 *
121	 * This requires to hold vector lock to prevent concurrent updates to
122	 * the affected vector.
123	 */
124	lock_vector_lock();
125
126	/*
127	 * Mark the new target vector on the local CPU if it is currently
128	 * unused. Reuse the VECTOR_RETRIGGERED state which is also used in
129	 * the CPU hotplug path for a similar purpose. This cannot be
130	 * undone here as the current CPU has interrupts disabled and
131	 * cannot handle the interrupt before the whole set_affinity()
132	 * section is done. In the CPU unplug case, the current CPU is
133	 * about to vanish and will not handle any interrupts anymore. The
134	 * vector is cleaned up when the CPU comes online again.
135	 */
136	if (IS_ERR_OR_NULL(this_cpu_read(vector_irq[cfg->vector])))
137		this_cpu_write(vector_irq[cfg->vector], VECTOR_RETRIGGERED);
138
139	/* Redirect it to the new vector on the local CPU temporarily */
140	old_cfg.vector = cfg->vector;
141	irq_msi_update_msg(irqd, &old_cfg);
142
143	/* Now transition it to the target CPU */
144	irq_msi_update_msg(irqd, cfg);
145
146	/*
147	 * All interrupts after this point are now targeted at the new
148	 * vector/CPU.
149	 *
150	 * Drop vector lock before testing whether the temporary assignment
151	 * to the local CPU was hit by an interrupt raised in the device,
152	 * because the retrigger function acquires vector lock again.
153	 */
154	unlock_vector_lock();
155
156	/*
157	 * Check whether the transition raced with a device interrupt and
158	 * is pending in the local APICs IRR. It is safe to do this outside
159	 * of vector lock as the irq_desc::lock of this interrupt is still
160	 * held and interrupts are disabled: The check is not accessing the
161	 * underlying vector store. It's just checking the local APIC's
162	 * IRR.
163	 */
164	if (lapic_vector_set_in_irr(cfg->vector))
165		irq_data_get_irq_chip(irqd)->irq_retrigger(irqd);
166
167	return ret;
168}
169
170/*
171 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
172 * which implement the MSI or MSI-X Capability Structure.
173 */
174static struct irq_chip pci_msi_controller = {
175	.name			= "PCI-MSI",
176	.irq_unmask		= pci_msi_unmask_irq,
177	.irq_mask		= pci_msi_mask_irq,
178	.irq_ack		= irq_chip_ack_parent,
179	.irq_retrigger		= irq_chip_retrigger_hierarchy,
180	.irq_compose_msi_msg	= irq_msi_compose_msg,
181	.irq_set_affinity	= msi_set_affinity,
182	.flags			= IRQCHIP_SKIP_SET_WAKE,
183};
184
185int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
186{
187	struct irq_domain *domain;
188	struct irq_alloc_info info;
189
190	init_irq_alloc_info(&info, NULL);
191	info.type = X86_IRQ_ALLOC_TYPE_MSI;
192	info.msi_dev = dev;
193
194	domain = irq_remapping_get_irq_domain(&info);
195	if (domain == NULL)
196		domain = msi_default_domain;
197	if (domain == NULL)
198		return -ENOSYS;
199
200	return msi_domain_alloc_irqs(domain, &dev->dev, nvec);
201}
202
203void native_teardown_msi_irq(unsigned int irq)
204{
205	irq_domain_free_irqs(irq, 1);
206}
207
208static irq_hw_number_t pci_msi_get_hwirq(struct msi_domain_info *info,
209					 msi_alloc_info_t *arg)
210{
211	return arg->msi_hwirq;
212}
213
214int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec,
215		    msi_alloc_info_t *arg)
216{
217	struct pci_dev *pdev = to_pci_dev(dev);
218	struct msi_desc *desc = first_pci_msi_entry(pdev);
219
220	init_irq_alloc_info(arg, NULL);
221	arg->msi_dev = pdev;
222	if (desc->msi_attrib.is_msix) {
223		arg->type = X86_IRQ_ALLOC_TYPE_MSIX;
224	} else {
225		arg->type = X86_IRQ_ALLOC_TYPE_MSI;
226		arg->flags |= X86_IRQ_ALLOC_CONTIGUOUS_VECTORS;
227	}
228
229	return 0;
230}
231EXPORT_SYMBOL_GPL(pci_msi_prepare);
232
233void pci_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
234{
235	arg->msi_hwirq = pci_msi_domain_calc_hwirq(arg->msi_dev, desc);
236}
237EXPORT_SYMBOL_GPL(pci_msi_set_desc);
238
239static struct msi_domain_ops pci_msi_domain_ops = {
240	.get_hwirq	= pci_msi_get_hwirq,
241	.msi_prepare	= pci_msi_prepare,
242	.set_desc	= pci_msi_set_desc,
243};
244
245static struct msi_domain_info pci_msi_domain_info = {
246	.flags		= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
247			  MSI_FLAG_PCI_MSIX,
248	.ops		= &pci_msi_domain_ops,
249	.chip		= &pci_msi_controller,
250	.handler	= handle_edge_irq,
251	.handler_name	= "edge",
252};
253
254void __init arch_init_msi_domain(struct irq_domain *parent)
255{
256	struct fwnode_handle *fn;
257
258	if (disable_apic)
259		return;
260
261	fn = irq_domain_alloc_named_fwnode("PCI-MSI");
262	if (fn) {
263		msi_default_domain =
264			pci_msi_create_irq_domain(fn, &pci_msi_domain_info,
265						  parent);
266	}
267	if (!msi_default_domain) {
268		irq_domain_free_fwnode(fn);
269		pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n");
270	} else {
271		msi_default_domain->flags |= IRQ_DOMAIN_MSI_NOMASK_QUIRK;
272	}
273}
274
275#ifdef CONFIG_IRQ_REMAP
276static struct irq_chip pci_msi_ir_controller = {
277	.name			= "IR-PCI-MSI",
278	.irq_unmask		= pci_msi_unmask_irq,
279	.irq_mask		= pci_msi_mask_irq,
280	.irq_ack		= irq_chip_ack_parent,
281	.irq_retrigger		= irq_chip_retrigger_hierarchy,
282	.irq_set_vcpu_affinity	= irq_chip_set_vcpu_affinity_parent,
283	.flags			= IRQCHIP_SKIP_SET_WAKE,
284};
285
286static struct msi_domain_info pci_msi_ir_domain_info = {
287	.flags		= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
288			  MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX,
289	.ops		= &pci_msi_domain_ops,
290	.chip		= &pci_msi_ir_controller,
291	.handler	= handle_edge_irq,
292	.handler_name	= "edge",
293};
294
295struct irq_domain *arch_create_remap_msi_irq_domain(struct irq_domain *parent,
296						    const char *name, int id)
297{
298	struct fwnode_handle *fn;
299	struct irq_domain *d;
300
301	fn = irq_domain_alloc_named_id_fwnode(name, id);
302	if (!fn)
303		return NULL;
304	d = pci_msi_create_irq_domain(fn, &pci_msi_ir_domain_info, parent);
305	if (!d)
306		irq_domain_free_fwnode(fn);
307	return d;
308}
309#endif
310
311#ifdef CONFIG_DMAR_TABLE
312static void dmar_msi_write_msg(struct irq_data *data, struct msi_msg *msg)
313{
314	dmar_msi_write(data->irq, msg);
315}
316
317static struct irq_chip dmar_msi_controller = {
318	.name			= "DMAR-MSI",
319	.irq_unmask		= dmar_msi_unmask,
320	.irq_mask		= dmar_msi_mask,
321	.irq_ack		= irq_chip_ack_parent,
322	.irq_set_affinity	= msi_domain_set_affinity,
323	.irq_retrigger		= irq_chip_retrigger_hierarchy,
324	.irq_compose_msi_msg	= irq_msi_compose_msg,
325	.irq_write_msi_msg	= dmar_msi_write_msg,
326	.flags			= IRQCHIP_SKIP_SET_WAKE,
327};
328
329static irq_hw_number_t dmar_msi_get_hwirq(struct msi_domain_info *info,
330					  msi_alloc_info_t *arg)
331{
332	return arg->dmar_id;
333}
334
335static int dmar_msi_init(struct irq_domain *domain,
336			 struct msi_domain_info *info, unsigned int virq,
337			 irq_hw_number_t hwirq, msi_alloc_info_t *arg)
338{
339	irq_domain_set_info(domain, virq, arg->dmar_id, info->chip, NULL,
340			    handle_edge_irq, arg->dmar_data, "edge");
341
342	return 0;
343}
344
345static struct msi_domain_ops dmar_msi_domain_ops = {
346	.get_hwirq	= dmar_msi_get_hwirq,
347	.msi_init	= dmar_msi_init,
348};
349
350static struct msi_domain_info dmar_msi_domain_info = {
351	.ops		= &dmar_msi_domain_ops,
352	.chip		= &dmar_msi_controller,
353};
354
355static struct irq_domain *dmar_get_irq_domain(void)
356{
357	static struct irq_domain *dmar_domain;
358	static DEFINE_MUTEX(dmar_lock);
359	struct fwnode_handle *fn;
360
361	mutex_lock(&dmar_lock);
362	if (dmar_domain)
363		goto out;
364
365	fn = irq_domain_alloc_named_fwnode("DMAR-MSI");
366	if (fn) {
367		dmar_domain = msi_create_irq_domain(fn, &dmar_msi_domain_info,
368						    x86_vector_domain);
369		if (!dmar_domain)
370			irq_domain_free_fwnode(fn);
371	}
372out:
373	mutex_unlock(&dmar_lock);
374	return dmar_domain;
375}
376
377int dmar_alloc_hwirq(int id, int node, void *arg)
378{
379	struct irq_domain *domain = dmar_get_irq_domain();
380	struct irq_alloc_info info;
381
382	if (!domain)
383		return -1;
384
385	init_irq_alloc_info(&info, NULL);
386	info.type = X86_IRQ_ALLOC_TYPE_DMAR;
387	info.dmar_id = id;
388	info.dmar_data = arg;
389
390	return irq_domain_alloc_irqs(domain, 1, node, &info);
391}
392
393void dmar_free_hwirq(int irq)
394{
395	irq_domain_free_irqs(irq, 1);
396}
397#endif
398
399/*
400 * MSI message composition
401 */
402#ifdef CONFIG_HPET_TIMER
403static inline int hpet_dev_id(struct irq_domain *domain)
404{
405	struct msi_domain_info *info = msi_get_domain_info(domain);
406
407	return (int)(long)info->data;
408}
409
410static void hpet_msi_write_msg(struct irq_data *data, struct msi_msg *msg)
411{
412	hpet_msi_write(irq_data_get_irq_handler_data(data), msg);
413}
414
415static struct irq_chip hpet_msi_controller __ro_after_init = {
416	.name = "HPET-MSI",
417	.irq_unmask = hpet_msi_unmask,
418	.irq_mask = hpet_msi_mask,
419	.irq_ack = irq_chip_ack_parent,
420	.irq_set_affinity = msi_domain_set_affinity,
421	.irq_retrigger = irq_chip_retrigger_hierarchy,
422	.irq_compose_msi_msg = irq_msi_compose_msg,
423	.irq_write_msi_msg = hpet_msi_write_msg,
424	.flags = IRQCHIP_SKIP_SET_WAKE,
425};
426
427static irq_hw_number_t hpet_msi_get_hwirq(struct msi_domain_info *info,
428					  msi_alloc_info_t *arg)
429{
430	return arg->hpet_index;
431}
432
433static int hpet_msi_init(struct irq_domain *domain,
434			 struct msi_domain_info *info, unsigned int virq,
435			 irq_hw_number_t hwirq, msi_alloc_info_t *arg)
436{
437	irq_set_status_flags(virq, IRQ_MOVE_PCNTXT);
438	irq_domain_set_info(domain, virq, arg->hpet_index, info->chip, NULL,
439			    handle_edge_irq, arg->hpet_data, "edge");
440
441	return 0;
442}
443
444static void hpet_msi_free(struct irq_domain *domain,
445			  struct msi_domain_info *info, unsigned int virq)
446{
447	irq_clear_status_flags(virq, IRQ_MOVE_PCNTXT);
448}
449
450static struct msi_domain_ops hpet_msi_domain_ops = {
451	.get_hwirq	= hpet_msi_get_hwirq,
452	.msi_init	= hpet_msi_init,
453	.msi_free	= hpet_msi_free,
454};
455
456static struct msi_domain_info hpet_msi_domain_info = {
457	.ops		= &hpet_msi_domain_ops,
458	.chip		= &hpet_msi_controller,
459};
460
461struct irq_domain *hpet_create_irq_domain(int hpet_id)
462{
463	struct msi_domain_info *domain_info;
464	struct irq_domain *parent, *d;
465	struct irq_alloc_info info;
466	struct fwnode_handle *fn;
467
468	if (x86_vector_domain == NULL)
469		return NULL;
470
471	domain_info = kzalloc(sizeof(*domain_info), GFP_KERNEL);
472	if (!domain_info)
473		return NULL;
474
475	*domain_info = hpet_msi_domain_info;
476	domain_info->data = (void *)(long)hpet_id;
477
478	init_irq_alloc_info(&info, NULL);
479	info.type = X86_IRQ_ALLOC_TYPE_HPET;
480	info.hpet_id = hpet_id;
481	parent = irq_remapping_get_ir_irq_domain(&info);
482	if (parent == NULL)
483		parent = x86_vector_domain;
484	else
485		hpet_msi_controller.name = "IR-HPET-MSI";
486
487	fn = irq_domain_alloc_named_id_fwnode(hpet_msi_controller.name,
488					      hpet_id);
489	if (!fn) {
490		kfree(domain_info);
491		return NULL;
492	}
493
494	d = msi_create_irq_domain(fn, domain_info, parent);
495	if (!d) {
496		irq_domain_free_fwnode(fn);
497		kfree(domain_info);
498	}
499	return d;
500}
501
502int hpet_assign_irq(struct irq_domain *domain, struct hpet_channel *hc,
503		    int dev_num)
504{
505	struct irq_alloc_info info;
506
507	init_irq_alloc_info(&info, NULL);
508	info.type = X86_IRQ_ALLOC_TYPE_HPET;
509	info.hpet_data = hc;
510	info.hpet_id = hpet_dev_id(domain);
511	info.hpet_index = dev_num;
512
513	return irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, &info);
514}
515#endif