Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Xen event channels
4 *
5 * Xen models interrupts with abstract event channels. Because each
6 * domain gets 1024 event channels, but NR_IRQ is not that large, we
7 * must dynamically map irqs<->event channels. The event channels
8 * interface with the rest of the kernel by defining a xen interrupt
9 * chip. When an event is received, it is mapped to an irq and sent
10 * through the normal interrupt processing path.
11 *
12 * There are four kinds of events which can be mapped to an event
13 * channel:
14 *
15 * 1. Inter-domain notifications. This includes all the virtual
16 * device events, since they're driven by front-ends in another domain
17 * (typically dom0).
18 * 2. VIRQs, typically used for timers. These are per-cpu events.
19 * 3. IPIs.
20 * 4. PIRQs - Hardware interrupts.
21 *
22 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
23 */
24
25#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
26
27#include <linux/linkage.h>
28#include <linux/interrupt.h>
29#include <linux/irq.h>
30#include <linux/moduleparam.h>
31#include <linux/string.h>
32#include <linux/memblock.h>
33#include <linux/slab.h>
34#include <linux/irqnr.h>
35#include <linux/pci.h>
36
37#ifdef CONFIG_X86
38#include <asm/desc.h>
39#include <asm/ptrace.h>
40#include <asm/idtentry.h>
41#include <asm/irq.h>
42#include <asm/io_apic.h>
43#include <asm/i8259.h>
44#include <asm/xen/pci.h>
45#endif
46#include <asm/sync_bitops.h>
47#include <asm/xen/hypercall.h>
48#include <asm/xen/hypervisor.h>
49#include <xen/page.h>
50
51#include <xen/xen.h>
52#include <xen/hvm.h>
53#include <xen/xen-ops.h>
54#include <xen/events.h>
55#include <xen/interface/xen.h>
56#include <xen/interface/event_channel.h>
57#include <xen/interface/hvm/hvm_op.h>
58#include <xen/interface/hvm/params.h>
59#include <xen/interface/physdev.h>
60#include <xen/interface/sched.h>
61#include <xen/interface/vcpu.h>
62#include <asm/hw_irq.h>
63
64#include "events_internal.h"
65
66const struct evtchn_ops *evtchn_ops;
67
68/*
69 * This lock protects updates to the following mapping and reference-count
70 * arrays. The lock does not need to be acquired to read the mapping tables.
71 */
72static DEFINE_MUTEX(irq_mapping_update_lock);
73
74static LIST_HEAD(xen_irq_list_head);
75
76/* IRQ <-> VIRQ mapping. */
77static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
78
79/* IRQ <-> IPI mapping */
80static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
81
82int **evtchn_to_irq;
83#ifdef CONFIG_X86
84static unsigned long *pirq_eoi_map;
85#endif
86static bool (*pirq_needs_eoi)(unsigned irq);
87
88#define EVTCHN_ROW(e) (e / (PAGE_SIZE/sizeof(**evtchn_to_irq)))
89#define EVTCHN_COL(e) (e % (PAGE_SIZE/sizeof(**evtchn_to_irq)))
90#define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq))
91
92/* Xen will never allocate port zero for any purpose. */
93#define VALID_EVTCHN(chn) ((chn) != 0)
94
95static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY];
96
97static struct irq_chip xen_dynamic_chip;
98static struct irq_chip xen_percpu_chip;
99static struct irq_chip xen_pirq_chip;
100static void enable_dynirq(struct irq_data *data);
101static void disable_dynirq(struct irq_data *data);
102
103static void clear_evtchn_to_irq_row(unsigned row)
104{
105 unsigned col;
106
107 for (col = 0; col < EVTCHN_PER_ROW; col++)
108 evtchn_to_irq[row][col] = -1;
109}
110
111static void clear_evtchn_to_irq_all(void)
112{
113 unsigned row;
114
115 for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) {
116 if (evtchn_to_irq[row] == NULL)
117 continue;
118 clear_evtchn_to_irq_row(row);
119 }
120}
121
122static int set_evtchn_to_irq(evtchn_port_t evtchn, unsigned int irq)
123{
124 unsigned row;
125 unsigned col;
126
127 if (evtchn >= xen_evtchn_max_channels())
128 return -EINVAL;
129
130 row = EVTCHN_ROW(evtchn);
131 col = EVTCHN_COL(evtchn);
132
133 if (evtchn_to_irq[row] == NULL) {
134 /* Unallocated irq entries return -1 anyway */
135 if (irq == -1)
136 return 0;
137
138 evtchn_to_irq[row] = (int *)get_zeroed_page(GFP_KERNEL);
139 if (evtchn_to_irq[row] == NULL)
140 return -ENOMEM;
141
142 clear_evtchn_to_irq_row(row);
143 }
144
145 evtchn_to_irq[row][col] = irq;
146 return 0;
147}
148
149int get_evtchn_to_irq(evtchn_port_t evtchn)
150{
151 if (evtchn >= xen_evtchn_max_channels())
152 return -1;
153 if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL)
154 return -1;
155 return evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)];
156}
157
158/* Get info for IRQ */
159struct irq_info *info_for_irq(unsigned irq)
160{
161 if (irq < nr_legacy_irqs())
162 return legacy_info_ptrs[irq];
163 else
164 return irq_get_chip_data(irq);
165}
166
167static void set_info_for_irq(unsigned int irq, struct irq_info *info)
168{
169 if (irq < nr_legacy_irqs())
170 legacy_info_ptrs[irq] = info;
171 else
172 irq_set_chip_data(irq, info);
173}
174
175/* Constructors for packed IRQ information. */
176static int xen_irq_info_common_setup(struct irq_info *info,
177 unsigned irq,
178 enum xen_irq_type type,
179 evtchn_port_t evtchn,
180 unsigned short cpu)
181{
182 int ret;
183
184 BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
185
186 info->type = type;
187 info->irq = irq;
188 info->evtchn = evtchn;
189 info->cpu = cpu;
190
191 ret = set_evtchn_to_irq(evtchn, irq);
192 if (ret < 0)
193 return ret;
194
195 irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN);
196
197 return xen_evtchn_port_setup(info);
198}
199
200static int xen_irq_info_evtchn_setup(unsigned irq,
201 evtchn_port_t evtchn)
202{
203 struct irq_info *info = info_for_irq(irq);
204
205 return xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0);
206}
207
208static int xen_irq_info_ipi_setup(unsigned cpu,
209 unsigned irq,
210 evtchn_port_t evtchn,
211 enum ipi_vector ipi)
212{
213 struct irq_info *info = info_for_irq(irq);
214
215 info->u.ipi = ipi;
216
217 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
218
219 return xen_irq_info_common_setup(info, irq, IRQT_IPI, evtchn, 0);
220}
221
222static int xen_irq_info_virq_setup(unsigned cpu,
223 unsigned irq,
224 evtchn_port_t evtchn,
225 unsigned virq)
226{
227 struct irq_info *info = info_for_irq(irq);
228
229 info->u.virq = virq;
230
231 per_cpu(virq_to_irq, cpu)[virq] = irq;
232
233 return xen_irq_info_common_setup(info, irq, IRQT_VIRQ, evtchn, 0);
234}
235
236static int xen_irq_info_pirq_setup(unsigned irq,
237 evtchn_port_t evtchn,
238 unsigned pirq,
239 unsigned gsi,
240 uint16_t domid,
241 unsigned char flags)
242{
243 struct irq_info *info = info_for_irq(irq);
244
245 info->u.pirq.pirq = pirq;
246 info->u.pirq.gsi = gsi;
247 info->u.pirq.domid = domid;
248 info->u.pirq.flags = flags;
249
250 return xen_irq_info_common_setup(info, irq, IRQT_PIRQ, evtchn, 0);
251}
252
253static void xen_irq_info_cleanup(struct irq_info *info)
254{
255 set_evtchn_to_irq(info->evtchn, -1);
256 info->evtchn = 0;
257}
258
259/*
260 * Accessors for packed IRQ information.
261 */
262evtchn_port_t evtchn_from_irq(unsigned irq)
263{
264 if (WARN(irq >= nr_irqs, "Invalid irq %d!\n", irq))
265 return 0;
266
267 return info_for_irq(irq)->evtchn;
268}
269
270unsigned int irq_from_evtchn(evtchn_port_t evtchn)
271{
272 return get_evtchn_to_irq(evtchn);
273}
274EXPORT_SYMBOL_GPL(irq_from_evtchn);
275
276int irq_from_virq(unsigned int cpu, unsigned int virq)
277{
278 return per_cpu(virq_to_irq, cpu)[virq];
279}
280
281static enum ipi_vector ipi_from_irq(unsigned irq)
282{
283 struct irq_info *info = info_for_irq(irq);
284
285 BUG_ON(info == NULL);
286 BUG_ON(info->type != IRQT_IPI);
287
288 return info->u.ipi;
289}
290
291static unsigned virq_from_irq(unsigned irq)
292{
293 struct irq_info *info = info_for_irq(irq);
294
295 BUG_ON(info == NULL);
296 BUG_ON(info->type != IRQT_VIRQ);
297
298 return info->u.virq;
299}
300
301static unsigned pirq_from_irq(unsigned irq)
302{
303 struct irq_info *info = info_for_irq(irq);
304
305 BUG_ON(info == NULL);
306 BUG_ON(info->type != IRQT_PIRQ);
307
308 return info->u.pirq.pirq;
309}
310
311static enum xen_irq_type type_from_irq(unsigned irq)
312{
313 return info_for_irq(irq)->type;
314}
315
316unsigned cpu_from_irq(unsigned irq)
317{
318 return info_for_irq(irq)->cpu;
319}
320
321unsigned int cpu_from_evtchn(evtchn_port_t evtchn)
322{
323 int irq = get_evtchn_to_irq(evtchn);
324 unsigned ret = 0;
325
326 if (irq != -1)
327 ret = cpu_from_irq(irq);
328
329 return ret;
330}
331
332#ifdef CONFIG_X86
333static bool pirq_check_eoi_map(unsigned irq)
334{
335 return test_bit(pirq_from_irq(irq), pirq_eoi_map);
336}
337#endif
338
339static bool pirq_needs_eoi_flag(unsigned irq)
340{
341 struct irq_info *info = info_for_irq(irq);
342 BUG_ON(info->type != IRQT_PIRQ);
343
344 return info->u.pirq.flags & PIRQ_NEEDS_EOI;
345}
346
347static void bind_evtchn_to_cpu(evtchn_port_t evtchn, unsigned int cpu)
348{
349 int irq = get_evtchn_to_irq(evtchn);
350 struct irq_info *info = info_for_irq(irq);
351
352 BUG_ON(irq == -1);
353#ifdef CONFIG_SMP
354 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(cpu));
355#endif
356 xen_evtchn_port_bind_to_cpu(info, cpu);
357
358 info->cpu = cpu;
359}
360
361/**
362 * notify_remote_via_irq - send event to remote end of event channel via irq
363 * @irq: irq of event channel to send event to
364 *
365 * Unlike notify_remote_via_evtchn(), this is safe to use across
366 * save/restore. Notifications on a broken connection are silently
367 * dropped.
368 */
369void notify_remote_via_irq(int irq)
370{
371 evtchn_port_t evtchn = evtchn_from_irq(irq);
372
373 if (VALID_EVTCHN(evtchn))
374 notify_remote_via_evtchn(evtchn);
375}
376EXPORT_SYMBOL_GPL(notify_remote_via_irq);
377
378static void xen_irq_init(unsigned irq)
379{
380 struct irq_info *info;
381#ifdef CONFIG_SMP
382 /* By default all event channels notify CPU#0. */
383 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(0));
384#endif
385
386 info = kzalloc(sizeof(*info), GFP_KERNEL);
387 if (info == NULL)
388 panic("Unable to allocate metadata for IRQ%d\n", irq);
389
390 info->type = IRQT_UNBOUND;
391 info->refcnt = -1;
392
393 set_info_for_irq(irq, info);
394
395 list_add_tail(&info->list, &xen_irq_list_head);
396}
397
398static int __must_check xen_allocate_irqs_dynamic(int nvec)
399{
400 int i, irq = irq_alloc_descs(-1, 0, nvec, -1);
401
402 if (irq >= 0) {
403 for (i = 0; i < nvec; i++)
404 xen_irq_init(irq + i);
405 }
406
407 return irq;
408}
409
410static inline int __must_check xen_allocate_irq_dynamic(void)
411{
412
413 return xen_allocate_irqs_dynamic(1);
414}
415
416static int __must_check xen_allocate_irq_gsi(unsigned gsi)
417{
418 int irq;
419
420 /*
421 * A PV guest has no concept of a GSI (since it has no ACPI
422 * nor access to/knowledge of the physical APICs). Therefore
423 * all IRQs are dynamically allocated from the entire IRQ
424 * space.
425 */
426 if (xen_pv_domain() && !xen_initial_domain())
427 return xen_allocate_irq_dynamic();
428
429 /* Legacy IRQ descriptors are already allocated by the arch. */
430 if (gsi < nr_legacy_irqs())
431 irq = gsi;
432 else
433 irq = irq_alloc_desc_at(gsi, -1);
434
435 xen_irq_init(irq);
436
437 return irq;
438}
439
440static void xen_free_irq(unsigned irq)
441{
442 struct irq_info *info = info_for_irq(irq);
443
444 if (WARN_ON(!info))
445 return;
446
447 list_del(&info->list);
448
449 set_info_for_irq(irq, NULL);
450
451 WARN_ON(info->refcnt > 0);
452
453 kfree(info);
454
455 /* Legacy IRQ descriptors are managed by the arch. */
456 if (irq < nr_legacy_irqs())
457 return;
458
459 irq_free_desc(irq);
460}
461
462static void xen_evtchn_close(evtchn_port_t port)
463{
464 struct evtchn_close close;
465
466 close.port = port;
467 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
468 BUG();
469}
470
471static void pirq_query_unmask(int irq)
472{
473 struct physdev_irq_status_query irq_status;
474 struct irq_info *info = info_for_irq(irq);
475
476 BUG_ON(info->type != IRQT_PIRQ);
477
478 irq_status.irq = pirq_from_irq(irq);
479 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
480 irq_status.flags = 0;
481
482 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
483 if (irq_status.flags & XENIRQSTAT_needs_eoi)
484 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
485}
486
487static void eoi_pirq(struct irq_data *data)
488{
489 evtchn_port_t evtchn = evtchn_from_irq(data->irq);
490 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
491 int rc = 0;
492
493 if (!VALID_EVTCHN(evtchn))
494 return;
495
496 if (unlikely(irqd_is_setaffinity_pending(data)) &&
497 likely(!irqd_irq_disabled(data))) {
498 int masked = test_and_set_mask(evtchn);
499
500 clear_evtchn(evtchn);
501
502 irq_move_masked_irq(data);
503
504 if (!masked)
505 unmask_evtchn(evtchn);
506 } else
507 clear_evtchn(evtchn);
508
509 if (pirq_needs_eoi(data->irq)) {
510 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
511 WARN_ON(rc);
512 }
513}
514
515static void mask_ack_pirq(struct irq_data *data)
516{
517 disable_dynirq(data);
518 eoi_pirq(data);
519}
520
521static unsigned int __startup_pirq(unsigned int irq)
522{
523 struct evtchn_bind_pirq bind_pirq;
524 struct irq_info *info = info_for_irq(irq);
525 evtchn_port_t evtchn = evtchn_from_irq(irq);
526 int rc;
527
528 BUG_ON(info->type != IRQT_PIRQ);
529
530 if (VALID_EVTCHN(evtchn))
531 goto out;
532
533 bind_pirq.pirq = pirq_from_irq(irq);
534 /* NB. We are happy to share unless we are probing. */
535 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
536 BIND_PIRQ__WILL_SHARE : 0;
537 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
538 if (rc != 0) {
539 pr_warn("Failed to obtain physical IRQ %d\n", irq);
540 return 0;
541 }
542 evtchn = bind_pirq.port;
543
544 pirq_query_unmask(irq);
545
546 rc = set_evtchn_to_irq(evtchn, irq);
547 if (rc)
548 goto err;
549
550 info->evtchn = evtchn;
551 bind_evtchn_to_cpu(evtchn, 0);
552
553 rc = xen_evtchn_port_setup(info);
554 if (rc)
555 goto err;
556
557out:
558 unmask_evtchn(evtchn);
559 eoi_pirq(irq_get_irq_data(irq));
560
561 return 0;
562
563err:
564 pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc);
565 xen_evtchn_close(evtchn);
566 return 0;
567}
568
569static unsigned int startup_pirq(struct irq_data *data)
570{
571 return __startup_pirq(data->irq);
572}
573
574static void shutdown_pirq(struct irq_data *data)
575{
576 unsigned int irq = data->irq;
577 struct irq_info *info = info_for_irq(irq);
578 evtchn_port_t evtchn = evtchn_from_irq(irq);
579
580 BUG_ON(info->type != IRQT_PIRQ);
581
582 if (!VALID_EVTCHN(evtchn))
583 return;
584
585 mask_evtchn(evtchn);
586 xen_evtchn_close(evtchn);
587 xen_irq_info_cleanup(info);
588}
589
590static void enable_pirq(struct irq_data *data)
591{
592 enable_dynirq(data);
593}
594
595static void disable_pirq(struct irq_data *data)
596{
597 disable_dynirq(data);
598}
599
600int xen_irq_from_gsi(unsigned gsi)
601{
602 struct irq_info *info;
603
604 list_for_each_entry(info, &xen_irq_list_head, list) {
605 if (info->type != IRQT_PIRQ)
606 continue;
607
608 if (info->u.pirq.gsi == gsi)
609 return info->irq;
610 }
611
612 return -1;
613}
614EXPORT_SYMBOL_GPL(xen_irq_from_gsi);
615
616static void __unbind_from_irq(unsigned int irq)
617{
618 evtchn_port_t evtchn = evtchn_from_irq(irq);
619 struct irq_info *info = info_for_irq(irq);
620
621 if (info->refcnt > 0) {
622 info->refcnt--;
623 if (info->refcnt != 0)
624 return;
625 }
626
627 if (VALID_EVTCHN(evtchn)) {
628 unsigned int cpu = cpu_from_irq(irq);
629
630 xen_evtchn_close(evtchn);
631
632 switch (type_from_irq(irq)) {
633 case IRQT_VIRQ:
634 per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1;
635 break;
636 case IRQT_IPI:
637 per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1;
638 break;
639 default:
640 break;
641 }
642
643 xen_irq_info_cleanup(info);
644 }
645
646 xen_free_irq(irq);
647}
648
649/*
650 * Do not make any assumptions regarding the relationship between the
651 * IRQ number returned here and the Xen pirq argument.
652 *
653 * Note: We don't assign an event channel until the irq actually started
654 * up. Return an existing irq if we've already got one for the gsi.
655 *
656 * Shareable implies level triggered, not shareable implies edge
657 * triggered here.
658 */
659int xen_bind_pirq_gsi_to_irq(unsigned gsi,
660 unsigned pirq, int shareable, char *name)
661{
662 int irq = -1;
663 struct physdev_irq irq_op;
664 int ret;
665
666 mutex_lock(&irq_mapping_update_lock);
667
668 irq = xen_irq_from_gsi(gsi);
669 if (irq != -1) {
670 pr_info("%s: returning irq %d for gsi %u\n",
671 __func__, irq, gsi);
672 goto out;
673 }
674
675 irq = xen_allocate_irq_gsi(gsi);
676 if (irq < 0)
677 goto out;
678
679 irq_op.irq = irq;
680 irq_op.vector = 0;
681
682 /* Only the privileged domain can do this. For non-priv, the pcifront
683 * driver provides a PCI bus that does the call to do exactly
684 * this in the priv domain. */
685 if (xen_initial_domain() &&
686 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
687 xen_free_irq(irq);
688 irq = -ENOSPC;
689 goto out;
690 }
691
692 ret = xen_irq_info_pirq_setup(irq, 0, pirq, gsi, DOMID_SELF,
693 shareable ? PIRQ_SHAREABLE : 0);
694 if (ret < 0) {
695 __unbind_from_irq(irq);
696 irq = ret;
697 goto out;
698 }
699
700 pirq_query_unmask(irq);
701 /* We try to use the handler with the appropriate semantic for the
702 * type of interrupt: if the interrupt is an edge triggered
703 * interrupt we use handle_edge_irq.
704 *
705 * On the other hand if the interrupt is level triggered we use
706 * handle_fasteoi_irq like the native code does for this kind of
707 * interrupts.
708 *
709 * Depending on the Xen version, pirq_needs_eoi might return true
710 * not only for level triggered interrupts but for edge triggered
711 * interrupts too. In any case Xen always honors the eoi mechanism,
712 * not injecting any more pirqs of the same kind if the first one
713 * hasn't received an eoi yet. Therefore using the fasteoi handler
714 * is the right choice either way.
715 */
716 if (shareable)
717 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
718 handle_fasteoi_irq, name);
719 else
720 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
721 handle_edge_irq, name);
722
723out:
724 mutex_unlock(&irq_mapping_update_lock);
725
726 return irq;
727}
728
729#ifdef CONFIG_PCI_MSI
730int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
731{
732 int rc;
733 struct physdev_get_free_pirq op_get_free_pirq;
734
735 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
736 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
737
738 WARN_ONCE(rc == -ENOSYS,
739 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
740
741 return rc ? -1 : op_get_free_pirq.pirq;
742}
743
744int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
745 int pirq, int nvec, const char *name, domid_t domid)
746{
747 int i, irq, ret;
748
749 mutex_lock(&irq_mapping_update_lock);
750
751 irq = xen_allocate_irqs_dynamic(nvec);
752 if (irq < 0)
753 goto out;
754
755 for (i = 0; i < nvec; i++) {
756 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name);
757
758 ret = xen_irq_info_pirq_setup(irq + i, 0, pirq + i, 0, domid,
759 i == 0 ? 0 : PIRQ_MSI_GROUP);
760 if (ret < 0)
761 goto error_irq;
762 }
763
764 ret = irq_set_msi_desc(irq, msidesc);
765 if (ret < 0)
766 goto error_irq;
767out:
768 mutex_unlock(&irq_mapping_update_lock);
769 return irq;
770error_irq:
771 while (nvec--)
772 __unbind_from_irq(irq + nvec);
773 mutex_unlock(&irq_mapping_update_lock);
774 return ret;
775}
776#endif
777
778int xen_destroy_irq(int irq)
779{
780 struct physdev_unmap_pirq unmap_irq;
781 struct irq_info *info = info_for_irq(irq);
782 int rc = -ENOENT;
783
784 mutex_lock(&irq_mapping_update_lock);
785
786 /*
787 * If trying to remove a vector in a MSI group different
788 * than the first one skip the PIRQ unmap unless this vector
789 * is the first one in the group.
790 */
791 if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) {
792 unmap_irq.pirq = info->u.pirq.pirq;
793 unmap_irq.domid = info->u.pirq.domid;
794 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
795 /* If another domain quits without making the pci_disable_msix
796 * call, the Xen hypervisor takes care of freeing the PIRQs
797 * (free_domain_pirqs).
798 */
799 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
800 pr_info("domain %d does not have %d anymore\n",
801 info->u.pirq.domid, info->u.pirq.pirq);
802 else if (rc) {
803 pr_warn("unmap irq failed %d\n", rc);
804 goto out;
805 }
806 }
807
808 xen_free_irq(irq);
809
810out:
811 mutex_unlock(&irq_mapping_update_lock);
812 return rc;
813}
814
815int xen_irq_from_pirq(unsigned pirq)
816{
817 int irq;
818
819 struct irq_info *info;
820
821 mutex_lock(&irq_mapping_update_lock);
822
823 list_for_each_entry(info, &xen_irq_list_head, list) {
824 if (info->type != IRQT_PIRQ)
825 continue;
826 irq = info->irq;
827 if (info->u.pirq.pirq == pirq)
828 goto out;
829 }
830 irq = -1;
831out:
832 mutex_unlock(&irq_mapping_update_lock);
833
834 return irq;
835}
836
837
838int xen_pirq_from_irq(unsigned irq)
839{
840 return pirq_from_irq(irq);
841}
842EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
843
844int bind_evtchn_to_irq(evtchn_port_t evtchn)
845{
846 int irq;
847 int ret;
848
849 if (evtchn >= xen_evtchn_max_channels())
850 return -ENOMEM;
851
852 mutex_lock(&irq_mapping_update_lock);
853
854 irq = get_evtchn_to_irq(evtchn);
855
856 if (irq == -1) {
857 irq = xen_allocate_irq_dynamic();
858 if (irq < 0)
859 goto out;
860
861 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
862 handle_edge_irq, "event");
863
864 ret = xen_irq_info_evtchn_setup(irq, evtchn);
865 if (ret < 0) {
866 __unbind_from_irq(irq);
867 irq = ret;
868 goto out;
869 }
870 /* New interdomain events are bound to VCPU 0. */
871 bind_evtchn_to_cpu(evtchn, 0);
872 } else {
873 struct irq_info *info = info_for_irq(irq);
874 WARN_ON(info == NULL || info->type != IRQT_EVTCHN);
875 }
876
877out:
878 mutex_unlock(&irq_mapping_update_lock);
879
880 return irq;
881}
882EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
883
884static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
885{
886 struct evtchn_bind_ipi bind_ipi;
887 evtchn_port_t evtchn;
888 int ret, irq;
889
890 mutex_lock(&irq_mapping_update_lock);
891
892 irq = per_cpu(ipi_to_irq, cpu)[ipi];
893
894 if (irq == -1) {
895 irq = xen_allocate_irq_dynamic();
896 if (irq < 0)
897 goto out;
898
899 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
900 handle_percpu_irq, "ipi");
901
902 bind_ipi.vcpu = xen_vcpu_nr(cpu);
903 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
904 &bind_ipi) != 0)
905 BUG();
906 evtchn = bind_ipi.port;
907
908 ret = xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
909 if (ret < 0) {
910 __unbind_from_irq(irq);
911 irq = ret;
912 goto out;
913 }
914 bind_evtchn_to_cpu(evtchn, cpu);
915 } else {
916 struct irq_info *info = info_for_irq(irq);
917 WARN_ON(info == NULL || info->type != IRQT_IPI);
918 }
919
920 out:
921 mutex_unlock(&irq_mapping_update_lock);
922 return irq;
923}
924
925int bind_interdomain_evtchn_to_irq(unsigned int remote_domain,
926 evtchn_port_t remote_port)
927{
928 struct evtchn_bind_interdomain bind_interdomain;
929 int err;
930
931 bind_interdomain.remote_dom = remote_domain;
932 bind_interdomain.remote_port = remote_port;
933
934 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
935 &bind_interdomain);
936
937 return err ? : bind_evtchn_to_irq(bind_interdomain.local_port);
938}
939EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq);
940
941static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn)
942{
943 struct evtchn_status status;
944 evtchn_port_t port;
945 int rc = -ENOENT;
946
947 memset(&status, 0, sizeof(status));
948 for (port = 0; port < xen_evtchn_max_channels(); port++) {
949 status.dom = DOMID_SELF;
950 status.port = port;
951 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
952 if (rc < 0)
953 continue;
954 if (status.status != EVTCHNSTAT_virq)
955 continue;
956 if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) {
957 *evtchn = port;
958 break;
959 }
960 }
961 return rc;
962}
963
964/**
965 * xen_evtchn_nr_channels - number of usable event channel ports
966 *
967 * This may be less than the maximum supported by the current
968 * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum
969 * supported.
970 */
971unsigned xen_evtchn_nr_channels(void)
972{
973 return evtchn_ops->nr_channels();
974}
975EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels);
976
977int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu)
978{
979 struct evtchn_bind_virq bind_virq;
980 evtchn_port_t evtchn = 0;
981 int irq, ret;
982
983 mutex_lock(&irq_mapping_update_lock);
984
985 irq = per_cpu(virq_to_irq, cpu)[virq];
986
987 if (irq == -1) {
988 irq = xen_allocate_irq_dynamic();
989 if (irq < 0)
990 goto out;
991
992 if (percpu)
993 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
994 handle_percpu_irq, "virq");
995 else
996 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
997 handle_edge_irq, "virq");
998
999 bind_virq.virq = virq;
1000 bind_virq.vcpu = xen_vcpu_nr(cpu);
1001 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1002 &bind_virq);
1003 if (ret == 0)
1004 evtchn = bind_virq.port;
1005 else {
1006 if (ret == -EEXIST)
1007 ret = find_virq(virq, cpu, &evtchn);
1008 BUG_ON(ret < 0);
1009 }
1010
1011 ret = xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
1012 if (ret < 0) {
1013 __unbind_from_irq(irq);
1014 irq = ret;
1015 goto out;
1016 }
1017
1018 bind_evtchn_to_cpu(evtchn, cpu);
1019 } else {
1020 struct irq_info *info = info_for_irq(irq);
1021 WARN_ON(info == NULL || info->type != IRQT_VIRQ);
1022 }
1023
1024out:
1025 mutex_unlock(&irq_mapping_update_lock);
1026
1027 return irq;
1028}
1029
1030static void unbind_from_irq(unsigned int irq)
1031{
1032 mutex_lock(&irq_mapping_update_lock);
1033 __unbind_from_irq(irq);
1034 mutex_unlock(&irq_mapping_update_lock);
1035}
1036
1037int bind_evtchn_to_irqhandler(evtchn_port_t evtchn,
1038 irq_handler_t handler,
1039 unsigned long irqflags,
1040 const char *devname, void *dev_id)
1041{
1042 int irq, retval;
1043
1044 irq = bind_evtchn_to_irq(evtchn);
1045 if (irq < 0)
1046 return irq;
1047 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1048 if (retval != 0) {
1049 unbind_from_irq(irq);
1050 return retval;
1051 }
1052
1053 return irq;
1054}
1055EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1056
1057int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
1058 evtchn_port_t remote_port,
1059 irq_handler_t handler,
1060 unsigned long irqflags,
1061 const char *devname,
1062 void *dev_id)
1063{
1064 int irq, retval;
1065
1066 irq = bind_interdomain_evtchn_to_irq(remote_domain, remote_port);
1067 if (irq < 0)
1068 return irq;
1069
1070 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1071 if (retval != 0) {
1072 unbind_from_irq(irq);
1073 return retval;
1074 }
1075
1076 return irq;
1077}
1078EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler);
1079
1080int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
1081 irq_handler_t handler,
1082 unsigned long irqflags, const char *devname, void *dev_id)
1083{
1084 int irq, retval;
1085
1086 irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU);
1087 if (irq < 0)
1088 return irq;
1089 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1090 if (retval != 0) {
1091 unbind_from_irq(irq);
1092 return retval;
1093 }
1094
1095 return irq;
1096}
1097EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1098
1099int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1100 unsigned int cpu,
1101 irq_handler_t handler,
1102 unsigned long irqflags,
1103 const char *devname,
1104 void *dev_id)
1105{
1106 int irq, retval;
1107
1108 irq = bind_ipi_to_irq(ipi, cpu);
1109 if (irq < 0)
1110 return irq;
1111
1112 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
1113 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1114 if (retval != 0) {
1115 unbind_from_irq(irq);
1116 return retval;
1117 }
1118
1119 return irq;
1120}
1121
1122void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1123{
1124 struct irq_info *info = info_for_irq(irq);
1125
1126 if (WARN_ON(!info))
1127 return;
1128 free_irq(irq, dev_id);
1129 unbind_from_irq(irq);
1130}
1131EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1132
1133/**
1134 * xen_set_irq_priority() - set an event channel priority.
1135 * @irq:irq bound to an event channel.
1136 * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN.
1137 */
1138int xen_set_irq_priority(unsigned irq, unsigned priority)
1139{
1140 struct evtchn_set_priority set_priority;
1141
1142 set_priority.port = evtchn_from_irq(irq);
1143 set_priority.priority = priority;
1144
1145 return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority,
1146 &set_priority);
1147}
1148EXPORT_SYMBOL_GPL(xen_set_irq_priority);
1149
1150int evtchn_make_refcounted(evtchn_port_t evtchn)
1151{
1152 int irq = get_evtchn_to_irq(evtchn);
1153 struct irq_info *info;
1154
1155 if (irq == -1)
1156 return -ENOENT;
1157
1158 info = info_for_irq(irq);
1159
1160 if (!info)
1161 return -ENOENT;
1162
1163 WARN_ON(info->refcnt != -1);
1164
1165 info->refcnt = 1;
1166
1167 return 0;
1168}
1169EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1170
1171int evtchn_get(evtchn_port_t evtchn)
1172{
1173 int irq;
1174 struct irq_info *info;
1175 int err = -ENOENT;
1176
1177 if (evtchn >= xen_evtchn_max_channels())
1178 return -EINVAL;
1179
1180 mutex_lock(&irq_mapping_update_lock);
1181
1182 irq = get_evtchn_to_irq(evtchn);
1183 if (irq == -1)
1184 goto done;
1185
1186 info = info_for_irq(irq);
1187
1188 if (!info)
1189 goto done;
1190
1191 err = -EINVAL;
1192 if (info->refcnt <= 0)
1193 goto done;
1194
1195 info->refcnt++;
1196 err = 0;
1197 done:
1198 mutex_unlock(&irq_mapping_update_lock);
1199
1200 return err;
1201}
1202EXPORT_SYMBOL_GPL(evtchn_get);
1203
1204void evtchn_put(evtchn_port_t evtchn)
1205{
1206 int irq = get_evtchn_to_irq(evtchn);
1207 if (WARN_ON(irq == -1))
1208 return;
1209 unbind_from_irq(irq);
1210}
1211EXPORT_SYMBOL_GPL(evtchn_put);
1212
1213void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1214{
1215 int irq;
1216
1217#ifdef CONFIG_X86
1218 if (unlikely(vector == XEN_NMI_VECTOR)) {
1219 int rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, xen_vcpu_nr(cpu),
1220 NULL);
1221 if (rc < 0)
1222 printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc);
1223 return;
1224 }
1225#endif
1226 irq = per_cpu(ipi_to_irq, cpu)[vector];
1227 BUG_ON(irq < 0);
1228 notify_remote_via_irq(irq);
1229}
1230
1231static void __xen_evtchn_do_upcall(void)
1232{
1233 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
1234 int cpu = smp_processor_id();
1235
1236 do {
1237 vcpu_info->evtchn_upcall_pending = 0;
1238
1239 xen_evtchn_handle_events(cpu);
1240
1241 BUG_ON(!irqs_disabled());
1242
1243 virt_rmb(); /* Hypervisor can set upcall pending. */
1244
1245 } while (vcpu_info->evtchn_upcall_pending);
1246}
1247
1248void xen_evtchn_do_upcall(struct pt_regs *regs)
1249{
1250 struct pt_regs *old_regs = set_irq_regs(regs);
1251
1252 irq_enter();
1253
1254 __xen_evtchn_do_upcall();
1255
1256 irq_exit();
1257 set_irq_regs(old_regs);
1258}
1259
1260void xen_hvm_evtchn_do_upcall(void)
1261{
1262 __xen_evtchn_do_upcall();
1263}
1264EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
1265
1266/* Rebind a new event channel to an existing irq. */
1267void rebind_evtchn_irq(evtchn_port_t evtchn, int irq)
1268{
1269 struct irq_info *info = info_for_irq(irq);
1270
1271 if (WARN_ON(!info))
1272 return;
1273
1274 /* Make sure the irq is masked, since the new event channel
1275 will also be masked. */
1276 disable_irq(irq);
1277
1278 mutex_lock(&irq_mapping_update_lock);
1279
1280 /* After resume the irq<->evtchn mappings are all cleared out */
1281 BUG_ON(get_evtchn_to_irq(evtchn) != -1);
1282 /* Expect irq to have been bound before,
1283 so there should be a proper type */
1284 BUG_ON(info->type == IRQT_UNBOUND);
1285
1286 (void)xen_irq_info_evtchn_setup(irq, evtchn);
1287
1288 mutex_unlock(&irq_mapping_update_lock);
1289
1290 bind_evtchn_to_cpu(evtchn, info->cpu);
1291 /* This will be deferred until interrupt is processed */
1292 irq_set_affinity(irq, cpumask_of(info->cpu));
1293
1294 /* Unmask the event channel. */
1295 enable_irq(irq);
1296}
1297
1298/* Rebind an evtchn so that it gets delivered to a specific cpu */
1299static int xen_rebind_evtchn_to_cpu(evtchn_port_t evtchn, unsigned int tcpu)
1300{
1301 struct evtchn_bind_vcpu bind_vcpu;
1302 int masked;
1303
1304 if (!VALID_EVTCHN(evtchn))
1305 return -1;
1306
1307 if (!xen_support_evtchn_rebind())
1308 return -1;
1309
1310 /* Send future instances of this interrupt to other vcpu. */
1311 bind_vcpu.port = evtchn;
1312 bind_vcpu.vcpu = xen_vcpu_nr(tcpu);
1313
1314 /*
1315 * Mask the event while changing the VCPU binding to prevent
1316 * it being delivered on an unexpected VCPU.
1317 */
1318 masked = test_and_set_mask(evtchn);
1319
1320 /*
1321 * If this fails, it usually just indicates that we're dealing with a
1322 * virq or IPI channel, which don't actually need to be rebound. Ignore
1323 * it, but don't do the xenlinux-level rebind in that case.
1324 */
1325 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1326 bind_evtchn_to_cpu(evtchn, tcpu);
1327
1328 if (!masked)
1329 unmask_evtchn(evtchn);
1330
1331 return 0;
1332}
1333
1334static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1335 bool force)
1336{
1337 unsigned tcpu = cpumask_first_and(dest, cpu_online_mask);
1338 int ret = xen_rebind_evtchn_to_cpu(evtchn_from_irq(data->irq), tcpu);
1339
1340 if (!ret)
1341 irq_data_update_effective_affinity(data, cpumask_of(tcpu));
1342
1343 return ret;
1344}
1345
1346/* To be called with desc->lock held. */
1347int xen_set_affinity_evtchn(struct irq_desc *desc, unsigned int tcpu)
1348{
1349 struct irq_data *d = irq_desc_get_irq_data(desc);
1350
1351 return set_affinity_irq(d, cpumask_of(tcpu), false);
1352}
1353EXPORT_SYMBOL_GPL(xen_set_affinity_evtchn);
1354
1355static void enable_dynirq(struct irq_data *data)
1356{
1357 evtchn_port_t evtchn = evtchn_from_irq(data->irq);
1358
1359 if (VALID_EVTCHN(evtchn))
1360 unmask_evtchn(evtchn);
1361}
1362
1363static void disable_dynirq(struct irq_data *data)
1364{
1365 evtchn_port_t evtchn = evtchn_from_irq(data->irq);
1366
1367 if (VALID_EVTCHN(evtchn))
1368 mask_evtchn(evtchn);
1369}
1370
1371static void ack_dynirq(struct irq_data *data)
1372{
1373 evtchn_port_t evtchn = evtchn_from_irq(data->irq);
1374
1375 if (!VALID_EVTCHN(evtchn))
1376 return;
1377
1378 if (unlikely(irqd_is_setaffinity_pending(data)) &&
1379 likely(!irqd_irq_disabled(data))) {
1380 int masked = test_and_set_mask(evtchn);
1381
1382 clear_evtchn(evtchn);
1383
1384 irq_move_masked_irq(data);
1385
1386 if (!masked)
1387 unmask_evtchn(evtchn);
1388 } else
1389 clear_evtchn(evtchn);
1390}
1391
1392static void mask_ack_dynirq(struct irq_data *data)
1393{
1394 disable_dynirq(data);
1395 ack_dynirq(data);
1396}
1397
1398static int retrigger_dynirq(struct irq_data *data)
1399{
1400 evtchn_port_t evtchn = evtchn_from_irq(data->irq);
1401 int masked;
1402
1403 if (!VALID_EVTCHN(evtchn))
1404 return 0;
1405
1406 masked = test_and_set_mask(evtchn);
1407 set_evtchn(evtchn);
1408 if (!masked)
1409 unmask_evtchn(evtchn);
1410
1411 return 1;
1412}
1413
1414static void restore_pirqs(void)
1415{
1416 int pirq, rc, irq, gsi;
1417 struct physdev_map_pirq map_irq;
1418 struct irq_info *info;
1419
1420 list_for_each_entry(info, &xen_irq_list_head, list) {
1421 if (info->type != IRQT_PIRQ)
1422 continue;
1423
1424 pirq = info->u.pirq.pirq;
1425 gsi = info->u.pirq.gsi;
1426 irq = info->irq;
1427
1428 /* save/restore of PT devices doesn't work, so at this point the
1429 * only devices present are GSI based emulated devices */
1430 if (!gsi)
1431 continue;
1432
1433 map_irq.domid = DOMID_SELF;
1434 map_irq.type = MAP_PIRQ_TYPE_GSI;
1435 map_irq.index = gsi;
1436 map_irq.pirq = pirq;
1437
1438 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1439 if (rc) {
1440 pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1441 gsi, irq, pirq, rc);
1442 xen_free_irq(irq);
1443 continue;
1444 }
1445
1446 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1447
1448 __startup_pirq(irq);
1449 }
1450}
1451
1452static void restore_cpu_virqs(unsigned int cpu)
1453{
1454 struct evtchn_bind_virq bind_virq;
1455 evtchn_port_t evtchn;
1456 int virq, irq;
1457
1458 for (virq = 0; virq < NR_VIRQS; virq++) {
1459 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1460 continue;
1461
1462 BUG_ON(virq_from_irq(irq) != virq);
1463
1464 /* Get a new binding from Xen. */
1465 bind_virq.virq = virq;
1466 bind_virq.vcpu = xen_vcpu_nr(cpu);
1467 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1468 &bind_virq) != 0)
1469 BUG();
1470 evtchn = bind_virq.port;
1471
1472 /* Record the new mapping. */
1473 (void)xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
1474 bind_evtchn_to_cpu(evtchn, cpu);
1475 }
1476}
1477
1478static void restore_cpu_ipis(unsigned int cpu)
1479{
1480 struct evtchn_bind_ipi bind_ipi;
1481 evtchn_port_t evtchn;
1482 int ipi, irq;
1483
1484 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1485 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1486 continue;
1487
1488 BUG_ON(ipi_from_irq(irq) != ipi);
1489
1490 /* Get a new binding from Xen. */
1491 bind_ipi.vcpu = xen_vcpu_nr(cpu);
1492 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1493 &bind_ipi) != 0)
1494 BUG();
1495 evtchn = bind_ipi.port;
1496
1497 /* Record the new mapping. */
1498 (void)xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
1499 bind_evtchn_to_cpu(evtchn, cpu);
1500 }
1501}
1502
1503/* Clear an irq's pending state, in preparation for polling on it */
1504void xen_clear_irq_pending(int irq)
1505{
1506 evtchn_port_t evtchn = evtchn_from_irq(irq);
1507
1508 if (VALID_EVTCHN(evtchn))
1509 clear_evtchn(evtchn);
1510}
1511EXPORT_SYMBOL(xen_clear_irq_pending);
1512void xen_set_irq_pending(int irq)
1513{
1514 evtchn_port_t evtchn = evtchn_from_irq(irq);
1515
1516 if (VALID_EVTCHN(evtchn))
1517 set_evtchn(evtchn);
1518}
1519
1520bool xen_test_irq_pending(int irq)
1521{
1522 evtchn_port_t evtchn = evtchn_from_irq(irq);
1523 bool ret = false;
1524
1525 if (VALID_EVTCHN(evtchn))
1526 ret = test_evtchn(evtchn);
1527
1528 return ret;
1529}
1530
1531/* Poll waiting for an irq to become pending with timeout. In the usual case,
1532 * the irq will be disabled so it won't deliver an interrupt. */
1533void xen_poll_irq_timeout(int irq, u64 timeout)
1534{
1535 evtchn_port_t evtchn = evtchn_from_irq(irq);
1536
1537 if (VALID_EVTCHN(evtchn)) {
1538 struct sched_poll poll;
1539
1540 poll.nr_ports = 1;
1541 poll.timeout = timeout;
1542 set_xen_guest_handle(poll.ports, &evtchn);
1543
1544 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
1545 BUG();
1546 }
1547}
1548EXPORT_SYMBOL(xen_poll_irq_timeout);
1549/* Poll waiting for an irq to become pending. In the usual case, the
1550 * irq will be disabled so it won't deliver an interrupt. */
1551void xen_poll_irq(int irq)
1552{
1553 xen_poll_irq_timeout(irq, 0 /* no timeout */);
1554}
1555
1556/* Check whether the IRQ line is shared with other guests. */
1557int xen_test_irq_shared(int irq)
1558{
1559 struct irq_info *info = info_for_irq(irq);
1560 struct physdev_irq_status_query irq_status;
1561
1562 if (WARN_ON(!info))
1563 return -ENOENT;
1564
1565 irq_status.irq = info->u.pirq.pirq;
1566
1567 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
1568 return 0;
1569 return !(irq_status.flags & XENIRQSTAT_shared);
1570}
1571EXPORT_SYMBOL_GPL(xen_test_irq_shared);
1572
1573void xen_irq_resume(void)
1574{
1575 unsigned int cpu;
1576 struct irq_info *info;
1577
1578 /* New event-channel space is not 'live' yet. */
1579 xen_evtchn_resume();
1580
1581 /* No IRQ <-> event-channel mappings. */
1582 list_for_each_entry(info, &xen_irq_list_head, list)
1583 info->evtchn = 0; /* zap event-channel binding */
1584
1585 clear_evtchn_to_irq_all();
1586
1587 for_each_possible_cpu(cpu) {
1588 restore_cpu_virqs(cpu);
1589 restore_cpu_ipis(cpu);
1590 }
1591
1592 restore_pirqs();
1593}
1594
1595static struct irq_chip xen_dynamic_chip __read_mostly = {
1596 .name = "xen-dyn",
1597
1598 .irq_disable = disable_dynirq,
1599 .irq_mask = disable_dynirq,
1600 .irq_unmask = enable_dynirq,
1601
1602 .irq_ack = ack_dynirq,
1603 .irq_mask_ack = mask_ack_dynirq,
1604
1605 .irq_set_affinity = set_affinity_irq,
1606 .irq_retrigger = retrigger_dynirq,
1607};
1608
1609static struct irq_chip xen_pirq_chip __read_mostly = {
1610 .name = "xen-pirq",
1611
1612 .irq_startup = startup_pirq,
1613 .irq_shutdown = shutdown_pirq,
1614 .irq_enable = enable_pirq,
1615 .irq_disable = disable_pirq,
1616
1617 .irq_mask = disable_dynirq,
1618 .irq_unmask = enable_dynirq,
1619
1620 .irq_ack = eoi_pirq,
1621 .irq_eoi = eoi_pirq,
1622 .irq_mask_ack = mask_ack_pirq,
1623
1624 .irq_set_affinity = set_affinity_irq,
1625
1626 .irq_retrigger = retrigger_dynirq,
1627};
1628
1629static struct irq_chip xen_percpu_chip __read_mostly = {
1630 .name = "xen-percpu",
1631
1632 .irq_disable = disable_dynirq,
1633 .irq_mask = disable_dynirq,
1634 .irq_unmask = enable_dynirq,
1635
1636 .irq_ack = ack_dynirq,
1637};
1638
1639int xen_set_callback_via(uint64_t via)
1640{
1641 struct xen_hvm_param a;
1642 a.domid = DOMID_SELF;
1643 a.index = HVM_PARAM_CALLBACK_IRQ;
1644 a.value = via;
1645 return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
1646}
1647EXPORT_SYMBOL_GPL(xen_set_callback_via);
1648
1649#ifdef CONFIG_XEN_PVHVM
1650/* Vector callbacks are better than PCI interrupts to receive event
1651 * channel notifications because we can receive vector callbacks on any
1652 * vcpu and we don't need PCI support or APIC interactions. */
1653void xen_setup_callback_vector(void)
1654{
1655 uint64_t callback_via;
1656
1657 if (xen_have_vector_callback) {
1658 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
1659 if (xen_set_callback_via(callback_via)) {
1660 pr_err("Request for Xen HVM callback vector failed\n");
1661 xen_have_vector_callback = 0;
1662 }
1663 }
1664}
1665
1666static __init void xen_alloc_callback_vector(void)
1667{
1668 if (!xen_have_vector_callback)
1669 return;
1670
1671 pr_info("Xen HVM callback vector for event delivery is enabled\n");
1672 alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback);
1673}
1674#else
1675void xen_setup_callback_vector(void) {}
1676static inline void xen_alloc_callback_vector(void) {}
1677#endif
1678
1679#undef MODULE_PARAM_PREFIX
1680#define MODULE_PARAM_PREFIX "xen."
1681
1682static bool fifo_events = true;
1683module_param(fifo_events, bool, 0);
1684
1685void __init xen_init_IRQ(void)
1686{
1687 int ret = -EINVAL;
1688 evtchn_port_t evtchn;
1689
1690 if (fifo_events)
1691 ret = xen_evtchn_fifo_init();
1692 if (ret < 0)
1693 xen_evtchn_2l_init();
1694
1695 evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()),
1696 sizeof(*evtchn_to_irq), GFP_KERNEL);
1697 BUG_ON(!evtchn_to_irq);
1698
1699 /* No event channels are 'live' right now. */
1700 for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++)
1701 mask_evtchn(evtchn);
1702
1703 pirq_needs_eoi = pirq_needs_eoi_flag;
1704
1705#ifdef CONFIG_X86
1706 if (xen_pv_domain()) {
1707 if (xen_initial_domain())
1708 pci_xen_initial_domain();
1709 }
1710 if (xen_feature(XENFEAT_hvm_callback_vector)) {
1711 xen_setup_callback_vector();
1712 xen_alloc_callback_vector();
1713 }
1714
1715 if (xen_hvm_domain()) {
1716 native_init_IRQ();
1717 /* pci_xen_hvm_init must be called after native_init_IRQ so that
1718 * __acpi_register_gsi can point at the right function */
1719 pci_xen_hvm_init();
1720 } else {
1721 int rc;
1722 struct physdev_pirq_eoi_gmfn eoi_gmfn;
1723
1724 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
1725 eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map);
1726 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
1727 if (rc != 0) {
1728 free_page((unsigned long) pirq_eoi_map);
1729 pirq_eoi_map = NULL;
1730 } else
1731 pirq_needs_eoi = pirq_check_eoi_map;
1732 }
1733#endif
1734}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Xen event channels
4 *
5 * Xen models interrupts with abstract event channels. Because each
6 * domain gets 1024 event channels, but NR_IRQ is not that large, we
7 * must dynamically map irqs<->event channels. The event channels
8 * interface with the rest of the kernel by defining a xen interrupt
9 * chip. When an event is received, it is mapped to an irq and sent
10 * through the normal interrupt processing path.
11 *
12 * There are four kinds of events which can be mapped to an event
13 * channel:
14 *
15 * 1. Inter-domain notifications. This includes all the virtual
16 * device events, since they're driven by front-ends in another domain
17 * (typically dom0).
18 * 2. VIRQs, typically used for timers. These are per-cpu events.
19 * 3. IPIs.
20 * 4. PIRQs - Hardware interrupts.
21 *
22 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
23 */
24
25#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
26
27#include <linux/linkage.h>
28#include <linux/interrupt.h>
29#include <linux/irq.h>
30#include <linux/moduleparam.h>
31#include <linux/string.h>
32#include <linux/memblock.h>
33#include <linux/slab.h>
34#include <linux/irqnr.h>
35#include <linux/pci.h>
36#include <linux/spinlock.h>
37#include <linux/cpuhotplug.h>
38#include <linux/atomic.h>
39#include <linux/ktime.h>
40
41#ifdef CONFIG_X86
42#include <asm/desc.h>
43#include <asm/ptrace.h>
44#include <asm/idtentry.h>
45#include <asm/irq.h>
46#include <asm/io_apic.h>
47#include <asm/i8259.h>
48#include <asm/xen/pci.h>
49#endif
50#include <asm/sync_bitops.h>
51#include <asm/xen/hypercall.h>
52#include <asm/xen/hypervisor.h>
53#include <xen/page.h>
54
55#include <xen/xen.h>
56#include <xen/hvm.h>
57#include <xen/xen-ops.h>
58#include <xen/events.h>
59#include <xen/interface/xen.h>
60#include <xen/interface/event_channel.h>
61#include <xen/interface/hvm/hvm_op.h>
62#include <xen/interface/hvm/params.h>
63#include <xen/interface/physdev.h>
64#include <xen/interface/sched.h>
65#include <xen/interface/vcpu.h>
66#include <xen/xenbus.h>
67#include <asm/hw_irq.h>
68
69#include "events_internal.h"
70
71#undef MODULE_PARAM_PREFIX
72#define MODULE_PARAM_PREFIX "xen."
73
74/* Interrupt types. */
75enum xen_irq_type {
76 IRQT_UNBOUND = 0,
77 IRQT_PIRQ,
78 IRQT_VIRQ,
79 IRQT_IPI,
80 IRQT_EVTCHN
81};
82
83/*
84 * Packed IRQ information:
85 * type - enum xen_irq_type
86 * event channel - irq->event channel mapping
87 * cpu - cpu this event channel is bound to
88 * index - type-specific information:
89 * PIRQ - vector, with MSB being "needs EIO", or physical IRQ of the HVM
90 * guest, or GSI (real passthrough IRQ) of the device.
91 * VIRQ - virq number
92 * IPI - IPI vector
93 * EVTCHN -
94 */
95struct irq_info {
96 struct list_head list;
97 struct list_head eoi_list;
98 short refcnt;
99 u8 spurious_cnt;
100 u8 is_accounted;
101 short type; /* type: IRQT_* */
102 u8 mask_reason; /* Why is event channel masked */
103#define EVT_MASK_REASON_EXPLICIT 0x01
104#define EVT_MASK_REASON_TEMPORARY 0x02
105#define EVT_MASK_REASON_EOI_PENDING 0x04
106 u8 is_active; /* Is event just being handled? */
107 unsigned irq;
108 evtchn_port_t evtchn; /* event channel */
109 unsigned short cpu; /* cpu bound */
110 unsigned short eoi_cpu; /* EOI must happen on this cpu-1 */
111 unsigned int irq_epoch; /* If eoi_cpu valid: irq_epoch of event */
112 u64 eoi_time; /* Time in jiffies when to EOI. */
113 raw_spinlock_t lock;
114
115 union {
116 unsigned short virq;
117 enum ipi_vector ipi;
118 struct {
119 unsigned short pirq;
120 unsigned short gsi;
121 unsigned char vector;
122 unsigned char flags;
123 uint16_t domid;
124 } pirq;
125 struct xenbus_device *interdomain;
126 } u;
127};
128
129#define PIRQ_NEEDS_EOI (1 << 0)
130#define PIRQ_SHAREABLE (1 << 1)
131#define PIRQ_MSI_GROUP (1 << 2)
132
133static uint __read_mostly event_loop_timeout = 2;
134module_param(event_loop_timeout, uint, 0644);
135
136static uint __read_mostly event_eoi_delay = 10;
137module_param(event_eoi_delay, uint, 0644);
138
139const struct evtchn_ops *evtchn_ops;
140
141/*
142 * This lock protects updates to the following mapping and reference-count
143 * arrays. The lock does not need to be acquired to read the mapping tables.
144 */
145static DEFINE_MUTEX(irq_mapping_update_lock);
146
147/*
148 * Lock protecting event handling loop against removing event channels.
149 * Adding of event channels is no issue as the associated IRQ becomes active
150 * only after everything is setup (before request_[threaded_]irq() the handler
151 * can't be entered for an event, as the event channel will be unmasked only
152 * then).
153 */
154static DEFINE_RWLOCK(evtchn_rwlock);
155
156/*
157 * Lock hierarchy:
158 *
159 * irq_mapping_update_lock
160 * evtchn_rwlock
161 * IRQ-desc lock
162 * percpu eoi_list_lock
163 * irq_info->lock
164 */
165
166static LIST_HEAD(xen_irq_list_head);
167
168/* IRQ <-> VIRQ mapping. */
169static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
170
171/* IRQ <-> IPI mapping */
172static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
173
174/* Event channel distribution data */
175static atomic_t channels_on_cpu[NR_CPUS];
176
177static int **evtchn_to_irq;
178#ifdef CONFIG_X86
179static unsigned long *pirq_eoi_map;
180#endif
181static bool (*pirq_needs_eoi)(unsigned irq);
182
183#define EVTCHN_ROW(e) (e / (PAGE_SIZE/sizeof(**evtchn_to_irq)))
184#define EVTCHN_COL(e) (e % (PAGE_SIZE/sizeof(**evtchn_to_irq)))
185#define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq))
186
187/* Xen will never allocate port zero for any purpose. */
188#define VALID_EVTCHN(chn) ((chn) != 0)
189
190static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY];
191
192static struct irq_chip xen_dynamic_chip;
193static struct irq_chip xen_lateeoi_chip;
194static struct irq_chip xen_percpu_chip;
195static struct irq_chip xen_pirq_chip;
196static void enable_dynirq(struct irq_data *data);
197static void disable_dynirq(struct irq_data *data);
198
199static DEFINE_PER_CPU(unsigned int, irq_epoch);
200
201static void clear_evtchn_to_irq_row(int *evtchn_row)
202{
203 unsigned col;
204
205 for (col = 0; col < EVTCHN_PER_ROW; col++)
206 WRITE_ONCE(evtchn_row[col], -1);
207}
208
209static void clear_evtchn_to_irq_all(void)
210{
211 unsigned row;
212
213 for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) {
214 if (evtchn_to_irq[row] == NULL)
215 continue;
216 clear_evtchn_to_irq_row(evtchn_to_irq[row]);
217 }
218}
219
220static int set_evtchn_to_irq(evtchn_port_t evtchn, unsigned int irq)
221{
222 unsigned row;
223 unsigned col;
224 int *evtchn_row;
225
226 if (evtchn >= xen_evtchn_max_channels())
227 return -EINVAL;
228
229 row = EVTCHN_ROW(evtchn);
230 col = EVTCHN_COL(evtchn);
231
232 if (evtchn_to_irq[row] == NULL) {
233 /* Unallocated irq entries return -1 anyway */
234 if (irq == -1)
235 return 0;
236
237 evtchn_row = (int *) __get_free_pages(GFP_KERNEL, 0);
238 if (evtchn_row == NULL)
239 return -ENOMEM;
240
241 clear_evtchn_to_irq_row(evtchn_row);
242
243 /*
244 * We've prepared an empty row for the mapping. If a different
245 * thread was faster inserting it, we can drop ours.
246 */
247 if (cmpxchg(&evtchn_to_irq[row], NULL, evtchn_row) != NULL)
248 free_page((unsigned long) evtchn_row);
249 }
250
251 WRITE_ONCE(evtchn_to_irq[row][col], irq);
252 return 0;
253}
254
255int get_evtchn_to_irq(evtchn_port_t evtchn)
256{
257 if (evtchn >= xen_evtchn_max_channels())
258 return -1;
259 if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL)
260 return -1;
261 return READ_ONCE(evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]);
262}
263
264/* Get info for IRQ */
265static struct irq_info *info_for_irq(unsigned irq)
266{
267 if (irq < nr_legacy_irqs())
268 return legacy_info_ptrs[irq];
269 else
270 return irq_get_chip_data(irq);
271}
272
273static void set_info_for_irq(unsigned int irq, struct irq_info *info)
274{
275 if (irq < nr_legacy_irqs())
276 legacy_info_ptrs[irq] = info;
277 else
278 irq_set_chip_data(irq, info);
279}
280
281/* Per CPU channel accounting */
282static void channels_on_cpu_dec(struct irq_info *info)
283{
284 if (!info->is_accounted)
285 return;
286
287 info->is_accounted = 0;
288
289 if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids))
290 return;
291
292 WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], -1 , 0));
293}
294
295static void channels_on_cpu_inc(struct irq_info *info)
296{
297 if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids))
298 return;
299
300 if (WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], 1,
301 INT_MAX)))
302 return;
303
304 info->is_accounted = 1;
305}
306
307/* Constructors for packed IRQ information. */
308static int xen_irq_info_common_setup(struct irq_info *info,
309 unsigned irq,
310 enum xen_irq_type type,
311 evtchn_port_t evtchn,
312 unsigned short cpu)
313{
314 int ret;
315
316 BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
317
318 info->type = type;
319 info->irq = irq;
320 info->evtchn = evtchn;
321 info->cpu = cpu;
322 info->mask_reason = EVT_MASK_REASON_EXPLICIT;
323 raw_spin_lock_init(&info->lock);
324
325 ret = set_evtchn_to_irq(evtchn, irq);
326 if (ret < 0)
327 return ret;
328
329 irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN);
330
331 return xen_evtchn_port_setup(evtchn);
332}
333
334static int xen_irq_info_evtchn_setup(unsigned irq,
335 evtchn_port_t evtchn,
336 struct xenbus_device *dev)
337{
338 struct irq_info *info = info_for_irq(irq);
339 int ret;
340
341 ret = xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0);
342 info->u.interdomain = dev;
343 if (dev)
344 atomic_inc(&dev->event_channels);
345
346 return ret;
347}
348
349static int xen_irq_info_ipi_setup(unsigned cpu,
350 unsigned irq,
351 evtchn_port_t evtchn,
352 enum ipi_vector ipi)
353{
354 struct irq_info *info = info_for_irq(irq);
355
356 info->u.ipi = ipi;
357
358 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
359
360 return xen_irq_info_common_setup(info, irq, IRQT_IPI, evtchn, 0);
361}
362
363static int xen_irq_info_virq_setup(unsigned cpu,
364 unsigned irq,
365 evtchn_port_t evtchn,
366 unsigned virq)
367{
368 struct irq_info *info = info_for_irq(irq);
369
370 info->u.virq = virq;
371
372 per_cpu(virq_to_irq, cpu)[virq] = irq;
373
374 return xen_irq_info_common_setup(info, irq, IRQT_VIRQ, evtchn, 0);
375}
376
377static int xen_irq_info_pirq_setup(unsigned irq,
378 evtchn_port_t evtchn,
379 unsigned pirq,
380 unsigned gsi,
381 uint16_t domid,
382 unsigned char flags)
383{
384 struct irq_info *info = info_for_irq(irq);
385
386 info->u.pirq.pirq = pirq;
387 info->u.pirq.gsi = gsi;
388 info->u.pirq.domid = domid;
389 info->u.pirq.flags = flags;
390
391 return xen_irq_info_common_setup(info, irq, IRQT_PIRQ, evtchn, 0);
392}
393
394static void xen_irq_info_cleanup(struct irq_info *info)
395{
396 set_evtchn_to_irq(info->evtchn, -1);
397 xen_evtchn_port_remove(info->evtchn, info->cpu);
398 info->evtchn = 0;
399 channels_on_cpu_dec(info);
400}
401
402/*
403 * Accessors for packed IRQ information.
404 */
405evtchn_port_t evtchn_from_irq(unsigned irq)
406{
407 const struct irq_info *info = NULL;
408
409 if (likely(irq < nr_irqs))
410 info = info_for_irq(irq);
411 if (!info)
412 return 0;
413
414 return info->evtchn;
415}
416
417unsigned int irq_from_evtchn(evtchn_port_t evtchn)
418{
419 return get_evtchn_to_irq(evtchn);
420}
421EXPORT_SYMBOL_GPL(irq_from_evtchn);
422
423int irq_from_virq(unsigned int cpu, unsigned int virq)
424{
425 return per_cpu(virq_to_irq, cpu)[virq];
426}
427
428static enum ipi_vector ipi_from_irq(unsigned irq)
429{
430 struct irq_info *info = info_for_irq(irq);
431
432 BUG_ON(info == NULL);
433 BUG_ON(info->type != IRQT_IPI);
434
435 return info->u.ipi;
436}
437
438static unsigned virq_from_irq(unsigned irq)
439{
440 struct irq_info *info = info_for_irq(irq);
441
442 BUG_ON(info == NULL);
443 BUG_ON(info->type != IRQT_VIRQ);
444
445 return info->u.virq;
446}
447
448static unsigned pirq_from_irq(unsigned irq)
449{
450 struct irq_info *info = info_for_irq(irq);
451
452 BUG_ON(info == NULL);
453 BUG_ON(info->type != IRQT_PIRQ);
454
455 return info->u.pirq.pirq;
456}
457
458static enum xen_irq_type type_from_irq(unsigned irq)
459{
460 return info_for_irq(irq)->type;
461}
462
463static unsigned cpu_from_irq(unsigned irq)
464{
465 return info_for_irq(irq)->cpu;
466}
467
468unsigned int cpu_from_evtchn(evtchn_port_t evtchn)
469{
470 int irq = get_evtchn_to_irq(evtchn);
471 unsigned ret = 0;
472
473 if (irq != -1)
474 ret = cpu_from_irq(irq);
475
476 return ret;
477}
478
479static void do_mask(struct irq_info *info, u8 reason)
480{
481 unsigned long flags;
482
483 raw_spin_lock_irqsave(&info->lock, flags);
484
485 if (!info->mask_reason)
486 mask_evtchn(info->evtchn);
487
488 info->mask_reason |= reason;
489
490 raw_spin_unlock_irqrestore(&info->lock, flags);
491}
492
493static void do_unmask(struct irq_info *info, u8 reason)
494{
495 unsigned long flags;
496
497 raw_spin_lock_irqsave(&info->lock, flags);
498
499 info->mask_reason &= ~reason;
500
501 if (!info->mask_reason)
502 unmask_evtchn(info->evtchn);
503
504 raw_spin_unlock_irqrestore(&info->lock, flags);
505}
506
507#ifdef CONFIG_X86
508static bool pirq_check_eoi_map(unsigned irq)
509{
510 return test_bit(pirq_from_irq(irq), pirq_eoi_map);
511}
512#endif
513
514static bool pirq_needs_eoi_flag(unsigned irq)
515{
516 struct irq_info *info = info_for_irq(irq);
517 BUG_ON(info->type != IRQT_PIRQ);
518
519 return info->u.pirq.flags & PIRQ_NEEDS_EOI;
520}
521
522static void bind_evtchn_to_cpu(evtchn_port_t evtchn, unsigned int cpu,
523 bool force_affinity)
524{
525 int irq = get_evtchn_to_irq(evtchn);
526 struct irq_info *info = info_for_irq(irq);
527
528 BUG_ON(irq == -1);
529
530 if (IS_ENABLED(CONFIG_SMP) && force_affinity) {
531 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(cpu));
532 cpumask_copy(irq_get_effective_affinity_mask(irq),
533 cpumask_of(cpu));
534 }
535
536 xen_evtchn_port_bind_to_cpu(evtchn, cpu, info->cpu);
537
538 channels_on_cpu_dec(info);
539 info->cpu = cpu;
540 channels_on_cpu_inc(info);
541}
542
543/**
544 * notify_remote_via_irq - send event to remote end of event channel via irq
545 * @irq: irq of event channel to send event to
546 *
547 * Unlike notify_remote_via_evtchn(), this is safe to use across
548 * save/restore. Notifications on a broken connection are silently
549 * dropped.
550 */
551void notify_remote_via_irq(int irq)
552{
553 evtchn_port_t evtchn = evtchn_from_irq(irq);
554
555 if (VALID_EVTCHN(evtchn))
556 notify_remote_via_evtchn(evtchn);
557}
558EXPORT_SYMBOL_GPL(notify_remote_via_irq);
559
560struct lateeoi_work {
561 struct delayed_work delayed;
562 spinlock_t eoi_list_lock;
563 struct list_head eoi_list;
564};
565
566static DEFINE_PER_CPU(struct lateeoi_work, lateeoi);
567
568static void lateeoi_list_del(struct irq_info *info)
569{
570 struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
571 unsigned long flags;
572
573 spin_lock_irqsave(&eoi->eoi_list_lock, flags);
574 list_del_init(&info->eoi_list);
575 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
576}
577
578static void lateeoi_list_add(struct irq_info *info)
579{
580 struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
581 struct irq_info *elem;
582 u64 now = get_jiffies_64();
583 unsigned long delay;
584 unsigned long flags;
585
586 if (now < info->eoi_time)
587 delay = info->eoi_time - now;
588 else
589 delay = 1;
590
591 spin_lock_irqsave(&eoi->eoi_list_lock, flags);
592
593 if (list_empty(&eoi->eoi_list)) {
594 list_add(&info->eoi_list, &eoi->eoi_list);
595 mod_delayed_work_on(info->eoi_cpu, system_wq,
596 &eoi->delayed, delay);
597 } else {
598 list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) {
599 if (elem->eoi_time <= info->eoi_time)
600 break;
601 }
602 list_add(&info->eoi_list, &elem->eoi_list);
603 }
604
605 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
606}
607
608static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious)
609{
610 evtchn_port_t evtchn;
611 unsigned int cpu;
612 unsigned int delay = 0;
613
614 evtchn = info->evtchn;
615 if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list))
616 return;
617
618 if (spurious) {
619 struct xenbus_device *dev = info->u.interdomain;
620 unsigned int threshold = 1;
621
622 if (dev && dev->spurious_threshold)
623 threshold = dev->spurious_threshold;
624
625 if ((1 << info->spurious_cnt) < (HZ << 2)) {
626 if (info->spurious_cnt != 0xFF)
627 info->spurious_cnt++;
628 }
629 if (info->spurious_cnt > threshold) {
630 delay = 1 << (info->spurious_cnt - 1 - threshold);
631 if (delay > HZ)
632 delay = HZ;
633 if (!info->eoi_time)
634 info->eoi_cpu = smp_processor_id();
635 info->eoi_time = get_jiffies_64() + delay;
636 if (dev)
637 atomic_add(delay, &dev->jiffies_eoi_delayed);
638 }
639 if (dev)
640 atomic_inc(&dev->spurious_events);
641 } else {
642 info->spurious_cnt = 0;
643 }
644
645 cpu = info->eoi_cpu;
646 if (info->eoi_time &&
647 (info->irq_epoch == per_cpu(irq_epoch, cpu) || delay)) {
648 lateeoi_list_add(info);
649 return;
650 }
651
652 info->eoi_time = 0;
653
654 /* is_active hasn't been reset yet, do it now. */
655 smp_store_release(&info->is_active, 0);
656 do_unmask(info, EVT_MASK_REASON_EOI_PENDING);
657}
658
659static void xen_irq_lateeoi_worker(struct work_struct *work)
660{
661 struct lateeoi_work *eoi;
662 struct irq_info *info;
663 u64 now = get_jiffies_64();
664 unsigned long flags;
665
666 eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed);
667
668 read_lock_irqsave(&evtchn_rwlock, flags);
669
670 while (true) {
671 spin_lock(&eoi->eoi_list_lock);
672
673 info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info,
674 eoi_list);
675
676 if (info == NULL || now < info->eoi_time) {
677 spin_unlock(&eoi->eoi_list_lock);
678 break;
679 }
680
681 list_del_init(&info->eoi_list);
682
683 spin_unlock(&eoi->eoi_list_lock);
684
685 info->eoi_time = 0;
686
687 xen_irq_lateeoi_locked(info, false);
688 }
689
690 if (info)
691 mod_delayed_work_on(info->eoi_cpu, system_wq,
692 &eoi->delayed, info->eoi_time - now);
693
694 read_unlock_irqrestore(&evtchn_rwlock, flags);
695}
696
697static void xen_cpu_init_eoi(unsigned int cpu)
698{
699 struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu);
700
701 INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker);
702 spin_lock_init(&eoi->eoi_list_lock);
703 INIT_LIST_HEAD(&eoi->eoi_list);
704}
705
706void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags)
707{
708 struct irq_info *info;
709 unsigned long flags;
710
711 read_lock_irqsave(&evtchn_rwlock, flags);
712
713 info = info_for_irq(irq);
714
715 if (info)
716 xen_irq_lateeoi_locked(info, eoi_flags & XEN_EOI_FLAG_SPURIOUS);
717
718 read_unlock_irqrestore(&evtchn_rwlock, flags);
719}
720EXPORT_SYMBOL_GPL(xen_irq_lateeoi);
721
722static void xen_irq_init(unsigned irq)
723{
724 struct irq_info *info;
725
726 info = kzalloc(sizeof(*info), GFP_KERNEL);
727 if (info == NULL)
728 panic("Unable to allocate metadata for IRQ%d\n", irq);
729
730 info->type = IRQT_UNBOUND;
731 info->refcnt = -1;
732
733 set_info_for_irq(irq, info);
734 /*
735 * Interrupt affinity setting can be immediate. No point
736 * in delaying it until an interrupt is handled.
737 */
738 irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
739
740 INIT_LIST_HEAD(&info->eoi_list);
741 list_add_tail(&info->list, &xen_irq_list_head);
742}
743
744static int __must_check xen_allocate_irqs_dynamic(int nvec)
745{
746 int i, irq = irq_alloc_descs(-1, 0, nvec, -1);
747
748 if (irq >= 0) {
749 for (i = 0; i < nvec; i++)
750 xen_irq_init(irq + i);
751 }
752
753 return irq;
754}
755
756static inline int __must_check xen_allocate_irq_dynamic(void)
757{
758
759 return xen_allocate_irqs_dynamic(1);
760}
761
762static int __must_check xen_allocate_irq_gsi(unsigned gsi)
763{
764 int irq;
765
766 /*
767 * A PV guest has no concept of a GSI (since it has no ACPI
768 * nor access to/knowledge of the physical APICs). Therefore
769 * all IRQs are dynamically allocated from the entire IRQ
770 * space.
771 */
772 if (xen_pv_domain() && !xen_initial_domain())
773 return xen_allocate_irq_dynamic();
774
775 /* Legacy IRQ descriptors are already allocated by the arch. */
776 if (gsi < nr_legacy_irqs())
777 irq = gsi;
778 else
779 irq = irq_alloc_desc_at(gsi, -1);
780
781 xen_irq_init(irq);
782
783 return irq;
784}
785
786static void xen_free_irq(unsigned irq)
787{
788 struct irq_info *info = info_for_irq(irq);
789 unsigned long flags;
790
791 if (WARN_ON(!info))
792 return;
793
794 write_lock_irqsave(&evtchn_rwlock, flags);
795
796 if (!list_empty(&info->eoi_list))
797 lateeoi_list_del(info);
798
799 list_del(&info->list);
800
801 set_info_for_irq(irq, NULL);
802
803 WARN_ON(info->refcnt > 0);
804
805 write_unlock_irqrestore(&evtchn_rwlock, flags);
806
807 kfree(info);
808
809 /* Legacy IRQ descriptors are managed by the arch. */
810 if (irq < nr_legacy_irqs())
811 return;
812
813 irq_free_desc(irq);
814}
815
816static void xen_evtchn_close(evtchn_port_t port)
817{
818 struct evtchn_close close;
819
820 close.port = port;
821 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
822 BUG();
823}
824
825/* Not called for lateeoi events. */
826static void event_handler_exit(struct irq_info *info)
827{
828 smp_store_release(&info->is_active, 0);
829 clear_evtchn(info->evtchn);
830}
831
832static void pirq_query_unmask(int irq)
833{
834 struct physdev_irq_status_query irq_status;
835 struct irq_info *info = info_for_irq(irq);
836
837 BUG_ON(info->type != IRQT_PIRQ);
838
839 irq_status.irq = pirq_from_irq(irq);
840 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
841 irq_status.flags = 0;
842
843 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
844 if (irq_status.flags & XENIRQSTAT_needs_eoi)
845 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
846}
847
848static void eoi_pirq(struct irq_data *data)
849{
850 struct irq_info *info = info_for_irq(data->irq);
851 evtchn_port_t evtchn = info ? info->evtchn : 0;
852 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
853 int rc = 0;
854
855 if (!VALID_EVTCHN(evtchn))
856 return;
857
858 event_handler_exit(info);
859
860 if (pirq_needs_eoi(data->irq)) {
861 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
862 WARN_ON(rc);
863 }
864}
865
866static void mask_ack_pirq(struct irq_data *data)
867{
868 disable_dynirq(data);
869 eoi_pirq(data);
870}
871
872static unsigned int __startup_pirq(unsigned int irq)
873{
874 struct evtchn_bind_pirq bind_pirq;
875 struct irq_info *info = info_for_irq(irq);
876 evtchn_port_t evtchn = evtchn_from_irq(irq);
877 int rc;
878
879 BUG_ON(info->type != IRQT_PIRQ);
880
881 if (VALID_EVTCHN(evtchn))
882 goto out;
883
884 bind_pirq.pirq = pirq_from_irq(irq);
885 /* NB. We are happy to share unless we are probing. */
886 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
887 BIND_PIRQ__WILL_SHARE : 0;
888 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
889 if (rc != 0) {
890 pr_warn("Failed to obtain physical IRQ %d\n", irq);
891 return 0;
892 }
893 evtchn = bind_pirq.port;
894
895 pirq_query_unmask(irq);
896
897 rc = set_evtchn_to_irq(evtchn, irq);
898 if (rc)
899 goto err;
900
901 info->evtchn = evtchn;
902 bind_evtchn_to_cpu(evtchn, 0, false);
903
904 rc = xen_evtchn_port_setup(evtchn);
905 if (rc)
906 goto err;
907
908out:
909 do_unmask(info, EVT_MASK_REASON_EXPLICIT);
910
911 eoi_pirq(irq_get_irq_data(irq));
912
913 return 0;
914
915err:
916 pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc);
917 xen_evtchn_close(evtchn);
918 return 0;
919}
920
921static unsigned int startup_pirq(struct irq_data *data)
922{
923 return __startup_pirq(data->irq);
924}
925
926static void shutdown_pirq(struct irq_data *data)
927{
928 unsigned int irq = data->irq;
929 struct irq_info *info = info_for_irq(irq);
930 evtchn_port_t evtchn = evtchn_from_irq(irq);
931
932 BUG_ON(info->type != IRQT_PIRQ);
933
934 if (!VALID_EVTCHN(evtchn))
935 return;
936
937 do_mask(info, EVT_MASK_REASON_EXPLICIT);
938 xen_evtchn_close(evtchn);
939 xen_irq_info_cleanup(info);
940}
941
942static void enable_pirq(struct irq_data *data)
943{
944 enable_dynirq(data);
945}
946
947static void disable_pirq(struct irq_data *data)
948{
949 disable_dynirq(data);
950}
951
952int xen_irq_from_gsi(unsigned gsi)
953{
954 struct irq_info *info;
955
956 list_for_each_entry(info, &xen_irq_list_head, list) {
957 if (info->type != IRQT_PIRQ)
958 continue;
959
960 if (info->u.pirq.gsi == gsi)
961 return info->irq;
962 }
963
964 return -1;
965}
966EXPORT_SYMBOL_GPL(xen_irq_from_gsi);
967
968static void __unbind_from_irq(unsigned int irq)
969{
970 evtchn_port_t evtchn = evtchn_from_irq(irq);
971 struct irq_info *info = info_for_irq(irq);
972
973 if (info->refcnt > 0) {
974 info->refcnt--;
975 if (info->refcnt != 0)
976 return;
977 }
978
979 if (VALID_EVTCHN(evtchn)) {
980 unsigned int cpu = cpu_from_irq(irq);
981 struct xenbus_device *dev;
982
983 xen_evtchn_close(evtchn);
984
985 switch (type_from_irq(irq)) {
986 case IRQT_VIRQ:
987 per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1;
988 break;
989 case IRQT_IPI:
990 per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1;
991 break;
992 case IRQT_EVTCHN:
993 dev = info->u.interdomain;
994 if (dev)
995 atomic_dec(&dev->event_channels);
996 break;
997 default:
998 break;
999 }
1000
1001 xen_irq_info_cleanup(info);
1002 }
1003
1004 xen_free_irq(irq);
1005}
1006
1007/*
1008 * Do not make any assumptions regarding the relationship between the
1009 * IRQ number returned here and the Xen pirq argument.
1010 *
1011 * Note: We don't assign an event channel until the irq actually started
1012 * up. Return an existing irq if we've already got one for the gsi.
1013 *
1014 * Shareable implies level triggered, not shareable implies edge
1015 * triggered here.
1016 */
1017int xen_bind_pirq_gsi_to_irq(unsigned gsi,
1018 unsigned pirq, int shareable, char *name)
1019{
1020 int irq;
1021 struct physdev_irq irq_op;
1022 int ret;
1023
1024 mutex_lock(&irq_mapping_update_lock);
1025
1026 irq = xen_irq_from_gsi(gsi);
1027 if (irq != -1) {
1028 pr_info("%s: returning irq %d for gsi %u\n",
1029 __func__, irq, gsi);
1030 goto out;
1031 }
1032
1033 irq = xen_allocate_irq_gsi(gsi);
1034 if (irq < 0)
1035 goto out;
1036
1037 irq_op.irq = irq;
1038 irq_op.vector = 0;
1039
1040 /* Only the privileged domain can do this. For non-priv, the pcifront
1041 * driver provides a PCI bus that does the call to do exactly
1042 * this in the priv domain. */
1043 if (xen_initial_domain() &&
1044 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
1045 xen_free_irq(irq);
1046 irq = -ENOSPC;
1047 goto out;
1048 }
1049
1050 ret = xen_irq_info_pirq_setup(irq, 0, pirq, gsi, DOMID_SELF,
1051 shareable ? PIRQ_SHAREABLE : 0);
1052 if (ret < 0) {
1053 __unbind_from_irq(irq);
1054 irq = ret;
1055 goto out;
1056 }
1057
1058 pirq_query_unmask(irq);
1059 /* We try to use the handler with the appropriate semantic for the
1060 * type of interrupt: if the interrupt is an edge triggered
1061 * interrupt we use handle_edge_irq.
1062 *
1063 * On the other hand if the interrupt is level triggered we use
1064 * handle_fasteoi_irq like the native code does for this kind of
1065 * interrupts.
1066 *
1067 * Depending on the Xen version, pirq_needs_eoi might return true
1068 * not only for level triggered interrupts but for edge triggered
1069 * interrupts too. In any case Xen always honors the eoi mechanism,
1070 * not injecting any more pirqs of the same kind if the first one
1071 * hasn't received an eoi yet. Therefore using the fasteoi handler
1072 * is the right choice either way.
1073 */
1074 if (shareable)
1075 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
1076 handle_fasteoi_irq, name);
1077 else
1078 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
1079 handle_edge_irq, name);
1080
1081out:
1082 mutex_unlock(&irq_mapping_update_lock);
1083
1084 return irq;
1085}
1086
1087#ifdef CONFIG_PCI_MSI
1088int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
1089{
1090 int rc;
1091 struct physdev_get_free_pirq op_get_free_pirq;
1092
1093 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
1094 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
1095
1096 WARN_ONCE(rc == -ENOSYS,
1097 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
1098
1099 return rc ? -1 : op_get_free_pirq.pirq;
1100}
1101
1102int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
1103 int pirq, int nvec, const char *name, domid_t domid)
1104{
1105 int i, irq, ret;
1106
1107 mutex_lock(&irq_mapping_update_lock);
1108
1109 irq = xen_allocate_irqs_dynamic(nvec);
1110 if (irq < 0)
1111 goto out;
1112
1113 for (i = 0; i < nvec; i++) {
1114 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name);
1115
1116 ret = xen_irq_info_pirq_setup(irq + i, 0, pirq + i, 0, domid,
1117 i == 0 ? 0 : PIRQ_MSI_GROUP);
1118 if (ret < 0)
1119 goto error_irq;
1120 }
1121
1122 ret = irq_set_msi_desc(irq, msidesc);
1123 if (ret < 0)
1124 goto error_irq;
1125out:
1126 mutex_unlock(&irq_mapping_update_lock);
1127 return irq;
1128error_irq:
1129 while (nvec--)
1130 __unbind_from_irq(irq + nvec);
1131 mutex_unlock(&irq_mapping_update_lock);
1132 return ret;
1133}
1134#endif
1135
1136int xen_destroy_irq(int irq)
1137{
1138 struct physdev_unmap_pirq unmap_irq;
1139 struct irq_info *info = info_for_irq(irq);
1140 int rc = -ENOENT;
1141
1142 mutex_lock(&irq_mapping_update_lock);
1143
1144 /*
1145 * If trying to remove a vector in a MSI group different
1146 * than the first one skip the PIRQ unmap unless this vector
1147 * is the first one in the group.
1148 */
1149 if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) {
1150 unmap_irq.pirq = info->u.pirq.pirq;
1151 unmap_irq.domid = info->u.pirq.domid;
1152 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
1153 /* If another domain quits without making the pci_disable_msix
1154 * call, the Xen hypervisor takes care of freeing the PIRQs
1155 * (free_domain_pirqs).
1156 */
1157 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
1158 pr_info("domain %d does not have %d anymore\n",
1159 info->u.pirq.domid, info->u.pirq.pirq);
1160 else if (rc) {
1161 pr_warn("unmap irq failed %d\n", rc);
1162 goto out;
1163 }
1164 }
1165
1166 xen_free_irq(irq);
1167
1168out:
1169 mutex_unlock(&irq_mapping_update_lock);
1170 return rc;
1171}
1172
1173int xen_irq_from_pirq(unsigned pirq)
1174{
1175 int irq;
1176
1177 struct irq_info *info;
1178
1179 mutex_lock(&irq_mapping_update_lock);
1180
1181 list_for_each_entry(info, &xen_irq_list_head, list) {
1182 if (info->type != IRQT_PIRQ)
1183 continue;
1184 irq = info->irq;
1185 if (info->u.pirq.pirq == pirq)
1186 goto out;
1187 }
1188 irq = -1;
1189out:
1190 mutex_unlock(&irq_mapping_update_lock);
1191
1192 return irq;
1193}
1194
1195
1196int xen_pirq_from_irq(unsigned irq)
1197{
1198 return pirq_from_irq(irq);
1199}
1200EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
1201
1202static int bind_evtchn_to_irq_chip(evtchn_port_t evtchn, struct irq_chip *chip,
1203 struct xenbus_device *dev)
1204{
1205 int irq;
1206 int ret;
1207
1208 if (evtchn >= xen_evtchn_max_channels())
1209 return -ENOMEM;
1210
1211 mutex_lock(&irq_mapping_update_lock);
1212
1213 irq = get_evtchn_to_irq(evtchn);
1214
1215 if (irq == -1) {
1216 irq = xen_allocate_irq_dynamic();
1217 if (irq < 0)
1218 goto out;
1219
1220 irq_set_chip_and_handler_name(irq, chip,
1221 handle_edge_irq, "event");
1222
1223 ret = xen_irq_info_evtchn_setup(irq, evtchn, dev);
1224 if (ret < 0) {
1225 __unbind_from_irq(irq);
1226 irq = ret;
1227 goto out;
1228 }
1229 /*
1230 * New interdomain events are initially bound to vCPU0 This
1231 * is required to setup the event channel in the first
1232 * place and also important for UP guests because the
1233 * affinity setting is not invoked on them so nothing would
1234 * bind the channel.
1235 */
1236 bind_evtchn_to_cpu(evtchn, 0, false);
1237 } else {
1238 struct irq_info *info = info_for_irq(irq);
1239 WARN_ON(info == NULL || info->type != IRQT_EVTCHN);
1240 }
1241
1242out:
1243 mutex_unlock(&irq_mapping_update_lock);
1244
1245 return irq;
1246}
1247
1248int bind_evtchn_to_irq(evtchn_port_t evtchn)
1249{
1250 return bind_evtchn_to_irq_chip(evtchn, &xen_dynamic_chip, NULL);
1251}
1252EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
1253
1254static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
1255{
1256 struct evtchn_bind_ipi bind_ipi;
1257 evtchn_port_t evtchn;
1258 int ret, irq;
1259
1260 mutex_lock(&irq_mapping_update_lock);
1261
1262 irq = per_cpu(ipi_to_irq, cpu)[ipi];
1263
1264 if (irq == -1) {
1265 irq = xen_allocate_irq_dynamic();
1266 if (irq < 0)
1267 goto out;
1268
1269 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
1270 handle_percpu_irq, "ipi");
1271
1272 bind_ipi.vcpu = xen_vcpu_nr(cpu);
1273 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1274 &bind_ipi) != 0)
1275 BUG();
1276 evtchn = bind_ipi.port;
1277
1278 ret = xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
1279 if (ret < 0) {
1280 __unbind_from_irq(irq);
1281 irq = ret;
1282 goto out;
1283 }
1284 /*
1285 * Force the affinity mask to the target CPU so proc shows
1286 * the correct target.
1287 */
1288 bind_evtchn_to_cpu(evtchn, cpu, true);
1289 } else {
1290 struct irq_info *info = info_for_irq(irq);
1291 WARN_ON(info == NULL || info->type != IRQT_IPI);
1292 }
1293
1294 out:
1295 mutex_unlock(&irq_mapping_update_lock);
1296 return irq;
1297}
1298
1299static int bind_interdomain_evtchn_to_irq_chip(struct xenbus_device *dev,
1300 evtchn_port_t remote_port,
1301 struct irq_chip *chip)
1302{
1303 struct evtchn_bind_interdomain bind_interdomain;
1304 int err;
1305
1306 bind_interdomain.remote_dom = dev->otherend_id;
1307 bind_interdomain.remote_port = remote_port;
1308
1309 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1310 &bind_interdomain);
1311
1312 return err ? : bind_evtchn_to_irq_chip(bind_interdomain.local_port,
1313 chip, dev);
1314}
1315
1316int bind_interdomain_evtchn_to_irq_lateeoi(struct xenbus_device *dev,
1317 evtchn_port_t remote_port)
1318{
1319 return bind_interdomain_evtchn_to_irq_chip(dev, remote_port,
1320 &xen_lateeoi_chip);
1321}
1322EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq_lateeoi);
1323
1324static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn)
1325{
1326 struct evtchn_status status;
1327 evtchn_port_t port;
1328 int rc = -ENOENT;
1329
1330 memset(&status, 0, sizeof(status));
1331 for (port = 0; port < xen_evtchn_max_channels(); port++) {
1332 status.dom = DOMID_SELF;
1333 status.port = port;
1334 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
1335 if (rc < 0)
1336 continue;
1337 if (status.status != EVTCHNSTAT_virq)
1338 continue;
1339 if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) {
1340 *evtchn = port;
1341 break;
1342 }
1343 }
1344 return rc;
1345}
1346
1347/**
1348 * xen_evtchn_nr_channels - number of usable event channel ports
1349 *
1350 * This may be less than the maximum supported by the current
1351 * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum
1352 * supported.
1353 */
1354unsigned xen_evtchn_nr_channels(void)
1355{
1356 return evtchn_ops->nr_channels();
1357}
1358EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels);
1359
1360int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu)
1361{
1362 struct evtchn_bind_virq bind_virq;
1363 evtchn_port_t evtchn = 0;
1364 int irq, ret;
1365
1366 mutex_lock(&irq_mapping_update_lock);
1367
1368 irq = per_cpu(virq_to_irq, cpu)[virq];
1369
1370 if (irq == -1) {
1371 irq = xen_allocate_irq_dynamic();
1372 if (irq < 0)
1373 goto out;
1374
1375 if (percpu)
1376 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
1377 handle_percpu_irq, "virq");
1378 else
1379 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
1380 handle_edge_irq, "virq");
1381
1382 bind_virq.virq = virq;
1383 bind_virq.vcpu = xen_vcpu_nr(cpu);
1384 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1385 &bind_virq);
1386 if (ret == 0)
1387 evtchn = bind_virq.port;
1388 else {
1389 if (ret == -EEXIST)
1390 ret = find_virq(virq, cpu, &evtchn);
1391 BUG_ON(ret < 0);
1392 }
1393
1394 ret = xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
1395 if (ret < 0) {
1396 __unbind_from_irq(irq);
1397 irq = ret;
1398 goto out;
1399 }
1400
1401 /*
1402 * Force the affinity mask for percpu interrupts so proc
1403 * shows the correct target.
1404 */
1405 bind_evtchn_to_cpu(evtchn, cpu, percpu);
1406 } else {
1407 struct irq_info *info = info_for_irq(irq);
1408 WARN_ON(info == NULL || info->type != IRQT_VIRQ);
1409 }
1410
1411out:
1412 mutex_unlock(&irq_mapping_update_lock);
1413
1414 return irq;
1415}
1416
1417static void unbind_from_irq(unsigned int irq)
1418{
1419 mutex_lock(&irq_mapping_update_lock);
1420 __unbind_from_irq(irq);
1421 mutex_unlock(&irq_mapping_update_lock);
1422}
1423
1424static int bind_evtchn_to_irqhandler_chip(evtchn_port_t evtchn,
1425 irq_handler_t handler,
1426 unsigned long irqflags,
1427 const char *devname, void *dev_id,
1428 struct irq_chip *chip)
1429{
1430 int irq, retval;
1431
1432 irq = bind_evtchn_to_irq_chip(evtchn, chip, NULL);
1433 if (irq < 0)
1434 return irq;
1435 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1436 if (retval != 0) {
1437 unbind_from_irq(irq);
1438 return retval;
1439 }
1440
1441 return irq;
1442}
1443
1444int bind_evtchn_to_irqhandler(evtchn_port_t evtchn,
1445 irq_handler_t handler,
1446 unsigned long irqflags,
1447 const char *devname, void *dev_id)
1448{
1449 return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags,
1450 devname, dev_id,
1451 &xen_dynamic_chip);
1452}
1453EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1454
1455int bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn,
1456 irq_handler_t handler,
1457 unsigned long irqflags,
1458 const char *devname, void *dev_id)
1459{
1460 return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags,
1461 devname, dev_id,
1462 &xen_lateeoi_chip);
1463}
1464EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler_lateeoi);
1465
1466static int bind_interdomain_evtchn_to_irqhandler_chip(
1467 struct xenbus_device *dev, evtchn_port_t remote_port,
1468 irq_handler_t handler, unsigned long irqflags,
1469 const char *devname, void *dev_id, struct irq_chip *chip)
1470{
1471 int irq, retval;
1472
1473 irq = bind_interdomain_evtchn_to_irq_chip(dev, remote_port, chip);
1474 if (irq < 0)
1475 return irq;
1476
1477 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1478 if (retval != 0) {
1479 unbind_from_irq(irq);
1480 return retval;
1481 }
1482
1483 return irq;
1484}
1485
1486int bind_interdomain_evtchn_to_irqhandler_lateeoi(struct xenbus_device *dev,
1487 evtchn_port_t remote_port,
1488 irq_handler_t handler,
1489 unsigned long irqflags,
1490 const char *devname,
1491 void *dev_id)
1492{
1493 return bind_interdomain_evtchn_to_irqhandler_chip(dev,
1494 remote_port, handler, irqflags, devname,
1495 dev_id, &xen_lateeoi_chip);
1496}
1497EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler_lateeoi);
1498
1499int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
1500 irq_handler_t handler,
1501 unsigned long irqflags, const char *devname, void *dev_id)
1502{
1503 int irq, retval;
1504
1505 irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU);
1506 if (irq < 0)
1507 return irq;
1508 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1509 if (retval != 0) {
1510 unbind_from_irq(irq);
1511 return retval;
1512 }
1513
1514 return irq;
1515}
1516EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1517
1518int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1519 unsigned int cpu,
1520 irq_handler_t handler,
1521 unsigned long irqflags,
1522 const char *devname,
1523 void *dev_id)
1524{
1525 int irq, retval;
1526
1527 irq = bind_ipi_to_irq(ipi, cpu);
1528 if (irq < 0)
1529 return irq;
1530
1531 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
1532 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1533 if (retval != 0) {
1534 unbind_from_irq(irq);
1535 return retval;
1536 }
1537
1538 return irq;
1539}
1540
1541void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1542{
1543 struct irq_info *info = info_for_irq(irq);
1544
1545 if (WARN_ON(!info))
1546 return;
1547 free_irq(irq, dev_id);
1548 unbind_from_irq(irq);
1549}
1550EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1551
1552/**
1553 * xen_set_irq_priority() - set an event channel priority.
1554 * @irq:irq bound to an event channel.
1555 * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN.
1556 */
1557int xen_set_irq_priority(unsigned irq, unsigned priority)
1558{
1559 struct evtchn_set_priority set_priority;
1560
1561 set_priority.port = evtchn_from_irq(irq);
1562 set_priority.priority = priority;
1563
1564 return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority,
1565 &set_priority);
1566}
1567EXPORT_SYMBOL_GPL(xen_set_irq_priority);
1568
1569int evtchn_make_refcounted(evtchn_port_t evtchn)
1570{
1571 int irq = get_evtchn_to_irq(evtchn);
1572 struct irq_info *info;
1573
1574 if (irq == -1)
1575 return -ENOENT;
1576
1577 info = info_for_irq(irq);
1578
1579 if (!info)
1580 return -ENOENT;
1581
1582 WARN_ON(info->refcnt != -1);
1583
1584 info->refcnt = 1;
1585
1586 return 0;
1587}
1588EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1589
1590int evtchn_get(evtchn_port_t evtchn)
1591{
1592 int irq;
1593 struct irq_info *info;
1594 int err = -ENOENT;
1595
1596 if (evtchn >= xen_evtchn_max_channels())
1597 return -EINVAL;
1598
1599 mutex_lock(&irq_mapping_update_lock);
1600
1601 irq = get_evtchn_to_irq(evtchn);
1602 if (irq == -1)
1603 goto done;
1604
1605 info = info_for_irq(irq);
1606
1607 if (!info)
1608 goto done;
1609
1610 err = -EINVAL;
1611 if (info->refcnt <= 0 || info->refcnt == SHRT_MAX)
1612 goto done;
1613
1614 info->refcnt++;
1615 err = 0;
1616 done:
1617 mutex_unlock(&irq_mapping_update_lock);
1618
1619 return err;
1620}
1621EXPORT_SYMBOL_GPL(evtchn_get);
1622
1623void evtchn_put(evtchn_port_t evtchn)
1624{
1625 int irq = get_evtchn_to_irq(evtchn);
1626 if (WARN_ON(irq == -1))
1627 return;
1628 unbind_from_irq(irq);
1629}
1630EXPORT_SYMBOL_GPL(evtchn_put);
1631
1632void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1633{
1634 int irq;
1635
1636#ifdef CONFIG_X86
1637 if (unlikely(vector == XEN_NMI_VECTOR)) {
1638 int rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, xen_vcpu_nr(cpu),
1639 NULL);
1640 if (rc < 0)
1641 printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc);
1642 return;
1643 }
1644#endif
1645 irq = per_cpu(ipi_to_irq, cpu)[vector];
1646 BUG_ON(irq < 0);
1647 notify_remote_via_irq(irq);
1648}
1649
1650struct evtchn_loop_ctrl {
1651 ktime_t timeout;
1652 unsigned count;
1653 bool defer_eoi;
1654};
1655
1656void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl)
1657{
1658 int irq;
1659 struct irq_info *info;
1660 struct xenbus_device *dev;
1661
1662 irq = get_evtchn_to_irq(port);
1663 if (irq == -1)
1664 return;
1665
1666 /*
1667 * Check for timeout every 256 events.
1668 * We are setting the timeout value only after the first 256
1669 * events in order to not hurt the common case of few loop
1670 * iterations. The 256 is basically an arbitrary value.
1671 *
1672 * In case we are hitting the timeout we need to defer all further
1673 * EOIs in order to ensure to leave the event handling loop rather
1674 * sooner than later.
1675 */
1676 if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) {
1677 ktime_t kt = ktime_get();
1678
1679 if (!ctrl->timeout) {
1680 kt = ktime_add_ms(kt,
1681 jiffies_to_msecs(event_loop_timeout));
1682 ctrl->timeout = kt;
1683 } else if (kt > ctrl->timeout) {
1684 ctrl->defer_eoi = true;
1685 }
1686 }
1687
1688 info = info_for_irq(irq);
1689 if (xchg_acquire(&info->is_active, 1))
1690 return;
1691
1692 dev = (info->type == IRQT_EVTCHN) ? info->u.interdomain : NULL;
1693 if (dev)
1694 atomic_inc(&dev->events);
1695
1696 if (ctrl->defer_eoi) {
1697 info->eoi_cpu = smp_processor_id();
1698 info->irq_epoch = __this_cpu_read(irq_epoch);
1699 info->eoi_time = get_jiffies_64() + event_eoi_delay;
1700 }
1701
1702 generic_handle_irq(irq);
1703}
1704
1705static void __xen_evtchn_do_upcall(void)
1706{
1707 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
1708 int cpu = smp_processor_id();
1709 struct evtchn_loop_ctrl ctrl = { 0 };
1710
1711 read_lock(&evtchn_rwlock);
1712
1713 do {
1714 vcpu_info->evtchn_upcall_pending = 0;
1715
1716 xen_evtchn_handle_events(cpu, &ctrl);
1717
1718 BUG_ON(!irqs_disabled());
1719
1720 virt_rmb(); /* Hypervisor can set upcall pending. */
1721
1722 } while (vcpu_info->evtchn_upcall_pending);
1723
1724 read_unlock(&evtchn_rwlock);
1725
1726 /*
1727 * Increment irq_epoch only now to defer EOIs only for
1728 * xen_irq_lateeoi() invocations occurring from inside the loop
1729 * above.
1730 */
1731 __this_cpu_inc(irq_epoch);
1732}
1733
1734void xen_evtchn_do_upcall(struct pt_regs *regs)
1735{
1736 struct pt_regs *old_regs = set_irq_regs(regs);
1737
1738 irq_enter();
1739
1740 __xen_evtchn_do_upcall();
1741
1742 irq_exit();
1743 set_irq_regs(old_regs);
1744}
1745
1746void xen_hvm_evtchn_do_upcall(void)
1747{
1748 __xen_evtchn_do_upcall();
1749}
1750EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
1751
1752/* Rebind a new event channel to an existing irq. */
1753void rebind_evtchn_irq(evtchn_port_t evtchn, int irq)
1754{
1755 struct irq_info *info = info_for_irq(irq);
1756
1757 if (WARN_ON(!info))
1758 return;
1759
1760 /* Make sure the irq is masked, since the new event channel
1761 will also be masked. */
1762 disable_irq(irq);
1763
1764 mutex_lock(&irq_mapping_update_lock);
1765
1766 /* After resume the irq<->evtchn mappings are all cleared out */
1767 BUG_ON(get_evtchn_to_irq(evtchn) != -1);
1768 /* Expect irq to have been bound before,
1769 so there should be a proper type */
1770 BUG_ON(info->type == IRQT_UNBOUND);
1771
1772 (void)xen_irq_info_evtchn_setup(irq, evtchn, NULL);
1773
1774 mutex_unlock(&irq_mapping_update_lock);
1775
1776 bind_evtchn_to_cpu(evtchn, info->cpu, false);
1777
1778 /* Unmask the event channel. */
1779 enable_irq(irq);
1780}
1781
1782/* Rebind an evtchn so that it gets delivered to a specific cpu */
1783static int xen_rebind_evtchn_to_cpu(struct irq_info *info, unsigned int tcpu)
1784{
1785 struct evtchn_bind_vcpu bind_vcpu;
1786 evtchn_port_t evtchn = info ? info->evtchn : 0;
1787
1788 if (!VALID_EVTCHN(evtchn))
1789 return -1;
1790
1791 if (!xen_support_evtchn_rebind())
1792 return -1;
1793
1794 /* Send future instances of this interrupt to other vcpu. */
1795 bind_vcpu.port = evtchn;
1796 bind_vcpu.vcpu = xen_vcpu_nr(tcpu);
1797
1798 /*
1799 * Mask the event while changing the VCPU binding to prevent
1800 * it being delivered on an unexpected VCPU.
1801 */
1802 do_mask(info, EVT_MASK_REASON_TEMPORARY);
1803
1804 /*
1805 * If this fails, it usually just indicates that we're dealing with a
1806 * virq or IPI channel, which don't actually need to be rebound. Ignore
1807 * it, but don't do the xenlinux-level rebind in that case.
1808 */
1809 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1810 bind_evtchn_to_cpu(evtchn, tcpu, false);
1811
1812 do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1813
1814 return 0;
1815}
1816
1817/*
1818 * Find the CPU within @dest mask which has the least number of channels
1819 * assigned. This is not precise as the per cpu counts can be modified
1820 * concurrently.
1821 */
1822static unsigned int select_target_cpu(const struct cpumask *dest)
1823{
1824 unsigned int cpu, best_cpu = UINT_MAX, minch = UINT_MAX;
1825
1826 for_each_cpu_and(cpu, dest, cpu_online_mask) {
1827 unsigned int curch = atomic_read(&channels_on_cpu[cpu]);
1828
1829 if (curch < minch) {
1830 minch = curch;
1831 best_cpu = cpu;
1832 }
1833 }
1834
1835 /*
1836 * Catch the unlikely case that dest contains no online CPUs. Can't
1837 * recurse.
1838 */
1839 if (best_cpu == UINT_MAX)
1840 return select_target_cpu(cpu_online_mask);
1841
1842 return best_cpu;
1843}
1844
1845static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1846 bool force)
1847{
1848 unsigned int tcpu = select_target_cpu(dest);
1849 int ret;
1850
1851 ret = xen_rebind_evtchn_to_cpu(info_for_irq(data->irq), tcpu);
1852 if (!ret)
1853 irq_data_update_effective_affinity(data, cpumask_of(tcpu));
1854
1855 return ret;
1856}
1857
1858static void enable_dynirq(struct irq_data *data)
1859{
1860 struct irq_info *info = info_for_irq(data->irq);
1861 evtchn_port_t evtchn = info ? info->evtchn : 0;
1862
1863 if (VALID_EVTCHN(evtchn))
1864 do_unmask(info, EVT_MASK_REASON_EXPLICIT);
1865}
1866
1867static void disable_dynirq(struct irq_data *data)
1868{
1869 struct irq_info *info = info_for_irq(data->irq);
1870 evtchn_port_t evtchn = info ? info->evtchn : 0;
1871
1872 if (VALID_EVTCHN(evtchn))
1873 do_mask(info, EVT_MASK_REASON_EXPLICIT);
1874}
1875
1876static void ack_dynirq(struct irq_data *data)
1877{
1878 struct irq_info *info = info_for_irq(data->irq);
1879 evtchn_port_t evtchn = info ? info->evtchn : 0;
1880
1881 if (VALID_EVTCHN(evtchn))
1882 event_handler_exit(info);
1883}
1884
1885static void mask_ack_dynirq(struct irq_data *data)
1886{
1887 disable_dynirq(data);
1888 ack_dynirq(data);
1889}
1890
1891static void lateeoi_ack_dynirq(struct irq_data *data)
1892{
1893 struct irq_info *info = info_for_irq(data->irq);
1894 evtchn_port_t evtchn = info ? info->evtchn : 0;
1895
1896 if (VALID_EVTCHN(evtchn)) {
1897 do_mask(info, EVT_MASK_REASON_EOI_PENDING);
1898 /*
1899 * Don't call event_handler_exit().
1900 * Need to keep is_active non-zero in order to ignore re-raised
1901 * events after cpu affinity changes while a lateeoi is pending.
1902 */
1903 clear_evtchn(evtchn);
1904 }
1905}
1906
1907static void lateeoi_mask_ack_dynirq(struct irq_data *data)
1908{
1909 struct irq_info *info = info_for_irq(data->irq);
1910 evtchn_port_t evtchn = info ? info->evtchn : 0;
1911
1912 if (VALID_EVTCHN(evtchn)) {
1913 do_mask(info, EVT_MASK_REASON_EXPLICIT);
1914 event_handler_exit(info);
1915 }
1916}
1917
1918static int retrigger_dynirq(struct irq_data *data)
1919{
1920 struct irq_info *info = info_for_irq(data->irq);
1921 evtchn_port_t evtchn = info ? info->evtchn : 0;
1922
1923 if (!VALID_EVTCHN(evtchn))
1924 return 0;
1925
1926 do_mask(info, EVT_MASK_REASON_TEMPORARY);
1927 set_evtchn(evtchn);
1928 do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1929
1930 return 1;
1931}
1932
1933static void restore_pirqs(void)
1934{
1935 int pirq, rc, irq, gsi;
1936 struct physdev_map_pirq map_irq;
1937 struct irq_info *info;
1938
1939 list_for_each_entry(info, &xen_irq_list_head, list) {
1940 if (info->type != IRQT_PIRQ)
1941 continue;
1942
1943 pirq = info->u.pirq.pirq;
1944 gsi = info->u.pirq.gsi;
1945 irq = info->irq;
1946
1947 /* save/restore of PT devices doesn't work, so at this point the
1948 * only devices present are GSI based emulated devices */
1949 if (!gsi)
1950 continue;
1951
1952 map_irq.domid = DOMID_SELF;
1953 map_irq.type = MAP_PIRQ_TYPE_GSI;
1954 map_irq.index = gsi;
1955 map_irq.pirq = pirq;
1956
1957 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1958 if (rc) {
1959 pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1960 gsi, irq, pirq, rc);
1961 xen_free_irq(irq);
1962 continue;
1963 }
1964
1965 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1966
1967 __startup_pirq(irq);
1968 }
1969}
1970
1971static void restore_cpu_virqs(unsigned int cpu)
1972{
1973 struct evtchn_bind_virq bind_virq;
1974 evtchn_port_t evtchn;
1975 int virq, irq;
1976
1977 for (virq = 0; virq < NR_VIRQS; virq++) {
1978 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1979 continue;
1980
1981 BUG_ON(virq_from_irq(irq) != virq);
1982
1983 /* Get a new binding from Xen. */
1984 bind_virq.virq = virq;
1985 bind_virq.vcpu = xen_vcpu_nr(cpu);
1986 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1987 &bind_virq) != 0)
1988 BUG();
1989 evtchn = bind_virq.port;
1990
1991 /* Record the new mapping. */
1992 (void)xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
1993 /* The affinity mask is still valid */
1994 bind_evtchn_to_cpu(evtchn, cpu, false);
1995 }
1996}
1997
1998static void restore_cpu_ipis(unsigned int cpu)
1999{
2000 struct evtchn_bind_ipi bind_ipi;
2001 evtchn_port_t evtchn;
2002 int ipi, irq;
2003
2004 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
2005 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
2006 continue;
2007
2008 BUG_ON(ipi_from_irq(irq) != ipi);
2009
2010 /* Get a new binding from Xen. */
2011 bind_ipi.vcpu = xen_vcpu_nr(cpu);
2012 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
2013 &bind_ipi) != 0)
2014 BUG();
2015 evtchn = bind_ipi.port;
2016
2017 /* Record the new mapping. */
2018 (void)xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
2019 /* The affinity mask is still valid */
2020 bind_evtchn_to_cpu(evtchn, cpu, false);
2021 }
2022}
2023
2024/* Clear an irq's pending state, in preparation for polling on it */
2025void xen_clear_irq_pending(int irq)
2026{
2027 struct irq_info *info = info_for_irq(irq);
2028 evtchn_port_t evtchn = info ? info->evtchn : 0;
2029
2030 if (VALID_EVTCHN(evtchn))
2031 event_handler_exit(info);
2032}
2033EXPORT_SYMBOL(xen_clear_irq_pending);
2034void xen_set_irq_pending(int irq)
2035{
2036 evtchn_port_t evtchn = evtchn_from_irq(irq);
2037
2038 if (VALID_EVTCHN(evtchn))
2039 set_evtchn(evtchn);
2040}
2041
2042bool xen_test_irq_pending(int irq)
2043{
2044 evtchn_port_t evtchn = evtchn_from_irq(irq);
2045 bool ret = false;
2046
2047 if (VALID_EVTCHN(evtchn))
2048 ret = test_evtchn(evtchn);
2049
2050 return ret;
2051}
2052
2053/* Poll waiting for an irq to become pending with timeout. In the usual case,
2054 * the irq will be disabled so it won't deliver an interrupt. */
2055void xen_poll_irq_timeout(int irq, u64 timeout)
2056{
2057 evtchn_port_t evtchn = evtchn_from_irq(irq);
2058
2059 if (VALID_EVTCHN(evtchn)) {
2060 struct sched_poll poll;
2061
2062 poll.nr_ports = 1;
2063 poll.timeout = timeout;
2064 set_xen_guest_handle(poll.ports, &evtchn);
2065
2066 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
2067 BUG();
2068 }
2069}
2070EXPORT_SYMBOL(xen_poll_irq_timeout);
2071/* Poll waiting for an irq to become pending. In the usual case, the
2072 * irq will be disabled so it won't deliver an interrupt. */
2073void xen_poll_irq(int irq)
2074{
2075 xen_poll_irq_timeout(irq, 0 /* no timeout */);
2076}
2077
2078/* Check whether the IRQ line is shared with other guests. */
2079int xen_test_irq_shared(int irq)
2080{
2081 struct irq_info *info = info_for_irq(irq);
2082 struct physdev_irq_status_query irq_status;
2083
2084 if (WARN_ON(!info))
2085 return -ENOENT;
2086
2087 irq_status.irq = info->u.pirq.pirq;
2088
2089 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
2090 return 0;
2091 return !(irq_status.flags & XENIRQSTAT_shared);
2092}
2093EXPORT_SYMBOL_GPL(xen_test_irq_shared);
2094
2095void xen_irq_resume(void)
2096{
2097 unsigned int cpu;
2098 struct irq_info *info;
2099
2100 /* New event-channel space is not 'live' yet. */
2101 xen_evtchn_resume();
2102
2103 /* No IRQ <-> event-channel mappings. */
2104 list_for_each_entry(info, &xen_irq_list_head, list) {
2105 /* Zap event-channel binding */
2106 info->evtchn = 0;
2107 /* Adjust accounting */
2108 channels_on_cpu_dec(info);
2109 }
2110
2111 clear_evtchn_to_irq_all();
2112
2113 for_each_possible_cpu(cpu) {
2114 restore_cpu_virqs(cpu);
2115 restore_cpu_ipis(cpu);
2116 }
2117
2118 restore_pirqs();
2119}
2120
2121static struct irq_chip xen_dynamic_chip __read_mostly = {
2122 .name = "xen-dyn",
2123
2124 .irq_disable = disable_dynirq,
2125 .irq_mask = disable_dynirq,
2126 .irq_unmask = enable_dynirq,
2127
2128 .irq_ack = ack_dynirq,
2129 .irq_mask_ack = mask_ack_dynirq,
2130
2131 .irq_set_affinity = set_affinity_irq,
2132 .irq_retrigger = retrigger_dynirq,
2133};
2134
2135static struct irq_chip xen_lateeoi_chip __read_mostly = {
2136 /* The chip name needs to contain "xen-dyn" for irqbalance to work. */
2137 .name = "xen-dyn-lateeoi",
2138
2139 .irq_disable = disable_dynirq,
2140 .irq_mask = disable_dynirq,
2141 .irq_unmask = enable_dynirq,
2142
2143 .irq_ack = lateeoi_ack_dynirq,
2144 .irq_mask_ack = lateeoi_mask_ack_dynirq,
2145
2146 .irq_set_affinity = set_affinity_irq,
2147 .irq_retrigger = retrigger_dynirq,
2148};
2149
2150static struct irq_chip xen_pirq_chip __read_mostly = {
2151 .name = "xen-pirq",
2152
2153 .irq_startup = startup_pirq,
2154 .irq_shutdown = shutdown_pirq,
2155 .irq_enable = enable_pirq,
2156 .irq_disable = disable_pirq,
2157
2158 .irq_mask = disable_dynirq,
2159 .irq_unmask = enable_dynirq,
2160
2161 .irq_ack = eoi_pirq,
2162 .irq_eoi = eoi_pirq,
2163 .irq_mask_ack = mask_ack_pirq,
2164
2165 .irq_set_affinity = set_affinity_irq,
2166
2167 .irq_retrigger = retrigger_dynirq,
2168};
2169
2170static struct irq_chip xen_percpu_chip __read_mostly = {
2171 .name = "xen-percpu",
2172
2173 .irq_disable = disable_dynirq,
2174 .irq_mask = disable_dynirq,
2175 .irq_unmask = enable_dynirq,
2176
2177 .irq_ack = ack_dynirq,
2178};
2179
2180#ifdef CONFIG_XEN_PVHVM
2181/* Vector callbacks are better than PCI interrupts to receive event
2182 * channel notifications because we can receive vector callbacks on any
2183 * vcpu and we don't need PCI support or APIC interactions. */
2184void xen_setup_callback_vector(void)
2185{
2186 uint64_t callback_via;
2187
2188 if (xen_have_vector_callback) {
2189 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
2190 if (xen_set_callback_via(callback_via)) {
2191 pr_err("Request for Xen HVM callback vector failed\n");
2192 xen_have_vector_callback = 0;
2193 }
2194 }
2195}
2196
2197static __init void xen_alloc_callback_vector(void)
2198{
2199 if (!xen_have_vector_callback)
2200 return;
2201
2202 pr_info("Xen HVM callback vector for event delivery is enabled\n");
2203 alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback);
2204}
2205#else
2206void xen_setup_callback_vector(void) {}
2207static inline void xen_alloc_callback_vector(void) {}
2208#endif
2209
2210bool xen_fifo_events = true;
2211module_param_named(fifo_events, xen_fifo_events, bool, 0);
2212
2213static int xen_evtchn_cpu_prepare(unsigned int cpu)
2214{
2215 int ret = 0;
2216
2217 xen_cpu_init_eoi(cpu);
2218
2219 if (evtchn_ops->percpu_init)
2220 ret = evtchn_ops->percpu_init(cpu);
2221
2222 return ret;
2223}
2224
2225static int xen_evtchn_cpu_dead(unsigned int cpu)
2226{
2227 int ret = 0;
2228
2229 if (evtchn_ops->percpu_deinit)
2230 ret = evtchn_ops->percpu_deinit(cpu);
2231
2232 return ret;
2233}
2234
2235void __init xen_init_IRQ(void)
2236{
2237 int ret = -EINVAL;
2238 evtchn_port_t evtchn;
2239
2240 if (xen_fifo_events)
2241 ret = xen_evtchn_fifo_init();
2242 if (ret < 0) {
2243 xen_evtchn_2l_init();
2244 xen_fifo_events = false;
2245 }
2246
2247 xen_cpu_init_eoi(smp_processor_id());
2248
2249 cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
2250 "xen/evtchn:prepare",
2251 xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
2252
2253 evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()),
2254 sizeof(*evtchn_to_irq), GFP_KERNEL);
2255 BUG_ON(!evtchn_to_irq);
2256
2257 /* No event channels are 'live' right now. */
2258 for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++)
2259 mask_evtchn(evtchn);
2260
2261 pirq_needs_eoi = pirq_needs_eoi_flag;
2262
2263#ifdef CONFIG_X86
2264 if (xen_pv_domain()) {
2265 if (xen_initial_domain())
2266 pci_xen_initial_domain();
2267 }
2268 if (xen_feature(XENFEAT_hvm_callback_vector)) {
2269 xen_setup_callback_vector();
2270 xen_alloc_callback_vector();
2271 }
2272
2273 if (xen_hvm_domain()) {
2274 native_init_IRQ();
2275 /* pci_xen_hvm_init must be called after native_init_IRQ so that
2276 * __acpi_register_gsi can point at the right function */
2277 pci_xen_hvm_init();
2278 } else {
2279 int rc;
2280 struct physdev_pirq_eoi_gmfn eoi_gmfn;
2281
2282 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
2283 eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map);
2284 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
2285 if (rc != 0) {
2286 free_page((unsigned long) pirq_eoi_map);
2287 pirq_eoi_map = NULL;
2288 } else
2289 pirq_needs_eoi = pirq_check_eoi_map;
2290 }
2291#endif
2292}