Loading...
1/*
2 * Intel IO-APIC support for multi-Pentium hosts.
3 *
4 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
5 *
6 * Many thanks to Stig Venaas for trying out countless experimental
7 * patches and reporting/debugging problems patiently!
8 *
9 * (c) 1999, Multiple IO-APIC support, developed by
10 * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
11 * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
12 * further tested and cleaned up by Zach Brown <zab@redhat.com>
13 * and Ingo Molnar <mingo@redhat.com>
14 *
15 * Fixes
16 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
17 * thanks to Eric Gilmore
18 * and Rolf G. Tews
19 * for testing these extensively
20 * Paul Diefenbaugh : Added full ACPI support
21 */
22
23#include <linux/mm.h>
24#include <linux/interrupt.h>
25#include <linux/init.h>
26#include <linux/delay.h>
27#include <linux/sched.h>
28#include <linux/pci.h>
29#include <linux/mc146818rtc.h>
30#include <linux/compiler.h>
31#include <linux/acpi.h>
32#include <linux/module.h>
33#include <linux/syscore_ops.h>
34#include <linux/msi.h>
35#include <linux/htirq.h>
36#include <linux/freezer.h>
37#include <linux/kthread.h>
38#include <linux/jiffies.h> /* time_after() */
39#include <linux/slab.h>
40#ifdef CONFIG_ACPI
41#include <acpi/acpi_bus.h>
42#endif
43#include <linux/bootmem.h>
44#include <linux/dmar.h>
45#include <linux/hpet.h>
46
47#include <asm/idle.h>
48#include <asm/io.h>
49#include <asm/smp.h>
50#include <asm/cpu.h>
51#include <asm/desc.h>
52#include <asm/proto.h>
53#include <asm/acpi.h>
54#include <asm/dma.h>
55#include <asm/timer.h>
56#include <asm/i8259.h>
57#include <asm/msidef.h>
58#include <asm/hypertransport.h>
59#include <asm/setup.h>
60#include <asm/irq_remapping.h>
61#include <asm/hpet.h>
62#include <asm/hw_irq.h>
63
64#include <asm/apic.h>
65
66#define __apicdebuginit(type) static type __init
67
68#define for_each_irq_pin(entry, head) \
69 for (entry = head; entry; entry = entry->next)
70
71#ifdef CONFIG_IRQ_REMAP
72static void irq_remap_modify_chip_defaults(struct irq_chip *chip);
73static inline bool irq_remapped(struct irq_cfg *cfg)
74{
75 return cfg->irq_2_iommu.iommu != NULL;
76}
77#else
78static inline bool irq_remapped(struct irq_cfg *cfg)
79{
80 return false;
81}
82static inline void irq_remap_modify_chip_defaults(struct irq_chip *chip)
83{
84}
85#endif
86
87/*
88 * Is the SiS APIC rmw bug present ?
89 * -1 = don't know, 0 = no, 1 = yes
90 */
91int sis_apic_bug = -1;
92
93static DEFINE_RAW_SPINLOCK(ioapic_lock);
94static DEFINE_RAW_SPINLOCK(vector_lock);
95
96static struct ioapic {
97 /*
98 * # of IRQ routing registers
99 */
100 int nr_registers;
101 /*
102 * Saved state during suspend/resume, or while enabling intr-remap.
103 */
104 struct IO_APIC_route_entry *saved_registers;
105 /* I/O APIC config */
106 struct mpc_ioapic mp_config;
107 /* IO APIC gsi routing info */
108 struct mp_ioapic_gsi gsi_config;
109 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
110} ioapics[MAX_IO_APICS];
111
112#define mpc_ioapic_ver(ioapic_idx) ioapics[ioapic_idx].mp_config.apicver
113
114int mpc_ioapic_id(int ioapic_idx)
115{
116 return ioapics[ioapic_idx].mp_config.apicid;
117}
118
119unsigned int mpc_ioapic_addr(int ioapic_idx)
120{
121 return ioapics[ioapic_idx].mp_config.apicaddr;
122}
123
124struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic_idx)
125{
126 return &ioapics[ioapic_idx].gsi_config;
127}
128
129int nr_ioapics;
130
131/* The one past the highest gsi number used */
132u32 gsi_top;
133
134/* MP IRQ source entries */
135struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
136
137/* # of MP IRQ source entries */
138int mp_irq_entries;
139
140/* GSI interrupts */
141static int nr_irqs_gsi = NR_IRQS_LEGACY;
142
143#ifdef CONFIG_EISA
144int mp_bus_id_to_type[MAX_MP_BUSSES];
145#endif
146
147DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
148
149int skip_ioapic_setup;
150
151/**
152 * disable_ioapic_support() - disables ioapic support at runtime
153 */
154void disable_ioapic_support(void)
155{
156#ifdef CONFIG_PCI
157 noioapicquirk = 1;
158 noioapicreroute = -1;
159#endif
160 skip_ioapic_setup = 1;
161}
162
163static int __init parse_noapic(char *str)
164{
165 /* disable IO-APIC */
166 disable_ioapic_support();
167 return 0;
168}
169early_param("noapic", parse_noapic);
170
171static int io_apic_setup_irq_pin(unsigned int irq, int node,
172 struct io_apic_irq_attr *attr);
173
174/* Will be called in mpparse/acpi/sfi codes for saving IRQ info */
175void mp_save_irq(struct mpc_intsrc *m)
176{
177 int i;
178
179 apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x,"
180 " IRQ %02x, APIC ID %x, APIC INT %02x\n",
181 m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus,
182 m->srcbusirq, m->dstapic, m->dstirq);
183
184 for (i = 0; i < mp_irq_entries; i++) {
185 if (!memcmp(&mp_irqs[i], m, sizeof(*m)))
186 return;
187 }
188
189 memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m));
190 if (++mp_irq_entries == MAX_IRQ_SOURCES)
191 panic("Max # of irq sources exceeded!!\n");
192}
193
194struct irq_pin_list {
195 int apic, pin;
196 struct irq_pin_list *next;
197};
198
199static struct irq_pin_list *alloc_irq_pin_list(int node)
200{
201 return kzalloc_node(sizeof(struct irq_pin_list), GFP_KERNEL, node);
202}
203
204
205/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
206static struct irq_cfg irq_cfgx[NR_IRQS_LEGACY];
207
208int __init arch_early_irq_init(void)
209{
210 struct irq_cfg *cfg;
211 int count, node, i;
212
213 if (!legacy_pic->nr_legacy_irqs)
214 io_apic_irqs = ~0UL;
215
216 for (i = 0; i < nr_ioapics; i++) {
217 ioapics[i].saved_registers =
218 kzalloc(sizeof(struct IO_APIC_route_entry) *
219 ioapics[i].nr_registers, GFP_KERNEL);
220 if (!ioapics[i].saved_registers)
221 pr_err("IOAPIC %d: suspend/resume impossible!\n", i);
222 }
223
224 cfg = irq_cfgx;
225 count = ARRAY_SIZE(irq_cfgx);
226 node = cpu_to_node(0);
227
228 /* Make sure the legacy interrupts are marked in the bitmap */
229 irq_reserve_irqs(0, legacy_pic->nr_legacy_irqs);
230
231 for (i = 0; i < count; i++) {
232 irq_set_chip_data(i, &cfg[i]);
233 zalloc_cpumask_var_node(&cfg[i].domain, GFP_KERNEL, node);
234 zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_KERNEL, node);
235 /*
236 * For legacy IRQ's, start with assigning irq0 to irq15 to
237 * IRQ0_VECTOR to IRQ15_VECTOR on cpu 0.
238 */
239 if (i < legacy_pic->nr_legacy_irqs) {
240 cfg[i].vector = IRQ0_VECTOR + i;
241 cpumask_set_cpu(0, cfg[i].domain);
242 }
243 }
244
245 return 0;
246}
247
248static struct irq_cfg *irq_cfg(unsigned int irq)
249{
250 return irq_get_chip_data(irq);
251}
252
253static struct irq_cfg *alloc_irq_cfg(unsigned int irq, int node)
254{
255 struct irq_cfg *cfg;
256
257 cfg = kzalloc_node(sizeof(*cfg), GFP_KERNEL, node);
258 if (!cfg)
259 return NULL;
260 if (!zalloc_cpumask_var_node(&cfg->domain, GFP_KERNEL, node))
261 goto out_cfg;
262 if (!zalloc_cpumask_var_node(&cfg->old_domain, GFP_KERNEL, node))
263 goto out_domain;
264 return cfg;
265out_domain:
266 free_cpumask_var(cfg->domain);
267out_cfg:
268 kfree(cfg);
269 return NULL;
270}
271
272static void free_irq_cfg(unsigned int at, struct irq_cfg *cfg)
273{
274 if (!cfg)
275 return;
276 irq_set_chip_data(at, NULL);
277 free_cpumask_var(cfg->domain);
278 free_cpumask_var(cfg->old_domain);
279 kfree(cfg);
280}
281
282static struct irq_cfg *alloc_irq_and_cfg_at(unsigned int at, int node)
283{
284 int res = irq_alloc_desc_at(at, node);
285 struct irq_cfg *cfg;
286
287 if (res < 0) {
288 if (res != -EEXIST)
289 return NULL;
290 cfg = irq_get_chip_data(at);
291 if (cfg)
292 return cfg;
293 }
294
295 cfg = alloc_irq_cfg(at, node);
296 if (cfg)
297 irq_set_chip_data(at, cfg);
298 else
299 irq_free_desc(at);
300 return cfg;
301}
302
303static int alloc_irq_from(unsigned int from, int node)
304{
305 return irq_alloc_desc_from(from, node);
306}
307
308static void free_irq_at(unsigned int at, struct irq_cfg *cfg)
309{
310 free_irq_cfg(at, cfg);
311 irq_free_desc(at);
312}
313
314
315struct io_apic {
316 unsigned int index;
317 unsigned int unused[3];
318 unsigned int data;
319 unsigned int unused2[11];
320 unsigned int eoi;
321};
322
323static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
324{
325 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
326 + (mpc_ioapic_addr(idx) & ~PAGE_MASK);
327}
328
329static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
330{
331 struct io_apic __iomem *io_apic = io_apic_base(apic);
332 writel(vector, &io_apic->eoi);
333}
334
335unsigned int native_io_apic_read(unsigned int apic, unsigned int reg)
336{
337 struct io_apic __iomem *io_apic = io_apic_base(apic);
338 writel(reg, &io_apic->index);
339 return readl(&io_apic->data);
340}
341
342void native_io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
343{
344 struct io_apic __iomem *io_apic = io_apic_base(apic);
345
346 writel(reg, &io_apic->index);
347 writel(value, &io_apic->data);
348}
349
350/*
351 * Re-write a value: to be used for read-modify-write
352 * cycles where the read already set up the index register.
353 *
354 * Older SiS APIC requires we rewrite the index register
355 */
356void native_io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
357{
358 struct io_apic __iomem *io_apic = io_apic_base(apic);
359
360 if (sis_apic_bug)
361 writel(reg, &io_apic->index);
362 writel(value, &io_apic->data);
363}
364
365union entry_union {
366 struct { u32 w1, w2; };
367 struct IO_APIC_route_entry entry;
368};
369
370static struct IO_APIC_route_entry __ioapic_read_entry(int apic, int pin)
371{
372 union entry_union eu;
373
374 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
375 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
376
377 return eu.entry;
378}
379
380static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
381{
382 union entry_union eu;
383 unsigned long flags;
384
385 raw_spin_lock_irqsave(&ioapic_lock, flags);
386 eu.entry = __ioapic_read_entry(apic, pin);
387 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
388
389 return eu.entry;
390}
391
392/*
393 * When we write a new IO APIC routing entry, we need to write the high
394 * word first! If the mask bit in the low word is clear, we will enable
395 * the interrupt, and we need to make sure the entry is fully populated
396 * before that happens.
397 */
398static void __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
399{
400 union entry_union eu = {{0, 0}};
401
402 eu.entry = e;
403 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
404 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
405}
406
407static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
408{
409 unsigned long flags;
410
411 raw_spin_lock_irqsave(&ioapic_lock, flags);
412 __ioapic_write_entry(apic, pin, e);
413 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
414}
415
416/*
417 * When we mask an IO APIC routing entry, we need to write the low
418 * word first, in order to set the mask bit before we change the
419 * high bits!
420 */
421static void ioapic_mask_entry(int apic, int pin)
422{
423 unsigned long flags;
424 union entry_union eu = { .entry.mask = 1 };
425
426 raw_spin_lock_irqsave(&ioapic_lock, flags);
427 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
428 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
429 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
430}
431
432/*
433 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
434 * shared ISA-space IRQs, so we have to support them. We are super
435 * fast in the common case, and fast for shared ISA-space IRQs.
436 */
437static int __add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
438{
439 struct irq_pin_list **last, *entry;
440
441 /* don't allow duplicates */
442 last = &cfg->irq_2_pin;
443 for_each_irq_pin(entry, cfg->irq_2_pin) {
444 if (entry->apic == apic && entry->pin == pin)
445 return 0;
446 last = &entry->next;
447 }
448
449 entry = alloc_irq_pin_list(node);
450 if (!entry) {
451 printk(KERN_ERR "can not alloc irq_pin_list (%d,%d,%d)\n",
452 node, apic, pin);
453 return -ENOMEM;
454 }
455 entry->apic = apic;
456 entry->pin = pin;
457
458 *last = entry;
459 return 0;
460}
461
462static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
463{
464 if (__add_pin_to_irq_node(cfg, node, apic, pin))
465 panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
466}
467
468/*
469 * Reroute an IRQ to a different pin.
470 */
471static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
472 int oldapic, int oldpin,
473 int newapic, int newpin)
474{
475 struct irq_pin_list *entry;
476
477 for_each_irq_pin(entry, cfg->irq_2_pin) {
478 if (entry->apic == oldapic && entry->pin == oldpin) {
479 entry->apic = newapic;
480 entry->pin = newpin;
481 /* every one is different, right? */
482 return;
483 }
484 }
485
486 /* old apic/pin didn't exist, so just add new ones */
487 add_pin_to_irq_node(cfg, node, newapic, newpin);
488}
489
490static void __io_apic_modify_irq(struct irq_pin_list *entry,
491 int mask_and, int mask_or,
492 void (*final)(struct irq_pin_list *entry))
493{
494 unsigned int reg, pin;
495
496 pin = entry->pin;
497 reg = io_apic_read(entry->apic, 0x10 + pin * 2);
498 reg &= mask_and;
499 reg |= mask_or;
500 io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
501 if (final)
502 final(entry);
503}
504
505static void io_apic_modify_irq(struct irq_cfg *cfg,
506 int mask_and, int mask_or,
507 void (*final)(struct irq_pin_list *entry))
508{
509 struct irq_pin_list *entry;
510
511 for_each_irq_pin(entry, cfg->irq_2_pin)
512 __io_apic_modify_irq(entry, mask_and, mask_or, final);
513}
514
515static void io_apic_sync(struct irq_pin_list *entry)
516{
517 /*
518 * Synchronize the IO-APIC and the CPU by doing
519 * a dummy read from the IO-APIC
520 */
521 struct io_apic __iomem *io_apic;
522
523 io_apic = io_apic_base(entry->apic);
524 readl(&io_apic->data);
525}
526
527static void mask_ioapic(struct irq_cfg *cfg)
528{
529 unsigned long flags;
530
531 raw_spin_lock_irqsave(&ioapic_lock, flags);
532 io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
533 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
534}
535
536static void mask_ioapic_irq(struct irq_data *data)
537{
538 mask_ioapic(data->chip_data);
539}
540
541static void __unmask_ioapic(struct irq_cfg *cfg)
542{
543 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
544}
545
546static void unmask_ioapic(struct irq_cfg *cfg)
547{
548 unsigned long flags;
549
550 raw_spin_lock_irqsave(&ioapic_lock, flags);
551 __unmask_ioapic(cfg);
552 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
553}
554
555static void unmask_ioapic_irq(struct irq_data *data)
556{
557 unmask_ioapic(data->chip_data);
558}
559
560/*
561 * IO-APIC versions below 0x20 don't support EOI register.
562 * For the record, here is the information about various versions:
563 * 0Xh 82489DX
564 * 1Xh I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant
565 * 2Xh I/O(x)APIC which is PCI 2.2 Compliant
566 * 30h-FFh Reserved
567 *
568 * Some of the Intel ICH Specs (ICH2 to ICH5) documents the io-apic
569 * version as 0x2. This is an error with documentation and these ICH chips
570 * use io-apic's of version 0x20.
571 *
572 * For IO-APIC's with EOI register, we use that to do an explicit EOI.
573 * Otherwise, we simulate the EOI message manually by changing the trigger
574 * mode to edge and then back to level, with RTE being masked during this.
575 */
576static void __eoi_ioapic_pin(int apic, int pin, int vector, struct irq_cfg *cfg)
577{
578 if (mpc_ioapic_ver(apic) >= 0x20) {
579 /*
580 * Intr-remapping uses pin number as the virtual vector
581 * in the RTE. Actual vector is programmed in
582 * intr-remapping table entry. Hence for the io-apic
583 * EOI we use the pin number.
584 */
585 if (cfg && irq_remapped(cfg))
586 io_apic_eoi(apic, pin);
587 else
588 io_apic_eoi(apic, vector);
589 } else {
590 struct IO_APIC_route_entry entry, entry1;
591
592 entry = entry1 = __ioapic_read_entry(apic, pin);
593
594 /*
595 * Mask the entry and change the trigger mode to edge.
596 */
597 entry1.mask = 1;
598 entry1.trigger = IOAPIC_EDGE;
599
600 __ioapic_write_entry(apic, pin, entry1);
601
602 /*
603 * Restore the previous level triggered entry.
604 */
605 __ioapic_write_entry(apic, pin, entry);
606 }
607}
608
609static void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
610{
611 struct irq_pin_list *entry;
612 unsigned long flags;
613
614 raw_spin_lock_irqsave(&ioapic_lock, flags);
615 for_each_irq_pin(entry, cfg->irq_2_pin)
616 __eoi_ioapic_pin(entry->apic, entry->pin, cfg->vector, cfg);
617 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
618}
619
620static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
621{
622 struct IO_APIC_route_entry entry;
623
624 /* Check delivery_mode to be sure we're not clearing an SMI pin */
625 entry = ioapic_read_entry(apic, pin);
626 if (entry.delivery_mode == dest_SMI)
627 return;
628
629 /*
630 * Make sure the entry is masked and re-read the contents to check
631 * if it is a level triggered pin and if the remote-IRR is set.
632 */
633 if (!entry.mask) {
634 entry.mask = 1;
635 ioapic_write_entry(apic, pin, entry);
636 entry = ioapic_read_entry(apic, pin);
637 }
638
639 if (entry.irr) {
640 unsigned long flags;
641
642 /*
643 * Make sure the trigger mode is set to level. Explicit EOI
644 * doesn't clear the remote-IRR if the trigger mode is not
645 * set to level.
646 */
647 if (!entry.trigger) {
648 entry.trigger = IOAPIC_LEVEL;
649 ioapic_write_entry(apic, pin, entry);
650 }
651
652 raw_spin_lock_irqsave(&ioapic_lock, flags);
653 __eoi_ioapic_pin(apic, pin, entry.vector, NULL);
654 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
655 }
656
657 /*
658 * Clear the rest of the bits in the IO-APIC RTE except for the mask
659 * bit.
660 */
661 ioapic_mask_entry(apic, pin);
662 entry = ioapic_read_entry(apic, pin);
663 if (entry.irr)
664 printk(KERN_ERR "Unable to reset IRR for apic: %d, pin :%d\n",
665 mpc_ioapic_id(apic), pin);
666}
667
668static void clear_IO_APIC (void)
669{
670 int apic, pin;
671
672 for (apic = 0; apic < nr_ioapics; apic++)
673 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
674 clear_IO_APIC_pin(apic, pin);
675}
676
677#ifdef CONFIG_X86_32
678/*
679 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
680 * specific CPU-side IRQs.
681 */
682
683#define MAX_PIRQS 8
684static int pirq_entries[MAX_PIRQS] = {
685 [0 ... MAX_PIRQS - 1] = -1
686};
687
688static int __init ioapic_pirq_setup(char *str)
689{
690 int i, max;
691 int ints[MAX_PIRQS+1];
692
693 get_options(str, ARRAY_SIZE(ints), ints);
694
695 apic_printk(APIC_VERBOSE, KERN_INFO
696 "PIRQ redirection, working around broken MP-BIOS.\n");
697 max = MAX_PIRQS;
698 if (ints[0] < MAX_PIRQS)
699 max = ints[0];
700
701 for (i = 0; i < max; i++) {
702 apic_printk(APIC_VERBOSE, KERN_DEBUG
703 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
704 /*
705 * PIRQs are mapped upside down, usually.
706 */
707 pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
708 }
709 return 1;
710}
711
712__setup("pirq=", ioapic_pirq_setup);
713#endif /* CONFIG_X86_32 */
714
715/*
716 * Saves all the IO-APIC RTE's
717 */
718int save_ioapic_entries(void)
719{
720 int apic, pin;
721 int err = 0;
722
723 for (apic = 0; apic < nr_ioapics; apic++) {
724 if (!ioapics[apic].saved_registers) {
725 err = -ENOMEM;
726 continue;
727 }
728
729 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
730 ioapics[apic].saved_registers[pin] =
731 ioapic_read_entry(apic, pin);
732 }
733
734 return err;
735}
736
737/*
738 * Mask all IO APIC entries.
739 */
740void mask_ioapic_entries(void)
741{
742 int apic, pin;
743
744 for (apic = 0; apic < nr_ioapics; apic++) {
745 if (!ioapics[apic].saved_registers)
746 continue;
747
748 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
749 struct IO_APIC_route_entry entry;
750
751 entry = ioapics[apic].saved_registers[pin];
752 if (!entry.mask) {
753 entry.mask = 1;
754 ioapic_write_entry(apic, pin, entry);
755 }
756 }
757 }
758}
759
760/*
761 * Restore IO APIC entries which was saved in the ioapic structure.
762 */
763int restore_ioapic_entries(void)
764{
765 int apic, pin;
766
767 for (apic = 0; apic < nr_ioapics; apic++) {
768 if (!ioapics[apic].saved_registers)
769 continue;
770
771 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
772 ioapic_write_entry(apic, pin,
773 ioapics[apic].saved_registers[pin]);
774 }
775 return 0;
776}
777
778/*
779 * Find the IRQ entry number of a certain pin.
780 */
781static int find_irq_entry(int ioapic_idx, int pin, int type)
782{
783 int i;
784
785 for (i = 0; i < mp_irq_entries; i++)
786 if (mp_irqs[i].irqtype == type &&
787 (mp_irqs[i].dstapic == mpc_ioapic_id(ioapic_idx) ||
788 mp_irqs[i].dstapic == MP_APIC_ALL) &&
789 mp_irqs[i].dstirq == pin)
790 return i;
791
792 return -1;
793}
794
795/*
796 * Find the pin to which IRQ[irq] (ISA) is connected
797 */
798static int __init find_isa_irq_pin(int irq, int type)
799{
800 int i;
801
802 for (i = 0; i < mp_irq_entries; i++) {
803 int lbus = mp_irqs[i].srcbus;
804
805 if (test_bit(lbus, mp_bus_not_pci) &&
806 (mp_irqs[i].irqtype == type) &&
807 (mp_irqs[i].srcbusirq == irq))
808
809 return mp_irqs[i].dstirq;
810 }
811 return -1;
812}
813
814static int __init find_isa_irq_apic(int irq, int type)
815{
816 int i;
817
818 for (i = 0; i < mp_irq_entries; i++) {
819 int lbus = mp_irqs[i].srcbus;
820
821 if (test_bit(lbus, mp_bus_not_pci) &&
822 (mp_irqs[i].irqtype == type) &&
823 (mp_irqs[i].srcbusirq == irq))
824 break;
825 }
826
827 if (i < mp_irq_entries) {
828 int ioapic_idx;
829
830 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
831 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic)
832 return ioapic_idx;
833 }
834
835 return -1;
836}
837
838#ifdef CONFIG_EISA
839/*
840 * EISA Edge/Level control register, ELCR
841 */
842static int EISA_ELCR(unsigned int irq)
843{
844 if (irq < legacy_pic->nr_legacy_irqs) {
845 unsigned int port = 0x4d0 + (irq >> 3);
846 return (inb(port) >> (irq & 7)) & 1;
847 }
848 apic_printk(APIC_VERBOSE, KERN_INFO
849 "Broken MPtable reports ISA irq %d\n", irq);
850 return 0;
851}
852
853#endif
854
855/* ISA interrupts are always polarity zero edge triggered,
856 * when listed as conforming in the MP table. */
857
858#define default_ISA_trigger(idx) (0)
859#define default_ISA_polarity(idx) (0)
860
861/* EISA interrupts are always polarity zero and can be edge or level
862 * trigger depending on the ELCR value. If an interrupt is listed as
863 * EISA conforming in the MP table, that means its trigger type must
864 * be read in from the ELCR */
865
866#define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].srcbusirq))
867#define default_EISA_polarity(idx) default_ISA_polarity(idx)
868
869/* PCI interrupts are always polarity one level triggered,
870 * when listed as conforming in the MP table. */
871
872#define default_PCI_trigger(idx) (1)
873#define default_PCI_polarity(idx) (1)
874
875static int irq_polarity(int idx)
876{
877 int bus = mp_irqs[idx].srcbus;
878 int polarity;
879
880 /*
881 * Determine IRQ line polarity (high active or low active):
882 */
883 switch (mp_irqs[idx].irqflag & 3)
884 {
885 case 0: /* conforms, ie. bus-type dependent polarity */
886 if (test_bit(bus, mp_bus_not_pci))
887 polarity = default_ISA_polarity(idx);
888 else
889 polarity = default_PCI_polarity(idx);
890 break;
891 case 1: /* high active */
892 {
893 polarity = 0;
894 break;
895 }
896 case 2: /* reserved */
897 {
898 printk(KERN_WARNING "broken BIOS!!\n");
899 polarity = 1;
900 break;
901 }
902 case 3: /* low active */
903 {
904 polarity = 1;
905 break;
906 }
907 default: /* invalid */
908 {
909 printk(KERN_WARNING "broken BIOS!!\n");
910 polarity = 1;
911 break;
912 }
913 }
914 return polarity;
915}
916
917static int irq_trigger(int idx)
918{
919 int bus = mp_irqs[idx].srcbus;
920 int trigger;
921
922 /*
923 * Determine IRQ trigger mode (edge or level sensitive):
924 */
925 switch ((mp_irqs[idx].irqflag>>2) & 3)
926 {
927 case 0: /* conforms, ie. bus-type dependent */
928 if (test_bit(bus, mp_bus_not_pci))
929 trigger = default_ISA_trigger(idx);
930 else
931 trigger = default_PCI_trigger(idx);
932#ifdef CONFIG_EISA
933 switch (mp_bus_id_to_type[bus]) {
934 case MP_BUS_ISA: /* ISA pin */
935 {
936 /* set before the switch */
937 break;
938 }
939 case MP_BUS_EISA: /* EISA pin */
940 {
941 trigger = default_EISA_trigger(idx);
942 break;
943 }
944 case MP_BUS_PCI: /* PCI pin */
945 {
946 /* set before the switch */
947 break;
948 }
949 default:
950 {
951 printk(KERN_WARNING "broken BIOS!!\n");
952 trigger = 1;
953 break;
954 }
955 }
956#endif
957 break;
958 case 1: /* edge */
959 {
960 trigger = 0;
961 break;
962 }
963 case 2: /* reserved */
964 {
965 printk(KERN_WARNING "broken BIOS!!\n");
966 trigger = 1;
967 break;
968 }
969 case 3: /* level */
970 {
971 trigger = 1;
972 break;
973 }
974 default: /* invalid */
975 {
976 printk(KERN_WARNING "broken BIOS!!\n");
977 trigger = 0;
978 break;
979 }
980 }
981 return trigger;
982}
983
984static int pin_2_irq(int idx, int apic, int pin)
985{
986 int irq;
987 int bus = mp_irqs[idx].srcbus;
988 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(apic);
989
990 /*
991 * Debugging check, we are in big trouble if this message pops up!
992 */
993 if (mp_irqs[idx].dstirq != pin)
994 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
995
996 if (test_bit(bus, mp_bus_not_pci)) {
997 irq = mp_irqs[idx].srcbusirq;
998 } else {
999 u32 gsi = gsi_cfg->gsi_base + pin;
1000
1001 if (gsi >= NR_IRQS_LEGACY)
1002 irq = gsi;
1003 else
1004 irq = gsi_top + gsi;
1005 }
1006
1007#ifdef CONFIG_X86_32
1008 /*
1009 * PCI IRQ command line redirection. Yes, limits are hardcoded.
1010 */
1011 if ((pin >= 16) && (pin <= 23)) {
1012 if (pirq_entries[pin-16] != -1) {
1013 if (!pirq_entries[pin-16]) {
1014 apic_printk(APIC_VERBOSE, KERN_DEBUG
1015 "disabling PIRQ%d\n", pin-16);
1016 } else {
1017 irq = pirq_entries[pin-16];
1018 apic_printk(APIC_VERBOSE, KERN_DEBUG
1019 "using PIRQ%d -> IRQ %d\n",
1020 pin-16, irq);
1021 }
1022 }
1023 }
1024#endif
1025
1026 return irq;
1027}
1028
1029/*
1030 * Find a specific PCI IRQ entry.
1031 * Not an __init, possibly needed by modules
1032 */
1033int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
1034 struct io_apic_irq_attr *irq_attr)
1035{
1036 int ioapic_idx, i, best_guess = -1;
1037
1038 apic_printk(APIC_DEBUG,
1039 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1040 bus, slot, pin);
1041 if (test_bit(bus, mp_bus_not_pci)) {
1042 apic_printk(APIC_VERBOSE,
1043 "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1044 return -1;
1045 }
1046 for (i = 0; i < mp_irq_entries; i++) {
1047 int lbus = mp_irqs[i].srcbus;
1048
1049 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1050 if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic ||
1051 mp_irqs[i].dstapic == MP_APIC_ALL)
1052 break;
1053
1054 if (!test_bit(lbus, mp_bus_not_pci) &&
1055 !mp_irqs[i].irqtype &&
1056 (bus == lbus) &&
1057 (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1058 int irq = pin_2_irq(i, ioapic_idx, mp_irqs[i].dstirq);
1059
1060 if (!(ioapic_idx || IO_APIC_IRQ(irq)))
1061 continue;
1062
1063 if (pin == (mp_irqs[i].srcbusirq & 3)) {
1064 set_io_apic_irq_attr(irq_attr, ioapic_idx,
1065 mp_irqs[i].dstirq,
1066 irq_trigger(i),
1067 irq_polarity(i));
1068 return irq;
1069 }
1070 /*
1071 * Use the first all-but-pin matching entry as a
1072 * best-guess fuzzy result for broken mptables.
1073 */
1074 if (best_guess < 0) {
1075 set_io_apic_irq_attr(irq_attr, ioapic_idx,
1076 mp_irqs[i].dstirq,
1077 irq_trigger(i),
1078 irq_polarity(i));
1079 best_guess = irq;
1080 }
1081 }
1082 }
1083 return best_guess;
1084}
1085EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1086
1087void lock_vector_lock(void)
1088{
1089 /* Used to the online set of cpus does not change
1090 * during assign_irq_vector.
1091 */
1092 raw_spin_lock(&vector_lock);
1093}
1094
1095void unlock_vector_lock(void)
1096{
1097 raw_spin_unlock(&vector_lock);
1098}
1099
1100static int
1101__assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1102{
1103 /*
1104 * NOTE! The local APIC isn't very good at handling
1105 * multiple interrupts at the same interrupt level.
1106 * As the interrupt level is determined by taking the
1107 * vector number and shifting that right by 4, we
1108 * want to spread these out a bit so that they don't
1109 * all fall in the same interrupt level.
1110 *
1111 * Also, we've got to be careful not to trash gate
1112 * 0x80, because int 0x80 is hm, kind of importantish. ;)
1113 */
1114 static int current_vector = FIRST_EXTERNAL_VECTOR + VECTOR_OFFSET_START;
1115 static int current_offset = VECTOR_OFFSET_START % 8;
1116 unsigned int old_vector;
1117 int cpu, err;
1118 cpumask_var_t tmp_mask;
1119
1120 if (cfg->move_in_progress)
1121 return -EBUSY;
1122
1123 if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
1124 return -ENOMEM;
1125
1126 old_vector = cfg->vector;
1127 if (old_vector) {
1128 cpumask_and(tmp_mask, mask, cpu_online_mask);
1129 cpumask_and(tmp_mask, cfg->domain, tmp_mask);
1130 if (!cpumask_empty(tmp_mask)) {
1131 free_cpumask_var(tmp_mask);
1132 return 0;
1133 }
1134 }
1135
1136 /* Only try and allocate irqs on cpus that are present */
1137 err = -ENOSPC;
1138 for_each_cpu_and(cpu, mask, cpu_online_mask) {
1139 int new_cpu;
1140 int vector, offset;
1141
1142 apic->vector_allocation_domain(cpu, tmp_mask);
1143
1144 vector = current_vector;
1145 offset = current_offset;
1146next:
1147 vector += 8;
1148 if (vector >= first_system_vector) {
1149 /* If out of vectors on large boxen, must share them. */
1150 offset = (offset + 1) % 8;
1151 vector = FIRST_EXTERNAL_VECTOR + offset;
1152 }
1153 if (unlikely(current_vector == vector))
1154 continue;
1155
1156 if (test_bit(vector, used_vectors))
1157 goto next;
1158
1159 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1160 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
1161 goto next;
1162 /* Found one! */
1163 current_vector = vector;
1164 current_offset = offset;
1165 if (old_vector) {
1166 cfg->move_in_progress = 1;
1167 cpumask_copy(cfg->old_domain, cfg->domain);
1168 }
1169 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1170 per_cpu(vector_irq, new_cpu)[vector] = irq;
1171 cfg->vector = vector;
1172 cpumask_copy(cfg->domain, tmp_mask);
1173 err = 0;
1174 break;
1175 }
1176 free_cpumask_var(tmp_mask);
1177 return err;
1178}
1179
1180int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1181{
1182 int err;
1183 unsigned long flags;
1184
1185 raw_spin_lock_irqsave(&vector_lock, flags);
1186 err = __assign_irq_vector(irq, cfg, mask);
1187 raw_spin_unlock_irqrestore(&vector_lock, flags);
1188 return err;
1189}
1190
1191static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
1192{
1193 int cpu, vector;
1194
1195 BUG_ON(!cfg->vector);
1196
1197 vector = cfg->vector;
1198 for_each_cpu(cpu, cfg->domain)
1199 per_cpu(vector_irq, cpu)[vector] = -1;
1200
1201 cfg->vector = 0;
1202 cpumask_clear(cfg->domain);
1203
1204 if (likely(!cfg->move_in_progress))
1205 return;
1206 for_each_cpu(cpu, cfg->old_domain) {
1207 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
1208 vector++) {
1209 if (per_cpu(vector_irq, cpu)[vector] != irq)
1210 continue;
1211 per_cpu(vector_irq, cpu)[vector] = -1;
1212 break;
1213 }
1214 }
1215 cfg->move_in_progress = 0;
1216}
1217
1218void __setup_vector_irq(int cpu)
1219{
1220 /* Initialize vector_irq on a new cpu */
1221 int irq, vector;
1222 struct irq_cfg *cfg;
1223
1224 /*
1225 * vector_lock will make sure that we don't run into irq vector
1226 * assignments that might be happening on another cpu in parallel,
1227 * while we setup our initial vector to irq mappings.
1228 */
1229 raw_spin_lock(&vector_lock);
1230 /* Mark the inuse vectors */
1231 for_each_active_irq(irq) {
1232 cfg = irq_get_chip_data(irq);
1233 if (!cfg)
1234 continue;
1235 /*
1236 * If it is a legacy IRQ handled by the legacy PIC, this cpu
1237 * will be part of the irq_cfg's domain.
1238 */
1239 if (irq < legacy_pic->nr_legacy_irqs && !IO_APIC_IRQ(irq))
1240 cpumask_set_cpu(cpu, cfg->domain);
1241
1242 if (!cpumask_test_cpu(cpu, cfg->domain))
1243 continue;
1244 vector = cfg->vector;
1245 per_cpu(vector_irq, cpu)[vector] = irq;
1246 }
1247 /* Mark the free vectors */
1248 for (vector = 0; vector < NR_VECTORS; ++vector) {
1249 irq = per_cpu(vector_irq, cpu)[vector];
1250 if (irq < 0)
1251 continue;
1252
1253 cfg = irq_cfg(irq);
1254 if (!cpumask_test_cpu(cpu, cfg->domain))
1255 per_cpu(vector_irq, cpu)[vector] = -1;
1256 }
1257 raw_spin_unlock(&vector_lock);
1258}
1259
1260static struct irq_chip ioapic_chip;
1261
1262#ifdef CONFIG_X86_32
1263static inline int IO_APIC_irq_trigger(int irq)
1264{
1265 int apic, idx, pin;
1266
1267 for (apic = 0; apic < nr_ioapics; apic++) {
1268 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1269 idx = find_irq_entry(apic, pin, mp_INT);
1270 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1271 return irq_trigger(idx);
1272 }
1273 }
1274 /*
1275 * nonexistent IRQs are edge default
1276 */
1277 return 0;
1278}
1279#else
1280static inline int IO_APIC_irq_trigger(int irq)
1281{
1282 return 1;
1283}
1284#endif
1285
1286static void ioapic_register_intr(unsigned int irq, struct irq_cfg *cfg,
1287 unsigned long trigger)
1288{
1289 struct irq_chip *chip = &ioapic_chip;
1290 irq_flow_handler_t hdl;
1291 bool fasteoi;
1292
1293 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1294 trigger == IOAPIC_LEVEL) {
1295 irq_set_status_flags(irq, IRQ_LEVEL);
1296 fasteoi = true;
1297 } else {
1298 irq_clear_status_flags(irq, IRQ_LEVEL);
1299 fasteoi = false;
1300 }
1301
1302 if (irq_remapped(cfg)) {
1303 irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
1304 irq_remap_modify_chip_defaults(chip);
1305 fasteoi = trigger != 0;
1306 }
1307
1308 hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq;
1309 irq_set_chip_and_handler_name(irq, chip, hdl,
1310 fasteoi ? "fasteoi" : "edge");
1311}
1312
1313static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
1314 unsigned int destination, int vector,
1315 struct io_apic_irq_attr *attr)
1316{
1317 if (irq_remapping_enabled)
1318 return setup_ioapic_remapped_entry(irq, entry, destination,
1319 vector, attr);
1320
1321 memset(entry, 0, sizeof(*entry));
1322
1323 entry->delivery_mode = apic->irq_delivery_mode;
1324 entry->dest_mode = apic->irq_dest_mode;
1325 entry->dest = destination;
1326 entry->vector = vector;
1327 entry->mask = 0; /* enable IRQ */
1328 entry->trigger = attr->trigger;
1329 entry->polarity = attr->polarity;
1330
1331 /*
1332 * Mask level triggered irqs.
1333 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
1334 */
1335 if (attr->trigger)
1336 entry->mask = 1;
1337
1338 return 0;
1339}
1340
1341static void setup_ioapic_irq(unsigned int irq, struct irq_cfg *cfg,
1342 struct io_apic_irq_attr *attr)
1343{
1344 struct IO_APIC_route_entry entry;
1345 unsigned int dest;
1346
1347 if (!IO_APIC_IRQ(irq))
1348 return;
1349 /*
1350 * For legacy irqs, cfg->domain starts with cpu 0 for legacy
1351 * controllers like 8259. Now that IO-APIC can handle this irq, update
1352 * the cfg->domain.
1353 */
1354 if (irq < legacy_pic->nr_legacy_irqs && cpumask_test_cpu(0, cfg->domain))
1355 apic->vector_allocation_domain(0, cfg->domain);
1356
1357 if (assign_irq_vector(irq, cfg, apic->target_cpus()))
1358 return;
1359
1360 dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
1361
1362 apic_printk(APIC_VERBOSE,KERN_DEBUG
1363 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1364 "IRQ %d Mode:%i Active:%i Dest:%d)\n",
1365 attr->ioapic, mpc_ioapic_id(attr->ioapic), attr->ioapic_pin,
1366 cfg->vector, irq, attr->trigger, attr->polarity, dest);
1367
1368 if (setup_ioapic_entry(irq, &entry, dest, cfg->vector, attr)) {
1369 pr_warn("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1370 mpc_ioapic_id(attr->ioapic), attr->ioapic_pin);
1371 __clear_irq_vector(irq, cfg);
1372
1373 return;
1374 }
1375
1376 ioapic_register_intr(irq, cfg, attr->trigger);
1377 if (irq < legacy_pic->nr_legacy_irqs)
1378 legacy_pic->mask(irq);
1379
1380 ioapic_write_entry(attr->ioapic, attr->ioapic_pin, entry);
1381}
1382
1383static bool __init io_apic_pin_not_connected(int idx, int ioapic_idx, int pin)
1384{
1385 if (idx != -1)
1386 return false;
1387
1388 apic_printk(APIC_VERBOSE, KERN_DEBUG " apic %d pin %d not connected\n",
1389 mpc_ioapic_id(ioapic_idx), pin);
1390 return true;
1391}
1392
1393static void __init __io_apic_setup_irqs(unsigned int ioapic_idx)
1394{
1395 int idx, node = cpu_to_node(0);
1396 struct io_apic_irq_attr attr;
1397 unsigned int pin, irq;
1398
1399 for (pin = 0; pin < ioapics[ioapic_idx].nr_registers; pin++) {
1400 idx = find_irq_entry(ioapic_idx, pin, mp_INT);
1401 if (io_apic_pin_not_connected(idx, ioapic_idx, pin))
1402 continue;
1403
1404 irq = pin_2_irq(idx, ioapic_idx, pin);
1405
1406 if ((ioapic_idx > 0) && (irq > 16))
1407 continue;
1408
1409 /*
1410 * Skip the timer IRQ if there's a quirk handler
1411 * installed and if it returns 1:
1412 */
1413 if (apic->multi_timer_check &&
1414 apic->multi_timer_check(ioapic_idx, irq))
1415 continue;
1416
1417 set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1418 irq_polarity(idx));
1419
1420 io_apic_setup_irq_pin(irq, node, &attr);
1421 }
1422}
1423
1424static void __init setup_IO_APIC_irqs(void)
1425{
1426 unsigned int ioapic_idx;
1427
1428 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1429
1430 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1431 __io_apic_setup_irqs(ioapic_idx);
1432}
1433
1434/*
1435 * for the gsit that is not in first ioapic
1436 * but could not use acpi_register_gsi()
1437 * like some special sci in IBM x3330
1438 */
1439void setup_IO_APIC_irq_extra(u32 gsi)
1440{
1441 int ioapic_idx = 0, pin, idx, irq, node = cpu_to_node(0);
1442 struct io_apic_irq_attr attr;
1443
1444 /*
1445 * Convert 'gsi' to 'ioapic.pin'.
1446 */
1447 ioapic_idx = mp_find_ioapic(gsi);
1448 if (ioapic_idx < 0)
1449 return;
1450
1451 pin = mp_find_ioapic_pin(ioapic_idx, gsi);
1452 idx = find_irq_entry(ioapic_idx, pin, mp_INT);
1453 if (idx == -1)
1454 return;
1455
1456 irq = pin_2_irq(idx, ioapic_idx, pin);
1457
1458 /* Only handle the non legacy irqs on secondary ioapics */
1459 if (ioapic_idx == 0 || irq < NR_IRQS_LEGACY)
1460 return;
1461
1462 set_io_apic_irq_attr(&attr, ioapic_idx, pin, irq_trigger(idx),
1463 irq_polarity(idx));
1464
1465 io_apic_setup_irq_pin_once(irq, node, &attr);
1466}
1467
1468/*
1469 * Set up the timer pin, possibly with the 8259A-master behind.
1470 */
1471static void __init setup_timer_IRQ0_pin(unsigned int ioapic_idx,
1472 unsigned int pin, int vector)
1473{
1474 struct IO_APIC_route_entry entry;
1475
1476 if (irq_remapping_enabled)
1477 return;
1478
1479 memset(&entry, 0, sizeof(entry));
1480
1481 /*
1482 * We use logical delivery to get the timer IRQ
1483 * to the first CPU.
1484 */
1485 entry.dest_mode = apic->irq_dest_mode;
1486 entry.mask = 0; /* don't mask IRQ for edge */
1487 entry.dest = apic->cpu_mask_to_apicid(apic->target_cpus());
1488 entry.delivery_mode = apic->irq_delivery_mode;
1489 entry.polarity = 0;
1490 entry.trigger = 0;
1491 entry.vector = vector;
1492
1493 /*
1494 * The timer IRQ doesn't have to know that behind the
1495 * scene we may have a 8259A-master in AEOI mode ...
1496 */
1497 irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
1498 "edge");
1499
1500 /*
1501 * Add it to the IO-APIC irq-routing table:
1502 */
1503 ioapic_write_entry(ioapic_idx, pin, entry);
1504}
1505
1506__apicdebuginit(void) print_IO_APIC(int ioapic_idx)
1507{
1508 int i;
1509 union IO_APIC_reg_00 reg_00;
1510 union IO_APIC_reg_01 reg_01;
1511 union IO_APIC_reg_02 reg_02;
1512 union IO_APIC_reg_03 reg_03;
1513 unsigned long flags;
1514
1515 raw_spin_lock_irqsave(&ioapic_lock, flags);
1516 reg_00.raw = io_apic_read(ioapic_idx, 0);
1517 reg_01.raw = io_apic_read(ioapic_idx, 1);
1518 if (reg_01.bits.version >= 0x10)
1519 reg_02.raw = io_apic_read(ioapic_idx, 2);
1520 if (reg_01.bits.version >= 0x20)
1521 reg_03.raw = io_apic_read(ioapic_idx, 3);
1522 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1523
1524 printk("\n");
1525 printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(ioapic_idx));
1526 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1527 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1528 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
1529 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
1530
1531 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)®_01);
1532 printk(KERN_DEBUG "....... : max redirection entries: %02X\n",
1533 reg_01.bits.entries);
1534
1535 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
1536 printk(KERN_DEBUG "....... : IO APIC version: %02X\n",
1537 reg_01.bits.version);
1538
1539 /*
1540 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
1541 * but the value of reg_02 is read as the previous read register
1542 * value, so ignore it if reg_02 == reg_01.
1543 */
1544 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1545 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1546 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
1547 }
1548
1549 /*
1550 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
1551 * or reg_03, but the value of reg_0[23] is read as the previous read
1552 * register value, so ignore it if reg_03 == reg_0[12].
1553 */
1554 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1555 reg_03.raw != reg_01.raw) {
1556 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1557 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
1558 }
1559
1560 printk(KERN_DEBUG ".... IRQ redirection table:\n");
1561
1562 if (irq_remapping_enabled) {
1563 printk(KERN_DEBUG " NR Indx Fmt Mask Trig IRR"
1564 " Pol Stat Indx2 Zero Vect:\n");
1565 } else {
1566 printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
1567 " Stat Dmod Deli Vect:\n");
1568 }
1569
1570 for (i = 0; i <= reg_01.bits.entries; i++) {
1571 if (irq_remapping_enabled) {
1572 struct IO_APIC_route_entry entry;
1573 struct IR_IO_APIC_route_entry *ir_entry;
1574
1575 entry = ioapic_read_entry(ioapic_idx, i);
1576 ir_entry = (struct IR_IO_APIC_route_entry *) &entry;
1577 printk(KERN_DEBUG " %02x %04X ",
1578 i,
1579 ir_entry->index
1580 );
1581 printk("%1d %1d %1d %1d %1d "
1582 "%1d %1d %X %02X\n",
1583 ir_entry->format,
1584 ir_entry->mask,
1585 ir_entry->trigger,
1586 ir_entry->irr,
1587 ir_entry->polarity,
1588 ir_entry->delivery_status,
1589 ir_entry->index2,
1590 ir_entry->zero,
1591 ir_entry->vector
1592 );
1593 } else {
1594 struct IO_APIC_route_entry entry;
1595
1596 entry = ioapic_read_entry(ioapic_idx, i);
1597 printk(KERN_DEBUG " %02x %02X ",
1598 i,
1599 entry.dest
1600 );
1601 printk("%1d %1d %1d %1d %1d "
1602 "%1d %1d %02X\n",
1603 entry.mask,
1604 entry.trigger,
1605 entry.irr,
1606 entry.polarity,
1607 entry.delivery_status,
1608 entry.dest_mode,
1609 entry.delivery_mode,
1610 entry.vector
1611 );
1612 }
1613 }
1614}
1615
1616__apicdebuginit(void) print_IO_APICs(void)
1617{
1618 int ioapic_idx;
1619 struct irq_cfg *cfg;
1620 unsigned int irq;
1621 struct irq_chip *chip;
1622
1623 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1624 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1625 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1626 mpc_ioapic_id(ioapic_idx),
1627 ioapics[ioapic_idx].nr_registers);
1628
1629 /*
1630 * We are a bit conservative about what we expect. We have to
1631 * know about every hardware change ASAP.
1632 */
1633 printk(KERN_INFO "testing the IO APIC.......................\n");
1634
1635 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++)
1636 print_IO_APIC(ioapic_idx);
1637
1638 printk(KERN_DEBUG "IRQ to pin mappings:\n");
1639 for_each_active_irq(irq) {
1640 struct irq_pin_list *entry;
1641
1642 chip = irq_get_chip(irq);
1643 if (chip != &ioapic_chip)
1644 continue;
1645
1646 cfg = irq_get_chip_data(irq);
1647 if (!cfg)
1648 continue;
1649 entry = cfg->irq_2_pin;
1650 if (!entry)
1651 continue;
1652 printk(KERN_DEBUG "IRQ%d ", irq);
1653 for_each_irq_pin(entry, cfg->irq_2_pin)
1654 printk("-> %d:%d", entry->apic, entry->pin);
1655 printk("\n");
1656 }
1657
1658 printk(KERN_INFO ".................................... done.\n");
1659}
1660
1661__apicdebuginit(void) print_APIC_field(int base)
1662{
1663 int i;
1664
1665 printk(KERN_DEBUG);
1666
1667 for (i = 0; i < 8; i++)
1668 printk(KERN_CONT "%08x", apic_read(base + i*0x10));
1669
1670 printk(KERN_CONT "\n");
1671}
1672
1673__apicdebuginit(void) print_local_APIC(void *dummy)
1674{
1675 unsigned int i, v, ver, maxlvt;
1676 u64 icr;
1677
1678 printk(KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1679 smp_processor_id(), hard_smp_processor_id());
1680 v = apic_read(APIC_ID);
1681 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, read_apic_id());
1682 v = apic_read(APIC_LVR);
1683 printk(KERN_INFO "... APIC VERSION: %08x\n", v);
1684 ver = GET_APIC_VERSION(v);
1685 maxlvt = lapic_get_maxlvt();
1686
1687 v = apic_read(APIC_TASKPRI);
1688 printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1689
1690 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1691 if (!APIC_XAPIC(ver)) {
1692 v = apic_read(APIC_ARBPRI);
1693 printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
1694 v & APIC_ARBPRI_MASK);
1695 }
1696 v = apic_read(APIC_PROCPRI);
1697 printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
1698 }
1699
1700 /*
1701 * Remote read supported only in the 82489DX and local APIC for
1702 * Pentium processors.
1703 */
1704 if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
1705 v = apic_read(APIC_RRR);
1706 printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
1707 }
1708
1709 v = apic_read(APIC_LDR);
1710 printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
1711 if (!x2apic_enabled()) {
1712 v = apic_read(APIC_DFR);
1713 printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
1714 }
1715 v = apic_read(APIC_SPIV);
1716 printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);
1717
1718 printk(KERN_DEBUG "... APIC ISR field:\n");
1719 print_APIC_field(APIC_ISR);
1720 printk(KERN_DEBUG "... APIC TMR field:\n");
1721 print_APIC_field(APIC_TMR);
1722 printk(KERN_DEBUG "... APIC IRR field:\n");
1723 print_APIC_field(APIC_IRR);
1724
1725 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1726 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1727 apic_write(APIC_ESR, 0);
1728
1729 v = apic_read(APIC_ESR);
1730 printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
1731 }
1732
1733 icr = apic_icr_read();
1734 printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
1735 printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
1736
1737 v = apic_read(APIC_LVTT);
1738 printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);
1739
1740 if (maxlvt > 3) { /* PC is LVT#4. */
1741 v = apic_read(APIC_LVTPC);
1742 printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
1743 }
1744 v = apic_read(APIC_LVT0);
1745 printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
1746 v = apic_read(APIC_LVT1);
1747 printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);
1748
1749 if (maxlvt > 2) { /* ERR is LVT#3. */
1750 v = apic_read(APIC_LVTERR);
1751 printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
1752 }
1753
1754 v = apic_read(APIC_TMICT);
1755 printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
1756 v = apic_read(APIC_TMCCT);
1757 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1758 v = apic_read(APIC_TDCR);
1759 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1760
1761 if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
1762 v = apic_read(APIC_EFEAT);
1763 maxlvt = (v >> 16) & 0xff;
1764 printk(KERN_DEBUG "... APIC EFEAT: %08x\n", v);
1765 v = apic_read(APIC_ECTRL);
1766 printk(KERN_DEBUG "... APIC ECTRL: %08x\n", v);
1767 for (i = 0; i < maxlvt; i++) {
1768 v = apic_read(APIC_EILVTn(i));
1769 printk(KERN_DEBUG "... APIC EILVT%d: %08x\n", i, v);
1770 }
1771 }
1772 printk("\n");
1773}
1774
1775__apicdebuginit(void) print_local_APICs(int maxcpu)
1776{
1777 int cpu;
1778
1779 if (!maxcpu)
1780 return;
1781
1782 preempt_disable();
1783 for_each_online_cpu(cpu) {
1784 if (cpu >= maxcpu)
1785 break;
1786 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1787 }
1788 preempt_enable();
1789}
1790
1791__apicdebuginit(void) print_PIC(void)
1792{
1793 unsigned int v;
1794 unsigned long flags;
1795
1796 if (!legacy_pic->nr_legacy_irqs)
1797 return;
1798
1799 printk(KERN_DEBUG "\nprinting PIC contents\n");
1800
1801 raw_spin_lock_irqsave(&i8259A_lock, flags);
1802
1803 v = inb(0xa1) << 8 | inb(0x21);
1804 printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
1805
1806 v = inb(0xa0) << 8 | inb(0x20);
1807 printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
1808
1809 outb(0x0b,0xa0);
1810 outb(0x0b,0x20);
1811 v = inb(0xa0) << 8 | inb(0x20);
1812 outb(0x0a,0xa0);
1813 outb(0x0a,0x20);
1814
1815 raw_spin_unlock_irqrestore(&i8259A_lock, flags);
1816
1817 printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
1818
1819 v = inb(0x4d1) << 8 | inb(0x4d0);
1820 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
1821}
1822
1823static int __initdata show_lapic = 1;
1824static __init int setup_show_lapic(char *arg)
1825{
1826 int num = -1;
1827
1828 if (strcmp(arg, "all") == 0) {
1829 show_lapic = CONFIG_NR_CPUS;
1830 } else {
1831 get_option(&arg, &num);
1832 if (num >= 0)
1833 show_lapic = num;
1834 }
1835
1836 return 1;
1837}
1838__setup("show_lapic=", setup_show_lapic);
1839
1840__apicdebuginit(int) print_ICs(void)
1841{
1842 if (apic_verbosity == APIC_QUIET)
1843 return 0;
1844
1845 print_PIC();
1846
1847 /* don't print out if apic is not there */
1848 if (!cpu_has_apic && !apic_from_smp_config())
1849 return 0;
1850
1851 print_local_APICs(show_lapic);
1852 print_IO_APICs();
1853
1854 return 0;
1855}
1856
1857late_initcall(print_ICs);
1858
1859
1860/* Where if anywhere is the i8259 connect in external int mode */
1861static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1862
1863void __init enable_IO_APIC(void)
1864{
1865 int i8259_apic, i8259_pin;
1866 int apic;
1867
1868 if (!legacy_pic->nr_legacy_irqs)
1869 return;
1870
1871 for(apic = 0; apic < nr_ioapics; apic++) {
1872 int pin;
1873 /* See if any of the pins is in ExtINT mode */
1874 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1875 struct IO_APIC_route_entry entry;
1876 entry = ioapic_read_entry(apic, pin);
1877
1878 /* If the interrupt line is enabled and in ExtInt mode
1879 * I have found the pin where the i8259 is connected.
1880 */
1881 if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
1882 ioapic_i8259.apic = apic;
1883 ioapic_i8259.pin = pin;
1884 goto found_i8259;
1885 }
1886 }
1887 }
1888 found_i8259:
1889 /* Look to see what if the MP table has reported the ExtINT */
1890 /* If we could not find the appropriate pin by looking at the ioapic
1891 * the i8259 probably is not connected the ioapic but give the
1892 * mptable a chance anyway.
1893 */
1894 i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
1895 i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1896 /* Trust the MP table if nothing is setup in the hardware */
1897 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1898 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1899 ioapic_i8259.pin = i8259_pin;
1900 ioapic_i8259.apic = i8259_apic;
1901 }
1902 /* Complain if the MP table and the hardware disagree */
1903 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1904 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1905 {
1906 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
1907 }
1908
1909 /*
1910 * Do not trust the IO-APIC being empty at bootup
1911 */
1912 clear_IO_APIC();
1913}
1914
1915/*
1916 * Not an __init, needed by the reboot code
1917 */
1918void disable_IO_APIC(void)
1919{
1920 /*
1921 * Clear the IO-APIC before rebooting:
1922 */
1923 clear_IO_APIC();
1924
1925 if (!legacy_pic->nr_legacy_irqs)
1926 return;
1927
1928 /*
1929 * If the i8259 is routed through an IOAPIC
1930 * Put that IOAPIC in virtual wire mode
1931 * so legacy interrupts can be delivered.
1932 *
1933 * With interrupt-remapping, for now we will use virtual wire A mode,
1934 * as virtual wire B is little complex (need to configure both
1935 * IOAPIC RTE as well as interrupt-remapping table entry).
1936 * As this gets called during crash dump, keep this simple for now.
1937 */
1938 if (ioapic_i8259.pin != -1 && !irq_remapping_enabled) {
1939 struct IO_APIC_route_entry entry;
1940
1941 memset(&entry, 0, sizeof(entry));
1942 entry.mask = 0; /* Enabled */
1943 entry.trigger = 0; /* Edge */
1944 entry.irr = 0;
1945 entry.polarity = 0; /* High */
1946 entry.delivery_status = 0;
1947 entry.dest_mode = 0; /* Physical */
1948 entry.delivery_mode = dest_ExtINT; /* ExtInt */
1949 entry.vector = 0;
1950 entry.dest = read_apic_id();
1951
1952 /*
1953 * Add it to the IO-APIC irq-routing table:
1954 */
1955 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1956 }
1957
1958 /*
1959 * Use virtual wire A mode when interrupt remapping is enabled.
1960 */
1961 if (cpu_has_apic || apic_from_smp_config())
1962 disconnect_bsp_APIC(!irq_remapping_enabled &&
1963 ioapic_i8259.pin != -1);
1964}
1965
1966#ifdef CONFIG_X86_32
1967/*
1968 * function to set the IO-APIC physical IDs based on the
1969 * values stored in the MPC table.
1970 *
1971 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999
1972 */
1973void __init setup_ioapic_ids_from_mpc_nocheck(void)
1974{
1975 union IO_APIC_reg_00 reg_00;
1976 physid_mask_t phys_id_present_map;
1977 int ioapic_idx;
1978 int i;
1979 unsigned char old_id;
1980 unsigned long flags;
1981
1982 /*
1983 * This is broken; anything with a real cpu count has to
1984 * circumvent this idiocy regardless.
1985 */
1986 apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map);
1987
1988 /*
1989 * Set the IOAPIC ID to the value stored in the MPC table.
1990 */
1991 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) {
1992 /* Read the register 0 value */
1993 raw_spin_lock_irqsave(&ioapic_lock, flags);
1994 reg_00.raw = io_apic_read(ioapic_idx, 0);
1995 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1996
1997 old_id = mpc_ioapic_id(ioapic_idx);
1998
1999 if (mpc_ioapic_id(ioapic_idx) >= get_physical_broadcast()) {
2000 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
2001 ioapic_idx, mpc_ioapic_id(ioapic_idx));
2002 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2003 reg_00.bits.ID);
2004 ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID;
2005 }
2006
2007 /*
2008 * Sanity check, is the ID really free? Every APIC in a
2009 * system must have a unique ID or we get lots of nice
2010 * 'stuck on smp_invalidate_needed IPI wait' messages.
2011 */
2012 if (apic->check_apicid_used(&phys_id_present_map,
2013 mpc_ioapic_id(ioapic_idx))) {
2014 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
2015 ioapic_idx, mpc_ioapic_id(ioapic_idx));
2016 for (i = 0; i < get_physical_broadcast(); i++)
2017 if (!physid_isset(i, phys_id_present_map))
2018 break;
2019 if (i >= get_physical_broadcast())
2020 panic("Max APIC ID exceeded!\n");
2021 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2022 i);
2023 physid_set(i, phys_id_present_map);
2024 ioapics[ioapic_idx].mp_config.apicid = i;
2025 } else {
2026 physid_mask_t tmp;
2027 apic->apicid_to_cpu_present(mpc_ioapic_id(ioapic_idx),
2028 &tmp);
2029 apic_printk(APIC_VERBOSE, "Setting %d in the "
2030 "phys_id_present_map\n",
2031 mpc_ioapic_id(ioapic_idx));
2032 physids_or(phys_id_present_map, phys_id_present_map, tmp);
2033 }
2034
2035 /*
2036 * We need to adjust the IRQ routing table
2037 * if the ID changed.
2038 */
2039 if (old_id != mpc_ioapic_id(ioapic_idx))
2040 for (i = 0; i < mp_irq_entries; i++)
2041 if (mp_irqs[i].dstapic == old_id)
2042 mp_irqs[i].dstapic
2043 = mpc_ioapic_id(ioapic_idx);
2044
2045 /*
2046 * Update the ID register according to the right value
2047 * from the MPC table if they are different.
2048 */
2049 if (mpc_ioapic_id(ioapic_idx) == reg_00.bits.ID)
2050 continue;
2051
2052 apic_printk(APIC_VERBOSE, KERN_INFO
2053 "...changing IO-APIC physical APIC ID to %d ...",
2054 mpc_ioapic_id(ioapic_idx));
2055
2056 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2057 raw_spin_lock_irqsave(&ioapic_lock, flags);
2058 io_apic_write(ioapic_idx, 0, reg_00.raw);
2059 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2060
2061 /*
2062 * Sanity check
2063 */
2064 raw_spin_lock_irqsave(&ioapic_lock, flags);
2065 reg_00.raw = io_apic_read(ioapic_idx, 0);
2066 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2067 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx))
2068 printk("could not set ID!\n");
2069 else
2070 apic_printk(APIC_VERBOSE, " ok.\n");
2071 }
2072}
2073
2074void __init setup_ioapic_ids_from_mpc(void)
2075{
2076
2077 if (acpi_ioapic)
2078 return;
2079 /*
2080 * Don't check I/O APIC IDs for xAPIC systems. They have
2081 * no meaning without the serial APIC bus.
2082 */
2083 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2084 || APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
2085 return;
2086 setup_ioapic_ids_from_mpc_nocheck();
2087}
2088#endif
2089
2090int no_timer_check __initdata;
2091
2092static int __init notimercheck(char *s)
2093{
2094 no_timer_check = 1;
2095 return 1;
2096}
2097__setup("no_timer_check", notimercheck);
2098
2099/*
2100 * There is a nasty bug in some older SMP boards, their mptable lies
2101 * about the timer IRQ. We do the following to work around the situation:
2102 *
2103 * - timer IRQ defaults to IO-APIC IRQ
2104 * - if this function detects that timer IRQs are defunct, then we fall
2105 * back to ISA timer IRQs
2106 */
2107static int __init timer_irq_works(void)
2108{
2109 unsigned long t1 = jiffies;
2110 unsigned long flags;
2111
2112 if (no_timer_check)
2113 return 1;
2114
2115 local_save_flags(flags);
2116 local_irq_enable();
2117 /* Let ten ticks pass... */
2118 mdelay((10 * 1000) / HZ);
2119 local_irq_restore(flags);
2120
2121 /*
2122 * Expect a few ticks at least, to be sure some possible
2123 * glue logic does not lock up after one or two first
2124 * ticks in a non-ExtINT mode. Also the local APIC
2125 * might have cached one ExtINT interrupt. Finally, at
2126 * least one tick may be lost due to delays.
2127 */
2128
2129 /* jiffies wrap? */
2130 if (time_after(jiffies, t1 + 4))
2131 return 1;
2132 return 0;
2133}
2134
2135/*
2136 * In the SMP+IOAPIC case it might happen that there are an unspecified
2137 * number of pending IRQ events unhandled. These cases are very rare,
2138 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
2139 * better to do it this way as thus we do not have to be aware of
2140 * 'pending' interrupts in the IRQ path, except at this point.
2141 */
2142/*
2143 * Edge triggered needs to resend any interrupt
2144 * that was delayed but this is now handled in the device
2145 * independent code.
2146 */
2147
2148/*
2149 * Starting up a edge-triggered IO-APIC interrupt is
2150 * nasty - we need to make sure that we get the edge.
2151 * If it is already asserted for some reason, we need
2152 * return 1 to indicate that is was pending.
2153 *
2154 * This is not complete - we should be able to fake
2155 * an edge even if it isn't on the 8259A...
2156 */
2157
2158static unsigned int startup_ioapic_irq(struct irq_data *data)
2159{
2160 int was_pending = 0, irq = data->irq;
2161 unsigned long flags;
2162
2163 raw_spin_lock_irqsave(&ioapic_lock, flags);
2164 if (irq < legacy_pic->nr_legacy_irqs) {
2165 legacy_pic->mask(irq);
2166 if (legacy_pic->irq_pending(irq))
2167 was_pending = 1;
2168 }
2169 __unmask_ioapic(data->chip_data);
2170 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2171
2172 return was_pending;
2173}
2174
2175static int ioapic_retrigger_irq(struct irq_data *data)
2176{
2177 struct irq_cfg *cfg = data->chip_data;
2178 unsigned long flags;
2179
2180 raw_spin_lock_irqsave(&vector_lock, flags);
2181 apic->send_IPI_mask(cpumask_of(cpumask_first(cfg->domain)), cfg->vector);
2182 raw_spin_unlock_irqrestore(&vector_lock, flags);
2183
2184 return 1;
2185}
2186
2187/*
2188 * Level and edge triggered IO-APIC interrupts need different handling,
2189 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
2190 * handled with the level-triggered descriptor, but that one has slightly
2191 * more overhead. Level-triggered interrupts cannot be handled with the
2192 * edge-triggered handler, without risking IRQ storms and other ugly
2193 * races.
2194 */
2195
2196#ifdef CONFIG_SMP
2197void send_cleanup_vector(struct irq_cfg *cfg)
2198{
2199 cpumask_var_t cleanup_mask;
2200
2201 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
2202 unsigned int i;
2203 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2204 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
2205 } else {
2206 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
2207 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2208 free_cpumask_var(cleanup_mask);
2209 }
2210 cfg->move_in_progress = 0;
2211}
2212
2213static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
2214{
2215 int apic, pin;
2216 struct irq_pin_list *entry;
2217 u8 vector = cfg->vector;
2218
2219 for_each_irq_pin(entry, cfg->irq_2_pin) {
2220 unsigned int reg;
2221
2222 apic = entry->apic;
2223 pin = entry->pin;
2224 /*
2225 * With interrupt-remapping, destination information comes
2226 * from interrupt-remapping table entry.
2227 */
2228 if (!irq_remapped(cfg))
2229 io_apic_write(apic, 0x11 + pin*2, dest);
2230 reg = io_apic_read(apic, 0x10 + pin*2);
2231 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
2232 reg |= vector;
2233 io_apic_modify(apic, 0x10 + pin*2, reg);
2234 }
2235}
2236
2237/*
2238 * Either sets data->affinity to a valid value, and returns
2239 * ->cpu_mask_to_apicid of that in dest_id, or returns -1 and
2240 * leaves data->affinity untouched.
2241 */
2242int __ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
2243 unsigned int *dest_id)
2244{
2245 struct irq_cfg *cfg = data->chip_data;
2246
2247 if (!cpumask_intersects(mask, cpu_online_mask))
2248 return -1;
2249
2250 if (assign_irq_vector(data->irq, data->chip_data, mask))
2251 return -1;
2252
2253 cpumask_copy(data->affinity, mask);
2254
2255 *dest_id = apic->cpu_mask_to_apicid_and(mask, cfg->domain);
2256 return 0;
2257}
2258
2259static int
2260ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
2261 bool force)
2262{
2263 unsigned int dest, irq = data->irq;
2264 unsigned long flags;
2265 int ret;
2266
2267 raw_spin_lock_irqsave(&ioapic_lock, flags);
2268 ret = __ioapic_set_affinity(data, mask, &dest);
2269 if (!ret) {
2270 /* Only the high 8 bits are valid. */
2271 dest = SET_APIC_LOGICAL_ID(dest);
2272 __target_IO_APIC_irq(irq, dest, data->chip_data);
2273 }
2274 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2275 return ret;
2276}
2277
2278asmlinkage void smp_irq_move_cleanup_interrupt(void)
2279{
2280 unsigned vector, me;
2281
2282 ack_APIC_irq();
2283 irq_enter();
2284 exit_idle();
2285
2286 me = smp_processor_id();
2287 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
2288 unsigned int irq;
2289 unsigned int irr;
2290 struct irq_desc *desc;
2291 struct irq_cfg *cfg;
2292 irq = __this_cpu_read(vector_irq[vector]);
2293
2294 if (irq == -1)
2295 continue;
2296
2297 desc = irq_to_desc(irq);
2298 if (!desc)
2299 continue;
2300
2301 cfg = irq_cfg(irq);
2302 raw_spin_lock(&desc->lock);
2303
2304 /*
2305 * Check if the irq migration is in progress. If so, we
2306 * haven't received the cleanup request yet for this irq.
2307 */
2308 if (cfg->move_in_progress)
2309 goto unlock;
2310
2311 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2312 goto unlock;
2313
2314 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
2315 /*
2316 * Check if the vector that needs to be cleanedup is
2317 * registered at the cpu's IRR. If so, then this is not
2318 * the best time to clean it up. Lets clean it up in the
2319 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
2320 * to myself.
2321 */
2322 if (irr & (1 << (vector % 32))) {
2323 apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
2324 goto unlock;
2325 }
2326 __this_cpu_write(vector_irq[vector], -1);
2327unlock:
2328 raw_spin_unlock(&desc->lock);
2329 }
2330
2331 irq_exit();
2332}
2333
2334static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
2335{
2336 unsigned me;
2337
2338 if (likely(!cfg->move_in_progress))
2339 return;
2340
2341 me = smp_processor_id();
2342
2343 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2344 send_cleanup_vector(cfg);
2345}
2346
2347static void irq_complete_move(struct irq_cfg *cfg)
2348{
2349 __irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
2350}
2351
2352void irq_force_complete_move(int irq)
2353{
2354 struct irq_cfg *cfg = irq_get_chip_data(irq);
2355
2356 if (!cfg)
2357 return;
2358
2359 __irq_complete_move(cfg, cfg->vector);
2360}
2361#else
2362static inline void irq_complete_move(struct irq_cfg *cfg) { }
2363#endif
2364
2365static void ack_apic_edge(struct irq_data *data)
2366{
2367 irq_complete_move(data->chip_data);
2368 irq_move_irq(data);
2369 ack_APIC_irq();
2370}
2371
2372atomic_t irq_mis_count;
2373
2374#ifdef CONFIG_GENERIC_PENDING_IRQ
2375static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
2376{
2377 struct irq_pin_list *entry;
2378 unsigned long flags;
2379
2380 raw_spin_lock_irqsave(&ioapic_lock, flags);
2381 for_each_irq_pin(entry, cfg->irq_2_pin) {
2382 unsigned int reg;
2383 int pin;
2384
2385 pin = entry->pin;
2386 reg = io_apic_read(entry->apic, 0x10 + pin*2);
2387 /* Is the remote IRR bit set? */
2388 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
2389 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2390 return true;
2391 }
2392 }
2393 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2394
2395 return false;
2396}
2397
2398static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg)
2399{
2400 /* If we are moving the irq we need to mask it */
2401 if (unlikely(irqd_is_setaffinity_pending(data))) {
2402 mask_ioapic(cfg);
2403 return true;
2404 }
2405 return false;
2406}
2407
2408static inline void ioapic_irqd_unmask(struct irq_data *data,
2409 struct irq_cfg *cfg, bool masked)
2410{
2411 if (unlikely(masked)) {
2412 /* Only migrate the irq if the ack has been received.
2413 *
2414 * On rare occasions the broadcast level triggered ack gets
2415 * delayed going to ioapics, and if we reprogram the
2416 * vector while Remote IRR is still set the irq will never
2417 * fire again.
2418 *
2419 * To prevent this scenario we read the Remote IRR bit
2420 * of the ioapic. This has two effects.
2421 * - On any sane system the read of the ioapic will
2422 * flush writes (and acks) going to the ioapic from
2423 * this cpu.
2424 * - We get to see if the ACK has actually been delivered.
2425 *
2426 * Based on failed experiments of reprogramming the
2427 * ioapic entry from outside of irq context starting
2428 * with masking the ioapic entry and then polling until
2429 * Remote IRR was clear before reprogramming the
2430 * ioapic I don't trust the Remote IRR bit to be
2431 * completey accurate.
2432 *
2433 * However there appears to be no other way to plug
2434 * this race, so if the Remote IRR bit is not
2435 * accurate and is causing problems then it is a hardware bug
2436 * and you can go talk to the chipset vendor about it.
2437 */
2438 if (!io_apic_level_ack_pending(cfg))
2439 irq_move_masked_irq(data);
2440 unmask_ioapic(cfg);
2441 }
2442}
2443#else
2444static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg)
2445{
2446 return false;
2447}
2448static inline void ioapic_irqd_unmask(struct irq_data *data,
2449 struct irq_cfg *cfg, bool masked)
2450{
2451}
2452#endif
2453
2454static void ack_apic_level(struct irq_data *data)
2455{
2456 struct irq_cfg *cfg = data->chip_data;
2457 int i, irq = data->irq;
2458 unsigned long v;
2459 bool masked;
2460
2461 irq_complete_move(cfg);
2462 masked = ioapic_irqd_mask(data, cfg);
2463
2464 /*
2465 * It appears there is an erratum which affects at least version 0x11
2466 * of I/O APIC (that's the 82093AA and cores integrated into various
2467 * chipsets). Under certain conditions a level-triggered interrupt is
2468 * erroneously delivered as edge-triggered one but the respective IRR
2469 * bit gets set nevertheless. As a result the I/O unit expects an EOI
2470 * message but it will never arrive and further interrupts are blocked
2471 * from the source. The exact reason is so far unknown, but the
2472 * phenomenon was observed when two consecutive interrupt requests
2473 * from a given source get delivered to the same CPU and the source is
2474 * temporarily disabled in between.
2475 *
2476 * A workaround is to simulate an EOI message manually. We achieve it
2477 * by setting the trigger mode to edge and then to level when the edge
2478 * trigger mode gets detected in the TMR of a local APIC for a
2479 * level-triggered interrupt. We mask the source for the time of the
2480 * operation to prevent an edge-triggered interrupt escaping meanwhile.
2481 * The idea is from Manfred Spraul. --macro
2482 *
2483 * Also in the case when cpu goes offline, fixup_irqs() will forward
2484 * any unhandled interrupt on the offlined cpu to the new cpu
2485 * destination that is handling the corresponding interrupt. This
2486 * interrupt forwarding is done via IPI's. Hence, in this case also
2487 * level-triggered io-apic interrupt will be seen as an edge
2488 * interrupt in the IRR. And we can't rely on the cpu's EOI
2489 * to be broadcasted to the IO-APIC's which will clear the remoteIRR
2490 * corresponding to the level-triggered interrupt. Hence on IO-APIC's
2491 * supporting EOI register, we do an explicit EOI to clear the
2492 * remote IRR and on IO-APIC's which don't have an EOI register,
2493 * we use the above logic (mask+edge followed by unmask+level) from
2494 * Manfred Spraul to clear the remote IRR.
2495 */
2496 i = cfg->vector;
2497 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2498
2499 /*
2500 * We must acknowledge the irq before we move it or the acknowledge will
2501 * not propagate properly.
2502 */
2503 ack_APIC_irq();
2504
2505 /*
2506 * Tail end of clearing remote IRR bit (either by delivering the EOI
2507 * message via io-apic EOI register write or simulating it using
2508 * mask+edge followed by unnask+level logic) manually when the
2509 * level triggered interrupt is seen as the edge triggered interrupt
2510 * at the cpu.
2511 */
2512 if (!(v & (1 << (i & 0x1f)))) {
2513 atomic_inc(&irq_mis_count);
2514
2515 eoi_ioapic_irq(irq, cfg);
2516 }
2517
2518 ioapic_irqd_unmask(data, cfg, masked);
2519}
2520
2521#ifdef CONFIG_IRQ_REMAP
2522static void ir_ack_apic_edge(struct irq_data *data)
2523{
2524 ack_APIC_irq();
2525}
2526
2527static void ir_ack_apic_level(struct irq_data *data)
2528{
2529 ack_APIC_irq();
2530 eoi_ioapic_irq(data->irq, data->chip_data);
2531}
2532
2533static void ir_print_prefix(struct irq_data *data, struct seq_file *p)
2534{
2535 seq_printf(p, " IR-%s", data->chip->name);
2536}
2537
2538static void irq_remap_modify_chip_defaults(struct irq_chip *chip)
2539{
2540 chip->irq_print_chip = ir_print_prefix;
2541 chip->irq_ack = ir_ack_apic_edge;
2542 chip->irq_eoi = ir_ack_apic_level;
2543
2544#ifdef CONFIG_SMP
2545 chip->irq_set_affinity = set_remapped_irq_affinity;
2546#endif
2547}
2548#endif /* CONFIG_IRQ_REMAP */
2549
2550static struct irq_chip ioapic_chip __read_mostly = {
2551 .name = "IO-APIC",
2552 .irq_startup = startup_ioapic_irq,
2553 .irq_mask = mask_ioapic_irq,
2554 .irq_unmask = unmask_ioapic_irq,
2555 .irq_ack = ack_apic_edge,
2556 .irq_eoi = ack_apic_level,
2557#ifdef CONFIG_SMP
2558 .irq_set_affinity = ioapic_set_affinity,
2559#endif
2560 .irq_retrigger = ioapic_retrigger_irq,
2561};
2562
2563static inline void init_IO_APIC_traps(void)
2564{
2565 struct irq_cfg *cfg;
2566 unsigned int irq;
2567
2568 /*
2569 * NOTE! The local APIC isn't very good at handling
2570 * multiple interrupts at the same interrupt level.
2571 * As the interrupt level is determined by taking the
2572 * vector number and shifting that right by 4, we
2573 * want to spread these out a bit so that they don't
2574 * all fall in the same interrupt level.
2575 *
2576 * Also, we've got to be careful not to trash gate
2577 * 0x80, because int 0x80 is hm, kind of importantish. ;)
2578 */
2579 for_each_active_irq(irq) {
2580 cfg = irq_get_chip_data(irq);
2581 if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
2582 /*
2583 * Hmm.. We don't have an entry for this,
2584 * so default to an old-fashioned 8259
2585 * interrupt if we can..
2586 */
2587 if (irq < legacy_pic->nr_legacy_irqs)
2588 legacy_pic->make_irq(irq);
2589 else
2590 /* Strange. Oh, well.. */
2591 irq_set_chip(irq, &no_irq_chip);
2592 }
2593 }
2594}
2595
2596/*
2597 * The local APIC irq-chip implementation:
2598 */
2599
2600static void mask_lapic_irq(struct irq_data *data)
2601{
2602 unsigned long v;
2603
2604 v = apic_read(APIC_LVT0);
2605 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
2606}
2607
2608static void unmask_lapic_irq(struct irq_data *data)
2609{
2610 unsigned long v;
2611
2612 v = apic_read(APIC_LVT0);
2613 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2614}
2615
2616static void ack_lapic_irq(struct irq_data *data)
2617{
2618 ack_APIC_irq();
2619}
2620
2621static struct irq_chip lapic_chip __read_mostly = {
2622 .name = "local-APIC",
2623 .irq_mask = mask_lapic_irq,
2624 .irq_unmask = unmask_lapic_irq,
2625 .irq_ack = ack_lapic_irq,
2626};
2627
2628static void lapic_register_intr(int irq)
2629{
2630 irq_clear_status_flags(irq, IRQ_LEVEL);
2631 irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2632 "edge");
2633}
2634
2635/*
2636 * This looks a bit hackish but it's about the only one way of sending
2637 * a few INTA cycles to 8259As and any associated glue logic. ICR does
2638 * not support the ExtINT mode, unfortunately. We need to send these
2639 * cycles as some i82489DX-based boards have glue logic that keeps the
2640 * 8259A interrupt line asserted until INTA. --macro
2641 */
2642static inline void __init unlock_ExtINT_logic(void)
2643{
2644 int apic, pin, i;
2645 struct IO_APIC_route_entry entry0, entry1;
2646 unsigned char save_control, save_freq_select;
2647
2648 pin = find_isa_irq_pin(8, mp_INT);
2649 if (pin == -1) {
2650 WARN_ON_ONCE(1);
2651 return;
2652 }
2653 apic = find_isa_irq_apic(8, mp_INT);
2654 if (apic == -1) {
2655 WARN_ON_ONCE(1);
2656 return;
2657 }
2658
2659 entry0 = ioapic_read_entry(apic, pin);
2660 clear_IO_APIC_pin(apic, pin);
2661
2662 memset(&entry1, 0, sizeof(entry1));
2663
2664 entry1.dest_mode = 0; /* physical delivery */
2665 entry1.mask = 0; /* unmask IRQ now */
2666 entry1.dest = hard_smp_processor_id();
2667 entry1.delivery_mode = dest_ExtINT;
2668 entry1.polarity = entry0.polarity;
2669 entry1.trigger = 0;
2670 entry1.vector = 0;
2671
2672 ioapic_write_entry(apic, pin, entry1);
2673
2674 save_control = CMOS_READ(RTC_CONTROL);
2675 save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2676 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2677 RTC_FREQ_SELECT);
2678 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2679
2680 i = 100;
2681 while (i-- > 0) {
2682 mdelay(10);
2683 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2684 i -= 10;
2685 }
2686
2687 CMOS_WRITE(save_control, RTC_CONTROL);
2688 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2689 clear_IO_APIC_pin(apic, pin);
2690
2691 ioapic_write_entry(apic, pin, entry0);
2692}
2693
2694static int disable_timer_pin_1 __initdata;
2695/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
2696static int __init disable_timer_pin_setup(char *arg)
2697{
2698 disable_timer_pin_1 = 1;
2699 return 0;
2700}
2701early_param("disable_timer_pin_1", disable_timer_pin_setup);
2702
2703int timer_through_8259 __initdata;
2704
2705/*
2706 * This code may look a bit paranoid, but it's supposed to cooperate with
2707 * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ
2708 * is so screwy. Thanks to Brian Perkins for testing/hacking this beast
2709 * fanatically on his truly buggy board.
2710 *
2711 * FIXME: really need to revamp this for all platforms.
2712 */
2713static inline void __init check_timer(void)
2714{
2715 struct irq_cfg *cfg = irq_get_chip_data(0);
2716 int node = cpu_to_node(0);
2717 int apic1, pin1, apic2, pin2;
2718 unsigned long flags;
2719 int no_pin1 = 0;
2720
2721 local_irq_save(flags);
2722
2723 /*
2724 * get/set the timer IRQ vector:
2725 */
2726 legacy_pic->mask(0);
2727 assign_irq_vector(0, cfg, apic->target_cpus());
2728
2729 /*
2730 * As IRQ0 is to be enabled in the 8259A, the virtual
2731 * wire has to be disabled in the local APIC. Also
2732 * timer interrupts need to be acknowledged manually in
2733 * the 8259A for the i82489DX when using the NMI
2734 * watchdog as that APIC treats NMIs as level-triggered.
2735 * The AEOI mode will finish them in the 8259A
2736 * automatically.
2737 */
2738 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2739 legacy_pic->init(1);
2740
2741 pin1 = find_isa_irq_pin(0, mp_INT);
2742 apic1 = find_isa_irq_apic(0, mp_INT);
2743 pin2 = ioapic_i8259.pin;
2744 apic2 = ioapic_i8259.apic;
2745
2746 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2747 "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2748 cfg->vector, apic1, pin1, apic2, pin2);
2749
2750 /*
2751 * Some BIOS writers are clueless and report the ExtINTA
2752 * I/O APIC input from the cascaded 8259A as the timer
2753 * interrupt input. So just in case, if only one pin
2754 * was found above, try it both directly and through the
2755 * 8259A.
2756 */
2757 if (pin1 == -1) {
2758 if (irq_remapping_enabled)
2759 panic("BIOS bug: timer not connected to IO-APIC");
2760 pin1 = pin2;
2761 apic1 = apic2;
2762 no_pin1 = 1;
2763 } else if (pin2 == -1) {
2764 pin2 = pin1;
2765 apic2 = apic1;
2766 }
2767
2768 if (pin1 != -1) {
2769 /*
2770 * Ok, does IRQ0 through the IOAPIC work?
2771 */
2772 if (no_pin1) {
2773 add_pin_to_irq_node(cfg, node, apic1, pin1);
2774 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2775 } else {
2776 /* for edge trigger, setup_ioapic_irq already
2777 * leave it unmasked.
2778 * so only need to unmask if it is level-trigger
2779 * do we really have level trigger timer?
2780 */
2781 int idx;
2782 idx = find_irq_entry(apic1, pin1, mp_INT);
2783 if (idx != -1 && irq_trigger(idx))
2784 unmask_ioapic(cfg);
2785 }
2786 if (timer_irq_works()) {
2787 if (disable_timer_pin_1 > 0)
2788 clear_IO_APIC_pin(0, pin1);
2789 goto out;
2790 }
2791 if (irq_remapping_enabled)
2792 panic("timer doesn't work through Interrupt-remapped IO-APIC");
2793 local_irq_disable();
2794 clear_IO_APIC_pin(apic1, pin1);
2795 if (!no_pin1)
2796 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2797 "8254 timer not connected to IO-APIC\n");
2798
2799 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2800 "(IRQ0) through the 8259A ...\n");
2801 apic_printk(APIC_QUIET, KERN_INFO
2802 "..... (found apic %d pin %d) ...\n", apic2, pin2);
2803 /*
2804 * legacy devices should be connected to IO APIC #0
2805 */
2806 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
2807 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2808 legacy_pic->unmask(0);
2809 if (timer_irq_works()) {
2810 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2811 timer_through_8259 = 1;
2812 goto out;
2813 }
2814 /*
2815 * Cleanup, just in case ...
2816 */
2817 local_irq_disable();
2818 legacy_pic->mask(0);
2819 clear_IO_APIC_pin(apic2, pin2);
2820 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
2821 }
2822
2823 apic_printk(APIC_QUIET, KERN_INFO
2824 "...trying to set up timer as Virtual Wire IRQ...\n");
2825
2826 lapic_register_intr(0);
2827 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
2828 legacy_pic->unmask(0);
2829
2830 if (timer_irq_works()) {
2831 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2832 goto out;
2833 }
2834 local_irq_disable();
2835 legacy_pic->mask(0);
2836 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2837 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
2838
2839 apic_printk(APIC_QUIET, KERN_INFO
2840 "...trying to set up timer as ExtINT IRQ...\n");
2841
2842 legacy_pic->init(0);
2843 legacy_pic->make_irq(0);
2844 apic_write(APIC_LVT0, APIC_DM_EXTINT);
2845
2846 unlock_ExtINT_logic();
2847
2848 if (timer_irq_works()) {
2849 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2850 goto out;
2851 }
2852 local_irq_disable();
2853 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
2854 if (x2apic_preenabled)
2855 apic_printk(APIC_QUIET, KERN_INFO
2856 "Perhaps problem with the pre-enabled x2apic mode\n"
2857 "Try booting with x2apic and interrupt-remapping disabled in the bios.\n");
2858 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
2859 "report. Then try booting with the 'noapic' option.\n");
2860out:
2861 local_irq_restore(flags);
2862}
2863
2864/*
2865 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2866 * to devices. However there may be an I/O APIC pin available for
2867 * this interrupt regardless. The pin may be left unconnected, but
2868 * typically it will be reused as an ExtINT cascade interrupt for
2869 * the master 8259A. In the MPS case such a pin will normally be
2870 * reported as an ExtINT interrupt in the MP table. With ACPI
2871 * there is no provision for ExtINT interrupts, and in the absence
2872 * of an override it would be treated as an ordinary ISA I/O APIC
2873 * interrupt, that is edge-triggered and unmasked by default. We
2874 * used to do this, but it caused problems on some systems because
2875 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
2876 * the same ExtINT cascade interrupt to drive the local APIC of the
2877 * bootstrap processor. Therefore we refrain from routing IRQ2 to
2878 * the I/O APIC in all cases now. No actual device should request
2879 * it anyway. --macro
2880 */
2881#define PIC_IRQS (1UL << PIC_CASCADE_IR)
2882
2883void __init setup_IO_APIC(void)
2884{
2885
2886 /*
2887 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
2888 */
2889 io_apic_irqs = legacy_pic->nr_legacy_irqs ? ~PIC_IRQS : ~0UL;
2890
2891 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
2892 /*
2893 * Set up IO-APIC IRQ routing.
2894 */
2895 x86_init.mpparse.setup_ioapic_ids();
2896
2897 sync_Arb_IDs();
2898 setup_IO_APIC_irqs();
2899 init_IO_APIC_traps();
2900 if (legacy_pic->nr_legacy_irqs)
2901 check_timer();
2902}
2903
2904/*
2905 * Called after all the initialization is done. If we didn't find any
2906 * APIC bugs then we can allow the modify fast path
2907 */
2908
2909static int __init io_apic_bug_finalize(void)
2910{
2911 if (sis_apic_bug == -1)
2912 sis_apic_bug = 0;
2913 return 0;
2914}
2915
2916late_initcall(io_apic_bug_finalize);
2917
2918static void resume_ioapic_id(int ioapic_idx)
2919{
2920 unsigned long flags;
2921 union IO_APIC_reg_00 reg_00;
2922
2923 raw_spin_lock_irqsave(&ioapic_lock, flags);
2924 reg_00.raw = io_apic_read(ioapic_idx, 0);
2925 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) {
2926 reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2927 io_apic_write(ioapic_idx, 0, reg_00.raw);
2928 }
2929 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2930}
2931
2932static void ioapic_resume(void)
2933{
2934 int ioapic_idx;
2935
2936 for (ioapic_idx = nr_ioapics - 1; ioapic_idx >= 0; ioapic_idx--)
2937 resume_ioapic_id(ioapic_idx);
2938
2939 restore_ioapic_entries();
2940}
2941
2942static struct syscore_ops ioapic_syscore_ops = {
2943 .suspend = save_ioapic_entries,
2944 .resume = ioapic_resume,
2945};
2946
2947static int __init ioapic_init_ops(void)
2948{
2949 register_syscore_ops(&ioapic_syscore_ops);
2950
2951 return 0;
2952}
2953
2954device_initcall(ioapic_init_ops);
2955
2956/*
2957 * Dynamic irq allocate and deallocation
2958 */
2959unsigned int create_irq_nr(unsigned int from, int node)
2960{
2961 struct irq_cfg *cfg;
2962 unsigned long flags;
2963 unsigned int ret = 0;
2964 int irq;
2965
2966 if (from < nr_irqs_gsi)
2967 from = nr_irqs_gsi;
2968
2969 irq = alloc_irq_from(from, node);
2970 if (irq < 0)
2971 return 0;
2972 cfg = alloc_irq_cfg(irq, node);
2973 if (!cfg) {
2974 free_irq_at(irq, NULL);
2975 return 0;
2976 }
2977
2978 raw_spin_lock_irqsave(&vector_lock, flags);
2979 if (!__assign_irq_vector(irq, cfg, apic->target_cpus()))
2980 ret = irq;
2981 raw_spin_unlock_irqrestore(&vector_lock, flags);
2982
2983 if (ret) {
2984 irq_set_chip_data(irq, cfg);
2985 irq_clear_status_flags(irq, IRQ_NOREQUEST);
2986 } else {
2987 free_irq_at(irq, cfg);
2988 }
2989 return ret;
2990}
2991
2992int create_irq(void)
2993{
2994 int node = cpu_to_node(0);
2995 unsigned int irq_want;
2996 int irq;
2997
2998 irq_want = nr_irqs_gsi;
2999 irq = create_irq_nr(irq_want, node);
3000
3001 if (irq == 0)
3002 irq = -1;
3003
3004 return irq;
3005}
3006
3007void destroy_irq(unsigned int irq)
3008{
3009 struct irq_cfg *cfg = irq_get_chip_data(irq);
3010 unsigned long flags;
3011
3012 irq_set_status_flags(irq, IRQ_NOREQUEST|IRQ_NOPROBE);
3013
3014 if (irq_remapped(cfg))
3015 free_remapped_irq(irq);
3016 raw_spin_lock_irqsave(&vector_lock, flags);
3017 __clear_irq_vector(irq, cfg);
3018 raw_spin_unlock_irqrestore(&vector_lock, flags);
3019 free_irq_at(irq, cfg);
3020}
3021
3022/*
3023 * MSI message composition
3024 */
3025#ifdef CONFIG_PCI_MSI
3026static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
3027 struct msi_msg *msg, u8 hpet_id)
3028{
3029 struct irq_cfg *cfg;
3030 int err;
3031 unsigned dest;
3032
3033 if (disable_apic)
3034 return -ENXIO;
3035
3036 cfg = irq_cfg(irq);
3037 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3038 if (err)
3039 return err;
3040
3041 dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
3042
3043 if (irq_remapped(cfg)) {
3044 compose_remapped_msi_msg(pdev, irq, dest, msg, hpet_id);
3045 return err;
3046 }
3047
3048 if (x2apic_enabled())
3049 msg->address_hi = MSI_ADDR_BASE_HI |
3050 MSI_ADDR_EXT_DEST_ID(dest);
3051 else
3052 msg->address_hi = MSI_ADDR_BASE_HI;
3053
3054 msg->address_lo =
3055 MSI_ADDR_BASE_LO |
3056 ((apic->irq_dest_mode == 0) ?
3057 MSI_ADDR_DEST_MODE_PHYSICAL:
3058 MSI_ADDR_DEST_MODE_LOGICAL) |
3059 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3060 MSI_ADDR_REDIRECTION_CPU:
3061 MSI_ADDR_REDIRECTION_LOWPRI) |
3062 MSI_ADDR_DEST_ID(dest);
3063
3064 msg->data =
3065 MSI_DATA_TRIGGER_EDGE |
3066 MSI_DATA_LEVEL_ASSERT |
3067 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3068 MSI_DATA_DELIVERY_FIXED:
3069 MSI_DATA_DELIVERY_LOWPRI) |
3070 MSI_DATA_VECTOR(cfg->vector);
3071
3072 return err;
3073}
3074
3075#ifdef CONFIG_SMP
3076static int
3077msi_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3078{
3079 struct irq_cfg *cfg = data->chip_data;
3080 struct msi_msg msg;
3081 unsigned int dest;
3082
3083 if (__ioapic_set_affinity(data, mask, &dest))
3084 return -1;
3085
3086 __get_cached_msi_msg(data->msi_desc, &msg);
3087
3088 msg.data &= ~MSI_DATA_VECTOR_MASK;
3089 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3090 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3091 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3092
3093 __write_msi_msg(data->msi_desc, &msg);
3094
3095 return 0;
3096}
3097#endif /* CONFIG_SMP */
3098
3099/*
3100 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
3101 * which implement the MSI or MSI-X Capability Structure.
3102 */
3103static struct irq_chip msi_chip = {
3104 .name = "PCI-MSI",
3105 .irq_unmask = unmask_msi_irq,
3106 .irq_mask = mask_msi_irq,
3107 .irq_ack = ack_apic_edge,
3108#ifdef CONFIG_SMP
3109 .irq_set_affinity = msi_set_affinity,
3110#endif
3111 .irq_retrigger = ioapic_retrigger_irq,
3112};
3113
3114static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
3115{
3116 struct irq_chip *chip = &msi_chip;
3117 struct msi_msg msg;
3118 int ret;
3119
3120 ret = msi_compose_msg(dev, irq, &msg, -1);
3121 if (ret < 0)
3122 return ret;
3123
3124 irq_set_msi_desc(irq, msidesc);
3125 write_msi_msg(irq, &msg);
3126
3127 if (irq_remapped(irq_get_chip_data(irq))) {
3128 irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
3129 irq_remap_modify_chip_defaults(chip);
3130 }
3131
3132 irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3133
3134 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq);
3135
3136 return 0;
3137}
3138
3139int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3140{
3141 int node, ret, sub_handle, index = 0;
3142 unsigned int irq, irq_want;
3143 struct msi_desc *msidesc;
3144
3145 /* x86 doesn't support multiple MSI yet */
3146 if (type == PCI_CAP_ID_MSI && nvec > 1)
3147 return 1;
3148
3149 node = dev_to_node(&dev->dev);
3150 irq_want = nr_irqs_gsi;
3151 sub_handle = 0;
3152 list_for_each_entry(msidesc, &dev->msi_list, list) {
3153 irq = create_irq_nr(irq_want, node);
3154 if (irq == 0)
3155 return -1;
3156 irq_want = irq + 1;
3157 if (!irq_remapping_enabled)
3158 goto no_ir;
3159
3160 if (!sub_handle) {
3161 /*
3162 * allocate the consecutive block of IRTE's
3163 * for 'nvec'
3164 */
3165 index = msi_alloc_remapped_irq(dev, irq, nvec);
3166 if (index < 0) {
3167 ret = index;
3168 goto error;
3169 }
3170 } else {
3171 ret = msi_setup_remapped_irq(dev, irq, index,
3172 sub_handle);
3173 if (ret < 0)
3174 goto error;
3175 }
3176no_ir:
3177 ret = setup_msi_irq(dev, msidesc, irq);
3178 if (ret < 0)
3179 goto error;
3180 sub_handle++;
3181 }
3182 return 0;
3183
3184error:
3185 destroy_irq(irq);
3186 return ret;
3187}
3188
3189void native_teardown_msi_irq(unsigned int irq)
3190{
3191 destroy_irq(irq);
3192}
3193
3194#ifdef CONFIG_DMAR_TABLE
3195#ifdef CONFIG_SMP
3196static int
3197dmar_msi_set_affinity(struct irq_data *data, const struct cpumask *mask,
3198 bool force)
3199{
3200 struct irq_cfg *cfg = data->chip_data;
3201 unsigned int dest, irq = data->irq;
3202 struct msi_msg msg;
3203
3204 if (__ioapic_set_affinity(data, mask, &dest))
3205 return -1;
3206
3207 dmar_msi_read(irq, &msg);
3208
3209 msg.data &= ~MSI_DATA_VECTOR_MASK;
3210 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3211 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3212 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3213 msg.address_hi = MSI_ADDR_BASE_HI | MSI_ADDR_EXT_DEST_ID(dest);
3214
3215 dmar_msi_write(irq, &msg);
3216
3217 return 0;
3218}
3219
3220#endif /* CONFIG_SMP */
3221
3222static struct irq_chip dmar_msi_type = {
3223 .name = "DMAR_MSI",
3224 .irq_unmask = dmar_msi_unmask,
3225 .irq_mask = dmar_msi_mask,
3226 .irq_ack = ack_apic_edge,
3227#ifdef CONFIG_SMP
3228 .irq_set_affinity = dmar_msi_set_affinity,
3229#endif
3230 .irq_retrigger = ioapic_retrigger_irq,
3231};
3232
3233int arch_setup_dmar_msi(unsigned int irq)
3234{
3235 int ret;
3236 struct msi_msg msg;
3237
3238 ret = msi_compose_msg(NULL, irq, &msg, -1);
3239 if (ret < 0)
3240 return ret;
3241 dmar_msi_write(irq, &msg);
3242 irq_set_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
3243 "edge");
3244 return 0;
3245}
3246#endif
3247
3248#ifdef CONFIG_HPET_TIMER
3249
3250#ifdef CONFIG_SMP
3251static int hpet_msi_set_affinity(struct irq_data *data,
3252 const struct cpumask *mask, bool force)
3253{
3254 struct irq_cfg *cfg = data->chip_data;
3255 struct msi_msg msg;
3256 unsigned int dest;
3257
3258 if (__ioapic_set_affinity(data, mask, &dest))
3259 return -1;
3260
3261 hpet_msi_read(data->handler_data, &msg);
3262
3263 msg.data &= ~MSI_DATA_VECTOR_MASK;
3264 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3265 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3266 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3267
3268 hpet_msi_write(data->handler_data, &msg);
3269
3270 return 0;
3271}
3272
3273#endif /* CONFIG_SMP */
3274
3275static struct irq_chip hpet_msi_type = {
3276 .name = "HPET_MSI",
3277 .irq_unmask = hpet_msi_unmask,
3278 .irq_mask = hpet_msi_mask,
3279 .irq_ack = ack_apic_edge,
3280#ifdef CONFIG_SMP
3281 .irq_set_affinity = hpet_msi_set_affinity,
3282#endif
3283 .irq_retrigger = ioapic_retrigger_irq,
3284};
3285
3286int arch_setup_hpet_msi(unsigned int irq, unsigned int id)
3287{
3288 struct irq_chip *chip = &hpet_msi_type;
3289 struct msi_msg msg;
3290 int ret;
3291
3292 if (irq_remapping_enabled) {
3293 if (!setup_hpet_msi_remapped(irq, id))
3294 return -1;
3295 }
3296
3297 ret = msi_compose_msg(NULL, irq, &msg, id);
3298 if (ret < 0)
3299 return ret;
3300
3301 hpet_msi_write(irq_get_handler_data(irq), &msg);
3302 irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
3303 if (irq_remapped(irq_get_chip_data(irq)))
3304 irq_remap_modify_chip_defaults(chip);
3305
3306 irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3307 return 0;
3308}
3309#endif
3310
3311#endif /* CONFIG_PCI_MSI */
3312/*
3313 * Hypertransport interrupt support
3314 */
3315#ifdef CONFIG_HT_IRQ
3316
3317#ifdef CONFIG_SMP
3318
3319static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3320{
3321 struct ht_irq_msg msg;
3322 fetch_ht_irq_msg(irq, &msg);
3323
3324 msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
3325 msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
3326
3327 msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
3328 msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
3329
3330 write_ht_irq_msg(irq, &msg);
3331}
3332
3333static int
3334ht_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3335{
3336 struct irq_cfg *cfg = data->chip_data;
3337 unsigned int dest;
3338
3339 if (__ioapic_set_affinity(data, mask, &dest))
3340 return -1;
3341
3342 target_ht_irq(data->irq, dest, cfg->vector);
3343 return 0;
3344}
3345
3346#endif
3347
3348static struct irq_chip ht_irq_chip = {
3349 .name = "PCI-HT",
3350 .irq_mask = mask_ht_irq,
3351 .irq_unmask = unmask_ht_irq,
3352 .irq_ack = ack_apic_edge,
3353#ifdef CONFIG_SMP
3354 .irq_set_affinity = ht_set_affinity,
3355#endif
3356 .irq_retrigger = ioapic_retrigger_irq,
3357};
3358
3359int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3360{
3361 struct irq_cfg *cfg;
3362 int err;
3363
3364 if (disable_apic)
3365 return -ENXIO;
3366
3367 cfg = irq_cfg(irq);
3368 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3369 if (!err) {
3370 struct ht_irq_msg msg;
3371 unsigned dest;
3372
3373 dest = apic->cpu_mask_to_apicid_and(cfg->domain,
3374 apic->target_cpus());
3375
3376 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
3377
3378 msg.address_lo =
3379 HT_IRQ_LOW_BASE |
3380 HT_IRQ_LOW_DEST_ID(dest) |
3381 HT_IRQ_LOW_VECTOR(cfg->vector) |
3382 ((apic->irq_dest_mode == 0) ?
3383 HT_IRQ_LOW_DM_PHYSICAL :
3384 HT_IRQ_LOW_DM_LOGICAL) |
3385 HT_IRQ_LOW_RQEOI_EDGE |
3386 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3387 HT_IRQ_LOW_MT_FIXED :
3388 HT_IRQ_LOW_MT_ARBITRATED) |
3389 HT_IRQ_LOW_IRQ_MASKED;
3390
3391 write_ht_irq_msg(irq, &msg);
3392
3393 irq_set_chip_and_handler_name(irq, &ht_irq_chip,
3394 handle_edge_irq, "edge");
3395
3396 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq);
3397 }
3398 return err;
3399}
3400#endif /* CONFIG_HT_IRQ */
3401
3402static int
3403io_apic_setup_irq_pin(unsigned int irq, int node, struct io_apic_irq_attr *attr)
3404{
3405 struct irq_cfg *cfg = alloc_irq_and_cfg_at(irq, node);
3406 int ret;
3407
3408 if (!cfg)
3409 return -EINVAL;
3410 ret = __add_pin_to_irq_node(cfg, node, attr->ioapic, attr->ioapic_pin);
3411 if (!ret)
3412 setup_ioapic_irq(irq, cfg, attr);
3413 return ret;
3414}
3415
3416int io_apic_setup_irq_pin_once(unsigned int irq, int node,
3417 struct io_apic_irq_attr *attr)
3418{
3419 unsigned int ioapic_idx = attr->ioapic, pin = attr->ioapic_pin;
3420 int ret;
3421
3422 /* Avoid redundant programming */
3423 if (test_bit(pin, ioapics[ioapic_idx].pin_programmed)) {
3424 pr_debug("Pin %d-%d already programmed\n",
3425 mpc_ioapic_id(ioapic_idx), pin);
3426 return 0;
3427 }
3428 ret = io_apic_setup_irq_pin(irq, node, attr);
3429 if (!ret)
3430 set_bit(pin, ioapics[ioapic_idx].pin_programmed);
3431 return ret;
3432}
3433
3434static int __init io_apic_get_redir_entries(int ioapic)
3435{
3436 union IO_APIC_reg_01 reg_01;
3437 unsigned long flags;
3438
3439 raw_spin_lock_irqsave(&ioapic_lock, flags);
3440 reg_01.raw = io_apic_read(ioapic, 1);
3441 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3442
3443 /* The register returns the maximum index redir index
3444 * supported, which is one less than the total number of redir
3445 * entries.
3446 */
3447 return reg_01.bits.entries + 1;
3448}
3449
3450static void __init probe_nr_irqs_gsi(void)
3451{
3452 int nr;
3453
3454 nr = gsi_top + NR_IRQS_LEGACY;
3455 if (nr > nr_irqs_gsi)
3456 nr_irqs_gsi = nr;
3457
3458 printk(KERN_DEBUG "nr_irqs_gsi: %d\n", nr_irqs_gsi);
3459}
3460
3461int get_nr_irqs_gsi(void)
3462{
3463 return nr_irqs_gsi;
3464}
3465
3466int __init arch_probe_nr_irqs(void)
3467{
3468 int nr;
3469
3470 if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
3471 nr_irqs = NR_VECTORS * nr_cpu_ids;
3472
3473 nr = nr_irqs_gsi + 8 * nr_cpu_ids;
3474#if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
3475 /*
3476 * for MSI and HT dyn irq
3477 */
3478 nr += nr_irqs_gsi * 16;
3479#endif
3480 if (nr < nr_irqs)
3481 nr_irqs = nr;
3482
3483 return NR_IRQS_LEGACY;
3484}
3485
3486int io_apic_set_pci_routing(struct device *dev, int irq,
3487 struct io_apic_irq_attr *irq_attr)
3488{
3489 int node;
3490
3491 if (!IO_APIC_IRQ(irq)) {
3492 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
3493 irq_attr->ioapic);
3494 return -EINVAL;
3495 }
3496
3497 node = dev ? dev_to_node(dev) : cpu_to_node(0);
3498
3499 return io_apic_setup_irq_pin_once(irq, node, irq_attr);
3500}
3501
3502#ifdef CONFIG_X86_32
3503static int __init io_apic_get_unique_id(int ioapic, int apic_id)
3504{
3505 union IO_APIC_reg_00 reg_00;
3506 static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
3507 physid_mask_t tmp;
3508 unsigned long flags;
3509 int i = 0;
3510
3511 /*
3512 * The P4 platform supports up to 256 APIC IDs on two separate APIC
3513 * buses (one for LAPICs, one for IOAPICs), where predecessors only
3514 * supports up to 16 on one shared APIC bus.
3515 *
3516 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
3517 * advantage of new APIC bus architecture.
3518 */
3519
3520 if (physids_empty(apic_id_map))
3521 apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map);
3522
3523 raw_spin_lock_irqsave(&ioapic_lock, flags);
3524 reg_00.raw = io_apic_read(ioapic, 0);
3525 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3526
3527 if (apic_id >= get_physical_broadcast()) {
3528 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
3529 "%d\n", ioapic, apic_id, reg_00.bits.ID);
3530 apic_id = reg_00.bits.ID;
3531 }
3532
3533 /*
3534 * Every APIC in a system must have a unique ID or we get lots of nice
3535 * 'stuck on smp_invalidate_needed IPI wait' messages.
3536 */
3537 if (apic->check_apicid_used(&apic_id_map, apic_id)) {
3538
3539 for (i = 0; i < get_physical_broadcast(); i++) {
3540 if (!apic->check_apicid_used(&apic_id_map, i))
3541 break;
3542 }
3543
3544 if (i == get_physical_broadcast())
3545 panic("Max apic_id exceeded!\n");
3546
3547 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
3548 "trying %d\n", ioapic, apic_id, i);
3549
3550 apic_id = i;
3551 }
3552
3553 apic->apicid_to_cpu_present(apic_id, &tmp);
3554 physids_or(apic_id_map, apic_id_map, tmp);
3555
3556 if (reg_00.bits.ID != apic_id) {
3557 reg_00.bits.ID = apic_id;
3558
3559 raw_spin_lock_irqsave(&ioapic_lock, flags);
3560 io_apic_write(ioapic, 0, reg_00.raw);
3561 reg_00.raw = io_apic_read(ioapic, 0);
3562 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3563
3564 /* Sanity check */
3565 if (reg_00.bits.ID != apic_id) {
3566 printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
3567 return -1;
3568 }
3569 }
3570
3571 apic_printk(APIC_VERBOSE, KERN_INFO
3572 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
3573
3574 return apic_id;
3575}
3576
3577static u8 __init io_apic_unique_id(u8 id)
3578{
3579 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
3580 !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
3581 return io_apic_get_unique_id(nr_ioapics, id);
3582 else
3583 return id;
3584}
3585#else
3586static u8 __init io_apic_unique_id(u8 id)
3587{
3588 int i;
3589 DECLARE_BITMAP(used, 256);
3590
3591 bitmap_zero(used, 256);
3592 for (i = 0; i < nr_ioapics; i++) {
3593 __set_bit(mpc_ioapic_id(i), used);
3594 }
3595 if (!test_bit(id, used))
3596 return id;
3597 return find_first_zero_bit(used, 256);
3598}
3599#endif
3600
3601static int __init io_apic_get_version(int ioapic)
3602{
3603 union IO_APIC_reg_01 reg_01;
3604 unsigned long flags;
3605
3606 raw_spin_lock_irqsave(&ioapic_lock, flags);
3607 reg_01.raw = io_apic_read(ioapic, 1);
3608 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3609
3610 return reg_01.bits.version;
3611}
3612
3613int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity)
3614{
3615 int ioapic, pin, idx;
3616
3617 if (skip_ioapic_setup)
3618 return -1;
3619
3620 ioapic = mp_find_ioapic(gsi);
3621 if (ioapic < 0)
3622 return -1;
3623
3624 pin = mp_find_ioapic_pin(ioapic, gsi);
3625 if (pin < 0)
3626 return -1;
3627
3628 idx = find_irq_entry(ioapic, pin, mp_INT);
3629 if (idx < 0)
3630 return -1;
3631
3632 *trigger = irq_trigger(idx);
3633 *polarity = irq_polarity(idx);
3634 return 0;
3635}
3636
3637/*
3638 * This function currently is only a helper for the i386 smp boot process where
3639 * we need to reprogram the ioredtbls to cater for the cpus which have come online
3640 * so mask in all cases should simply be apic->target_cpus()
3641 */
3642#ifdef CONFIG_SMP
3643void __init setup_ioapic_dest(void)
3644{
3645 int pin, ioapic, irq, irq_entry;
3646 const struct cpumask *mask;
3647 struct irq_data *idata;
3648
3649 if (skip_ioapic_setup == 1)
3650 return;
3651
3652 for (ioapic = 0; ioapic < nr_ioapics; ioapic++)
3653 for (pin = 0; pin < ioapics[ioapic].nr_registers; pin++) {
3654 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
3655 if (irq_entry == -1)
3656 continue;
3657 irq = pin_2_irq(irq_entry, ioapic, pin);
3658
3659 if ((ioapic > 0) && (irq > 16))
3660 continue;
3661
3662 idata = irq_get_irq_data(irq);
3663
3664 /*
3665 * Honour affinities which have been set in early boot
3666 */
3667 if (!irqd_can_balance(idata) || irqd_affinity_was_set(idata))
3668 mask = idata->affinity;
3669 else
3670 mask = apic->target_cpus();
3671
3672 if (irq_remapping_enabled)
3673 set_remapped_irq_affinity(idata, mask, false);
3674 else
3675 ioapic_set_affinity(idata, mask, false);
3676 }
3677
3678}
3679#endif
3680
3681#define IOAPIC_RESOURCE_NAME_SIZE 11
3682
3683static struct resource *ioapic_resources;
3684
3685static struct resource * __init ioapic_setup_resources(int nr_ioapics)
3686{
3687 unsigned long n;
3688 struct resource *res;
3689 char *mem;
3690 int i;
3691
3692 if (nr_ioapics <= 0)
3693 return NULL;
3694
3695 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
3696 n *= nr_ioapics;
3697
3698 mem = alloc_bootmem(n);
3699 res = (void *)mem;
3700
3701 mem += sizeof(struct resource) * nr_ioapics;
3702
3703 for (i = 0; i < nr_ioapics; i++) {
3704 res[i].name = mem;
3705 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
3706 snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i);
3707 mem += IOAPIC_RESOURCE_NAME_SIZE;
3708 }
3709
3710 ioapic_resources = res;
3711
3712 return res;
3713}
3714
3715void __init native_io_apic_init_mappings(void)
3716{
3717 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
3718 struct resource *ioapic_res;
3719 int i;
3720
3721 ioapic_res = ioapic_setup_resources(nr_ioapics);
3722 for (i = 0; i < nr_ioapics; i++) {
3723 if (smp_found_config) {
3724 ioapic_phys = mpc_ioapic_addr(i);
3725#ifdef CONFIG_X86_32
3726 if (!ioapic_phys) {
3727 printk(KERN_ERR
3728 "WARNING: bogus zero IO-APIC "
3729 "address found in MPTABLE, "
3730 "disabling IO/APIC support!\n");
3731 smp_found_config = 0;
3732 skip_ioapic_setup = 1;
3733 goto fake_ioapic_page;
3734 }
3735#endif
3736 } else {
3737#ifdef CONFIG_X86_32
3738fake_ioapic_page:
3739#endif
3740 ioapic_phys = (unsigned long)alloc_bootmem_pages(PAGE_SIZE);
3741 ioapic_phys = __pa(ioapic_phys);
3742 }
3743 set_fixmap_nocache(idx, ioapic_phys);
3744 apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n",
3745 __fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK),
3746 ioapic_phys);
3747 idx++;
3748
3749 ioapic_res->start = ioapic_phys;
3750 ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1;
3751 ioapic_res++;
3752 }
3753
3754 probe_nr_irqs_gsi();
3755}
3756
3757void __init ioapic_insert_resources(void)
3758{
3759 int i;
3760 struct resource *r = ioapic_resources;
3761
3762 if (!r) {
3763 if (nr_ioapics > 0)
3764 printk(KERN_ERR
3765 "IO APIC resources couldn't be allocated.\n");
3766 return;
3767 }
3768
3769 for (i = 0; i < nr_ioapics; i++) {
3770 insert_resource(&iomem_resource, r);
3771 r++;
3772 }
3773}
3774
3775int mp_find_ioapic(u32 gsi)
3776{
3777 int i = 0;
3778
3779 if (nr_ioapics == 0)
3780 return -1;
3781
3782 /* Find the IOAPIC that manages this GSI. */
3783 for (i = 0; i < nr_ioapics; i++) {
3784 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i);
3785 if ((gsi >= gsi_cfg->gsi_base)
3786 && (gsi <= gsi_cfg->gsi_end))
3787 return i;
3788 }
3789
3790 printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
3791 return -1;
3792}
3793
3794int mp_find_ioapic_pin(int ioapic, u32 gsi)
3795{
3796 struct mp_ioapic_gsi *gsi_cfg;
3797
3798 if (WARN_ON(ioapic == -1))
3799 return -1;
3800
3801 gsi_cfg = mp_ioapic_gsi_routing(ioapic);
3802 if (WARN_ON(gsi > gsi_cfg->gsi_end))
3803 return -1;
3804
3805 return gsi - gsi_cfg->gsi_base;
3806}
3807
3808static __init int bad_ioapic(unsigned long address)
3809{
3810 if (nr_ioapics >= MAX_IO_APICS) {
3811 pr_warn("WARNING: Max # of I/O APICs (%d) exceeded (found %d), skipping\n",
3812 MAX_IO_APICS, nr_ioapics);
3813 return 1;
3814 }
3815 if (!address) {
3816 pr_warn("WARNING: Bogus (zero) I/O APIC address found in table, skipping!\n");
3817 return 1;
3818 }
3819 return 0;
3820}
3821
3822static __init int bad_ioapic_register(int idx)
3823{
3824 union IO_APIC_reg_00 reg_00;
3825 union IO_APIC_reg_01 reg_01;
3826 union IO_APIC_reg_02 reg_02;
3827
3828 reg_00.raw = io_apic_read(idx, 0);
3829 reg_01.raw = io_apic_read(idx, 1);
3830 reg_02.raw = io_apic_read(idx, 2);
3831
3832 if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) {
3833 pr_warn("I/O APIC 0x%x registers return all ones, skipping!\n",
3834 mpc_ioapic_addr(idx));
3835 return 1;
3836 }
3837
3838 return 0;
3839}
3840
3841void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
3842{
3843 int idx = 0;
3844 int entries;
3845 struct mp_ioapic_gsi *gsi_cfg;
3846
3847 if (bad_ioapic(address))
3848 return;
3849
3850 idx = nr_ioapics;
3851
3852 ioapics[idx].mp_config.type = MP_IOAPIC;
3853 ioapics[idx].mp_config.flags = MPC_APIC_USABLE;
3854 ioapics[idx].mp_config.apicaddr = address;
3855
3856 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
3857
3858 if (bad_ioapic_register(idx)) {
3859 clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
3860 return;
3861 }
3862
3863 ioapics[idx].mp_config.apicid = io_apic_unique_id(id);
3864 ioapics[idx].mp_config.apicver = io_apic_get_version(idx);
3865
3866 /*
3867 * Build basic GSI lookup table to facilitate gsi->io_apic lookups
3868 * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
3869 */
3870 entries = io_apic_get_redir_entries(idx);
3871 gsi_cfg = mp_ioapic_gsi_routing(idx);
3872 gsi_cfg->gsi_base = gsi_base;
3873 gsi_cfg->gsi_end = gsi_base + entries - 1;
3874
3875 /*
3876 * The number of IO-APIC IRQ registers (== #pins):
3877 */
3878 ioapics[idx].nr_registers = entries;
3879
3880 if (gsi_cfg->gsi_end >= gsi_top)
3881 gsi_top = gsi_cfg->gsi_end + 1;
3882
3883 pr_info("IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n",
3884 idx, mpc_ioapic_id(idx),
3885 mpc_ioapic_ver(idx), mpc_ioapic_addr(idx),
3886 gsi_cfg->gsi_base, gsi_cfg->gsi_end);
3887
3888 nr_ioapics++;
3889}
3890
3891/* Enable IOAPIC early just for system timer */
3892void __init pre_init_apic_IRQ0(void)
3893{
3894 struct io_apic_irq_attr attr = { 0, 0, 0, 0 };
3895
3896 printk(KERN_INFO "Early APIC setup for system timer0\n");
3897#ifndef CONFIG_SMP
3898 physid_set_mask_of_physid(boot_cpu_physical_apicid,
3899 &phys_cpu_present_map);
3900#endif
3901 setup_local_APIC();
3902
3903 io_apic_setup_irq_pin(0, 0, &attr);
3904 irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
3905 "edge");
3906}
1/*
2 * Intel IO-APIC support for multi-Pentium hosts.
3 *
4 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
5 *
6 * Many thanks to Stig Venaas for trying out countless experimental
7 * patches and reporting/debugging problems patiently!
8 *
9 * (c) 1999, Multiple IO-APIC support, developed by
10 * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
11 * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
12 * further tested and cleaned up by Zach Brown <zab@redhat.com>
13 * and Ingo Molnar <mingo@redhat.com>
14 *
15 * Fixes
16 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
17 * thanks to Eric Gilmore
18 * and Rolf G. Tews
19 * for testing these extensively
20 * Paul Diefenbaugh : Added full ACPI support
21 */
22
23#include <linux/mm.h>
24#include <linux/interrupt.h>
25#include <linux/init.h>
26#include <linux/delay.h>
27#include <linux/sched.h>
28#include <linux/pci.h>
29#include <linux/mc146818rtc.h>
30#include <linux/compiler.h>
31#include <linux/acpi.h>
32#include <linux/module.h>
33#include <linux/syscore_ops.h>
34#include <linux/msi.h>
35#include <linux/htirq.h>
36#include <linux/freezer.h>
37#include <linux/kthread.h>
38#include <linux/jiffies.h> /* time_after() */
39#include <linux/slab.h>
40#ifdef CONFIG_ACPI
41#include <acpi/acpi_bus.h>
42#endif
43#include <linux/bootmem.h>
44#include <linux/dmar.h>
45#include <linux/hpet.h>
46
47#include <asm/idle.h>
48#include <asm/io.h>
49#include <asm/smp.h>
50#include <asm/cpu.h>
51#include <asm/desc.h>
52#include <asm/proto.h>
53#include <asm/acpi.h>
54#include <asm/dma.h>
55#include <asm/timer.h>
56#include <asm/i8259.h>
57#include <asm/msidef.h>
58#include <asm/hypertransport.h>
59#include <asm/setup.h>
60#include <asm/irq_remapping.h>
61#include <asm/hpet.h>
62#include <asm/hw_irq.h>
63
64#include <asm/apic.h>
65
66#define __apicdebuginit(type) static type __init
67#define for_each_irq_pin(entry, head) \
68 for (entry = head; entry; entry = entry->next)
69
70/*
71 * Is the SiS APIC rmw bug present ?
72 * -1 = don't know, 0 = no, 1 = yes
73 */
74int sis_apic_bug = -1;
75
76static DEFINE_RAW_SPINLOCK(ioapic_lock);
77static DEFINE_RAW_SPINLOCK(vector_lock);
78
79static struct ioapic {
80 /*
81 * # of IRQ routing registers
82 */
83 int nr_registers;
84 /*
85 * Saved state during suspend/resume, or while enabling intr-remap.
86 */
87 struct IO_APIC_route_entry *saved_registers;
88 /* I/O APIC config */
89 struct mpc_ioapic mp_config;
90 /* IO APIC gsi routing info */
91 struct mp_ioapic_gsi gsi_config;
92 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
93} ioapics[MAX_IO_APICS];
94
95#define mpc_ioapic_ver(id) ioapics[id].mp_config.apicver
96
97int mpc_ioapic_id(int id)
98{
99 return ioapics[id].mp_config.apicid;
100}
101
102unsigned int mpc_ioapic_addr(int id)
103{
104 return ioapics[id].mp_config.apicaddr;
105}
106
107struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int id)
108{
109 return &ioapics[id].gsi_config;
110}
111
112int nr_ioapics;
113
114/* The one past the highest gsi number used */
115u32 gsi_top;
116
117/* MP IRQ source entries */
118struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
119
120/* # of MP IRQ source entries */
121int mp_irq_entries;
122
123/* GSI interrupts */
124static int nr_irqs_gsi = NR_IRQS_LEGACY;
125
126#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
127int mp_bus_id_to_type[MAX_MP_BUSSES];
128#endif
129
130DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
131
132int skip_ioapic_setup;
133
134/**
135 * disable_ioapic_support() - disables ioapic support at runtime
136 */
137void disable_ioapic_support(void)
138{
139#ifdef CONFIG_PCI
140 noioapicquirk = 1;
141 noioapicreroute = -1;
142#endif
143 skip_ioapic_setup = 1;
144}
145
146static int __init parse_noapic(char *str)
147{
148 /* disable IO-APIC */
149 disable_ioapic_support();
150 return 0;
151}
152early_param("noapic", parse_noapic);
153
154static int io_apic_setup_irq_pin(unsigned int irq, int node,
155 struct io_apic_irq_attr *attr);
156
157/* Will be called in mpparse/acpi/sfi codes for saving IRQ info */
158void mp_save_irq(struct mpc_intsrc *m)
159{
160 int i;
161
162 apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x,"
163 " IRQ %02x, APIC ID %x, APIC INT %02x\n",
164 m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus,
165 m->srcbusirq, m->dstapic, m->dstirq);
166
167 for (i = 0; i < mp_irq_entries; i++) {
168 if (!memcmp(&mp_irqs[i], m, sizeof(*m)))
169 return;
170 }
171
172 memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m));
173 if (++mp_irq_entries == MAX_IRQ_SOURCES)
174 panic("Max # of irq sources exceeded!!\n");
175}
176
177struct irq_pin_list {
178 int apic, pin;
179 struct irq_pin_list *next;
180};
181
182static struct irq_pin_list *alloc_irq_pin_list(int node)
183{
184 return kzalloc_node(sizeof(struct irq_pin_list), GFP_KERNEL, node);
185}
186
187
188/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
189#ifdef CONFIG_SPARSE_IRQ
190static struct irq_cfg irq_cfgx[NR_IRQS_LEGACY];
191#else
192static struct irq_cfg irq_cfgx[NR_IRQS];
193#endif
194
195int __init arch_early_irq_init(void)
196{
197 struct irq_cfg *cfg;
198 int count, node, i;
199
200 if (!legacy_pic->nr_legacy_irqs) {
201 nr_irqs_gsi = 0;
202 io_apic_irqs = ~0UL;
203 }
204
205 for (i = 0; i < nr_ioapics; i++) {
206 ioapics[i].saved_registers =
207 kzalloc(sizeof(struct IO_APIC_route_entry) *
208 ioapics[i].nr_registers, GFP_KERNEL);
209 if (!ioapics[i].saved_registers)
210 pr_err("IOAPIC %d: suspend/resume impossible!\n", i);
211 }
212
213 cfg = irq_cfgx;
214 count = ARRAY_SIZE(irq_cfgx);
215 node = cpu_to_node(0);
216
217 /* Make sure the legacy interrupts are marked in the bitmap */
218 irq_reserve_irqs(0, legacy_pic->nr_legacy_irqs);
219
220 for (i = 0; i < count; i++) {
221 irq_set_chip_data(i, &cfg[i]);
222 zalloc_cpumask_var_node(&cfg[i].domain, GFP_KERNEL, node);
223 zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_KERNEL, node);
224 /*
225 * For legacy IRQ's, start with assigning irq0 to irq15 to
226 * IRQ0_VECTOR to IRQ15_VECTOR on cpu 0.
227 */
228 if (i < legacy_pic->nr_legacy_irqs) {
229 cfg[i].vector = IRQ0_VECTOR + i;
230 cpumask_set_cpu(0, cfg[i].domain);
231 }
232 }
233
234 return 0;
235}
236
237#ifdef CONFIG_SPARSE_IRQ
238static struct irq_cfg *irq_cfg(unsigned int irq)
239{
240 return irq_get_chip_data(irq);
241}
242
243static struct irq_cfg *alloc_irq_cfg(unsigned int irq, int node)
244{
245 struct irq_cfg *cfg;
246
247 cfg = kzalloc_node(sizeof(*cfg), GFP_KERNEL, node);
248 if (!cfg)
249 return NULL;
250 if (!zalloc_cpumask_var_node(&cfg->domain, GFP_KERNEL, node))
251 goto out_cfg;
252 if (!zalloc_cpumask_var_node(&cfg->old_domain, GFP_KERNEL, node))
253 goto out_domain;
254 return cfg;
255out_domain:
256 free_cpumask_var(cfg->domain);
257out_cfg:
258 kfree(cfg);
259 return NULL;
260}
261
262static void free_irq_cfg(unsigned int at, struct irq_cfg *cfg)
263{
264 if (!cfg)
265 return;
266 irq_set_chip_data(at, NULL);
267 free_cpumask_var(cfg->domain);
268 free_cpumask_var(cfg->old_domain);
269 kfree(cfg);
270}
271
272#else
273
274struct irq_cfg *irq_cfg(unsigned int irq)
275{
276 return irq < nr_irqs ? irq_cfgx + irq : NULL;
277}
278
279static struct irq_cfg *alloc_irq_cfg(unsigned int irq, int node)
280{
281 return irq_cfgx + irq;
282}
283
284static inline void free_irq_cfg(unsigned int at, struct irq_cfg *cfg) { }
285
286#endif
287
288static struct irq_cfg *alloc_irq_and_cfg_at(unsigned int at, int node)
289{
290 int res = irq_alloc_desc_at(at, node);
291 struct irq_cfg *cfg;
292
293 if (res < 0) {
294 if (res != -EEXIST)
295 return NULL;
296 cfg = irq_get_chip_data(at);
297 if (cfg)
298 return cfg;
299 }
300
301 cfg = alloc_irq_cfg(at, node);
302 if (cfg)
303 irq_set_chip_data(at, cfg);
304 else
305 irq_free_desc(at);
306 return cfg;
307}
308
309static int alloc_irq_from(unsigned int from, int node)
310{
311 return irq_alloc_desc_from(from, node);
312}
313
314static void free_irq_at(unsigned int at, struct irq_cfg *cfg)
315{
316 free_irq_cfg(at, cfg);
317 irq_free_desc(at);
318}
319
320struct io_apic {
321 unsigned int index;
322 unsigned int unused[3];
323 unsigned int data;
324 unsigned int unused2[11];
325 unsigned int eoi;
326};
327
328static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
329{
330 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
331 + (mpc_ioapic_addr(idx) & ~PAGE_MASK);
332}
333
334static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
335{
336 struct io_apic __iomem *io_apic = io_apic_base(apic);
337 writel(vector, &io_apic->eoi);
338}
339
340static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
341{
342 struct io_apic __iomem *io_apic = io_apic_base(apic);
343 writel(reg, &io_apic->index);
344 return readl(&io_apic->data);
345}
346
347static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
348{
349 struct io_apic __iomem *io_apic = io_apic_base(apic);
350 writel(reg, &io_apic->index);
351 writel(value, &io_apic->data);
352}
353
354/*
355 * Re-write a value: to be used for read-modify-write
356 * cycles where the read already set up the index register.
357 *
358 * Older SiS APIC requires we rewrite the index register
359 */
360static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
361{
362 struct io_apic __iomem *io_apic = io_apic_base(apic);
363
364 if (sis_apic_bug)
365 writel(reg, &io_apic->index);
366 writel(value, &io_apic->data);
367}
368
369static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
370{
371 struct irq_pin_list *entry;
372 unsigned long flags;
373
374 raw_spin_lock_irqsave(&ioapic_lock, flags);
375 for_each_irq_pin(entry, cfg->irq_2_pin) {
376 unsigned int reg;
377 int pin;
378
379 pin = entry->pin;
380 reg = io_apic_read(entry->apic, 0x10 + pin*2);
381 /* Is the remote IRR bit set? */
382 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
383 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
384 return true;
385 }
386 }
387 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
388
389 return false;
390}
391
392union entry_union {
393 struct { u32 w1, w2; };
394 struct IO_APIC_route_entry entry;
395};
396
397static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
398{
399 union entry_union eu;
400 unsigned long flags;
401 raw_spin_lock_irqsave(&ioapic_lock, flags);
402 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
403 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
404 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
405 return eu.entry;
406}
407
408/*
409 * When we write a new IO APIC routing entry, we need to write the high
410 * word first! If the mask bit in the low word is clear, we will enable
411 * the interrupt, and we need to make sure the entry is fully populated
412 * before that happens.
413 */
414static void
415__ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
416{
417 union entry_union eu = {{0, 0}};
418
419 eu.entry = e;
420 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
421 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
422}
423
424static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
425{
426 unsigned long flags;
427 raw_spin_lock_irqsave(&ioapic_lock, flags);
428 __ioapic_write_entry(apic, pin, e);
429 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
430}
431
432/*
433 * When we mask an IO APIC routing entry, we need to write the low
434 * word first, in order to set the mask bit before we change the
435 * high bits!
436 */
437static void ioapic_mask_entry(int apic, int pin)
438{
439 unsigned long flags;
440 union entry_union eu = { .entry.mask = 1 };
441
442 raw_spin_lock_irqsave(&ioapic_lock, flags);
443 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
444 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
445 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
446}
447
448/*
449 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
450 * shared ISA-space IRQs, so we have to support them. We are super
451 * fast in the common case, and fast for shared ISA-space IRQs.
452 */
453static int
454__add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
455{
456 struct irq_pin_list **last, *entry;
457
458 /* don't allow duplicates */
459 last = &cfg->irq_2_pin;
460 for_each_irq_pin(entry, cfg->irq_2_pin) {
461 if (entry->apic == apic && entry->pin == pin)
462 return 0;
463 last = &entry->next;
464 }
465
466 entry = alloc_irq_pin_list(node);
467 if (!entry) {
468 printk(KERN_ERR "can not alloc irq_pin_list (%d,%d,%d)\n",
469 node, apic, pin);
470 return -ENOMEM;
471 }
472 entry->apic = apic;
473 entry->pin = pin;
474
475 *last = entry;
476 return 0;
477}
478
479static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
480{
481 if (__add_pin_to_irq_node(cfg, node, apic, pin))
482 panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
483}
484
485/*
486 * Reroute an IRQ to a different pin.
487 */
488static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
489 int oldapic, int oldpin,
490 int newapic, int newpin)
491{
492 struct irq_pin_list *entry;
493
494 for_each_irq_pin(entry, cfg->irq_2_pin) {
495 if (entry->apic == oldapic && entry->pin == oldpin) {
496 entry->apic = newapic;
497 entry->pin = newpin;
498 /* every one is different, right? */
499 return;
500 }
501 }
502
503 /* old apic/pin didn't exist, so just add new ones */
504 add_pin_to_irq_node(cfg, node, newapic, newpin);
505}
506
507static void __io_apic_modify_irq(struct irq_pin_list *entry,
508 int mask_and, int mask_or,
509 void (*final)(struct irq_pin_list *entry))
510{
511 unsigned int reg, pin;
512
513 pin = entry->pin;
514 reg = io_apic_read(entry->apic, 0x10 + pin * 2);
515 reg &= mask_and;
516 reg |= mask_or;
517 io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
518 if (final)
519 final(entry);
520}
521
522static void io_apic_modify_irq(struct irq_cfg *cfg,
523 int mask_and, int mask_or,
524 void (*final)(struct irq_pin_list *entry))
525{
526 struct irq_pin_list *entry;
527
528 for_each_irq_pin(entry, cfg->irq_2_pin)
529 __io_apic_modify_irq(entry, mask_and, mask_or, final);
530}
531
532static void __mask_and_edge_IO_APIC_irq(struct irq_pin_list *entry)
533{
534 __io_apic_modify_irq(entry, ~IO_APIC_REDIR_LEVEL_TRIGGER,
535 IO_APIC_REDIR_MASKED, NULL);
536}
537
538static void __unmask_and_level_IO_APIC_irq(struct irq_pin_list *entry)
539{
540 __io_apic_modify_irq(entry, ~IO_APIC_REDIR_MASKED,
541 IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
542}
543
544static void io_apic_sync(struct irq_pin_list *entry)
545{
546 /*
547 * Synchronize the IO-APIC and the CPU by doing
548 * a dummy read from the IO-APIC
549 */
550 struct io_apic __iomem *io_apic;
551 io_apic = io_apic_base(entry->apic);
552 readl(&io_apic->data);
553}
554
555static void mask_ioapic(struct irq_cfg *cfg)
556{
557 unsigned long flags;
558
559 raw_spin_lock_irqsave(&ioapic_lock, flags);
560 io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
561 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
562}
563
564static void mask_ioapic_irq(struct irq_data *data)
565{
566 mask_ioapic(data->chip_data);
567}
568
569static void __unmask_ioapic(struct irq_cfg *cfg)
570{
571 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
572}
573
574static void unmask_ioapic(struct irq_cfg *cfg)
575{
576 unsigned long flags;
577
578 raw_spin_lock_irqsave(&ioapic_lock, flags);
579 __unmask_ioapic(cfg);
580 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
581}
582
583static void unmask_ioapic_irq(struct irq_data *data)
584{
585 unmask_ioapic(data->chip_data);
586}
587
588static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
589{
590 struct IO_APIC_route_entry entry;
591
592 /* Check delivery_mode to be sure we're not clearing an SMI pin */
593 entry = ioapic_read_entry(apic, pin);
594 if (entry.delivery_mode == dest_SMI)
595 return;
596 /*
597 * Disable it in the IO-APIC irq-routing table:
598 */
599 ioapic_mask_entry(apic, pin);
600}
601
602static void clear_IO_APIC (void)
603{
604 int apic, pin;
605
606 for (apic = 0; apic < nr_ioapics; apic++)
607 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
608 clear_IO_APIC_pin(apic, pin);
609}
610
611#ifdef CONFIG_X86_32
612/*
613 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
614 * specific CPU-side IRQs.
615 */
616
617#define MAX_PIRQS 8
618static int pirq_entries[MAX_PIRQS] = {
619 [0 ... MAX_PIRQS - 1] = -1
620};
621
622static int __init ioapic_pirq_setup(char *str)
623{
624 int i, max;
625 int ints[MAX_PIRQS+1];
626
627 get_options(str, ARRAY_SIZE(ints), ints);
628
629 apic_printk(APIC_VERBOSE, KERN_INFO
630 "PIRQ redirection, working around broken MP-BIOS.\n");
631 max = MAX_PIRQS;
632 if (ints[0] < MAX_PIRQS)
633 max = ints[0];
634
635 for (i = 0; i < max; i++) {
636 apic_printk(APIC_VERBOSE, KERN_DEBUG
637 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
638 /*
639 * PIRQs are mapped upside down, usually.
640 */
641 pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
642 }
643 return 1;
644}
645
646__setup("pirq=", ioapic_pirq_setup);
647#endif /* CONFIG_X86_32 */
648
649/*
650 * Saves all the IO-APIC RTE's
651 */
652int save_ioapic_entries(void)
653{
654 int apic, pin;
655 int err = 0;
656
657 for (apic = 0; apic < nr_ioapics; apic++) {
658 if (!ioapics[apic].saved_registers) {
659 err = -ENOMEM;
660 continue;
661 }
662
663 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
664 ioapics[apic].saved_registers[pin] =
665 ioapic_read_entry(apic, pin);
666 }
667
668 return err;
669}
670
671/*
672 * Mask all IO APIC entries.
673 */
674void mask_ioapic_entries(void)
675{
676 int apic, pin;
677
678 for (apic = 0; apic < nr_ioapics; apic++) {
679 if (!ioapics[apic].saved_registers)
680 continue;
681
682 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
683 struct IO_APIC_route_entry entry;
684
685 entry = ioapics[apic].saved_registers[pin];
686 if (!entry.mask) {
687 entry.mask = 1;
688 ioapic_write_entry(apic, pin, entry);
689 }
690 }
691 }
692}
693
694/*
695 * Restore IO APIC entries which was saved in the ioapic structure.
696 */
697int restore_ioapic_entries(void)
698{
699 int apic, pin;
700
701 for (apic = 0; apic < nr_ioapics; apic++) {
702 if (!ioapics[apic].saved_registers)
703 continue;
704
705 for (pin = 0; pin < ioapics[apic].nr_registers; pin++)
706 ioapic_write_entry(apic, pin,
707 ioapics[apic].saved_registers[pin]);
708 }
709 return 0;
710}
711
712/*
713 * Find the IRQ entry number of a certain pin.
714 */
715static int find_irq_entry(int apic, int pin, int type)
716{
717 int i;
718
719 for (i = 0; i < mp_irq_entries; i++)
720 if (mp_irqs[i].irqtype == type &&
721 (mp_irqs[i].dstapic == mpc_ioapic_id(apic) ||
722 mp_irqs[i].dstapic == MP_APIC_ALL) &&
723 mp_irqs[i].dstirq == pin)
724 return i;
725
726 return -1;
727}
728
729/*
730 * Find the pin to which IRQ[irq] (ISA) is connected
731 */
732static int __init find_isa_irq_pin(int irq, int type)
733{
734 int i;
735
736 for (i = 0; i < mp_irq_entries; i++) {
737 int lbus = mp_irqs[i].srcbus;
738
739 if (test_bit(lbus, mp_bus_not_pci) &&
740 (mp_irqs[i].irqtype == type) &&
741 (mp_irqs[i].srcbusirq == irq))
742
743 return mp_irqs[i].dstirq;
744 }
745 return -1;
746}
747
748static int __init find_isa_irq_apic(int irq, int type)
749{
750 int i;
751
752 for (i = 0; i < mp_irq_entries; i++) {
753 int lbus = mp_irqs[i].srcbus;
754
755 if (test_bit(lbus, mp_bus_not_pci) &&
756 (mp_irqs[i].irqtype == type) &&
757 (mp_irqs[i].srcbusirq == irq))
758 break;
759 }
760 if (i < mp_irq_entries) {
761 int apic;
762 for(apic = 0; apic < nr_ioapics; apic++) {
763 if (mpc_ioapic_id(apic) == mp_irqs[i].dstapic)
764 return apic;
765 }
766 }
767
768 return -1;
769}
770
771#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
772/*
773 * EISA Edge/Level control register, ELCR
774 */
775static int EISA_ELCR(unsigned int irq)
776{
777 if (irq < legacy_pic->nr_legacy_irqs) {
778 unsigned int port = 0x4d0 + (irq >> 3);
779 return (inb(port) >> (irq & 7)) & 1;
780 }
781 apic_printk(APIC_VERBOSE, KERN_INFO
782 "Broken MPtable reports ISA irq %d\n", irq);
783 return 0;
784}
785
786#endif
787
788/* ISA interrupts are always polarity zero edge triggered,
789 * when listed as conforming in the MP table. */
790
791#define default_ISA_trigger(idx) (0)
792#define default_ISA_polarity(idx) (0)
793
794/* EISA interrupts are always polarity zero and can be edge or level
795 * trigger depending on the ELCR value. If an interrupt is listed as
796 * EISA conforming in the MP table, that means its trigger type must
797 * be read in from the ELCR */
798
799#define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].srcbusirq))
800#define default_EISA_polarity(idx) default_ISA_polarity(idx)
801
802/* PCI interrupts are always polarity one level triggered,
803 * when listed as conforming in the MP table. */
804
805#define default_PCI_trigger(idx) (1)
806#define default_PCI_polarity(idx) (1)
807
808/* MCA interrupts are always polarity zero level triggered,
809 * when listed as conforming in the MP table. */
810
811#define default_MCA_trigger(idx) (1)
812#define default_MCA_polarity(idx) default_ISA_polarity(idx)
813
814static int irq_polarity(int idx)
815{
816 int bus = mp_irqs[idx].srcbus;
817 int polarity;
818
819 /*
820 * Determine IRQ line polarity (high active or low active):
821 */
822 switch (mp_irqs[idx].irqflag & 3)
823 {
824 case 0: /* conforms, ie. bus-type dependent polarity */
825 if (test_bit(bus, mp_bus_not_pci))
826 polarity = default_ISA_polarity(idx);
827 else
828 polarity = default_PCI_polarity(idx);
829 break;
830 case 1: /* high active */
831 {
832 polarity = 0;
833 break;
834 }
835 case 2: /* reserved */
836 {
837 printk(KERN_WARNING "broken BIOS!!\n");
838 polarity = 1;
839 break;
840 }
841 case 3: /* low active */
842 {
843 polarity = 1;
844 break;
845 }
846 default: /* invalid */
847 {
848 printk(KERN_WARNING "broken BIOS!!\n");
849 polarity = 1;
850 break;
851 }
852 }
853 return polarity;
854}
855
856static int irq_trigger(int idx)
857{
858 int bus = mp_irqs[idx].srcbus;
859 int trigger;
860
861 /*
862 * Determine IRQ trigger mode (edge or level sensitive):
863 */
864 switch ((mp_irqs[idx].irqflag>>2) & 3)
865 {
866 case 0: /* conforms, ie. bus-type dependent */
867 if (test_bit(bus, mp_bus_not_pci))
868 trigger = default_ISA_trigger(idx);
869 else
870 trigger = default_PCI_trigger(idx);
871#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
872 switch (mp_bus_id_to_type[bus]) {
873 case MP_BUS_ISA: /* ISA pin */
874 {
875 /* set before the switch */
876 break;
877 }
878 case MP_BUS_EISA: /* EISA pin */
879 {
880 trigger = default_EISA_trigger(idx);
881 break;
882 }
883 case MP_BUS_PCI: /* PCI pin */
884 {
885 /* set before the switch */
886 break;
887 }
888 case MP_BUS_MCA: /* MCA pin */
889 {
890 trigger = default_MCA_trigger(idx);
891 break;
892 }
893 default:
894 {
895 printk(KERN_WARNING "broken BIOS!!\n");
896 trigger = 1;
897 break;
898 }
899 }
900#endif
901 break;
902 case 1: /* edge */
903 {
904 trigger = 0;
905 break;
906 }
907 case 2: /* reserved */
908 {
909 printk(KERN_WARNING "broken BIOS!!\n");
910 trigger = 1;
911 break;
912 }
913 case 3: /* level */
914 {
915 trigger = 1;
916 break;
917 }
918 default: /* invalid */
919 {
920 printk(KERN_WARNING "broken BIOS!!\n");
921 trigger = 0;
922 break;
923 }
924 }
925 return trigger;
926}
927
928static int pin_2_irq(int idx, int apic, int pin)
929{
930 int irq;
931 int bus = mp_irqs[idx].srcbus;
932 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(apic);
933
934 /*
935 * Debugging check, we are in big trouble if this message pops up!
936 */
937 if (mp_irqs[idx].dstirq != pin)
938 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
939
940 if (test_bit(bus, mp_bus_not_pci)) {
941 irq = mp_irqs[idx].srcbusirq;
942 } else {
943 u32 gsi = gsi_cfg->gsi_base + pin;
944
945 if (gsi >= NR_IRQS_LEGACY)
946 irq = gsi;
947 else
948 irq = gsi_top + gsi;
949 }
950
951#ifdef CONFIG_X86_32
952 /*
953 * PCI IRQ command line redirection. Yes, limits are hardcoded.
954 */
955 if ((pin >= 16) && (pin <= 23)) {
956 if (pirq_entries[pin-16] != -1) {
957 if (!pirq_entries[pin-16]) {
958 apic_printk(APIC_VERBOSE, KERN_DEBUG
959 "disabling PIRQ%d\n", pin-16);
960 } else {
961 irq = pirq_entries[pin-16];
962 apic_printk(APIC_VERBOSE, KERN_DEBUG
963 "using PIRQ%d -> IRQ %d\n",
964 pin-16, irq);
965 }
966 }
967 }
968#endif
969
970 return irq;
971}
972
973/*
974 * Find a specific PCI IRQ entry.
975 * Not an __init, possibly needed by modules
976 */
977int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
978 struct io_apic_irq_attr *irq_attr)
979{
980 int apic, i, best_guess = -1;
981
982 apic_printk(APIC_DEBUG,
983 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
984 bus, slot, pin);
985 if (test_bit(bus, mp_bus_not_pci)) {
986 apic_printk(APIC_VERBOSE,
987 "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
988 return -1;
989 }
990 for (i = 0; i < mp_irq_entries; i++) {
991 int lbus = mp_irqs[i].srcbus;
992
993 for (apic = 0; apic < nr_ioapics; apic++)
994 if (mpc_ioapic_id(apic) == mp_irqs[i].dstapic ||
995 mp_irqs[i].dstapic == MP_APIC_ALL)
996 break;
997
998 if (!test_bit(lbus, mp_bus_not_pci) &&
999 !mp_irqs[i].irqtype &&
1000 (bus == lbus) &&
1001 (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1002 int irq = pin_2_irq(i, apic, mp_irqs[i].dstirq);
1003
1004 if (!(apic || IO_APIC_IRQ(irq)))
1005 continue;
1006
1007 if (pin == (mp_irqs[i].srcbusirq & 3)) {
1008 set_io_apic_irq_attr(irq_attr, apic,
1009 mp_irqs[i].dstirq,
1010 irq_trigger(i),
1011 irq_polarity(i));
1012 return irq;
1013 }
1014 /*
1015 * Use the first all-but-pin matching entry as a
1016 * best-guess fuzzy result for broken mptables.
1017 */
1018 if (best_guess < 0) {
1019 set_io_apic_irq_attr(irq_attr, apic,
1020 mp_irqs[i].dstirq,
1021 irq_trigger(i),
1022 irq_polarity(i));
1023 best_guess = irq;
1024 }
1025 }
1026 }
1027 return best_guess;
1028}
1029EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1030
1031void lock_vector_lock(void)
1032{
1033 /* Used to the online set of cpus does not change
1034 * during assign_irq_vector.
1035 */
1036 raw_spin_lock(&vector_lock);
1037}
1038
1039void unlock_vector_lock(void)
1040{
1041 raw_spin_unlock(&vector_lock);
1042}
1043
1044static int
1045__assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1046{
1047 /*
1048 * NOTE! The local APIC isn't very good at handling
1049 * multiple interrupts at the same interrupt level.
1050 * As the interrupt level is determined by taking the
1051 * vector number and shifting that right by 4, we
1052 * want to spread these out a bit so that they don't
1053 * all fall in the same interrupt level.
1054 *
1055 * Also, we've got to be careful not to trash gate
1056 * 0x80, because int 0x80 is hm, kind of importantish. ;)
1057 */
1058 static int current_vector = FIRST_EXTERNAL_VECTOR + VECTOR_OFFSET_START;
1059 static int current_offset = VECTOR_OFFSET_START % 8;
1060 unsigned int old_vector;
1061 int cpu, err;
1062 cpumask_var_t tmp_mask;
1063
1064 if (cfg->move_in_progress)
1065 return -EBUSY;
1066
1067 if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
1068 return -ENOMEM;
1069
1070 old_vector = cfg->vector;
1071 if (old_vector) {
1072 cpumask_and(tmp_mask, mask, cpu_online_mask);
1073 cpumask_and(tmp_mask, cfg->domain, tmp_mask);
1074 if (!cpumask_empty(tmp_mask)) {
1075 free_cpumask_var(tmp_mask);
1076 return 0;
1077 }
1078 }
1079
1080 /* Only try and allocate irqs on cpus that are present */
1081 err = -ENOSPC;
1082 for_each_cpu_and(cpu, mask, cpu_online_mask) {
1083 int new_cpu;
1084 int vector, offset;
1085
1086 apic->vector_allocation_domain(cpu, tmp_mask);
1087
1088 vector = current_vector;
1089 offset = current_offset;
1090next:
1091 vector += 8;
1092 if (vector >= first_system_vector) {
1093 /* If out of vectors on large boxen, must share them. */
1094 offset = (offset + 1) % 8;
1095 vector = FIRST_EXTERNAL_VECTOR + offset;
1096 }
1097 if (unlikely(current_vector == vector))
1098 continue;
1099
1100 if (test_bit(vector, used_vectors))
1101 goto next;
1102
1103 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1104 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
1105 goto next;
1106 /* Found one! */
1107 current_vector = vector;
1108 current_offset = offset;
1109 if (old_vector) {
1110 cfg->move_in_progress = 1;
1111 cpumask_copy(cfg->old_domain, cfg->domain);
1112 }
1113 for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask)
1114 per_cpu(vector_irq, new_cpu)[vector] = irq;
1115 cfg->vector = vector;
1116 cpumask_copy(cfg->domain, tmp_mask);
1117 err = 0;
1118 break;
1119 }
1120 free_cpumask_var(tmp_mask);
1121 return err;
1122}
1123
1124int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1125{
1126 int err;
1127 unsigned long flags;
1128
1129 raw_spin_lock_irqsave(&vector_lock, flags);
1130 err = __assign_irq_vector(irq, cfg, mask);
1131 raw_spin_unlock_irqrestore(&vector_lock, flags);
1132 return err;
1133}
1134
1135static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
1136{
1137 int cpu, vector;
1138
1139 BUG_ON(!cfg->vector);
1140
1141 vector = cfg->vector;
1142 for_each_cpu_and(cpu, cfg->domain, cpu_online_mask)
1143 per_cpu(vector_irq, cpu)[vector] = -1;
1144
1145 cfg->vector = 0;
1146 cpumask_clear(cfg->domain);
1147
1148 if (likely(!cfg->move_in_progress))
1149 return;
1150 for_each_cpu_and(cpu, cfg->old_domain, cpu_online_mask) {
1151 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
1152 vector++) {
1153 if (per_cpu(vector_irq, cpu)[vector] != irq)
1154 continue;
1155 per_cpu(vector_irq, cpu)[vector] = -1;
1156 break;
1157 }
1158 }
1159 cfg->move_in_progress = 0;
1160}
1161
1162void __setup_vector_irq(int cpu)
1163{
1164 /* Initialize vector_irq on a new cpu */
1165 int irq, vector;
1166 struct irq_cfg *cfg;
1167
1168 /*
1169 * vector_lock will make sure that we don't run into irq vector
1170 * assignments that might be happening on another cpu in parallel,
1171 * while we setup our initial vector to irq mappings.
1172 */
1173 raw_spin_lock(&vector_lock);
1174 /* Mark the inuse vectors */
1175 for_each_active_irq(irq) {
1176 cfg = irq_get_chip_data(irq);
1177 if (!cfg)
1178 continue;
1179 /*
1180 * If it is a legacy IRQ handled by the legacy PIC, this cpu
1181 * will be part of the irq_cfg's domain.
1182 */
1183 if (irq < legacy_pic->nr_legacy_irqs && !IO_APIC_IRQ(irq))
1184 cpumask_set_cpu(cpu, cfg->domain);
1185
1186 if (!cpumask_test_cpu(cpu, cfg->domain))
1187 continue;
1188 vector = cfg->vector;
1189 per_cpu(vector_irq, cpu)[vector] = irq;
1190 }
1191 /* Mark the free vectors */
1192 for (vector = 0; vector < NR_VECTORS; ++vector) {
1193 irq = per_cpu(vector_irq, cpu)[vector];
1194 if (irq < 0)
1195 continue;
1196
1197 cfg = irq_cfg(irq);
1198 if (!cpumask_test_cpu(cpu, cfg->domain))
1199 per_cpu(vector_irq, cpu)[vector] = -1;
1200 }
1201 raw_spin_unlock(&vector_lock);
1202}
1203
1204static struct irq_chip ioapic_chip;
1205static struct irq_chip ir_ioapic_chip;
1206
1207#ifdef CONFIG_X86_32
1208static inline int IO_APIC_irq_trigger(int irq)
1209{
1210 int apic, idx, pin;
1211
1212 for (apic = 0; apic < nr_ioapics; apic++) {
1213 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1214 idx = find_irq_entry(apic, pin, mp_INT);
1215 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1216 return irq_trigger(idx);
1217 }
1218 }
1219 /*
1220 * nonexistent IRQs are edge default
1221 */
1222 return 0;
1223}
1224#else
1225static inline int IO_APIC_irq_trigger(int irq)
1226{
1227 return 1;
1228}
1229#endif
1230
1231static void ioapic_register_intr(unsigned int irq, struct irq_cfg *cfg,
1232 unsigned long trigger)
1233{
1234 struct irq_chip *chip = &ioapic_chip;
1235 irq_flow_handler_t hdl;
1236 bool fasteoi;
1237
1238 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1239 trigger == IOAPIC_LEVEL) {
1240 irq_set_status_flags(irq, IRQ_LEVEL);
1241 fasteoi = true;
1242 } else {
1243 irq_clear_status_flags(irq, IRQ_LEVEL);
1244 fasteoi = false;
1245 }
1246
1247 if (irq_remapped(cfg)) {
1248 irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
1249 chip = &ir_ioapic_chip;
1250 fasteoi = trigger != 0;
1251 }
1252
1253 hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq;
1254 irq_set_chip_and_handler_name(irq, chip, hdl,
1255 fasteoi ? "fasteoi" : "edge");
1256}
1257
1258static int setup_ioapic_entry(int apic_id, int irq,
1259 struct IO_APIC_route_entry *entry,
1260 unsigned int destination, int trigger,
1261 int polarity, int vector, int pin)
1262{
1263 /*
1264 * add it to the IO-APIC irq-routing table:
1265 */
1266 memset(entry,0,sizeof(*entry));
1267
1268 if (intr_remapping_enabled) {
1269 struct intel_iommu *iommu = map_ioapic_to_ir(apic_id);
1270 struct irte irte;
1271 struct IR_IO_APIC_route_entry *ir_entry =
1272 (struct IR_IO_APIC_route_entry *) entry;
1273 int index;
1274
1275 if (!iommu)
1276 panic("No mapping iommu for ioapic %d\n", apic_id);
1277
1278 index = alloc_irte(iommu, irq, 1);
1279 if (index < 0)
1280 panic("Failed to allocate IRTE for ioapic %d\n", apic_id);
1281
1282 prepare_irte(&irte, vector, destination);
1283
1284 /* Set source-id of interrupt request */
1285 set_ioapic_sid(&irte, apic_id);
1286
1287 modify_irte(irq, &irte);
1288
1289 ir_entry->index2 = (index >> 15) & 0x1;
1290 ir_entry->zero = 0;
1291 ir_entry->format = 1;
1292 ir_entry->index = (index & 0x7fff);
1293 /*
1294 * IO-APIC RTE will be configured with virtual vector.
1295 * irq handler will do the explicit EOI to the io-apic.
1296 */
1297 ir_entry->vector = pin;
1298
1299 apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: "
1300 "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d "
1301 "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X "
1302 "Avail:%X Vector:%02X Dest:%08X "
1303 "SID:%04X SQ:%X SVT:%X)\n",
1304 apic_id, irte.present, irte.fpd, irte.dst_mode,
1305 irte.redir_hint, irte.trigger_mode, irte.dlvry_mode,
1306 irte.avail, irte.vector, irte.dest_id,
1307 irte.sid, irte.sq, irte.svt);
1308 } else {
1309 entry->delivery_mode = apic->irq_delivery_mode;
1310 entry->dest_mode = apic->irq_dest_mode;
1311 entry->dest = destination;
1312 entry->vector = vector;
1313 }
1314
1315 entry->mask = 0; /* enable IRQ */
1316 entry->trigger = trigger;
1317 entry->polarity = polarity;
1318
1319 /* Mask level triggered irqs.
1320 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
1321 */
1322 if (trigger)
1323 entry->mask = 1;
1324 return 0;
1325}
1326
1327static void setup_ioapic_irq(int apic_id, int pin, unsigned int irq,
1328 struct irq_cfg *cfg, int trigger, int polarity)
1329{
1330 struct IO_APIC_route_entry entry;
1331 unsigned int dest;
1332
1333 if (!IO_APIC_IRQ(irq))
1334 return;
1335 /*
1336 * For legacy irqs, cfg->domain starts with cpu 0 for legacy
1337 * controllers like 8259. Now that IO-APIC can handle this irq, update
1338 * the cfg->domain.
1339 */
1340 if (irq < legacy_pic->nr_legacy_irqs && cpumask_test_cpu(0, cfg->domain))
1341 apic->vector_allocation_domain(0, cfg->domain);
1342
1343 if (assign_irq_vector(irq, cfg, apic->target_cpus()))
1344 return;
1345
1346 dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
1347
1348 apic_printk(APIC_VERBOSE,KERN_DEBUG
1349 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1350 "IRQ %d Mode:%i Active:%i Dest:%d)\n",
1351 apic_id, mpc_ioapic_id(apic_id), pin, cfg->vector,
1352 irq, trigger, polarity, dest);
1353
1354
1355 if (setup_ioapic_entry(mpc_ioapic_id(apic_id), irq, &entry,
1356 dest, trigger, polarity, cfg->vector, pin)) {
1357 printk("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1358 mpc_ioapic_id(apic_id), pin);
1359 __clear_irq_vector(irq, cfg);
1360 return;
1361 }
1362
1363 ioapic_register_intr(irq, cfg, trigger);
1364 if (irq < legacy_pic->nr_legacy_irqs)
1365 legacy_pic->mask(irq);
1366
1367 ioapic_write_entry(apic_id, pin, entry);
1368}
1369
1370static bool __init io_apic_pin_not_connected(int idx, int apic_id, int pin)
1371{
1372 if (idx != -1)
1373 return false;
1374
1375 apic_printk(APIC_VERBOSE, KERN_DEBUG " apic %d pin %d not connected\n",
1376 mpc_ioapic_id(apic_id), pin);
1377 return true;
1378}
1379
1380static void __init __io_apic_setup_irqs(unsigned int apic_id)
1381{
1382 int idx, node = cpu_to_node(0);
1383 struct io_apic_irq_attr attr;
1384 unsigned int pin, irq;
1385
1386 for (pin = 0; pin < ioapics[apic_id].nr_registers; pin++) {
1387 idx = find_irq_entry(apic_id, pin, mp_INT);
1388 if (io_apic_pin_not_connected(idx, apic_id, pin))
1389 continue;
1390
1391 irq = pin_2_irq(idx, apic_id, pin);
1392
1393 if ((apic_id > 0) && (irq > 16))
1394 continue;
1395
1396 /*
1397 * Skip the timer IRQ if there's a quirk handler
1398 * installed and if it returns 1:
1399 */
1400 if (apic->multi_timer_check &&
1401 apic->multi_timer_check(apic_id, irq))
1402 continue;
1403
1404 set_io_apic_irq_attr(&attr, apic_id, pin, irq_trigger(idx),
1405 irq_polarity(idx));
1406
1407 io_apic_setup_irq_pin(irq, node, &attr);
1408 }
1409}
1410
1411static void __init setup_IO_APIC_irqs(void)
1412{
1413 unsigned int apic_id;
1414
1415 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1416
1417 for (apic_id = 0; apic_id < nr_ioapics; apic_id++)
1418 __io_apic_setup_irqs(apic_id);
1419}
1420
1421/*
1422 * for the gsit that is not in first ioapic
1423 * but could not use acpi_register_gsi()
1424 * like some special sci in IBM x3330
1425 */
1426void setup_IO_APIC_irq_extra(u32 gsi)
1427{
1428 int apic_id = 0, pin, idx, irq, node = cpu_to_node(0);
1429 struct io_apic_irq_attr attr;
1430
1431 /*
1432 * Convert 'gsi' to 'ioapic.pin'.
1433 */
1434 apic_id = mp_find_ioapic(gsi);
1435 if (apic_id < 0)
1436 return;
1437
1438 pin = mp_find_ioapic_pin(apic_id, gsi);
1439 idx = find_irq_entry(apic_id, pin, mp_INT);
1440 if (idx == -1)
1441 return;
1442
1443 irq = pin_2_irq(idx, apic_id, pin);
1444
1445 /* Only handle the non legacy irqs on secondary ioapics */
1446 if (apic_id == 0 || irq < NR_IRQS_LEGACY)
1447 return;
1448
1449 set_io_apic_irq_attr(&attr, apic_id, pin, irq_trigger(idx),
1450 irq_polarity(idx));
1451
1452 io_apic_setup_irq_pin_once(irq, node, &attr);
1453}
1454
1455/*
1456 * Set up the timer pin, possibly with the 8259A-master behind.
1457 */
1458static void __init setup_timer_IRQ0_pin(unsigned int apic_id, unsigned int pin,
1459 int vector)
1460{
1461 struct IO_APIC_route_entry entry;
1462
1463 if (intr_remapping_enabled)
1464 return;
1465
1466 memset(&entry, 0, sizeof(entry));
1467
1468 /*
1469 * We use logical delivery to get the timer IRQ
1470 * to the first CPU.
1471 */
1472 entry.dest_mode = apic->irq_dest_mode;
1473 entry.mask = 0; /* don't mask IRQ for edge */
1474 entry.dest = apic->cpu_mask_to_apicid(apic->target_cpus());
1475 entry.delivery_mode = apic->irq_delivery_mode;
1476 entry.polarity = 0;
1477 entry.trigger = 0;
1478 entry.vector = vector;
1479
1480 /*
1481 * The timer IRQ doesn't have to know that behind the
1482 * scene we may have a 8259A-master in AEOI mode ...
1483 */
1484 irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
1485 "edge");
1486
1487 /*
1488 * Add it to the IO-APIC irq-routing table:
1489 */
1490 ioapic_write_entry(apic_id, pin, entry);
1491}
1492
1493
1494__apicdebuginit(void) print_IO_APIC(void)
1495{
1496 int apic, i;
1497 union IO_APIC_reg_00 reg_00;
1498 union IO_APIC_reg_01 reg_01;
1499 union IO_APIC_reg_02 reg_02;
1500 union IO_APIC_reg_03 reg_03;
1501 unsigned long flags;
1502 struct irq_cfg *cfg;
1503 unsigned int irq;
1504
1505 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1506 for (i = 0; i < nr_ioapics; i++)
1507 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1508 mpc_ioapic_id(i), ioapics[i].nr_registers);
1509
1510 /*
1511 * We are a bit conservative about what we expect. We have to
1512 * know about every hardware change ASAP.
1513 */
1514 printk(KERN_INFO "testing the IO APIC.......................\n");
1515
1516 for (apic = 0; apic < nr_ioapics; apic++) {
1517
1518 raw_spin_lock_irqsave(&ioapic_lock, flags);
1519 reg_00.raw = io_apic_read(apic, 0);
1520 reg_01.raw = io_apic_read(apic, 1);
1521 if (reg_01.bits.version >= 0x10)
1522 reg_02.raw = io_apic_read(apic, 2);
1523 if (reg_01.bits.version >= 0x20)
1524 reg_03.raw = io_apic_read(apic, 3);
1525 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1526
1527 printk("\n");
1528 printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(apic));
1529 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1530 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1531 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
1532 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
1533
1534 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)®_01);
1535 printk(KERN_DEBUG "....... : max redirection entries: %02X\n",
1536 reg_01.bits.entries);
1537
1538 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
1539 printk(KERN_DEBUG "....... : IO APIC version: %02X\n",
1540 reg_01.bits.version);
1541
1542 /*
1543 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
1544 * but the value of reg_02 is read as the previous read register
1545 * value, so ignore it if reg_02 == reg_01.
1546 */
1547 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1548 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1549 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
1550 }
1551
1552 /*
1553 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
1554 * or reg_03, but the value of reg_0[23] is read as the previous read
1555 * register value, so ignore it if reg_03 == reg_0[12].
1556 */
1557 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1558 reg_03.raw != reg_01.raw) {
1559 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1560 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
1561 }
1562
1563 printk(KERN_DEBUG ".... IRQ redirection table:\n");
1564
1565 if (intr_remapping_enabled) {
1566 printk(KERN_DEBUG " NR Indx Fmt Mask Trig IRR"
1567 " Pol Stat Indx2 Zero Vect:\n");
1568 } else {
1569 printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
1570 " Stat Dmod Deli Vect:\n");
1571 }
1572
1573 for (i = 0; i <= reg_01.bits.entries; i++) {
1574 if (intr_remapping_enabled) {
1575 struct IO_APIC_route_entry entry;
1576 struct IR_IO_APIC_route_entry *ir_entry;
1577
1578 entry = ioapic_read_entry(apic, i);
1579 ir_entry = (struct IR_IO_APIC_route_entry *) &entry;
1580 printk(KERN_DEBUG " %02x %04X ",
1581 i,
1582 ir_entry->index
1583 );
1584 printk("%1d %1d %1d %1d %1d "
1585 "%1d %1d %X %02X\n",
1586 ir_entry->format,
1587 ir_entry->mask,
1588 ir_entry->trigger,
1589 ir_entry->irr,
1590 ir_entry->polarity,
1591 ir_entry->delivery_status,
1592 ir_entry->index2,
1593 ir_entry->zero,
1594 ir_entry->vector
1595 );
1596 } else {
1597 struct IO_APIC_route_entry entry;
1598
1599 entry = ioapic_read_entry(apic, i);
1600 printk(KERN_DEBUG " %02x %02X ",
1601 i,
1602 entry.dest
1603 );
1604 printk("%1d %1d %1d %1d %1d "
1605 "%1d %1d %02X\n",
1606 entry.mask,
1607 entry.trigger,
1608 entry.irr,
1609 entry.polarity,
1610 entry.delivery_status,
1611 entry.dest_mode,
1612 entry.delivery_mode,
1613 entry.vector
1614 );
1615 }
1616 }
1617 }
1618
1619 printk(KERN_DEBUG "IRQ to pin mappings:\n");
1620 for_each_active_irq(irq) {
1621 struct irq_pin_list *entry;
1622
1623 cfg = irq_get_chip_data(irq);
1624 if (!cfg)
1625 continue;
1626 entry = cfg->irq_2_pin;
1627 if (!entry)
1628 continue;
1629 printk(KERN_DEBUG "IRQ%d ", irq);
1630 for_each_irq_pin(entry, cfg->irq_2_pin)
1631 printk("-> %d:%d", entry->apic, entry->pin);
1632 printk("\n");
1633 }
1634
1635 printk(KERN_INFO ".................................... done.\n");
1636
1637 return;
1638}
1639
1640__apicdebuginit(void) print_APIC_field(int base)
1641{
1642 int i;
1643
1644 printk(KERN_DEBUG);
1645
1646 for (i = 0; i < 8; i++)
1647 printk(KERN_CONT "%08x", apic_read(base + i*0x10));
1648
1649 printk(KERN_CONT "\n");
1650}
1651
1652__apicdebuginit(void) print_local_APIC(void *dummy)
1653{
1654 unsigned int i, v, ver, maxlvt;
1655 u64 icr;
1656
1657 printk(KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1658 smp_processor_id(), hard_smp_processor_id());
1659 v = apic_read(APIC_ID);
1660 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, read_apic_id());
1661 v = apic_read(APIC_LVR);
1662 printk(KERN_INFO "... APIC VERSION: %08x\n", v);
1663 ver = GET_APIC_VERSION(v);
1664 maxlvt = lapic_get_maxlvt();
1665
1666 v = apic_read(APIC_TASKPRI);
1667 printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1668
1669 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1670 if (!APIC_XAPIC(ver)) {
1671 v = apic_read(APIC_ARBPRI);
1672 printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
1673 v & APIC_ARBPRI_MASK);
1674 }
1675 v = apic_read(APIC_PROCPRI);
1676 printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
1677 }
1678
1679 /*
1680 * Remote read supported only in the 82489DX and local APIC for
1681 * Pentium processors.
1682 */
1683 if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
1684 v = apic_read(APIC_RRR);
1685 printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
1686 }
1687
1688 v = apic_read(APIC_LDR);
1689 printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
1690 if (!x2apic_enabled()) {
1691 v = apic_read(APIC_DFR);
1692 printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
1693 }
1694 v = apic_read(APIC_SPIV);
1695 printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);
1696
1697 printk(KERN_DEBUG "... APIC ISR field:\n");
1698 print_APIC_field(APIC_ISR);
1699 printk(KERN_DEBUG "... APIC TMR field:\n");
1700 print_APIC_field(APIC_TMR);
1701 printk(KERN_DEBUG "... APIC IRR field:\n");
1702 print_APIC_field(APIC_IRR);
1703
1704 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1705 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1706 apic_write(APIC_ESR, 0);
1707
1708 v = apic_read(APIC_ESR);
1709 printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
1710 }
1711
1712 icr = apic_icr_read();
1713 printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
1714 printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
1715
1716 v = apic_read(APIC_LVTT);
1717 printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);
1718
1719 if (maxlvt > 3) { /* PC is LVT#4. */
1720 v = apic_read(APIC_LVTPC);
1721 printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
1722 }
1723 v = apic_read(APIC_LVT0);
1724 printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
1725 v = apic_read(APIC_LVT1);
1726 printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);
1727
1728 if (maxlvt > 2) { /* ERR is LVT#3. */
1729 v = apic_read(APIC_LVTERR);
1730 printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
1731 }
1732
1733 v = apic_read(APIC_TMICT);
1734 printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
1735 v = apic_read(APIC_TMCCT);
1736 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1737 v = apic_read(APIC_TDCR);
1738 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1739
1740 if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
1741 v = apic_read(APIC_EFEAT);
1742 maxlvt = (v >> 16) & 0xff;
1743 printk(KERN_DEBUG "... APIC EFEAT: %08x\n", v);
1744 v = apic_read(APIC_ECTRL);
1745 printk(KERN_DEBUG "... APIC ECTRL: %08x\n", v);
1746 for (i = 0; i < maxlvt; i++) {
1747 v = apic_read(APIC_EILVTn(i));
1748 printk(KERN_DEBUG "... APIC EILVT%d: %08x\n", i, v);
1749 }
1750 }
1751 printk("\n");
1752}
1753
1754__apicdebuginit(void) print_local_APICs(int maxcpu)
1755{
1756 int cpu;
1757
1758 if (!maxcpu)
1759 return;
1760
1761 preempt_disable();
1762 for_each_online_cpu(cpu) {
1763 if (cpu >= maxcpu)
1764 break;
1765 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1766 }
1767 preempt_enable();
1768}
1769
1770__apicdebuginit(void) print_PIC(void)
1771{
1772 unsigned int v;
1773 unsigned long flags;
1774
1775 if (!legacy_pic->nr_legacy_irqs)
1776 return;
1777
1778 printk(KERN_DEBUG "\nprinting PIC contents\n");
1779
1780 raw_spin_lock_irqsave(&i8259A_lock, flags);
1781
1782 v = inb(0xa1) << 8 | inb(0x21);
1783 printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
1784
1785 v = inb(0xa0) << 8 | inb(0x20);
1786 printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
1787
1788 outb(0x0b,0xa0);
1789 outb(0x0b,0x20);
1790 v = inb(0xa0) << 8 | inb(0x20);
1791 outb(0x0a,0xa0);
1792 outb(0x0a,0x20);
1793
1794 raw_spin_unlock_irqrestore(&i8259A_lock, flags);
1795
1796 printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
1797
1798 v = inb(0x4d1) << 8 | inb(0x4d0);
1799 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
1800}
1801
1802static int __initdata show_lapic = 1;
1803static __init int setup_show_lapic(char *arg)
1804{
1805 int num = -1;
1806
1807 if (strcmp(arg, "all") == 0) {
1808 show_lapic = CONFIG_NR_CPUS;
1809 } else {
1810 get_option(&arg, &num);
1811 if (num >= 0)
1812 show_lapic = num;
1813 }
1814
1815 return 1;
1816}
1817__setup("show_lapic=", setup_show_lapic);
1818
1819__apicdebuginit(int) print_ICs(void)
1820{
1821 if (apic_verbosity == APIC_QUIET)
1822 return 0;
1823
1824 print_PIC();
1825
1826 /* don't print out if apic is not there */
1827 if (!cpu_has_apic && !apic_from_smp_config())
1828 return 0;
1829
1830 print_local_APICs(show_lapic);
1831 print_IO_APIC();
1832
1833 return 0;
1834}
1835
1836late_initcall(print_ICs);
1837
1838
1839/* Where if anywhere is the i8259 connect in external int mode */
1840static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1841
1842void __init enable_IO_APIC(void)
1843{
1844 int i8259_apic, i8259_pin;
1845 int apic;
1846
1847 if (!legacy_pic->nr_legacy_irqs)
1848 return;
1849
1850 for(apic = 0; apic < nr_ioapics; apic++) {
1851 int pin;
1852 /* See if any of the pins is in ExtINT mode */
1853 for (pin = 0; pin < ioapics[apic].nr_registers; pin++) {
1854 struct IO_APIC_route_entry entry;
1855 entry = ioapic_read_entry(apic, pin);
1856
1857 /* If the interrupt line is enabled and in ExtInt mode
1858 * I have found the pin where the i8259 is connected.
1859 */
1860 if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
1861 ioapic_i8259.apic = apic;
1862 ioapic_i8259.pin = pin;
1863 goto found_i8259;
1864 }
1865 }
1866 }
1867 found_i8259:
1868 /* Look to see what if the MP table has reported the ExtINT */
1869 /* If we could not find the appropriate pin by looking at the ioapic
1870 * the i8259 probably is not connected the ioapic but give the
1871 * mptable a chance anyway.
1872 */
1873 i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
1874 i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1875 /* Trust the MP table if nothing is setup in the hardware */
1876 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1877 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1878 ioapic_i8259.pin = i8259_pin;
1879 ioapic_i8259.apic = i8259_apic;
1880 }
1881 /* Complain if the MP table and the hardware disagree */
1882 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1883 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1884 {
1885 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
1886 }
1887
1888 /*
1889 * Do not trust the IO-APIC being empty at bootup
1890 */
1891 clear_IO_APIC();
1892}
1893
1894/*
1895 * Not an __init, needed by the reboot code
1896 */
1897void disable_IO_APIC(void)
1898{
1899 /*
1900 * Clear the IO-APIC before rebooting:
1901 */
1902 clear_IO_APIC();
1903
1904 if (!legacy_pic->nr_legacy_irqs)
1905 return;
1906
1907 /*
1908 * If the i8259 is routed through an IOAPIC
1909 * Put that IOAPIC in virtual wire mode
1910 * so legacy interrupts can be delivered.
1911 *
1912 * With interrupt-remapping, for now we will use virtual wire A mode,
1913 * as virtual wire B is little complex (need to configure both
1914 * IOAPIC RTE as well as interrupt-remapping table entry).
1915 * As this gets called during crash dump, keep this simple for now.
1916 */
1917 if (ioapic_i8259.pin != -1 && !intr_remapping_enabled) {
1918 struct IO_APIC_route_entry entry;
1919
1920 memset(&entry, 0, sizeof(entry));
1921 entry.mask = 0; /* Enabled */
1922 entry.trigger = 0; /* Edge */
1923 entry.irr = 0;
1924 entry.polarity = 0; /* High */
1925 entry.delivery_status = 0;
1926 entry.dest_mode = 0; /* Physical */
1927 entry.delivery_mode = dest_ExtINT; /* ExtInt */
1928 entry.vector = 0;
1929 entry.dest = read_apic_id();
1930
1931 /*
1932 * Add it to the IO-APIC irq-routing table:
1933 */
1934 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1935 }
1936
1937 /*
1938 * Use virtual wire A mode when interrupt remapping is enabled.
1939 */
1940 if (cpu_has_apic || apic_from_smp_config())
1941 disconnect_bsp_APIC(!intr_remapping_enabled &&
1942 ioapic_i8259.pin != -1);
1943}
1944
1945#ifdef CONFIG_X86_32
1946/*
1947 * function to set the IO-APIC physical IDs based on the
1948 * values stored in the MPC table.
1949 *
1950 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999
1951 */
1952void __init setup_ioapic_ids_from_mpc_nocheck(void)
1953{
1954 union IO_APIC_reg_00 reg_00;
1955 physid_mask_t phys_id_present_map;
1956 int apic_id;
1957 int i;
1958 unsigned char old_id;
1959 unsigned long flags;
1960
1961 /*
1962 * This is broken; anything with a real cpu count has to
1963 * circumvent this idiocy regardless.
1964 */
1965 apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map);
1966
1967 /*
1968 * Set the IOAPIC ID to the value stored in the MPC table.
1969 */
1970 for (apic_id = 0; apic_id < nr_ioapics; apic_id++) {
1971
1972 /* Read the register 0 value */
1973 raw_spin_lock_irqsave(&ioapic_lock, flags);
1974 reg_00.raw = io_apic_read(apic_id, 0);
1975 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1976
1977 old_id = mpc_ioapic_id(apic_id);
1978
1979 if (mpc_ioapic_id(apic_id) >= get_physical_broadcast()) {
1980 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
1981 apic_id, mpc_ioapic_id(apic_id));
1982 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1983 reg_00.bits.ID);
1984 ioapics[apic_id].mp_config.apicid = reg_00.bits.ID;
1985 }
1986
1987 /*
1988 * Sanity check, is the ID really free? Every APIC in a
1989 * system must have a unique ID or we get lots of nice
1990 * 'stuck on smp_invalidate_needed IPI wait' messages.
1991 */
1992 if (apic->check_apicid_used(&phys_id_present_map,
1993 mpc_ioapic_id(apic_id))) {
1994 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
1995 apic_id, mpc_ioapic_id(apic_id));
1996 for (i = 0; i < get_physical_broadcast(); i++)
1997 if (!physid_isset(i, phys_id_present_map))
1998 break;
1999 if (i >= get_physical_broadcast())
2000 panic("Max APIC ID exceeded!\n");
2001 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2002 i);
2003 physid_set(i, phys_id_present_map);
2004 ioapics[apic_id].mp_config.apicid = i;
2005 } else {
2006 physid_mask_t tmp;
2007 apic->apicid_to_cpu_present(mpc_ioapic_id(apic_id),
2008 &tmp);
2009 apic_printk(APIC_VERBOSE, "Setting %d in the "
2010 "phys_id_present_map\n",
2011 mpc_ioapic_id(apic_id));
2012 physids_or(phys_id_present_map, phys_id_present_map, tmp);
2013 }
2014
2015 /*
2016 * We need to adjust the IRQ routing table
2017 * if the ID changed.
2018 */
2019 if (old_id != mpc_ioapic_id(apic_id))
2020 for (i = 0; i < mp_irq_entries; i++)
2021 if (mp_irqs[i].dstapic == old_id)
2022 mp_irqs[i].dstapic
2023 = mpc_ioapic_id(apic_id);
2024
2025 /*
2026 * Update the ID register according to the right value
2027 * from the MPC table if they are different.
2028 */
2029 if (mpc_ioapic_id(apic_id) == reg_00.bits.ID)
2030 continue;
2031
2032 apic_printk(APIC_VERBOSE, KERN_INFO
2033 "...changing IO-APIC physical APIC ID to %d ...",
2034 mpc_ioapic_id(apic_id));
2035
2036 reg_00.bits.ID = mpc_ioapic_id(apic_id);
2037 raw_spin_lock_irqsave(&ioapic_lock, flags);
2038 io_apic_write(apic_id, 0, reg_00.raw);
2039 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2040
2041 /*
2042 * Sanity check
2043 */
2044 raw_spin_lock_irqsave(&ioapic_lock, flags);
2045 reg_00.raw = io_apic_read(apic_id, 0);
2046 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2047 if (reg_00.bits.ID != mpc_ioapic_id(apic_id))
2048 printk("could not set ID!\n");
2049 else
2050 apic_printk(APIC_VERBOSE, " ok.\n");
2051 }
2052}
2053
2054void __init setup_ioapic_ids_from_mpc(void)
2055{
2056
2057 if (acpi_ioapic)
2058 return;
2059 /*
2060 * Don't check I/O APIC IDs for xAPIC systems. They have
2061 * no meaning without the serial APIC bus.
2062 */
2063 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2064 || APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
2065 return;
2066 setup_ioapic_ids_from_mpc_nocheck();
2067}
2068#endif
2069
2070int no_timer_check __initdata;
2071
2072static int __init notimercheck(char *s)
2073{
2074 no_timer_check = 1;
2075 return 1;
2076}
2077__setup("no_timer_check", notimercheck);
2078
2079/*
2080 * There is a nasty bug in some older SMP boards, their mptable lies
2081 * about the timer IRQ. We do the following to work around the situation:
2082 *
2083 * - timer IRQ defaults to IO-APIC IRQ
2084 * - if this function detects that timer IRQs are defunct, then we fall
2085 * back to ISA timer IRQs
2086 */
2087static int __init timer_irq_works(void)
2088{
2089 unsigned long t1 = jiffies;
2090 unsigned long flags;
2091
2092 if (no_timer_check)
2093 return 1;
2094
2095 local_save_flags(flags);
2096 local_irq_enable();
2097 /* Let ten ticks pass... */
2098 mdelay((10 * 1000) / HZ);
2099 local_irq_restore(flags);
2100
2101 /*
2102 * Expect a few ticks at least, to be sure some possible
2103 * glue logic does not lock up after one or two first
2104 * ticks in a non-ExtINT mode. Also the local APIC
2105 * might have cached one ExtINT interrupt. Finally, at
2106 * least one tick may be lost due to delays.
2107 */
2108
2109 /* jiffies wrap? */
2110 if (time_after(jiffies, t1 + 4))
2111 return 1;
2112 return 0;
2113}
2114
2115/*
2116 * In the SMP+IOAPIC case it might happen that there are an unspecified
2117 * number of pending IRQ events unhandled. These cases are very rare,
2118 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
2119 * better to do it this way as thus we do not have to be aware of
2120 * 'pending' interrupts in the IRQ path, except at this point.
2121 */
2122/*
2123 * Edge triggered needs to resend any interrupt
2124 * that was delayed but this is now handled in the device
2125 * independent code.
2126 */
2127
2128/*
2129 * Starting up a edge-triggered IO-APIC interrupt is
2130 * nasty - we need to make sure that we get the edge.
2131 * If it is already asserted for some reason, we need
2132 * return 1 to indicate that is was pending.
2133 *
2134 * This is not complete - we should be able to fake
2135 * an edge even if it isn't on the 8259A...
2136 */
2137
2138static unsigned int startup_ioapic_irq(struct irq_data *data)
2139{
2140 int was_pending = 0, irq = data->irq;
2141 unsigned long flags;
2142
2143 raw_spin_lock_irqsave(&ioapic_lock, flags);
2144 if (irq < legacy_pic->nr_legacy_irqs) {
2145 legacy_pic->mask(irq);
2146 if (legacy_pic->irq_pending(irq))
2147 was_pending = 1;
2148 }
2149 __unmask_ioapic(data->chip_data);
2150 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2151
2152 return was_pending;
2153}
2154
2155static int ioapic_retrigger_irq(struct irq_data *data)
2156{
2157 struct irq_cfg *cfg = data->chip_data;
2158 unsigned long flags;
2159
2160 raw_spin_lock_irqsave(&vector_lock, flags);
2161 apic->send_IPI_mask(cpumask_of(cpumask_first(cfg->domain)), cfg->vector);
2162 raw_spin_unlock_irqrestore(&vector_lock, flags);
2163
2164 return 1;
2165}
2166
2167/*
2168 * Level and edge triggered IO-APIC interrupts need different handling,
2169 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
2170 * handled with the level-triggered descriptor, but that one has slightly
2171 * more overhead. Level-triggered interrupts cannot be handled with the
2172 * edge-triggered handler, without risking IRQ storms and other ugly
2173 * races.
2174 */
2175
2176#ifdef CONFIG_SMP
2177void send_cleanup_vector(struct irq_cfg *cfg)
2178{
2179 cpumask_var_t cleanup_mask;
2180
2181 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
2182 unsigned int i;
2183 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2184 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
2185 } else {
2186 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
2187 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2188 free_cpumask_var(cleanup_mask);
2189 }
2190 cfg->move_in_progress = 0;
2191}
2192
2193static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
2194{
2195 int apic, pin;
2196 struct irq_pin_list *entry;
2197 u8 vector = cfg->vector;
2198
2199 for_each_irq_pin(entry, cfg->irq_2_pin) {
2200 unsigned int reg;
2201
2202 apic = entry->apic;
2203 pin = entry->pin;
2204 /*
2205 * With interrupt-remapping, destination information comes
2206 * from interrupt-remapping table entry.
2207 */
2208 if (!irq_remapped(cfg))
2209 io_apic_write(apic, 0x11 + pin*2, dest);
2210 reg = io_apic_read(apic, 0x10 + pin*2);
2211 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
2212 reg |= vector;
2213 io_apic_modify(apic, 0x10 + pin*2, reg);
2214 }
2215}
2216
2217/*
2218 * Either sets data->affinity to a valid value, and returns
2219 * ->cpu_mask_to_apicid of that in dest_id, or returns -1 and
2220 * leaves data->affinity untouched.
2221 */
2222int __ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
2223 unsigned int *dest_id)
2224{
2225 struct irq_cfg *cfg = data->chip_data;
2226
2227 if (!cpumask_intersects(mask, cpu_online_mask))
2228 return -1;
2229
2230 if (assign_irq_vector(data->irq, data->chip_data, mask))
2231 return -1;
2232
2233 cpumask_copy(data->affinity, mask);
2234
2235 *dest_id = apic->cpu_mask_to_apicid_and(mask, cfg->domain);
2236 return 0;
2237}
2238
2239static int
2240ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
2241 bool force)
2242{
2243 unsigned int dest, irq = data->irq;
2244 unsigned long flags;
2245 int ret;
2246
2247 raw_spin_lock_irqsave(&ioapic_lock, flags);
2248 ret = __ioapic_set_affinity(data, mask, &dest);
2249 if (!ret) {
2250 /* Only the high 8 bits are valid. */
2251 dest = SET_APIC_LOGICAL_ID(dest);
2252 __target_IO_APIC_irq(irq, dest, data->chip_data);
2253 }
2254 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2255 return ret;
2256}
2257
2258#ifdef CONFIG_INTR_REMAP
2259
2260/*
2261 * Migrate the IO-APIC irq in the presence of intr-remapping.
2262 *
2263 * For both level and edge triggered, irq migration is a simple atomic
2264 * update(of vector and cpu destination) of IRTE and flush the hardware cache.
2265 *
2266 * For level triggered, we eliminate the io-apic RTE modification (with the
2267 * updated vector information), by using a virtual vector (io-apic pin number).
2268 * Real vector that is used for interrupting cpu will be coming from
2269 * the interrupt-remapping table entry.
2270 */
2271static int
2272ir_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
2273 bool force)
2274{
2275 struct irq_cfg *cfg = data->chip_data;
2276 unsigned int dest, irq = data->irq;
2277 struct irte irte;
2278
2279 if (!cpumask_intersects(mask, cpu_online_mask))
2280 return -EINVAL;
2281
2282 if (get_irte(irq, &irte))
2283 return -EBUSY;
2284
2285 if (assign_irq_vector(irq, cfg, mask))
2286 return -EBUSY;
2287
2288 dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask);
2289
2290 irte.vector = cfg->vector;
2291 irte.dest_id = IRTE_DEST(dest);
2292
2293 /*
2294 * Modified the IRTE and flushes the Interrupt entry cache.
2295 */
2296 modify_irte(irq, &irte);
2297
2298 if (cfg->move_in_progress)
2299 send_cleanup_vector(cfg);
2300
2301 cpumask_copy(data->affinity, mask);
2302 return 0;
2303}
2304
2305#else
2306static inline int
2307ir_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
2308 bool force)
2309{
2310 return 0;
2311}
2312#endif
2313
2314asmlinkage void smp_irq_move_cleanup_interrupt(void)
2315{
2316 unsigned vector, me;
2317
2318 ack_APIC_irq();
2319 exit_idle();
2320 irq_enter();
2321
2322 me = smp_processor_id();
2323 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
2324 unsigned int irq;
2325 unsigned int irr;
2326 struct irq_desc *desc;
2327 struct irq_cfg *cfg;
2328 irq = __this_cpu_read(vector_irq[vector]);
2329
2330 if (irq == -1)
2331 continue;
2332
2333 desc = irq_to_desc(irq);
2334 if (!desc)
2335 continue;
2336
2337 cfg = irq_cfg(irq);
2338 raw_spin_lock(&desc->lock);
2339
2340 /*
2341 * Check if the irq migration is in progress. If so, we
2342 * haven't received the cleanup request yet for this irq.
2343 */
2344 if (cfg->move_in_progress)
2345 goto unlock;
2346
2347 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2348 goto unlock;
2349
2350 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
2351 /*
2352 * Check if the vector that needs to be cleanedup is
2353 * registered at the cpu's IRR. If so, then this is not
2354 * the best time to clean it up. Lets clean it up in the
2355 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
2356 * to myself.
2357 */
2358 if (irr & (1 << (vector % 32))) {
2359 apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR);
2360 goto unlock;
2361 }
2362 __this_cpu_write(vector_irq[vector], -1);
2363unlock:
2364 raw_spin_unlock(&desc->lock);
2365 }
2366
2367 irq_exit();
2368}
2369
2370static void __irq_complete_move(struct irq_cfg *cfg, unsigned vector)
2371{
2372 unsigned me;
2373
2374 if (likely(!cfg->move_in_progress))
2375 return;
2376
2377 me = smp_processor_id();
2378
2379 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2380 send_cleanup_vector(cfg);
2381}
2382
2383static void irq_complete_move(struct irq_cfg *cfg)
2384{
2385 __irq_complete_move(cfg, ~get_irq_regs()->orig_ax);
2386}
2387
2388void irq_force_complete_move(int irq)
2389{
2390 struct irq_cfg *cfg = irq_get_chip_data(irq);
2391
2392 if (!cfg)
2393 return;
2394
2395 __irq_complete_move(cfg, cfg->vector);
2396}
2397#else
2398static inline void irq_complete_move(struct irq_cfg *cfg) { }
2399#endif
2400
2401static void ack_apic_edge(struct irq_data *data)
2402{
2403 irq_complete_move(data->chip_data);
2404 irq_move_irq(data);
2405 ack_APIC_irq();
2406}
2407
2408atomic_t irq_mis_count;
2409
2410/*
2411 * IO-APIC versions below 0x20 don't support EOI register.
2412 * For the record, here is the information about various versions:
2413 * 0Xh 82489DX
2414 * 1Xh I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant
2415 * 2Xh I/O(x)APIC which is PCI 2.2 Compliant
2416 * 30h-FFh Reserved
2417 *
2418 * Some of the Intel ICH Specs (ICH2 to ICH5) documents the io-apic
2419 * version as 0x2. This is an error with documentation and these ICH chips
2420 * use io-apic's of version 0x20.
2421 *
2422 * For IO-APIC's with EOI register, we use that to do an explicit EOI.
2423 * Otherwise, we simulate the EOI message manually by changing the trigger
2424 * mode to edge and then back to level, with RTE being masked during this.
2425*/
2426static void eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2427{
2428 struct irq_pin_list *entry;
2429 unsigned long flags;
2430
2431 raw_spin_lock_irqsave(&ioapic_lock, flags);
2432 for_each_irq_pin(entry, cfg->irq_2_pin) {
2433 if (mpc_ioapic_ver(entry->apic) >= 0x20) {
2434 /*
2435 * Intr-remapping uses pin number as the virtual vector
2436 * in the RTE. Actual vector is programmed in
2437 * intr-remapping table entry. Hence for the io-apic
2438 * EOI we use the pin number.
2439 */
2440 if (irq_remapped(cfg))
2441 io_apic_eoi(entry->apic, entry->pin);
2442 else
2443 io_apic_eoi(entry->apic, cfg->vector);
2444 } else {
2445 __mask_and_edge_IO_APIC_irq(entry);
2446 __unmask_and_level_IO_APIC_irq(entry);
2447 }
2448 }
2449 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2450}
2451
2452static void ack_apic_level(struct irq_data *data)
2453{
2454 struct irq_cfg *cfg = data->chip_data;
2455 int i, do_unmask_irq = 0, irq = data->irq;
2456 unsigned long v;
2457
2458 irq_complete_move(cfg);
2459#ifdef CONFIG_GENERIC_PENDING_IRQ
2460 /* If we are moving the irq we need to mask it */
2461 if (unlikely(irqd_is_setaffinity_pending(data))) {
2462 do_unmask_irq = 1;
2463 mask_ioapic(cfg);
2464 }
2465#endif
2466
2467 /*
2468 * It appears there is an erratum which affects at least version 0x11
2469 * of I/O APIC (that's the 82093AA and cores integrated into various
2470 * chipsets). Under certain conditions a level-triggered interrupt is
2471 * erroneously delivered as edge-triggered one but the respective IRR
2472 * bit gets set nevertheless. As a result the I/O unit expects an EOI
2473 * message but it will never arrive and further interrupts are blocked
2474 * from the source. The exact reason is so far unknown, but the
2475 * phenomenon was observed when two consecutive interrupt requests
2476 * from a given source get delivered to the same CPU and the source is
2477 * temporarily disabled in between.
2478 *
2479 * A workaround is to simulate an EOI message manually. We achieve it
2480 * by setting the trigger mode to edge and then to level when the edge
2481 * trigger mode gets detected in the TMR of a local APIC for a
2482 * level-triggered interrupt. We mask the source for the time of the
2483 * operation to prevent an edge-triggered interrupt escaping meanwhile.
2484 * The idea is from Manfred Spraul. --macro
2485 *
2486 * Also in the case when cpu goes offline, fixup_irqs() will forward
2487 * any unhandled interrupt on the offlined cpu to the new cpu
2488 * destination that is handling the corresponding interrupt. This
2489 * interrupt forwarding is done via IPI's. Hence, in this case also
2490 * level-triggered io-apic interrupt will be seen as an edge
2491 * interrupt in the IRR. And we can't rely on the cpu's EOI
2492 * to be broadcasted to the IO-APIC's which will clear the remoteIRR
2493 * corresponding to the level-triggered interrupt. Hence on IO-APIC's
2494 * supporting EOI register, we do an explicit EOI to clear the
2495 * remote IRR and on IO-APIC's which don't have an EOI register,
2496 * we use the above logic (mask+edge followed by unmask+level) from
2497 * Manfred Spraul to clear the remote IRR.
2498 */
2499 i = cfg->vector;
2500 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2501
2502 /*
2503 * We must acknowledge the irq before we move it or the acknowledge will
2504 * not propagate properly.
2505 */
2506 ack_APIC_irq();
2507
2508 /*
2509 * Tail end of clearing remote IRR bit (either by delivering the EOI
2510 * message via io-apic EOI register write or simulating it using
2511 * mask+edge followed by unnask+level logic) manually when the
2512 * level triggered interrupt is seen as the edge triggered interrupt
2513 * at the cpu.
2514 */
2515 if (!(v & (1 << (i & 0x1f)))) {
2516 atomic_inc(&irq_mis_count);
2517
2518 eoi_ioapic_irq(irq, cfg);
2519 }
2520
2521 /* Now we can move and renable the irq */
2522 if (unlikely(do_unmask_irq)) {
2523 /* Only migrate the irq if the ack has been received.
2524 *
2525 * On rare occasions the broadcast level triggered ack gets
2526 * delayed going to ioapics, and if we reprogram the
2527 * vector while Remote IRR is still set the irq will never
2528 * fire again.
2529 *
2530 * To prevent this scenario we read the Remote IRR bit
2531 * of the ioapic. This has two effects.
2532 * - On any sane system the read of the ioapic will
2533 * flush writes (and acks) going to the ioapic from
2534 * this cpu.
2535 * - We get to see if the ACK has actually been delivered.
2536 *
2537 * Based on failed experiments of reprogramming the
2538 * ioapic entry from outside of irq context starting
2539 * with masking the ioapic entry and then polling until
2540 * Remote IRR was clear before reprogramming the
2541 * ioapic I don't trust the Remote IRR bit to be
2542 * completey accurate.
2543 *
2544 * However there appears to be no other way to plug
2545 * this race, so if the Remote IRR bit is not
2546 * accurate and is causing problems then it is a hardware bug
2547 * and you can go talk to the chipset vendor about it.
2548 */
2549 if (!io_apic_level_ack_pending(cfg))
2550 irq_move_masked_irq(data);
2551 unmask_ioapic(cfg);
2552 }
2553}
2554
2555#ifdef CONFIG_INTR_REMAP
2556static void ir_ack_apic_edge(struct irq_data *data)
2557{
2558 ack_APIC_irq();
2559}
2560
2561static void ir_ack_apic_level(struct irq_data *data)
2562{
2563 ack_APIC_irq();
2564 eoi_ioapic_irq(data->irq, data->chip_data);
2565}
2566#endif /* CONFIG_INTR_REMAP */
2567
2568static struct irq_chip ioapic_chip __read_mostly = {
2569 .name = "IO-APIC",
2570 .irq_startup = startup_ioapic_irq,
2571 .irq_mask = mask_ioapic_irq,
2572 .irq_unmask = unmask_ioapic_irq,
2573 .irq_ack = ack_apic_edge,
2574 .irq_eoi = ack_apic_level,
2575#ifdef CONFIG_SMP
2576 .irq_set_affinity = ioapic_set_affinity,
2577#endif
2578 .irq_retrigger = ioapic_retrigger_irq,
2579};
2580
2581static struct irq_chip ir_ioapic_chip __read_mostly = {
2582 .name = "IR-IO-APIC",
2583 .irq_startup = startup_ioapic_irq,
2584 .irq_mask = mask_ioapic_irq,
2585 .irq_unmask = unmask_ioapic_irq,
2586#ifdef CONFIG_INTR_REMAP
2587 .irq_ack = ir_ack_apic_edge,
2588 .irq_eoi = ir_ack_apic_level,
2589#ifdef CONFIG_SMP
2590 .irq_set_affinity = ir_ioapic_set_affinity,
2591#endif
2592#endif
2593 .irq_retrigger = ioapic_retrigger_irq,
2594};
2595
2596static inline void init_IO_APIC_traps(void)
2597{
2598 struct irq_cfg *cfg;
2599 unsigned int irq;
2600
2601 /*
2602 * NOTE! The local APIC isn't very good at handling
2603 * multiple interrupts at the same interrupt level.
2604 * As the interrupt level is determined by taking the
2605 * vector number and shifting that right by 4, we
2606 * want to spread these out a bit so that they don't
2607 * all fall in the same interrupt level.
2608 *
2609 * Also, we've got to be careful not to trash gate
2610 * 0x80, because int 0x80 is hm, kind of importantish. ;)
2611 */
2612 for_each_active_irq(irq) {
2613 cfg = irq_get_chip_data(irq);
2614 if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
2615 /*
2616 * Hmm.. We don't have an entry for this,
2617 * so default to an old-fashioned 8259
2618 * interrupt if we can..
2619 */
2620 if (irq < legacy_pic->nr_legacy_irqs)
2621 legacy_pic->make_irq(irq);
2622 else
2623 /* Strange. Oh, well.. */
2624 irq_set_chip(irq, &no_irq_chip);
2625 }
2626 }
2627}
2628
2629/*
2630 * The local APIC irq-chip implementation:
2631 */
2632
2633static void mask_lapic_irq(struct irq_data *data)
2634{
2635 unsigned long v;
2636
2637 v = apic_read(APIC_LVT0);
2638 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
2639}
2640
2641static void unmask_lapic_irq(struct irq_data *data)
2642{
2643 unsigned long v;
2644
2645 v = apic_read(APIC_LVT0);
2646 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2647}
2648
2649static void ack_lapic_irq(struct irq_data *data)
2650{
2651 ack_APIC_irq();
2652}
2653
2654static struct irq_chip lapic_chip __read_mostly = {
2655 .name = "local-APIC",
2656 .irq_mask = mask_lapic_irq,
2657 .irq_unmask = unmask_lapic_irq,
2658 .irq_ack = ack_lapic_irq,
2659};
2660
2661static void lapic_register_intr(int irq)
2662{
2663 irq_clear_status_flags(irq, IRQ_LEVEL);
2664 irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2665 "edge");
2666}
2667
2668/*
2669 * This looks a bit hackish but it's about the only one way of sending
2670 * a few INTA cycles to 8259As and any associated glue logic. ICR does
2671 * not support the ExtINT mode, unfortunately. We need to send these
2672 * cycles as some i82489DX-based boards have glue logic that keeps the
2673 * 8259A interrupt line asserted until INTA. --macro
2674 */
2675static inline void __init unlock_ExtINT_logic(void)
2676{
2677 int apic, pin, i;
2678 struct IO_APIC_route_entry entry0, entry1;
2679 unsigned char save_control, save_freq_select;
2680
2681 pin = find_isa_irq_pin(8, mp_INT);
2682 if (pin == -1) {
2683 WARN_ON_ONCE(1);
2684 return;
2685 }
2686 apic = find_isa_irq_apic(8, mp_INT);
2687 if (apic == -1) {
2688 WARN_ON_ONCE(1);
2689 return;
2690 }
2691
2692 entry0 = ioapic_read_entry(apic, pin);
2693 clear_IO_APIC_pin(apic, pin);
2694
2695 memset(&entry1, 0, sizeof(entry1));
2696
2697 entry1.dest_mode = 0; /* physical delivery */
2698 entry1.mask = 0; /* unmask IRQ now */
2699 entry1.dest = hard_smp_processor_id();
2700 entry1.delivery_mode = dest_ExtINT;
2701 entry1.polarity = entry0.polarity;
2702 entry1.trigger = 0;
2703 entry1.vector = 0;
2704
2705 ioapic_write_entry(apic, pin, entry1);
2706
2707 save_control = CMOS_READ(RTC_CONTROL);
2708 save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2709 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2710 RTC_FREQ_SELECT);
2711 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2712
2713 i = 100;
2714 while (i-- > 0) {
2715 mdelay(10);
2716 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2717 i -= 10;
2718 }
2719
2720 CMOS_WRITE(save_control, RTC_CONTROL);
2721 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2722 clear_IO_APIC_pin(apic, pin);
2723
2724 ioapic_write_entry(apic, pin, entry0);
2725}
2726
2727static int disable_timer_pin_1 __initdata;
2728/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
2729static int __init disable_timer_pin_setup(char *arg)
2730{
2731 disable_timer_pin_1 = 1;
2732 return 0;
2733}
2734early_param("disable_timer_pin_1", disable_timer_pin_setup);
2735
2736int timer_through_8259 __initdata;
2737
2738/*
2739 * This code may look a bit paranoid, but it's supposed to cooperate with
2740 * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ
2741 * is so screwy. Thanks to Brian Perkins for testing/hacking this beast
2742 * fanatically on his truly buggy board.
2743 *
2744 * FIXME: really need to revamp this for all platforms.
2745 */
2746static inline void __init check_timer(void)
2747{
2748 struct irq_cfg *cfg = irq_get_chip_data(0);
2749 int node = cpu_to_node(0);
2750 int apic1, pin1, apic2, pin2;
2751 unsigned long flags;
2752 int no_pin1 = 0;
2753
2754 local_irq_save(flags);
2755
2756 /*
2757 * get/set the timer IRQ vector:
2758 */
2759 legacy_pic->mask(0);
2760 assign_irq_vector(0, cfg, apic->target_cpus());
2761
2762 /*
2763 * As IRQ0 is to be enabled in the 8259A, the virtual
2764 * wire has to be disabled in the local APIC. Also
2765 * timer interrupts need to be acknowledged manually in
2766 * the 8259A for the i82489DX when using the NMI
2767 * watchdog as that APIC treats NMIs as level-triggered.
2768 * The AEOI mode will finish them in the 8259A
2769 * automatically.
2770 */
2771 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2772 legacy_pic->init(1);
2773
2774 pin1 = find_isa_irq_pin(0, mp_INT);
2775 apic1 = find_isa_irq_apic(0, mp_INT);
2776 pin2 = ioapic_i8259.pin;
2777 apic2 = ioapic_i8259.apic;
2778
2779 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2780 "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2781 cfg->vector, apic1, pin1, apic2, pin2);
2782
2783 /*
2784 * Some BIOS writers are clueless and report the ExtINTA
2785 * I/O APIC input from the cascaded 8259A as the timer
2786 * interrupt input. So just in case, if only one pin
2787 * was found above, try it both directly and through the
2788 * 8259A.
2789 */
2790 if (pin1 == -1) {
2791 if (intr_remapping_enabled)
2792 panic("BIOS bug: timer not connected to IO-APIC");
2793 pin1 = pin2;
2794 apic1 = apic2;
2795 no_pin1 = 1;
2796 } else if (pin2 == -1) {
2797 pin2 = pin1;
2798 apic2 = apic1;
2799 }
2800
2801 if (pin1 != -1) {
2802 /*
2803 * Ok, does IRQ0 through the IOAPIC work?
2804 */
2805 if (no_pin1) {
2806 add_pin_to_irq_node(cfg, node, apic1, pin1);
2807 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2808 } else {
2809 /* for edge trigger, setup_ioapic_irq already
2810 * leave it unmasked.
2811 * so only need to unmask if it is level-trigger
2812 * do we really have level trigger timer?
2813 */
2814 int idx;
2815 idx = find_irq_entry(apic1, pin1, mp_INT);
2816 if (idx != -1 && irq_trigger(idx))
2817 unmask_ioapic(cfg);
2818 }
2819 if (timer_irq_works()) {
2820 if (disable_timer_pin_1 > 0)
2821 clear_IO_APIC_pin(0, pin1);
2822 goto out;
2823 }
2824 if (intr_remapping_enabled)
2825 panic("timer doesn't work through Interrupt-remapped IO-APIC");
2826 local_irq_disable();
2827 clear_IO_APIC_pin(apic1, pin1);
2828 if (!no_pin1)
2829 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2830 "8254 timer not connected to IO-APIC\n");
2831
2832 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2833 "(IRQ0) through the 8259A ...\n");
2834 apic_printk(APIC_QUIET, KERN_INFO
2835 "..... (found apic %d pin %d) ...\n", apic2, pin2);
2836 /*
2837 * legacy devices should be connected to IO APIC #0
2838 */
2839 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
2840 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2841 legacy_pic->unmask(0);
2842 if (timer_irq_works()) {
2843 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2844 timer_through_8259 = 1;
2845 goto out;
2846 }
2847 /*
2848 * Cleanup, just in case ...
2849 */
2850 local_irq_disable();
2851 legacy_pic->mask(0);
2852 clear_IO_APIC_pin(apic2, pin2);
2853 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
2854 }
2855
2856 apic_printk(APIC_QUIET, KERN_INFO
2857 "...trying to set up timer as Virtual Wire IRQ...\n");
2858
2859 lapic_register_intr(0);
2860 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
2861 legacy_pic->unmask(0);
2862
2863 if (timer_irq_works()) {
2864 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2865 goto out;
2866 }
2867 local_irq_disable();
2868 legacy_pic->mask(0);
2869 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2870 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
2871
2872 apic_printk(APIC_QUIET, KERN_INFO
2873 "...trying to set up timer as ExtINT IRQ...\n");
2874
2875 legacy_pic->init(0);
2876 legacy_pic->make_irq(0);
2877 apic_write(APIC_LVT0, APIC_DM_EXTINT);
2878
2879 unlock_ExtINT_logic();
2880
2881 if (timer_irq_works()) {
2882 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2883 goto out;
2884 }
2885 local_irq_disable();
2886 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
2887 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
2888 "report. Then try booting with the 'noapic' option.\n");
2889out:
2890 local_irq_restore(flags);
2891}
2892
2893/*
2894 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2895 * to devices. However there may be an I/O APIC pin available for
2896 * this interrupt regardless. The pin may be left unconnected, but
2897 * typically it will be reused as an ExtINT cascade interrupt for
2898 * the master 8259A. In the MPS case such a pin will normally be
2899 * reported as an ExtINT interrupt in the MP table. With ACPI
2900 * there is no provision for ExtINT interrupts, and in the absence
2901 * of an override it would be treated as an ordinary ISA I/O APIC
2902 * interrupt, that is edge-triggered and unmasked by default. We
2903 * used to do this, but it caused problems on some systems because
2904 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
2905 * the same ExtINT cascade interrupt to drive the local APIC of the
2906 * bootstrap processor. Therefore we refrain from routing IRQ2 to
2907 * the I/O APIC in all cases now. No actual device should request
2908 * it anyway. --macro
2909 */
2910#define PIC_IRQS (1UL << PIC_CASCADE_IR)
2911
2912void __init setup_IO_APIC(void)
2913{
2914
2915 /*
2916 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
2917 */
2918 io_apic_irqs = legacy_pic->nr_legacy_irqs ? ~PIC_IRQS : ~0UL;
2919
2920 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
2921 /*
2922 * Set up IO-APIC IRQ routing.
2923 */
2924 x86_init.mpparse.setup_ioapic_ids();
2925
2926 sync_Arb_IDs();
2927 setup_IO_APIC_irqs();
2928 init_IO_APIC_traps();
2929 if (legacy_pic->nr_legacy_irqs)
2930 check_timer();
2931}
2932
2933/*
2934 * Called after all the initialization is done. If we didn't find any
2935 * APIC bugs then we can allow the modify fast path
2936 */
2937
2938static int __init io_apic_bug_finalize(void)
2939{
2940 if (sis_apic_bug == -1)
2941 sis_apic_bug = 0;
2942 return 0;
2943}
2944
2945late_initcall(io_apic_bug_finalize);
2946
2947static void resume_ioapic_id(int ioapic_id)
2948{
2949 unsigned long flags;
2950 union IO_APIC_reg_00 reg_00;
2951
2952
2953 raw_spin_lock_irqsave(&ioapic_lock, flags);
2954 reg_00.raw = io_apic_read(ioapic_id, 0);
2955 if (reg_00.bits.ID != mpc_ioapic_id(ioapic_id)) {
2956 reg_00.bits.ID = mpc_ioapic_id(ioapic_id);
2957 io_apic_write(ioapic_id, 0, reg_00.raw);
2958 }
2959 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2960}
2961
2962static void ioapic_resume(void)
2963{
2964 int ioapic_id;
2965
2966 for (ioapic_id = nr_ioapics - 1; ioapic_id >= 0; ioapic_id--)
2967 resume_ioapic_id(ioapic_id);
2968
2969 restore_ioapic_entries();
2970}
2971
2972static struct syscore_ops ioapic_syscore_ops = {
2973 .suspend = save_ioapic_entries,
2974 .resume = ioapic_resume,
2975};
2976
2977static int __init ioapic_init_ops(void)
2978{
2979 register_syscore_ops(&ioapic_syscore_ops);
2980
2981 return 0;
2982}
2983
2984device_initcall(ioapic_init_ops);
2985
2986/*
2987 * Dynamic irq allocate and deallocation
2988 */
2989unsigned int create_irq_nr(unsigned int from, int node)
2990{
2991 struct irq_cfg *cfg;
2992 unsigned long flags;
2993 unsigned int ret = 0;
2994 int irq;
2995
2996 if (from < nr_irqs_gsi)
2997 from = nr_irqs_gsi;
2998
2999 irq = alloc_irq_from(from, node);
3000 if (irq < 0)
3001 return 0;
3002 cfg = alloc_irq_cfg(irq, node);
3003 if (!cfg) {
3004 free_irq_at(irq, NULL);
3005 return 0;
3006 }
3007
3008 raw_spin_lock_irqsave(&vector_lock, flags);
3009 if (!__assign_irq_vector(irq, cfg, apic->target_cpus()))
3010 ret = irq;
3011 raw_spin_unlock_irqrestore(&vector_lock, flags);
3012
3013 if (ret) {
3014 irq_set_chip_data(irq, cfg);
3015 irq_clear_status_flags(irq, IRQ_NOREQUEST);
3016 } else {
3017 free_irq_at(irq, cfg);
3018 }
3019 return ret;
3020}
3021
3022int create_irq(void)
3023{
3024 int node = cpu_to_node(0);
3025 unsigned int irq_want;
3026 int irq;
3027
3028 irq_want = nr_irqs_gsi;
3029 irq = create_irq_nr(irq_want, node);
3030
3031 if (irq == 0)
3032 irq = -1;
3033
3034 return irq;
3035}
3036
3037void destroy_irq(unsigned int irq)
3038{
3039 struct irq_cfg *cfg = irq_get_chip_data(irq);
3040 unsigned long flags;
3041
3042 irq_set_status_flags(irq, IRQ_NOREQUEST|IRQ_NOPROBE);
3043
3044 if (irq_remapped(cfg))
3045 free_irte(irq);
3046 raw_spin_lock_irqsave(&vector_lock, flags);
3047 __clear_irq_vector(irq, cfg);
3048 raw_spin_unlock_irqrestore(&vector_lock, flags);
3049 free_irq_at(irq, cfg);
3050}
3051
3052/*
3053 * MSI message composition
3054 */
3055#ifdef CONFIG_PCI_MSI
3056static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
3057 struct msi_msg *msg, u8 hpet_id)
3058{
3059 struct irq_cfg *cfg;
3060 int err;
3061 unsigned dest;
3062
3063 if (disable_apic)
3064 return -ENXIO;
3065
3066 cfg = irq_cfg(irq);
3067 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3068 if (err)
3069 return err;
3070
3071 dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus());
3072
3073 if (irq_remapped(cfg)) {
3074 struct irte irte;
3075 int ir_index;
3076 u16 sub_handle;
3077
3078 ir_index = map_irq_to_irte_handle(irq, &sub_handle);
3079 BUG_ON(ir_index == -1);
3080
3081 prepare_irte(&irte, cfg->vector, dest);
3082
3083 /* Set source-id of interrupt request */
3084 if (pdev)
3085 set_msi_sid(&irte, pdev);
3086 else
3087 set_hpet_sid(&irte, hpet_id);
3088
3089 modify_irte(irq, &irte);
3090
3091 msg->address_hi = MSI_ADDR_BASE_HI;
3092 msg->data = sub_handle;
3093 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
3094 MSI_ADDR_IR_SHV |
3095 MSI_ADDR_IR_INDEX1(ir_index) |
3096 MSI_ADDR_IR_INDEX2(ir_index);
3097 } else {
3098 if (x2apic_enabled())
3099 msg->address_hi = MSI_ADDR_BASE_HI |
3100 MSI_ADDR_EXT_DEST_ID(dest);
3101 else
3102 msg->address_hi = MSI_ADDR_BASE_HI;
3103
3104 msg->address_lo =
3105 MSI_ADDR_BASE_LO |
3106 ((apic->irq_dest_mode == 0) ?
3107 MSI_ADDR_DEST_MODE_PHYSICAL:
3108 MSI_ADDR_DEST_MODE_LOGICAL) |
3109 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3110 MSI_ADDR_REDIRECTION_CPU:
3111 MSI_ADDR_REDIRECTION_LOWPRI) |
3112 MSI_ADDR_DEST_ID(dest);
3113
3114 msg->data =
3115 MSI_DATA_TRIGGER_EDGE |
3116 MSI_DATA_LEVEL_ASSERT |
3117 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3118 MSI_DATA_DELIVERY_FIXED:
3119 MSI_DATA_DELIVERY_LOWPRI) |
3120 MSI_DATA_VECTOR(cfg->vector);
3121 }
3122 return err;
3123}
3124
3125#ifdef CONFIG_SMP
3126static int
3127msi_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3128{
3129 struct irq_cfg *cfg = data->chip_data;
3130 struct msi_msg msg;
3131 unsigned int dest;
3132
3133 if (__ioapic_set_affinity(data, mask, &dest))
3134 return -1;
3135
3136 __get_cached_msi_msg(data->msi_desc, &msg);
3137
3138 msg.data &= ~MSI_DATA_VECTOR_MASK;
3139 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3140 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3141 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3142
3143 __write_msi_msg(data->msi_desc, &msg);
3144
3145 return 0;
3146}
3147#ifdef CONFIG_INTR_REMAP
3148/*
3149 * Migrate the MSI irq to another cpumask. This migration is
3150 * done in the process context using interrupt-remapping hardware.
3151 */
3152static int
3153ir_msi_set_affinity(struct irq_data *data, const struct cpumask *mask,
3154 bool force)
3155{
3156 struct irq_cfg *cfg = data->chip_data;
3157 unsigned int dest, irq = data->irq;
3158 struct irte irte;
3159
3160 if (get_irte(irq, &irte))
3161 return -1;
3162
3163 if (__ioapic_set_affinity(data, mask, &dest))
3164 return -1;
3165
3166 irte.vector = cfg->vector;
3167 irte.dest_id = IRTE_DEST(dest);
3168
3169 /*
3170 * atomically update the IRTE with the new destination and vector.
3171 */
3172 modify_irte(irq, &irte);
3173
3174 /*
3175 * After this point, all the interrupts will start arriving
3176 * at the new destination. So, time to cleanup the previous
3177 * vector allocation.
3178 */
3179 if (cfg->move_in_progress)
3180 send_cleanup_vector(cfg);
3181
3182 return 0;
3183}
3184
3185#endif
3186#endif /* CONFIG_SMP */
3187
3188/*
3189 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
3190 * which implement the MSI or MSI-X Capability Structure.
3191 */
3192static struct irq_chip msi_chip = {
3193 .name = "PCI-MSI",
3194 .irq_unmask = unmask_msi_irq,
3195 .irq_mask = mask_msi_irq,
3196 .irq_ack = ack_apic_edge,
3197#ifdef CONFIG_SMP
3198 .irq_set_affinity = msi_set_affinity,
3199#endif
3200 .irq_retrigger = ioapic_retrigger_irq,
3201};
3202
3203static struct irq_chip msi_ir_chip = {
3204 .name = "IR-PCI-MSI",
3205 .irq_unmask = unmask_msi_irq,
3206 .irq_mask = mask_msi_irq,
3207#ifdef CONFIG_INTR_REMAP
3208 .irq_ack = ir_ack_apic_edge,
3209#ifdef CONFIG_SMP
3210 .irq_set_affinity = ir_msi_set_affinity,
3211#endif
3212#endif
3213 .irq_retrigger = ioapic_retrigger_irq,
3214};
3215
3216/*
3217 * Map the PCI dev to the corresponding remapping hardware unit
3218 * and allocate 'nvec' consecutive interrupt-remapping table entries
3219 * in it.
3220 */
3221static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
3222{
3223 struct intel_iommu *iommu;
3224 int index;
3225
3226 iommu = map_dev_to_ir(dev);
3227 if (!iommu) {
3228 printk(KERN_ERR
3229 "Unable to map PCI %s to iommu\n", pci_name(dev));
3230 return -ENOENT;
3231 }
3232
3233 index = alloc_irte(iommu, irq, nvec);
3234 if (index < 0) {
3235 printk(KERN_ERR
3236 "Unable to allocate %d IRTE for PCI %s\n", nvec,
3237 pci_name(dev));
3238 return -ENOSPC;
3239 }
3240 return index;
3241}
3242
3243static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
3244{
3245 struct irq_chip *chip = &msi_chip;
3246 struct msi_msg msg;
3247 int ret;
3248
3249 ret = msi_compose_msg(dev, irq, &msg, -1);
3250 if (ret < 0)
3251 return ret;
3252
3253 irq_set_msi_desc(irq, msidesc);
3254 write_msi_msg(irq, &msg);
3255
3256 if (irq_remapped(irq_get_chip_data(irq))) {
3257 irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
3258 chip = &msi_ir_chip;
3259 }
3260
3261 irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3262
3263 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq);
3264
3265 return 0;
3266}
3267
3268int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3269{
3270 int node, ret, sub_handle, index = 0;
3271 unsigned int irq, irq_want;
3272 struct msi_desc *msidesc;
3273 struct intel_iommu *iommu = NULL;
3274
3275 /* x86 doesn't support multiple MSI yet */
3276 if (type == PCI_CAP_ID_MSI && nvec > 1)
3277 return 1;
3278
3279 node = dev_to_node(&dev->dev);
3280 irq_want = nr_irqs_gsi;
3281 sub_handle = 0;
3282 list_for_each_entry(msidesc, &dev->msi_list, list) {
3283 irq = create_irq_nr(irq_want, node);
3284 if (irq == 0)
3285 return -1;
3286 irq_want = irq + 1;
3287 if (!intr_remapping_enabled)
3288 goto no_ir;
3289
3290 if (!sub_handle) {
3291 /*
3292 * allocate the consecutive block of IRTE's
3293 * for 'nvec'
3294 */
3295 index = msi_alloc_irte(dev, irq, nvec);
3296 if (index < 0) {
3297 ret = index;
3298 goto error;
3299 }
3300 } else {
3301 iommu = map_dev_to_ir(dev);
3302 if (!iommu) {
3303 ret = -ENOENT;
3304 goto error;
3305 }
3306 /*
3307 * setup the mapping between the irq and the IRTE
3308 * base index, the sub_handle pointing to the
3309 * appropriate interrupt remap table entry.
3310 */
3311 set_irte_irq(irq, iommu, index, sub_handle);
3312 }
3313no_ir:
3314 ret = setup_msi_irq(dev, msidesc, irq);
3315 if (ret < 0)
3316 goto error;
3317 sub_handle++;
3318 }
3319 return 0;
3320
3321error:
3322 destroy_irq(irq);
3323 return ret;
3324}
3325
3326void native_teardown_msi_irq(unsigned int irq)
3327{
3328 destroy_irq(irq);
3329}
3330
3331#if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP)
3332#ifdef CONFIG_SMP
3333static int
3334dmar_msi_set_affinity(struct irq_data *data, const struct cpumask *mask,
3335 bool force)
3336{
3337 struct irq_cfg *cfg = data->chip_data;
3338 unsigned int dest, irq = data->irq;
3339 struct msi_msg msg;
3340
3341 if (__ioapic_set_affinity(data, mask, &dest))
3342 return -1;
3343
3344 dmar_msi_read(irq, &msg);
3345
3346 msg.data &= ~MSI_DATA_VECTOR_MASK;
3347 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3348 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3349 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3350 msg.address_hi = MSI_ADDR_BASE_HI | MSI_ADDR_EXT_DEST_ID(dest);
3351
3352 dmar_msi_write(irq, &msg);
3353
3354 return 0;
3355}
3356
3357#endif /* CONFIG_SMP */
3358
3359static struct irq_chip dmar_msi_type = {
3360 .name = "DMAR_MSI",
3361 .irq_unmask = dmar_msi_unmask,
3362 .irq_mask = dmar_msi_mask,
3363 .irq_ack = ack_apic_edge,
3364#ifdef CONFIG_SMP
3365 .irq_set_affinity = dmar_msi_set_affinity,
3366#endif
3367 .irq_retrigger = ioapic_retrigger_irq,
3368};
3369
3370int arch_setup_dmar_msi(unsigned int irq)
3371{
3372 int ret;
3373 struct msi_msg msg;
3374
3375 ret = msi_compose_msg(NULL, irq, &msg, -1);
3376 if (ret < 0)
3377 return ret;
3378 dmar_msi_write(irq, &msg);
3379 irq_set_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
3380 "edge");
3381 return 0;
3382}
3383#endif
3384
3385#ifdef CONFIG_HPET_TIMER
3386
3387#ifdef CONFIG_SMP
3388static int hpet_msi_set_affinity(struct irq_data *data,
3389 const struct cpumask *mask, bool force)
3390{
3391 struct irq_cfg *cfg = data->chip_data;
3392 struct msi_msg msg;
3393 unsigned int dest;
3394
3395 if (__ioapic_set_affinity(data, mask, &dest))
3396 return -1;
3397
3398 hpet_msi_read(data->handler_data, &msg);
3399
3400 msg.data &= ~MSI_DATA_VECTOR_MASK;
3401 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3402 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3403 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3404
3405 hpet_msi_write(data->handler_data, &msg);
3406
3407 return 0;
3408}
3409
3410#endif /* CONFIG_SMP */
3411
3412static struct irq_chip ir_hpet_msi_type = {
3413 .name = "IR-HPET_MSI",
3414 .irq_unmask = hpet_msi_unmask,
3415 .irq_mask = hpet_msi_mask,
3416#ifdef CONFIG_INTR_REMAP
3417 .irq_ack = ir_ack_apic_edge,
3418#ifdef CONFIG_SMP
3419 .irq_set_affinity = ir_msi_set_affinity,
3420#endif
3421#endif
3422 .irq_retrigger = ioapic_retrigger_irq,
3423};
3424
3425static struct irq_chip hpet_msi_type = {
3426 .name = "HPET_MSI",
3427 .irq_unmask = hpet_msi_unmask,
3428 .irq_mask = hpet_msi_mask,
3429 .irq_ack = ack_apic_edge,
3430#ifdef CONFIG_SMP
3431 .irq_set_affinity = hpet_msi_set_affinity,
3432#endif
3433 .irq_retrigger = ioapic_retrigger_irq,
3434};
3435
3436int arch_setup_hpet_msi(unsigned int irq, unsigned int id)
3437{
3438 struct irq_chip *chip = &hpet_msi_type;
3439 struct msi_msg msg;
3440 int ret;
3441
3442 if (intr_remapping_enabled) {
3443 struct intel_iommu *iommu = map_hpet_to_ir(id);
3444 int index;
3445
3446 if (!iommu)
3447 return -1;
3448
3449 index = alloc_irte(iommu, irq, 1);
3450 if (index < 0)
3451 return -1;
3452 }
3453
3454 ret = msi_compose_msg(NULL, irq, &msg, id);
3455 if (ret < 0)
3456 return ret;
3457
3458 hpet_msi_write(irq_get_handler_data(irq), &msg);
3459 irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
3460 if (irq_remapped(irq_get_chip_data(irq)))
3461 chip = &ir_hpet_msi_type;
3462
3463 irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
3464 return 0;
3465}
3466#endif
3467
3468#endif /* CONFIG_PCI_MSI */
3469/*
3470 * Hypertransport interrupt support
3471 */
3472#ifdef CONFIG_HT_IRQ
3473
3474#ifdef CONFIG_SMP
3475
3476static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3477{
3478 struct ht_irq_msg msg;
3479 fetch_ht_irq_msg(irq, &msg);
3480
3481 msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
3482 msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
3483
3484 msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
3485 msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
3486
3487 write_ht_irq_msg(irq, &msg);
3488}
3489
3490static int
3491ht_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
3492{
3493 struct irq_cfg *cfg = data->chip_data;
3494 unsigned int dest;
3495
3496 if (__ioapic_set_affinity(data, mask, &dest))
3497 return -1;
3498
3499 target_ht_irq(data->irq, dest, cfg->vector);
3500 return 0;
3501}
3502
3503#endif
3504
3505static struct irq_chip ht_irq_chip = {
3506 .name = "PCI-HT",
3507 .irq_mask = mask_ht_irq,
3508 .irq_unmask = unmask_ht_irq,
3509 .irq_ack = ack_apic_edge,
3510#ifdef CONFIG_SMP
3511 .irq_set_affinity = ht_set_affinity,
3512#endif
3513 .irq_retrigger = ioapic_retrigger_irq,
3514};
3515
3516int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3517{
3518 struct irq_cfg *cfg;
3519 int err;
3520
3521 if (disable_apic)
3522 return -ENXIO;
3523
3524 cfg = irq_cfg(irq);
3525 err = assign_irq_vector(irq, cfg, apic->target_cpus());
3526 if (!err) {
3527 struct ht_irq_msg msg;
3528 unsigned dest;
3529
3530 dest = apic->cpu_mask_to_apicid_and(cfg->domain,
3531 apic->target_cpus());
3532
3533 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
3534
3535 msg.address_lo =
3536 HT_IRQ_LOW_BASE |
3537 HT_IRQ_LOW_DEST_ID(dest) |
3538 HT_IRQ_LOW_VECTOR(cfg->vector) |
3539 ((apic->irq_dest_mode == 0) ?
3540 HT_IRQ_LOW_DM_PHYSICAL :
3541 HT_IRQ_LOW_DM_LOGICAL) |
3542 HT_IRQ_LOW_RQEOI_EDGE |
3543 ((apic->irq_delivery_mode != dest_LowestPrio) ?
3544 HT_IRQ_LOW_MT_FIXED :
3545 HT_IRQ_LOW_MT_ARBITRATED) |
3546 HT_IRQ_LOW_IRQ_MASKED;
3547
3548 write_ht_irq_msg(irq, &msg);
3549
3550 irq_set_chip_and_handler_name(irq, &ht_irq_chip,
3551 handle_edge_irq, "edge");
3552
3553 dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq);
3554 }
3555 return err;
3556}
3557#endif /* CONFIG_HT_IRQ */
3558
3559static int
3560io_apic_setup_irq_pin(unsigned int irq, int node, struct io_apic_irq_attr *attr)
3561{
3562 struct irq_cfg *cfg = alloc_irq_and_cfg_at(irq, node);
3563 int ret;
3564
3565 if (!cfg)
3566 return -EINVAL;
3567 ret = __add_pin_to_irq_node(cfg, node, attr->ioapic, attr->ioapic_pin);
3568 if (!ret)
3569 setup_ioapic_irq(attr->ioapic, attr->ioapic_pin, irq, cfg,
3570 attr->trigger, attr->polarity);
3571 return ret;
3572}
3573
3574int io_apic_setup_irq_pin_once(unsigned int irq, int node,
3575 struct io_apic_irq_attr *attr)
3576{
3577 unsigned int id = attr->ioapic, pin = attr->ioapic_pin;
3578 int ret;
3579
3580 /* Avoid redundant programming */
3581 if (test_bit(pin, ioapics[id].pin_programmed)) {
3582 pr_debug("Pin %d-%d already programmed\n",
3583 mpc_ioapic_id(id), pin);
3584 return 0;
3585 }
3586 ret = io_apic_setup_irq_pin(irq, node, attr);
3587 if (!ret)
3588 set_bit(pin, ioapics[id].pin_programmed);
3589 return ret;
3590}
3591
3592static int __init io_apic_get_redir_entries(int ioapic)
3593{
3594 union IO_APIC_reg_01 reg_01;
3595 unsigned long flags;
3596
3597 raw_spin_lock_irqsave(&ioapic_lock, flags);
3598 reg_01.raw = io_apic_read(ioapic, 1);
3599 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3600
3601 /* The register returns the maximum index redir index
3602 * supported, which is one less than the total number of redir
3603 * entries.
3604 */
3605 return reg_01.bits.entries + 1;
3606}
3607
3608static void __init probe_nr_irqs_gsi(void)
3609{
3610 int nr;
3611
3612 nr = gsi_top + NR_IRQS_LEGACY;
3613 if (nr > nr_irqs_gsi)
3614 nr_irqs_gsi = nr;
3615
3616 printk(KERN_DEBUG "nr_irqs_gsi: %d\n", nr_irqs_gsi);
3617}
3618
3619int get_nr_irqs_gsi(void)
3620{
3621 return nr_irqs_gsi;
3622}
3623
3624#ifdef CONFIG_SPARSE_IRQ
3625int __init arch_probe_nr_irqs(void)
3626{
3627 int nr;
3628
3629 if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
3630 nr_irqs = NR_VECTORS * nr_cpu_ids;
3631
3632 nr = nr_irqs_gsi + 8 * nr_cpu_ids;
3633#if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
3634 /*
3635 * for MSI and HT dyn irq
3636 */
3637 nr += nr_irqs_gsi * 16;
3638#endif
3639 if (nr < nr_irqs)
3640 nr_irqs = nr;
3641
3642 return NR_IRQS_LEGACY;
3643}
3644#endif
3645
3646int io_apic_set_pci_routing(struct device *dev, int irq,
3647 struct io_apic_irq_attr *irq_attr)
3648{
3649 int node;
3650
3651 if (!IO_APIC_IRQ(irq)) {
3652 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
3653 irq_attr->ioapic);
3654 return -EINVAL;
3655 }
3656
3657 node = dev ? dev_to_node(dev) : cpu_to_node(0);
3658
3659 return io_apic_setup_irq_pin_once(irq, node, irq_attr);
3660}
3661
3662#ifdef CONFIG_X86_32
3663static int __init io_apic_get_unique_id(int ioapic, int apic_id)
3664{
3665 union IO_APIC_reg_00 reg_00;
3666 static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
3667 physid_mask_t tmp;
3668 unsigned long flags;
3669 int i = 0;
3670
3671 /*
3672 * The P4 platform supports up to 256 APIC IDs on two separate APIC
3673 * buses (one for LAPICs, one for IOAPICs), where predecessors only
3674 * supports up to 16 on one shared APIC bus.
3675 *
3676 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
3677 * advantage of new APIC bus architecture.
3678 */
3679
3680 if (physids_empty(apic_id_map))
3681 apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map);
3682
3683 raw_spin_lock_irqsave(&ioapic_lock, flags);
3684 reg_00.raw = io_apic_read(ioapic, 0);
3685 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3686
3687 if (apic_id >= get_physical_broadcast()) {
3688 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
3689 "%d\n", ioapic, apic_id, reg_00.bits.ID);
3690 apic_id = reg_00.bits.ID;
3691 }
3692
3693 /*
3694 * Every APIC in a system must have a unique ID or we get lots of nice
3695 * 'stuck on smp_invalidate_needed IPI wait' messages.
3696 */
3697 if (apic->check_apicid_used(&apic_id_map, apic_id)) {
3698
3699 for (i = 0; i < get_physical_broadcast(); i++) {
3700 if (!apic->check_apicid_used(&apic_id_map, i))
3701 break;
3702 }
3703
3704 if (i == get_physical_broadcast())
3705 panic("Max apic_id exceeded!\n");
3706
3707 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
3708 "trying %d\n", ioapic, apic_id, i);
3709
3710 apic_id = i;
3711 }
3712
3713 apic->apicid_to_cpu_present(apic_id, &tmp);
3714 physids_or(apic_id_map, apic_id_map, tmp);
3715
3716 if (reg_00.bits.ID != apic_id) {
3717 reg_00.bits.ID = apic_id;
3718
3719 raw_spin_lock_irqsave(&ioapic_lock, flags);
3720 io_apic_write(ioapic, 0, reg_00.raw);
3721 reg_00.raw = io_apic_read(ioapic, 0);
3722 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3723
3724 /* Sanity check */
3725 if (reg_00.bits.ID != apic_id) {
3726 printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
3727 return -1;
3728 }
3729 }
3730
3731 apic_printk(APIC_VERBOSE, KERN_INFO
3732 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
3733
3734 return apic_id;
3735}
3736
3737static u8 __init io_apic_unique_id(u8 id)
3738{
3739 if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
3740 !APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
3741 return io_apic_get_unique_id(nr_ioapics, id);
3742 else
3743 return id;
3744}
3745#else
3746static u8 __init io_apic_unique_id(u8 id)
3747{
3748 int i;
3749 DECLARE_BITMAP(used, 256);
3750
3751 bitmap_zero(used, 256);
3752 for (i = 0; i < nr_ioapics; i++) {
3753 __set_bit(mpc_ioapic_id(i), used);
3754 }
3755 if (!test_bit(id, used))
3756 return id;
3757 return find_first_zero_bit(used, 256);
3758}
3759#endif
3760
3761static int __init io_apic_get_version(int ioapic)
3762{
3763 union IO_APIC_reg_01 reg_01;
3764 unsigned long flags;
3765
3766 raw_spin_lock_irqsave(&ioapic_lock, flags);
3767 reg_01.raw = io_apic_read(ioapic, 1);
3768 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3769
3770 return reg_01.bits.version;
3771}
3772
3773int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity)
3774{
3775 int ioapic, pin, idx;
3776
3777 if (skip_ioapic_setup)
3778 return -1;
3779
3780 ioapic = mp_find_ioapic(gsi);
3781 if (ioapic < 0)
3782 return -1;
3783
3784 pin = mp_find_ioapic_pin(ioapic, gsi);
3785 if (pin < 0)
3786 return -1;
3787
3788 idx = find_irq_entry(ioapic, pin, mp_INT);
3789 if (idx < 0)
3790 return -1;
3791
3792 *trigger = irq_trigger(idx);
3793 *polarity = irq_polarity(idx);
3794 return 0;
3795}
3796
3797/*
3798 * This function currently is only a helper for the i386 smp boot process where
3799 * we need to reprogram the ioredtbls to cater for the cpus which have come online
3800 * so mask in all cases should simply be apic->target_cpus()
3801 */
3802#ifdef CONFIG_SMP
3803void __init setup_ioapic_dest(void)
3804{
3805 int pin, ioapic, irq, irq_entry;
3806 const struct cpumask *mask;
3807 struct irq_data *idata;
3808
3809 if (skip_ioapic_setup == 1)
3810 return;
3811
3812 for (ioapic = 0; ioapic < nr_ioapics; ioapic++)
3813 for (pin = 0; pin < ioapics[ioapic].nr_registers; pin++) {
3814 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
3815 if (irq_entry == -1)
3816 continue;
3817 irq = pin_2_irq(irq_entry, ioapic, pin);
3818
3819 if ((ioapic > 0) && (irq > 16))
3820 continue;
3821
3822 idata = irq_get_irq_data(irq);
3823
3824 /*
3825 * Honour affinities which have been set in early boot
3826 */
3827 if (!irqd_can_balance(idata) || irqd_affinity_was_set(idata))
3828 mask = idata->affinity;
3829 else
3830 mask = apic->target_cpus();
3831
3832 if (intr_remapping_enabled)
3833 ir_ioapic_set_affinity(idata, mask, false);
3834 else
3835 ioapic_set_affinity(idata, mask, false);
3836 }
3837
3838}
3839#endif
3840
3841#define IOAPIC_RESOURCE_NAME_SIZE 11
3842
3843static struct resource *ioapic_resources;
3844
3845static struct resource * __init ioapic_setup_resources(int nr_ioapics)
3846{
3847 unsigned long n;
3848 struct resource *res;
3849 char *mem;
3850 int i;
3851
3852 if (nr_ioapics <= 0)
3853 return NULL;
3854
3855 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
3856 n *= nr_ioapics;
3857
3858 mem = alloc_bootmem(n);
3859 res = (void *)mem;
3860
3861 mem += sizeof(struct resource) * nr_ioapics;
3862
3863 for (i = 0; i < nr_ioapics; i++) {
3864 res[i].name = mem;
3865 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
3866 snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i);
3867 mem += IOAPIC_RESOURCE_NAME_SIZE;
3868 }
3869
3870 ioapic_resources = res;
3871
3872 return res;
3873}
3874
3875void __init ioapic_and_gsi_init(void)
3876{
3877 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
3878 struct resource *ioapic_res;
3879 int i;
3880
3881 ioapic_res = ioapic_setup_resources(nr_ioapics);
3882 for (i = 0; i < nr_ioapics; i++) {
3883 if (smp_found_config) {
3884 ioapic_phys = mpc_ioapic_addr(i);
3885#ifdef CONFIG_X86_32
3886 if (!ioapic_phys) {
3887 printk(KERN_ERR
3888 "WARNING: bogus zero IO-APIC "
3889 "address found in MPTABLE, "
3890 "disabling IO/APIC support!\n");
3891 smp_found_config = 0;
3892 skip_ioapic_setup = 1;
3893 goto fake_ioapic_page;
3894 }
3895#endif
3896 } else {
3897#ifdef CONFIG_X86_32
3898fake_ioapic_page:
3899#endif
3900 ioapic_phys = (unsigned long)alloc_bootmem_pages(PAGE_SIZE);
3901 ioapic_phys = __pa(ioapic_phys);
3902 }
3903 set_fixmap_nocache(idx, ioapic_phys);
3904 apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n",
3905 __fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK),
3906 ioapic_phys);
3907 idx++;
3908
3909 ioapic_res->start = ioapic_phys;
3910 ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1;
3911 ioapic_res++;
3912 }
3913
3914 probe_nr_irqs_gsi();
3915}
3916
3917void __init ioapic_insert_resources(void)
3918{
3919 int i;
3920 struct resource *r = ioapic_resources;
3921
3922 if (!r) {
3923 if (nr_ioapics > 0)
3924 printk(KERN_ERR
3925 "IO APIC resources couldn't be allocated.\n");
3926 return;
3927 }
3928
3929 for (i = 0; i < nr_ioapics; i++) {
3930 insert_resource(&iomem_resource, r);
3931 r++;
3932 }
3933}
3934
3935int mp_find_ioapic(u32 gsi)
3936{
3937 int i = 0;
3938
3939 if (nr_ioapics == 0)
3940 return -1;
3941
3942 /* Find the IOAPIC that manages this GSI. */
3943 for (i = 0; i < nr_ioapics; i++) {
3944 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i);
3945 if ((gsi >= gsi_cfg->gsi_base)
3946 && (gsi <= gsi_cfg->gsi_end))
3947 return i;
3948 }
3949
3950 printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
3951 return -1;
3952}
3953
3954int mp_find_ioapic_pin(int ioapic, u32 gsi)
3955{
3956 struct mp_ioapic_gsi *gsi_cfg;
3957
3958 if (WARN_ON(ioapic == -1))
3959 return -1;
3960
3961 gsi_cfg = mp_ioapic_gsi_routing(ioapic);
3962 if (WARN_ON(gsi > gsi_cfg->gsi_end))
3963 return -1;
3964
3965 return gsi - gsi_cfg->gsi_base;
3966}
3967
3968static __init int bad_ioapic(unsigned long address)
3969{
3970 if (nr_ioapics >= MAX_IO_APICS) {
3971 printk(KERN_WARNING "WARNING: Max # of I/O APICs (%d) exceeded "
3972 "(found %d), skipping\n", MAX_IO_APICS, nr_ioapics);
3973 return 1;
3974 }
3975 if (!address) {
3976 printk(KERN_WARNING "WARNING: Bogus (zero) I/O APIC address"
3977 " found in table, skipping!\n");
3978 return 1;
3979 }
3980 return 0;
3981}
3982
3983void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
3984{
3985 int idx = 0;
3986 int entries;
3987 struct mp_ioapic_gsi *gsi_cfg;
3988
3989 if (bad_ioapic(address))
3990 return;
3991
3992 idx = nr_ioapics;
3993
3994 ioapics[idx].mp_config.type = MP_IOAPIC;
3995 ioapics[idx].mp_config.flags = MPC_APIC_USABLE;
3996 ioapics[idx].mp_config.apicaddr = address;
3997
3998 set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
3999 ioapics[idx].mp_config.apicid = io_apic_unique_id(id);
4000 ioapics[idx].mp_config.apicver = io_apic_get_version(idx);
4001
4002 /*
4003 * Build basic GSI lookup table to facilitate gsi->io_apic lookups
4004 * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
4005 */
4006 entries = io_apic_get_redir_entries(idx);
4007 gsi_cfg = mp_ioapic_gsi_routing(idx);
4008 gsi_cfg->gsi_base = gsi_base;
4009 gsi_cfg->gsi_end = gsi_base + entries - 1;
4010
4011 /*
4012 * The number of IO-APIC IRQ registers (== #pins):
4013 */
4014 ioapics[idx].nr_registers = entries;
4015
4016 if (gsi_cfg->gsi_end >= gsi_top)
4017 gsi_top = gsi_cfg->gsi_end + 1;
4018
4019 printk(KERN_INFO "IOAPIC[%d]: apic_id %d, version %d, address 0x%x, "
4020 "GSI %d-%d\n", idx, mpc_ioapic_id(idx),
4021 mpc_ioapic_ver(idx), mpc_ioapic_addr(idx),
4022 gsi_cfg->gsi_base, gsi_cfg->gsi_end);
4023
4024 nr_ioapics++;
4025}
4026
4027/* Enable IOAPIC early just for system timer */
4028void __init pre_init_apic_IRQ0(void)
4029{
4030 struct io_apic_irq_attr attr = { 0, 0, 0, 0 };
4031
4032 printk(KERN_INFO "Early APIC setup for system timer0\n");
4033#ifndef CONFIG_SMP
4034 physid_set_mask_of_physid(boot_cpu_physical_apicid,
4035 &phys_cpu_present_map);
4036#endif
4037 setup_local_APIC();
4038
4039 io_apic_setup_irq_pin(0, 0, &attr);
4040 irq_set_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq,
4041 "edge");
4042}