1/*
   2 *	Local APIC handling, local APIC timers
   3 *
   4 *	(c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
   5 *
   6 *	Fixes
   7 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
   8 *					thanks to Eric Gilmore
   9 *					and Rolf G. Tews
  10 *					for testing these extensively.
  11 *	Maciej W. Rozycki	:	Various updates and fixes.
  12 *	Mikael Pettersson	:	Power Management for UP-APIC.
  13 *	Pavel Machek and
  14 *	Mikael Pettersson	:	PM converted to driver model.
  15 */
  16
  17#include <linux/perf_event.h>
  18#include <linux/kernel_stat.h>
  19#include <linux/mc146818rtc.h>
  20#include <linux/acpi_pmtmr.h>
  21#include <linux/clockchips.h>
  22#include <linux/interrupt.h>
  23#include <linux/bootmem.h>
  24#include <linux/ftrace.h>
  25#include <linux/ioport.h>
  26#include <linux/module.h>
  27#include <linux/syscore_ops.h>
  28#include <linux/delay.h>
  29#include <linux/timex.h>
  30#include <linux/i8253.h>
  31#include <linux/dmar.h>
  32#include <linux/init.h>
  33#include <linux/cpu.h>
  34#include <linux/dmi.h>
  35#include <linux/smp.h>
  36#include <linux/mm.h>
  37
 
  38#include <asm/irq_remapping.h>
  39#include <asm/perf_event.h>
  40#include <asm/x86_init.h>
  41#include <asm/pgalloc.h>
  42#include <linux/atomic.h>
  43#include <asm/mpspec.h>
  44#include <asm/i8259.h>
  45#include <asm/proto.h>
  46#include <asm/apic.h>
  47#include <asm/io_apic.h>
  48#include <asm/desc.h>
  49#include <asm/hpet.h>
  50#include <asm/idle.h>
  51#include <asm/mtrr.h>
  52#include <asm/time.h>
  53#include <asm/smp.h>
  54#include <asm/mce.h>
  55#include <asm/tsc.h>
  56#include <asm/hypervisor.h>
  57
  58unsigned int num_processors;
  59
  60unsigned disabled_cpus __cpuinitdata;
  61
  62/* Processor that is doing the boot up */
  63unsigned int boot_cpu_physical_apicid = -1U;
 
  64
  65/*
  66 * The highest APIC ID seen during enumeration.
  67 */
  68unsigned int max_physical_apicid;
  69
  70/*
  71 * Bitmask of physically existing CPUs:
  72 */
  73physid_mask_t phys_cpu_present_map;
  74
  75/*
 
 
 
 
 
 
 
 
 
 
 
 
 
  76 * Map cpu index to physical APIC ID
  77 */
  78DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
  79DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
  80EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
  81EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
  82
  83#ifdef CONFIG_X86_32
  84
  85/*
  86 * On x86_32, the mapping between cpu and logical apicid may vary
  87 * depending on apic in use.  The following early percpu variable is
  88 * used for the mapping.  This is where the behaviors of x86_64 and 32
  89 * actually diverge.  Let's keep it ugly for now.
  90 */
  91DEFINE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid, BAD_APICID);
  92
  93/*
  94 * Knob to control our willingness to enable the local APIC.
  95 *
  96 * +1=force-enable
  97 */
  98static int force_enable_local_apic __initdata;
  99/*
 100 * APIC command line parameters
 101 */
 102static int __init parse_lapic(char *arg)
 103{
 104	force_enable_local_apic = 1;
 105	return 0;
 106}
 107early_param("lapic", parse_lapic);
 108/* Local APIC was disabled by the BIOS and enabled by the kernel */
 109static int enabled_via_apicbase;
 110
 111/*
 112 * Handle interrupt mode configuration register (IMCR).
 113 * This register controls whether the interrupt signals
 114 * that reach the BSP come from the master PIC or from the
 115 * local APIC. Before entering Symmetric I/O Mode, either
 116 * the BIOS or the operating system must switch out of
 117 * PIC Mode by changing the IMCR.
 118 */
 119static inline void imcr_pic_to_apic(void)
 120{
 121	/* select IMCR register */
 122	outb(0x70, 0x22);
 123	/* NMI and 8259 INTR go through APIC */
 124	outb(0x01, 0x23);
 125}
 126
 127static inline void imcr_apic_to_pic(void)
 128{
 129	/* select IMCR register */
 130	outb(0x70, 0x22);
 131	/* NMI and 8259 INTR go directly to BSP */
 132	outb(0x00, 0x23);
 133}
 134#endif
 135
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 136#ifdef CONFIG_X86_64
 137static int apic_calibrate_pmtmr __initdata;
 138static __init int setup_apicpmtimer(char *s)
 139{
 140	apic_calibrate_pmtmr = 1;
 141	notsc_setup(NULL);
 142	return 0;
 143}
 144__setup("apicpmtimer", setup_apicpmtimer);
 145#endif
 146
 147int x2apic_mode;
 148#ifdef CONFIG_X86_X2APIC
 149/* x2apic enabled before OS handover */
 150int x2apic_preenabled;
 151static int x2apic_disabled;
 152static int nox2apic;
 153static __init int setup_nox2apic(char *str)
 154{
 155	if (x2apic_enabled()) {
 156		int apicid = native_apic_msr_read(APIC_ID);
 157
 158		if (apicid >= 255) {
 159			pr_warning("Apicid: %08x, cannot enforce nox2apic\n",
 160				   apicid);
 161			return 0;
 162		}
 163
 164		pr_warning("x2apic already enabled. will disable it\n");
 165	} else
 166		setup_clear_cpu_cap(X86_FEATURE_X2APIC);
 167
 168	nox2apic = 1;
 169
 170	return 0;
 171}
 172early_param("nox2apic", setup_nox2apic);
 173#endif
 174
 175unsigned long mp_lapic_addr;
 176int disable_apic;
 177/* Disable local APIC timer from the kernel commandline or via dmi quirk */
 178static int disable_apic_timer __initdata;
 179/* Local APIC timer works in C2 */
 180int local_apic_timer_c2_ok;
 181EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
 182
 183int first_system_vector = 0xfe;
 184
 185/*
 186 * Debug level, exported for io_apic.c
 187 */
 188unsigned int apic_verbosity;
 189
 190int pic_mode;
 191
 192/* Have we found an MP table */
 193int smp_found_config;
 194
 195static struct resource lapic_resource = {
 196	.name = "Local APIC",
 197	.flags = IORESOURCE_MEM | IORESOURCE_BUSY,
 198};
 199
 200unsigned int lapic_timer_frequency = 0;
 201
 202static void apic_pm_activate(void);
 203
 204static unsigned long apic_phys;
 205
 206/*
 207 * Get the LAPIC version
 208 */
 209static inline int lapic_get_version(void)
 210{
 211	return GET_APIC_VERSION(apic_read(APIC_LVR));
 212}
 213
 214/*
 215 * Check, if the APIC is integrated or a separate chip
 216 */
 217static inline int lapic_is_integrated(void)
 218{
 219#ifdef CONFIG_X86_64
 220	return 1;
 221#else
 222	return APIC_INTEGRATED(lapic_get_version());
 223#endif
 224}
 225
 226/*
 227 * Check, whether this is a modern or a first generation APIC
 228 */
 229static int modern_apic(void)
 230{
 231	/* AMD systems use old APIC versions, so check the CPU */
 232	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
 233	    boot_cpu_data.x86 >= 0xf)
 234		return 1;
 235	return lapic_get_version() >= 0x14;
 236}
 237
 238/*
 239 * right after this call apic become NOOP driven
 240 * so apic->write/read doesn't do anything
 241 */
 242static void __init apic_disable(void)
 243{
 244	pr_info("APIC: switched to apic NOOP\n");
 245	apic = &apic_noop;
 246}
 247
 248void native_apic_wait_icr_idle(void)
 249{
 250	while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
 251		cpu_relax();
 252}
 253
 254u32 native_safe_apic_wait_icr_idle(void)
 255{
 256	u32 send_status;
 257	int timeout;
 258
 259	timeout = 0;
 260	do {
 261		send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
 262		if (!send_status)
 263			break;
 264		inc_irq_stat(icr_read_retry_count);
 265		udelay(100);
 266	} while (timeout++ < 1000);
 267
 268	return send_status;
 269}
 270
 271void native_apic_icr_write(u32 low, u32 id)
 272{
 
 
 
 273	apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
 274	apic_write(APIC_ICR, low);
 
 275}
 276
 277u64 native_apic_icr_read(void)
 278{
 279	u32 icr1, icr2;
 280
 281	icr2 = apic_read(APIC_ICR2);
 282	icr1 = apic_read(APIC_ICR);
 283
 284	return icr1 | ((u64)icr2 << 32);
 285}
 286
 287#ifdef CONFIG_X86_32
 288/**
 289 * get_physical_broadcast - Get number of physical broadcast IDs
 290 */
 291int get_physical_broadcast(void)
 292{
 293	return modern_apic() ? 0xff : 0xf;
 294}
 295#endif
 296
 297/**
 298 * lapic_get_maxlvt - get the maximum number of local vector table entries
 299 */
 300int lapic_get_maxlvt(void)
 301{
 302	unsigned int v;
 303
 304	v = apic_read(APIC_LVR);
 305	/*
 306	 * - we always have APIC integrated on 64bit mode
 307	 * - 82489DXs do not report # of LVT entries
 308	 */
 309	return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2;
 310}
 311
 312/*
 313 * Local APIC timer
 314 */
 315
 316/* Clock divisor */
 317#define APIC_DIVISOR 16
 
 318
 319/*
 320 * This function sets up the local APIC timer, with a timeout of
 321 * 'clocks' APIC bus clock. During calibration we actually call
 322 * this function twice on the boot CPU, once with a bogus timeout
 323 * value, second time for real. The other (noncalibrating) CPUs
 324 * call this function only once, with the real, calibrated value.
 325 *
 326 * We do reads before writes even if unnecessary, to get around the
 327 * P5 APIC double write bug.
 328 */
 329static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
 330{
 331	unsigned int lvtt_value, tmp_value;
 332
 333	lvtt_value = LOCAL_TIMER_VECTOR;
 334	if (!oneshot)
 335		lvtt_value |= APIC_LVT_TIMER_PERIODIC;
 
 
 
 336	if (!lapic_is_integrated())
 337		lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);
 338
 339	if (!irqen)
 340		lvtt_value |= APIC_LVT_MASKED;
 341
 342	apic_write(APIC_LVTT, lvtt_value);
 343
 
 
 
 
 
 
 
 
 
 
 
 
 344	/*
 345	 * Divide PICLK by 16
 346	 */
 347	tmp_value = apic_read(APIC_TDCR);
 348	apic_write(APIC_TDCR,
 349		(tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
 350		APIC_TDR_DIV_16);
 351
 352	if (!oneshot)
 353		apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
 354}
 355
 356/*
 357 * Setup extended LVT, AMD specific
 358 *
 359 * Software should use the LVT offsets the BIOS provides.  The offsets
 360 * are determined by the subsystems using it like those for MCE
 361 * threshold or IBS.  On K8 only offset 0 (APIC500) and MCE interrupts
 362 * are supported. Beginning with family 10h at least 4 offsets are
 363 * available.
 364 *
 365 * Since the offsets must be consistent for all cores, we keep track
 366 * of the LVT offsets in software and reserve the offset for the same
 367 * vector also to be used on other cores. An offset is freed by
 368 * setting the entry to APIC_EILVT_MASKED.
 369 *
 370 * If the BIOS is right, there should be no conflicts. Otherwise a
 371 * "[Firmware Bug]: ..." error message is generated. However, if
 372 * software does not properly determines the offsets, it is not
 373 * necessarily a BIOS bug.
 374 */
 375
 376static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
 377
 378static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
 379{
 380	return (old & APIC_EILVT_MASKED)
 381		|| (new == APIC_EILVT_MASKED)
 382		|| ((new & ~APIC_EILVT_MASKED) == old);
 383}
 384
 385static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
 386{
 387	unsigned int rsvd, vector;
 388
 389	if (offset >= APIC_EILVT_NR_MAX)
 390		return ~0;
 391
 392	rsvd = atomic_read(&eilvt_offsets[offset]);
 393	do {
 394		vector = rsvd & ~APIC_EILVT_MASKED;	/* 0: unassigned */
 395		if (vector && !eilvt_entry_is_changeable(vector, new))
 396			/* may not change if vectors are different */
 397			return rsvd;
 398		rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new);
 399	} while (rsvd != new);
 400
 401	rsvd &= ~APIC_EILVT_MASKED;
 402	if (rsvd && rsvd != vector)
 403		pr_info("LVT offset %d assigned for vector 0x%02x\n",
 404			offset, rsvd);
 405
 406	return new;
 407}
 408
 409/*
 410 * If mask=1, the LVT entry does not generate interrupts while mask=0
 411 * enables the vector. See also the BKDGs. Must be called with
 412 * preemption disabled.
 413 */
 414
 415int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
 416{
 417	unsigned long reg = APIC_EILVTn(offset);
 418	unsigned int new, old, reserved;
 419
 420	new = (mask << 16) | (msg_type << 8) | vector;
 421	old = apic_read(reg);
 422	reserved = reserve_eilvt_offset(offset, new);
 423
 424	if (reserved != new) {
 425		pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
 426		       "vector 0x%x, but the register is already in use for "
 427		       "vector 0x%x on another cpu\n",
 428		       smp_processor_id(), reg, offset, new, reserved);
 429		return -EINVAL;
 430	}
 431
 432	if (!eilvt_entry_is_changeable(old, new)) {
 433		pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
 434		       "vector 0x%x, but the register is already in use for "
 435		       "vector 0x%x on this cpu\n",
 436		       smp_processor_id(), reg, offset, new, old);
 437		return -EBUSY;
 438	}
 439
 440	apic_write(reg, new);
 441
 442	return 0;
 443}
 444EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
 445
 446/*
 447 * Program the next event, relative to now
 448 */
 449static int lapic_next_event(unsigned long delta,
 450			    struct clock_event_device *evt)
 451{
 452	apic_write(APIC_TMICT, delta);
 453	return 0;
 454}
 455
 456/*
 457 * Setup the lapic timer in periodic or oneshot mode
 458 */
 459static void lapic_timer_setup(enum clock_event_mode mode,
 460			      struct clock_event_device *evt)
 
 
 
 
 
 
 461{
 462	unsigned long flags;
 463	unsigned int v;
 464
 465	/* Lapic used as dummy for broadcast ? */
 466	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
 467		return;
 468
 469	local_irq_save(flags);
 
 
 
 
 
 470
 471	switch (mode) {
 472	case CLOCK_EVT_MODE_PERIODIC:
 473	case CLOCK_EVT_MODE_ONESHOT:
 474		__setup_APIC_LVTT(lapic_timer_frequency,
 475				  mode != CLOCK_EVT_MODE_PERIODIC, 1);
 476		break;
 477	case CLOCK_EVT_MODE_UNUSED:
 478	case CLOCK_EVT_MODE_SHUTDOWN:
 479		v = apic_read(APIC_LVTT);
 480		v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
 481		apic_write(APIC_LVTT, v);
 482		apic_write(APIC_TMICT, 0);
 483		break;
 484	case CLOCK_EVT_MODE_RESUME:
 485		/* Nothing to do here */
 486		break;
 487	}
 488
 489	local_irq_restore(flags);
 
 
 
 
 
 
 
 
 
 
 
 490}
 491
 492/*
 493 * Local APIC timer broadcast function
 494 */
 495static void lapic_timer_broadcast(const struct cpumask *mask)
 496{
 497#ifdef CONFIG_SMP
 498	apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
 499#endif
 500}
 501
 502
 503/*
 504 * The local apic timer can be used for any function which is CPU local.
 505 */
 506static struct clock_event_device lapic_clockevent = {
 507	.name		= "lapic",
 508	.features	= CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT
 509			| CLOCK_EVT_FEAT_C3STOP | CLOCK_EVT_FEAT_DUMMY,
 510	.shift		= 32,
 511	.set_mode	= lapic_timer_setup,
 512	.set_next_event	= lapic_next_event,
 513	.broadcast	= lapic_timer_broadcast,
 514	.rating		= 100,
 515	.irq		= -1,
 
 
 
 516};
 517static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
 518
 519/*
 520 * Setup the local APIC timer for this CPU. Copy the initialized values
 521 * of the boot CPU and register the clock event in the framework.
 522 */
 523static void __cpuinit setup_APIC_timer(void)
 524{
 525	struct clock_event_device *levt = &__get_cpu_var(lapic_events);
 526
 527	if (this_cpu_has(X86_FEATURE_ARAT)) {
 528		lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
 529		/* Make LAPIC timer preferrable over percpu HPET */
 530		lapic_clockevent.rating = 150;
 531	}
 532
 533	memcpy(levt, &lapic_clockevent, sizeof(*levt));
 534	levt->cpumask = cpumask_of(smp_processor_id());
 535
 536	clockevents_register_device(levt);
 
 
 
 
 
 
 
 
 537}
 538
 539/*
 540 * In this functions we calibrate APIC bus clocks to the external timer.
 541 *
 542 * We want to do the calibration only once since we want to have local timer
 543 * irqs syncron. CPUs connected by the same APIC bus have the very same bus
 544 * frequency.
 545 *
 546 * This was previously done by reading the PIT/HPET and waiting for a wrap
 547 * around to find out, that a tick has elapsed. I have a box, where the PIT
 548 * readout is broken, so it never gets out of the wait loop again. This was
 549 * also reported by others.
 550 *
 551 * Monitoring the jiffies value is inaccurate and the clockevents
 552 * infrastructure allows us to do a simple substitution of the interrupt
 553 * handler.
 554 *
 555 * The calibration routine also uses the pm_timer when possible, as the PIT
 556 * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
 557 * back to normal later in the boot process).
 558 */
 559
 560#define LAPIC_CAL_LOOPS		(HZ/10)
 561
 562static __initdata int lapic_cal_loops = -1;
 563static __initdata long lapic_cal_t1, lapic_cal_t2;
 564static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
 565static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
 566static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
 567
 568/*
 569 * Temporary interrupt handler.
 570 */
 571static void __init lapic_cal_handler(struct clock_event_device *dev)
 572{
 573	unsigned long long tsc = 0;
 574	long tapic = apic_read(APIC_TMCCT);
 575	unsigned long pm = acpi_pm_read_early();
 576
 577	if (cpu_has_tsc)
 578		rdtscll(tsc);
 579
 580	switch (lapic_cal_loops++) {
 581	case 0:
 582		lapic_cal_t1 = tapic;
 583		lapic_cal_tsc1 = tsc;
 584		lapic_cal_pm1 = pm;
 585		lapic_cal_j1 = jiffies;
 586		break;
 587
 588	case LAPIC_CAL_LOOPS:
 589		lapic_cal_t2 = tapic;
 590		lapic_cal_tsc2 = tsc;
 591		if (pm < lapic_cal_pm1)
 592			pm += ACPI_PM_OVRRUN;
 593		lapic_cal_pm2 = pm;
 594		lapic_cal_j2 = jiffies;
 595		break;
 596	}
 597}
 598
 599static int __init
 600calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
 601{
 602	const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
 603	const long pm_thresh = pm_100ms / 100;
 604	unsigned long mult;
 605	u64 res;
 606
 607#ifndef CONFIG_X86_PM_TIMER
 608	return -1;
 609#endif
 610
 611	apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
 612
 613	/* Check, if the PM timer is available */
 614	if (!deltapm)
 615		return -1;
 616
 617	mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
 618
 619	if (deltapm > (pm_100ms - pm_thresh) &&
 620	    deltapm < (pm_100ms + pm_thresh)) {
 621		apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
 622		return 0;
 623	}
 624
 625	res = (((u64)deltapm) *  mult) >> 22;
 626	do_div(res, 1000000);
 627	pr_warning("APIC calibration not consistent "
 628		   "with PM-Timer: %ldms instead of 100ms\n",(long)res);
 629
 630	/* Correct the lapic counter value */
 631	res = (((u64)(*delta)) * pm_100ms);
 632	do_div(res, deltapm);
 633	pr_info("APIC delta adjusted to PM-Timer: "
 634		"%lu (%ld)\n", (unsigned long)res, *delta);
 635	*delta = (long)res;
 636
 637	/* Correct the tsc counter value */
 638	if (cpu_has_tsc) {
 639		res = (((u64)(*deltatsc)) * pm_100ms);
 640		do_div(res, deltapm);
 641		apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
 642					  "PM-Timer: %lu (%ld)\n",
 643					(unsigned long)res, *deltatsc);
 644		*deltatsc = (long)res;
 645	}
 646
 647	return 0;
 648}
 649
 650static int __init calibrate_APIC_clock(void)
 651{
 652	struct clock_event_device *levt = &__get_cpu_var(lapic_events);
 653	void (*real_handler)(struct clock_event_device *dev);
 654	unsigned long deltaj;
 655	long delta, deltatsc;
 656	int pm_referenced = 0;
 657
 658	/**
 659	 * check if lapic timer has already been calibrated by platform
 660	 * specific routine, such as tsc calibration code. if so, we just fill
 661	 * in the clockevent structure and return.
 662	 */
 663
 664	if (lapic_timer_frequency) {
 
 
 665		apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
 666				lapic_timer_frequency);
 667		lapic_clockevent.mult = div_sc(lapic_timer_frequency/APIC_DIVISOR,
 668					TICK_NSEC, lapic_clockevent.shift);
 669		lapic_clockevent.max_delta_ns =
 670			clockevent_delta2ns(0x7FFFFF, &lapic_clockevent);
 671		lapic_clockevent.min_delta_ns =
 672			clockevent_delta2ns(0xF, &lapic_clockevent);
 673		lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
 674		return 0;
 675	}
 676
 
 
 
 677	local_irq_disable();
 678
 679	/* Replace the global interrupt handler */
 680	real_handler = global_clock_event->event_handler;
 681	global_clock_event->event_handler = lapic_cal_handler;
 682
 683	/*
 684	 * Setup the APIC counter to maximum. There is no way the lapic
 685	 * can underflow in the 100ms detection time frame
 686	 */
 687	__setup_APIC_LVTT(0xffffffff, 0, 0);
 688
 689	/* Let the interrupts run */
 690	local_irq_enable();
 691
 692	while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
 693		cpu_relax();
 694
 695	local_irq_disable();
 696
 697	/* Restore the real event handler */
 698	global_clock_event->event_handler = real_handler;
 699
 700	/* Build delta t1-t2 as apic timer counts down */
 701	delta = lapic_cal_t1 - lapic_cal_t2;
 702	apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
 703
 704	deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
 705
 706	/* we trust the PM based calibration if possible */
 707	pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
 708					&delta, &deltatsc);
 709
 710	/* Calculate the scaled math multiplication factor */
 711	lapic_clockevent.mult = div_sc(delta, TICK_NSEC * LAPIC_CAL_LOOPS,
 712				       lapic_clockevent.shift);
 713	lapic_clockevent.max_delta_ns =
 714		clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
 715	lapic_clockevent.min_delta_ns =
 716		clockevent_delta2ns(0xF, &lapic_clockevent);
 717
 718	lapic_timer_frequency = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
 719
 720	apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
 721	apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
 722	apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
 723		    lapic_timer_frequency);
 724
 725	if (cpu_has_tsc) {
 726		apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
 727			    "%ld.%04ld MHz.\n",
 728			    (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
 729			    (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
 730	}
 731
 732	apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
 733		    "%u.%04u MHz.\n",
 734		    lapic_timer_frequency / (1000000 / HZ),
 735		    lapic_timer_frequency % (1000000 / HZ));
 736
 737	/*
 738	 * Do a sanity check on the APIC calibration result
 739	 */
 740	if (lapic_timer_frequency < (1000000 / HZ)) {
 741		local_irq_enable();
 742		pr_warning("APIC frequency too slow, disabling apic timer\n");
 743		return -1;
 744	}
 745
 746	levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
 747
 748	/*
 749	 * PM timer calibration failed or not turned on
 750	 * so lets try APIC timer based calibration
 751	 */
 752	if (!pm_referenced) {
 753		apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
 754
 755		/*
 756		 * Setup the apic timer manually
 757		 */
 758		levt->event_handler = lapic_cal_handler;
 759		lapic_timer_setup(CLOCK_EVT_MODE_PERIODIC, levt);
 760		lapic_cal_loops = -1;
 761
 762		/* Let the interrupts run */
 763		local_irq_enable();
 764
 765		while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
 766			cpu_relax();
 767
 768		/* Stop the lapic timer */
 769		lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, levt);
 
 770
 771		/* Jiffies delta */
 772		deltaj = lapic_cal_j2 - lapic_cal_j1;
 773		apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
 774
 775		/* Check, if the jiffies result is consistent */
 776		if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
 777			apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
 778		else
 779			levt->features |= CLOCK_EVT_FEAT_DUMMY;
 780	} else
 781		local_irq_enable();
 782
 783	if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
 784		pr_warning("APIC timer disabled due to verification failure\n");
 785			return -1;
 786	}
 787
 788	return 0;
 789}
 790
 791/*
 792 * Setup the boot APIC
 793 *
 794 * Calibrate and verify the result.
 795 */
 796void __init setup_boot_APIC_clock(void)
 797{
 798	/*
 799	 * The local apic timer can be disabled via the kernel
 800	 * commandline or from the CPU detection code. Register the lapic
 801	 * timer as a dummy clock event source on SMP systems, so the
 802	 * broadcast mechanism is used. On UP systems simply ignore it.
 803	 */
 804	if (disable_apic_timer) {
 805		pr_info("Disabling APIC timer\n");
 806		/* No broadcast on UP ! */
 807		if (num_possible_cpus() > 1) {
 808			lapic_clockevent.mult = 1;
 809			setup_APIC_timer();
 810		}
 811		return;
 812	}
 813
 814	apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
 815		    "calibrating APIC timer ...\n");
 816
 817	if (calibrate_APIC_clock()) {
 818		/* No broadcast on UP ! */
 819		if (num_possible_cpus() > 1)
 820			setup_APIC_timer();
 821		return;
 822	}
 823
 824	/*
 825	 * If nmi_watchdog is set to IO_APIC, we need the
 826	 * PIT/HPET going.  Otherwise register lapic as a dummy
 827	 * device.
 828	 */
 829	lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
 830
 831	/* Setup the lapic or request the broadcast */
 832	setup_APIC_timer();
 833}
 834
 835void __cpuinit setup_secondary_APIC_clock(void)
 836{
 837	setup_APIC_timer();
 838}
 839
 840/*
 841 * The guts of the apic timer interrupt
 842 */
 843static void local_apic_timer_interrupt(void)
 844{
 845	int cpu = smp_processor_id();
 846	struct clock_event_device *evt = &per_cpu(lapic_events, cpu);
 847
 848	/*
 849	 * Normally we should not be here till LAPIC has been initialized but
 850	 * in some cases like kdump, its possible that there is a pending LAPIC
 851	 * timer interrupt from previous kernel's context and is delivered in
 852	 * new kernel the moment interrupts are enabled.
 853	 *
 854	 * Interrupts are enabled early and LAPIC is setup much later, hence
 855	 * its possible that when we get here evt->event_handler is NULL.
 856	 * Check for event_handler being NULL and discard the interrupt as
 857	 * spurious.
 858	 */
 859	if (!evt->event_handler) {
 860		pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu);
 861		/* Switch it off */
 862		lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, evt);
 863		return;
 864	}
 865
 866	/*
 867	 * the NMI deadlock-detector uses this.
 868	 */
 869	inc_irq_stat(apic_timer_irqs);
 870
 871	evt->event_handler(evt);
 872}
 873
 874/*
 875 * Local APIC timer interrupt. This is the most natural way for doing
 876 * local interrupts, but local timer interrupts can be emulated by
 877 * broadcast interrupts too. [in case the hw doesn't support APIC timers]
 878 *
 879 * [ if a single-CPU system runs an SMP kernel then we call the local
 880 *   interrupt as well. Thus we cannot inline the local irq ... ]
 881 */
 882void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
 883{
 884	struct pt_regs *old_regs = set_irq_regs(regs);
 885
 886	/*
 887	 * NOTE! We'd better ACK the irq immediately,
 888	 * because timer handling can be slow.
 
 
 
 
 889	 */
 890	ack_APIC_irq();
 
 
 
 
 
 
 
 
 
 
 891	/*
 
 
 
 892	 * update_process_times() expects us to have done irq_enter().
 893	 * Besides, if we don't timer interrupts ignore the global
 894	 * interrupt lock, which is the WrongThing (tm) to do.
 895	 */
 896	irq_enter();
 897	exit_idle();
 898	local_apic_timer_interrupt();
 899	irq_exit();
 
 900
 901	set_irq_regs(old_regs);
 902}
 903
 904int setup_profiling_timer(unsigned int multiplier)
 905{
 906	return -EINVAL;
 907}
 908
 909/*
 910 * Local APIC start and shutdown
 911 */
 912
 913/**
 914 * clear_local_APIC - shutdown the local APIC
 915 *
 916 * This is called, when a CPU is disabled and before rebooting, so the state of
 917 * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
 918 * leftovers during boot.
 919 */
 920void clear_local_APIC(void)
 921{
 922	int maxlvt;
 923	u32 v;
 924
 925	/* APIC hasn't been mapped yet */
 926	if (!x2apic_mode && !apic_phys)
 927		return;
 928
 929	maxlvt = lapic_get_maxlvt();
 930	/*
 931	 * Masking an LVT entry can trigger a local APIC error
 932	 * if the vector is zero. Mask LVTERR first to prevent this.
 933	 */
 934	if (maxlvt >= 3) {
 935		v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
 936		apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
 937	}
 938	/*
 939	 * Careful: we have to set masks only first to deassert
 940	 * any level-triggered sources.
 941	 */
 942	v = apic_read(APIC_LVTT);
 943	apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
 944	v = apic_read(APIC_LVT0);
 945	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
 946	v = apic_read(APIC_LVT1);
 947	apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
 948	if (maxlvt >= 4) {
 949		v = apic_read(APIC_LVTPC);
 950		apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
 951	}
 952
 953	/* lets not touch this if we didn't frob it */
 954#ifdef CONFIG_X86_THERMAL_VECTOR
 955	if (maxlvt >= 5) {
 956		v = apic_read(APIC_LVTTHMR);
 957		apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
 958	}
 959#endif
 960#ifdef CONFIG_X86_MCE_INTEL
 961	if (maxlvt >= 6) {
 962		v = apic_read(APIC_LVTCMCI);
 963		if (!(v & APIC_LVT_MASKED))
 964			apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
 965	}
 966#endif
 967
 968	/*
 969	 * Clean APIC state for other OSs:
 970	 */
 971	apic_write(APIC_LVTT, APIC_LVT_MASKED);
 972	apic_write(APIC_LVT0, APIC_LVT_MASKED);
 973	apic_write(APIC_LVT1, APIC_LVT_MASKED);
 974	if (maxlvt >= 3)
 975		apic_write(APIC_LVTERR, APIC_LVT_MASKED);
 976	if (maxlvt >= 4)
 977		apic_write(APIC_LVTPC, APIC_LVT_MASKED);
 978
 979	/* Integrated APIC (!82489DX) ? */
 980	if (lapic_is_integrated()) {
 981		if (maxlvt > 3)
 982			/* Clear ESR due to Pentium errata 3AP and 11AP */
 983			apic_write(APIC_ESR, 0);
 984		apic_read(APIC_ESR);
 985	}
 986}
 987
 988/**
 989 * disable_local_APIC - clear and disable the local APIC
 990 */
 991void disable_local_APIC(void)
 992{
 993	unsigned int value;
 994
 995	/* APIC hasn't been mapped yet */
 996	if (!x2apic_mode && !apic_phys)
 997		return;
 998
 999	clear_local_APIC();
1000
1001	/*
1002	 * Disable APIC (implies clearing of registers
1003	 * for 82489DX!).
1004	 */
1005	value = apic_read(APIC_SPIV);
1006	value &= ~APIC_SPIV_APIC_ENABLED;
1007	apic_write(APIC_SPIV, value);
1008
1009#ifdef CONFIG_X86_32
1010	/*
1011	 * When LAPIC was disabled by the BIOS and enabled by the kernel,
1012	 * restore the disabled state.
1013	 */
1014	if (enabled_via_apicbase) {
1015		unsigned int l, h;
1016
1017		rdmsr(MSR_IA32_APICBASE, l, h);
1018		l &= ~MSR_IA32_APICBASE_ENABLE;
1019		wrmsr(MSR_IA32_APICBASE, l, h);
1020	}
1021#endif
1022}
1023
1024/*
1025 * If Linux enabled the LAPIC against the BIOS default disable it down before
1026 * re-entering the BIOS on shutdown.  Otherwise the BIOS may get confused and
1027 * not power-off.  Additionally clear all LVT entries before disable_local_APIC
1028 * for the case where Linux didn't enable the LAPIC.
1029 */
1030void lapic_shutdown(void)
1031{
1032	unsigned long flags;
1033
1034	if (!cpu_has_apic && !apic_from_smp_config())
1035		return;
1036
1037	local_irq_save(flags);
1038
1039#ifdef CONFIG_X86_32
1040	if (!enabled_via_apicbase)
1041		clear_local_APIC();
1042	else
1043#endif
1044		disable_local_APIC();
1045
1046
1047	local_irq_restore(flags);
1048}
1049
1050/*
1051 * This is to verify that we're looking at a real local APIC.
1052 * Check these against your board if the CPUs aren't getting
1053 * started for no apparent reason.
1054 */
1055int __init verify_local_APIC(void)
1056{
1057	unsigned int reg0, reg1;
1058
1059	/*
1060	 * The version register is read-only in a real APIC.
1061	 */
1062	reg0 = apic_read(APIC_LVR);
1063	apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0);
1064	apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK);
1065	reg1 = apic_read(APIC_LVR);
1066	apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1);
1067
1068	/*
1069	 * The two version reads above should print the same
1070	 * numbers.  If the second one is different, then we
1071	 * poke at a non-APIC.
1072	 */
1073	if (reg1 != reg0)
1074		return 0;
1075
1076	/*
1077	 * Check if the version looks reasonably.
1078	 */
1079	reg1 = GET_APIC_VERSION(reg0);
1080	if (reg1 == 0x00 || reg1 == 0xff)
1081		return 0;
1082	reg1 = lapic_get_maxlvt();
1083	if (reg1 < 0x02 || reg1 == 0xff)
1084		return 0;
1085
1086	/*
1087	 * The ID register is read/write in a real APIC.
1088	 */
1089	reg0 = apic_read(APIC_ID);
1090	apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0);
1091	apic_write(APIC_ID, reg0 ^ apic->apic_id_mask);
1092	reg1 = apic_read(APIC_ID);
1093	apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg1);
1094	apic_write(APIC_ID, reg0);
1095	if (reg1 != (reg0 ^ apic->apic_id_mask))
1096		return 0;
1097
1098	/*
1099	 * The next two are just to see if we have sane values.
1100	 * They're only really relevant if we're in Virtual Wire
1101	 * compatibility mode, but most boxes are anymore.
1102	 */
1103	reg0 = apic_read(APIC_LVT0);
1104	apic_printk(APIC_DEBUG, "Getting LVT0: %x\n", reg0);
1105	reg1 = apic_read(APIC_LVT1);
1106	apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1);
1107
1108	return 1;
1109}
1110
1111/**
1112 * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1113 */
1114void __init sync_Arb_IDs(void)
1115{
1116	/*
1117	 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1118	 * needed on AMD.
1119	 */
1120	if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1121		return;
1122
1123	/*
1124	 * Wait for idle.
1125	 */
1126	apic_wait_icr_idle();
1127
1128	apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
1129	apic_write(APIC_ICR, APIC_DEST_ALLINC |
1130			APIC_INT_LEVELTRIG | APIC_DM_INIT);
1131}
1132
1133/*
1134 * An initial setup of the virtual wire mode.
1135 */
1136void __init init_bsp_APIC(void)
1137{
1138	unsigned int value;
1139
1140	/*
1141	 * Don't do the setup now if we have a SMP BIOS as the
1142	 * through-I/O-APIC virtual wire mode might be active.
1143	 */
1144	if (smp_found_config || !cpu_has_apic)
1145		return;
1146
1147	/*
1148	 * Do not trust the local APIC being empty at bootup.
1149	 */
1150	clear_local_APIC();
1151
1152	/*
1153	 * Enable APIC.
1154	 */
1155	value = apic_read(APIC_SPIV);
1156	value &= ~APIC_VECTOR_MASK;
1157	value |= APIC_SPIV_APIC_ENABLED;
1158
1159#ifdef CONFIG_X86_32
1160	/* This bit is reserved on P4/Xeon and should be cleared */
1161	if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1162	    (boot_cpu_data.x86 == 15))
1163		value &= ~APIC_SPIV_FOCUS_DISABLED;
1164	else
1165#endif
1166		value |= APIC_SPIV_FOCUS_DISABLED;
1167	value |= SPURIOUS_APIC_VECTOR;
1168	apic_write(APIC_SPIV, value);
1169
1170	/*
1171	 * Set up the virtual wire mode.
1172	 */
1173	apic_write(APIC_LVT0, APIC_DM_EXTINT);
1174	value = APIC_DM_NMI;
1175	if (!lapic_is_integrated())		/* 82489DX */
1176		value |= APIC_LVT_LEVEL_TRIGGER;
 
 
1177	apic_write(APIC_LVT1, value);
1178}
1179
1180static void __cpuinit lapic_setup_esr(void)
1181{
1182	unsigned int oldvalue, value, maxlvt;
1183
1184	if (!lapic_is_integrated()) {
1185		pr_info("No ESR for 82489DX.\n");
1186		return;
1187	}
1188
1189	if (apic->disable_esr) {
1190		/*
1191		 * Something untraceable is creating bad interrupts on
1192		 * secondary quads ... for the moment, just leave the
1193		 * ESR disabled - we can't do anything useful with the
1194		 * errors anyway - mbligh
1195		 */
1196		pr_info("Leaving ESR disabled.\n");
1197		return;
1198	}
1199
1200	maxlvt = lapic_get_maxlvt();
1201	if (maxlvt > 3)		/* Due to the Pentium erratum 3AP. */
1202		apic_write(APIC_ESR, 0);
1203	oldvalue = apic_read(APIC_ESR);
1204
1205	/* enables sending errors */
1206	value = ERROR_APIC_VECTOR;
1207	apic_write(APIC_LVTERR, value);
1208
1209	/*
1210	 * spec says clear errors after enabling vector.
1211	 */
1212	if (maxlvt > 3)
1213		apic_write(APIC_ESR, 0);
1214	value = apic_read(APIC_ESR);
1215	if (value != oldvalue)
1216		apic_printk(APIC_VERBOSE, "ESR value before enabling "
1217			"vector: 0x%08x  after: 0x%08x\n",
1218			oldvalue, value);
1219}
1220
1221/**
1222 * setup_local_APIC - setup the local APIC
1223 *
1224 * Used to setup local APIC while initializing BSP or bringin up APs.
1225 * Always called with preemption disabled.
1226 */
1227void __cpuinit setup_local_APIC(void)
1228{
1229	int cpu = smp_processor_id();
1230	unsigned int value, queued;
1231	int i, j, acked = 0;
1232	unsigned long long tsc = 0, ntsc;
1233	long long max_loops = cpu_khz;
1234
1235	if (cpu_has_tsc)
1236		rdtscll(tsc);
1237
1238	if (disable_apic) {
1239		disable_ioapic_support();
1240		return;
1241	}
1242
1243#ifdef CONFIG_X86_32
1244	/* Pound the ESR really hard over the head with a big hammer - mbligh */
1245	if (lapic_is_integrated() && apic->disable_esr) {
1246		apic_write(APIC_ESR, 0);
1247		apic_write(APIC_ESR, 0);
1248		apic_write(APIC_ESR, 0);
1249		apic_write(APIC_ESR, 0);
1250	}
1251#endif
1252	perf_events_lapic_init();
1253
1254	/*
1255	 * Double-check whether this APIC is really registered.
1256	 * This is meaningless in clustered apic mode, so we skip it.
1257	 */
1258	BUG_ON(!apic->apic_id_registered());
1259
1260	/*
1261	 * Intel recommends to set DFR, LDR and TPR before enabling
1262	 * an APIC.  See e.g. "AP-388 82489DX User's Manual" (Intel
1263	 * document number 292116).  So here it goes...
1264	 */
1265	apic->init_apic_ldr();
1266
1267#ifdef CONFIG_X86_32
1268	/*
1269	 * APIC LDR is initialized.  If logical_apicid mapping was
1270	 * initialized during get_smp_config(), make sure it matches the
1271	 * actual value.
1272	 */
1273	i = early_per_cpu(x86_cpu_to_logical_apicid, cpu);
1274	WARN_ON(i != BAD_APICID && i != logical_smp_processor_id());
1275	/* always use the value from LDR */
1276	early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
1277		logical_smp_processor_id();
1278
1279	/*
1280	 * Some NUMA implementations (NUMAQ) don't initialize apicid to
1281	 * node mapping during NUMA init.  Now that logical apicid is
1282	 * guaranteed to be known, give it another chance.  This is already
1283	 * a bit too late - percpu allocation has already happened without
1284	 * proper NUMA affinity.
1285	 */
1286	if (apic->x86_32_numa_cpu_node)
1287		set_apicid_to_node(early_per_cpu(x86_cpu_to_apicid, cpu),
1288				   apic->x86_32_numa_cpu_node(cpu));
1289#endif
1290
1291	/*
1292	 * Set Task Priority to 'accept all'. We never change this
1293	 * later on.
1294	 */
1295	value = apic_read(APIC_TASKPRI);
1296	value &= ~APIC_TPRI_MASK;
1297	apic_write(APIC_TASKPRI, value);
1298
1299	/*
1300	 * After a crash, we no longer service the interrupts and a pending
1301	 * interrupt from previous kernel might still have ISR bit set.
1302	 *
1303	 * Most probably by now CPU has serviced that pending interrupt and
1304	 * it might not have done the ack_APIC_irq() because it thought,
1305	 * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it
1306	 * does not clear the ISR bit and cpu thinks it has already serivced
1307	 * the interrupt. Hence a vector might get locked. It was noticed
1308	 * for timer irq (vector 0x31). Issue an extra EOI to clear ISR.
1309	 */
1310	do {
1311		queued = 0;
1312		for (i = APIC_ISR_NR - 1; i >= 0; i--)
1313			queued |= apic_read(APIC_IRR + i*0x10);
1314
1315		for (i = APIC_ISR_NR - 1; i >= 0; i--) {
1316			value = apic_read(APIC_ISR + i*0x10);
1317			for (j = 31; j >= 0; j--) {
1318				if (value & (1<<j)) {
1319					ack_APIC_irq();
1320					acked++;
1321				}
1322			}
1323		}
1324		if (acked > 256) {
1325			printk(KERN_ERR "LAPIC pending interrupts after %d EOI\n",
1326			       acked);
1327			break;
1328		}
1329		if (queued) {
1330			if (cpu_has_tsc) {
1331				rdtscll(ntsc);
1332				max_loops = (cpu_khz << 10) - (ntsc - tsc);
1333			} else
1334				max_loops--;
1335		}
1336	} while (queued && max_loops > 0);
1337	WARN_ON(max_loops <= 0);
1338
1339	/*
1340	 * Now that we are all set up, enable the APIC
1341	 */
1342	value = apic_read(APIC_SPIV);
1343	value &= ~APIC_VECTOR_MASK;
1344	/*
1345	 * Enable APIC
1346	 */
1347	value |= APIC_SPIV_APIC_ENABLED;
1348
1349#ifdef CONFIG_X86_32
1350	/*
1351	 * Some unknown Intel IO/APIC (or APIC) errata is biting us with
1352	 * certain networking cards. If high frequency interrupts are
1353	 * happening on a particular IOAPIC pin, plus the IOAPIC routing
1354	 * entry is masked/unmasked at a high rate as well then sooner or
1355	 * later IOAPIC line gets 'stuck', no more interrupts are received
1356	 * from the device. If focus CPU is disabled then the hang goes
1357	 * away, oh well :-(
1358	 *
1359	 * [ This bug can be reproduced easily with a level-triggered
1360	 *   PCI Ne2000 networking cards and PII/PIII processors, dual
1361	 *   BX chipset. ]
1362	 */
1363	/*
1364	 * Actually disabling the focus CPU check just makes the hang less
1365	 * frequent as it makes the interrupt distributon model be more
1366	 * like LRU than MRU (the short-term load is more even across CPUs).
1367	 * See also the comment in end_level_ioapic_irq().  --macro
1368	 */
1369
1370	/*
1371	 * - enable focus processor (bit==0)
1372	 * - 64bit mode always use processor focus
1373	 *   so no need to set it
1374	 */
1375	value &= ~APIC_SPIV_FOCUS_DISABLED;
1376#endif
1377
1378	/*
1379	 * Set spurious IRQ vector
1380	 */
1381	value |= SPURIOUS_APIC_VECTOR;
1382	apic_write(APIC_SPIV, value);
1383
1384	/*
1385	 * Set up LVT0, LVT1:
1386	 *
1387	 * set up through-local-APIC on the BP's LINT0. This is not
1388	 * strictly necessary in pure symmetric-IO mode, but sometimes
1389	 * we delegate interrupts to the 8259A.
1390	 */
1391	/*
1392	 * TODO: set up through-local-APIC from through-I/O-APIC? --macro
1393	 */
1394	value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1395	if (!cpu && (pic_mode || !value)) {
1396		value = APIC_DM_EXTINT;
1397		apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
1398	} else {
1399		value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1400		apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
1401	}
1402	apic_write(APIC_LVT0, value);
1403
1404	/*
1405	 * only the BP should see the LINT1 NMI signal, obviously.
 
1406	 */
1407	if (!cpu)
 
1408		value = APIC_DM_NMI;
1409	else
1410		value = APIC_DM_NMI | APIC_LVT_MASKED;
1411	if (!lapic_is_integrated())		/* 82489DX */
1412		value |= APIC_LVT_LEVEL_TRIGGER;
1413	apic_write(APIC_LVT1, value);
1414
1415#ifdef CONFIG_X86_MCE_INTEL
1416	/* Recheck CMCI information after local APIC is up on CPU #0 */
1417	if (!cpu)
1418		cmci_recheck();
1419#endif
1420}
1421
1422void __cpuinit end_local_APIC_setup(void)
1423{
1424	lapic_setup_esr();
1425
1426#ifdef CONFIG_X86_32
1427	{
1428		unsigned int value;
1429		/* Disable the local apic timer */
1430		value = apic_read(APIC_LVTT);
1431		value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1432		apic_write(APIC_LVTT, value);
1433	}
1434#endif
1435
1436	apic_pm_activate();
1437}
1438
1439void __init bsp_end_local_APIC_setup(void)
 
 
 
1440{
 
1441	end_local_APIC_setup();
1442
1443	/*
1444	 * Now that local APIC setup is completed for BP, configure the fault
1445	 * handling for interrupt remapping.
1446	 */
1447	if (irq_remapping_enabled)
1448		irq_remap_enable_fault_handling();
1449
1450}
1451
1452#ifdef CONFIG_X86_X2APIC
1453/*
1454 * Need to disable xapic and x2apic at the same time and then enable xapic mode
1455 */
1456static inline void __disable_x2apic(u64 msr)
1457{
1458	wrmsrl(MSR_IA32_APICBASE,
1459	       msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1460	wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
1461}
1462
1463static __init void disable_x2apic(void)
1464{
1465	u64 msr;
1466
1467	if (!cpu_has_x2apic)
1468		return;
1469
1470	rdmsrl(MSR_IA32_APICBASE, msr);
1471	if (msr & X2APIC_ENABLE) {
1472		u32 x2apic_id = read_apic_id();
1473
1474		if (x2apic_id >= 255)
1475			panic("Cannot disable x2apic, id: %08x\n", x2apic_id);
 
 
1476
1477		pr_info("Disabling x2apic\n");
1478		__disable_x2apic(msr);
 
1479
1480		if (nox2apic) {
1481			clear_cpu_cap(&cpu_data(0), X86_FEATURE_X2APIC);
1482			setup_clear_cpu_cap(X86_FEATURE_X2APIC);
1483		}
 
 
1484
1485		x2apic_disabled = 1;
1486		x2apic_mode = 0;
 
 
1487
1488		register_lapic_address(mp_lapic_addr);
 
 
 
 
 
 
1489	}
 
 
 
 
1490}
 
1491
1492void check_x2apic(void)
 
1493{
1494	if (x2apic_enabled()) {
1495		pr_info("x2apic enabled by BIOS, switching to x2apic ops\n");
1496		x2apic_preenabled = x2apic_mode = 1;
 
 
 
 
1497	}
 
1498}
1499
1500void enable_x2apic(void)
1501{
1502	u64 msr;
1503
1504	rdmsrl(MSR_IA32_APICBASE, msr);
1505	if (x2apic_disabled) {
1506		__disable_x2apic(msr);
 
1507		return;
1508	}
1509
1510	if (!x2apic_mode)
 
 
 
 
 
 
 
 
 
 
1511		return;
1512
1513	if (!(msr & X2APIC_ENABLE)) {
1514		printk_once(KERN_INFO "Enabling x2apic\n");
1515		wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1516	}
 
1517}
1518#endif /* CONFIG_X86_X2APIC */
1519
1520int __init enable_IR(void)
1521{
1522#ifdef CONFIG_IRQ_REMAP
1523	if (!irq_remapping_supported()) {
1524		pr_debug("intr-remapping not supported\n");
1525		return -1;
 
 
1526	}
 
 
 
 
 
 
 
 
 
 
 
 
1527
1528	if (!x2apic_preenabled && skip_ioapic_setup) {
1529		pr_info("Skipped enabling intr-remap because of skipping "
1530			"io-apic setup\n");
 
 
 
 
 
 
1531		return -1;
1532	}
1533
1534	return irq_remapping_enable();
1535#endif
1536	return -1;
1537}
1538
1539void __init enable_IR_x2apic(void)
1540{
1541	unsigned long flags;
1542	int ret, x2apic_enabled = 0;
1543	int hardware_init_ret;
1544
1545	/* Make sure irq_remap_ops are initialized */
1546	setup_irq_remapping_ops();
1547
1548	hardware_init_ret = irq_remapping_prepare();
1549	if (hardware_init_ret && !x2apic_supported())
1550		return;
1551
1552	ret = save_ioapic_entries();
1553	if (ret) {
1554		pr_info("Saving IO-APIC state failed: %d\n", ret);
1555		return;
1556	}
1557
1558	local_irq_save(flags);
1559	legacy_pic->mask_all();
1560	mask_ioapic_entries();
1561
1562	if (x2apic_preenabled && nox2apic)
1563		disable_x2apic();
1564
1565	if (hardware_init_ret)
1566		ret = -1;
1567	else
1568		ret = enable_IR();
1569
1570	if (!x2apic_supported())
1571		goto skip_x2apic;
1572
1573	if (ret < 0) {
1574		/* IR is required if there is APIC ID > 255 even when running
1575		 * under KVM
1576		 */
1577		if (max_physical_apicid > 255 ||
1578		    !hypervisor_x2apic_available()) {
1579			if (x2apic_preenabled)
1580				disable_x2apic();
1581			goto skip_x2apic;
1582		}
1583		/*
1584		 * without IR all CPUs can be addressed by IOAPIC/MSI
1585		 * only in physical mode
1586		 */
1587		x2apic_force_phys();
1588	}
1589
1590	if (ret == IRQ_REMAP_XAPIC_MODE) {
1591		pr_info("x2apic not enabled, IRQ remapping is in xapic mode\n");
1592		goto skip_x2apic;
1593	}
1594
1595	x2apic_enabled = 1;
1596
1597	if (x2apic_supported() && !x2apic_mode) {
1598		x2apic_mode = 1;
1599		enable_x2apic();
1600		pr_info("Enabled x2apic\n");
1601	}
1602
1603skip_x2apic:
1604	if (ret < 0) /* IR enabling failed */
1605		restore_ioapic_entries();
1606	legacy_pic->restore_mask();
1607	local_irq_restore(flags);
1608}
1609
1610#ifdef CONFIG_X86_64
1611/*
1612 * Detect and enable local APICs on non-SMP boards.
1613 * Original code written by Keir Fraser.
1614 * On AMD64 we trust the BIOS - if it says no APIC it is likely
1615 * not correctly set up (usually the APIC timer won't work etc.)
1616 */
1617static int __init detect_init_APIC(void)
1618{
1619	if (!cpu_has_apic) {
1620		pr_info("No local APIC present\n");
1621		return -1;
1622	}
1623
1624	mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1625	return 0;
1626}
1627#else
1628
1629static int __init apic_verify(void)
1630{
1631	u32 features, h, l;
1632
1633	/*
1634	 * The APIC feature bit should now be enabled
1635	 * in `cpuid'
1636	 */
1637	features = cpuid_edx(1);
1638	if (!(features & (1 << X86_FEATURE_APIC))) {
1639		pr_warning("Could not enable APIC!\n");
1640		return -1;
1641	}
1642	set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1643	mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1644
1645	/* The BIOS may have set up the APIC at some other address */
1646	if (boot_cpu_data.x86 >= 6) {
1647		rdmsr(MSR_IA32_APICBASE, l, h);
1648		if (l & MSR_IA32_APICBASE_ENABLE)
1649			mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
1650	}
1651
1652	pr_info("Found and enabled local APIC!\n");
1653	return 0;
1654}
1655
1656int __init apic_force_enable(unsigned long addr)
1657{
1658	u32 h, l;
1659
1660	if (disable_apic)
1661		return -1;
1662
1663	/*
1664	 * Some BIOSes disable the local APIC in the APIC_BASE
1665	 * MSR. This can only be done in software for Intel P6 or later
1666	 * and AMD K7 (Model > 1) or later.
1667	 */
1668	if (boot_cpu_data.x86 >= 6) {
1669		rdmsr(MSR_IA32_APICBASE, l, h);
1670		if (!(l & MSR_IA32_APICBASE_ENABLE)) {
1671			pr_info("Local APIC disabled by BIOS -- reenabling.\n");
1672			l &= ~MSR_IA32_APICBASE_BASE;
1673			l |= MSR_IA32_APICBASE_ENABLE | addr;
1674			wrmsr(MSR_IA32_APICBASE, l, h);
1675			enabled_via_apicbase = 1;
1676		}
1677	}
1678	return apic_verify();
1679}
1680
1681/*
1682 * Detect and initialize APIC
1683 */
1684static int __init detect_init_APIC(void)
1685{
1686	/* Disabled by kernel option? */
1687	if (disable_apic)
1688		return -1;
1689
1690	switch (boot_cpu_data.x86_vendor) {
1691	case X86_VENDOR_AMD:
1692		if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
1693		    (boot_cpu_data.x86 >= 15))
1694			break;
1695		goto no_apic;
1696	case X86_VENDOR_INTEL:
1697		if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
1698		    (boot_cpu_data.x86 == 5 && cpu_has_apic))
1699			break;
1700		goto no_apic;
1701	default:
1702		goto no_apic;
1703	}
1704
1705	if (!cpu_has_apic) {
1706		/*
1707		 * Over-ride BIOS and try to enable the local APIC only if
1708		 * "lapic" specified.
1709		 */
1710		if (!force_enable_local_apic) {
1711			pr_info("Local APIC disabled by BIOS -- "
1712				"you can enable it with \"lapic\"\n");
1713			return -1;
1714		}
1715		if (apic_force_enable(APIC_DEFAULT_PHYS_BASE))
1716			return -1;
1717	} else {
1718		if (apic_verify())
1719			return -1;
1720	}
1721
1722	apic_pm_activate();
1723
1724	return 0;
1725
1726no_apic:
1727	pr_info("No local APIC present or hardware disabled\n");
1728	return -1;
1729}
1730#endif
1731
1732/**
1733 * init_apic_mappings - initialize APIC mappings
1734 */
1735void __init init_apic_mappings(void)
1736{
1737	unsigned int new_apicid;
1738
1739	if (x2apic_mode) {
1740		boot_cpu_physical_apicid = read_apic_id();
1741		return;
1742	}
1743
1744	/* If no local APIC can be found return early */
1745	if (!smp_found_config && detect_init_APIC()) {
1746		/* lets NOP'ify apic operations */
1747		pr_info("APIC: disable apic facility\n");
1748		apic_disable();
1749	} else {
1750		apic_phys = mp_lapic_addr;
1751
1752		/*
1753		 * acpi lapic path already maps that address in
1754		 * acpi_register_lapic_address()
1755		 */
1756		if (!acpi_lapic && !smp_found_config)
1757			register_lapic_address(apic_phys);
1758	}
1759
1760	/*
1761	 * Fetch the APIC ID of the BSP in case we have a
1762	 * default configuration (or the MP table is broken).
1763	 */
1764	new_apicid = read_apic_id();
1765	if (boot_cpu_physical_apicid != new_apicid) {
1766		boot_cpu_physical_apicid = new_apicid;
1767		/*
1768		 * yeah -- we lie about apic_version
1769		 * in case if apic was disabled via boot option
1770		 * but it's not a problem for SMP compiled kernel
1771		 * since smp_sanity_check is prepared for such a case
1772		 * and disable smp mode
1773		 */
1774		apic_version[new_apicid] =
1775			 GET_APIC_VERSION(apic_read(APIC_LVR));
1776	}
1777}
1778
1779void __init register_lapic_address(unsigned long address)
1780{
1781	mp_lapic_addr = address;
1782
1783	if (!x2apic_mode) {
1784		set_fixmap_nocache(FIX_APIC_BASE, address);
1785		apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
1786			    APIC_BASE, mp_lapic_addr);
1787	}
1788	if (boot_cpu_physical_apicid == -1U) {
1789		boot_cpu_physical_apicid  = read_apic_id();
1790		apic_version[boot_cpu_physical_apicid] =
1791			 GET_APIC_VERSION(apic_read(APIC_LVR));
1792	}
1793}
1794
1795/*
1796 * This initializes the IO-APIC and APIC hardware if this is
1797 * a UP kernel.
1798 */
1799int apic_version[MAX_LOCAL_APIC];
1800
1801int __init APIC_init_uniprocessor(void)
1802{
1803	if (disable_apic) {
1804		pr_info("Apic disabled\n");
1805		return -1;
1806	}
1807#ifdef CONFIG_X86_64
1808	if (!cpu_has_apic) {
1809		disable_apic = 1;
1810		pr_info("Apic disabled by BIOS\n");
1811		return -1;
1812	}
1813#else
1814	if (!smp_found_config && !cpu_has_apic)
1815		return -1;
1816
1817	/*
1818	 * Complain if the BIOS pretends there is one.
1819	 */
1820	if (!cpu_has_apic &&
1821	    APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
1822		pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
1823			boot_cpu_physical_apicid);
1824		return -1;
1825	}
1826#endif
1827
1828	default_setup_apic_routing();
1829
1830	verify_local_APIC();
1831	connect_bsp_APIC();
1832
1833#ifdef CONFIG_X86_64
1834	apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid));
1835#else
1836	/*
1837	 * Hack: In case of kdump, after a crash, kernel might be booting
1838	 * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
1839	 * might be zero if read from MP tables. Get it from LAPIC.
1840	 */
1841# ifdef CONFIG_CRASH_DUMP
1842	boot_cpu_physical_apicid = read_apic_id();
1843# endif
1844#endif
1845	physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
1846	setup_local_APIC();
1847
1848#ifdef CONFIG_X86_IO_APIC
1849	/*
1850	 * Now enable IO-APICs, actually call clear_IO_APIC
1851	 * We need clear_IO_APIC before enabling error vector
1852	 */
1853	if (!skip_ioapic_setup && nr_ioapics)
1854		enable_IO_APIC();
1855#endif
1856
1857	bsp_end_local_APIC_setup();
1858
1859#ifdef CONFIG_X86_IO_APIC
1860	if (smp_found_config && !skip_ioapic_setup && nr_ioapics)
1861		setup_IO_APIC();
1862	else {
1863		nr_ioapics = 0;
1864	}
1865#endif
1866
1867	x86_init.timers.setup_percpu_clockev();
1868	return 0;
1869}
1870
1871/*
1872 * Local APIC interrupts
1873 */
1874
1875/*
1876 * This interrupt should _never_ happen with our APIC/SMP architecture
1877 */
1878void smp_spurious_interrupt(struct pt_regs *regs)
1879{
1880	u32 v;
1881
1882	irq_enter();
1883	exit_idle();
1884	/*
1885	 * Check if this really is a spurious interrupt and ACK it
1886	 * if it is a vectored one.  Just in case...
1887	 * Spurious interrupts should not be ACKed.
1888	 */
1889	v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1));
1890	if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
1891		ack_APIC_irq();
1892
1893	inc_irq_stat(irq_spurious_count);
1894
1895	/* see sw-dev-man vol 3, chapter 7.4.13.5 */
1896	pr_info("spurious APIC interrupt on CPU#%d, "
1897		"should never happen.\n", smp_processor_id());
1898	irq_exit();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1899}
1900
1901/*
1902 * This interrupt should never happen with our APIC/SMP architecture
1903 */
1904void smp_error_interrupt(struct pt_regs *regs)
1905{
1906	u32 v0, v1;
1907	u32 i = 0;
1908	static const char * const error_interrupt_reason[] = {
1909		"Send CS error",		/* APIC Error Bit 0 */
1910		"Receive CS error",		/* APIC Error Bit 1 */
1911		"Send accept error",		/* APIC Error Bit 2 */
1912		"Receive accept error",		/* APIC Error Bit 3 */
1913		"Redirectable IPI",		/* APIC Error Bit 4 */
1914		"Send illegal vector",		/* APIC Error Bit 5 */
1915		"Received illegal vector",	/* APIC Error Bit 6 */
1916		"Illegal register address",	/* APIC Error Bit 7 */
1917	};
1918
1919	irq_enter();
1920	exit_idle();
1921	/* First tickle the hardware, only then report what went on. -- REW */
1922	v0 = apic_read(APIC_ESR);
1923	apic_write(APIC_ESR, 0);
1924	v1 = apic_read(APIC_ESR);
1925	ack_APIC_irq();
1926	atomic_inc(&irq_err_count);
1927
1928	apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x(%02x)",
1929		    smp_processor_id(), v0 , v1);
1930
1931	v1 = v1 & 0xff;
1932	while (v1) {
1933		if (v1 & 0x1)
1934			apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
1935		i++;
1936		v1 >>= 1;
1937	};
1938
1939	apic_printk(APIC_DEBUG, KERN_CONT "\n");
1940
1941	irq_exit();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1942}
1943
1944/**
1945 * connect_bsp_APIC - attach the APIC to the interrupt system
1946 */
1947void __init connect_bsp_APIC(void)
1948{
1949#ifdef CONFIG_X86_32
1950	if (pic_mode) {
1951		/*
1952		 * Do not trust the local APIC being empty at bootup.
1953		 */
1954		clear_local_APIC();
1955		/*
1956		 * PIC mode, enable APIC mode in the IMCR, i.e.  connect BSP's
1957		 * local APIC to INT and NMI lines.
1958		 */
1959		apic_printk(APIC_VERBOSE, "leaving PIC mode, "
1960				"enabling APIC mode.\n");
1961		imcr_pic_to_apic();
1962	}
1963#endif
1964	if (apic->enable_apic_mode)
1965		apic->enable_apic_mode();
1966}
1967
1968/**
1969 * disconnect_bsp_APIC - detach the APIC from the interrupt system
1970 * @virt_wire_setup:	indicates, whether virtual wire mode is selected
1971 *
1972 * Virtual wire mode is necessary to deliver legacy interrupts even when the
1973 * APIC is disabled.
1974 */
1975void disconnect_bsp_APIC(int virt_wire_setup)
1976{
1977	unsigned int value;
1978
1979#ifdef CONFIG_X86_32
1980	if (pic_mode) {
1981		/*
1982		 * Put the board back into PIC mode (has an effect only on
1983		 * certain older boards).  Note that APIC interrupts, including
1984		 * IPIs, won't work beyond this point!  The only exception are
1985		 * INIT IPIs.
1986		 */
1987		apic_printk(APIC_VERBOSE, "disabling APIC mode, "
1988				"entering PIC mode.\n");
1989		imcr_apic_to_pic();
1990		return;
1991	}
1992#endif
1993
1994	/* Go back to Virtual Wire compatibility mode */
1995
1996	/* For the spurious interrupt use vector F, and enable it */
1997	value = apic_read(APIC_SPIV);
1998	value &= ~APIC_VECTOR_MASK;
1999	value |= APIC_SPIV_APIC_ENABLED;
2000	value |= 0xf;
2001	apic_write(APIC_SPIV, value);
2002
2003	if (!virt_wire_setup) {
2004		/*
2005		 * For LVT0 make it edge triggered, active high,
2006		 * external and enabled
2007		 */
2008		value = apic_read(APIC_LVT0);
2009		value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2010			APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2011			APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2012		value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2013		value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2014		apic_write(APIC_LVT0, value);
2015	} else {
2016		/* Disable LVT0 */
2017		apic_write(APIC_LVT0, APIC_LVT_MASKED);
2018	}
2019
2020	/*
2021	 * For LVT1 make it edge triggered, active high,
2022	 * nmi and enabled
2023	 */
2024	value = apic_read(APIC_LVT1);
2025	value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2026			APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2027			APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2028	value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2029	value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2030	apic_write(APIC_LVT1, value);
2031}
2032
2033void __cpuinit generic_processor_info(int apicid, int version)
2034{
2035	int cpu, max = nr_cpu_ids;
2036	bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
2037				phys_cpu_present_map);
2038
2039	/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2040	 * If boot cpu has not been detected yet, then only allow upto
2041	 * nr_cpu_ids - 1 processors and keep one slot free for boot cpu
2042	 */
2043	if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 &&
2044	    apicid != boot_cpu_physical_apicid) {
2045		int thiscpu = max + disabled_cpus - 1;
2046
2047		pr_warning(
2048			"ACPI: NR_CPUS/possible_cpus limit of %i almost"
2049			" reached. Keeping one slot for boot cpu."
2050			"  Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2051
2052		disabled_cpus++;
2053		return;
2054	}
2055
2056	if (num_processors >= nr_cpu_ids) {
2057		int thiscpu = max + disabled_cpus;
2058
2059		pr_warning(
2060			"ACPI: NR_CPUS/possible_cpus limit of %i reached."
2061			"  Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2062
2063		disabled_cpus++;
2064		return;
2065	}
2066
2067	num_processors++;
2068	if (apicid == boot_cpu_physical_apicid) {
2069		/*
2070		 * x86_bios_cpu_apicid is required to have processors listed
2071		 * in same order as logical cpu numbers. Hence the first
2072		 * entry is BSP, and so on.
2073		 * boot_cpu_init() already hold bit 0 in cpu_present_mask
2074		 * for BSP.
2075		 */
2076		cpu = 0;
2077	} else
2078		cpu = cpumask_next_zero(-1, cpu_present_mask);
2079
2080	/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2081	 * Validate version
2082	 */
2083	if (version == 0x0) {
2084		pr_warning("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
2085			   cpu, apicid);
2086		version = 0x10;
2087	}
2088	apic_version[apicid] = version;
2089
2090	if (version != apic_version[boot_cpu_physical_apicid]) {
2091		pr_warning("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
2092			apic_version[boot_cpu_physical_apicid], cpu, version);
2093	}
2094
2095	physid_set(apicid, phys_cpu_present_map);
2096	if (apicid > max_physical_apicid)
2097		max_physical_apicid = apicid;
2098
2099#if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
2100	early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
2101	early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
2102#endif
2103#ifdef CONFIG_X86_32
2104	early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
2105		apic->x86_32_early_logical_apicid(cpu);
2106#endif
2107	set_cpu_possible(cpu, true);
2108	set_cpu_present(cpu, true);
 
 
2109}
2110
2111int hard_smp_processor_id(void)
2112{
2113	return read_apic_id();
2114}
2115
2116void default_init_apic_ldr(void)
2117{
2118	unsigned long val;
2119
2120	apic_write(APIC_DFR, APIC_DFR_VALUE);
2121	val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
2122	val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
2123	apic_write(APIC_LDR, val);
2124}
2125
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2126/*
2127 * Power management
2128 */
2129#ifdef CONFIG_PM
2130
2131static struct {
2132	/*
2133	 * 'active' is true if the local APIC was enabled by us and
2134	 * not the BIOS; this signifies that we are also responsible
2135	 * for disabling it before entering apm/acpi suspend
2136	 */
2137	int active;
2138	/* r/w apic fields */
2139	unsigned int apic_id;
2140	unsigned int apic_taskpri;
2141	unsigned int apic_ldr;
2142	unsigned int apic_dfr;
2143	unsigned int apic_spiv;
2144	unsigned int apic_lvtt;
2145	unsigned int apic_lvtpc;
2146	unsigned int apic_lvt0;
2147	unsigned int apic_lvt1;
2148	unsigned int apic_lvterr;
2149	unsigned int apic_tmict;
2150	unsigned int apic_tdcr;
2151	unsigned int apic_thmr;
 
2152} apic_pm_state;
2153
2154static int lapic_suspend(void)
2155{
2156	unsigned long flags;
2157	int maxlvt;
2158
2159	if (!apic_pm_state.active)
2160		return 0;
2161
2162	maxlvt = lapic_get_maxlvt();
2163
2164	apic_pm_state.apic_id = apic_read(APIC_ID);
2165	apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2166	apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2167	apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2168	apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2169	apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2170	if (maxlvt >= 4)
2171		apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2172	apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2173	apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2174	apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2175	apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2176	apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
2177#ifdef CONFIG_X86_THERMAL_VECTOR
2178	if (maxlvt >= 5)
2179		apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2180#endif
 
 
 
 
2181
2182	local_irq_save(flags);
2183	disable_local_APIC();
2184
2185	if (irq_remapping_enabled)
2186		irq_remapping_disable();
2187
2188	local_irq_restore(flags);
2189	return 0;
2190}
2191
2192static void lapic_resume(void)
2193{
2194	unsigned int l, h;
2195	unsigned long flags;
2196	int maxlvt;
2197
2198	if (!apic_pm_state.active)
2199		return;
2200
2201	local_irq_save(flags);
2202	if (irq_remapping_enabled) {
2203		/*
2204		 * IO-APIC and PIC have their own resume routines.
2205		 * We just mask them here to make sure the interrupt
2206		 * subsystem is completely quiet while we enable x2apic
2207		 * and interrupt-remapping.
2208		 */
2209		mask_ioapic_entries();
2210		legacy_pic->mask_all();
2211	}
2212
2213	if (x2apic_mode)
2214		enable_x2apic();
2215	else {
 
 
 
 
 
 
 
 
 
2216		/*
2217		 * Make sure the APICBASE points to the right address
2218		 *
2219		 * FIXME! This will be wrong if we ever support suspend on
2220		 * SMP! We'll need to do this as part of the CPU restore!
2221		 */
2222		if (boot_cpu_data.x86 >= 6) {
2223			rdmsr(MSR_IA32_APICBASE, l, h);
2224			l &= ~MSR_IA32_APICBASE_BASE;
2225			l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2226			wrmsr(MSR_IA32_APICBASE, l, h);
2227		}
2228	}
2229
2230	maxlvt = lapic_get_maxlvt();
2231	apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2232	apic_write(APIC_ID, apic_pm_state.apic_id);
2233	apic_write(APIC_DFR, apic_pm_state.apic_dfr);
2234	apic_write(APIC_LDR, apic_pm_state.apic_ldr);
2235	apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
2236	apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
2237	apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
2238	apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
2239#if defined(CONFIG_X86_MCE_P4THERMAL) || defined(CONFIG_X86_MCE_INTEL)
2240	if (maxlvt >= 5)
2241		apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
2242#endif
 
 
 
 
2243	if (maxlvt >= 4)
2244		apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
2245	apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
2246	apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
2247	apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
2248	apic_write(APIC_ESR, 0);
2249	apic_read(APIC_ESR);
2250	apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
2251	apic_write(APIC_ESR, 0);
2252	apic_read(APIC_ESR);
2253
2254	if (irq_remapping_enabled)
2255		irq_remapping_reenable(x2apic_mode);
2256
2257	local_irq_restore(flags);
2258}
2259
2260/*
2261 * This device has no shutdown method - fully functioning local APICs
2262 * are needed on every CPU up until machine_halt/restart/poweroff.
2263 */
2264
2265static struct syscore_ops lapic_syscore_ops = {
2266	.resume		= lapic_resume,
2267	.suspend	= lapic_suspend,
2268};
2269
2270static void __cpuinit apic_pm_activate(void)
2271{
2272	apic_pm_state.active = 1;
2273}
2274
2275static int __init init_lapic_sysfs(void)
2276{
2277	/* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2278	if (cpu_has_apic)
2279		register_syscore_ops(&lapic_syscore_ops);
2280
2281	return 0;
2282}
2283
2284/* local apic needs to resume before other devices access its registers. */
2285core_initcall(init_lapic_sysfs);
2286
2287#else	/* CONFIG_PM */
2288
2289static void apic_pm_activate(void) { }
2290
2291#endif	/* CONFIG_PM */
2292
2293#ifdef CONFIG_X86_64
2294
2295static int __cpuinit apic_cluster_num(void)
2296{
2297	int i, clusters, zeros;
2298	unsigned id;
2299	u16 *bios_cpu_apicid;
2300	DECLARE_BITMAP(clustermap, NUM_APIC_CLUSTERS);
2301
2302	bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid);
2303	bitmap_zero(clustermap, NUM_APIC_CLUSTERS);
2304
2305	for (i = 0; i < nr_cpu_ids; i++) {
2306		/* are we being called early in kernel startup? */
2307		if (bios_cpu_apicid) {
2308			id = bios_cpu_apicid[i];
2309		} else if (i < nr_cpu_ids) {
2310			if (cpu_present(i))
2311				id = per_cpu(x86_bios_cpu_apicid, i);
2312			else
2313				continue;
2314		} else
2315			break;
2316
2317		if (id != BAD_APICID)
2318			__set_bit(APIC_CLUSTERID(id), clustermap);
2319	}
2320
2321	/* Problem:  Partially populated chassis may not have CPUs in some of
2322	 * the APIC clusters they have been allocated.  Only present CPUs have
2323	 * x86_bios_cpu_apicid entries, thus causing zeroes in the bitmap.
2324	 * Since clusters are allocated sequentially, count zeros only if
2325	 * they are bounded by ones.
2326	 */
2327	clusters = 0;
2328	zeros = 0;
2329	for (i = 0; i < NUM_APIC_CLUSTERS; i++) {
2330		if (test_bit(i, clustermap)) {
2331			clusters += 1 + zeros;
2332			zeros = 0;
2333		} else
2334			++zeros;
2335	}
2336
2337	return clusters;
2338}
2339
2340static int __cpuinitdata multi_checked;
2341static int __cpuinitdata multi;
2342
2343static int __cpuinit set_multi(const struct dmi_system_id *d)
2344{
2345	if (multi)
2346		return 0;
2347	pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2348	multi = 1;
2349	return 0;
2350}
2351
2352static const __cpuinitconst struct dmi_system_id multi_dmi_table[] = {
2353	{
2354		.callback = set_multi,
2355		.ident = "IBM System Summit2",
2356		.matches = {
2357			DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2358			DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2359		},
2360	},
2361	{}
2362};
2363
2364static void __cpuinit dmi_check_multi(void)
2365{
2366	if (multi_checked)
2367		return;
2368
2369	dmi_check_system(multi_dmi_table);
2370	multi_checked = 1;
2371}
2372
2373/*
2374 * apic_is_clustered_box() -- Check if we can expect good TSC
2375 *
2376 * Thus far, the major user of this is IBM's Summit2 series:
2377 * Clustered boxes may have unsynced TSC problems if they are
2378 * multi-chassis.
2379 * Use DMI to check them
2380 */
2381__cpuinit int apic_is_clustered_box(void)
2382{
2383	dmi_check_multi();
2384	if (multi)
2385		return 1;
2386
2387	if (!is_vsmp_box())
2388		return 0;
2389
2390	/*
2391	 * ScaleMP vSMPowered boxes have one cluster per board and TSCs are
2392	 * not guaranteed to be synced between boards
2393	 */
2394	if (apic_cluster_num() > 1)
2395		return 1;
2396
2397	return 0;
2398}
2399#endif
2400
2401/*
2402 * APIC command line parameters
2403 */
2404static int __init setup_disableapic(char *arg)
2405{
2406	disable_apic = 1;
2407	setup_clear_cpu_cap(X86_FEATURE_APIC);
2408	return 0;
2409}
2410early_param("disableapic", setup_disableapic);
2411
2412/* same as disableapic, for compatibility */
2413static int __init setup_nolapic(char *arg)
2414{
2415	return setup_disableapic(arg);
2416}
2417early_param("nolapic", setup_nolapic);
2418
2419static int __init parse_lapic_timer_c2_ok(char *arg)
2420{
2421	local_apic_timer_c2_ok = 1;
2422	return 0;
2423}
2424early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2425
2426static int __init parse_disable_apic_timer(char *arg)
2427{
2428	disable_apic_timer = 1;
2429	return 0;
2430}
2431early_param("noapictimer", parse_disable_apic_timer);
2432
2433static int __init parse_nolapic_timer(char *arg)
2434{
2435	disable_apic_timer = 1;
2436	return 0;
2437}
2438early_param("nolapic_timer", parse_nolapic_timer);
2439
2440static int __init apic_set_verbosity(char *arg)
2441{
2442	if (!arg)  {
2443#ifdef CONFIG_X86_64
2444		skip_ioapic_setup = 0;
2445		return 0;
2446#endif
2447		return -EINVAL;
2448	}
2449
2450	if (strcmp("debug", arg) == 0)
2451		apic_verbosity = APIC_DEBUG;
2452	else if (strcmp("verbose", arg) == 0)
2453		apic_verbosity = APIC_VERBOSE;
2454	else {
2455		pr_warning("APIC Verbosity level %s not recognised"
2456			" use apic=verbose or apic=debug\n", arg);
2457		return -EINVAL;
2458	}
2459
2460	return 0;
2461}
2462early_param("apic", apic_set_verbosity);
2463
2464static int __init lapic_insert_resource(void)
2465{
2466	if (!apic_phys)
2467		return -1;
2468
2469	/* Put local APIC into the resource map. */
2470	lapic_resource.start = apic_phys;
2471	lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2472	insert_resource(&iomem_resource, &lapic_resource);
2473
2474	return 0;
2475}
2476
2477/*
2478 * need call insert after e820_reserve_resources()
2479 * that is using request_resource
2480 */
2481late_initcall(lapic_insert_resource);