1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 *  linux/arch/x86_64/entry.S
   4 *
   5 *  Copyright (C) 1991, 1992  Linus Torvalds
   6 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
   7 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
   8 *
   9 * entry.S contains the system-call and fault low-level handling routines.
  10 *
  11 * Some of this is documented in Documentation/x86/entry_64.rst
  12 *
  13 * A note on terminology:
  14 * - iret frame:	Architecture defined interrupt frame from SS to RIP
  15 *			at the top of the kernel process stack.
  16 *
  17 * Some macro usage:
  18 * - ENTRY/END:		Define functions in the symbol table.
  19 * - TRACE_IRQ_*:	Trace hardirq state for lock debugging.
  20 * - idtentry:		Define exception entry points.
  21 */
  22#include <linux/linkage.h>
  23#include <asm/segment.h>
  24#include <asm/cache.h>
  25#include <asm/errno.h>
  26#include <asm/asm-offsets.h>
  27#include <asm/msr.h>
  28#include <asm/unistd.h>
  29#include <asm/thread_info.h>
  30#include <asm/hw_irq.h>
  31#include <asm/page_types.h>
  32#include <asm/irqflags.h>
  33#include <asm/paravirt.h>
  34#include <asm/percpu.h>
  35#include <asm/asm.h>
  36#include <asm/smap.h>
  37#include <asm/pgtable_types.h>
  38#include <asm/export.h>
  39#include <asm/frame.h>
 
  40#include <asm/nospec-branch.h>
 
  41#include <linux/err.h>
  42
  43#include "calling.h"
  44
  45.code64
  46.section .entry.text, "ax"
  47
  48#ifdef CONFIG_PARAVIRT
  49ENTRY(native_usergs_sysret64)
  50	UNWIND_HINT_EMPTY
  51	swapgs
  52	sysretq
  53END(native_usergs_sysret64)
  54#endif /* CONFIG_PARAVIRT */
  55
  56.macro TRACE_IRQS_FLAGS flags:req
  57#ifdef CONFIG_TRACE_IRQFLAGS
  58	btl	$9, \flags		/* interrupts off? */
  59	jnc	1f
  60	TRACE_IRQS_ON
  611:
  62#endif
  63.endm
  64
  65.macro TRACE_IRQS_IRETQ
  66	TRACE_IRQS_FLAGS EFLAGS(%rsp)
  67.endm
  68
  69/*
  70 * When dynamic function tracer is enabled it will add a breakpoint
  71 * to all locations that it is about to modify, sync CPUs, update
  72 * all the code, sync CPUs, then remove the breakpoints. In this time
  73 * if lockdep is enabled, it might jump back into the debug handler
  74 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF).
  75 *
  76 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to
  77 * make sure the stack pointer does not get reset back to the top
  78 * of the debug stack, and instead just reuses the current stack.
  79 */
  80#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)
  81
  82.macro TRACE_IRQS_OFF_DEBUG
  83	call	debug_stack_set_zero
  84	TRACE_IRQS_OFF
  85	call	debug_stack_reset
  86.endm
  87
  88.macro TRACE_IRQS_ON_DEBUG
  89	call	debug_stack_set_zero
  90	TRACE_IRQS_ON
  91	call	debug_stack_reset
  92.endm
  93
  94.macro TRACE_IRQS_IRETQ_DEBUG
  95	btl	$9, EFLAGS(%rsp)		/* interrupts off? */
  96	jnc	1f
  97	TRACE_IRQS_ON_DEBUG
  981:
  99.endm
 100
 101#else
 102# define TRACE_IRQS_OFF_DEBUG			TRACE_IRQS_OFF
 103# define TRACE_IRQS_ON_DEBUG			TRACE_IRQS_ON
 104# define TRACE_IRQS_IRETQ_DEBUG			TRACE_IRQS_IRETQ
 105#endif
 106
 107/*
 108 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
 109 *
 110 * This is the only entry point used for 64-bit system calls.  The
 111 * hardware interface is reasonably well designed and the register to
 112 * argument mapping Linux uses fits well with the registers that are
 113 * available when SYSCALL is used.
 114 *
 115 * SYSCALL instructions can be found inlined in libc implementations as
 116 * well as some other programs and libraries.  There are also a handful
 117 * of SYSCALL instructions in the vDSO used, for example, as a
 118 * clock_gettimeofday fallback.
 119 *
 120 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
 121 * then loads new ss, cs, and rip from previously programmed MSRs.
 122 * rflags gets masked by a value from another MSR (so CLD and CLAC
 123 * are not needed). SYSCALL does not save anything on the stack
 124 * and does not change rsp.
 125 *
 126 * Registers on entry:
 127 * rax  system call number
 128 * rcx  return address
 129 * r11  saved rflags (note: r11 is callee-clobbered register in C ABI)
 130 * rdi  arg0
 131 * rsi  arg1
 132 * rdx  arg2
 133 * r10  arg3 (needs to be moved to rcx to conform to C ABI)
 134 * r8   arg4
 135 * r9   arg5
 136 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
 137 *
 138 * Only called from user space.
 139 *
 140 * When user can change pt_regs->foo always force IRET. That is because
 141 * it deals with uncanonical addresses better. SYSRET has trouble
 142 * with them due to bugs in both AMD and Intel CPUs.
 143 */
 144
 145ENTRY(entry_SYSCALL_64)
 146	UNWIND_HINT_EMPTY
 147	/*
 148	 * Interrupts are off on entry.
 149	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
 150	 * it is too small to ever cause noticeable irq latency.
 151	 */
 152
 153	swapgs
 154	/* tss.sp2 is scratch space. */
 155	movq	%rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
 156	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
 157	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 158
 
 
 159	/* Construct struct pt_regs on stack */
 160	pushq	$__USER_DS				/* pt_regs->ss */
 161	pushq	PER_CPU_VAR(cpu_tss_rw + TSS_sp2)	/* pt_regs->sp */
 162	pushq	%r11					/* pt_regs->flags */
 163	pushq	$__USER_CS				/* pt_regs->cs */
 164	pushq	%rcx					/* pt_regs->ip */
 165GLOBAL(entry_SYSCALL_64_after_hwframe)
 166	pushq	%rax					/* pt_regs->orig_ax */
 167
 168	PUSH_AND_CLEAR_REGS rax=$-ENOSYS
 169
 170	TRACE_IRQS_OFF
 171
 172	/* IRQs are off. */
 173	movq	%rax, %rdi
 174	movq	%rsp, %rsi
 
 175	call	do_syscall_64		/* returns with IRQs disabled */
 176
 177	TRACE_IRQS_IRETQ		/* we're about to change IF */
 178
 179	/*
 180	 * Try to use SYSRET instead of IRET if we're returning to
 181	 * a completely clean 64-bit userspace context.  If we're not,
 182	 * go to the slow exit path.
 
 183	 */
 
 
 
 
 184	movq	RCX(%rsp), %rcx
 185	movq	RIP(%rsp), %r11
 186
 187	cmpq	%rcx, %r11	/* SYSRET requires RCX == RIP */
 188	jne	swapgs_restore_regs_and_return_to_usermode
 189
 190	/*
 191	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
 192	 * in kernel space.  This essentially lets the user take over
 193	 * the kernel, since userspace controls RSP.
 194	 *
 195	 * If width of "canonical tail" ever becomes variable, this will need
 196	 * to be updated to remain correct on both old and new CPUs.
 197	 *
 198	 * Change top bits to match most significant bit (47th or 56th bit
 199	 * depending on paging mode) in the address.
 200	 */
 201#ifdef CONFIG_X86_5LEVEL
 202	ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \
 203		"shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57
 204#else
 205	shl	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
 206	sar	$(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
 207#endif
 208
 209	/* If this changed %rcx, it was not canonical */
 210	cmpq	%rcx, %r11
 211	jne	swapgs_restore_regs_and_return_to_usermode
 212
 213	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */
 214	jne	swapgs_restore_regs_and_return_to_usermode
 215
 216	movq	R11(%rsp), %r11
 217	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */
 218	jne	swapgs_restore_regs_and_return_to_usermode
 219
 220	/*
 221	 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
 222	 * restore RF properly. If the slowpath sets it for whatever reason, we
 223	 * need to restore it correctly.
 224	 *
 225	 * SYSRET can restore TF, but unlike IRET, restoring TF results in a
 226	 * trap from userspace immediately after SYSRET.  This would cause an
 227	 * infinite loop whenever #DB happens with register state that satisfies
 228	 * the opportunistic SYSRET conditions.  For example, single-stepping
 229	 * this user code:
 230	 *
 231	 *           movq	$stuck_here, %rcx
 232	 *           pushfq
 233	 *           popq %r11
 234	 *   stuck_here:
 235	 *
 236	 * would never get past 'stuck_here'.
 237	 */
 238	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
 239	jnz	swapgs_restore_regs_and_return_to_usermode
 240
 241	/* nothing to check for RSP */
 242
 243	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */
 244	jne	swapgs_restore_regs_and_return_to_usermode
 245
 246	/*
 247	 * We win! This label is here just for ease of understanding
 248	 * perf profiles. Nothing jumps here.
 249	 */
 250syscall_return_via_sysret:
 251	/* rcx and r11 are already restored (see code above) */
 252	UNWIND_HINT_EMPTY
 253	POP_REGS pop_rdi=0 skip_r11rcx=1
 254
 255	/*
 256	 * Now all regs are restored except RSP and RDI.
 257	 * Save old stack pointer and switch to trampoline stack.
 258	 */
 259	movq	%rsp, %rdi
 260	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
 
 261
 262	pushq	RSP-RDI(%rdi)	/* RSP */
 263	pushq	(%rdi)		/* RDI */
 264
 265	/*
 266	 * We are on the trampoline stack.  All regs except RDI are live.
 267	 * We can do future final exit work right here.
 268	 */
 269	STACKLEAK_ERASE_NOCLOBBER
 270
 271	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
 272
 273	popq	%rdi
 274	popq	%rsp
 275	USERGS_SYSRET64
 276END(entry_SYSCALL_64)
 
 277
 278/*
 279 * %rdi: prev task
 280 * %rsi: next task
 281 */
 282ENTRY(__switch_to_asm)
 283	UNWIND_HINT_FUNC
 284	/*
 285	 * Save callee-saved registers
 286	 * This must match the order in inactive_task_frame
 287	 */
 288	pushq	%rbp
 289	pushq	%rbx
 290	pushq	%r12
 291	pushq	%r13
 292	pushq	%r14
 293	pushq	%r15
 294
 295	/* switch stack */
 296	movq	%rsp, TASK_threadsp(%rdi)
 297	movq	TASK_threadsp(%rsi), %rsp
 298
 299#ifdef CONFIG_STACKPROTECTOR
 300	movq	TASK_stack_canary(%rsi), %rbx
 301	movq	%rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset
 302#endif
 303
 304#ifdef CONFIG_RETPOLINE
 305	/*
 306	 * When switching from a shallower to a deeper call stack
 307	 * the RSB may either underflow or use entries populated
 308	 * with userspace addresses. On CPUs where those concerns
 309	 * exist, overwrite the RSB with entries which capture
 310	 * speculative execution to prevent attack.
 311	 */
 312	FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
 313#endif
 314
 315	/* restore callee-saved registers */
 316	popq	%r15
 317	popq	%r14
 318	popq	%r13
 319	popq	%r12
 320	popq	%rbx
 321	popq	%rbp
 322
 323	jmp	__switch_to
 324END(__switch_to_asm)
 
 325
 326/*
 327 * A newly forked process directly context switches into this address.
 328 *
 329 * rax: prev task we switched from
 330 * rbx: kernel thread func (NULL for user thread)
 331 * r12: kernel thread arg
 332 */
 333ENTRY(ret_from_fork)
 
 334	UNWIND_HINT_EMPTY
 335	movq	%rax, %rdi
 336	call	schedule_tail			/* rdi: 'prev' task parameter */
 337
 338	testq	%rbx, %rbx			/* from kernel_thread? */
 339	jnz	1f				/* kernel threads are uncommon */
 340
 3412:
 342	UNWIND_HINT_REGS
 343	movq	%rsp, %rdi
 344	call	syscall_return_slowpath	/* returns with IRQs disabled */
 345	TRACE_IRQS_ON			/* user mode is traced as IRQS on */
 346	jmp	swapgs_restore_regs_and_return_to_usermode
 347
 3481:
 349	/* kernel thread */
 350	UNWIND_HINT_EMPTY
 351	movq	%r12, %rdi
 352	CALL_NOSPEC %rbx
 353	/*
 354	 * A kernel thread is allowed to return here after successfully
 355	 * calling do_execve().  Exit to userspace to complete the execve()
 356	 * syscall.
 357	 */
 358	movq	$0, RAX(%rsp)
 359	jmp	2b
 360END(ret_from_fork)
 361
 362/*
 363 * Build the entry stubs with some assembler magic.
 364 * We pack 1 stub into every 8-byte block.
 365 */
 366	.align 8
 367ENTRY(irq_entries_start)
 368    vector=FIRST_EXTERNAL_VECTOR
 369    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
 370	UNWIND_HINT_IRET_REGS
 371	pushq	$(~vector+0x80)			/* Note: always in signed byte range */
 372	jmp	common_interrupt
 373	.align	8
 374	vector=vector+1
 375    .endr
 376END(irq_entries_start)
 377
 378	.align 8
 379ENTRY(spurious_entries_start)
 380    vector=FIRST_SYSTEM_VECTOR
 381    .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR)
 382	UNWIND_HINT_IRET_REGS
 383	pushq	$(~vector+0x80)			/* Note: always in signed byte range */
 384	jmp	common_spurious
 385	.align	8
 386	vector=vector+1
 387    .endr
 388END(spurious_entries_start)
 389
 390.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
 391#ifdef CONFIG_DEBUG_ENTRY
 392	pushq %rax
 393	SAVE_FLAGS(CLBR_RAX)
 394	testl $X86_EFLAGS_IF, %eax
 395	jz .Lokay_\@
 396	ud2
 397.Lokay_\@:
 398	popq %rax
 399#endif
 400.endm
 401
 402/*
 403 * Enters the IRQ stack if we're not already using it.  NMI-safe.  Clobbers
 404 * flags and puts old RSP into old_rsp, and leaves all other GPRs alone.
 405 * Requires kernel GSBASE.
 406 *
 407 * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
 408 */
 409.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0
 410	DEBUG_ENTRY_ASSERT_IRQS_OFF
 411
 412	.if \save_ret
 413	/*
 414	 * If save_ret is set, the original stack contains one additional
 415	 * entry -- the return address. Therefore, move the address one
 416	 * entry below %rsp to \old_rsp.
 417	 */
 418	leaq	8(%rsp), \old_rsp
 419	.else
 420	movq	%rsp, \old_rsp
 421	.endif
 422
 423	.if \regs
 424	UNWIND_HINT_REGS base=\old_rsp
 425	.endif
 426
 427	incl	PER_CPU_VAR(irq_count)
 428	jnz	.Lirq_stack_push_old_rsp_\@
 429
 430	/*
 431	 * Right now, if we just incremented irq_count to zero, we've
 432	 * claimed the IRQ stack but we haven't switched to it yet.
 433	 *
 434	 * If anything is added that can interrupt us here without using IST,
 435	 * it must be *extremely* careful to limit its stack usage.  This
 436	 * could include kprobes and a hypothetical future IST-less #DB
 437	 * handler.
 438	 *
 439	 * The OOPS unwinder relies on the word at the top of the IRQ
 440	 * stack linking back to the previous RSP for the entire time we're
 441	 * on the IRQ stack.  For this to work reliably, we need to write
 442	 * it before we actually move ourselves to the IRQ stack.
 443	 */
 444
 445	movq	\old_rsp, PER_CPU_VAR(irq_stack_backing_store + IRQ_STACK_SIZE - 8)
 446	movq	PER_CPU_VAR(hardirq_stack_ptr), %rsp
 447
 448#ifdef CONFIG_DEBUG_ENTRY
 449	/*
 450	 * If the first movq above becomes wrong due to IRQ stack layout
 451	 * changes, the only way we'll notice is if we try to unwind right
 452	 * here.  Assert that we set up the stack right to catch this type
 453	 * of bug quickly.
 454	 */
 455	cmpq	-8(%rsp), \old_rsp
 456	je	.Lirq_stack_okay\@
 457	ud2
 458	.Lirq_stack_okay\@:
 459#endif
 460
 461.Lirq_stack_push_old_rsp_\@:
 462	pushq	\old_rsp
 
 
 
 
 
 
 
 
 
 
 
 
 463
 464	.if \regs
 465	UNWIND_HINT_REGS indirect=1
 466	.endif
 467
 468	.if \save_ret
 469	/*
 470	 * Push the return address to the stack. This return address can
 471	 * be found at the "real" original RSP, which was offset by 8 at
 472	 * the beginning of this macro.
 473	 */
 474	pushq	-8(\old_rsp)
 
 
 
 
 
 475	.endif
 
 
 
 
 
 476.endm
 477
 478/*
 479 * Undoes ENTER_IRQ_STACK.
 
 
 
 
 
 
 480 */
 481.macro LEAVE_IRQ_STACK regs=1
 482	DEBUG_ENTRY_ASSERT_IRQS_OFF
 483	/* We need to be off the IRQ stack before decrementing irq_count. */
 484	popq	%rsp
 485
 486	.if \regs
 487	UNWIND_HINT_REGS
 488	.endif
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 489
 490	/*
 491	 * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming
 492	 * the irq stack but we're not on it.
 493	 */
 
 
 494
 495	decl	PER_CPU_VAR(irq_count)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 496.endm
 497
 498/*
 499 * Interrupt entry helper function.
 
 
 
 
 
 
 
 
 
 
 
 
 500 *
 501 * Entry runs with interrupts off. Stack layout at entry:
 502 * +----------------------------------------------------+
 503 * | regs->ss						|
 504 * | regs->rsp						|
 505 * | regs->eflags					|
 506 * | regs->cs						|
 507 * | regs->ip						|
 508 * +----------------------------------------------------+
 509 * | regs->orig_ax = ~(interrupt number)		|
 510 * +----------------------------------------------------+
 511 * | return address					|
 512 * +----------------------------------------------------+
 513 */
 514ENTRY(interrupt_entry)
 515	UNWIND_HINT_FUNC
 
 516	ASM_CLAC
 517	cld
 518
 519	testb	$3, CS-ORIG_RAX+8(%rsp)
 520	jz	1f
 521	SWAPGS
 522	FENCE_SWAPGS_USER_ENTRY
 523	/*
 524	 * Switch to the thread stack. The IRET frame and orig_ax are
 525	 * on the stack, as well as the return address. RDI..R12 are
 526	 * not (yet) on the stack and space has not (yet) been
 527	 * allocated for them.
 528	 */
 529	pushq	%rdi
 
 530
 531	/* Need to switch before accessing the thread stack. */
 532	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi
 533	movq	%rsp, %rdi
 534	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
 535
 536	 /*
 537	  * We have RDI, return address, and orig_ax on the stack on
 538	  * top of the IRET frame. That means offset=24
 539	  */
 540	UNWIND_HINT_IRET_REGS base=%rdi offset=24
 541
 542	pushq	7*8(%rdi)		/* regs->ss */
 543	pushq	6*8(%rdi)		/* regs->rsp */
 544	pushq	5*8(%rdi)		/* regs->eflags */
 545	pushq	4*8(%rdi)		/* regs->cs */
 546	pushq	3*8(%rdi)		/* regs->ip */
 547	pushq	2*8(%rdi)		/* regs->orig_ax */
 548	pushq	8(%rdi)			/* return address */
 549	UNWIND_HINT_FUNC
 550
 551	movq	(%rdi), %rdi
 552	jmp	2f
 5531:
 554	FENCE_SWAPGS_KERNEL_ENTRY
 5552:
 556	PUSH_AND_CLEAR_REGS save_ret=1
 557	ENCODE_FRAME_POINTER 8
 
 
 558
 559	testb	$3, CS+8(%rsp)
 560	jz	1f
 561
 562	/*
 563	 * IRQ from user mode.
 564	 *
 565	 * We need to tell lockdep that IRQs are off.  We can't do this until
 566	 * we fix gsbase, and we should do it before enter_from_user_mode
 567	 * (which can take locks).  Since TRACE_IRQS_OFF is idempotent,
 568	 * the simplest way to handle it is to just call it twice if
 569	 * we enter from user mode.  There's no reason to optimize this since
 570	 * TRACE_IRQS_OFF is a no-op if lockdep is off.
 571	 */
 572	TRACE_IRQS_OFF
 573
 574	CALL_enter_from_user_mode
 575
 5761:
 577	ENTER_IRQ_STACK old_rsp=%rdi save_ret=1
 578	/* We entered an interrupt context - irqs are off: */
 579	TRACE_IRQS_OFF
 580
 581	ret
 582END(interrupt_entry)
 583_ASM_NOKPROBE(interrupt_entry)
 
 
 
 584
 
 
 
 585
 586/* Interrupt entry/exit. */
 
 
 
 587
 588/*
 589 * The interrupt stubs push (~vector+0x80) onto the stack and
 590 * then jump to common_spurious/interrupt.
 
 591 */
 592common_spurious:
 593	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
 594	call	interrupt_entry
 595	UNWIND_HINT_REGS indirect=1
 596	call	smp_spurious_interrupt		/* rdi points to pt_regs */
 597	jmp	ret_from_intr
 598END(common_spurious)
 599_ASM_NOKPROBE(common_spurious)
 600
 601/* common_interrupt is a hotpath. Align it */
 602	.p2align CONFIG_X86_L1_CACHE_SHIFT
 603common_interrupt:
 604	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
 605	call	interrupt_entry
 606	UNWIND_HINT_REGS indirect=1
 607	call	do_IRQ	/* rdi points to pt_regs */
 608	/* 0(%rsp): old RSP */
 609ret_from_intr:
 610	DISABLE_INTERRUPTS(CLBR_ANY)
 611	TRACE_IRQS_OFF
 612
 613	LEAVE_IRQ_STACK
 
 
 
 614
 615	testb	$3, CS(%rsp)
 616	jz	retint_kernel
 
 
 
 617
 618	/* Interrupt came from user space */
 619GLOBAL(retint_user)
 620	mov	%rsp,%rdi
 621	call	prepare_exit_to_usermode
 622	TRACE_IRQS_IRETQ
 
 
 
 
 
 
 
 
 
 623
 624GLOBAL(swapgs_restore_regs_and_return_to_usermode)
 
 625#ifdef CONFIG_DEBUG_ENTRY
 626	/* Assert that pt_regs indicates user mode. */
 627	testb	$3, CS(%rsp)
 628	jnz	1f
 629	ud2
 6301:
 631#endif
 632	POP_REGS pop_rdi=0
 633
 634	/*
 635	 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
 636	 * Save old stack pointer and switch to trampoline stack.
 637	 */
 638	movq	%rsp, %rdi
 639	movq	PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
 
 640
 641	/* Copy the IRET frame to the trampoline stack. */
 642	pushq	6*8(%rdi)	/* SS */
 643	pushq	5*8(%rdi)	/* RSP */
 644	pushq	4*8(%rdi)	/* EFLAGS */
 645	pushq	3*8(%rdi)	/* CS */
 646	pushq	2*8(%rdi)	/* RIP */
 647
 648	/* Push user RDI on the trampoline stack. */
 649	pushq	(%rdi)
 650
 651	/*
 652	 * We are on the trampoline stack.  All regs except RDI are live.
 653	 * We can do future final exit work right here.
 654	 */
 655	STACKLEAK_ERASE_NOCLOBBER
 656
 657	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
 658
 659	/* Restore RDI. */
 660	popq	%rdi
 661	SWAPGS
 662	INTERRUPT_RETURN
 663
 664
 665/* Returning to kernel space */
 666retint_kernel:
 667#ifdef CONFIG_PREEMPTION
 668	/* Interrupts are off */
 669	/* Check if we need preemption */
 670	btl	$9, EFLAGS(%rsp)		/* were interrupts off? */
 671	jnc	1f
 672	cmpl	$0, PER_CPU_VAR(__preempt_count)
 673	jnz	1f
 674	call	preempt_schedule_irq
 6751:
 676#endif
 677	/*
 678	 * The iretq could re-enable interrupts:
 679	 */
 680	TRACE_IRQS_IRETQ
 681
 682GLOBAL(restore_regs_and_return_to_kernel)
 683#ifdef CONFIG_DEBUG_ENTRY
 684	/* Assert that pt_regs indicates kernel mode. */
 685	testb	$3, CS(%rsp)
 686	jz	1f
 687	ud2
 6881:
 689#endif
 690	POP_REGS
 691	addq	$8, %rsp	/* skip regs->orig_ax */
 692	/*
 693	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
 694	 * when returning from IPI handler.
 695	 */
 696	INTERRUPT_RETURN
 697
 698ENTRY(native_iret)
 699	UNWIND_HINT_IRET_REGS
 700	/*
 701	 * Are we returning to a stack segment from the LDT?  Note: in
 702	 * 64-bit mode SS:RSP on the exception stack is always valid.
 703	 */
 704#ifdef CONFIG_X86_ESPFIX64
 705	testb	$4, (SS-RIP)(%rsp)
 706	jnz	native_irq_return_ldt
 707#endif
 708
 709.global native_irq_return_iret
 710native_irq_return_iret:
 711	/*
 712	 * This may fault.  Non-paranoid faults on return to userspace are
 713	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
 714	 * Double-faults due to espfix64 are handled in do_double_fault.
 715	 * Other faults here are fatal.
 716	 */
 717	iretq
 718
 719#ifdef CONFIG_X86_ESPFIX64
 720native_irq_return_ldt:
 721	/*
 722	 * We are running with user GSBASE.  All GPRs contain their user
 723	 * values.  We have a percpu ESPFIX stack that is eight slots
 724	 * long (see ESPFIX_STACK_SIZE).  espfix_waddr points to the bottom
 725	 * of the ESPFIX stack.
 726	 *
 727	 * We clobber RAX and RDI in this code.  We stash RDI on the
 728	 * normal stack and RAX on the ESPFIX stack.
 729	 *
 730	 * The ESPFIX stack layout we set up looks like this:
 731	 *
 732	 * --- top of ESPFIX stack ---
 733	 * SS
 734	 * RSP
 735	 * RFLAGS
 736	 * CS
 737	 * RIP  <-- RSP points here when we're done
 738	 * RAX  <-- espfix_waddr points here
 739	 * --- bottom of ESPFIX stack ---
 740	 */
 741
 742	pushq	%rdi				/* Stash user RDI */
 743	SWAPGS					/* to kernel GS */
 744	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi	/* to kernel CR3 */
 745
 746	movq	PER_CPU_VAR(espfix_waddr), %rdi
 747	movq	%rax, (0*8)(%rdi)		/* user RAX */
 748	movq	(1*8)(%rsp), %rax		/* user RIP */
 749	movq	%rax, (1*8)(%rdi)
 750	movq	(2*8)(%rsp), %rax		/* user CS */
 751	movq	%rax, (2*8)(%rdi)
 752	movq	(3*8)(%rsp), %rax		/* user RFLAGS */
 753	movq	%rax, (3*8)(%rdi)
 754	movq	(5*8)(%rsp), %rax		/* user SS */
 755	movq	%rax, (5*8)(%rdi)
 756	movq	(4*8)(%rsp), %rax		/* user RSP */
 757	movq	%rax, (4*8)(%rdi)
 758	/* Now RAX == RSP. */
 759
 760	andl	$0xffff0000, %eax		/* RAX = (RSP & 0xffff0000) */
 761
 762	/*
 763	 * espfix_stack[31:16] == 0.  The page tables are set up such that
 764	 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
 765	 * espfix_waddr for any X.  That is, there are 65536 RO aliases of
 766	 * the same page.  Set up RSP so that RSP[31:16] contains the
 767	 * respective 16 bits of the /userspace/ RSP and RSP nonetheless
 768	 * still points to an RO alias of the ESPFIX stack.
 769	 */
 770	orq	PER_CPU_VAR(espfix_stack), %rax
 771
 772	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
 773	SWAPGS					/* to user GS */
 774	popq	%rdi				/* Restore user RDI */
 775
 776	movq	%rax, %rsp
 777	UNWIND_HINT_IRET_REGS offset=8
 778
 779	/*
 780	 * At this point, we cannot write to the stack any more, but we can
 781	 * still read.
 782	 */
 783	popq	%rax				/* Restore user RAX */
 784
 785	/*
 786	 * RSP now points to an ordinary IRET frame, except that the page
 787	 * is read-only and RSP[31:16] are preloaded with the userspace
 788	 * values.  We can now IRET back to userspace.
 789	 */
 790	jmp	native_irq_return_iret
 791#endif
 792END(common_interrupt)
 793_ASM_NOKPROBE(common_interrupt)
 794
 795/*
 796 * APIC interrupts.
 797 */
 798.macro apicinterrupt3 num sym do_sym
 799ENTRY(\sym)
 800	UNWIND_HINT_IRET_REGS
 801	pushq	$~(\num)
 802.Lcommon_\sym:
 803	call	interrupt_entry
 804	UNWIND_HINT_REGS indirect=1
 805	call	\do_sym	/* rdi points to pt_regs */
 806	jmp	ret_from_intr
 807END(\sym)
 808_ASM_NOKPROBE(\sym)
 809.endm
 810
 811/* Make sure APIC interrupt handlers end up in the irqentry section: */
 812#define PUSH_SECTION_IRQENTRY	.pushsection .irqentry.text, "ax"
 813#define POP_SECTION_IRQENTRY	.popsection
 814
 815.macro apicinterrupt num sym do_sym
 816PUSH_SECTION_IRQENTRY
 817apicinterrupt3 \num \sym \do_sym
 818POP_SECTION_IRQENTRY
 819.endm
 820
 821#ifdef CONFIG_SMP
 822apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR		irq_move_cleanup_interrupt	smp_irq_move_cleanup_interrupt
 823apicinterrupt3 REBOOT_VECTOR			reboot_interrupt		smp_reboot_interrupt
 824#endif
 825
 826#ifdef CONFIG_X86_UV
 827apicinterrupt3 UV_BAU_MESSAGE			uv_bau_message_intr1		uv_bau_message_interrupt
 828#endif
 829
 830apicinterrupt LOCAL_TIMER_VECTOR		apic_timer_interrupt		smp_apic_timer_interrupt
 831apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
 832
 833#ifdef CONFIG_HAVE_KVM
 834apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
 835apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
 836apicinterrupt3 POSTED_INTR_NESTED_VECTOR	kvm_posted_intr_nested_ipi	smp_kvm_posted_intr_nested_ipi
 837#endif
 838
 839#ifdef CONFIG_X86_MCE_THRESHOLD
 840apicinterrupt THRESHOLD_APIC_VECTOR		threshold_interrupt		smp_threshold_interrupt
 841#endif
 842
 843#ifdef CONFIG_X86_MCE_AMD
 844apicinterrupt DEFERRED_ERROR_VECTOR		deferred_error_interrupt	smp_deferred_error_interrupt
 845#endif
 846
 847#ifdef CONFIG_X86_THERMAL_VECTOR
 848apicinterrupt THERMAL_APIC_VECTOR		thermal_interrupt		smp_thermal_interrupt
 849#endif
 850
 851#ifdef CONFIG_SMP
 852apicinterrupt CALL_FUNCTION_SINGLE_VECTOR	call_function_single_interrupt	smp_call_function_single_interrupt
 853apicinterrupt CALL_FUNCTION_VECTOR		call_function_interrupt		smp_call_function_interrupt
 854apicinterrupt RESCHEDULE_VECTOR			reschedule_interrupt		smp_reschedule_interrupt
 855#endif
 856
 857apicinterrupt ERROR_APIC_VECTOR			error_interrupt			smp_error_interrupt
 858apicinterrupt SPURIOUS_APIC_VECTOR		spurious_interrupt		smp_spurious_interrupt
 859
 860#ifdef CONFIG_IRQ_WORK
 861apicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
 862#endif
 863
 864/*
 865 * Exception entry points.
 866 */
 867#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + (x) * 8)
 868
 869.macro idtentry_part do_sym, has_error_code:req, read_cr2:req, paranoid:req, shift_ist=-1, ist_offset=0
 870
 871	.if \paranoid
 872	call	paranoid_entry
 873	/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
 874	.else
 875	call	error_entry
 876	.endif
 877	UNWIND_HINT_REGS
 878
 879	.if \read_cr2
 880	/*
 881	 * Store CR2 early so subsequent faults cannot clobber it. Use R12 as
 882	 * intermediate storage as RDX can be clobbered in enter_from_user_mode().
 883	 * GET_CR2_INTO can clobber RAX.
 884	 */
 885	GET_CR2_INTO(%r12);
 886	.endif
 887
 888	.if \shift_ist != -1
 889	TRACE_IRQS_OFF_DEBUG			/* reload IDT in case of recursion */
 890	.else
 891	TRACE_IRQS_OFF
 892	.endif
 893
 894	.if \paranoid == 0
 895	testb	$3, CS(%rsp)
 896	jz	.Lfrom_kernel_no_context_tracking_\@
 897	CALL_enter_from_user_mode
 898.Lfrom_kernel_no_context_tracking_\@:
 899	.endif
 900
 901	movq	%rsp, %rdi			/* pt_regs pointer */
 902
 903	.if \has_error_code
 904	movq	ORIG_RAX(%rsp), %rsi		/* get error code */
 905	movq	$-1, ORIG_RAX(%rsp)		/* no syscall to restart */
 906	.else
 907	xorl	%esi, %esi			/* no error code */
 908	.endif
 909
 910	.if \shift_ist != -1
 911	subq	$\ist_offset, CPU_TSS_IST(\shift_ist)
 912	.endif
 913
 914	.if \read_cr2
 915	movq	%r12, %rdx			/* Move CR2 into 3rd argument */
 916	.endif
 917
 918	call	\do_sym
 919
 920	.if \shift_ist != -1
 921	addq	$\ist_offset, CPU_TSS_IST(\shift_ist)
 922	.endif
 923
 924	.if \paranoid
 925	/* this procedure expect "no swapgs" flag in ebx */
 926	jmp	paranoid_exit
 927	.else
 928	jmp	error_exit
 929	.endif
 930
 931.endm
 932
 933/**
 934 * idtentry - Generate an IDT entry stub
 935 * @sym:		Name of the generated entry point
 936 * @do_sym:		C function to be called
 937 * @has_error_code:	True if this IDT vector has an error code on the stack
 938 * @paranoid:		non-zero means that this vector may be invoked from
 939 *			kernel mode with user GSBASE and/or user CR3.
 940 *			2 is special -- see below.
 941 * @shift_ist:		Set to an IST index if entries from kernel mode should
 942 *			decrement the IST stack so that nested entries get a
 943 *			fresh stack.  (This is for #DB, which has a nasty habit
 944 *			of recursing.)
 945 * @create_gap:		create a 6-word stack gap when coming from kernel mode.
 946 * @read_cr2:		load CR2 into the 3rd argument; done before calling any C code
 947 *
 948 * idtentry generates an IDT stub that sets up a usable kernel context,
 949 * creates struct pt_regs, and calls @do_sym.  The stub has the following
 950 * special behaviors:
 951 *
 952 * On an entry from user mode, the stub switches from the trampoline or
 953 * IST stack to the normal thread stack.  On an exit to user mode, the
 954 * normal exit-to-usermode path is invoked.
 955 *
 956 * On an exit to kernel mode, if @paranoid == 0, we check for preemption,
 957 * whereas we omit the preemption check if @paranoid != 0.  This is purely
 958 * because the implementation is simpler this way.  The kernel only needs
 959 * to check for asynchronous kernel preemption when IRQ handlers return.
 960 *
 961 * If @paranoid == 0, then the stub will handle IRET faults by pretending
 962 * that the fault came from user mode.  It will handle gs_change faults by
 963 * pretending that the fault happened with kernel GSBASE.  Since this handling
 964 * is omitted for @paranoid != 0, the #GP, #SS, and #NP stubs must have
 965 * @paranoid == 0.  This special handling will do the wrong thing for
 966 * espfix-induced #DF on IRET, so #DF must not use @paranoid == 0.
 967 *
 968 * @paranoid == 2 is special: the stub will never switch stacks.  This is for
 969 * #DF: if the thread stack is somehow unusable, we'll still get a useful OOPS.
 970 */
 971.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ist_offset=0 create_gap=0 read_cr2=0
 972ENTRY(\sym)
 973	UNWIND_HINT_IRET_REGS offset=\has_error_code*8
 974
 975	/* Sanity check */
 976	.if \shift_ist != -1 && \paranoid != 1
 977	.error "using shift_ist requires paranoid=1"
 978	.endif
 979
 980	.if \create_gap && \paranoid
 981	.error "using create_gap requires paranoid=0"
 982	.endif
 983
 984	ASM_CLAC
 985
 986	.if \has_error_code == 0
 987	pushq	$-1				/* ORIG_RAX: no syscall to restart */
 988	.endif
 989
 990	.if \paranoid == 1
 991	testb	$3, CS-ORIG_RAX(%rsp)		/* If coming from userspace, switch stacks */
 992	jnz	.Lfrom_usermode_switch_stack_\@
 993	.endif
 994
 995	.if \create_gap == 1
 996	/*
 997	 * If coming from kernel space, create a 6-word gap to allow the
 998	 * int3 handler to emulate a call instruction.
 999	 */
1000	testb	$3, CS-ORIG_RAX(%rsp)
1001	jnz	.Lfrom_usermode_no_gap_\@
1002	.rept	6
1003	pushq	5*8(%rsp)
1004	.endr
1005	UNWIND_HINT_IRET_REGS offset=8
1006.Lfrom_usermode_no_gap_\@:
1007	.endif
1008
1009	idtentry_part \do_sym, \has_error_code, \read_cr2, \paranoid, \shift_ist, \ist_offset
1010
1011	.if \paranoid == 1
1012	/*
1013	 * Entry from userspace.  Switch stacks and treat it
1014	 * as a normal entry.  This means that paranoid handlers
1015	 * run in real process context if user_mode(regs).
1016	 */
1017.Lfrom_usermode_switch_stack_\@:
1018	idtentry_part \do_sym, \has_error_code, \read_cr2, paranoid=0
1019	.endif
1020
1021_ASM_NOKPROBE(\sym)
1022END(\sym)
1023.endm
1024
1025idtentry divide_error			do_divide_error			has_error_code=0
1026idtentry overflow			do_overflow			has_error_code=0
1027idtentry bounds				do_bounds			has_error_code=0
1028idtentry invalid_op			do_invalid_op			has_error_code=0
1029idtentry device_not_available		do_device_not_available		has_error_code=0
1030idtentry double_fault			do_double_fault			has_error_code=1 paranoid=2 read_cr2=1
1031idtentry coprocessor_segment_overrun	do_coprocessor_segment_overrun	has_error_code=0
1032idtentry invalid_TSS			do_invalid_TSS			has_error_code=1
1033idtentry segment_not_present		do_segment_not_present		has_error_code=1
1034idtentry spurious_interrupt_bug		do_spurious_interrupt_bug	has_error_code=0
1035idtentry coprocessor_error		do_coprocessor_error		has_error_code=0
1036idtentry alignment_check		do_alignment_check		has_error_code=1
1037idtentry simd_coprocessor_error		do_simd_coprocessor_error	has_error_code=0
1038
1039
1040	/*
1041	 * Reload gs selector with exception handling
1042	 * edi:  new selector
1043	 */
1044ENTRY(native_load_gs_index)
1045	FRAME_BEGIN
1046	pushfq
1047	DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
1048	TRACE_IRQS_OFF
1049	SWAPGS
1050.Lgs_change:
1051	movl	%edi, %gs
10522:	ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
1053	SWAPGS
1054	TRACE_IRQS_FLAGS (%rsp)
1055	popfq
1056	FRAME_END
1057	ret
1058ENDPROC(native_load_gs_index)
1059EXPORT_SYMBOL(native_load_gs_index)
1060
1061	_ASM_EXTABLE(.Lgs_change, .Lbad_gs)
1062	.section .fixup, "ax"
1063	/* running with kernelgs */
1064.Lbad_gs:
1065	SWAPGS					/* switch back to user gs */
1066.macro ZAP_GS
1067	/* This can't be a string because the preprocessor needs to see it. */
1068	movl $__USER_DS, %eax
1069	movl %eax, %gs
1070.endm
1071	ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
1072	xorl	%eax, %eax
1073	movl	%eax, %gs
1074	jmp	2b
 
1075	.previous
1076
1077/* Call softirq on interrupt stack. Interrupts are off. */
1078ENTRY(do_softirq_own_stack)
1079	pushq	%rbp
1080	mov	%rsp, %rbp
1081	ENTER_IRQ_STACK regs=0 old_rsp=%r11
1082	call	__do_softirq
1083	LEAVE_IRQ_STACK regs=0
1084	leaveq
1085	ret
1086ENDPROC(do_softirq_own_stack)
1087
1088#ifdef CONFIG_XEN_PV
1089idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0
1090
1091/*
1092 * A note on the "critical region" in our callback handler.
1093 * We want to avoid stacking callback handlers due to events occurring
1094 * during handling of the last event. To do this, we keep events disabled
1095 * until we've done all processing. HOWEVER, we must enable events before
1096 * popping the stack frame (can't be done atomically) and so it would still
1097 * be possible to get enough handler activations to overflow the stack.
1098 * Although unlikely, bugs of that kind are hard to track down, so we'd
1099 * like to avoid the possibility.
1100 * So, on entry to the handler we detect whether we interrupted an
1101 * existing activation in its critical region -- if so, we pop the current
1102 * activation and restart the handler using the previous one.
 
 
1103 */
1104ENTRY(xen_do_hypervisor_callback)		/* do_hypervisor_callback(struct *pt_regs) */
1105
1106/*
1107 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
1108 * see the correct pointer to the pt_regs
1109 */
1110	UNWIND_HINT_FUNC
1111	movq	%rdi, %rsp			/* we don't return, adjust the stack frame */
1112	UNWIND_HINT_REGS
1113
1114	ENTER_IRQ_STACK old_rsp=%r10
1115	call	xen_evtchn_do_upcall
1116	LEAVE_IRQ_STACK
1117
1118#ifndef CONFIG_PREEMPTION
1119	call	xen_maybe_preempt_hcall
1120#endif
1121	jmp	error_exit
1122END(xen_do_hypervisor_callback)
1123
1124/*
1125 * Hypervisor uses this for application faults while it executes.
1126 * We get here for two reasons:
1127 *  1. Fault while reloading DS, ES, FS or GS
1128 *  2. Fault while executing IRET
1129 * Category 1 we do not need to fix up as Xen has already reloaded all segment
1130 * registers that could be reloaded and zeroed the others.
1131 * Category 2 we fix up by killing the current process. We cannot use the
1132 * normal Linux return path in this case because if we use the IRET hypercall
1133 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1134 * We distinguish between categories by comparing each saved segment register
1135 * with its current contents: any discrepancy means we in category 1.
1136 */
1137ENTRY(xen_failsafe_callback)
1138	UNWIND_HINT_EMPTY
1139	movl	%ds, %ecx
1140	cmpw	%cx, 0x10(%rsp)
1141	jne	1f
1142	movl	%es, %ecx
1143	cmpw	%cx, 0x18(%rsp)
1144	jne	1f
1145	movl	%fs, %ecx
1146	cmpw	%cx, 0x20(%rsp)
1147	jne	1f
1148	movl	%gs, %ecx
1149	cmpw	%cx, 0x28(%rsp)
1150	jne	1f
1151	/* All segments match their saved values => Category 2 (Bad IRET). */
1152	movq	(%rsp), %rcx
1153	movq	8(%rsp), %r11
1154	addq	$0x30, %rsp
1155	pushq	$0				/* RIP */
1156	UNWIND_HINT_IRET_REGS offset=8
1157	jmp	general_protection
11581:	/* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
1159	movq	(%rsp), %rcx
1160	movq	8(%rsp), %r11
1161	addq	$0x30, %rsp
1162	UNWIND_HINT_IRET_REGS
1163	pushq	$-1 /* orig_ax = -1 => not a system call */
1164	PUSH_AND_CLEAR_REGS
1165	ENCODE_FRAME_POINTER
1166	jmp	error_exit
1167END(xen_failsafe_callback)
1168#endif /* CONFIG_XEN_PV */
1169
1170#ifdef CONFIG_XEN_PVHVM
1171apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1172	xen_hvm_callback_vector xen_evtchn_do_upcall
1173#endif
1174
1175
1176#if IS_ENABLED(CONFIG_HYPERV)
1177apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1178	hyperv_callback_vector hyperv_vector_handler
1179
1180apicinterrupt3 HYPERV_REENLIGHTENMENT_VECTOR \
1181	hyperv_reenlightenment_vector hyperv_reenlightenment_intr
1182
1183apicinterrupt3 HYPERV_STIMER0_VECTOR \
1184	hv_stimer0_callback_vector hv_stimer0_vector_handler
1185#endif /* CONFIG_HYPERV */
1186
1187#if IS_ENABLED(CONFIG_ACRN_GUEST)
1188apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1189	acrn_hv_callback_vector acrn_hv_vector_handler
1190#endif
1191
1192idtentry debug			do_debug		has_error_code=0	paranoid=1 shift_ist=IST_INDEX_DB ist_offset=DB_STACK_OFFSET
1193idtentry int3			do_int3			has_error_code=0	create_gap=1
1194idtentry stack_segment		do_stack_segment	has_error_code=1
1195
1196#ifdef CONFIG_XEN_PV
1197idtentry xennmi			do_nmi			has_error_code=0
1198idtentry xendebug		do_debug		has_error_code=0
1199#endif
1200
1201idtentry general_protection	do_general_protection	has_error_code=1
1202idtentry page_fault		do_page_fault		has_error_code=1	read_cr2=1
1203
1204#ifdef CONFIG_KVM_GUEST
1205idtentry async_page_fault	do_async_page_fault	has_error_code=1	read_cr2=1
1206#endif
1207
1208#ifdef CONFIG_X86_MCE
1209idtentry machine_check		do_mce			has_error_code=0	paranoid=1
1210#endif
1211
1212/*
1213 * Save all registers in pt_regs, and switch gs if needed.
1214 * Use slow, but surefire "are we in kernel?" check.
1215 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
 
 
 
 
 
1216 */
1217ENTRY(paranoid_entry)
1218	UNWIND_HINT_FUNC
1219	cld
1220	PUSH_AND_CLEAR_REGS save_ret=1
1221	ENCODE_FRAME_POINTER 8
1222	movl	$1, %ebx
1223	movl	$MSR_GS_BASE, %ecx
1224	rdmsr
1225	testl	%edx, %edx
1226	js	1f				/* negative -> in kernel */
1227	SWAPGS
1228	xorl	%ebx, %ebx
1229
12301:
1231	/*
1232	 * Always stash CR3 in %r14.  This value will be restored,
1233	 * verbatim, at exit.  Needed if paranoid_entry interrupted
1234	 * another entry that already switched to the user CR3 value
1235	 * but has not yet returned to userspace.
1236	 *
1237	 * This is also why CS (stashed in the "iret frame" by the
1238	 * hardware at entry) can not be used: this may be a return
1239	 * to kernel code, but with a user CR3 value.
 
 
 
 
 
1240	 */
1241	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
1242
1243	/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1244	 * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
1245	 * unconditional CR3 write, even in the PTI case.  So do an lfence
1246	 * to prevent GS speculation, regardless of whether PTI is enabled.
1247	 */
1248	FENCE_SWAPGS_KERNEL_ENTRY
1249
 
 
1250	ret
1251END(paranoid_entry)
1252
1253/*
1254 * "Paranoid" exit path from exception stack.  This is invoked
1255 * only on return from non-NMI IST interrupts that came
1256 * from kernel space.
1257 *
1258 * We may be returning to very strange contexts (e.g. very early
1259 * in syscall entry), so checking for preemption here would
1260 * be complicated.  Fortunately, we there's no good reason
1261 * to try to handle preemption here.
 
 
 
 
 
 
 
1262 *
1263 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
1264 */
1265ENTRY(paranoid_exit)
1266	UNWIND_HINT_REGS
1267	DISABLE_INTERRUPTS(CLBR_ANY)
1268	TRACE_IRQS_OFF_DEBUG
1269	testl	%ebx, %ebx			/* swapgs needed? */
1270	jnz	.Lparanoid_exit_no_swapgs
1271	TRACE_IRQS_IRETQ
1272	/* Always restore stashed CR3 value (see paranoid_entry) */
1273	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
1274	SWAPGS_UNSAFE_STACK
1275	jmp	.Lparanoid_exit_restore
1276.Lparanoid_exit_no_swapgs:
1277	TRACE_IRQS_IRETQ_DEBUG
1278	/* Always restore stashed CR3 value (see paranoid_entry) */
1279	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
1280.Lparanoid_exit_restore:
1281	jmp restore_regs_and_return_to_kernel
1282END(paranoid_exit)
 
 
 
 
 
 
 
 
 
 
1283
1284/*
1285 * Save all registers in pt_regs, and switch GS if needed.
1286 */
1287ENTRY(error_entry)
1288	UNWIND_HINT_FUNC
1289	cld
1290	PUSH_AND_CLEAR_REGS save_ret=1
1291	ENCODE_FRAME_POINTER 8
1292	testb	$3, CS+8(%rsp)
1293	jz	.Lerror_kernelspace
1294
1295	/*
1296	 * We entered from user mode or we're pretending to have entered
1297	 * from user mode due to an IRET fault.
1298	 */
1299	SWAPGS
1300	FENCE_SWAPGS_USER_ENTRY
1301	/* We have user CR3.  Change to kernel CR3. */
1302	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1303
1304.Lerror_entry_from_usermode_after_swapgs:
1305	/* Put us onto the real thread stack. */
1306	popq	%r12				/* save return addr in %12 */
1307	movq	%rsp, %rdi			/* arg0 = pt_regs pointer */
1308	call	sync_regs
1309	movq	%rax, %rsp			/* switch stack */
1310	ENCODE_FRAME_POINTER
1311	pushq	%r12
1312	ret
1313
1314.Lerror_entry_done_lfence:
1315	FENCE_SWAPGS_KERNEL_ENTRY
1316.Lerror_entry_done:
1317	ret
1318
1319	/*
1320	 * There are two places in the kernel that can potentially fault with
1321	 * usergs. Handle them here.  B stepping K8s sometimes report a
1322	 * truncated RIP for IRET exceptions returning to compat mode. Check
1323	 * for these here too.
1324	 */
1325.Lerror_kernelspace:
1326	leaq	native_irq_return_iret(%rip), %rcx
1327	cmpq	%rcx, RIP+8(%rsp)
1328	je	.Lerror_bad_iret
1329	movl	%ecx, %eax			/* zero extend */
1330	cmpq	%rax, RIP+8(%rsp)
1331	je	.Lbstep_iret
1332	cmpq	$.Lgs_change, RIP+8(%rsp)
1333	jne	.Lerror_entry_done_lfence
1334
1335	/*
1336	 * hack: .Lgs_change can fail with user gsbase.  If this happens, fix up
1337	 * gsbase and proceed.  We'll fix up the exception and land in
1338	 * .Lgs_change's error handler with kernel gsbase.
1339	 */
1340	SWAPGS
1341	FENCE_SWAPGS_USER_ENTRY
1342	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1343	jmp .Lerror_entry_done
1344
1345.Lbstep_iret:
1346	/* Fix truncated RIP */
1347	movq	%rcx, RIP+8(%rsp)
1348	/* fall through */
1349
1350.Lerror_bad_iret:
1351	/*
1352	 * We came from an IRET to user mode, so we have user
1353	 * gsbase and CR3.  Switch to kernel gsbase and CR3:
1354	 */
1355	SWAPGS
1356	FENCE_SWAPGS_USER_ENTRY
1357	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1358
1359	/*
1360	 * Pretend that the exception came from user mode: set up pt_regs
1361	 * as if we faulted immediately after IRET.
1362	 */
1363	mov	%rsp, %rdi
1364	call	fixup_bad_iret
1365	mov	%rax, %rsp
1366	jmp	.Lerror_entry_from_usermode_after_swapgs
1367END(error_entry)
1368
1369ENTRY(error_exit)
1370	UNWIND_HINT_REGS
1371	DISABLE_INTERRUPTS(CLBR_ANY)
1372	TRACE_IRQS_OFF
1373	testb	$3, CS(%rsp)
1374	jz	retint_kernel
1375	jmp	retint_user
1376END(error_exit)
1377
1378/*
1379 * Runs on exception stack.  Xen PV does not go through this path at all,
1380 * so we can use real assembly here.
1381 *
1382 * Registers:
1383 *	%r14: Used to save/restore the CR3 of the interrupted context
1384 *	      when PAGE_TABLE_ISOLATION is in use.  Do not clobber.
1385 */
1386ENTRY(nmi)
1387	UNWIND_HINT_IRET_REGS
1388
1389	/*
1390	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
1391	 * the iretq it performs will take us out of NMI context.
1392	 * This means that we can have nested NMIs where the next
1393	 * NMI is using the top of the stack of the previous NMI. We
1394	 * can't let it execute because the nested NMI will corrupt the
1395	 * stack of the previous NMI. NMI handlers are not re-entrant
1396	 * anyway.
1397	 *
1398	 * To handle this case we do the following:
1399	 *  Check the a special location on the stack that contains
1400	 *  a variable that is set when NMIs are executing.
1401	 *  The interrupted task's stack is also checked to see if it
1402	 *  is an NMI stack.
1403	 *  If the variable is not set and the stack is not the NMI
1404	 *  stack then:
1405	 *    o Set the special variable on the stack
1406	 *    o Copy the interrupt frame into an "outermost" location on the
1407	 *      stack
1408	 *    o Copy the interrupt frame into an "iret" location on the stack
1409	 *    o Continue processing the NMI
1410	 *  If the variable is set or the previous stack is the NMI stack:
1411	 *    o Modify the "iret" location to jump to the repeat_nmi
1412	 *    o return back to the first NMI
1413	 *
1414	 * Now on exit of the first NMI, we first clear the stack variable
1415	 * The NMI stack will tell any nested NMIs at that point that it is
1416	 * nested. Then we pop the stack normally with iret, and if there was
1417	 * a nested NMI that updated the copy interrupt stack frame, a
1418	 * jump will be made to the repeat_nmi code that will handle the second
1419	 * NMI.
1420	 *
1421	 * However, espfix prevents us from directly returning to userspace
1422	 * with a single IRET instruction.  Similarly, IRET to user mode
1423	 * can fault.  We therefore handle NMIs from user space like
1424	 * other IST entries.
1425	 */
1426
1427	ASM_CLAC
1428
1429	/* Use %rdx as our temp variable throughout */
1430	pushq	%rdx
1431
1432	testb	$3, CS-RIP+8(%rsp)
1433	jz	.Lnmi_from_kernel
1434
1435	/*
1436	 * NMI from user mode.  We need to run on the thread stack, but we
1437	 * can't go through the normal entry paths: NMIs are masked, and
1438	 * we don't want to enable interrupts, because then we'll end
1439	 * up in an awkward situation in which IRQs are on but NMIs
1440	 * are off.
1441	 *
1442	 * We also must not push anything to the stack before switching
1443	 * stacks lest we corrupt the "NMI executing" variable.
1444	 */
1445
1446	swapgs
1447	cld
1448	FENCE_SWAPGS_USER_ENTRY
1449	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
1450	movq	%rsp, %rdx
1451	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
1452	UNWIND_HINT_IRET_REGS base=%rdx offset=8
1453	pushq	5*8(%rdx)	/* pt_regs->ss */
1454	pushq	4*8(%rdx)	/* pt_regs->rsp */
1455	pushq	3*8(%rdx)	/* pt_regs->flags */
1456	pushq	2*8(%rdx)	/* pt_regs->cs */
1457	pushq	1*8(%rdx)	/* pt_regs->rip */
1458	UNWIND_HINT_IRET_REGS
1459	pushq   $-1		/* pt_regs->orig_ax */
1460	PUSH_AND_CLEAR_REGS rdx=(%rdx)
1461	ENCODE_FRAME_POINTER
1462
1463	/*
1464	 * At this point we no longer need to worry about stack damage
1465	 * due to nesting -- we're on the normal thread stack and we're
1466	 * done with the NMI stack.
1467	 */
1468
1469	movq	%rsp, %rdi
1470	movq	$-1, %rsi
1471	call	do_nmi
1472
1473	/*
1474	 * Return back to user mode.  We must *not* do the normal exit
1475	 * work, because we don't want to enable interrupts.
1476	 */
1477	jmp	swapgs_restore_regs_and_return_to_usermode
1478
1479.Lnmi_from_kernel:
1480	/*
1481	 * Here's what our stack frame will look like:
1482	 * +---------------------------------------------------------+
1483	 * | original SS                                             |
1484	 * | original Return RSP                                     |
1485	 * | original RFLAGS                                         |
1486	 * | original CS                                             |
1487	 * | original RIP                                            |
1488	 * +---------------------------------------------------------+
1489	 * | temp storage for rdx                                    |
1490	 * +---------------------------------------------------------+
1491	 * | "NMI executing" variable                                |
1492	 * +---------------------------------------------------------+
1493	 * | iret SS          } Copied from "outermost" frame        |
1494	 * | iret Return RSP  } on each loop iteration; overwritten  |
1495	 * | iret RFLAGS      } by a nested NMI to force another     |
1496	 * | iret CS          } iteration if needed.                 |
1497	 * | iret RIP         }                                      |
1498	 * +---------------------------------------------------------+
1499	 * | outermost SS          } initialized in first_nmi;       |
1500	 * | outermost Return RSP  } will not be changed before      |
1501	 * | outermost RFLAGS      } NMI processing is done.         |
1502	 * | outermost CS          } Copied to "iret" frame on each  |
1503	 * | outermost RIP         } iteration.                      |
1504	 * +---------------------------------------------------------+
1505	 * | pt_regs                                                 |
1506	 * +---------------------------------------------------------+
1507	 *
1508	 * The "original" frame is used by hardware.  Before re-enabling
1509	 * NMIs, we need to be done with it, and we need to leave enough
1510	 * space for the asm code here.
1511	 *
1512	 * We return by executing IRET while RSP points to the "iret" frame.
1513	 * That will either return for real or it will loop back into NMI
1514	 * processing.
1515	 *
1516	 * The "outermost" frame is copied to the "iret" frame on each
1517	 * iteration of the loop, so each iteration starts with the "iret"
1518	 * frame pointing to the final return target.
1519	 */
1520
1521	/*
1522	 * Determine whether we're a nested NMI.
1523	 *
1524	 * If we interrupted kernel code between repeat_nmi and
1525	 * end_repeat_nmi, then we are a nested NMI.  We must not
1526	 * modify the "iret" frame because it's being written by
1527	 * the outer NMI.  That's okay; the outer NMI handler is
1528	 * about to about to call do_nmi anyway, so we can just
1529	 * resume the outer NMI.
1530	 */
1531
1532	movq	$repeat_nmi, %rdx
1533	cmpq	8(%rsp), %rdx
1534	ja	1f
1535	movq	$end_repeat_nmi, %rdx
1536	cmpq	8(%rsp), %rdx
1537	ja	nested_nmi_out
15381:
1539
1540	/*
1541	 * Now check "NMI executing".  If it's set, then we're nested.
1542	 * This will not detect if we interrupted an outer NMI just
1543	 * before IRET.
1544	 */
1545	cmpl	$1, -8(%rsp)
1546	je	nested_nmi
1547
1548	/*
1549	 * Now test if the previous stack was an NMI stack.  This covers
1550	 * the case where we interrupt an outer NMI after it clears
1551	 * "NMI executing" but before IRET.  We need to be careful, though:
1552	 * there is one case in which RSP could point to the NMI stack
1553	 * despite there being no NMI active: naughty userspace controls
1554	 * RSP at the very beginning of the SYSCALL targets.  We can
1555	 * pull a fast one on naughty userspace, though: we program
1556	 * SYSCALL to mask DF, so userspace cannot cause DF to be set
1557	 * if it controls the kernel's RSP.  We set DF before we clear
1558	 * "NMI executing".
1559	 */
1560	lea	6*8(%rsp), %rdx
1561	/* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
1562	cmpq	%rdx, 4*8(%rsp)
1563	/* If the stack pointer is above the NMI stack, this is a normal NMI */
1564	ja	first_nmi
1565
1566	subq	$EXCEPTION_STKSZ, %rdx
1567	cmpq	%rdx, 4*8(%rsp)
1568	/* If it is below the NMI stack, it is a normal NMI */
1569	jb	first_nmi
1570
1571	/* Ah, it is within the NMI stack. */
1572
1573	testb	$(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
1574	jz	first_nmi	/* RSP was user controlled. */
1575
1576	/* This is a nested NMI. */
1577
1578nested_nmi:
1579	/*
1580	 * Modify the "iret" frame to point to repeat_nmi, forcing another
1581	 * iteration of NMI handling.
1582	 */
1583	subq	$8, %rsp
1584	leaq	-10*8(%rsp), %rdx
1585	pushq	$__KERNEL_DS
1586	pushq	%rdx
1587	pushfq
1588	pushq	$__KERNEL_CS
1589	pushq	$repeat_nmi
1590
1591	/* Put stack back */
1592	addq	$(6*8), %rsp
1593
1594nested_nmi_out:
1595	popq	%rdx
1596
1597	/* We are returning to kernel mode, so this cannot result in a fault. */
1598	iretq
1599
1600first_nmi:
1601	/* Restore rdx. */
1602	movq	(%rsp), %rdx
1603
1604	/* Make room for "NMI executing". */
1605	pushq	$0
1606
1607	/* Leave room for the "iret" frame */
1608	subq	$(5*8), %rsp
1609
1610	/* Copy the "original" frame to the "outermost" frame */
1611	.rept 5
1612	pushq	11*8(%rsp)
1613	.endr
1614	UNWIND_HINT_IRET_REGS
1615
1616	/* Everything up to here is safe from nested NMIs */
1617
1618#ifdef CONFIG_DEBUG_ENTRY
1619	/*
1620	 * For ease of testing, unmask NMIs right away.  Disabled by
1621	 * default because IRET is very expensive.
1622	 */
1623	pushq	$0		/* SS */
1624	pushq	%rsp		/* RSP (minus 8 because of the previous push) */
1625	addq	$8, (%rsp)	/* Fix up RSP */
1626	pushfq			/* RFLAGS */
1627	pushq	$__KERNEL_CS	/* CS */
1628	pushq	$1f		/* RIP */
1629	iretq			/* continues at repeat_nmi below */
1630	UNWIND_HINT_IRET_REGS
16311:
1632#endif
1633
1634repeat_nmi:
1635	/*
1636	 * If there was a nested NMI, the first NMI's iret will return
1637	 * here. But NMIs are still enabled and we can take another
1638	 * nested NMI. The nested NMI checks the interrupted RIP to see
1639	 * if it is between repeat_nmi and end_repeat_nmi, and if so
1640	 * it will just return, as we are about to repeat an NMI anyway.
1641	 * This makes it safe to copy to the stack frame that a nested
1642	 * NMI will update.
1643	 *
1644	 * RSP is pointing to "outermost RIP".  gsbase is unknown, but, if
1645	 * we're repeating an NMI, gsbase has the same value that it had on
1646	 * the first iteration.  paranoid_entry will load the kernel
1647	 * gsbase if needed before we call do_nmi.  "NMI executing"
1648	 * is zero.
1649	 */
1650	movq	$1, 10*8(%rsp)		/* Set "NMI executing". */
1651
1652	/*
1653	 * Copy the "outermost" frame to the "iret" frame.  NMIs that nest
1654	 * here must not modify the "iret" frame while we're writing to
1655	 * it or it will end up containing garbage.
1656	 */
1657	addq	$(10*8), %rsp
1658	.rept 5
1659	pushq	-6*8(%rsp)
1660	.endr
1661	subq	$(5*8), %rsp
1662end_repeat_nmi:
1663
1664	/*
1665	 * Everything below this point can be preempted by a nested NMI.
1666	 * If this happens, then the inner NMI will change the "iret"
1667	 * frame to point back to repeat_nmi.
1668	 */
1669	pushq	$-1				/* ORIG_RAX: no syscall to restart */
1670
1671	/*
1672	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1673	 * as we should not be calling schedule in NMI context.
1674	 * Even with normal interrupts enabled. An NMI should not be
1675	 * setting NEED_RESCHED or anything that normal interrupts and
1676	 * exceptions might do.
1677	 */
1678	call	paranoid_entry
1679	UNWIND_HINT_REGS
1680
1681	/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1682	movq	%rsp, %rdi
1683	movq	$-1, %rsi
1684	call	do_nmi
1685
1686	/* Always restore stashed CR3 value (see paranoid_entry) */
1687	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
1688
1689	testl	%ebx, %ebx			/* swapgs needed? */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1690	jnz	nmi_restore
 
1691nmi_swapgs:
1692	SWAPGS_UNSAFE_STACK
 
1693nmi_restore:
1694	POP_REGS
1695
1696	/*
1697	 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
1698	 * at the "iret" frame.
1699	 */
1700	addq	$6*8, %rsp
1701
1702	/*
1703	 * Clear "NMI executing".  Set DF first so that we can easily
1704	 * distinguish the remaining code between here and IRET from
1705	 * the SYSCALL entry and exit paths.
1706	 *
1707	 * We arguably should just inspect RIP instead, but I (Andy) wrote
1708	 * this code when I had the misapprehension that Xen PV supported
1709	 * NMIs, and Xen PV would break that approach.
1710	 */
1711	std
1712	movq	$0, 5*8(%rsp)		/* clear "NMI executing" */
1713
1714	/*
1715	 * iretq reads the "iret" frame and exits the NMI stack in a
1716	 * single instruction.  We are returning to kernel mode, so this
1717	 * cannot result in a fault.  Similarly, we don't need to worry
1718	 * about espfix64 on the way back to kernel mode.
1719	 */
1720	iretq
1721END(nmi)
1722
1723#ifndef CONFIG_IA32_EMULATION
1724/*
1725 * This handles SYSCALL from 32-bit code.  There is no way to program
1726 * MSRs to fully disable 32-bit SYSCALL.
1727 */
1728ENTRY(ignore_sysret)
1729	UNWIND_HINT_EMPTY
1730	mov	$-ENOSYS, %eax
1731	sysret
1732END(ignore_sysret)
1733#endif
1734
1735ENTRY(rewind_stack_do_exit)
 
1736	UNWIND_HINT_FUNC
1737	/* Prevent any naive code from trying to unwind to our caller. */
1738	xorl	%ebp, %ebp
1739
1740	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax
1741	leaq	-PTREGS_SIZE(%rax), %rsp
1742	UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE
1743
1744	call	do_exit
1745END(rewind_stack_do_exit)