Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 1991,1992 Linus Torvalds
4 *
5 * entry_32.S contains the system-call and low-level fault and trap handling routines.
6 *
7 * Stack layout while running C code:
8 * ptrace needs to have all registers on the stack.
9 * If the order here is changed, it needs to be
10 * updated in fork.c:copy_process(), signal.c:do_signal(),
11 * ptrace.c and ptrace.h
12 *
13 * 0(%esp) - %ebx
14 * 4(%esp) - %ecx
15 * 8(%esp) - %edx
16 * C(%esp) - %esi
17 * 10(%esp) - %edi
18 * 14(%esp) - %ebp
19 * 18(%esp) - %eax
20 * 1C(%esp) - %ds
21 * 20(%esp) - %es
22 * 24(%esp) - %fs
23 * 28(%esp) - unused -- was %gs on old stackprotector kernels
24 * 2C(%esp) - orig_eax
25 * 30(%esp) - %eip
26 * 34(%esp) - %cs
27 * 38(%esp) - %eflags
28 * 3C(%esp) - %oldesp
29 * 40(%esp) - %oldss
30 */
31
32#include <linux/linkage.h>
33#include <linux/err.h>
34#include <asm/thread_info.h>
35#include <asm/irqflags.h>
36#include <asm/errno.h>
37#include <asm/segment.h>
38#include <asm/smp.h>
39#include <asm/percpu.h>
40#include <asm/processor-flags.h>
41#include <asm/irq_vectors.h>
42#include <asm/cpufeatures.h>
43#include <asm/alternative.h>
44#include <asm/asm.h>
45#include <asm/smap.h>
46#include <asm/frame.h>
47#include <asm/trapnr.h>
48#include <asm/nospec-branch.h>
49
50#include "calling.h"
51
52 .section .entry.text, "ax"
53
54#define PTI_SWITCH_MASK (1 << PAGE_SHIFT)
55
56/* Unconditionally switch to user cr3 */
57.macro SWITCH_TO_USER_CR3 scratch_reg:req
58 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
59
60 movl %cr3, \scratch_reg
61 orl $PTI_SWITCH_MASK, \scratch_reg
62 movl \scratch_reg, %cr3
63.Lend_\@:
64.endm
65
66.macro BUG_IF_WRONG_CR3 no_user_check=0
67#ifdef CONFIG_DEBUG_ENTRY
68 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
69 .if \no_user_check == 0
70 /* coming from usermode? */
71 testl $USER_SEGMENT_RPL_MASK, PT_CS(%esp)
72 jz .Lend_\@
73 .endif
74 /* On user-cr3? */
75 movl %cr3, %eax
76 testl $PTI_SWITCH_MASK, %eax
77 jnz .Lend_\@
78 /* From userspace with kernel cr3 - BUG */
79 ud2
80.Lend_\@:
81#endif
82.endm
83
84/*
85 * Switch to kernel cr3 if not already loaded and return current cr3 in
86 * \scratch_reg
87 */
88.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
89 ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
90 movl %cr3, \scratch_reg
91 /* Test if we are already on kernel CR3 */
92 testl $PTI_SWITCH_MASK, \scratch_reg
93 jz .Lend_\@
94 andl $(~PTI_SWITCH_MASK), \scratch_reg
95 movl \scratch_reg, %cr3
96 /* Return original CR3 in \scratch_reg */
97 orl $PTI_SWITCH_MASK, \scratch_reg
98.Lend_\@:
99.endm
100
101#define CS_FROM_ENTRY_STACK (1 << 31)
102#define CS_FROM_USER_CR3 (1 << 30)
103#define CS_FROM_KERNEL (1 << 29)
104#define CS_FROM_ESPFIX (1 << 28)
105
106.macro FIXUP_FRAME
107 /*
108 * The high bits of the CS dword (__csh) are used for CS_FROM_*.
109 * Clear them in case hardware didn't do this for us.
110 */
111 andl $0x0000ffff, 4*4(%esp)
112
113#ifdef CONFIG_VM86
114 testl $X86_EFLAGS_VM, 5*4(%esp)
115 jnz .Lfrom_usermode_no_fixup_\@
116#endif
117 testl $USER_SEGMENT_RPL_MASK, 4*4(%esp)
118 jnz .Lfrom_usermode_no_fixup_\@
119
120 orl $CS_FROM_KERNEL, 4*4(%esp)
121
122 /*
123 * When we're here from kernel mode; the (exception) stack looks like:
124 *
125 * 6*4(%esp) - <previous context>
126 * 5*4(%esp) - flags
127 * 4*4(%esp) - cs
128 * 3*4(%esp) - ip
129 * 2*4(%esp) - orig_eax
130 * 1*4(%esp) - gs / function
131 * 0*4(%esp) - fs
132 *
133 * Lets build a 5 entry IRET frame after that, such that struct pt_regs
134 * is complete and in particular regs->sp is correct. This gives us
135 * the original 6 entries as gap:
136 *
137 * 14*4(%esp) - <previous context>
138 * 13*4(%esp) - gap / flags
139 * 12*4(%esp) - gap / cs
140 * 11*4(%esp) - gap / ip
141 * 10*4(%esp) - gap / orig_eax
142 * 9*4(%esp) - gap / gs / function
143 * 8*4(%esp) - gap / fs
144 * 7*4(%esp) - ss
145 * 6*4(%esp) - sp
146 * 5*4(%esp) - flags
147 * 4*4(%esp) - cs
148 * 3*4(%esp) - ip
149 * 2*4(%esp) - orig_eax
150 * 1*4(%esp) - gs / function
151 * 0*4(%esp) - fs
152 */
153
154 pushl %ss # ss
155 pushl %esp # sp (points at ss)
156 addl $7*4, (%esp) # point sp back at the previous context
157 pushl 7*4(%esp) # flags
158 pushl 7*4(%esp) # cs
159 pushl 7*4(%esp) # ip
160 pushl 7*4(%esp) # orig_eax
161 pushl 7*4(%esp) # gs / function
162 pushl 7*4(%esp) # fs
163.Lfrom_usermode_no_fixup_\@:
164.endm
165
166.macro IRET_FRAME
167 /*
168 * We're called with %ds, %es, %fs, and %gs from the interrupted
169 * frame, so we shouldn't use them. Also, we may be in ESPFIX
170 * mode and therefore have a nonzero SS base and an offset ESP,
171 * so any attempt to access the stack needs to use SS. (except for
172 * accesses through %esp, which automatically use SS.)
173 */
174 testl $CS_FROM_KERNEL, 1*4(%esp)
175 jz .Lfinished_frame_\@
176
177 /*
178 * Reconstruct the 3 entry IRET frame right after the (modified)
179 * regs->sp without lowering %esp in between, such that an NMI in the
180 * middle doesn't scribble our stack.
181 */
182 pushl %eax
183 pushl %ecx
184 movl 5*4(%esp), %eax # (modified) regs->sp
185
186 movl 4*4(%esp), %ecx # flags
187 movl %ecx, %ss:-1*4(%eax)
188
189 movl 3*4(%esp), %ecx # cs
190 andl $0x0000ffff, %ecx
191 movl %ecx, %ss:-2*4(%eax)
192
193 movl 2*4(%esp), %ecx # ip
194 movl %ecx, %ss:-3*4(%eax)
195
196 movl 1*4(%esp), %ecx # eax
197 movl %ecx, %ss:-4*4(%eax)
198
199 popl %ecx
200 lea -4*4(%eax), %esp
201 popl %eax
202.Lfinished_frame_\@:
203.endm
204
205.macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 skip_gs=0 unwind_espfix=0
206 cld
207.if \skip_gs == 0
208 pushl $0
209.endif
210 pushl %fs
211
212 pushl %eax
213 movl $(__KERNEL_PERCPU), %eax
214 movl %eax, %fs
215.if \unwind_espfix > 0
216 UNWIND_ESPFIX_STACK
217.endif
218 popl %eax
219
220 FIXUP_FRAME
221 pushl %es
222 pushl %ds
223 pushl \pt_regs_ax
224 pushl %ebp
225 pushl %edi
226 pushl %esi
227 pushl %edx
228 pushl %ecx
229 pushl %ebx
230 movl $(__USER_DS), %edx
231 movl %edx, %ds
232 movl %edx, %es
233 /* Switch to kernel stack if necessary */
234.if \switch_stacks > 0
235 SWITCH_TO_KERNEL_STACK
236.endif
237.endm
238
239.macro SAVE_ALL_NMI cr3_reg:req unwind_espfix=0
240 SAVE_ALL unwind_espfix=\unwind_espfix
241
242 BUG_IF_WRONG_CR3
243
244 /*
245 * Now switch the CR3 when PTI is enabled.
246 *
247 * We can enter with either user or kernel cr3, the code will
248 * store the old cr3 in \cr3_reg and switches to the kernel cr3
249 * if necessary.
250 */
251 SWITCH_TO_KERNEL_CR3 scratch_reg=\cr3_reg
252
253.Lend_\@:
254.endm
255
256.macro RESTORE_INT_REGS
257 popl %ebx
258 popl %ecx
259 popl %edx
260 popl %esi
261 popl %edi
262 popl %ebp
263 popl %eax
264.endm
265
266.macro RESTORE_REGS pop=0
267 RESTORE_INT_REGS
2681: popl %ds
2692: popl %es
2703: popl %fs
2714: addl $(4 + \pop), %esp /* pop the unused "gs" slot */
272 IRET_FRAME
273
274 /*
275 * There is no _ASM_EXTABLE_TYPE_REG() for ASM, however since this is
276 * ASM the registers are known and we can trivially hard-code them.
277 */
278 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_POP_ZERO|EX_REG_DS)
279 _ASM_EXTABLE_TYPE(2b, 3b, EX_TYPE_POP_ZERO|EX_REG_ES)
280 _ASM_EXTABLE_TYPE(3b, 4b, EX_TYPE_POP_ZERO|EX_REG_FS)
281.endm
282
283.macro RESTORE_ALL_NMI cr3_reg:req pop=0
284 /*
285 * Now switch the CR3 when PTI is enabled.
286 *
287 * We enter with kernel cr3 and switch the cr3 to the value
288 * stored on \cr3_reg, which is either a user or a kernel cr3.
289 */
290 ALTERNATIVE "jmp .Lswitched_\@", "", X86_FEATURE_PTI
291
292 testl $PTI_SWITCH_MASK, \cr3_reg
293 jz .Lswitched_\@
294
295 /* User cr3 in \cr3_reg - write it to hardware cr3 */
296 movl \cr3_reg, %cr3
297
298.Lswitched_\@:
299
300 BUG_IF_WRONG_CR3
301
302 RESTORE_REGS pop=\pop
303.endm
304
305.macro CHECK_AND_APPLY_ESPFIX
306#ifdef CONFIG_X86_ESPFIX32
307#define GDT_ESPFIX_OFFSET (GDT_ENTRY_ESPFIX_SS * 8)
308#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + GDT_ESPFIX_OFFSET
309
310 ALTERNATIVE "jmp .Lend_\@", "", X86_BUG_ESPFIX
311
312 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
313 /*
314 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we
315 * are returning to the kernel.
316 * See comments in process.c:copy_thread() for details.
317 */
318 movb PT_OLDSS(%esp), %ah
319 movb PT_CS(%esp), %al
320 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
321 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
322 jne .Lend_\@ # returning to user-space with LDT SS
323
324 /*
325 * Setup and switch to ESPFIX stack
326 *
327 * We're returning to userspace with a 16 bit stack. The CPU will not
328 * restore the high word of ESP for us on executing iret... This is an
329 * "official" bug of all the x86-compatible CPUs, which we can work
330 * around to make dosemu and wine happy. We do this by preloading the
331 * high word of ESP with the high word of the userspace ESP while
332 * compensating for the offset by changing to the ESPFIX segment with
333 * a base address that matches for the difference.
334 */
335 mov %esp, %edx /* load kernel esp */
336 mov PT_OLDESP(%esp), %eax /* load userspace esp */
337 mov %dx, %ax /* eax: new kernel esp */
338 sub %eax, %edx /* offset (low word is 0) */
339 shr $16, %edx
340 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
341 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
342 pushl $__ESPFIX_SS
343 pushl %eax /* new kernel esp */
344 /*
345 * Disable interrupts, but do not irqtrace this section: we
346 * will soon execute iret and the tracer was already set to
347 * the irqstate after the IRET:
348 */
349 cli
350 lss (%esp), %esp /* switch to espfix segment */
351.Lend_\@:
352#endif /* CONFIG_X86_ESPFIX32 */
353.endm
354
355/*
356 * Called with pt_regs fully populated and kernel segments loaded,
357 * so we can access PER_CPU and use the integer registers.
358 *
359 * We need to be very careful here with the %esp switch, because an NMI
360 * can happen everywhere. If the NMI handler finds itself on the
361 * entry-stack, it will overwrite the task-stack and everything we
362 * copied there. So allocate the stack-frame on the task-stack and
363 * switch to it before we do any copying.
364 */
365
366.macro SWITCH_TO_KERNEL_STACK
367
368 BUG_IF_WRONG_CR3
369
370 SWITCH_TO_KERNEL_CR3 scratch_reg=%eax
371
372 /*
373 * %eax now contains the entry cr3 and we carry it forward in
374 * that register for the time this macro runs
375 */
376
377 /* Are we on the entry stack? Bail out if not! */
378 movl PER_CPU_VAR(cpu_entry_area), %ecx
379 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
380 subl %esp, %ecx /* ecx = (end of entry_stack) - esp */
381 cmpl $SIZEOF_entry_stack, %ecx
382 jae .Lend_\@
383
384 /* Load stack pointer into %esi and %edi */
385 movl %esp, %esi
386 movl %esi, %edi
387
388 /* Move %edi to the top of the entry stack */
389 andl $(MASK_entry_stack), %edi
390 addl $(SIZEOF_entry_stack), %edi
391
392 /* Load top of task-stack into %edi */
393 movl TSS_entry2task_stack(%edi), %edi
394
395 /* Special case - entry from kernel mode via entry stack */
396#ifdef CONFIG_VM86
397 movl PT_EFLAGS(%esp), %ecx # mix EFLAGS and CS
398 movb PT_CS(%esp), %cl
399 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %ecx
400#else
401 movl PT_CS(%esp), %ecx
402 andl $SEGMENT_RPL_MASK, %ecx
403#endif
404 cmpl $USER_RPL, %ecx
405 jb .Lentry_from_kernel_\@
406
407 /* Bytes to copy */
408 movl $PTREGS_SIZE, %ecx
409
410#ifdef CONFIG_VM86
411 testl $X86_EFLAGS_VM, PT_EFLAGS(%esi)
412 jz .Lcopy_pt_regs_\@
413
414 /*
415 * Stack-frame contains 4 additional segment registers when
416 * coming from VM86 mode
417 */
418 addl $(4 * 4), %ecx
419
420#endif
421.Lcopy_pt_regs_\@:
422
423 /* Allocate frame on task-stack */
424 subl %ecx, %edi
425
426 /* Switch to task-stack */
427 movl %edi, %esp
428
429 /*
430 * We are now on the task-stack and can safely copy over the
431 * stack-frame
432 */
433 shrl $2, %ecx
434 cld
435 rep movsl
436
437 jmp .Lend_\@
438
439.Lentry_from_kernel_\@:
440
441 /*
442 * This handles the case when we enter the kernel from
443 * kernel-mode and %esp points to the entry-stack. When this
444 * happens we need to switch to the task-stack to run C code,
445 * but switch back to the entry-stack again when we approach
446 * iret and return to the interrupted code-path. This usually
447 * happens when we hit an exception while restoring user-space
448 * segment registers on the way back to user-space or when the
449 * sysenter handler runs with eflags.tf set.
450 *
451 * When we switch to the task-stack here, we can't trust the
452 * contents of the entry-stack anymore, as the exception handler
453 * might be scheduled out or moved to another CPU. Therefore we
454 * copy the complete entry-stack to the task-stack and set a
455 * marker in the iret-frame (bit 31 of the CS dword) to detect
456 * what we've done on the iret path.
457 *
458 * On the iret path we copy everything back and switch to the
459 * entry-stack, so that the interrupted kernel code-path
460 * continues on the same stack it was interrupted with.
461 *
462 * Be aware that an NMI can happen anytime in this code.
463 *
464 * %esi: Entry-Stack pointer (same as %esp)
465 * %edi: Top of the task stack
466 * %eax: CR3 on kernel entry
467 */
468
469 /* Calculate number of bytes on the entry stack in %ecx */
470 movl %esi, %ecx
471
472 /* %ecx to the top of entry-stack */
473 andl $(MASK_entry_stack), %ecx
474 addl $(SIZEOF_entry_stack), %ecx
475
476 /* Number of bytes on the entry stack to %ecx */
477 sub %esi, %ecx
478
479 /* Mark stackframe as coming from entry stack */
480 orl $CS_FROM_ENTRY_STACK, PT_CS(%esp)
481
482 /*
483 * Test the cr3 used to enter the kernel and add a marker
484 * so that we can switch back to it before iret.
485 */
486 testl $PTI_SWITCH_MASK, %eax
487 jz .Lcopy_pt_regs_\@
488 orl $CS_FROM_USER_CR3, PT_CS(%esp)
489
490 /*
491 * %esi and %edi are unchanged, %ecx contains the number of
492 * bytes to copy. The code at .Lcopy_pt_regs_\@ will allocate
493 * the stack-frame on task-stack and copy everything over
494 */
495 jmp .Lcopy_pt_regs_\@
496
497.Lend_\@:
498.endm
499
500/*
501 * Switch back from the kernel stack to the entry stack.
502 *
503 * The %esp register must point to pt_regs on the task stack. It will
504 * first calculate the size of the stack-frame to copy, depending on
505 * whether we return to VM86 mode or not. With that it uses 'rep movsl'
506 * to copy the contents of the stack over to the entry stack.
507 *
508 * We must be very careful here, as we can't trust the contents of the
509 * task-stack once we switched to the entry-stack. When an NMI happens
510 * while on the entry-stack, the NMI handler will switch back to the top
511 * of the task stack, overwriting our stack-frame we are about to copy.
512 * Therefore we switch the stack only after everything is copied over.
513 */
514.macro SWITCH_TO_ENTRY_STACK
515
516 /* Bytes to copy */
517 movl $PTREGS_SIZE, %ecx
518
519#ifdef CONFIG_VM86
520 testl $(X86_EFLAGS_VM), PT_EFLAGS(%esp)
521 jz .Lcopy_pt_regs_\@
522
523 /* Additional 4 registers to copy when returning to VM86 mode */
524 addl $(4 * 4), %ecx
525
526.Lcopy_pt_regs_\@:
527#endif
528
529 /* Initialize source and destination for movsl */
530 movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi
531 subl %ecx, %edi
532 movl %esp, %esi
533
534 /* Save future stack pointer in %ebx */
535 movl %edi, %ebx
536
537 /* Copy over the stack-frame */
538 shrl $2, %ecx
539 cld
540 rep movsl
541
542 /*
543 * Switch to entry-stack - needs to happen after everything is
544 * copied because the NMI handler will overwrite the task-stack
545 * when on entry-stack
546 */
547 movl %ebx, %esp
548
549.Lend_\@:
550.endm
551
552/*
553 * This macro handles the case when we return to kernel-mode on the iret
554 * path and have to switch back to the entry stack and/or user-cr3
555 *
556 * See the comments below the .Lentry_from_kernel_\@ label in the
557 * SWITCH_TO_KERNEL_STACK macro for more details.
558 */
559.macro PARANOID_EXIT_TO_KERNEL_MODE
560
561 /*
562 * Test if we entered the kernel with the entry-stack. Most
563 * likely we did not, because this code only runs on the
564 * return-to-kernel path.
565 */
566 testl $CS_FROM_ENTRY_STACK, PT_CS(%esp)
567 jz .Lend_\@
568
569 /* Unlikely slow-path */
570
571 /* Clear marker from stack-frame */
572 andl $(~CS_FROM_ENTRY_STACK), PT_CS(%esp)
573
574 /* Copy the remaining task-stack contents to entry-stack */
575 movl %esp, %esi
576 movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi
577
578 /* Bytes on the task-stack to ecx */
579 movl PER_CPU_VAR(cpu_tss_rw + TSS_sp1), %ecx
580 subl %esi, %ecx
581
582 /* Allocate stack-frame on entry-stack */
583 subl %ecx, %edi
584
585 /*
586 * Save future stack-pointer, we must not switch until the
587 * copy is done, otherwise the NMI handler could destroy the
588 * contents of the task-stack we are about to copy.
589 */
590 movl %edi, %ebx
591
592 /* Do the copy */
593 shrl $2, %ecx
594 cld
595 rep movsl
596
597 /* Safe to switch to entry-stack now */
598 movl %ebx, %esp
599
600 /*
601 * We came from entry-stack and need to check if we also need to
602 * switch back to user cr3.
603 */
604 testl $CS_FROM_USER_CR3, PT_CS(%esp)
605 jz .Lend_\@
606
607 /* Clear marker from stack-frame */
608 andl $(~CS_FROM_USER_CR3), PT_CS(%esp)
609
610 SWITCH_TO_USER_CR3 scratch_reg=%eax
611
612.Lend_\@:
613.endm
614
615/**
616 * idtentry - Macro to generate entry stubs for simple IDT entries
617 * @vector: Vector number
618 * @asmsym: ASM symbol for the entry point
619 * @cfunc: C function to be called
620 * @has_error_code: Hardware pushed error code on stack
621 */
622.macro idtentry vector asmsym cfunc has_error_code:req
623SYM_CODE_START(\asmsym)
624 ASM_CLAC
625 cld
626
627 .if \has_error_code == 0
628 pushl $0 /* Clear the error code */
629 .endif
630
631 /* Push the C-function address into the GS slot */
632 pushl $\cfunc
633 /* Invoke the common exception entry */
634 jmp handle_exception
635SYM_CODE_END(\asmsym)
636.endm
637
638.macro idtentry_irq vector cfunc
639 .p2align CONFIG_X86_L1_CACHE_SHIFT
640SYM_CODE_START_LOCAL(asm_\cfunc)
641 ASM_CLAC
642 SAVE_ALL switch_stacks=1
643 ENCODE_FRAME_POINTER
644 movl %esp, %eax
645 movl PT_ORIG_EAX(%esp), %edx /* get the vector from stack */
646 movl $-1, PT_ORIG_EAX(%esp) /* no syscall to restart */
647 call \cfunc
648 jmp handle_exception_return
649SYM_CODE_END(asm_\cfunc)
650.endm
651
652.macro idtentry_sysvec vector cfunc
653 idtentry \vector asm_\cfunc \cfunc has_error_code=0
654.endm
655
656/*
657 * Include the defines which emit the idt entries which are shared
658 * shared between 32 and 64 bit and emit the __irqentry_text_* markers
659 * so the stacktrace boundary checks work.
660 */
661 .align 16
662 .globl __irqentry_text_start
663__irqentry_text_start:
664
665#include <asm/idtentry.h>
666
667 .align 16
668 .globl __irqentry_text_end
669__irqentry_text_end:
670
671/*
672 * %eax: prev task
673 * %edx: next task
674 */
675.pushsection .text, "ax"
676SYM_CODE_START(__switch_to_asm)
677 /*
678 * Save callee-saved registers
679 * This must match the order in struct inactive_task_frame
680 */
681 pushl %ebp
682 pushl %ebx
683 pushl %edi
684 pushl %esi
685 /*
686 * Flags are saved to prevent AC leakage. This could go
687 * away if objtool would have 32bit support to verify
688 * the STAC/CLAC correctness.
689 */
690 pushfl
691
692 /* switch stack */
693 movl %esp, TASK_threadsp(%eax)
694 movl TASK_threadsp(%edx), %esp
695
696#ifdef CONFIG_STACKPROTECTOR
697 movl TASK_stack_canary(%edx), %ebx
698 movl %ebx, PER_CPU_VAR(__stack_chk_guard)
699#endif
700
701 /*
702 * When switching from a shallower to a deeper call stack
703 * the RSB may either underflow or use entries populated
704 * with userspace addresses. On CPUs where those concerns
705 * exist, overwrite the RSB with entries which capture
706 * speculative execution to prevent attack.
707 */
708 FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
709
710 /* Restore flags or the incoming task to restore AC state. */
711 popfl
712 /* restore callee-saved registers */
713 popl %esi
714 popl %edi
715 popl %ebx
716 popl %ebp
717
718 jmp __switch_to
719SYM_CODE_END(__switch_to_asm)
720.popsection
721
722/*
723 * The unwinder expects the last frame on the stack to always be at the same
724 * offset from the end of the page, which allows it to validate the stack.
725 * Calling schedule_tail() directly would break that convention because its an
726 * asmlinkage function so its argument has to be pushed on the stack. This
727 * wrapper creates a proper "end of stack" frame header before the call.
728 */
729.pushsection .text, "ax"
730SYM_FUNC_START(schedule_tail_wrapper)
731 FRAME_BEGIN
732
733 pushl %eax
734 call schedule_tail
735 popl %eax
736
737 FRAME_END
738 RET
739SYM_FUNC_END(schedule_tail_wrapper)
740.popsection
741
742/*
743 * A newly forked process directly context switches into this address.
744 *
745 * eax: prev task we switched from
746 * ebx: kernel thread func (NULL for user thread)
747 * edi: kernel thread arg
748 */
749.pushsection .text, "ax"
750SYM_CODE_START(ret_from_fork)
751 call schedule_tail_wrapper
752
753 testl %ebx, %ebx
754 jnz 1f /* kernel threads are uncommon */
755
7562:
757 /* When we fork, we trace the syscall return in the child, too. */
758 movl %esp, %eax
759 call syscall_exit_to_user_mode
760 jmp .Lsyscall_32_done
761
762 /* kernel thread */
7631: movl %edi, %eax
764 CALL_NOSPEC ebx
765 /*
766 * A kernel thread is allowed to return here after successfully
767 * calling kernel_execve(). Exit to userspace to complete the execve()
768 * syscall.
769 */
770 movl $0, PT_EAX(%esp)
771 jmp 2b
772SYM_CODE_END(ret_from_fork)
773.popsection
774
775SYM_ENTRY(__begin_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE)
776/*
777 * All code from here through __end_SYSENTER_singlestep_region is subject
778 * to being single-stepped if a user program sets TF and executes SYSENTER.
779 * There is absolutely nothing that we can do to prevent this from happening
780 * (thanks Intel!). To keep our handling of this situation as simple as
781 * possible, we handle TF just like AC and NT, except that our #DB handler
782 * will ignore all of the single-step traps generated in this range.
783 */
784
785/*
786 * 32-bit SYSENTER entry.
787 *
788 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
789 * if X86_FEATURE_SEP is available. This is the preferred system call
790 * entry on 32-bit systems.
791 *
792 * The SYSENTER instruction, in principle, should *only* occur in the
793 * vDSO. In practice, a small number of Android devices were shipped
794 * with a copy of Bionic that inlined a SYSENTER instruction. This
795 * never happened in any of Google's Bionic versions -- it only happened
796 * in a narrow range of Intel-provided versions.
797 *
798 * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs.
799 * IF and VM in RFLAGS are cleared (IOW: interrupts are off).
800 * SYSENTER does not save anything on the stack,
801 * and does not save old EIP (!!!), ESP, or EFLAGS.
802 *
803 * To avoid losing track of EFLAGS.VM (and thus potentially corrupting
804 * user and/or vm86 state), we explicitly disable the SYSENTER
805 * instruction in vm86 mode by reprogramming the MSRs.
806 *
807 * Arguments:
808 * eax system call number
809 * ebx arg1
810 * ecx arg2
811 * edx arg3
812 * esi arg4
813 * edi arg5
814 * ebp user stack
815 * 0(%ebp) arg6
816 */
817SYM_FUNC_START(entry_SYSENTER_32)
818 /*
819 * On entry-stack with all userspace-regs live - save and
820 * restore eflags and %eax to use it as scratch-reg for the cr3
821 * switch.
822 */
823 pushfl
824 pushl %eax
825 BUG_IF_WRONG_CR3 no_user_check=1
826 SWITCH_TO_KERNEL_CR3 scratch_reg=%eax
827 popl %eax
828 popfl
829
830 /* Stack empty again, switch to task stack */
831 movl TSS_entry2task_stack(%esp), %esp
832
833.Lsysenter_past_esp:
834 pushl $__USER_DS /* pt_regs->ss */
835 pushl $0 /* pt_regs->sp (placeholder) */
836 pushfl /* pt_regs->flags (except IF = 0) */
837 pushl $__USER_CS /* pt_regs->cs */
838 pushl $0 /* pt_regs->ip = 0 (placeholder) */
839 pushl %eax /* pt_regs->orig_ax */
840 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest, stack already switched */
841
842 /*
843 * SYSENTER doesn't filter flags, so we need to clear NT, AC
844 * and TF ourselves. To save a few cycles, we can check whether
845 * either was set instead of doing an unconditional popfq.
846 * This needs to happen before enabling interrupts so that
847 * we don't get preempted with NT set.
848 *
849 * If TF is set, we will single-step all the way to here -- do_debug
850 * will ignore all the traps. (Yes, this is slow, but so is
851 * single-stepping in general. This allows us to avoid having
852 * a more complicated code to handle the case where a user program
853 * forces us to single-step through the SYSENTER entry code.)
854 *
855 * NB.: .Lsysenter_fix_flags is a label with the code under it moved
856 * out-of-line as an optimization: NT is unlikely to be set in the
857 * majority of the cases and instead of polluting the I$ unnecessarily,
858 * we're keeping that code behind a branch which will predict as
859 * not-taken and therefore its instructions won't be fetched.
860 */
861 testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp)
862 jnz .Lsysenter_fix_flags
863.Lsysenter_flags_fixed:
864
865 movl %esp, %eax
866 call do_SYSENTER_32
867 testl %eax, %eax
868 jz .Lsyscall_32_done
869
870 STACKLEAK_ERASE
871
872 /* Opportunistic SYSEXIT */
873
874 /*
875 * Setup entry stack - we keep the pointer in %eax and do the
876 * switch after almost all user-state is restored.
877 */
878
879 /* Load entry stack pointer and allocate frame for eflags/eax */
880 movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %eax
881 subl $(2*4), %eax
882
883 /* Copy eflags and eax to entry stack */
884 movl PT_EFLAGS(%esp), %edi
885 movl PT_EAX(%esp), %esi
886 movl %edi, (%eax)
887 movl %esi, 4(%eax)
888
889 /* Restore user registers and segments */
890 movl PT_EIP(%esp), %edx /* pt_regs->ip */
891 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */
8921: mov PT_FS(%esp), %fs
893
894 popl %ebx /* pt_regs->bx */
895 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */
896 popl %esi /* pt_regs->si */
897 popl %edi /* pt_regs->di */
898 popl %ebp /* pt_regs->bp */
899
900 /* Switch to entry stack */
901 movl %eax, %esp
902
903 /* Now ready to switch the cr3 */
904 SWITCH_TO_USER_CR3 scratch_reg=%eax
905
906 /*
907 * Restore all flags except IF. (We restore IF separately because
908 * STI gives a one-instruction window in which we won't be interrupted,
909 * whereas POPF does not.)
910 */
911 btrl $X86_EFLAGS_IF_BIT, (%esp)
912 BUG_IF_WRONG_CR3 no_user_check=1
913 popfl
914 popl %eax
915
916 /*
917 * Return back to the vDSO, which will pop ecx and edx.
918 * Don't bother with DS and ES (they already contain __USER_DS).
919 */
920 sti
921 sysexit
922
9232: movl $0, PT_FS(%esp)
924 jmp 1b
925 _ASM_EXTABLE(1b, 2b)
926
927.Lsysenter_fix_flags:
928 pushl $X86_EFLAGS_FIXED
929 popfl
930 jmp .Lsysenter_flags_fixed
931SYM_ENTRY(__end_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE)
932SYM_FUNC_END(entry_SYSENTER_32)
933
934/*
935 * 32-bit legacy system call entry.
936 *
937 * 32-bit x86 Linux system calls traditionally used the INT $0x80
938 * instruction. INT $0x80 lands here.
939 *
940 * This entry point can be used by any 32-bit perform system calls.
941 * Instances of INT $0x80 can be found inline in various programs and
942 * libraries. It is also used by the vDSO's __kernel_vsyscall
943 * fallback for hardware that doesn't support a faster entry method.
944 * Restarted 32-bit system calls also fall back to INT $0x80
945 * regardless of what instruction was originally used to do the system
946 * call. (64-bit programs can use INT $0x80 as well, but they can
947 * only run on 64-bit kernels and therefore land in
948 * entry_INT80_compat.)
949 *
950 * This is considered a slow path. It is not used by most libc
951 * implementations on modern hardware except during process startup.
952 *
953 * Arguments:
954 * eax system call number
955 * ebx arg1
956 * ecx arg2
957 * edx arg3
958 * esi arg4
959 * edi arg5
960 * ebp arg6
961 */
962SYM_FUNC_START(entry_INT80_32)
963 ASM_CLAC
964 pushl %eax /* pt_regs->orig_ax */
965
966 SAVE_ALL pt_regs_ax=$-ENOSYS switch_stacks=1 /* save rest */
967
968 movl %esp, %eax
969 call do_int80_syscall_32
970.Lsyscall_32_done:
971 STACKLEAK_ERASE
972
973restore_all_switch_stack:
974 SWITCH_TO_ENTRY_STACK
975 CHECK_AND_APPLY_ESPFIX
976
977 /* Switch back to user CR3 */
978 SWITCH_TO_USER_CR3 scratch_reg=%eax
979
980 BUG_IF_WRONG_CR3
981
982 /* Restore user state */
983 RESTORE_REGS pop=4 # skip orig_eax/error_code
984.Lirq_return:
985 /*
986 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
987 * when returning from IPI handler and when returning from
988 * scheduler to user-space.
989 */
990 iret
991
992.Lasm_iret_error:
993 pushl $0 # no error code
994 pushl $iret_error
995
996#ifdef CONFIG_DEBUG_ENTRY
997 /*
998 * The stack-frame here is the one that iret faulted on, so its a
999 * return-to-user frame. We are on kernel-cr3 because we come here from
1000 * the fixup code. This confuses the CR3 checker, so switch to user-cr3
1001 * as the checker expects it.
1002 */
1003 pushl %eax
1004 SWITCH_TO_USER_CR3 scratch_reg=%eax
1005 popl %eax
1006#endif
1007
1008 jmp handle_exception
1009
1010 _ASM_EXTABLE(.Lirq_return, .Lasm_iret_error)
1011SYM_FUNC_END(entry_INT80_32)
1012
1013.macro FIXUP_ESPFIX_STACK
1014/*
1015 * Switch back for ESPFIX stack to the normal zerobased stack
1016 *
1017 * We can't call C functions using the ESPFIX stack. This code reads
1018 * the high word of the segment base from the GDT and swiches to the
1019 * normal stack and adjusts ESP with the matching offset.
1020 *
1021 * We might be on user CR3 here, so percpu data is not mapped and we can't
1022 * access the GDT through the percpu segment. Instead, use SGDT to find
1023 * the cpu_entry_area alias of the GDT.
1024 */
1025#ifdef CONFIG_X86_ESPFIX32
1026 /* fixup the stack */
1027 pushl %ecx
1028 subl $2*4, %esp
1029 sgdt (%esp)
1030 movl 2(%esp), %ecx /* GDT address */
1031 /*
1032 * Careful: ECX is a linear pointer, so we need to force base
1033 * zero. %cs is the only known-linear segment we have right now.
1034 */
1035 mov %cs:GDT_ESPFIX_OFFSET + 4(%ecx), %al /* bits 16..23 */
1036 mov %cs:GDT_ESPFIX_OFFSET + 7(%ecx), %ah /* bits 24..31 */
1037 shl $16, %eax
1038 addl $2*4, %esp
1039 popl %ecx
1040 addl %esp, %eax /* the adjusted stack pointer */
1041 pushl $__KERNEL_DS
1042 pushl %eax
1043 lss (%esp), %esp /* switch to the normal stack segment */
1044#endif
1045.endm
1046
1047.macro UNWIND_ESPFIX_STACK
1048 /* It's safe to clobber %eax, all other regs need to be preserved */
1049#ifdef CONFIG_X86_ESPFIX32
1050 movl %ss, %eax
1051 /* see if on espfix stack */
1052 cmpw $__ESPFIX_SS, %ax
1053 jne .Lno_fixup_\@
1054 /* switch to normal stack */
1055 FIXUP_ESPFIX_STACK
1056.Lno_fixup_\@:
1057#endif
1058.endm
1059
1060SYM_CODE_START_LOCAL_NOALIGN(handle_exception)
1061 /* the function address is in %gs's slot on the stack */
1062 SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1
1063 ENCODE_FRAME_POINTER
1064
1065 movl PT_GS(%esp), %edi # get the function address
1066
1067 /* fixup orig %eax */
1068 movl PT_ORIG_EAX(%esp), %edx # get the error code
1069 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
1070
1071 movl %esp, %eax # pt_regs pointer
1072 CALL_NOSPEC edi
1073
1074handle_exception_return:
1075#ifdef CONFIG_VM86
1076 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
1077 movb PT_CS(%esp), %al
1078 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
1079#else
1080 /*
1081 * We can be coming here from child spawned by kernel_thread().
1082 */
1083 movl PT_CS(%esp), %eax
1084 andl $SEGMENT_RPL_MASK, %eax
1085#endif
1086 cmpl $USER_RPL, %eax # returning to v8086 or userspace ?
1087 jnb ret_to_user
1088
1089 PARANOID_EXIT_TO_KERNEL_MODE
1090 BUG_IF_WRONG_CR3
1091 RESTORE_REGS 4
1092 jmp .Lirq_return
1093
1094ret_to_user:
1095 movl %esp, %eax
1096 jmp restore_all_switch_stack
1097SYM_CODE_END(handle_exception)
1098
1099SYM_CODE_START(asm_exc_double_fault)
11001:
1101 /*
1102 * This is a task gate handler, not an interrupt gate handler.
1103 * The error code is on the stack, but the stack is otherwise
1104 * empty. Interrupts are off. Our state is sane with the following
1105 * exceptions:
1106 *
1107 * - CR0.TS is set. "TS" literally means "task switched".
1108 * - EFLAGS.NT is set because we're a "nested task".
1109 * - The doublefault TSS has back_link set and has been marked busy.
1110 * - TR points to the doublefault TSS and the normal TSS is busy.
1111 * - CR3 is the normal kernel PGD. This would be delightful, except
1112 * that the CPU didn't bother to save the old CR3 anywhere. This
1113 * would make it very awkward to return back to the context we came
1114 * from.
1115 *
1116 * The rest of EFLAGS is sanitized for us, so we don't need to
1117 * worry about AC or DF.
1118 *
1119 * Don't even bother popping the error code. It's always zero,
1120 * and ignoring it makes us a bit more robust against buggy
1121 * hypervisor task gate implementations.
1122 *
1123 * We will manually undo the task switch instead of doing a
1124 * task-switching IRET.
1125 */
1126
1127 clts /* clear CR0.TS */
1128 pushl $X86_EFLAGS_FIXED
1129 popfl /* clear EFLAGS.NT */
1130
1131 call doublefault_shim
1132
1133 /* We don't support returning, so we have no IRET here. */
11341:
1135 hlt
1136 jmp 1b
1137SYM_CODE_END(asm_exc_double_fault)
1138
1139/*
1140 * NMI is doubly nasty. It can happen on the first instruction of
1141 * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning
1142 * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32
1143 * switched stacks. We handle both conditions by simply checking whether we
1144 * interrupted kernel code running on the SYSENTER stack.
1145 */
1146SYM_CODE_START(asm_exc_nmi)
1147 ASM_CLAC
1148
1149#ifdef CONFIG_X86_ESPFIX32
1150 /*
1151 * ESPFIX_SS is only ever set on the return to user path
1152 * after we've switched to the entry stack.
1153 */
1154 pushl %eax
1155 movl %ss, %eax
1156 cmpw $__ESPFIX_SS, %ax
1157 popl %eax
1158 je .Lnmi_espfix_stack
1159#endif
1160
1161 pushl %eax # pt_regs->orig_ax
1162 SAVE_ALL_NMI cr3_reg=%edi
1163 ENCODE_FRAME_POINTER
1164 xorl %edx, %edx # zero error code
1165 movl %esp, %eax # pt_regs pointer
1166
1167 /* Are we currently on the SYSENTER stack? */
1168 movl PER_CPU_VAR(cpu_entry_area), %ecx
1169 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
1170 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */
1171 cmpl $SIZEOF_entry_stack, %ecx
1172 jb .Lnmi_from_sysenter_stack
1173
1174 /* Not on SYSENTER stack. */
1175 call exc_nmi
1176 jmp .Lnmi_return
1177
1178.Lnmi_from_sysenter_stack:
1179 /*
1180 * We're on the SYSENTER stack. Switch off. No one (not even debug)
1181 * is using the thread stack right now, so it's safe for us to use it.
1182 */
1183 movl %esp, %ebx
1184 movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esp
1185 call exc_nmi
1186 movl %ebx, %esp
1187
1188.Lnmi_return:
1189#ifdef CONFIG_X86_ESPFIX32
1190 testl $CS_FROM_ESPFIX, PT_CS(%esp)
1191 jnz .Lnmi_from_espfix
1192#endif
1193
1194 CHECK_AND_APPLY_ESPFIX
1195 RESTORE_ALL_NMI cr3_reg=%edi pop=4
1196 jmp .Lirq_return
1197
1198#ifdef CONFIG_X86_ESPFIX32
1199.Lnmi_espfix_stack:
1200 /*
1201 * Create the pointer to LSS back
1202 */
1203 pushl %ss
1204 pushl %esp
1205 addl $4, (%esp)
1206
1207 /* Copy the (short) IRET frame */
1208 pushl 4*4(%esp) # flags
1209 pushl 4*4(%esp) # cs
1210 pushl 4*4(%esp) # ip
1211
1212 pushl %eax # orig_ax
1213
1214 SAVE_ALL_NMI cr3_reg=%edi unwind_espfix=1
1215 ENCODE_FRAME_POINTER
1216
1217 /* clear CS_FROM_KERNEL, set CS_FROM_ESPFIX */
1218 xorl $(CS_FROM_ESPFIX | CS_FROM_KERNEL), PT_CS(%esp)
1219
1220 xorl %edx, %edx # zero error code
1221 movl %esp, %eax # pt_regs pointer
1222 jmp .Lnmi_from_sysenter_stack
1223
1224.Lnmi_from_espfix:
1225 RESTORE_ALL_NMI cr3_reg=%edi
1226 /*
1227 * Because we cleared CS_FROM_KERNEL, IRET_FRAME 'forgot' to
1228 * fix up the gap and long frame:
1229 *
1230 * 3 - original frame (exception)
1231 * 2 - ESPFIX block (above)
1232 * 6 - gap (FIXUP_FRAME)
1233 * 5 - long frame (FIXUP_FRAME)
1234 * 1 - orig_ax
1235 */
1236 lss (1+5+6)*4(%esp), %esp # back to espfix stack
1237 jmp .Lirq_return
1238#endif
1239SYM_CODE_END(asm_exc_nmi)
1240
1241.pushsection .text, "ax"
1242SYM_CODE_START(rewind_stack_and_make_dead)
1243 /* Prevent any naive code from trying to unwind to our caller. */
1244 xorl %ebp, %ebp
1245
1246 movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esi
1247 leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp
1248
1249 call make_task_dead
12501: jmp 1b
1251SYM_CODE_END(rewind_stack_and_make_dead)
1252.popsection
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 1991,1992 Linus Torvalds
4 *
5 * entry_32.S contains the system-call and low-level fault and trap handling routines.
6 *
7 * Stack layout while running C code:
8 * ptrace needs to have all registers on the stack.
9 * If the order here is changed, it needs to be
10 * updated in fork.c:copy_process(), signal.c:do_signal(),
11 * ptrace.c and ptrace.h
12 *
13 * 0(%esp) - %ebx
14 * 4(%esp) - %ecx
15 * 8(%esp) - %edx
16 * C(%esp) - %esi
17 * 10(%esp) - %edi
18 * 14(%esp) - %ebp
19 * 18(%esp) - %eax
20 * 1C(%esp) - %ds
21 * 20(%esp) - %es
22 * 24(%esp) - %fs
23 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS
24 * 2C(%esp) - orig_eax
25 * 30(%esp) - %eip
26 * 34(%esp) - %cs
27 * 38(%esp) - %eflags
28 * 3C(%esp) - %oldesp
29 * 40(%esp) - %oldss
30 */
31
32#include <linux/linkage.h>
33#include <linux/err.h>
34#include <asm/thread_info.h>
35#include <asm/irqflags.h>
36#include <asm/errno.h>
37#include <asm/segment.h>
38#include <asm/smp.h>
39#include <asm/percpu.h>
40#include <asm/processor-flags.h>
41#include <asm/irq_vectors.h>
42#include <asm/cpufeatures.h>
43#include <asm/alternative-asm.h>
44#include <asm/asm.h>
45#include <asm/smap.h>
46#include <asm/frame.h>
47#include <asm/nospec-branch.h>
48
49 .section .entry.text, "ax"
50
51/*
52 * We use macros for low-level operations which need to be overridden
53 * for paravirtualization. The following will never clobber any registers:
54 * INTERRUPT_RETURN (aka. "iret")
55 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
56 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
57 *
58 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
59 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
60 * Allowing a register to be clobbered can shrink the paravirt replacement
61 * enough to patch inline, increasing performance.
62 */
63
64#ifdef CONFIG_PREEMPT
65# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
66#else
67# define preempt_stop(clobbers)
68# define resume_kernel restore_all
69#endif
70
71.macro TRACE_IRQS_IRET
72#ifdef CONFIG_TRACE_IRQFLAGS
73 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off?
74 jz 1f
75 TRACE_IRQS_ON
761:
77#endif
78.endm
79
80/*
81 * User gs save/restore
82 *
83 * %gs is used for userland TLS and kernel only uses it for stack
84 * canary which is required to be at %gs:20 by gcc. Read the comment
85 * at the top of stackprotector.h for more info.
86 *
87 * Local labels 98 and 99 are used.
88 */
89#ifdef CONFIG_X86_32_LAZY_GS
90
91 /* unfortunately push/pop can't be no-op */
92.macro PUSH_GS
93 pushl $0
94.endm
95.macro POP_GS pop=0
96 addl $(4 + \pop), %esp
97.endm
98.macro POP_GS_EX
99.endm
100
101 /* all the rest are no-op */
102.macro PTGS_TO_GS
103.endm
104.macro PTGS_TO_GS_EX
105.endm
106.macro GS_TO_REG reg
107.endm
108.macro REG_TO_PTGS reg
109.endm
110.macro SET_KERNEL_GS reg
111.endm
112
113#else /* CONFIG_X86_32_LAZY_GS */
114
115.macro PUSH_GS
116 pushl %gs
117.endm
118
119.macro POP_GS pop=0
12098: popl %gs
121 .if \pop <> 0
122 add $\pop, %esp
123 .endif
124.endm
125.macro POP_GS_EX
126.pushsection .fixup, "ax"
12799: movl $0, (%esp)
128 jmp 98b
129.popsection
130 _ASM_EXTABLE(98b, 99b)
131.endm
132
133.macro PTGS_TO_GS
13498: mov PT_GS(%esp), %gs
135.endm
136.macro PTGS_TO_GS_EX
137.pushsection .fixup, "ax"
13899: movl $0, PT_GS(%esp)
139 jmp 98b
140.popsection
141 _ASM_EXTABLE(98b, 99b)
142.endm
143
144.macro GS_TO_REG reg
145 movl %gs, \reg
146.endm
147.macro REG_TO_PTGS reg
148 movl \reg, PT_GS(%esp)
149.endm
150.macro SET_KERNEL_GS reg
151 movl $(__KERNEL_STACK_CANARY), \reg
152 movl \reg, %gs
153.endm
154
155#endif /* CONFIG_X86_32_LAZY_GS */
156
157.macro SAVE_ALL pt_regs_ax=%eax
158 cld
159 PUSH_GS
160 pushl %fs
161 pushl %es
162 pushl %ds
163 pushl \pt_regs_ax
164 pushl %ebp
165 pushl %edi
166 pushl %esi
167 pushl %edx
168 pushl %ecx
169 pushl %ebx
170 movl $(__USER_DS), %edx
171 movl %edx, %ds
172 movl %edx, %es
173 movl $(__KERNEL_PERCPU), %edx
174 movl %edx, %fs
175 SET_KERNEL_GS %edx
176.endm
177
178/*
179 * This is a sneaky trick to help the unwinder find pt_regs on the stack. The
180 * frame pointer is replaced with an encoded pointer to pt_regs. The encoding
181 * is just clearing the MSB, which makes it an invalid stack address and is also
182 * a signal to the unwinder that it's a pt_regs pointer in disguise.
183 *
184 * NOTE: This macro must be used *after* SAVE_ALL because it corrupts the
185 * original rbp.
186 */
187.macro ENCODE_FRAME_POINTER
188#ifdef CONFIG_FRAME_POINTER
189 mov %esp, %ebp
190 andl $0x7fffffff, %ebp
191#endif
192.endm
193
194.macro RESTORE_INT_REGS
195 popl %ebx
196 popl %ecx
197 popl %edx
198 popl %esi
199 popl %edi
200 popl %ebp
201 popl %eax
202.endm
203
204.macro RESTORE_REGS pop=0
205 RESTORE_INT_REGS
2061: popl %ds
2072: popl %es
2083: popl %fs
209 POP_GS \pop
210.pushsection .fixup, "ax"
2114: movl $0, (%esp)
212 jmp 1b
2135: movl $0, (%esp)
214 jmp 2b
2156: movl $0, (%esp)
216 jmp 3b
217.popsection
218 _ASM_EXTABLE(1b, 4b)
219 _ASM_EXTABLE(2b, 5b)
220 _ASM_EXTABLE(3b, 6b)
221 POP_GS_EX
222.endm
223
224/*
225 * %eax: prev task
226 * %edx: next task
227 */
228ENTRY(__switch_to_asm)
229 /*
230 * Save callee-saved registers
231 * This must match the order in struct inactive_task_frame
232 */
233 pushl %ebp
234 pushl %ebx
235 pushl %edi
236 pushl %esi
237
238 /* switch stack */
239 movl %esp, TASK_threadsp(%eax)
240 movl TASK_threadsp(%edx), %esp
241
242#ifdef CONFIG_CC_STACKPROTECTOR
243 movl TASK_stack_canary(%edx), %ebx
244 movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
245#endif
246
247#ifdef CONFIG_RETPOLINE
248 /*
249 * When switching from a shallower to a deeper call stack
250 * the RSB may either underflow or use entries populated
251 * with userspace addresses. On CPUs where those concerns
252 * exist, overwrite the RSB with entries which capture
253 * speculative execution to prevent attack.
254 */
255 FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
256#endif
257
258 /* restore callee-saved registers */
259 popl %esi
260 popl %edi
261 popl %ebx
262 popl %ebp
263
264 jmp __switch_to
265END(__switch_to_asm)
266
267/*
268 * The unwinder expects the last frame on the stack to always be at the same
269 * offset from the end of the page, which allows it to validate the stack.
270 * Calling schedule_tail() directly would break that convention because its an
271 * asmlinkage function so its argument has to be pushed on the stack. This
272 * wrapper creates a proper "end of stack" frame header before the call.
273 */
274ENTRY(schedule_tail_wrapper)
275 FRAME_BEGIN
276
277 pushl %eax
278 call schedule_tail
279 popl %eax
280
281 FRAME_END
282 ret
283ENDPROC(schedule_tail_wrapper)
284/*
285 * A newly forked process directly context switches into this address.
286 *
287 * eax: prev task we switched from
288 * ebx: kernel thread func (NULL for user thread)
289 * edi: kernel thread arg
290 */
291ENTRY(ret_from_fork)
292 call schedule_tail_wrapper
293
294 testl %ebx, %ebx
295 jnz 1f /* kernel threads are uncommon */
296
2972:
298 /* When we fork, we trace the syscall return in the child, too. */
299 movl %esp, %eax
300 call syscall_return_slowpath
301 jmp restore_all
302
303 /* kernel thread */
3041: movl %edi, %eax
305 CALL_NOSPEC %ebx
306 /*
307 * A kernel thread is allowed to return here after successfully
308 * calling do_execve(). Exit to userspace to complete the execve()
309 * syscall.
310 */
311 movl $0, PT_EAX(%esp)
312 jmp 2b
313END(ret_from_fork)
314
315/*
316 * Return to user mode is not as complex as all this looks,
317 * but we want the default path for a system call return to
318 * go as quickly as possible which is why some of this is
319 * less clear than it otherwise should be.
320 */
321
322 # userspace resumption stub bypassing syscall exit tracing
323 ALIGN
324ret_from_exception:
325 preempt_stop(CLBR_ANY)
326ret_from_intr:
327#ifdef CONFIG_VM86
328 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
329 movb PT_CS(%esp), %al
330 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
331#else
332 /*
333 * We can be coming here from child spawned by kernel_thread().
334 */
335 movl PT_CS(%esp), %eax
336 andl $SEGMENT_RPL_MASK, %eax
337#endif
338 cmpl $USER_RPL, %eax
339 jb resume_kernel # not returning to v8086 or userspace
340
341ENTRY(resume_userspace)
342 DISABLE_INTERRUPTS(CLBR_ANY)
343 TRACE_IRQS_OFF
344 movl %esp, %eax
345 call prepare_exit_to_usermode
346 jmp restore_all
347END(ret_from_exception)
348
349#ifdef CONFIG_PREEMPT
350ENTRY(resume_kernel)
351 DISABLE_INTERRUPTS(CLBR_ANY)
352.Lneed_resched:
353 cmpl $0, PER_CPU_VAR(__preempt_count)
354 jnz restore_all
355 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ?
356 jz restore_all
357 call preempt_schedule_irq
358 jmp .Lneed_resched
359END(resume_kernel)
360#endif
361
362GLOBAL(__begin_SYSENTER_singlestep_region)
363/*
364 * All code from here through __end_SYSENTER_singlestep_region is subject
365 * to being single-stepped if a user program sets TF and executes SYSENTER.
366 * There is absolutely nothing that we can do to prevent this from happening
367 * (thanks Intel!). To keep our handling of this situation as simple as
368 * possible, we handle TF just like AC and NT, except that our #DB handler
369 * will ignore all of the single-step traps generated in this range.
370 */
371
372#ifdef CONFIG_XEN
373/*
374 * Xen doesn't set %esp to be precisely what the normal SYSENTER
375 * entry point expects, so fix it up before using the normal path.
376 */
377ENTRY(xen_sysenter_target)
378 addl $5*4, %esp /* remove xen-provided frame */
379 jmp .Lsysenter_past_esp
380#endif
381
382/*
383 * 32-bit SYSENTER entry.
384 *
385 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
386 * if X86_FEATURE_SEP is available. This is the preferred system call
387 * entry on 32-bit systems.
388 *
389 * The SYSENTER instruction, in principle, should *only* occur in the
390 * vDSO. In practice, a small number of Android devices were shipped
391 * with a copy of Bionic that inlined a SYSENTER instruction. This
392 * never happened in any of Google's Bionic versions -- it only happened
393 * in a narrow range of Intel-provided versions.
394 *
395 * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs.
396 * IF and VM in RFLAGS are cleared (IOW: interrupts are off).
397 * SYSENTER does not save anything on the stack,
398 * and does not save old EIP (!!!), ESP, or EFLAGS.
399 *
400 * To avoid losing track of EFLAGS.VM (and thus potentially corrupting
401 * user and/or vm86 state), we explicitly disable the SYSENTER
402 * instruction in vm86 mode by reprogramming the MSRs.
403 *
404 * Arguments:
405 * eax system call number
406 * ebx arg1
407 * ecx arg2
408 * edx arg3
409 * esi arg4
410 * edi arg5
411 * ebp user stack
412 * 0(%ebp) arg6
413 */
414ENTRY(entry_SYSENTER_32)
415 movl TSS_sysenter_sp0(%esp), %esp
416.Lsysenter_past_esp:
417 pushl $__USER_DS /* pt_regs->ss */
418 pushl %ebp /* pt_regs->sp (stashed in bp) */
419 pushfl /* pt_regs->flags (except IF = 0) */
420 orl $X86_EFLAGS_IF, (%esp) /* Fix IF */
421 pushl $__USER_CS /* pt_regs->cs */
422 pushl $0 /* pt_regs->ip = 0 (placeholder) */
423 pushl %eax /* pt_regs->orig_ax */
424 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */
425
426 /*
427 * SYSENTER doesn't filter flags, so we need to clear NT, AC
428 * and TF ourselves. To save a few cycles, we can check whether
429 * either was set instead of doing an unconditional popfq.
430 * This needs to happen before enabling interrupts so that
431 * we don't get preempted with NT set.
432 *
433 * If TF is set, we will single-step all the way to here -- do_debug
434 * will ignore all the traps. (Yes, this is slow, but so is
435 * single-stepping in general. This allows us to avoid having
436 * a more complicated code to handle the case where a user program
437 * forces us to single-step through the SYSENTER entry code.)
438 *
439 * NB.: .Lsysenter_fix_flags is a label with the code under it moved
440 * out-of-line as an optimization: NT is unlikely to be set in the
441 * majority of the cases and instead of polluting the I$ unnecessarily,
442 * we're keeping that code behind a branch which will predict as
443 * not-taken and therefore its instructions won't be fetched.
444 */
445 testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp)
446 jnz .Lsysenter_fix_flags
447.Lsysenter_flags_fixed:
448
449 /*
450 * User mode is traced as though IRQs are on, and SYSENTER
451 * turned them off.
452 */
453 TRACE_IRQS_OFF
454
455 movl %esp, %eax
456 call do_fast_syscall_32
457 /* XEN PV guests always use IRET path */
458 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \
459 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV
460
461/* Opportunistic SYSEXIT */
462 TRACE_IRQS_ON /* User mode traces as IRQs on. */
463 movl PT_EIP(%esp), %edx /* pt_regs->ip */
464 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */
4651: mov PT_FS(%esp), %fs
466 PTGS_TO_GS
467 popl %ebx /* pt_regs->bx */
468 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */
469 popl %esi /* pt_regs->si */
470 popl %edi /* pt_regs->di */
471 popl %ebp /* pt_regs->bp */
472 popl %eax /* pt_regs->ax */
473
474 /*
475 * Restore all flags except IF. (We restore IF separately because
476 * STI gives a one-instruction window in which we won't be interrupted,
477 * whereas POPF does not.)
478 */
479 addl $PT_EFLAGS-PT_DS, %esp /* point esp at pt_regs->flags */
480 btr $X86_EFLAGS_IF_BIT, (%esp)
481 popfl
482
483 /*
484 * Return back to the vDSO, which will pop ecx and edx.
485 * Don't bother with DS and ES (they already contain __USER_DS).
486 */
487 sti
488 sysexit
489
490.pushsection .fixup, "ax"
4912: movl $0, PT_FS(%esp)
492 jmp 1b
493.popsection
494 _ASM_EXTABLE(1b, 2b)
495 PTGS_TO_GS_EX
496
497.Lsysenter_fix_flags:
498 pushl $X86_EFLAGS_FIXED
499 popfl
500 jmp .Lsysenter_flags_fixed
501GLOBAL(__end_SYSENTER_singlestep_region)
502ENDPROC(entry_SYSENTER_32)
503
504/*
505 * 32-bit legacy system call entry.
506 *
507 * 32-bit x86 Linux system calls traditionally used the INT $0x80
508 * instruction. INT $0x80 lands here.
509 *
510 * This entry point can be used by any 32-bit perform system calls.
511 * Instances of INT $0x80 can be found inline in various programs and
512 * libraries. It is also used by the vDSO's __kernel_vsyscall
513 * fallback for hardware that doesn't support a faster entry method.
514 * Restarted 32-bit system calls also fall back to INT $0x80
515 * regardless of what instruction was originally used to do the system
516 * call. (64-bit programs can use INT $0x80 as well, but they can
517 * only run on 64-bit kernels and therefore land in
518 * entry_INT80_compat.)
519 *
520 * This is considered a slow path. It is not used by most libc
521 * implementations on modern hardware except during process startup.
522 *
523 * Arguments:
524 * eax system call number
525 * ebx arg1
526 * ecx arg2
527 * edx arg3
528 * esi arg4
529 * edi arg5
530 * ebp arg6
531 */
532ENTRY(entry_INT80_32)
533 ASM_CLAC
534 pushl %eax /* pt_regs->orig_ax */
535 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */
536
537 /*
538 * User mode is traced as though IRQs are on, and the interrupt gate
539 * turned them off.
540 */
541 TRACE_IRQS_OFF
542
543 movl %esp, %eax
544 call do_int80_syscall_32
545.Lsyscall_32_done:
546
547restore_all:
548 TRACE_IRQS_IRET
549.Lrestore_all_notrace:
550#ifdef CONFIG_X86_ESPFIX32
551 ALTERNATIVE "jmp .Lrestore_nocheck", "", X86_BUG_ESPFIX
552
553 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
554 /*
555 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we
556 * are returning to the kernel.
557 * See comments in process.c:copy_thread() for details.
558 */
559 movb PT_OLDSS(%esp), %ah
560 movb PT_CS(%esp), %al
561 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
562 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
563 je .Lldt_ss # returning to user-space with LDT SS
564#endif
565.Lrestore_nocheck:
566 RESTORE_REGS 4 # skip orig_eax/error_code
567.Lirq_return:
568 /*
569 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
570 * when returning from IPI handler and when returning from
571 * scheduler to user-space.
572 */
573 INTERRUPT_RETURN
574
575.section .fixup, "ax"
576ENTRY(iret_exc )
577 pushl $0 # no error code
578 pushl $do_iret_error
579 jmp common_exception
580.previous
581 _ASM_EXTABLE(.Lirq_return, iret_exc)
582
583#ifdef CONFIG_X86_ESPFIX32
584.Lldt_ss:
585/*
586 * Setup and switch to ESPFIX stack
587 *
588 * We're returning to userspace with a 16 bit stack. The CPU will not
589 * restore the high word of ESP for us on executing iret... This is an
590 * "official" bug of all the x86-compatible CPUs, which we can work
591 * around to make dosemu and wine happy. We do this by preloading the
592 * high word of ESP with the high word of the userspace ESP while
593 * compensating for the offset by changing to the ESPFIX segment with
594 * a base address that matches for the difference.
595 */
596#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
597 mov %esp, %edx /* load kernel esp */
598 mov PT_OLDESP(%esp), %eax /* load userspace esp */
599 mov %dx, %ax /* eax: new kernel esp */
600 sub %eax, %edx /* offset (low word is 0) */
601 shr $16, %edx
602 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
603 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
604 pushl $__ESPFIX_SS
605 pushl %eax /* new kernel esp */
606 /*
607 * Disable interrupts, but do not irqtrace this section: we
608 * will soon execute iret and the tracer was already set to
609 * the irqstate after the IRET:
610 */
611 DISABLE_INTERRUPTS(CLBR_ANY)
612 lss (%esp), %esp /* switch to espfix segment */
613 jmp .Lrestore_nocheck
614#endif
615ENDPROC(entry_INT80_32)
616
617.macro FIXUP_ESPFIX_STACK
618/*
619 * Switch back for ESPFIX stack to the normal zerobased stack
620 *
621 * We can't call C functions using the ESPFIX stack. This code reads
622 * the high word of the segment base from the GDT and swiches to the
623 * normal stack and adjusts ESP with the matching offset.
624 */
625#ifdef CONFIG_X86_ESPFIX32
626 /* fixup the stack */
627 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
628 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
629 shl $16, %eax
630 addl %esp, %eax /* the adjusted stack pointer */
631 pushl $__KERNEL_DS
632 pushl %eax
633 lss (%esp), %esp /* switch to the normal stack segment */
634#endif
635.endm
636.macro UNWIND_ESPFIX_STACK
637#ifdef CONFIG_X86_ESPFIX32
638 movl %ss, %eax
639 /* see if on espfix stack */
640 cmpw $__ESPFIX_SS, %ax
641 jne 27f
642 movl $__KERNEL_DS, %eax
643 movl %eax, %ds
644 movl %eax, %es
645 /* switch to normal stack */
646 FIXUP_ESPFIX_STACK
64727:
648#endif
649.endm
650
651/*
652 * Build the entry stubs with some assembler magic.
653 * We pack 1 stub into every 8-byte block.
654 */
655 .align 8
656ENTRY(irq_entries_start)
657 vector=FIRST_EXTERNAL_VECTOR
658 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
659 pushl $(~vector+0x80) /* Note: always in signed byte range */
660 vector=vector+1
661 jmp common_interrupt
662 .align 8
663 .endr
664END(irq_entries_start)
665
666/*
667 * the CPU automatically disables interrupts when executing an IRQ vector,
668 * so IRQ-flags tracing has to follow that:
669 */
670 .p2align CONFIG_X86_L1_CACHE_SHIFT
671common_interrupt:
672 ASM_CLAC
673 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */
674 SAVE_ALL
675 ENCODE_FRAME_POINTER
676 TRACE_IRQS_OFF
677 movl %esp, %eax
678 call do_IRQ
679 jmp ret_from_intr
680ENDPROC(common_interrupt)
681
682#define BUILD_INTERRUPT3(name, nr, fn) \
683ENTRY(name) \
684 ASM_CLAC; \
685 pushl $~(nr); \
686 SAVE_ALL; \
687 ENCODE_FRAME_POINTER; \
688 TRACE_IRQS_OFF \
689 movl %esp, %eax; \
690 call fn; \
691 jmp ret_from_intr; \
692ENDPROC(name)
693
694#define BUILD_INTERRUPT(name, nr) \
695 BUILD_INTERRUPT3(name, nr, smp_##name); \
696
697/* The include is where all of the SMP etc. interrupts come from */
698#include <asm/entry_arch.h>
699
700ENTRY(coprocessor_error)
701 ASM_CLAC
702 pushl $0
703 pushl $do_coprocessor_error
704 jmp common_exception
705END(coprocessor_error)
706
707ENTRY(simd_coprocessor_error)
708 ASM_CLAC
709 pushl $0
710#ifdef CONFIG_X86_INVD_BUG
711 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
712 ALTERNATIVE "pushl $do_general_protection", \
713 "pushl $do_simd_coprocessor_error", \
714 X86_FEATURE_XMM
715#else
716 pushl $do_simd_coprocessor_error
717#endif
718 jmp common_exception
719END(simd_coprocessor_error)
720
721ENTRY(device_not_available)
722 ASM_CLAC
723 pushl $-1 # mark this as an int
724 pushl $do_device_not_available
725 jmp common_exception
726END(device_not_available)
727
728#ifdef CONFIG_PARAVIRT
729ENTRY(native_iret)
730 iret
731 _ASM_EXTABLE(native_iret, iret_exc)
732END(native_iret)
733#endif
734
735ENTRY(overflow)
736 ASM_CLAC
737 pushl $0
738 pushl $do_overflow
739 jmp common_exception
740END(overflow)
741
742ENTRY(bounds)
743 ASM_CLAC
744 pushl $0
745 pushl $do_bounds
746 jmp common_exception
747END(bounds)
748
749ENTRY(invalid_op)
750 ASM_CLAC
751 pushl $0
752 pushl $do_invalid_op
753 jmp common_exception
754END(invalid_op)
755
756ENTRY(coprocessor_segment_overrun)
757 ASM_CLAC
758 pushl $0
759 pushl $do_coprocessor_segment_overrun
760 jmp common_exception
761END(coprocessor_segment_overrun)
762
763ENTRY(invalid_TSS)
764 ASM_CLAC
765 pushl $do_invalid_TSS
766 jmp common_exception
767END(invalid_TSS)
768
769ENTRY(segment_not_present)
770 ASM_CLAC
771 pushl $do_segment_not_present
772 jmp common_exception
773END(segment_not_present)
774
775ENTRY(stack_segment)
776 ASM_CLAC
777 pushl $do_stack_segment
778 jmp common_exception
779END(stack_segment)
780
781ENTRY(alignment_check)
782 ASM_CLAC
783 pushl $do_alignment_check
784 jmp common_exception
785END(alignment_check)
786
787ENTRY(divide_error)
788 ASM_CLAC
789 pushl $0 # no error code
790 pushl $do_divide_error
791 jmp common_exception
792END(divide_error)
793
794#ifdef CONFIG_X86_MCE
795ENTRY(machine_check)
796 ASM_CLAC
797 pushl $0
798 pushl machine_check_vector
799 jmp common_exception
800END(machine_check)
801#endif
802
803ENTRY(spurious_interrupt_bug)
804 ASM_CLAC
805 pushl $0
806 pushl $do_spurious_interrupt_bug
807 jmp common_exception
808END(spurious_interrupt_bug)
809
810#ifdef CONFIG_XEN
811ENTRY(xen_hypervisor_callback)
812 pushl $-1 /* orig_ax = -1 => not a system call */
813 SAVE_ALL
814 ENCODE_FRAME_POINTER
815 TRACE_IRQS_OFF
816
817 /*
818 * Check to see if we got the event in the critical
819 * region in xen_iret_direct, after we've reenabled
820 * events and checked for pending events. This simulates
821 * iret instruction's behaviour where it delivers a
822 * pending interrupt when enabling interrupts:
823 */
824 movl PT_EIP(%esp), %eax
825 cmpl $xen_iret_start_crit, %eax
826 jb 1f
827 cmpl $xen_iret_end_crit, %eax
828 jae 1f
829
830 jmp xen_iret_crit_fixup
831
832ENTRY(xen_do_upcall)
8331: mov %esp, %eax
834 call xen_evtchn_do_upcall
835#ifndef CONFIG_PREEMPT
836 call xen_maybe_preempt_hcall
837#endif
838 jmp ret_from_intr
839ENDPROC(xen_hypervisor_callback)
840
841/*
842 * Hypervisor uses this for application faults while it executes.
843 * We get here for two reasons:
844 * 1. Fault while reloading DS, ES, FS or GS
845 * 2. Fault while executing IRET
846 * Category 1 we fix up by reattempting the load, and zeroing the segment
847 * register if the load fails.
848 * Category 2 we fix up by jumping to do_iret_error. We cannot use the
849 * normal Linux return path in this case because if we use the IRET hypercall
850 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
851 * We distinguish between categories by maintaining a status value in EAX.
852 */
853ENTRY(xen_failsafe_callback)
854 pushl %eax
855 movl $1, %eax
8561: mov 4(%esp), %ds
8572: mov 8(%esp), %es
8583: mov 12(%esp), %fs
8594: mov 16(%esp), %gs
860 /* EAX == 0 => Category 1 (Bad segment)
861 EAX != 0 => Category 2 (Bad IRET) */
862 testl %eax, %eax
863 popl %eax
864 lea 16(%esp), %esp
865 jz 5f
866 jmp iret_exc
8675: pushl $-1 /* orig_ax = -1 => not a system call */
868 SAVE_ALL
869 ENCODE_FRAME_POINTER
870 jmp ret_from_exception
871
872.section .fixup, "ax"
8736: xorl %eax, %eax
874 movl %eax, 4(%esp)
875 jmp 1b
8767: xorl %eax, %eax
877 movl %eax, 8(%esp)
878 jmp 2b
8798: xorl %eax, %eax
880 movl %eax, 12(%esp)
881 jmp 3b
8829: xorl %eax, %eax
883 movl %eax, 16(%esp)
884 jmp 4b
885.previous
886 _ASM_EXTABLE(1b, 6b)
887 _ASM_EXTABLE(2b, 7b)
888 _ASM_EXTABLE(3b, 8b)
889 _ASM_EXTABLE(4b, 9b)
890ENDPROC(xen_failsafe_callback)
891
892BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
893 xen_evtchn_do_upcall)
894
895#endif /* CONFIG_XEN */
896
897#if IS_ENABLED(CONFIG_HYPERV)
898
899BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
900 hyperv_vector_handler)
901
902BUILD_INTERRUPT3(hyperv_reenlightenment_vector, HYPERV_REENLIGHTENMENT_VECTOR,
903 hyperv_reenlightenment_intr)
904
905BUILD_INTERRUPT3(hv_stimer0_callback_vector, HYPERV_STIMER0_VECTOR,
906 hv_stimer0_vector_handler)
907
908#endif /* CONFIG_HYPERV */
909
910ENTRY(page_fault)
911 ASM_CLAC
912 pushl $do_page_fault
913 ALIGN
914 jmp common_exception
915END(page_fault)
916
917common_exception:
918 /* the function address is in %gs's slot on the stack */
919 pushl %fs
920 pushl %es
921 pushl %ds
922 pushl %eax
923 pushl %ebp
924 pushl %edi
925 pushl %esi
926 pushl %edx
927 pushl %ecx
928 pushl %ebx
929 ENCODE_FRAME_POINTER
930 cld
931 movl $(__KERNEL_PERCPU), %ecx
932 movl %ecx, %fs
933 UNWIND_ESPFIX_STACK
934 GS_TO_REG %ecx
935 movl PT_GS(%esp), %edi # get the function address
936 movl PT_ORIG_EAX(%esp), %edx # get the error code
937 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
938 REG_TO_PTGS %ecx
939 SET_KERNEL_GS %ecx
940 movl $(__USER_DS), %ecx
941 movl %ecx, %ds
942 movl %ecx, %es
943 TRACE_IRQS_OFF
944 movl %esp, %eax # pt_regs pointer
945 CALL_NOSPEC %edi
946 jmp ret_from_exception
947END(common_exception)
948
949ENTRY(debug)
950 /*
951 * #DB can happen at the first instruction of
952 * entry_SYSENTER_32 or in Xen's SYSENTER prologue. If this
953 * happens, then we will be running on a very small stack. We
954 * need to detect this condition and switch to the thread
955 * stack before calling any C code at all.
956 *
957 * If you edit this code, keep in mind that NMIs can happen in here.
958 */
959 ASM_CLAC
960 pushl $-1 # mark this as an int
961 SAVE_ALL
962 ENCODE_FRAME_POINTER
963 xorl %edx, %edx # error code 0
964 movl %esp, %eax # pt_regs pointer
965
966 /* Are we currently on the SYSENTER stack? */
967 movl PER_CPU_VAR(cpu_entry_area), %ecx
968 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
969 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */
970 cmpl $SIZEOF_entry_stack, %ecx
971 jb .Ldebug_from_sysenter_stack
972
973 TRACE_IRQS_OFF
974 call do_debug
975 jmp ret_from_exception
976
977.Ldebug_from_sysenter_stack:
978 /* We're on the SYSENTER stack. Switch off. */
979 movl %esp, %ebx
980 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp
981 TRACE_IRQS_OFF
982 call do_debug
983 movl %ebx, %esp
984 jmp ret_from_exception
985END(debug)
986
987/*
988 * NMI is doubly nasty. It can happen on the first instruction of
989 * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning
990 * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32
991 * switched stacks. We handle both conditions by simply checking whether we
992 * interrupted kernel code running on the SYSENTER stack.
993 */
994ENTRY(nmi)
995 ASM_CLAC
996#ifdef CONFIG_X86_ESPFIX32
997 pushl %eax
998 movl %ss, %eax
999 cmpw $__ESPFIX_SS, %ax
1000 popl %eax
1001 je .Lnmi_espfix_stack
1002#endif
1003
1004 pushl %eax # pt_regs->orig_ax
1005 SAVE_ALL
1006 ENCODE_FRAME_POINTER
1007 xorl %edx, %edx # zero error code
1008 movl %esp, %eax # pt_regs pointer
1009
1010 /* Are we currently on the SYSENTER stack? */
1011 movl PER_CPU_VAR(cpu_entry_area), %ecx
1012 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
1013 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */
1014 cmpl $SIZEOF_entry_stack, %ecx
1015 jb .Lnmi_from_sysenter_stack
1016
1017 /* Not on SYSENTER stack. */
1018 call do_nmi
1019 jmp .Lrestore_all_notrace
1020
1021.Lnmi_from_sysenter_stack:
1022 /*
1023 * We're on the SYSENTER stack. Switch off. No one (not even debug)
1024 * is using the thread stack right now, so it's safe for us to use it.
1025 */
1026 movl %esp, %ebx
1027 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp
1028 call do_nmi
1029 movl %ebx, %esp
1030 jmp .Lrestore_all_notrace
1031
1032#ifdef CONFIG_X86_ESPFIX32
1033.Lnmi_espfix_stack:
1034 /*
1035 * create the pointer to lss back
1036 */
1037 pushl %ss
1038 pushl %esp
1039 addl $4, (%esp)
1040 /* copy the iret frame of 12 bytes */
1041 .rept 3
1042 pushl 16(%esp)
1043 .endr
1044 pushl %eax
1045 SAVE_ALL
1046 ENCODE_FRAME_POINTER
1047 FIXUP_ESPFIX_STACK # %eax == %esp
1048 xorl %edx, %edx # zero error code
1049 call do_nmi
1050 RESTORE_REGS
1051 lss 12+4(%esp), %esp # back to espfix stack
1052 jmp .Lirq_return
1053#endif
1054END(nmi)
1055
1056ENTRY(int3)
1057 ASM_CLAC
1058 pushl $-1 # mark this as an int
1059 SAVE_ALL
1060 ENCODE_FRAME_POINTER
1061 TRACE_IRQS_OFF
1062 xorl %edx, %edx # zero error code
1063 movl %esp, %eax # pt_regs pointer
1064 call do_int3
1065 jmp ret_from_exception
1066END(int3)
1067
1068ENTRY(general_protection)
1069 pushl $do_general_protection
1070 jmp common_exception
1071END(general_protection)
1072
1073#ifdef CONFIG_KVM_GUEST
1074ENTRY(async_page_fault)
1075 ASM_CLAC
1076 pushl $do_async_page_fault
1077 jmp common_exception
1078END(async_page_fault)
1079#endif
1080
1081ENTRY(rewind_stack_do_exit)
1082 /* Prevent any naive code from trying to unwind to our caller. */
1083 xorl %ebp, %ebp
1084
1085 movl PER_CPU_VAR(cpu_current_top_of_stack), %esi
1086 leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp
1087
1088 call do_exit
10891: jmp 1b
1090END(rewind_stack_do_exit)