Loading...
1/*
2 * Asm versions of Xen pv-ops, suitable for either direct use or
3 * inlining. The inline versions are the same as the direct-use
4 * versions, with the pre- and post-amble chopped off.
5 *
6 * This code is encoded for size rather than absolute efficiency, with
7 * a view to being able to inline as much as possible.
8 *
9 * We only bother with direct forms (ie, vcpu in percpu data) of the
10 * operations here; the indirect forms are better handled in C, since
11 * they're generally too large to inline anyway.
12 */
13
14#include <asm/asm-offsets.h>
15#include <asm/percpu.h>
16#include <asm/processor-flags.h>
17
18#include "xen-asm.h"
19
20/*
21 * Enable events. This clears the event mask and tests the pending
22 * event status with one and operation. If there are pending events,
23 * then enter the hypervisor to get them handled.
24 */
25ENTRY(xen_irq_enable_direct)
26 /* Unmask events */
27 movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
28
29 /*
30 * Preempt here doesn't matter because that will deal with any
31 * pending interrupts. The pending check may end up being run
32 * on the wrong CPU, but that doesn't hurt.
33 */
34
35 /* Test for pending */
36 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
37 jz 1f
38
392: call check_events
401:
41ENDPATCH(xen_irq_enable_direct)
42 ret
43 ENDPROC(xen_irq_enable_direct)
44 RELOC(xen_irq_enable_direct, 2b+1)
45
46
47/*
48 * Disabling events is simply a matter of making the event mask
49 * non-zero.
50 */
51ENTRY(xen_irq_disable_direct)
52 movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
53ENDPATCH(xen_irq_disable_direct)
54 ret
55 ENDPROC(xen_irq_disable_direct)
56 RELOC(xen_irq_disable_direct, 0)
57
58/*
59 * (xen_)save_fl is used to get the current interrupt enable status.
60 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
61 * may be set in the return value. We take advantage of this by
62 * making sure that X86_EFLAGS_IF has the right value (and other bits
63 * in that byte are 0), but other bits in the return value are
64 * undefined. We need to toggle the state of the bit, because Xen and
65 * x86 use opposite senses (mask vs enable).
66 */
67ENTRY(xen_save_fl_direct)
68 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
69 setz %ah
70 addb %ah, %ah
71ENDPATCH(xen_save_fl_direct)
72 ret
73 ENDPROC(xen_save_fl_direct)
74 RELOC(xen_save_fl_direct, 0)
75
76
77/*
78 * In principle the caller should be passing us a value return from
79 * xen_save_fl_direct, but for robustness sake we test only the
80 * X86_EFLAGS_IF flag rather than the whole byte. After setting the
81 * interrupt mask state, it checks for unmasked pending events and
82 * enters the hypervisor to get them delivered if so.
83 */
84ENTRY(xen_restore_fl_direct)
85#ifdef CONFIG_X86_64
86 testw $X86_EFLAGS_IF, %di
87#else
88 testb $X86_EFLAGS_IF>>8, %ah
89#endif
90 setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
91 /*
92 * Preempt here doesn't matter because that will deal with any
93 * pending interrupts. The pending check may end up being run
94 * on the wrong CPU, but that doesn't hurt.
95 */
96
97 /* check for unmasked and pending */
98 cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
99 jnz 1f
1002: call check_events
1011:
102ENDPATCH(xen_restore_fl_direct)
103 ret
104 ENDPROC(xen_restore_fl_direct)
105 RELOC(xen_restore_fl_direct, 2b+1)
106
107
108/*
109 * Force an event check by making a hypercall, but preserve regs
110 * before making the call.
111 */
112check_events:
113#ifdef CONFIG_X86_32
114 push %eax
115 push %ecx
116 push %edx
117 call xen_force_evtchn_callback
118 pop %edx
119 pop %ecx
120 pop %eax
121#else
122 push %rax
123 push %rcx
124 push %rdx
125 push %rsi
126 push %rdi
127 push %r8
128 push %r9
129 push %r10
130 push %r11
131 call xen_force_evtchn_callback
132 pop %r11
133 pop %r10
134 pop %r9
135 pop %r8
136 pop %rdi
137 pop %rsi
138 pop %rdx
139 pop %rcx
140 pop %rax
141#endif
142 ret
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Asm versions of Xen pv-ops, suitable for direct use.
4 *
5 * We only bother with direct forms (ie, vcpu in percpu data) of the
6 * operations here; the indirect forms are better handled in C.
7 */
8
9#include <asm/errno.h>
10#include <asm/asm-offsets.h>
11#include <asm/percpu.h>
12#include <asm/processor-flags.h>
13#include <asm/segment.h>
14#include <asm/thread_info.h>
15#include <asm/asm.h>
16#include <asm/frame.h>
17#include <asm/unwind_hints.h>
18
19#include <xen/interface/xen.h>
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <../entry/calling.h>
24
25.pushsection .noinstr.text, "ax"
26/*
27 * Disabling events is simply a matter of making the event mask
28 * non-zero.
29 */
30SYM_FUNC_START(xen_irq_disable_direct)
31 movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
32 RET
33SYM_FUNC_END(xen_irq_disable_direct)
34
35/*
36 * Force an event check by making a hypercall, but preserve regs
37 * before making the call.
38 */
39SYM_FUNC_START(check_events)
40 FRAME_BEGIN
41 push %rax
42 push %rcx
43 push %rdx
44 push %rsi
45 push %rdi
46 push %r8
47 push %r9
48 push %r10
49 push %r11
50 call xen_force_evtchn_callback
51 pop %r11
52 pop %r10
53 pop %r9
54 pop %r8
55 pop %rdi
56 pop %rsi
57 pop %rdx
58 pop %rcx
59 pop %rax
60 FRAME_END
61 RET
62SYM_FUNC_END(check_events)
63
64/*
65 * Enable events. This clears the event mask and tests the pending
66 * event status with one and operation. If there are pending events,
67 * then enter the hypervisor to get them handled.
68 */
69SYM_FUNC_START(xen_irq_enable_direct)
70 FRAME_BEGIN
71 /* Unmask events */
72 movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
73
74 /*
75 * Preempt here doesn't matter because that will deal with any
76 * pending interrupts. The pending check may end up being run
77 * on the wrong CPU, but that doesn't hurt.
78 */
79
80 /* Test for pending */
81 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
82 jz 1f
83
84 call check_events
851:
86 FRAME_END
87 RET
88SYM_FUNC_END(xen_irq_enable_direct)
89
90/*
91 * (xen_)save_fl is used to get the current interrupt enable status.
92 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
93 * may be set in the return value. We take advantage of this by
94 * making sure that X86_EFLAGS_IF has the right value (and other bits
95 * in that byte are 0), but other bits in the return value are
96 * undefined. We need to toggle the state of the bit, because Xen and
97 * x86 use opposite senses (mask vs enable).
98 */
99SYM_FUNC_START(xen_save_fl_direct)
100 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
101 setz %ah
102 addb %ah, %ah
103 RET
104SYM_FUNC_END(xen_save_fl_direct)
105
106SYM_FUNC_START(xen_read_cr2)
107 FRAME_BEGIN
108 _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
109 _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
110 FRAME_END
111 RET
112SYM_FUNC_END(xen_read_cr2);
113
114SYM_FUNC_START(xen_read_cr2_direct)
115 FRAME_BEGIN
116 _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
117 FRAME_END
118 RET
119SYM_FUNC_END(xen_read_cr2_direct);
120.popsection
121
122.macro xen_pv_trap name
123SYM_CODE_START(xen_\name)
124 UNWIND_HINT_ENTRY
125 ENDBR
126 pop %rcx
127 pop %r11
128 jmp \name
129SYM_CODE_END(xen_\name)
130_ASM_NOKPROBE(xen_\name)
131.endm
132
133xen_pv_trap asm_exc_divide_error
134xen_pv_trap asm_xenpv_exc_debug
135xen_pv_trap asm_exc_int3
136xen_pv_trap asm_xenpv_exc_nmi
137xen_pv_trap asm_exc_overflow
138xen_pv_trap asm_exc_bounds
139xen_pv_trap asm_exc_invalid_op
140xen_pv_trap asm_exc_device_not_available
141xen_pv_trap asm_xenpv_exc_double_fault
142xen_pv_trap asm_exc_coproc_segment_overrun
143xen_pv_trap asm_exc_invalid_tss
144xen_pv_trap asm_exc_segment_not_present
145xen_pv_trap asm_exc_stack_segment
146xen_pv_trap asm_exc_general_protection
147xen_pv_trap asm_exc_page_fault
148xen_pv_trap asm_exc_spurious_interrupt_bug
149xen_pv_trap asm_exc_coprocessor_error
150xen_pv_trap asm_exc_alignment_check
151#ifdef CONFIG_X86_KERNEL_IBT
152xen_pv_trap asm_exc_control_protection
153#endif
154#ifdef CONFIG_X86_MCE
155xen_pv_trap asm_xenpv_exc_machine_check
156#endif /* CONFIG_X86_MCE */
157xen_pv_trap asm_exc_simd_coprocessor_error
158#ifdef CONFIG_IA32_EMULATION
159xen_pv_trap entry_INT80_compat
160#endif
161xen_pv_trap asm_exc_xen_unknown_trap
162xen_pv_trap asm_exc_xen_hypervisor_callback
163
164 __INIT
165SYM_CODE_START(xen_early_idt_handler_array)
166 i = 0
167 .rept NUM_EXCEPTION_VECTORS
168 UNWIND_HINT_EMPTY
169 ENDBR
170 pop %rcx
171 pop %r11
172 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
173 i = i + 1
174 .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
175 .endr
176SYM_CODE_END(xen_early_idt_handler_array)
177 __FINIT
178
179hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
180/*
181 * Xen64 iret frame:
182 *
183 * ss
184 * rsp
185 * rflags
186 * cs
187 * rip <-- standard iret frame
188 *
189 * flags
190 *
191 * rcx }
192 * r11 }<-- pushed by hypercall page
193 * rsp->rax }
194 */
195SYM_CODE_START(xen_iret)
196 UNWIND_HINT_EMPTY
197 ANNOTATE_NOENDBR
198 pushq $0
199 jmp hypercall_iret
200SYM_CODE_END(xen_iret)
201
202/*
203 * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
204 * also the kernel stack. Reusing swapgs_restore_regs_and_return_to_usermode()
205 * in XEN pv would cause %rsp to move up to the top of the kernel stack and
206 * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
207 * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
208 * frame at the same address is useless.
209 */
210SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
211 UNWIND_HINT_REGS
212 POP_REGS
213
214 /* stackleak_erase() can work safely on the kernel stack. */
215 STACKLEAK_ERASE_NOCLOBBER
216
217 addq $8, %rsp /* skip regs->orig_ax */
218 jmp xen_iret
219SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
220
221/*
222 * Xen handles syscall callbacks much like ordinary exceptions, which
223 * means we have:
224 * - kernel gs
225 * - kernel rsp
226 * - an iret-like stack frame on the stack (including rcx and r11):
227 * ss
228 * rsp
229 * rflags
230 * cs
231 * rip
232 * r11
233 * rsp->rcx
234 */
235
236/* Normal 64-bit system call target */
237SYM_CODE_START(xen_entry_SYSCALL_64)
238 UNWIND_HINT_ENTRY
239 ENDBR
240 popq %rcx
241 popq %r11
242
243 /*
244 * Neither Xen nor the kernel really knows what the old SS and
245 * CS were. The kernel expects __USER_DS and __USER_CS, so
246 * report those values even though Xen will guess its own values.
247 */
248 movq $__USER_DS, 4*8(%rsp)
249 movq $__USER_CS, 1*8(%rsp)
250
251 jmp entry_SYSCALL_64_after_hwframe
252SYM_CODE_END(xen_entry_SYSCALL_64)
253
254#ifdef CONFIG_IA32_EMULATION
255
256/* 32-bit compat syscall target */
257SYM_CODE_START(xen_entry_SYSCALL_compat)
258 UNWIND_HINT_ENTRY
259 ENDBR
260 popq %rcx
261 popq %r11
262
263 /*
264 * Neither Xen nor the kernel really knows what the old SS and
265 * CS were. The kernel expects __USER_DS and __USER32_CS, so
266 * report those values even though Xen will guess its own values.
267 */
268 movq $__USER_DS, 4*8(%rsp)
269 movq $__USER32_CS, 1*8(%rsp)
270
271 jmp entry_SYSCALL_compat_after_hwframe
272SYM_CODE_END(xen_entry_SYSCALL_compat)
273
274/* 32-bit compat sysenter target */
275SYM_CODE_START(xen_entry_SYSENTER_compat)
276 UNWIND_HINT_ENTRY
277 ENDBR
278 /*
279 * NB: Xen is polite and clears TF from EFLAGS for us. This means
280 * that we don't need to guard against single step exceptions here.
281 */
282 popq %rcx
283 popq %r11
284
285 /*
286 * Neither Xen nor the kernel really knows what the old SS and
287 * CS were. The kernel expects __USER_DS and __USER32_CS, so
288 * report those values even though Xen will guess its own values.
289 */
290 movq $__USER_DS, 4*8(%rsp)
291 movq $__USER32_CS, 1*8(%rsp)
292
293 jmp entry_SYSENTER_compat_after_hwframe
294SYM_CODE_END(xen_entry_SYSENTER_compat)
295
296#else /* !CONFIG_IA32_EMULATION */
297
298SYM_CODE_START(xen_entry_SYSCALL_compat)
299SYM_CODE_START(xen_entry_SYSENTER_compat)
300 UNWIND_HINT_ENTRY
301 ENDBR
302 lea 16(%rsp), %rsp /* strip %rcx, %r11 */
303 mov $-ENOSYS, %rax
304 pushq $0
305 jmp hypercall_iret
306SYM_CODE_END(xen_entry_SYSENTER_compat)
307SYM_CODE_END(xen_entry_SYSCALL_compat)
308
309#endif /* CONFIG_IA32_EMULATION */