1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Asm versions of Xen pv-ops, suitable for direct use.
  4 *
  5 * We only bother with direct forms (ie, vcpu in percpu data) of the
  6 * operations here; the indirect forms are better handled in C.
  7 */
  8
  9#include <asm/errno.h>
 10#include <asm/asm-offsets.h>
 11#include <asm/percpu.h>
 12#include <asm/processor-flags.h>
 13#include <asm/segment.h>
 14#include <asm/thread_info.h>
 15#include <asm/asm.h>
 16#include <asm/frame.h>
 17#include <asm/unwind_hints.h>
 18
 19#include <xen/interface/xen.h>
 20
 21#include <linux/init.h>
 22#include <linux/linkage.h>
 23#include <../entry/calling.h>
 24
 25.pushsection .noinstr.text, "ax"
 26/*
 27 * Disabling events is simply a matter of making the event mask
 28 * non-zero.
 29 */
 30SYM_FUNC_START(xen_irq_disable_direct)
 31	movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
 32	RET
 33SYM_FUNC_END(xen_irq_disable_direct)
 34
 35/*
 36 * Force an event check by making a hypercall, but preserve regs
 37 * before making the call.
 38 */
 39SYM_FUNC_START(check_events)
 40	FRAME_BEGIN
 41	push %rax
 42	push %rcx
 43	push %rdx
 44	push %rsi
 45	push %rdi
 46	push %r8
 47	push %r9
 48	push %r10
 49	push %r11
 50	call xen_force_evtchn_callback
 51	pop %r11
 52	pop %r10
 53	pop %r9
 54	pop %r8
 55	pop %rdi
 56	pop %rsi
 57	pop %rdx
 58	pop %rcx
 59	pop %rax
 60	FRAME_END
 61	RET
 62SYM_FUNC_END(check_events)
 63
 64/*
 65 * Enable events.  This clears the event mask and tests the pending
 66 * event status with one and operation.  If there are pending events,
 67 * then enter the hypervisor to get them handled.
 68 */
 69SYM_FUNC_START(xen_irq_enable_direct)
 70	FRAME_BEGIN
 71	/* Unmask events */
 72	movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
 73
 74	/*
 75	 * Preempt here doesn't matter because that will deal with any
 76	 * pending interrupts.  The pending check may end up being run
 77	 * on the wrong CPU, but that doesn't hurt.
 78	 */
 79
 80	/* Test for pending */
 81	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
 82	jz 1f
 83
 84	call check_events
 851:
 86	FRAME_END
 87	RET
 88SYM_FUNC_END(xen_irq_enable_direct)
 
 
 
 
 
 
 
 
 
 
 89
 90/*
 91 * (xen_)save_fl is used to get the current interrupt enable status.
 92 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
 93 * may be set in the return value.  We take advantage of this by
 94 * making sure that X86_EFLAGS_IF has the right value (and other bits
 95 * in that byte are 0), but other bits in the return value are
 96 * undefined.  We need to toggle the state of the bit, because Xen and
 97 * x86 use opposite senses (mask vs enable).
 98 */
 99SYM_FUNC_START(xen_save_fl_direct)
100	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
101	setz %ah
102	addb %ah, %ah
103	RET
104SYM_FUNC_END(xen_save_fl_direct)
105
106SYM_FUNC_START(xen_read_cr2)
107	FRAME_BEGIN
108	_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
109	_ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
110	FRAME_END
111	RET
112SYM_FUNC_END(xen_read_cr2);
113
114SYM_FUNC_START(xen_read_cr2_direct)
115	FRAME_BEGIN
116	_ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
117	FRAME_END
118	RET
119SYM_FUNC_END(xen_read_cr2_direct);
120.popsection
121
122.macro xen_pv_trap name
123SYM_CODE_START(xen_\name)
124	UNWIND_HINT_ENTRY
125	ENDBR
126	pop %rcx
127	pop %r11
128	jmp  \name
129SYM_CODE_END(xen_\name)
130_ASM_NOKPROBE(xen_\name)
131.endm
132
133xen_pv_trap asm_exc_divide_error
134xen_pv_trap asm_xenpv_exc_debug
135xen_pv_trap asm_exc_int3
136xen_pv_trap asm_xenpv_exc_nmi
137xen_pv_trap asm_exc_overflow
138xen_pv_trap asm_exc_bounds
139xen_pv_trap asm_exc_invalid_op
140xen_pv_trap asm_exc_device_not_available
141xen_pv_trap asm_xenpv_exc_double_fault
142xen_pv_trap asm_exc_coproc_segment_overrun
143xen_pv_trap asm_exc_invalid_tss
144xen_pv_trap asm_exc_segment_not_present
145xen_pv_trap asm_exc_stack_segment
146xen_pv_trap asm_exc_general_protection
147xen_pv_trap asm_exc_page_fault
148xen_pv_trap asm_exc_spurious_interrupt_bug
149xen_pv_trap asm_exc_coprocessor_error
150xen_pv_trap asm_exc_alignment_check
151#ifdef CONFIG_X86_CET
152xen_pv_trap asm_exc_control_protection
153#endif
154#ifdef CONFIG_X86_MCE
155xen_pv_trap asm_xenpv_exc_machine_check
156#endif /* CONFIG_X86_MCE */
157xen_pv_trap asm_exc_simd_coprocessor_error
158#ifdef CONFIG_IA32_EMULATION
159xen_pv_trap asm_int80_emulation
160#endif
161xen_pv_trap asm_exc_xen_unknown_trap
162xen_pv_trap asm_exc_xen_hypervisor_callback
163
164	__INIT
165SYM_CODE_START(xen_early_idt_handler_array)
166	i = 0
167	.rept NUM_EXCEPTION_VECTORS
168	UNWIND_HINT_UNDEFINED
169	ENDBR
170	pop %rcx
171	pop %r11
172	jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
173	i = i + 1
174	.fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
175	.endr
176SYM_CODE_END(xen_early_idt_handler_array)
177	__FINIT
178
179hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
180/*
181 * Xen64 iret frame:
182 *
183 *	ss
184 *	rsp
185 *	rflags
186 *	cs
187 *	rip		<-- standard iret frame
188 *
189 *	flags
190 *
191 *	rcx		}
192 *	r11		}<-- pushed by hypercall page
193 * rsp->rax		}
194 */
195SYM_CODE_START(xen_iret)
196	UNWIND_HINT_UNDEFINED
197	ANNOTATE_NOENDBR
198	pushq $0
199	jmp hypercall_iret
200SYM_CODE_END(xen_iret)
201
202/*
203 * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
204 * also the kernel stack.  Reusing swapgs_restore_regs_and_return_to_usermode()
205 * in XEN pv would cause %rsp to move up to the top of the kernel stack and
206 * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
207 * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
208 * frame at the same address is useless.
209 */
210SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
211	UNWIND_HINT_REGS
212	POP_REGS
213
214	/* stackleak_erase() can work safely on the kernel stack. */
215	STACKLEAK_ERASE_NOCLOBBER
216
217	addq	$8, %rsp	/* skip regs->orig_ax */
218	jmp xen_iret
219SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
220
221/*
222 * Xen handles syscall callbacks much like ordinary exceptions, which
223 * means we have:
224 * - kernel gs
225 * - kernel rsp
226 * - an iret-like stack frame on the stack (including rcx and r11):
227 *	ss
228 *	rsp
229 *	rflags
230 *	cs
231 *	rip
232 *	r11
233 * rsp->rcx
234 */
235
236/* Normal 64-bit system call target */
237SYM_CODE_START(xen_entry_SYSCALL_64)
238	UNWIND_HINT_ENTRY
239	ENDBR
240	popq %rcx
241	popq %r11
242
243	/*
244	 * Neither Xen nor the kernel really knows what the old SS and
245	 * CS were.  The kernel expects __USER_DS and __USER_CS, so
246	 * report those values even though Xen will guess its own values.
247	 */
248	movq $__USER_DS, 4*8(%rsp)
249	movq $__USER_CS, 1*8(%rsp)
250
251	jmp entry_SYSCALL_64_after_hwframe
252SYM_CODE_END(xen_entry_SYSCALL_64)
253
254#ifdef CONFIG_IA32_EMULATION
255
256/* 32-bit compat syscall target */
257SYM_CODE_START(xen_entry_SYSCALL_compat)
258	UNWIND_HINT_ENTRY
259	ENDBR
260	popq %rcx
261	popq %r11
262
263	/*
264	 * Neither Xen nor the kernel really knows what the old SS and
265	 * CS were.  The kernel expects __USER_DS and __USER32_CS, so
266	 * report those values even though Xen will guess its own values.
267	 */
268	movq $__USER_DS, 4*8(%rsp)
269	movq $__USER32_CS, 1*8(%rsp)
270
271	jmp entry_SYSCALL_compat_after_hwframe
272SYM_CODE_END(xen_entry_SYSCALL_compat)
273
274/* 32-bit compat sysenter target */
275SYM_CODE_START(xen_entry_SYSENTER_compat)
276	UNWIND_HINT_ENTRY
277	ENDBR
278	/*
279	 * NB: Xen is polite and clears TF from EFLAGS for us.  This means
280	 * that we don't need to guard against single step exceptions here.
 
281	 */
282	popq %rcx
283	popq %r11
284
285	/*
286	 * Neither Xen nor the kernel really knows what the old SS and
287	 * CS were.  The kernel expects __USER_DS and __USER32_CS, so
288	 * report those values even though Xen will guess its own values.
289	 */
290	movq $__USER_DS, 4*8(%rsp)
291	movq $__USER32_CS, 1*8(%rsp)
 
292
293	jmp entry_SYSENTER_compat_after_hwframe
294SYM_CODE_END(xen_entry_SYSENTER_compat)
295
296#else /* !CONFIG_IA32_EMULATION */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
297
298SYM_CODE_START(xen_entry_SYSCALL_compat)
299SYM_CODE_START(xen_entry_SYSENTER_compat)
300	UNWIND_HINT_ENTRY
301	ENDBR
302	lea 16(%rsp), %rsp	/* strip %rcx, %r11 */
303	mov $-ENOSYS, %rax
304	pushq $0
305	jmp hypercall_iret
306SYM_CODE_END(xen_entry_SYSENTER_compat)
307SYM_CODE_END(xen_entry_SYSCALL_compat)
308
309#endif	/* CONFIG_IA32_EMULATION */
 
 
 
 
 

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Asm versions of Xen pv-ops, suitable for direct use.
  4 *
  5 * We only bother with direct forms (ie, vcpu in percpu data) of the
  6 * operations here; the indirect forms are better handled in C.
  7 */
  8
 
  9#include <asm/asm-offsets.h>
 10#include <asm/percpu.h>
 11#include <asm/processor-flags.h>
 
 
 
 12#include <asm/frame.h>
 13#include <asm/asm.h>
 
 
 14
 
 15#include <linux/linkage.h>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 16
 17/*
 18 * Enable events.  This clears the event mask and tests the pending
 19 * event status with one and operation.  If there are pending events,
 20 * then enter the hypervisor to get them handled.
 21 */
 22ENTRY(xen_irq_enable_direct)
 23	FRAME_BEGIN
 24	/* Unmask events */
 25	movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
 26
 27	/*
 28	 * Preempt here doesn't matter because that will deal with any
 29	 * pending interrupts.  The pending check may end up being run
 30	 * on the wrong CPU, but that doesn't hurt.
 31	 */
 32
 33	/* Test for pending */
 34	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
 35	jz 1f
 36
 37	call check_events
 381:
 39	FRAME_END
 40	ret
 41	ENDPROC(xen_irq_enable_direct)
 42
 43
 44/*
 45 * Disabling events is simply a matter of making the event mask
 46 * non-zero.
 47 */
 48ENTRY(xen_irq_disable_direct)
 49	movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
 50	ret
 51ENDPROC(xen_irq_disable_direct)
 52
 53/*
 54 * (xen_)save_fl is used to get the current interrupt enable status.
 55 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
 56 * may be set in the return value.  We take advantage of this by
 57 * making sure that X86_EFLAGS_IF has the right value (and other bits
 58 * in that byte are 0), but other bits in the return value are
 59 * undefined.  We need to toggle the state of the bit, because Xen and
 60 * x86 use opposite senses (mask vs enable).
 61 */
 62ENTRY(xen_save_fl_direct)
 63	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
 64	setz %ah
 65	addb %ah, %ah
 66	ret
 67	ENDPROC(xen_save_fl_direct)
 
 
 
 
 
 
 
 
 68
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 69
 70/*
 71 * In principle the caller should be passing us a value return from
 72 * xen_save_fl_direct, but for robustness sake we test only the
 73 * X86_EFLAGS_IF flag rather than the whole byte. After setting the
 74 * interrupt mask state, it checks for unmasked pending events and
 75 * enters the hypervisor to get them delivered if so.
 
 76 */
 77ENTRY(xen_restore_fl_direct)
 78	FRAME_BEGIN
 79#ifdef CONFIG_X86_64
 80	testw $X86_EFLAGS_IF, %di
 81#else
 82	testb $X86_EFLAGS_IF>>8, %ah
 83#endif
 84	setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 85	/*
 86	 * Preempt here doesn't matter because that will deal with any
 87	 * pending interrupts.  The pending check may end up being run
 88	 * on the wrong CPU, but that doesn't hurt.
 89	 */
 
 
 90
 91	/* check for unmasked and pending */
 92	cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
 93	jnz 1f
 94	call check_events
 951:
 96	FRAME_END
 97	ret
 98	ENDPROC(xen_restore_fl_direct)
 99
 
 
100
101/*
102 * Force an event check by making a hypercall, but preserve regs
103 * before making the call.
104 */
105ENTRY(check_events)
106	FRAME_BEGIN
107#ifdef CONFIG_X86_32
108	push %eax
109	push %ecx
110	push %edx
111	call xen_force_evtchn_callback
112	pop %edx
113	pop %ecx
114	pop %eax
115#else
116	push %rax
117	push %rcx
118	push %rdx
119	push %rsi
120	push %rdi
121	push %r8
122	push %r9
123	push %r10
124	push %r11
125	call xen_force_evtchn_callback
126	pop %r11
127	pop %r10
128	pop %r9
129	pop %r8
130	pop %rdi
131	pop %rsi
132	pop %rdx
133	pop %rcx
134	pop %rax
135#endif
136	FRAME_END
137	ret
138ENDPROC(check_events)
139
140ENTRY(xen_read_cr2)
141	FRAME_BEGIN
142	_ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
143	_ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
144	FRAME_END
145	ret
146	ENDPROC(xen_read_cr2);
 
 
 
147
148ENTRY(xen_read_cr2_direct)
149	FRAME_BEGIN
150	_ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
151	FRAME_END
152	ret
153	ENDPROC(xen_read_cr2_direct);