Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2#ifndef _ASM_X86_ENTRY_COMMON_H
3#define _ASM_X86_ENTRY_COMMON_H
4
5#include <linux/randomize_kstack.h>
6#include <linux/user-return-notifier.h>
7
8#include <asm/nospec-branch.h>
9#include <asm/io_bitmap.h>
10#include <asm/fpu/api.h>
11
12/* Check that the stack and regs on entry from user mode are sane. */
13static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs)
14{
15 if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) {
16 /*
17 * Make sure that the entry code gave us a sensible EFLAGS
18 * register. Native because we want to check the actual CPU
19 * state, not the interrupt state as imagined by Xen.
20 */
21 unsigned long flags = native_save_fl();
22 unsigned long mask = X86_EFLAGS_DF | X86_EFLAGS_NT;
23
24 /*
25 * For !SMAP hardware we patch out CLAC on entry.
26 */
27 if (cpu_feature_enabled(X86_FEATURE_SMAP) ||
28 cpu_feature_enabled(X86_FEATURE_XENPV))
29 mask |= X86_EFLAGS_AC;
30
31 WARN_ON_ONCE(flags & mask);
32
33 /* We think we came from user mode. Make sure pt_regs agrees. */
34 WARN_ON_ONCE(!user_mode(regs));
35
36 /*
37 * All entries from user mode (except #DF) should be on the
38 * normal thread stack and should have user pt_regs in the
39 * correct location.
40 */
41 WARN_ON_ONCE(!on_thread_stack());
42 WARN_ON_ONCE(regs != task_pt_regs(current));
43 }
44}
45#define arch_enter_from_user_mode arch_enter_from_user_mode
46
47static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
48 unsigned long ti_work)
49{
50 if (ti_work & _TIF_USER_RETURN_NOTIFY)
51 fire_user_return_notifiers();
52
53 if (unlikely(ti_work & _TIF_IO_BITMAP))
54 tss_update_io_bitmap();
55
56 fpregs_assert_state_consistent();
57 if (unlikely(ti_work & _TIF_NEED_FPU_LOAD))
58 switch_fpu_return();
59
60#ifdef CONFIG_COMPAT
61 /*
62 * Compat syscalls set TS_COMPAT. Make sure we clear it before
63 * returning to user mode. We need to clear it *after* signal
64 * handling, because syscall restart has a fixup for compat
65 * syscalls. The fixup is exercised by the ptrace_syscall_32
66 * selftest.
67 *
68 * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
69 * special case only applies after poking regs and before the
70 * very next return to user mode.
71 */
72 current_thread_info()->status &= ~(TS_COMPAT | TS_I386_REGS_POKED);
73#endif
74
75 /*
76 * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(),
77 * but not enough for x86 stack utilization comfort. To keep
78 * reasonable stack head room, reduce the maximum offset to 8 bits.
79 *
80 * The actual entropy will be further reduced by the compiler when
81 * applying stack alignment constraints (see cc_stack_align4/8 in
82 * arch/x86/Makefile), which will remove the 3 (x86_64) or 2 (ia32)
83 * low bits from any entropy chosen here.
84 *
85 * Therefore, final stack offset entropy will be 5 (x86_64) or
86 * 6 (ia32) bits.
87 */
88 choose_random_kstack_offset(rdtsc() & 0xFF);
89}
90#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
91
92static __always_inline void arch_exit_to_user_mode(void)
93{
94 mds_user_clear_cpu_buffers();
95}
96#define arch_exit_to_user_mode arch_exit_to_user_mode
97
98#endif
1/* SPDX-License-Identifier: GPL-2.0-only */
2#ifndef _ASM_X86_ENTRY_COMMON_H
3#define _ASM_X86_ENTRY_COMMON_H
4
5#include <linux/user-return-notifier.h>
6
7#include <asm/nospec-branch.h>
8#include <asm/io_bitmap.h>
9#include <asm/fpu/api.h>
10
11/* Check that the stack and regs on entry from user mode are sane. */
12static __always_inline void arch_check_user_regs(struct pt_regs *regs)
13{
14 if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) {
15 /*
16 * Make sure that the entry code gave us a sensible EFLAGS
17 * register. Native because we want to check the actual CPU
18 * state, not the interrupt state as imagined by Xen.
19 */
20 unsigned long flags = native_save_fl();
21 unsigned long mask = X86_EFLAGS_DF | X86_EFLAGS_NT;
22
23 /*
24 * For !SMAP hardware we patch out CLAC on entry.
25 */
26 if (boot_cpu_has(X86_FEATURE_SMAP) ||
27 (IS_ENABLED(CONFIG_64_BIT) && boot_cpu_has(X86_FEATURE_XENPV)))
28 mask |= X86_EFLAGS_AC;
29
30 WARN_ON_ONCE(flags & mask);
31
32 /* We think we came from user mode. Make sure pt_regs agrees. */
33 WARN_ON_ONCE(!user_mode(regs));
34
35 /*
36 * All entries from user mode (except #DF) should be on the
37 * normal thread stack and should have user pt_regs in the
38 * correct location.
39 */
40 WARN_ON_ONCE(!on_thread_stack());
41 WARN_ON_ONCE(regs != task_pt_regs(current));
42 }
43}
44#define arch_check_user_regs arch_check_user_regs
45
46#define ARCH_SYSCALL_EXIT_WORK (_TIF_SINGLESTEP)
47
48static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
49 unsigned long ti_work)
50{
51 if (ti_work & _TIF_USER_RETURN_NOTIFY)
52 fire_user_return_notifiers();
53
54 if (unlikely(ti_work & _TIF_IO_BITMAP))
55 tss_update_io_bitmap();
56
57 fpregs_assert_state_consistent();
58 if (unlikely(ti_work & _TIF_NEED_FPU_LOAD))
59 switch_fpu_return();
60
61#ifdef CONFIG_COMPAT
62 /*
63 * Compat syscalls set TS_COMPAT. Make sure we clear it before
64 * returning to user mode. We need to clear it *after* signal
65 * handling, because syscall restart has a fixup for compat
66 * syscalls. The fixup is exercised by the ptrace_syscall_32
67 * selftest.
68 *
69 * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
70 * special case only applies after poking regs and before the
71 * very next return to user mode.
72 */
73 current_thread_info()->status &= ~(TS_COMPAT | TS_I386_REGS_POKED);
74#endif
75}
76#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
77
78static __always_inline void arch_exit_to_user_mode(void)
79{
80 mds_user_clear_cpu_buffers();
81}
82#define arch_exit_to_user_mode arch_exit_to_user_mode
83
84#endif