Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * __get_user functions.
4 *
5 * (C) Copyright 1998 Linus Torvalds
6 * (C) Copyright 2005 Andi Kleen
7 * (C) Copyright 2008 Glauber Costa
8 *
9 * These functions have a non-standard call interface
10 * to make them more efficient, especially as they
11 * return an error value in addition to the "real"
12 * return value.
13 */
14
15/*
16 * __get_user_X
17 *
18 * Inputs: %[r|e]ax contains the address.
19 *
20 * Outputs: %[r|e]ax is error code (0 or -EFAULT)
21 * %[r|e]dx contains zero-extended value
22 * %ecx contains the high half for 32-bit __get_user_8
23 *
24 *
25 * These functions should not modify any other registers,
26 * as they get called from within inline assembly.
27 */
28
29#include <linux/export.h>
30#include <linux/linkage.h>
31#include <asm/page_types.h>
32#include <asm/errno.h>
33#include <asm/asm-offsets.h>
34#include <asm/thread_info.h>
35#include <asm/asm.h>
36#include <asm/smap.h>
37
38#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
39
40.macro check_range size:req
41.if IS_ENABLED(CONFIG_X86_64)
42 movq $0x0123456789abcdef,%rdx
43 1:
44 .pushsection runtime_ptr_USER_PTR_MAX,"a"
45 .long 1b - 8 - .
46 .popsection
47 cmp %rax, %rdx
48 sbb %rdx, %rdx
49 or %rdx, %rax
50.else
51 cmp $TASK_SIZE_MAX-\size+1, %eax
52 jae .Lbad_get_user
53 sbb %edx, %edx /* array_index_mask_nospec() */
54 and %edx, %eax
55.endif
56.endm
57
58.macro UACCESS op src dst
591: \op \src,\dst
60 _ASM_EXTABLE_UA(1b, __get_user_handle_exception)
61.endm
62
63
64 .text
65SYM_FUNC_START(__get_user_1)
66 check_range size=1
67 ASM_STAC
68 UACCESS movzbl (%_ASM_AX),%edx
69 xor %eax,%eax
70 ASM_CLAC
71 RET
72SYM_FUNC_END(__get_user_1)
73EXPORT_SYMBOL(__get_user_1)
74
75SYM_FUNC_START(__get_user_2)
76 check_range size=2
77 ASM_STAC
78 UACCESS movzwl (%_ASM_AX),%edx
79 xor %eax,%eax
80 ASM_CLAC
81 RET
82SYM_FUNC_END(__get_user_2)
83EXPORT_SYMBOL(__get_user_2)
84
85SYM_FUNC_START(__get_user_4)
86 check_range size=4
87 ASM_STAC
88 UACCESS movl (%_ASM_AX),%edx
89 xor %eax,%eax
90 ASM_CLAC
91 RET
92SYM_FUNC_END(__get_user_4)
93EXPORT_SYMBOL(__get_user_4)
94
95SYM_FUNC_START(__get_user_8)
96#ifndef CONFIG_X86_64
97 xor %ecx,%ecx
98#endif
99 check_range size=8
100 ASM_STAC
101#ifdef CONFIG_X86_64
102 UACCESS movq (%_ASM_AX),%rdx
103#else
104 UACCESS movl (%_ASM_AX),%edx
105 UACCESS movl 4(%_ASM_AX),%ecx
106#endif
107 xor %eax,%eax
108 ASM_CLAC
109 RET
110SYM_FUNC_END(__get_user_8)
111EXPORT_SYMBOL(__get_user_8)
112
113/* .. and the same for __get_user, just without the range checks */
114SYM_FUNC_START(__get_user_nocheck_1)
115 ASM_STAC
116 ASM_BARRIER_NOSPEC
117 UACCESS movzbl (%_ASM_AX),%edx
118 xor %eax,%eax
119 ASM_CLAC
120 RET
121SYM_FUNC_END(__get_user_nocheck_1)
122EXPORT_SYMBOL(__get_user_nocheck_1)
123
124SYM_FUNC_START(__get_user_nocheck_2)
125 ASM_STAC
126 ASM_BARRIER_NOSPEC
127 UACCESS movzwl (%_ASM_AX),%edx
128 xor %eax,%eax
129 ASM_CLAC
130 RET
131SYM_FUNC_END(__get_user_nocheck_2)
132EXPORT_SYMBOL(__get_user_nocheck_2)
133
134SYM_FUNC_START(__get_user_nocheck_4)
135 ASM_STAC
136 ASM_BARRIER_NOSPEC
137 UACCESS movl (%_ASM_AX),%edx
138 xor %eax,%eax
139 ASM_CLAC
140 RET
141SYM_FUNC_END(__get_user_nocheck_4)
142EXPORT_SYMBOL(__get_user_nocheck_4)
143
144SYM_FUNC_START(__get_user_nocheck_8)
145 ASM_STAC
146 ASM_BARRIER_NOSPEC
147#ifdef CONFIG_X86_64
148 UACCESS movq (%_ASM_AX),%rdx
149#else
150 xor %ecx,%ecx
151 UACCESS movl (%_ASM_AX),%edx
152 UACCESS movl 4(%_ASM_AX),%ecx
153#endif
154 xor %eax,%eax
155 ASM_CLAC
156 RET
157SYM_FUNC_END(__get_user_nocheck_8)
158EXPORT_SYMBOL(__get_user_nocheck_8)
159
160
161SYM_CODE_START_LOCAL(__get_user_handle_exception)
162 ASM_CLAC
163.Lbad_get_user:
164 xor %edx,%edx
165 mov $(-EFAULT),%_ASM_AX
166 RET
167SYM_CODE_END(__get_user_handle_exception)
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * __get_user functions.
4 *
5 * (C) Copyright 1998 Linus Torvalds
6 * (C) Copyright 2005 Andi Kleen
7 * (C) Copyright 2008 Glauber Costa
8 *
9 * These functions have a non-standard call interface
10 * to make them more efficient, especially as they
11 * return an error value in addition to the "real"
12 * return value.
13 */
14
15/*
16 * __get_user_X
17 *
18 * Inputs: %[r|e]ax contains the address.
19 *
20 * Outputs: %[r|e]ax is error code (0 or -EFAULT)
21 * %[r|e]dx contains zero-extended value
22 * %ecx contains the high half for 32-bit __get_user_8
23 *
24 *
25 * These functions should not modify any other registers,
26 * as they get called from within inline assembly.
27 */
28
29#include <linux/linkage.h>
30#include <asm/page_types.h>
31#include <asm/errno.h>
32#include <asm/asm-offsets.h>
33#include <asm/thread_info.h>
34#include <asm/asm.h>
35#include <asm/smap.h>
36#include <asm/export.h>
37
38 .text
39ENTRY(__get_user_1)
40 mov PER_CPU_VAR(current_task), %_ASM_DX
41 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
42 jae bad_get_user
43 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
44 and %_ASM_DX, %_ASM_AX
45 ASM_STAC
461: movzbl (%_ASM_AX),%edx
47 xor %eax,%eax
48 ASM_CLAC
49 ret
50ENDPROC(__get_user_1)
51EXPORT_SYMBOL(__get_user_1)
52
53ENTRY(__get_user_2)
54 add $1,%_ASM_AX
55 jc bad_get_user
56 mov PER_CPU_VAR(current_task), %_ASM_DX
57 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
58 jae bad_get_user
59 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
60 and %_ASM_DX, %_ASM_AX
61 ASM_STAC
622: movzwl -1(%_ASM_AX),%edx
63 xor %eax,%eax
64 ASM_CLAC
65 ret
66ENDPROC(__get_user_2)
67EXPORT_SYMBOL(__get_user_2)
68
69ENTRY(__get_user_4)
70 add $3,%_ASM_AX
71 jc bad_get_user
72 mov PER_CPU_VAR(current_task), %_ASM_DX
73 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
74 jae bad_get_user
75 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
76 and %_ASM_DX, %_ASM_AX
77 ASM_STAC
783: movl -3(%_ASM_AX),%edx
79 xor %eax,%eax
80 ASM_CLAC
81 ret
82ENDPROC(__get_user_4)
83EXPORT_SYMBOL(__get_user_4)
84
85ENTRY(__get_user_8)
86#ifdef CONFIG_X86_64
87 add $7,%_ASM_AX
88 jc bad_get_user
89 mov PER_CPU_VAR(current_task), %_ASM_DX
90 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
91 jae bad_get_user
92 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
93 and %_ASM_DX, %_ASM_AX
94 ASM_STAC
954: movq -7(%_ASM_AX),%rdx
96 xor %eax,%eax
97 ASM_CLAC
98 ret
99#else
100 add $7,%_ASM_AX
101 jc bad_get_user_8
102 mov PER_CPU_VAR(current_task), %_ASM_DX
103 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
104 jae bad_get_user_8
105 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
106 and %_ASM_DX, %_ASM_AX
107 ASM_STAC
1084: movl -7(%_ASM_AX),%edx
1095: movl -3(%_ASM_AX),%ecx
110 xor %eax,%eax
111 ASM_CLAC
112 ret
113#endif
114ENDPROC(__get_user_8)
115EXPORT_SYMBOL(__get_user_8)
116
117
118.Lbad_get_user_clac:
119 ASM_CLAC
120bad_get_user:
121 xor %edx,%edx
122 mov $(-EFAULT),%_ASM_AX
123 ret
124
125#ifdef CONFIG_X86_32
126.Lbad_get_user_8_clac:
127 ASM_CLAC
128bad_get_user_8:
129 xor %edx,%edx
130 xor %ecx,%ecx
131 mov $(-EFAULT),%_ASM_AX
132 ret
133#endif
134
135 _ASM_EXTABLE_UA(1b, .Lbad_get_user_clac)
136 _ASM_EXTABLE_UA(2b, .Lbad_get_user_clac)
137 _ASM_EXTABLE_UA(3b, .Lbad_get_user_clac)
138#ifdef CONFIG_X86_64
139 _ASM_EXTABLE_UA(4b, .Lbad_get_user_clac)
140#else
141 _ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac)
142 _ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac)
143#endif