Loading...
1/*
2 * __get_user functions.
3 *
4 * (C) Copyright 1998 Linus Torvalds
5 * (C) Copyright 2005 Andi Kleen
6 * (C) Copyright 2008 Glauber Costa
7 *
8 * These functions have a non-standard call interface
9 * to make them more efficient, especially as they
10 * return an error value in addition to the "real"
11 * return value.
12 */
13
14/*
15 * __get_user_X
16 *
17 * Inputs: %[r|e]ax contains the address.
18 *
19 * Outputs: %[r|e]ax is error code (0 or -EFAULT)
20 * %[r|e]dx contains zero-extended value
21 * %ecx contains the high half for 32-bit __get_user_8
22 *
23 *
24 * These functions should not modify any other registers,
25 * as they get called from within inline assembly.
26 */
27
28#include <linux/linkage.h>
29#include <asm/page_types.h>
30#include <asm/errno.h>
31#include <asm/asm-offsets.h>
32#include <asm/thread_info.h>
33#include <asm/asm.h>
34#include <asm/smap.h>
35#include <asm/export.h>
36
37 .text
38ENTRY(__get_user_1)
39 mov PER_CPU_VAR(current_task), %_ASM_DX
40 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
41 jae bad_get_user
42 ASM_STAC
431: movzbl (%_ASM_AX),%edx
44 xor %eax,%eax
45 ASM_CLAC
46 ret
47ENDPROC(__get_user_1)
48EXPORT_SYMBOL(__get_user_1)
49
50ENTRY(__get_user_2)
51 add $1,%_ASM_AX
52 jc bad_get_user
53 mov PER_CPU_VAR(current_task), %_ASM_DX
54 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
55 jae bad_get_user
56 ASM_STAC
572: movzwl -1(%_ASM_AX),%edx
58 xor %eax,%eax
59 ASM_CLAC
60 ret
61ENDPROC(__get_user_2)
62EXPORT_SYMBOL(__get_user_2)
63
64ENTRY(__get_user_4)
65 add $3,%_ASM_AX
66 jc bad_get_user
67 mov PER_CPU_VAR(current_task), %_ASM_DX
68 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
69 jae bad_get_user
70 ASM_STAC
713: movl -3(%_ASM_AX),%edx
72 xor %eax,%eax
73 ASM_CLAC
74 ret
75ENDPROC(__get_user_4)
76EXPORT_SYMBOL(__get_user_4)
77
78ENTRY(__get_user_8)
79#ifdef CONFIG_X86_64
80 add $7,%_ASM_AX
81 jc bad_get_user
82 mov PER_CPU_VAR(current_task), %_ASM_DX
83 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
84 jae bad_get_user
85 ASM_STAC
864: movq -7(%_ASM_AX),%rdx
87 xor %eax,%eax
88 ASM_CLAC
89 ret
90#else
91 add $7,%_ASM_AX
92 jc bad_get_user_8
93 mov PER_CPU_VAR(current_task), %_ASM_DX
94 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
95 jae bad_get_user_8
96 ASM_STAC
974: movl -7(%_ASM_AX),%edx
985: movl -3(%_ASM_AX),%ecx
99 xor %eax,%eax
100 ASM_CLAC
101 ret
102#endif
103ENDPROC(__get_user_8)
104EXPORT_SYMBOL(__get_user_8)
105
106
107bad_get_user:
108 xor %edx,%edx
109 mov $(-EFAULT),%_ASM_AX
110 ASM_CLAC
111 ret
112END(bad_get_user)
113
114#ifdef CONFIG_X86_32
115bad_get_user_8:
116 xor %edx,%edx
117 xor %ecx,%ecx
118 mov $(-EFAULT),%_ASM_AX
119 ASM_CLAC
120 ret
121END(bad_get_user_8)
122#endif
123
124 _ASM_EXTABLE(1b,bad_get_user)
125 _ASM_EXTABLE(2b,bad_get_user)
126 _ASM_EXTABLE(3b,bad_get_user)
127#ifdef CONFIG_X86_64
128 _ASM_EXTABLE(4b,bad_get_user)
129#else
130 _ASM_EXTABLE(4b,bad_get_user_8)
131 _ASM_EXTABLE(5b,bad_get_user_8)
132#endif
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * __get_user functions.
4 *
5 * (C) Copyright 1998 Linus Torvalds
6 * (C) Copyright 2005 Andi Kleen
7 * (C) Copyright 2008 Glauber Costa
8 *
9 * These functions have a non-standard call interface
10 * to make them more efficient, especially as they
11 * return an error value in addition to the "real"
12 * return value.
13 */
14
15/*
16 * __get_user_X
17 *
18 * Inputs: %[r|e]ax contains the address.
19 *
20 * Outputs: %[r|e]ax is error code (0 or -EFAULT)
21 * %[r|e]dx contains zero-extended value
22 * %ecx contains the high half for 32-bit __get_user_8
23 *
24 *
25 * These functions should not modify any other registers,
26 * as they get called from within inline assembly.
27 */
28
29#include <linux/linkage.h>
30#include <asm/page_types.h>
31#include <asm/errno.h>
32#include <asm/asm-offsets.h>
33#include <asm/thread_info.h>
34#include <asm/asm.h>
35#include <asm/smap.h>
36#include <asm/export.h>
37
38 .text
39SYM_FUNC_START(__get_user_1)
40 mov PER_CPU_VAR(current_task), %_ASM_DX
41 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
42 jae bad_get_user
43 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
44 and %_ASM_DX, %_ASM_AX
45 ASM_STAC
461: movzbl (%_ASM_AX),%edx
47 xor %eax,%eax
48 ASM_CLAC
49 ret
50SYM_FUNC_END(__get_user_1)
51EXPORT_SYMBOL(__get_user_1)
52
53SYM_FUNC_START(__get_user_2)
54 add $1,%_ASM_AX
55 jc bad_get_user
56 mov PER_CPU_VAR(current_task), %_ASM_DX
57 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
58 jae bad_get_user
59 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
60 and %_ASM_DX, %_ASM_AX
61 ASM_STAC
622: movzwl -1(%_ASM_AX),%edx
63 xor %eax,%eax
64 ASM_CLAC
65 ret
66SYM_FUNC_END(__get_user_2)
67EXPORT_SYMBOL(__get_user_2)
68
69SYM_FUNC_START(__get_user_4)
70 add $3,%_ASM_AX
71 jc bad_get_user
72 mov PER_CPU_VAR(current_task), %_ASM_DX
73 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
74 jae bad_get_user
75 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
76 and %_ASM_DX, %_ASM_AX
77 ASM_STAC
783: movl -3(%_ASM_AX),%edx
79 xor %eax,%eax
80 ASM_CLAC
81 ret
82SYM_FUNC_END(__get_user_4)
83EXPORT_SYMBOL(__get_user_4)
84
85SYM_FUNC_START(__get_user_8)
86#ifdef CONFIG_X86_64
87 add $7,%_ASM_AX
88 jc bad_get_user
89 mov PER_CPU_VAR(current_task), %_ASM_DX
90 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
91 jae bad_get_user
92 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
93 and %_ASM_DX, %_ASM_AX
94 ASM_STAC
954: movq -7(%_ASM_AX),%rdx
96 xor %eax,%eax
97 ASM_CLAC
98 ret
99#else
100 add $7,%_ASM_AX
101 jc bad_get_user_8
102 mov PER_CPU_VAR(current_task), %_ASM_DX
103 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
104 jae bad_get_user_8
105 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
106 and %_ASM_DX, %_ASM_AX
107 ASM_STAC
1084: movl -7(%_ASM_AX),%edx
1095: movl -3(%_ASM_AX),%ecx
110 xor %eax,%eax
111 ASM_CLAC
112 ret
113#endif
114SYM_FUNC_END(__get_user_8)
115EXPORT_SYMBOL(__get_user_8)
116
117
118SYM_CODE_START_LOCAL(.Lbad_get_user_clac)
119 ASM_CLAC
120bad_get_user:
121 xor %edx,%edx
122 mov $(-EFAULT),%_ASM_AX
123 ret
124SYM_CODE_END(.Lbad_get_user_clac)
125
126#ifdef CONFIG_X86_32
127SYM_CODE_START_LOCAL(.Lbad_get_user_8_clac)
128 ASM_CLAC
129bad_get_user_8:
130 xor %edx,%edx
131 xor %ecx,%ecx
132 mov $(-EFAULT),%_ASM_AX
133 ret
134SYM_CODE_END(.Lbad_get_user_8_clac)
135#endif
136
137 _ASM_EXTABLE_UA(1b, .Lbad_get_user_clac)
138 _ASM_EXTABLE_UA(2b, .Lbad_get_user_clac)
139 _ASM_EXTABLE_UA(3b, .Lbad_get_user_clac)
140#ifdef CONFIG_X86_64
141 _ASM_EXTABLE_UA(4b, .Lbad_get_user_clac)
142#else
143 _ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac)
144 _ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac)
145#endif