1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * __get_user functions.
  4 *
  5 * (C) Copyright 1998 Linus Torvalds
  6 * (C) Copyright 2005 Andi Kleen
  7 * (C) Copyright 2008 Glauber Costa
  8 *
  9 * These functions have a non-standard call interface
 10 * to make them more efficient, especially as they
 11 * return an error value in addition to the "real"
 12 * return value.
 13 */
 14
 15/*
 16 * __get_user_X
 17 *
 18 * Inputs:	%[r|e]ax contains the address.
 19 *
 20 * Outputs:	%[r|e]ax is error code (0 or -EFAULT)
 21 *		%[r|e]dx contains zero-extended value
 22 *		%ecx contains the high half for 32-bit __get_user_8
 23 *
 24 *
 25 * These functions should not modify any other registers,
 26 * as they get called from within inline assembly.
 27 */
 28
 29#include <linux/export.h>
 30#include <linux/linkage.h>
 31#include <asm/page_types.h>
 32#include <asm/errno.h>
 33#include <asm/asm-offsets.h>
 34#include <asm/thread_info.h>
 35#include <asm/asm.h>
 36#include <asm/smap.h>
 37
 38#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
 39
 40.macro check_range size:req
 41.if IS_ENABLED(CONFIG_X86_64)
 42	mov %rax, %rdx
 43	sar $63, %rdx
 
 
 
 
 
 44	or %rdx, %rax
 45.else
 46	cmp $TASK_SIZE_MAX-\size+1, %eax
 47	jae .Lbad_get_user
 48	sbb %edx, %edx		/* array_index_mask_nospec() */
 49	and %edx, %eax
 50.endif
 51.endm
 52
 
 
 
 
 
 
 53	.text
 54SYM_FUNC_START(__get_user_1)
 55	check_range size=1
 56	ASM_STAC
 571:	movzbl (%_ASM_AX),%edx
 58	xor %eax,%eax
 59	ASM_CLAC
 60	RET
 61SYM_FUNC_END(__get_user_1)
 62EXPORT_SYMBOL(__get_user_1)
 63
 64SYM_FUNC_START(__get_user_2)
 65	check_range size=2
 66	ASM_STAC
 672:	movzwl (%_ASM_AX),%edx
 68	xor %eax,%eax
 69	ASM_CLAC
 70	RET
 71SYM_FUNC_END(__get_user_2)
 72EXPORT_SYMBOL(__get_user_2)
 73
 74SYM_FUNC_START(__get_user_4)
 75	check_range size=4
 76	ASM_STAC
 773:	movl (%_ASM_AX),%edx
 78	xor %eax,%eax
 79	ASM_CLAC
 80	RET
 81SYM_FUNC_END(__get_user_4)
 82EXPORT_SYMBOL(__get_user_4)
 83
 84SYM_FUNC_START(__get_user_8)
 
 
 
 85	check_range size=8
 86	ASM_STAC
 87#ifdef CONFIG_X86_64
 884:	movq (%_ASM_AX),%rdx
 89#else
 904:	movl (%_ASM_AX),%edx
 915:	movl 4(%_ASM_AX),%ecx
 92#endif
 93	xor %eax,%eax
 94	ASM_CLAC
 95	RET
 96SYM_FUNC_END(__get_user_8)
 97EXPORT_SYMBOL(__get_user_8)
 98
 99/* .. and the same for __get_user, just without the range checks */
100SYM_FUNC_START(__get_user_nocheck_1)
101	ASM_STAC
102	ASM_BARRIER_NOSPEC
1036:	movzbl (%_ASM_AX),%edx
104	xor %eax,%eax
105	ASM_CLAC
106	RET
107SYM_FUNC_END(__get_user_nocheck_1)
108EXPORT_SYMBOL(__get_user_nocheck_1)
109
110SYM_FUNC_START(__get_user_nocheck_2)
111	ASM_STAC
112	ASM_BARRIER_NOSPEC
1137:	movzwl (%_ASM_AX),%edx
114	xor %eax,%eax
115	ASM_CLAC
116	RET
117SYM_FUNC_END(__get_user_nocheck_2)
118EXPORT_SYMBOL(__get_user_nocheck_2)
119
120SYM_FUNC_START(__get_user_nocheck_4)
121	ASM_STAC
122	ASM_BARRIER_NOSPEC
1238:	movl (%_ASM_AX),%edx
124	xor %eax,%eax
125	ASM_CLAC
126	RET
127SYM_FUNC_END(__get_user_nocheck_4)
128EXPORT_SYMBOL(__get_user_nocheck_4)
129
130SYM_FUNC_START(__get_user_nocheck_8)
131	ASM_STAC
132	ASM_BARRIER_NOSPEC
133#ifdef CONFIG_X86_64
1349:	movq (%_ASM_AX),%rdx
135#else
1369:	movl (%_ASM_AX),%edx
13710:	movl 4(%_ASM_AX),%ecx
 
138#endif
139	xor %eax,%eax
140	ASM_CLAC
141	RET
142SYM_FUNC_END(__get_user_nocheck_8)
143EXPORT_SYMBOL(__get_user_nocheck_8)
144
145
146SYM_CODE_START_LOCAL(__get_user_handle_exception)
147	ASM_CLAC
148.Lbad_get_user:
149	xor %edx,%edx
150	mov $(-EFAULT),%_ASM_AX
151	RET
152SYM_CODE_END(__get_user_handle_exception)
153
154#ifdef CONFIG_X86_32
155SYM_CODE_START_LOCAL(__get_user_8_handle_exception)
156	ASM_CLAC
157bad_get_user_8:
158	xor %edx,%edx
159	xor %ecx,%ecx
160	mov $(-EFAULT),%_ASM_AX
161	RET
162SYM_CODE_END(__get_user_8_handle_exception)
163#endif
164
165/* get_user */
166	_ASM_EXTABLE_UA(1b, __get_user_handle_exception)
167	_ASM_EXTABLE_UA(2b, __get_user_handle_exception)
168	_ASM_EXTABLE_UA(3b, __get_user_handle_exception)
169#ifdef CONFIG_X86_64
170	_ASM_EXTABLE_UA(4b, __get_user_handle_exception)
171#else
172	_ASM_EXTABLE_UA(4b, __get_user_8_handle_exception)
173	_ASM_EXTABLE_UA(5b, __get_user_8_handle_exception)
174#endif
175
176/* __get_user */
177	_ASM_EXTABLE_UA(6b, __get_user_handle_exception)
178	_ASM_EXTABLE_UA(7b, __get_user_handle_exception)
179	_ASM_EXTABLE_UA(8b, __get_user_handle_exception)
180#ifdef CONFIG_X86_64
181	_ASM_EXTABLE_UA(9b, __get_user_handle_exception)
182#else
183	_ASM_EXTABLE_UA(9b, __get_user_8_handle_exception)
184	_ASM_EXTABLE_UA(10b, __get_user_8_handle_exception)
185#endif

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * __get_user functions.
  4 *
  5 * (C) Copyright 1998 Linus Torvalds
  6 * (C) Copyright 2005 Andi Kleen
  7 * (C) Copyright 2008 Glauber Costa
  8 *
  9 * These functions have a non-standard call interface
 10 * to make them more efficient, especially as they
 11 * return an error value in addition to the "real"
 12 * return value.
 13 */
 14
 15/*
 16 * __get_user_X
 17 *
 18 * Inputs:	%[r|e]ax contains the address.
 19 *
 20 * Outputs:	%[r|e]ax is error code (0 or -EFAULT)
 21 *		%[r|e]dx contains zero-extended value
 22 *		%ecx contains the high half for 32-bit __get_user_8
 23 *
 24 *
 25 * These functions should not modify any other registers,
 26 * as they get called from within inline assembly.
 27 */
 28
 29#include <linux/export.h>
 30#include <linux/linkage.h>
 31#include <asm/page_types.h>
 32#include <asm/errno.h>
 33#include <asm/asm-offsets.h>
 34#include <asm/thread_info.h>
 35#include <asm/asm.h>
 36#include <asm/smap.h>
 37
 38#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
 39
 40.macro check_range size:req
 41.if IS_ENABLED(CONFIG_X86_64)
 42	movq $0x0123456789abcdef,%rdx
 43  1:
 44  .pushsection runtime_ptr_USER_PTR_MAX,"a"
 45	.long 1b - 8 - .
 46  .popsection
 47	cmp %rax, %rdx
 48	sbb %rdx, %rdx
 49	or %rdx, %rax
 50.else
 51	cmp $TASK_SIZE_MAX-\size+1, %eax
 52	jae .Lbad_get_user
 53	sbb %edx, %edx		/* array_index_mask_nospec() */
 54	and %edx, %eax
 55.endif
 56.endm
 57
 58.macro UACCESS op src dst
 591:	\op \src,\dst
 60	_ASM_EXTABLE_UA(1b, __get_user_handle_exception)
 61.endm
 62
 63
 64	.text
 65SYM_FUNC_START(__get_user_1)
 66	check_range size=1
 67	ASM_STAC
 68	UACCESS movzbl (%_ASM_AX),%edx
 69	xor %eax,%eax
 70	ASM_CLAC
 71	RET
 72SYM_FUNC_END(__get_user_1)
 73EXPORT_SYMBOL(__get_user_1)
 74
 75SYM_FUNC_START(__get_user_2)
 76	check_range size=2
 77	ASM_STAC
 78	UACCESS movzwl (%_ASM_AX),%edx
 79	xor %eax,%eax
 80	ASM_CLAC
 81	RET
 82SYM_FUNC_END(__get_user_2)
 83EXPORT_SYMBOL(__get_user_2)
 84
 85SYM_FUNC_START(__get_user_4)
 86	check_range size=4
 87	ASM_STAC
 88	UACCESS movl (%_ASM_AX),%edx
 89	xor %eax,%eax
 90	ASM_CLAC
 91	RET
 92SYM_FUNC_END(__get_user_4)
 93EXPORT_SYMBOL(__get_user_4)
 94
 95SYM_FUNC_START(__get_user_8)
 96#ifndef CONFIG_X86_64
 97	xor %ecx,%ecx
 98#endif
 99	check_range size=8
100	ASM_STAC
101#ifdef CONFIG_X86_64
102	UACCESS movq (%_ASM_AX),%rdx
103#else
104	UACCESS movl (%_ASM_AX),%edx
105	UACCESS movl 4(%_ASM_AX),%ecx
106#endif
107	xor %eax,%eax
108	ASM_CLAC
109	RET
110SYM_FUNC_END(__get_user_8)
111EXPORT_SYMBOL(__get_user_8)
112
113/* .. and the same for __get_user, just without the range checks */
114SYM_FUNC_START(__get_user_nocheck_1)
115	ASM_STAC
116	ASM_BARRIER_NOSPEC
117	UACCESS movzbl (%_ASM_AX),%edx
118	xor %eax,%eax
119	ASM_CLAC
120	RET
121SYM_FUNC_END(__get_user_nocheck_1)
122EXPORT_SYMBOL(__get_user_nocheck_1)
123
124SYM_FUNC_START(__get_user_nocheck_2)
125	ASM_STAC
126	ASM_BARRIER_NOSPEC
127	UACCESS movzwl (%_ASM_AX),%edx
128	xor %eax,%eax
129	ASM_CLAC
130	RET
131SYM_FUNC_END(__get_user_nocheck_2)
132EXPORT_SYMBOL(__get_user_nocheck_2)
133
134SYM_FUNC_START(__get_user_nocheck_4)
135	ASM_STAC
136	ASM_BARRIER_NOSPEC
137	UACCESS movl (%_ASM_AX),%edx
138	xor %eax,%eax
139	ASM_CLAC
140	RET
141SYM_FUNC_END(__get_user_nocheck_4)
142EXPORT_SYMBOL(__get_user_nocheck_4)
143
144SYM_FUNC_START(__get_user_nocheck_8)
145	ASM_STAC
146	ASM_BARRIER_NOSPEC
147#ifdef CONFIG_X86_64
148	UACCESS movq (%_ASM_AX),%rdx
149#else
150	xor %ecx,%ecx
151	UACCESS movl (%_ASM_AX),%edx
152	UACCESS movl 4(%_ASM_AX),%ecx
153#endif
154	xor %eax,%eax
155	ASM_CLAC
156	RET
157SYM_FUNC_END(__get_user_nocheck_8)
158EXPORT_SYMBOL(__get_user_nocheck_8)
159
160
161SYM_CODE_START_LOCAL(__get_user_handle_exception)
162	ASM_CLAC
163.Lbad_get_user:
164	xor %edx,%edx
165	mov $(-EFAULT),%_ASM_AX
166	RET
167SYM_CODE_END(__get_user_handle_exception)