1/* SPDX-License-Identifier: GPL-2.0-only */
  2/* Copyright 2002 Andi Kleen */
  3
  4#include <linux/linkage.h>
  5#include <asm/errno.h>
  6#include <asm/cpufeatures.h>
  7#include <asm/mcsafe_test.h>
  8#include <asm/alternative-asm.h>
  9#include <asm/export.h>
 10
 11.pushsection .noinstr.text, "ax"
 12
 13/*
 14 * We build a jump to memcpy_orig by default which gets NOPped out on
 15 * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which
 16 * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs
 17 * to a jmp to memcpy_erms which does the REP; MOVSB mem copy.
 18 */
 19
 20.weak memcpy
 21
 22/*
 23 * memcpy - Copy a memory block.
 24 *
 25 * Input:
 26 *  rdi destination
 27 *  rsi source
 28 *  rdx count
 29 *
 30 * Output:
 31 * rax original destination
 32 */
 33SYM_FUNC_START_ALIAS(__memcpy)
 34SYM_FUNC_START_LOCAL(memcpy)
 35	ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
 36		      "jmp memcpy_erms", X86_FEATURE_ERMS
 37
 38	movq %rdi, %rax
 39	movq %rdx, %rcx
 40	shrq $3, %rcx
 41	andl $7, %edx
 42	rep movsq
 43	movl %edx, %ecx
 44	rep movsb
 45	ret
 46SYM_FUNC_END(memcpy)
 47SYM_FUNC_END_ALIAS(__memcpy)
 48EXPORT_SYMBOL(memcpy)
 49EXPORT_SYMBOL(__memcpy)
 50
 51/*
 52 * memcpy_erms() - enhanced fast string memcpy. This is faster and
 53 * simpler than memcpy. Use memcpy_erms when possible.
 54 */
 55SYM_FUNC_START(memcpy_erms)
 56	movq %rdi, %rax
 57	movq %rdx, %rcx
 58	rep movsb
 59	ret
 60SYM_FUNC_END(memcpy_erms)
 61
 62SYM_FUNC_START(memcpy_orig)
 63	movq %rdi, %rax
 64
 65	cmpq $0x20, %rdx
 66	jb .Lhandle_tail
 67
 68	/*
 69	 * We check whether memory false dependence could occur,
 70	 * then jump to corresponding copy mode.
 71	 */
 72	cmp  %dil, %sil
 73	jl .Lcopy_backward
 74	subq $0x20, %rdx
 75.Lcopy_forward_loop:
 76	subq $0x20,	%rdx
 77
 78	/*
 79	 * Move in blocks of 4x8 bytes:
 80	 */
 81	movq 0*8(%rsi),	%r8
 82	movq 1*8(%rsi),	%r9
 83	movq 2*8(%rsi),	%r10
 84	movq 3*8(%rsi),	%r11
 85	leaq 4*8(%rsi),	%rsi
 86
 87	movq %r8,	0*8(%rdi)
 88	movq %r9,	1*8(%rdi)
 89	movq %r10,	2*8(%rdi)
 90	movq %r11,	3*8(%rdi)
 91	leaq 4*8(%rdi),	%rdi
 92	jae  .Lcopy_forward_loop
 93	addl $0x20,	%edx
 94	jmp  .Lhandle_tail
 95
 96.Lcopy_backward:
 97	/*
 98	 * Calculate copy position to tail.
 99	 */
100	addq %rdx,	%rsi
101	addq %rdx,	%rdi
102	subq $0x20,	%rdx
103	/*
104	 * At most 3 ALU operations in one cycle,
105	 * so append NOPS in the same 16 bytes trunk.
106	 */
107	.p2align 4
108.Lcopy_backward_loop:
109	subq $0x20,	%rdx
110	movq -1*8(%rsi),	%r8
111	movq -2*8(%rsi),	%r9
112	movq -3*8(%rsi),	%r10
113	movq -4*8(%rsi),	%r11
114	leaq -4*8(%rsi),	%rsi
115	movq %r8,		-1*8(%rdi)
116	movq %r9,		-2*8(%rdi)
117	movq %r10,		-3*8(%rdi)
118	movq %r11,		-4*8(%rdi)
119	leaq -4*8(%rdi),	%rdi
120	jae  .Lcopy_backward_loop
121
122	/*
123	 * Calculate copy position to head.
124	 */
125	addl $0x20,	%edx
126	subq %rdx,	%rsi
127	subq %rdx,	%rdi
128.Lhandle_tail:
129	cmpl $16,	%edx
130	jb   .Lless_16bytes
131
132	/*
133	 * Move data from 16 bytes to 31 bytes.
134	 */
135	movq 0*8(%rsi), %r8
136	movq 1*8(%rsi),	%r9
137	movq -2*8(%rsi, %rdx),	%r10
138	movq -1*8(%rsi, %rdx),	%r11
139	movq %r8,	0*8(%rdi)
140	movq %r9,	1*8(%rdi)
141	movq %r10,	-2*8(%rdi, %rdx)
142	movq %r11,	-1*8(%rdi, %rdx)
143	retq
144	.p2align 4
145.Lless_16bytes:
146	cmpl $8,	%edx
147	jb   .Lless_8bytes
148	/*
149	 * Move data from 8 bytes to 15 bytes.
150	 */
151	movq 0*8(%rsi),	%r8
152	movq -1*8(%rsi, %rdx),	%r9
153	movq %r8,	0*8(%rdi)
154	movq %r9,	-1*8(%rdi, %rdx)
155	retq
156	.p2align 4
157.Lless_8bytes:
158	cmpl $4,	%edx
159	jb   .Lless_3bytes
160
161	/*
162	 * Move data from 4 bytes to 7 bytes.
163	 */
164	movl (%rsi), %ecx
165	movl -4(%rsi, %rdx), %r8d
166	movl %ecx, (%rdi)
167	movl %r8d, -4(%rdi, %rdx)
168	retq
169	.p2align 4
170.Lless_3bytes:
171	subl $1, %edx
172	jb .Lend
173	/*
174	 * Move data from 1 bytes to 3 bytes.
175	 */
176	movzbl (%rsi), %ecx
177	jz .Lstore_1byte
178	movzbq 1(%rsi), %r8
179	movzbq (%rsi, %rdx), %r9
180	movb %r8b, 1(%rdi)
181	movb %r9b, (%rdi, %rdx)
182.Lstore_1byte:
183	movb %cl, (%rdi)
184
185.Lend:
186	retq
187SYM_FUNC_END(memcpy_orig)
188
189.popsection
190
191#ifndef CONFIG_UML
192
193MCSAFE_TEST_CTL
194
195/*
196 * __memcpy_mcsafe - memory copy with machine check exception handling
197 * Note that we only catch machine checks when reading the source addresses.
198 * Writes to target are posted and don't generate machine checks.
199 */
200SYM_FUNC_START(__memcpy_mcsafe)
201	cmpl $8, %edx
202	/* Less than 8 bytes? Go to byte copy loop */
203	jb .L_no_whole_words
204
205	/* Check for bad alignment of source */
206	testl $7, %esi
207	/* Already aligned */
208	jz .L_8byte_aligned
209
210	/* Copy one byte at a time until source is 8-byte aligned */
211	movl %esi, %ecx
212	andl $7, %ecx
213	subl $8, %ecx
214	negl %ecx
215	subl %ecx, %edx
216.L_read_leading_bytes:
217	movb (%rsi), %al
218	MCSAFE_TEST_SRC %rsi 1 .E_leading_bytes
219	MCSAFE_TEST_DST %rdi 1 .E_leading_bytes
220.L_write_leading_bytes:
221	movb %al, (%rdi)
222	incq %rsi
223	incq %rdi
224	decl %ecx
225	jnz .L_read_leading_bytes
226
227.L_8byte_aligned:
228	movl %edx, %ecx
229	andl $7, %edx
230	shrl $3, %ecx
231	jz .L_no_whole_words
232
233.L_read_words:
234	movq (%rsi), %r8
235	MCSAFE_TEST_SRC %rsi 8 .E_read_words
236	MCSAFE_TEST_DST %rdi 8 .E_write_words
237.L_write_words:
238	movq %r8, (%rdi)
239	addq $8, %rsi
240	addq $8, %rdi
241	decl %ecx
242	jnz .L_read_words
243
244	/* Any trailing bytes? */
245.L_no_whole_words:
246	andl %edx, %edx
247	jz .L_done_memcpy_trap
248
249	/* Copy trailing bytes */
250	movl %edx, %ecx
251.L_read_trailing_bytes:
252	movb (%rsi), %al
253	MCSAFE_TEST_SRC %rsi 1 .E_trailing_bytes
254	MCSAFE_TEST_DST %rdi 1 .E_trailing_bytes
255.L_write_trailing_bytes:
256	movb %al, (%rdi)
257	incq %rsi
258	incq %rdi
259	decl %ecx
260	jnz .L_read_trailing_bytes
261
262	/* Copy successful. Return zero */
263.L_done_memcpy_trap:
264	xorl %eax, %eax
265.L_done:
266	ret
267SYM_FUNC_END(__memcpy_mcsafe)
268EXPORT_SYMBOL_GPL(__memcpy_mcsafe)
269
270	.section .fixup, "ax"
271	/*
272	 * Return number of bytes not copied for any failure. Note that
273	 * there is no "tail" handling since the source buffer is 8-byte
274	 * aligned and poison is cacheline aligned.
275	 */
276.E_read_words:
277	shll	$3, %ecx
278.E_leading_bytes:
279	addl	%edx, %ecx
280.E_trailing_bytes:
281	mov	%ecx, %eax
282	jmp	.L_done
283
284	/*
285	 * For write fault handling, given the destination is unaligned,
286	 * we handle faults on multi-byte writes with a byte-by-byte
287	 * copy up to the write-protected page.
288	 */
289.E_write_words:
290	shll	$3, %ecx
291	addl	%edx, %ecx
292	movl	%ecx, %edx
293	jmp mcsafe_handle_tail
294
295	.previous
296
297	_ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)
298	_ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)
299	_ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)
300	_ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
301	_ASM_EXTABLE(.L_write_words, .E_write_words)
302	_ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)
303#endif