1/* SPDX-License-Identifier: GPL-2.0 */
  2
  3#include <linux/export.h>
  4#include <linux/linkage.h>
  5
  6SYM_FUNC_START(memmove)
  7/*
  8 * void *memmove(void *dest_in, const void *src_in, size_t n)
  9 * -mregparm=3 passes these in registers:
 10 * dest_in: %eax
 11 * src_in: %edx
 12 * n: %ecx
 13 * See also: arch/x86/entry/calling.h for description of the calling convention.
 14 *
 15 * n can remain in %ecx, but for `rep movsl`, we'll need dest in %edi and src
 16 * in %esi.
 17 */
 18.set dest_in, %eax
 19.set dest, %edi
 20.set src_in, %edx
 21.set src, %esi
 22.set n, %ecx
 23.set tmp0, %edx
 24.set tmp0w, %dx
 25.set tmp1, %ebx
 26.set tmp1w, %bx
 27.set tmp2, %eax
 28.set tmp3b, %cl
 29
 30/*
 31 * Save all callee-saved registers, because this function is going to clobber
 32 * all of them:
 33 */
 34	pushl	%ebp
 35	movl	%esp, %ebp	// set standard frame pointer
 36
 37	pushl	%ebx
 38	pushl	%edi
 39	pushl	%esi
 40	pushl	%eax		// save 'dest_in' parameter [eax] as the return value
 41
 42	movl src_in, src
 43	movl dest_in, dest
 44
 45	/* Handle more 16 bytes in loop */
 46	cmpl	$0x10, n
 47	jb	.Lmove_16B
 48
 49	/* Decide forward/backward copy mode */
 50	cmpl	dest, src
 51	jb	.Lbackwards_header
 52
 53	/*
 54	 * movs instruction have many startup latency
 55	 * so we handle small size by general register.
 56	 */
 57	cmpl	$680, n
 58	jb	.Ltoo_small_forwards
 59	/* movs instruction is only good for aligned case. */
 60	movl	src, tmp0
 61	xorl	dest, tmp0
 62	andl	$0xff, tmp0
 63	jz	.Lforward_movs
 64.Ltoo_small_forwards:
 65	subl	$0x10, n
 66
 67	/* We gobble 16 bytes forward in each loop. */
 68.Lmove_16B_forwards_loop:
 69	subl	$0x10, n
 70	movl	0*4(src), tmp0
 71	movl	1*4(src), tmp1
 72	movl	tmp0, 0*4(dest)
 73	movl	tmp1, 1*4(dest)
 74	movl	2*4(src), tmp0
 75	movl	3*4(src), tmp1
 76	movl	tmp0, 2*4(dest)
 77	movl	tmp1, 3*4(dest)
 78	leal	0x10(src), src
 79	leal	0x10(dest), dest
 80	jae	.Lmove_16B_forwards_loop
 81	addl	$0x10, n
 82	jmp	.Lmove_16B
 83
 84	/* Handle data forward by movs. */
 85.p2align 4
 86.Lforward_movs:
 87	movl	-4(src, n), tmp0
 88	leal	-4(dest, n), tmp1
 89	shrl	$2, n
 90	rep	movsl
 91	movl	tmp0, (tmp1)
 92	jmp	.Ldone
 93
 94	/* Handle data backward by movs. */
 95.p2align 4
 96.Lbackwards_movs:
 97	movl	(src), tmp0
 98	movl	dest, tmp1
 99	leal	-4(src, n), src
100	leal	-4(dest, n), dest
101	shrl	$2, n
102	std
103	rep	movsl
104	movl	tmp0,(tmp1)
105	cld
106	jmp	.Ldone
107
108	/* Start to prepare for backward copy. */
109.p2align 4
110.Lbackwards_header:
111	cmpl	$680, n
112	jb	.Ltoo_small_backwards
113	movl	src, tmp0
114	xorl	dest, tmp0
115	andl	$0xff, tmp0
116	jz	.Lbackwards_movs
117
118	/* Calculate copy position to tail. */
119.Ltoo_small_backwards:
120	addl	n, src
121	addl	n, dest
122	subl	$0x10, n
123
124	/* We gobble 16 bytes backward in each loop. */
125.Lmove_16B_backwards_loop:
126	subl	$0x10, n
127
128	movl	-1*4(src), tmp0
129	movl	-2*4(src), tmp1
130	movl	tmp0, -1*4(dest)
131	movl	tmp1, -2*4(dest)
132	movl	-3*4(src), tmp0
133	movl	-4*4(src), tmp1
134	movl	tmp0, -3*4(dest)
135	movl	tmp1, -4*4(dest)
136	leal	-0x10(src), src
137	leal	-0x10(dest), dest
138	jae	.Lmove_16B_backwards_loop
139	/* Calculate copy position to head. */
140	addl	$0x10, n
141	subl	n, src
142	subl	n, dest
143
144	/* Move data from 8 bytes to 15 bytes. */
145.p2align 4
146.Lmove_16B:
147	cmpl	$8, n
148	jb	.Lmove_8B
149	movl	0*4(src), tmp0
150	movl	1*4(src), tmp1
151	movl	-2*4(src, n), tmp2
152	movl	-1*4(src, n), src
153
154	movl	tmp0, 0*4(dest)
155	movl	tmp1, 1*4(dest)
156	movl	tmp2, -2*4(dest, n)
157	movl	src, -1*4(dest, n)
158	jmp	.Ldone
159
160	/* Move data from 4 bytes to 7 bytes. */
161.p2align 4
162.Lmove_8B:
163	cmpl	$4, n
164	jb	.Lmove_4B
165	movl	0*4(src), tmp0
166	movl	-1*4(src, n), tmp1
167	movl	tmp0, 0*4(dest)
168	movl	tmp1, -1*4(dest, n)
169	jmp	.Ldone
170
171	/* Move data from 2 bytes to 3 bytes. */
172.p2align 4
173.Lmove_4B:
174	cmpl	$2, n
175	jb	.Lmove_1B
176	movw	0*2(src), tmp0w
177	movw	-1*2(src, n), tmp1w
178	movw	tmp0w, 0*2(dest)
179	movw	tmp1w, -1*2(dest, n)
180	jmp	.Ldone
181
182	/* Move data for 1 byte. */
183.p2align 4
184.Lmove_1B:
185	cmpl	$1, n
186	jb	.Ldone
187	movb	(src), tmp3b
188	movb	tmp3b, (dest)
189.p2align 4
190.Ldone:
191	popl	dest_in	// restore 'dest_in' [eax] as the return value
192	/* Restore all callee-saved registers: */
193	popl	%esi
194	popl	%edi
195	popl	%ebx
196	popl	%ebp
197
198	RET
199SYM_FUNC_END(memmove)
200EXPORT_SYMBOL(memmove)