Loading...
1/*
2 * Normally compiler builtins are used, but sometimes the compiler calls out
3 * of line code. Based on asm-i386/string.h.
4 *
5 * This assembly file is re-written from memmove_64.c file.
6 * - Copyright 2011 Fenghua Yu <fenghua.yu@intel.com>
7 */
8#include <linux/linkage.h>
9#include <asm/cpufeatures.h>
10#include <asm/alternative-asm.h>
11#include <asm/export.h>
12
13#undef memmove
14
15/*
16 * Implement memmove(). This can handle overlap between src and dst.
17 *
18 * Input:
19 * rdi: dest
20 * rsi: src
21 * rdx: count
22 *
23 * Output:
24 * rax: dest
25 */
26.weak memmove
27
28ENTRY(memmove)
29ENTRY(__memmove)
30
31 /* Handle more 32 bytes in loop */
32 mov %rdi, %rax
33 cmp $0x20, %rdx
34 jb 1f
35
36 /* Decide forward/backward copy mode */
37 cmp %rdi, %rsi
38 jge .Lmemmove_begin_forward
39 mov %rsi, %r8
40 add %rdx, %r8
41 cmp %rdi, %r8
42 jg 2f
43
44.Lmemmove_begin_forward:
45 ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; retq", X86_FEATURE_ERMS
46
47 /*
48 * movsq instruction have many startup latency
49 * so we handle small size by general register.
50 */
51 cmp $680, %rdx
52 jb 3f
53 /*
54 * movsq instruction is only good for aligned case.
55 */
56
57 cmpb %dil, %sil
58 je 4f
593:
60 sub $0x20, %rdx
61 /*
62 * We gobble 32 bytes forward in each loop.
63 */
645:
65 sub $0x20, %rdx
66 movq 0*8(%rsi), %r11
67 movq 1*8(%rsi), %r10
68 movq 2*8(%rsi), %r9
69 movq 3*8(%rsi), %r8
70 leaq 4*8(%rsi), %rsi
71
72 movq %r11, 0*8(%rdi)
73 movq %r10, 1*8(%rdi)
74 movq %r9, 2*8(%rdi)
75 movq %r8, 3*8(%rdi)
76 leaq 4*8(%rdi), %rdi
77 jae 5b
78 addq $0x20, %rdx
79 jmp 1f
80 /*
81 * Handle data forward by movsq.
82 */
83 .p2align 4
844:
85 movq %rdx, %rcx
86 movq -8(%rsi, %rdx), %r11
87 lea -8(%rdi, %rdx), %r10
88 shrq $3, %rcx
89 rep movsq
90 movq %r11, (%r10)
91 jmp 13f
92.Lmemmove_end_forward:
93
94 /*
95 * Handle data backward by movsq.
96 */
97 .p2align 4
987:
99 movq %rdx, %rcx
100 movq (%rsi), %r11
101 movq %rdi, %r10
102 leaq -8(%rsi, %rdx), %rsi
103 leaq -8(%rdi, %rdx), %rdi
104 shrq $3, %rcx
105 std
106 rep movsq
107 cld
108 movq %r11, (%r10)
109 jmp 13f
110
111 /*
112 * Start to prepare for backward copy.
113 */
114 .p2align 4
1152:
116 cmp $680, %rdx
117 jb 6f
118 cmp %dil, %sil
119 je 7b
1206:
121 /*
122 * Calculate copy position to tail.
123 */
124 addq %rdx, %rsi
125 addq %rdx, %rdi
126 subq $0x20, %rdx
127 /*
128 * We gobble 32 bytes backward in each loop.
129 */
1308:
131 subq $0x20, %rdx
132 movq -1*8(%rsi), %r11
133 movq -2*8(%rsi), %r10
134 movq -3*8(%rsi), %r9
135 movq -4*8(%rsi), %r8
136 leaq -4*8(%rsi), %rsi
137
138 movq %r11, -1*8(%rdi)
139 movq %r10, -2*8(%rdi)
140 movq %r9, -3*8(%rdi)
141 movq %r8, -4*8(%rdi)
142 leaq -4*8(%rdi), %rdi
143 jae 8b
144 /*
145 * Calculate copy position to head.
146 */
147 addq $0x20, %rdx
148 subq %rdx, %rsi
149 subq %rdx, %rdi
1501:
151 cmpq $16, %rdx
152 jb 9f
153 /*
154 * Move data from 16 bytes to 31 bytes.
155 */
156 movq 0*8(%rsi), %r11
157 movq 1*8(%rsi), %r10
158 movq -2*8(%rsi, %rdx), %r9
159 movq -1*8(%rsi, %rdx), %r8
160 movq %r11, 0*8(%rdi)
161 movq %r10, 1*8(%rdi)
162 movq %r9, -2*8(%rdi, %rdx)
163 movq %r8, -1*8(%rdi, %rdx)
164 jmp 13f
165 .p2align 4
1669:
167 cmpq $8, %rdx
168 jb 10f
169 /*
170 * Move data from 8 bytes to 15 bytes.
171 */
172 movq 0*8(%rsi), %r11
173 movq -1*8(%rsi, %rdx), %r10
174 movq %r11, 0*8(%rdi)
175 movq %r10, -1*8(%rdi, %rdx)
176 jmp 13f
17710:
178 cmpq $4, %rdx
179 jb 11f
180 /*
181 * Move data from 4 bytes to 7 bytes.
182 */
183 movl (%rsi), %r11d
184 movl -4(%rsi, %rdx), %r10d
185 movl %r11d, (%rdi)
186 movl %r10d, -4(%rdi, %rdx)
187 jmp 13f
18811:
189 cmp $2, %rdx
190 jb 12f
191 /*
192 * Move data from 2 bytes to 3 bytes.
193 */
194 movw (%rsi), %r11w
195 movw -2(%rsi, %rdx), %r10w
196 movw %r11w, (%rdi)
197 movw %r10w, -2(%rdi, %rdx)
198 jmp 13f
19912:
200 cmp $1, %rdx
201 jb 13f
202 /*
203 * Move data for 1 byte.
204 */
205 movb (%rsi), %r11b
206 movb %r11b, (%rdi)
20713:
208 retq
209ENDPROC(__memmove)
210ENDPROC(memmove)
211EXPORT_SYMBOL(__memmove)
212EXPORT_SYMBOL(memmove)
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Normally compiler builtins are used, but sometimes the compiler calls out
4 * of line code. Based on asm-i386/string.h.
5 *
6 * This assembly file is re-written from memmove_64.c file.
7 * - Copyright 2011 Fenghua Yu <fenghua.yu@intel.com>
8 */
9#include <linux/linkage.h>
10#include <asm/cpufeatures.h>
11#include <asm/alternative-asm.h>
12#include <asm/export.h>
13
14#undef memmove
15
16/*
17 * Implement memmove(). This can handle overlap between src and dst.
18 *
19 * Input:
20 * rdi: dest
21 * rsi: src
22 * rdx: count
23 *
24 * Output:
25 * rax: dest
26 */
27.weak memmove
28
29ENTRY(memmove)
30ENTRY(__memmove)
31
32 /* Handle more 32 bytes in loop */
33 mov %rdi, %rax
34 cmp $0x20, %rdx
35 jb 1f
36
37 /* Decide forward/backward copy mode */
38 cmp %rdi, %rsi
39 jge .Lmemmove_begin_forward
40 mov %rsi, %r8
41 add %rdx, %r8
42 cmp %rdi, %r8
43 jg 2f
44
45.Lmemmove_begin_forward:
46 ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; retq", X86_FEATURE_ERMS
47
48 /*
49 * movsq instruction have many startup latency
50 * so we handle small size by general register.
51 */
52 cmp $680, %rdx
53 jb 3f
54 /*
55 * movsq instruction is only good for aligned case.
56 */
57
58 cmpb %dil, %sil
59 je 4f
603:
61 sub $0x20, %rdx
62 /*
63 * We gobble 32 bytes forward in each loop.
64 */
655:
66 sub $0x20, %rdx
67 movq 0*8(%rsi), %r11
68 movq 1*8(%rsi), %r10
69 movq 2*8(%rsi), %r9
70 movq 3*8(%rsi), %r8
71 leaq 4*8(%rsi), %rsi
72
73 movq %r11, 0*8(%rdi)
74 movq %r10, 1*8(%rdi)
75 movq %r9, 2*8(%rdi)
76 movq %r8, 3*8(%rdi)
77 leaq 4*8(%rdi), %rdi
78 jae 5b
79 addq $0x20, %rdx
80 jmp 1f
81 /*
82 * Handle data forward by movsq.
83 */
84 .p2align 4
854:
86 movq %rdx, %rcx
87 movq -8(%rsi, %rdx), %r11
88 lea -8(%rdi, %rdx), %r10
89 shrq $3, %rcx
90 rep movsq
91 movq %r11, (%r10)
92 jmp 13f
93.Lmemmove_end_forward:
94
95 /*
96 * Handle data backward by movsq.
97 */
98 .p2align 4
997:
100 movq %rdx, %rcx
101 movq (%rsi), %r11
102 movq %rdi, %r10
103 leaq -8(%rsi, %rdx), %rsi
104 leaq -8(%rdi, %rdx), %rdi
105 shrq $3, %rcx
106 std
107 rep movsq
108 cld
109 movq %r11, (%r10)
110 jmp 13f
111
112 /*
113 * Start to prepare for backward copy.
114 */
115 .p2align 4
1162:
117 cmp $680, %rdx
118 jb 6f
119 cmp %dil, %sil
120 je 7b
1216:
122 /*
123 * Calculate copy position to tail.
124 */
125 addq %rdx, %rsi
126 addq %rdx, %rdi
127 subq $0x20, %rdx
128 /*
129 * We gobble 32 bytes backward in each loop.
130 */
1318:
132 subq $0x20, %rdx
133 movq -1*8(%rsi), %r11
134 movq -2*8(%rsi), %r10
135 movq -3*8(%rsi), %r9
136 movq -4*8(%rsi), %r8
137 leaq -4*8(%rsi), %rsi
138
139 movq %r11, -1*8(%rdi)
140 movq %r10, -2*8(%rdi)
141 movq %r9, -3*8(%rdi)
142 movq %r8, -4*8(%rdi)
143 leaq -4*8(%rdi), %rdi
144 jae 8b
145 /*
146 * Calculate copy position to head.
147 */
148 addq $0x20, %rdx
149 subq %rdx, %rsi
150 subq %rdx, %rdi
1511:
152 cmpq $16, %rdx
153 jb 9f
154 /*
155 * Move data from 16 bytes to 31 bytes.
156 */
157 movq 0*8(%rsi), %r11
158 movq 1*8(%rsi), %r10
159 movq -2*8(%rsi, %rdx), %r9
160 movq -1*8(%rsi, %rdx), %r8
161 movq %r11, 0*8(%rdi)
162 movq %r10, 1*8(%rdi)
163 movq %r9, -2*8(%rdi, %rdx)
164 movq %r8, -1*8(%rdi, %rdx)
165 jmp 13f
166 .p2align 4
1679:
168 cmpq $8, %rdx
169 jb 10f
170 /*
171 * Move data from 8 bytes to 15 bytes.
172 */
173 movq 0*8(%rsi), %r11
174 movq -1*8(%rsi, %rdx), %r10
175 movq %r11, 0*8(%rdi)
176 movq %r10, -1*8(%rdi, %rdx)
177 jmp 13f
17810:
179 cmpq $4, %rdx
180 jb 11f
181 /*
182 * Move data from 4 bytes to 7 bytes.
183 */
184 movl (%rsi), %r11d
185 movl -4(%rsi, %rdx), %r10d
186 movl %r11d, (%rdi)
187 movl %r10d, -4(%rdi, %rdx)
188 jmp 13f
18911:
190 cmp $2, %rdx
191 jb 12f
192 /*
193 * Move data from 2 bytes to 3 bytes.
194 */
195 movw (%rsi), %r11w
196 movw -2(%rsi, %rdx), %r10w
197 movw %r11w, (%rdi)
198 movw %r10w, -2(%rdi, %rdx)
199 jmp 13f
20012:
201 cmp $1, %rdx
202 jb 13f
203 /*
204 * Move data for 1 byte.
205 */
206 movb (%rsi), %r11b
207 movb %r11b, (%rdi)
20813:
209 retq
210ENDPROC(__memmove)
211ENDPROC(memmove)
212EXPORT_SYMBOL(__memmove)
213EXPORT_SYMBOL(memmove)