Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * linux/arch/arm/lib/memmove.S
4 *
5 * Author: Nicolas Pitre
6 * Created: Sep 28, 2005
7 * Copyright: (C) MontaVista Software Inc.
8 */
9
10#include <linux/linkage.h>
11#include <asm/assembler.h>
12#include <asm/unwind.h>
13
14 .text
15
16/*
17 * Prototype: void *memmove(void *dest, const void *src, size_t n);
18 *
19 * Note:
20 *
21 * If the memory regions don't overlap, we simply branch to memcpy which is
22 * normally a bit faster. Otherwise the copy is done going downwards. This
23 * is a transposition of the code from copy_template.S but with the copy
24 * occurring in the opposite direction.
25 */
26
27ENTRY(__memmove)
28WEAK(memmove)
29 UNWIND( .fnstart )
30
31 subs ip, r0, r1
32 cmphi r2, ip
33 bls __memcpy
34 UNWIND( .fnend )
35
36 UNWIND( .fnstart )
37 UNWIND( .save {r0, r4, fpreg, lr} )
38 stmfd sp!, {r0, r4, UNWIND(fpreg,) lr}
39 UNWIND( .setfp fpreg, sp )
40 UNWIND( mov fpreg, sp )
41 add r1, r1, r2
42 add r0, r0, r2
43 subs r2, r2, #4
44 blt 8f
45 ands ip, r0, #3
46 PLD( pld [r1, #-4] )
47 bne 9f
48 ands ip, r1, #3
49 bne 10f
50
511: subs r2, r2, #(28)
52 stmfd sp!, {r5, r6, r8, r9}
53 blt 5f
54
55 CALGN( ands ip, r0, #31 )
56 CALGN( sbcsne r4, ip, r2 ) @ C is always set here
57 CALGN( bcs 2f )
58 CALGN( adr r4, 6f )
59 CALGN( subs r2, r2, ip ) @ C is set here
60 CALGN( rsb ip, ip, #32 )
61 CALGN( add pc, r4, ip )
62
63 PLD( pld [r1, #-4] )
642: PLD( subs r2, r2, #96 )
65 PLD( pld [r1, #-32] )
66 PLD( blt 4f )
67 PLD( pld [r1, #-64] )
68 PLD( pld [r1, #-96] )
69
703: PLD( pld [r1, #-128] )
714: ldmdb r1!, {r3, r4, r5, r6, r8, r9, ip, lr}
72 subs r2, r2, #32
73 stmdb r0!, {r3, r4, r5, r6, r8, r9, ip, lr}
74 bge 3b
75 PLD( cmn r2, #96 )
76 PLD( bge 4b )
77
785: ands ip, r2, #28
79 rsb ip, ip, #32
80 addne pc, pc, ip @ C is always clear here
81 b 7f
826: W(nop)
83 W(ldr) r3, [r1, #-4]!
84 W(ldr) r4, [r1, #-4]!
85 W(ldr) r5, [r1, #-4]!
86 W(ldr) r6, [r1, #-4]!
87 W(ldr) r8, [r1, #-4]!
88 W(ldr) r9, [r1, #-4]!
89 W(ldr) lr, [r1, #-4]!
90
91 add pc, pc, ip
92 nop
93 W(nop)
94 W(str) r3, [r0, #-4]!
95 W(str) r4, [r0, #-4]!
96 W(str) r5, [r0, #-4]!
97 W(str) r6, [r0, #-4]!
98 W(str) r8, [r0, #-4]!
99 W(str) r9, [r0, #-4]!
100 W(str) lr, [r0, #-4]!
101
102 CALGN( bcs 2b )
103
1047: ldmfd sp!, {r5, r6, r8, r9}
105
1068: movs r2, r2, lsl #31
107 ldrbne r3, [r1, #-1]!
108 ldrbcs r4, [r1, #-1]!
109 ldrbcs ip, [r1, #-1]
110 strbne r3, [r0, #-1]!
111 strbcs r4, [r0, #-1]!
112 strbcs ip, [r0, #-1]
113 ldmfd sp!, {r0, r4, UNWIND(fpreg,) pc}
114
1159: cmp ip, #2
116 ldrbgt r3, [r1, #-1]!
117 ldrbge r4, [r1, #-1]!
118 ldrb lr, [r1, #-1]!
119 strbgt r3, [r0, #-1]!
120 strbge r4, [r0, #-1]!
121 subs r2, r2, ip
122 strb lr, [r0, #-1]!
123 blt 8b
124 ands ip, r1, #3
125 beq 1b
126
12710: bic r1, r1, #3
128 cmp ip, #2
129 ldr r3, [r1, #0]
130 beq 17f
131 blt 18f
132
133
134 .macro backward_copy_shift push pull
135
136 subs r2, r2, #28
137 blt 14f
138
139 CALGN( ands ip, r0, #31 )
140 CALGN( sbcsne r4, ip, r2 ) @ C is always set here
141 CALGN( subcc r2, r2, ip )
142 CALGN( bcc 15f )
143
14411: stmfd sp!, {r5, r6, r8 - r10}
145
146 PLD( pld [r1, #-4] )
147 PLD( subs r2, r2, #96 )
148 PLD( pld [r1, #-32] )
149 PLD( blt 13f )
150 PLD( pld [r1, #-64] )
151 PLD( pld [r1, #-96] )
152
15312: PLD( pld [r1, #-128] )
15413: ldmdb r1!, {r8, r9, r10, ip}
155 mov lr, r3, lspush #\push
156 subs r2, r2, #32
157 ldmdb r1!, {r3, r4, r5, r6}
158 orr lr, lr, ip, lspull #\pull
159 mov ip, ip, lspush #\push
160 orr ip, ip, r10, lspull #\pull
161 mov r10, r10, lspush #\push
162 orr r10, r10, r9, lspull #\pull
163 mov r9, r9, lspush #\push
164 orr r9, r9, r8, lspull #\pull
165 mov r8, r8, lspush #\push
166 orr r8, r8, r6, lspull #\pull
167 mov r6, r6, lspush #\push
168 orr r6, r6, r5, lspull #\pull
169 mov r5, r5, lspush #\push
170 orr r5, r5, r4, lspull #\pull
171 mov r4, r4, lspush #\push
172 orr r4, r4, r3, lspull #\pull
173 stmdb r0!, {r4 - r6, r8 - r10, ip, lr}
174 bge 12b
175 PLD( cmn r2, #96 )
176 PLD( bge 13b )
177
178 ldmfd sp!, {r5, r6, r8 - r10}
179
18014: ands ip, r2, #28
181 beq 16f
182
18315: mov lr, r3, lspush #\push
184 ldr r3, [r1, #-4]!
185 subs ip, ip, #4
186 orr lr, lr, r3, lspull #\pull
187 str lr, [r0, #-4]!
188 bgt 15b
189 CALGN( cmp r2, #0 )
190 CALGN( bge 11b )
191
19216: add r1, r1, #(\pull / 8)
193 b 8b
194
195 .endm
196
197
198 backward_copy_shift push=8 pull=24
199
20017: backward_copy_shift push=16 pull=16
201
20218: backward_copy_shift push=24 pull=8
203
204 UNWIND( .fnend )
205ENDPROC(memmove)
206ENDPROC(__memmove)
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * linux/arch/arm/lib/memmove.S
4 *
5 * Author: Nicolas Pitre
6 * Created: Sep 28, 2005
7 * Copyright: (C) MontaVista Software Inc.
8 */
9
10#include <linux/linkage.h>
11#include <asm/assembler.h>
12#include <asm/unwind.h>
13
14 .text
15
16/*
17 * Prototype: void *memmove(void *dest, const void *src, size_t n);
18 *
19 * Note:
20 *
21 * If the memory regions don't overlap, we simply branch to memcpy which is
22 * normally a bit faster. Otherwise the copy is done going downwards. This
23 * is a transposition of the code from copy_template.S but with the copy
24 * occurring in the opposite direction.
25 */
26
27ENTRY(__memmove)
28WEAK(memmove)
29 UNWIND( .fnstart )
30
31 subs ip, r0, r1
32 cmphi r2, ip
33 bls __memcpy
34
35 stmfd sp!, {r0, r4, lr}
36 UNWIND( .fnend )
37
38 UNWIND( .fnstart )
39 UNWIND( .save {r0, r4, lr} ) @ in first stmfd block
40 add r1, r1, r2
41 add r0, r0, r2
42 subs r2, r2, #4
43 blt 8f
44 ands ip, r0, #3
45 PLD( pld [r1, #-4] )
46 bne 9f
47 ands ip, r1, #3
48 bne 10f
49
501: subs r2, r2, #(28)
51 stmfd sp!, {r5 - r8}
52 UNWIND( .fnend )
53
54 UNWIND( .fnstart )
55 UNWIND( .save {r0, r4, lr} )
56 UNWIND( .save {r5 - r8} ) @ in second stmfd block
57 blt 5f
58
59 CALGN( ands ip, r0, #31 )
60 CALGN( sbcsne r4, ip, r2 ) @ C is always set here
61 CALGN( bcs 2f )
62 CALGN( adr r4, 6f )
63 CALGN( subs r2, r2, ip ) @ C is set here
64 CALGN( rsb ip, ip, #32 )
65 CALGN( add pc, r4, ip )
66
67 PLD( pld [r1, #-4] )
682: PLD( subs r2, r2, #96 )
69 PLD( pld [r1, #-32] )
70 PLD( blt 4f )
71 PLD( pld [r1, #-64] )
72 PLD( pld [r1, #-96] )
73
743: PLD( pld [r1, #-128] )
754: ldmdb r1!, {r3, r4, r5, r6, r7, r8, ip, lr}
76 subs r2, r2, #32
77 stmdb r0!, {r3, r4, r5, r6, r7, r8, ip, lr}
78 bge 3b
79 PLD( cmn r2, #96 )
80 PLD( bge 4b )
81
825: ands ip, r2, #28
83 rsb ip, ip, #32
84 addne pc, pc, ip @ C is always clear here
85 b 7f
866: W(nop)
87 W(ldr) r3, [r1, #-4]!
88 W(ldr) r4, [r1, #-4]!
89 W(ldr) r5, [r1, #-4]!
90 W(ldr) r6, [r1, #-4]!
91 W(ldr) r7, [r1, #-4]!
92 W(ldr) r8, [r1, #-4]!
93 W(ldr) lr, [r1, #-4]!
94
95 add pc, pc, ip
96 nop
97 W(nop)
98 W(str) r3, [r0, #-4]!
99 W(str) r4, [r0, #-4]!
100 W(str) r5, [r0, #-4]!
101 W(str) r6, [r0, #-4]!
102 W(str) r7, [r0, #-4]!
103 W(str) r8, [r0, #-4]!
104 W(str) lr, [r0, #-4]!
105
106 CALGN( bcs 2b )
107
1087: ldmfd sp!, {r5 - r8}
109 UNWIND( .fnend ) @ end of second stmfd block
110
111 UNWIND( .fnstart )
112 UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block
113
1148: movs r2, r2, lsl #31
115 ldrbne r3, [r1, #-1]!
116 ldrbcs r4, [r1, #-1]!
117 ldrbcs ip, [r1, #-1]
118 strbne r3, [r0, #-1]!
119 strbcs r4, [r0, #-1]!
120 strbcs ip, [r0, #-1]
121 ldmfd sp!, {r0, r4, pc}
122
1239: cmp ip, #2
124 ldrbgt r3, [r1, #-1]!
125 ldrbge r4, [r1, #-1]!
126 ldrb lr, [r1, #-1]!
127 strbgt r3, [r0, #-1]!
128 strbge r4, [r0, #-1]!
129 subs r2, r2, ip
130 strb lr, [r0, #-1]!
131 blt 8b
132 ands ip, r1, #3
133 beq 1b
134
13510: bic r1, r1, #3
136 cmp ip, #2
137 ldr r3, [r1, #0]
138 beq 17f
139 blt 18f
140 UNWIND( .fnend )
141
142
143 .macro backward_copy_shift push pull
144
145 UNWIND( .fnstart )
146 UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block
147 subs r2, r2, #28
148 blt 14f
149
150 CALGN( ands ip, r0, #31 )
151 CALGN( sbcsne r4, ip, r2 ) @ C is always set here
152 CALGN( subcc r2, r2, ip )
153 CALGN( bcc 15f )
154
15511: stmfd sp!, {r5 - r9}
156 UNWIND( .fnend )
157
158 UNWIND( .fnstart )
159 UNWIND( .save {r0, r4, lr} )
160 UNWIND( .save {r5 - r9} ) @ in new second stmfd block
161
162 PLD( pld [r1, #-4] )
163 PLD( subs r2, r2, #96 )
164 PLD( pld [r1, #-32] )
165 PLD( blt 13f )
166 PLD( pld [r1, #-64] )
167 PLD( pld [r1, #-96] )
168
16912: PLD( pld [r1, #-128] )
17013: ldmdb r1!, {r7, r8, r9, ip}
171 mov lr, r3, lspush #\push
172 subs r2, r2, #32
173 ldmdb r1!, {r3, r4, r5, r6}
174 orr lr, lr, ip, lspull #\pull
175 mov ip, ip, lspush #\push
176 orr ip, ip, r9, lspull #\pull
177 mov r9, r9, lspush #\push
178 orr r9, r9, r8, lspull #\pull
179 mov r8, r8, lspush #\push
180 orr r8, r8, r7, lspull #\pull
181 mov r7, r7, lspush #\push
182 orr r7, r7, r6, lspull #\pull
183 mov r6, r6, lspush #\push
184 orr r6, r6, r5, lspull #\pull
185 mov r5, r5, lspush #\push
186 orr r5, r5, r4, lspull #\pull
187 mov r4, r4, lspush #\push
188 orr r4, r4, r3, lspull #\pull
189 stmdb r0!, {r4 - r9, ip, lr}
190 bge 12b
191 PLD( cmn r2, #96 )
192 PLD( bge 13b )
193
194 ldmfd sp!, {r5 - r9}
195 UNWIND( .fnend ) @ end of the second stmfd block
196
197 UNWIND( .fnstart )
198 UNWIND( .save {r0, r4, lr} ) @ still in first stmfd block
199
20014: ands ip, r2, #28
201 beq 16f
202
20315: mov lr, r3, lspush #\push
204 ldr r3, [r1, #-4]!
205 subs ip, ip, #4
206 orr lr, lr, r3, lspull #\pull
207 str lr, [r0, #-4]!
208 bgt 15b
209 CALGN( cmp r2, #0 )
210 CALGN( bge 11b )
211
21216: add r1, r1, #(\pull / 8)
213 b 8b
214 UNWIND( .fnend )
215
216 .endm
217
218
219 backward_copy_shift push=8 pull=24
220
22117: backward_copy_shift push=16 pull=16
222
22318: backward_copy_shift push=24 pull=8
224
225ENDPROC(memmove)
226ENDPROC(__memmove)