Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * linux/arch/arm/lib/copy_template.s
4 *
5 * Code template for optimized memory copy functions
6 *
7 * Author: Nicolas Pitre
8 * Created: Sep 28, 2005
9 * Copyright: MontaVista Software, Inc.
10 */
11
12/*
13 * Theory of operation
14 * -------------------
15 *
16 * This file provides the core code for a forward memory copy used in
17 * the implementation of memcopy(), copy_to_user() and copy_from_user().
18 *
19 * The including file must define the following accessor macros
20 * according to the need of the given function:
21 *
22 * ldr1w ptr reg abort
23 *
24 * This loads one word from 'ptr', stores it in 'reg' and increments
25 * 'ptr' to the next word. The 'abort' argument is used for fixup tables.
26 *
27 * ldr4w ptr reg1 reg2 reg3 reg4 abort
28 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
29 *
30 * This loads four or eight words starting from 'ptr', stores them
31 * in provided registers and increments 'ptr' past those words.
32 * The'abort' argument is used for fixup tables.
33 *
34 * ldr1b ptr reg cond abort
35 *
36 * Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
37 * It also must apply the condition code if provided, otherwise the
38 * "al" condition is assumed by default.
39 *
40 * str1w ptr reg abort
41 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
42 * str1b ptr reg cond abort
43 *
44 * Same as their ldr* counterparts, but data is stored to 'ptr' location
45 * rather than being loaded.
46 *
47 * enter reg1 reg2
48 *
49 * Preserve the provided registers on the stack plus any additional
50 * data as needed by the implementation including this code. Called
51 * upon code entry.
52 *
53 * usave reg1 reg2
54 *
55 * Unwind annotation macro is corresponding for 'enter' macro.
56 * It tell unwinder that preserved some provided registers on the stack
57 * and additional data by a prior 'enter' macro.
58 *
59 * exit reg1 reg2
60 *
61 * Restore registers with the values previously saved with the
62 * 'preserv' macro. Called upon code termination.
63 *
64 * LDR1W_SHIFT
65 * STR1W_SHIFT
66 *
67 * Correction to be applied to the "ip" register when branching into
68 * the ldr1w or str1w instructions (some of these macros may expand to
69 * than one 32bit instruction in Thumb-2)
70 */
71
72 UNWIND( .fnstart )
73 enter r4, UNWIND(fpreg,) lr
74 UNWIND( .setfp fpreg, sp )
75 UNWIND( mov fpreg, sp )
76
77 subs r2, r2, #4
78 blt 8f
79 ands ip, r0, #3
80 PLD( pld [r1, #0] )
81 bne 9f
82 ands ip, r1, #3
83 bne 10f
84
851: subs r2, r2, #(28)
86 stmfd sp!, {r5, r6, r8, r9}
87 blt 5f
88
89 CALGN( ands ip, r0, #31 )
90 CALGN( rsb r3, ip, #32 )
91 CALGN( sbcsne r4, r3, r2 ) @ C is always set here
92 CALGN( bcs 2f )
93 CALGN( adr r4, 6f )
94 CALGN( subs r2, r2, r3 ) @ C gets set
95 CALGN( add pc, r4, ip )
96
97 PLD( pld [r1, #0] )
982: PLD( subs r2, r2, #96 )
99 PLD( pld [r1, #28] )
100 PLD( blt 4f )
101 PLD( pld [r1, #60] )
102 PLD( pld [r1, #92] )
103
1043: PLD( pld [r1, #124] )
1054: ldr8w r1, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
106 subs r2, r2, #32
107 str8w r0, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
108 bge 3b
109 PLD( cmn r2, #96 )
110 PLD( bge 4b )
111
1125: ands ip, r2, #28
113 rsb ip, ip, #32
114#if LDR1W_SHIFT > 0
115 lsl ip, ip, #LDR1W_SHIFT
116#endif
117 addne pc, pc, ip @ C is always clear here
118 b 7f
1196:
120 .rept (1 << LDR1W_SHIFT)
121 W(nop)
122 .endr
123 ldr1w r1, r3, abort=20f
124 ldr1w r1, r4, abort=20f
125 ldr1w r1, r5, abort=20f
126 ldr1w r1, r6, abort=20f
127 ldr1w r1, r8, abort=20f
128 ldr1w r1, r9, abort=20f
129 ldr1w r1, lr, abort=20f
130
131#if LDR1W_SHIFT < STR1W_SHIFT
132 lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
133#elif LDR1W_SHIFT > STR1W_SHIFT
134 lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
135#endif
136 add pc, pc, ip
137 nop
138 .rept (1 << STR1W_SHIFT)
139 W(nop)
140 .endr
141 str1w r0, r3, abort=20f
142 str1w r0, r4, abort=20f
143 str1w r0, r5, abort=20f
144 str1w r0, r6, abort=20f
145 str1w r0, r8, abort=20f
146 str1w r0, r9, abort=20f
147 str1w r0, lr, abort=20f
148
149 CALGN( bcs 2b )
150
1517: ldmfd sp!, {r5, r6, r8, r9}
152
1538: movs r2, r2, lsl #31
154 ldr1b r1, r3, ne, abort=21f
155 ldr1b r1, r4, cs, abort=21f
156 ldr1b r1, ip, cs, abort=21f
157 str1b r0, r3, ne, abort=21f
158 str1b r0, r4, cs, abort=21f
159 str1b r0, ip, cs, abort=21f
160
161 exit r4, UNWIND(fpreg,) pc
162
1639: rsb ip, ip, #4
164 cmp ip, #2
165 ldr1b r1, r3, gt, abort=21f
166 ldr1b r1, r4, ge, abort=21f
167 ldr1b r1, lr, abort=21f
168 str1b r0, r3, gt, abort=21f
169 str1b r0, r4, ge, abort=21f
170 subs r2, r2, ip
171 str1b r0, lr, abort=21f
172 blt 8b
173 ands ip, r1, #3
174 beq 1b
175
17610: bic r1, r1, #3
177 cmp ip, #2
178 ldr1w r1, lr, abort=21f
179 beq 17f
180 bgt 18f
181
182
183 .macro forward_copy_shift pull push
184
185 subs r2, r2, #28
186 blt 14f
187
188 CALGN( ands ip, r0, #31 )
189 CALGN( rsb ip, ip, #32 )
190 CALGN( sbcsne r4, ip, r2 ) @ C is always set here
191 CALGN( subcc r2, r2, ip )
192 CALGN( bcc 15f )
193
19411: stmfd sp!, {r5, r6, r8 - r10}
195
196 PLD( pld [r1, #0] )
197 PLD( subs r2, r2, #96 )
198 PLD( pld [r1, #28] )
199 PLD( blt 13f )
200 PLD( pld [r1, #60] )
201 PLD( pld [r1, #92] )
202
20312: PLD( pld [r1, #124] )
20413: ldr4w r1, r4, r5, r6, r8, abort=19f
205 mov r3, lr, lspull #\pull
206 subs r2, r2, #32
207 ldr4w r1, r9, r10, ip, lr, abort=19f
208 orr r3, r3, r4, lspush #\push
209 mov r4, r4, lspull #\pull
210 orr r4, r4, r5, lspush #\push
211 mov r5, r5, lspull #\pull
212 orr r5, r5, r6, lspush #\push
213 mov r6, r6, lspull #\pull
214 orr r6, r6, r8, lspush #\push
215 mov r8, r8, lspull #\pull
216 orr r8, r8, r9, lspush #\push
217 mov r9, r9, lspull #\pull
218 orr r9, r9, r10, lspush #\push
219 mov r10, r10, lspull #\pull
220 orr r10, r10, ip, lspush #\push
221 mov ip, ip, lspull #\pull
222 orr ip, ip, lr, lspush #\push
223 str8w r0, r3, r4, r5, r6, r8, r9, r10, ip, abort=19f
224 bge 12b
225 PLD( cmn r2, #96 )
226 PLD( bge 13b )
227
228 ldmfd sp!, {r5, r6, r8 - r10}
229
23014: ands ip, r2, #28
231 beq 16f
232
23315: mov r3, lr, lspull #\pull
234 ldr1w r1, lr, abort=21f
235 subs ip, ip, #4
236 orr r3, r3, lr, lspush #\push
237 str1w r0, r3, abort=21f
238 bgt 15b
239 CALGN( cmp r2, #0 )
240 CALGN( bge 11b )
241
24216: sub r1, r1, #(\push / 8)
243 b 8b
244
245 .endm
246
247
248 forward_copy_shift pull=8 push=24
249
25017: forward_copy_shift pull=16 push=16
251
25218: forward_copy_shift pull=24 push=8
253
254 UNWIND( .fnend )
255
256/*
257 * Abort preamble and completion macros.
258 * If a fixup handler is required then those macros must surround it.
259 * It is assumed that the fixup code will handle the private part of
260 * the exit macro.
261 */
262
263 .macro copy_abort_preamble
26419: ldmfd sp!, {r5, r6, r8 - r10}
265 b 21f
26620: ldmfd sp!, {r5, r6, r8, r9}
26721:
268 .endm
269
270 .macro copy_abort_end
271 ldmfd sp!, {r4, UNWIND(fpreg,) pc}
272 .endm
273
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * linux/arch/arm/lib/copy_template.s
4 *
5 * Code template for optimized memory copy functions
6 *
7 * Author: Nicolas Pitre
8 * Created: Sep 28, 2005
9 * Copyright: MontaVista Software, Inc.
10 */
11
12/*
13 * Theory of operation
14 * -------------------
15 *
16 * This file provides the core code for a forward memory copy used in
17 * the implementation of memcopy(), copy_to_user() and copy_from_user().
18 *
19 * The including file must define the following accessor macros
20 * according to the need of the given function:
21 *
22 * ldr1w ptr reg abort
23 *
24 * This loads one word from 'ptr', stores it in 'reg' and increments
25 * 'ptr' to the next word. The 'abort' argument is used for fixup tables.
26 *
27 * ldr4w ptr reg1 reg2 reg3 reg4 abort
28 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
29 *
30 * This loads four or eight words starting from 'ptr', stores them
31 * in provided registers and increments 'ptr' past those words.
32 * The'abort' argument is used for fixup tables.
33 *
34 * ldr1b ptr reg cond abort
35 *
36 * Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
37 * It also must apply the condition code if provided, otherwise the
38 * "al" condition is assumed by default.
39 *
40 * str1w ptr reg abort
41 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
42 * str1b ptr reg cond abort
43 *
44 * Same as their ldr* counterparts, but data is stored to 'ptr' location
45 * rather than being loaded.
46 *
47 * enter reg1 reg2
48 *
49 * Preserve the provided registers on the stack plus any additional
50 * data as needed by the implementation including this code. Called
51 * upon code entry.
52 *
53 * usave reg1 reg2
54 *
55 * Unwind annotation macro is corresponding for 'enter' macro.
56 * It tell unwinder that preserved some provided registers on the stack
57 * and additional data by a prior 'enter' macro.
58 *
59 * exit reg1 reg2
60 *
61 * Restore registers with the values previously saved with the
62 * 'preserv' macro. Called upon code termination.
63 *
64 * LDR1W_SHIFT
65 * STR1W_SHIFT
66 *
67 * Correction to be applied to the "ip" register when branching into
68 * the ldr1w or str1w instructions (some of these macros may expand to
69 * than one 32bit instruction in Thumb-2)
70 */
71
72 UNWIND( .fnstart )
73 enter r4, UNWIND(fpreg,) lr
74 UNWIND( .setfp fpreg, sp )
75 UNWIND( mov fpreg, sp )
76
77 subs r2, r2, #4
78 blt 8f
79 ands ip, r0, #3
80 PLD( pld [r1, #0] )
81 bne 9f
82 ands ip, r1, #3
83 bne 10f
84
851: subs r2, r2, #(28)
86 stmfd sp!, {r5, r6, r8, r9}
87 blt 5f
88
89 CALGN( ands ip, r0, #31 )
90 CALGN( rsb r3, ip, #32 )
91 CALGN( sbcsne r4, r3, r2 ) @ C is always set here
92 CALGN( bcs 2f )
93 CALGN( adr r4, 6f )
94 CALGN( subs r2, r2, r3 ) @ C gets set
95 CALGN( add pc, r4, ip )
96
97 PLD( pld [r1, #0] )
982: PLD( subs r2, r2, #96 )
99 PLD( pld [r1, #28] )
100 PLD( blt 4f )
101 PLD( pld [r1, #60] )
102 PLD( pld [r1, #92] )
103
1043: PLD( pld [r1, #124] )
1054: ldr8w r1, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
106 subs r2, r2, #32
107 str8w r0, r3, r4, r5, r6, r8, r9, ip, lr, abort=20f
108 bge 3b
109 PLD( cmn r2, #96 )
110 PLD( bge 4b )
111
1125: ands ip, r2, #28
113 rsb ip, ip, #32
114#if LDR1W_SHIFT > 0
115 lsl ip, ip, #LDR1W_SHIFT
116#endif
117 addne pc, pc, ip @ C is always clear here
118 b 7f
1196:
120 .rept (1 << LDR1W_SHIFT)
121 W(nop)
122 .endr
123 ldr1w r1, r3, abort=20f
124 ldr1w r1, r4, abort=20f
125 ldr1w r1, r5, abort=20f
126 ldr1w r1, r6, abort=20f
127 ldr1w r1, r8, abort=20f
128 ldr1w r1, r9, abort=20f
129 ldr1w r1, lr, abort=20f
130
131#if LDR1W_SHIFT < STR1W_SHIFT
132 lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
133#elif LDR1W_SHIFT > STR1W_SHIFT
134 lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
135#endif
136 add pc, pc, ip
137 nop
138 .rept (1 << STR1W_SHIFT)
139 W(nop)
140 .endr
141 str1w r0, r3, abort=20f
142 str1w r0, r4, abort=20f
143 str1w r0, r5, abort=20f
144 str1w r0, r6, abort=20f
145 str1w r0, r8, abort=20f
146 str1w r0, r9, abort=20f
147 str1w r0, lr, abort=20f
148
149 CALGN( bcs 2b )
150
1517: ldmfd sp!, {r5, r6, r8, r9}
152
1538: movs r2, r2, lsl #31
154 ldr1b r1, r3, ne, abort=21f
155 ldr1b r1, r4, cs, abort=21f
156 ldr1b r1, ip, cs, abort=21f
157 str1b r0, r3, ne, abort=21f
158 str1b r0, r4, cs, abort=21f
159 str1b r0, ip, cs, abort=21f
160
161 exit r4, UNWIND(fpreg,) pc
162
1639: rsb ip, ip, #4
164 cmp ip, #2
165 ldr1b r1, r3, gt, abort=21f
166 ldr1b r1, r4, ge, abort=21f
167 ldr1b r1, lr, abort=21f
168 str1b r0, r3, gt, abort=21f
169 str1b r0, r4, ge, abort=21f
170 subs r2, r2, ip
171 str1b r0, lr, abort=21f
172 blt 8b
173 ands ip, r1, #3
174 beq 1b
175
17610: bic r1, r1, #3
177 cmp ip, #2
178 ldr1w r1, lr, abort=21f
179 beq 17f
180 bgt 18f
181
182
183 .macro forward_copy_shift pull push
184
185 subs r2, r2, #28
186 blt 14f
187
188 CALGN( ands ip, r0, #31 )
189 CALGN( rsb ip, ip, #32 )
190 CALGN( sbcsne r4, ip, r2 ) @ C is always set here
191 CALGN( subcc r2, r2, ip )
192 CALGN( bcc 15f )
193
19411: stmfd sp!, {r5, r6, r8 - r10}
195
196 PLD( pld [r1, #0] )
197 PLD( subs r2, r2, #96 )
198 PLD( pld [r1, #28] )
199 PLD( blt 13f )
200 PLD( pld [r1, #60] )
201 PLD( pld [r1, #92] )
202
20312: PLD( pld [r1, #124] )
20413: ldr4w r1, r4, r5, r6, r8, abort=19f
205 mov r3, lr, lspull #\pull
206 subs r2, r2, #32
207 ldr4w r1, r9, r10, ip, lr, abort=19f
208 orr r3, r3, r4, lspush #\push
209 mov r4, r4, lspull #\pull
210 orr r4, r4, r5, lspush #\push
211 mov r5, r5, lspull #\pull
212 orr r5, r5, r6, lspush #\push
213 mov r6, r6, lspull #\pull
214 orr r6, r6, r8, lspush #\push
215 mov r8, r8, lspull #\pull
216 orr r8, r8, r9, lspush #\push
217 mov r9, r9, lspull #\pull
218 orr r9, r9, r10, lspush #\push
219 mov r10, r10, lspull #\pull
220 orr r10, r10, ip, lspush #\push
221 mov ip, ip, lspull #\pull
222 orr ip, ip, lr, lspush #\push
223 str8w r0, r3, r4, r5, r6, r8, r9, r10, ip, abort=19f
224 bge 12b
225 PLD( cmn r2, #96 )
226 PLD( bge 13b )
227
228 ldmfd sp!, {r5, r6, r8 - r10}
229
23014: ands ip, r2, #28
231 beq 16f
232
23315: mov r3, lr, lspull #\pull
234 ldr1w r1, lr, abort=21f
235 subs ip, ip, #4
236 orr r3, r3, lr, lspush #\push
237 str1w r0, r3, abort=21f
238 bgt 15b
239 CALGN( cmp r2, #0 )
240 CALGN( bge 11b )
241
24216: sub r1, r1, #(\push / 8)
243 b 8b
244
245 .endm
246
247
248 forward_copy_shift pull=8 push=24
249
25017: forward_copy_shift pull=16 push=16
251
25218: forward_copy_shift pull=24 push=8
253
254 UNWIND( .fnend )
255
256/*
257 * Abort preamble and completion macros.
258 * If a fixup handler is required then those macros must surround it.
259 * It is assumed that the fixup code will handle the private part of
260 * the exit macro.
261 */
262
263 .macro copy_abort_preamble
26419: ldmfd sp!, {r5, r6, r8 - r10}
265 b 21f
26620: ldmfd sp!, {r5, r6, r8, r9}
26721:
268 .endm
269
270 .macro copy_abort_end
271 ldmfd sp!, {r4, UNWIND(fpreg,) pc}
272 .endm
273