Loading...
1/*
2 * linux/arch/arm/lib/copy_template.s
3 *
4 * Code template for optimized memory copy functions
5 *
6 * Author: Nicolas Pitre
7 * Created: Sep 28, 2005
8 * Copyright: MontaVista Software, Inc.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
13 */
14
15/*
16 * Theory of operation
17 * -------------------
18 *
19 * This file provides the core code for a forward memory copy used in
20 * the implementation of memcopy(), copy_to_user() and copy_from_user().
21 *
22 * The including file must define the following accessor macros
23 * according to the need of the given function:
24 *
25 * ldr1w ptr reg abort
26 *
27 * This loads one word from 'ptr', stores it in 'reg' and increments
28 * 'ptr' to the next word. The 'abort' argument is used for fixup tables.
29 *
30 * ldr4w ptr reg1 reg2 reg3 reg4 abort
31 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
32 *
33 * This loads four or eight words starting from 'ptr', stores them
34 * in provided registers and increments 'ptr' past those words.
35 * The'abort' argument is used for fixup tables.
36 *
37 * ldr1b ptr reg cond abort
38 *
39 * Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
40 * It also must apply the condition code if provided, otherwise the
41 * "al" condition is assumed by default.
42 *
43 * str1w ptr reg abort
44 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
45 * str1b ptr reg cond abort
46 *
47 * Same as their ldr* counterparts, but data is stored to 'ptr' location
48 * rather than being loaded.
49 *
50 * enter reg1 reg2
51 *
52 * Preserve the provided registers on the stack plus any additional
53 * data as needed by the implementation including this code. Called
54 * upon code entry.
55 *
56 * exit reg1 reg2
57 *
58 * Restore registers with the values previously saved with the
59 * 'preserv' macro. Called upon code termination.
60 *
61 * LDR1W_SHIFT
62 * STR1W_SHIFT
63 *
64 * Correction to be applied to the "ip" register when branching into
65 * the ldr1w or str1w instructions (some of these macros may expand to
66 * than one 32bit instruction in Thumb-2)
67 */
68
69
70 enter r4, lr
71
72 subs r2, r2, #4
73 blt 8f
74 ands ip, r0, #3
75 PLD( pld [r1, #0] )
76 bne 9f
77 ands ip, r1, #3
78 bne 10f
79
801: subs r2, r2, #(28)
81 stmfd sp!, {r5 - r8}
82 blt 5f
83
84 CALGN( ands ip, r0, #31 )
85 CALGN( rsb r3, ip, #32 )
86 CALGN( sbcnes r4, r3, r2 ) @ C is always set here
87 CALGN( bcs 2f )
88 CALGN( adr r4, 6f )
89 CALGN( subs r2, r2, r3 ) @ C gets set
90 CALGN( add pc, r4, ip )
91
92 PLD( pld [r1, #0] )
932: PLD( subs r2, r2, #96 )
94 PLD( pld [r1, #28] )
95 PLD( blt 4f )
96 PLD( pld [r1, #60] )
97 PLD( pld [r1, #92] )
98
993: PLD( pld [r1, #124] )
1004: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
101 subs r2, r2, #32
102 str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
103 bge 3b
104 PLD( cmn r2, #96 )
105 PLD( bge 4b )
106
1075: ands ip, r2, #28
108 rsb ip, ip, #32
109#if LDR1W_SHIFT > 0
110 lsl ip, ip, #LDR1W_SHIFT
111#endif
112 addne pc, pc, ip @ C is always clear here
113 b 7f
1146:
115 .rept (1 << LDR1W_SHIFT)
116 W(nop)
117 .endr
118 ldr1w r1, r3, abort=20f
119 ldr1w r1, r4, abort=20f
120 ldr1w r1, r5, abort=20f
121 ldr1w r1, r6, abort=20f
122 ldr1w r1, r7, abort=20f
123 ldr1w r1, r8, abort=20f
124 ldr1w r1, lr, abort=20f
125
126#if LDR1W_SHIFT < STR1W_SHIFT
127 lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
128#elif LDR1W_SHIFT > STR1W_SHIFT
129 lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
130#endif
131 add pc, pc, ip
132 nop
133 .rept (1 << STR1W_SHIFT)
134 W(nop)
135 .endr
136 str1w r0, r3, abort=20f
137 str1w r0, r4, abort=20f
138 str1w r0, r5, abort=20f
139 str1w r0, r6, abort=20f
140 str1w r0, r7, abort=20f
141 str1w r0, r8, abort=20f
142 str1w r0, lr, abort=20f
143
144 CALGN( bcs 2b )
145
1467: ldmfd sp!, {r5 - r8}
147
1488: movs r2, r2, lsl #31
149 ldr1b r1, r3, ne, abort=21f
150 ldr1b r1, r4, cs, abort=21f
151 ldr1b r1, ip, cs, abort=21f
152 str1b r0, r3, ne, abort=21f
153 str1b r0, r4, cs, abort=21f
154 str1b r0, ip, cs, abort=21f
155
156 exit r4, pc
157
1589: rsb ip, ip, #4
159 cmp ip, #2
160 ldr1b r1, r3, gt, abort=21f
161 ldr1b r1, r4, ge, abort=21f
162 ldr1b r1, lr, abort=21f
163 str1b r0, r3, gt, abort=21f
164 str1b r0, r4, ge, abort=21f
165 subs r2, r2, ip
166 str1b r0, lr, abort=21f
167 blt 8b
168 ands ip, r1, #3
169 beq 1b
170
17110: bic r1, r1, #3
172 cmp ip, #2
173 ldr1w r1, lr, abort=21f
174 beq 17f
175 bgt 18f
176
177
178 .macro forward_copy_shift pull push
179
180 subs r2, r2, #28
181 blt 14f
182
183 CALGN( ands ip, r0, #31 )
184 CALGN( rsb ip, ip, #32 )
185 CALGN( sbcnes r4, ip, r2 ) @ C is always set here
186 CALGN( subcc r2, r2, ip )
187 CALGN( bcc 15f )
188
18911: stmfd sp!, {r5 - r9}
190
191 PLD( pld [r1, #0] )
192 PLD( subs r2, r2, #96 )
193 PLD( pld [r1, #28] )
194 PLD( blt 13f )
195 PLD( pld [r1, #60] )
196 PLD( pld [r1, #92] )
197
19812: PLD( pld [r1, #124] )
19913: ldr4w r1, r4, r5, r6, r7, abort=19f
200 mov r3, lr, pull #\pull
201 subs r2, r2, #32
202 ldr4w r1, r8, r9, ip, lr, abort=19f
203 orr r3, r3, r4, push #\push
204 mov r4, r4, pull #\pull
205 orr r4, r4, r5, push #\push
206 mov r5, r5, pull #\pull
207 orr r5, r5, r6, push #\push
208 mov r6, r6, pull #\pull
209 orr r6, r6, r7, push #\push
210 mov r7, r7, pull #\pull
211 orr r7, r7, r8, push #\push
212 mov r8, r8, pull #\pull
213 orr r8, r8, r9, push #\push
214 mov r9, r9, pull #\pull
215 orr r9, r9, ip, push #\push
216 mov ip, ip, pull #\pull
217 orr ip, ip, lr, push #\push
218 str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
219 bge 12b
220 PLD( cmn r2, #96 )
221 PLD( bge 13b )
222
223 ldmfd sp!, {r5 - r9}
224
22514: ands ip, r2, #28
226 beq 16f
227
22815: mov r3, lr, pull #\pull
229 ldr1w r1, lr, abort=21f
230 subs ip, ip, #4
231 orr r3, r3, lr, push #\push
232 str1w r0, r3, abort=21f
233 bgt 15b
234 CALGN( cmp r2, #0 )
235 CALGN( bge 11b )
236
23716: sub r1, r1, #(\push / 8)
238 b 8b
239
240 .endm
241
242
243 forward_copy_shift pull=8 push=24
244
24517: forward_copy_shift pull=16 push=16
246
24718: forward_copy_shift pull=24 push=8
248
249
250/*
251 * Abort preamble and completion macros.
252 * If a fixup handler is required then those macros must surround it.
253 * It is assumed that the fixup code will handle the private part of
254 * the exit macro.
255 */
256
257 .macro copy_abort_preamble
25819: ldmfd sp!, {r5 - r9}
259 b 21f
26020: ldmfd sp!, {r5 - r8}
26121:
262 .endm
263
264 .macro copy_abort_end
265 ldmfd sp!, {r4, pc}
266 .endm
267
1/*
2 * linux/arch/arm/lib/copy_template.s
3 *
4 * Code template for optimized memory copy functions
5 *
6 * Author: Nicolas Pitre
7 * Created: Sep 28, 2005
8 * Copyright: MontaVista Software, Inc.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
13 */
14
15/*
16 * Theory of operation
17 * -------------------
18 *
19 * This file provides the core code for a forward memory copy used in
20 * the implementation of memcopy(), copy_to_user() and copy_from_user().
21 *
22 * The including file must define the following accessor macros
23 * according to the need of the given function:
24 *
25 * ldr1w ptr reg abort
26 *
27 * This loads one word from 'ptr', stores it in 'reg' and increments
28 * 'ptr' to the next word. The 'abort' argument is used for fixup tables.
29 *
30 * ldr4w ptr reg1 reg2 reg3 reg4 abort
31 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
32 *
33 * This loads four or eight words starting from 'ptr', stores them
34 * in provided registers and increments 'ptr' past those words.
35 * The'abort' argument is used for fixup tables.
36 *
37 * ldr1b ptr reg cond abort
38 *
39 * Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
40 * It also must apply the condition code if provided, otherwise the
41 * "al" condition is assumed by default.
42 *
43 * str1w ptr reg abort
44 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
45 * str1b ptr reg cond abort
46 *
47 * Same as their ldr* counterparts, but data is stored to 'ptr' location
48 * rather than being loaded.
49 *
50 * enter reg1 reg2
51 *
52 * Preserve the provided registers on the stack plus any additional
53 * data as needed by the implementation including this code. Called
54 * upon code entry.
55 *
56 * usave reg1 reg2
57 *
58 * Unwind annotation macro is corresponding for 'enter' macro.
59 * It tell unwinder that preserved some provided registers on the stack
60 * and additional data by a prior 'enter' macro.
61 *
62 * exit reg1 reg2
63 *
64 * Restore registers with the values previously saved with the
65 * 'preserv' macro. Called upon code termination.
66 *
67 * LDR1W_SHIFT
68 * STR1W_SHIFT
69 *
70 * Correction to be applied to the "ip" register when branching into
71 * the ldr1w or str1w instructions (some of these macros may expand to
72 * than one 32bit instruction in Thumb-2)
73 */
74
75
76 UNWIND( .fnstart )
77 enter r4, lr
78 UNWIND( .fnend )
79
80 UNWIND( .fnstart )
81 usave r4, lr @ in first stmdb block
82
83 subs r2, r2, #4
84 blt 8f
85 ands ip, r0, #3
86 PLD( pld [r1, #0] )
87 bne 9f
88 ands ip, r1, #3
89 bne 10f
90
911: subs r2, r2, #(28)
92 stmfd sp!, {r5 - r8}
93 UNWIND( .fnend )
94
95 UNWIND( .fnstart )
96 usave r4, lr
97 UNWIND( .save {r5 - r8} ) @ in second stmfd block
98 blt 5f
99
100 CALGN( ands ip, r0, #31 )
101 CALGN( rsb r3, ip, #32 )
102 CALGN( sbcnes r4, r3, r2 ) @ C is always set here
103 CALGN( bcs 2f )
104 CALGN( adr r4, 6f )
105 CALGN( subs r2, r2, r3 ) @ C gets set
106 CALGN( add pc, r4, ip )
107
108 PLD( pld [r1, #0] )
1092: PLD( subs r2, r2, #96 )
110 PLD( pld [r1, #28] )
111 PLD( blt 4f )
112 PLD( pld [r1, #60] )
113 PLD( pld [r1, #92] )
114
1153: PLD( pld [r1, #124] )
1164: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
117 subs r2, r2, #32
118 str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
119 bge 3b
120 PLD( cmn r2, #96 )
121 PLD( bge 4b )
122
1235: ands ip, r2, #28
124 rsb ip, ip, #32
125#if LDR1W_SHIFT > 0
126 lsl ip, ip, #LDR1W_SHIFT
127#endif
128 addne pc, pc, ip @ C is always clear here
129 b 7f
1306:
131 .rept (1 << LDR1W_SHIFT)
132 W(nop)
133 .endr
134 ldr1w r1, r3, abort=20f
135 ldr1w r1, r4, abort=20f
136 ldr1w r1, r5, abort=20f
137 ldr1w r1, r6, abort=20f
138 ldr1w r1, r7, abort=20f
139 ldr1w r1, r8, abort=20f
140 ldr1w r1, lr, abort=20f
141
142#if LDR1W_SHIFT < STR1W_SHIFT
143 lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
144#elif LDR1W_SHIFT > STR1W_SHIFT
145 lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
146#endif
147 add pc, pc, ip
148 nop
149 .rept (1 << STR1W_SHIFT)
150 W(nop)
151 .endr
152 str1w r0, r3, abort=20f
153 str1w r0, r4, abort=20f
154 str1w r0, r5, abort=20f
155 str1w r0, r6, abort=20f
156 str1w r0, r7, abort=20f
157 str1w r0, r8, abort=20f
158 str1w r0, lr, abort=20f
159
160 CALGN( bcs 2b )
161
1627: ldmfd sp!, {r5 - r8}
163 UNWIND( .fnend ) @ end of second stmfd block
164
165 UNWIND( .fnstart )
166 usave r4, lr @ still in first stmdb block
1678: movs r2, r2, lsl #31
168 ldr1b r1, r3, ne, abort=21f
169 ldr1b r1, r4, cs, abort=21f
170 ldr1b r1, ip, cs, abort=21f
171 str1b r0, r3, ne, abort=21f
172 str1b r0, r4, cs, abort=21f
173 str1b r0, ip, cs, abort=21f
174
175 exit r4, pc
176
1779: rsb ip, ip, #4
178 cmp ip, #2
179 ldr1b r1, r3, gt, abort=21f
180 ldr1b r1, r4, ge, abort=21f
181 ldr1b r1, lr, abort=21f
182 str1b r0, r3, gt, abort=21f
183 str1b r0, r4, ge, abort=21f
184 subs r2, r2, ip
185 str1b r0, lr, abort=21f
186 blt 8b
187 ands ip, r1, #3
188 beq 1b
189
19010: bic r1, r1, #3
191 cmp ip, #2
192 ldr1w r1, lr, abort=21f
193 beq 17f
194 bgt 18f
195 UNWIND( .fnend )
196
197
198 .macro forward_copy_shift pull push
199
200 UNWIND( .fnstart )
201 usave r4, lr @ still in first stmdb block
202 subs r2, r2, #28
203 blt 14f
204
205 CALGN( ands ip, r0, #31 )
206 CALGN( rsb ip, ip, #32 )
207 CALGN( sbcnes r4, ip, r2 ) @ C is always set here
208 CALGN( subcc r2, r2, ip )
209 CALGN( bcc 15f )
210
21111: stmfd sp!, {r5 - r9}
212 UNWIND( .fnend )
213
214 UNWIND( .fnstart )
215 usave r4, lr
216 UNWIND( .save {r5 - r9} ) @ in new second stmfd block
217 PLD( pld [r1, #0] )
218 PLD( subs r2, r2, #96 )
219 PLD( pld [r1, #28] )
220 PLD( blt 13f )
221 PLD( pld [r1, #60] )
222 PLD( pld [r1, #92] )
223
22412: PLD( pld [r1, #124] )
22513: ldr4w r1, r4, r5, r6, r7, abort=19f
226 mov r3, lr, lspull #\pull
227 subs r2, r2, #32
228 ldr4w r1, r8, r9, ip, lr, abort=19f
229 orr r3, r3, r4, lspush #\push
230 mov r4, r4, lspull #\pull
231 orr r4, r4, r5, lspush #\push
232 mov r5, r5, lspull #\pull
233 orr r5, r5, r6, lspush #\push
234 mov r6, r6, lspull #\pull
235 orr r6, r6, r7, lspush #\push
236 mov r7, r7, lspull #\pull
237 orr r7, r7, r8, lspush #\push
238 mov r8, r8, lspull #\pull
239 orr r8, r8, r9, lspush #\push
240 mov r9, r9, lspull #\pull
241 orr r9, r9, ip, lspush #\push
242 mov ip, ip, lspull #\pull
243 orr ip, ip, lr, lspush #\push
244 str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
245 bge 12b
246 PLD( cmn r2, #96 )
247 PLD( bge 13b )
248
249 ldmfd sp!, {r5 - r9}
250 UNWIND( .fnend ) @ end of the second stmfd block
251
252 UNWIND( .fnstart )
253 usave r4, lr @ still in first stmdb block
25414: ands ip, r2, #28
255 beq 16f
256
25715: mov r3, lr, lspull #\pull
258 ldr1w r1, lr, abort=21f
259 subs ip, ip, #4
260 orr r3, r3, lr, lspush #\push
261 str1w r0, r3, abort=21f
262 bgt 15b
263 CALGN( cmp r2, #0 )
264 CALGN( bge 11b )
265
26616: sub r1, r1, #(\push / 8)
267 b 8b
268 UNWIND( .fnend )
269
270 .endm
271
272
273 forward_copy_shift pull=8 push=24
274
27517: forward_copy_shift pull=16 push=16
276
27718: forward_copy_shift pull=24 push=8
278
279
280/*
281 * Abort preamble and completion macros.
282 * If a fixup handler is required then those macros must surround it.
283 * It is assumed that the fixup code will handle the private part of
284 * the exit macro.
285 */
286
287 .macro copy_abort_preamble
28819: ldmfd sp!, {r5 - r9}
289 b 21f
29020: ldmfd sp!, {r5 - r8}
29121:
292 .endm
293
294 .macro copy_abort_end
295 ldmfd sp!, {r4, pc}
296 .endm
297