Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *
4 * Optimized version of the standard memcpy() function
5 *
6 * Inputs:
7 * in0: destination address
8 * in1: source address
9 * in2: number of bytes to copy
10 * Output:
11 * no return value
12 *
13 * Copyright (C) 2000-2001 Hewlett-Packard Co
14 * Stephane Eranian <eranian@hpl.hp.com>
15 * David Mosberger-Tang <davidm@hpl.hp.com>
16 */
17#include <asm/asmmacro.h>
18#include <asm/export.h>
19
20GLOBAL_ENTRY(memcpy)
21
22# define MEM_LAT 21 /* latency to memory */
23
24# define dst r2
25# define src r3
26# define retval r8
27# define saved_pfs r9
28# define saved_lc r10
29# define saved_pr r11
30# define cnt r16
31# define src2 r17
32# define t0 r18
33# define t1 r19
34# define t2 r20
35# define t3 r21
36# define t4 r22
37# define src_end r23
38
39# define N (MEM_LAT + 4)
40# define Nrot ((N + 7) & ~7)
41
42 /*
43 * First, check if everything (src, dst, len) is a multiple of eight. If
44 * so, we handle everything with no taken branches (other than the loop
45 * itself) and a small icache footprint. Otherwise, we jump off to
46 * the more general copy routine handling arbitrary
47 * sizes/alignment etc.
48 */
49 .prologue
50 .save ar.pfs, saved_pfs
51 alloc saved_pfs=ar.pfs,3,Nrot,0,Nrot
52 .save ar.lc, saved_lc
53 mov saved_lc=ar.lc
54 or t0=in0,in1
55 ;;
56
57 or t0=t0,in2
58 .save pr, saved_pr
59 mov saved_pr=pr
60
61 .body
62
63 cmp.eq p6,p0=in2,r0 // zero length?
64 mov retval=in0 // return dst
65(p6) br.ret.spnt.many rp // zero length, return immediately
66 ;;
67
68 mov dst=in0 // copy because of rotation
69 shr.u cnt=in2,3 // number of 8-byte words to copy
70 mov pr.rot=1<<16
71 ;;
72
73 adds cnt=-1,cnt // br.ctop is repeat/until
74 cmp.gtu p7,p0=16,in2 // copying less than 16 bytes?
75 mov ar.ec=N
76 ;;
77
78 and t0=0x7,t0
79 mov ar.lc=cnt
80 ;;
81 cmp.ne p6,p0=t0,r0
82
83 mov src=in1 // copy because of rotation
84(p7) br.cond.spnt.few .memcpy_short
85(p6) br.cond.spnt.few .memcpy_long
86 ;;
87 nop.m 0
88 ;;
89 nop.m 0
90 nop.i 0
91 ;;
92 nop.m 0
93 ;;
94 .rotr val[N]
95 .rotp p[N]
96 .align 32
971: { .mib
98(p[0]) ld8 val[0]=[src],8
99 nop.i 0
100 brp.loop.imp 1b, 2f
101}
1022: { .mfb
103(p[N-1])st8 [dst]=val[N-1],8
104 nop.f 0
105 br.ctop.dptk.few 1b
106}
107 ;;
108 mov ar.lc=saved_lc
109 mov pr=saved_pr,-1
110 mov ar.pfs=saved_pfs
111 br.ret.sptk.many rp
112
113 /*
114 * Small (<16 bytes) unaligned copying is done via a simple byte-at-the-time
115 * copy loop. This performs relatively poorly on Itanium, but it doesn't
116 * get used very often (gcc inlines small copies) and due to atomicity
117 * issues, we want to avoid read-modify-write of entire words.
118 */
119 .align 32
120.memcpy_short:
121 adds cnt=-1,in2 // br.ctop is repeat/until
122 mov ar.ec=MEM_LAT
123 brp.loop.imp 1f, 2f
124 ;;
125 mov ar.lc=cnt
126 ;;
127 nop.m 0
128 ;;
129 nop.m 0
130 nop.i 0
131 ;;
132 nop.m 0
133 ;;
134 nop.m 0
135 ;;
136 /*
137 * It is faster to put a stop bit in the loop here because it makes
138 * the pipeline shorter (and latency is what matters on short copies).
139 */
140 .align 32
1411: { .mib
142(p[0]) ld1 val[0]=[src],1
143 nop.i 0
144 brp.loop.imp 1b, 2f
145} ;;
1462: { .mfb
147(p[MEM_LAT-1])st1 [dst]=val[MEM_LAT-1],1
148 nop.f 0
149 br.ctop.dptk.few 1b
150} ;;
151 mov ar.lc=saved_lc
152 mov pr=saved_pr,-1
153 mov ar.pfs=saved_pfs
154 br.ret.sptk.many rp
155
156 /*
157 * Large (>= 16 bytes) copying is done in a fancy way. Latency isn't
158 * an overriding concern here, but throughput is. We first do
159 * sub-word copying until the destination is aligned, then we check
160 * if the source is also aligned. If so, we do a simple load/store-loop
161 * until there are less than 8 bytes left over and then we do the tail,
162 * by storing the last few bytes using sub-word copying. If the source
163 * is not aligned, we branch off to the non-congruent loop.
164 *
165 * stage: op:
166 * 0 ld
167 * :
168 * MEM_LAT+3 shrp
169 * MEM_LAT+4 st
170 *
171 * On Itanium, the pipeline itself runs without stalls. However, br.ctop
172 * seems to introduce an unavoidable bubble in the pipeline so the overall
173 * latency is 2 cycles/iteration. This gives us a _copy_ throughput
174 * of 4 byte/cycle. Still not bad.
175 */
176# undef N
177# undef Nrot
178# define N (MEM_LAT + 5) /* number of stages */
179# define Nrot ((N+1 + 2 + 7) & ~7) /* number of rotating regs */
180
181#define LOG_LOOP_SIZE 6
182
183.memcpy_long:
184 alloc t3=ar.pfs,3,Nrot,0,Nrot // resize register frame
185 and t0=-8,src // t0 = src & ~7
186 and t2=7,src // t2 = src & 7
187 ;;
188 ld8 t0=[t0] // t0 = 1st source word
189 adds src2=7,src // src2 = (src + 7)
190 sub t4=r0,dst // t4 = -dst
191 ;;
192 and src2=-8,src2 // src2 = (src + 7) & ~7
193 shl t2=t2,3 // t2 = 8*(src & 7)
194 shl t4=t4,3 // t4 = 8*(dst & 7)
195 ;;
196 ld8 t1=[src2] // t1 = 1st source word if src is 8-byte aligned, 2nd otherwise
197 sub t3=64,t2 // t3 = 64-8*(src & 7)
198 shr.u t0=t0,t2
199 ;;
200 add src_end=src,in2
201 shl t1=t1,t3
202 mov pr=t4,0x38 // (p5,p4,p3)=(dst & 7)
203 ;;
204 or t0=t0,t1
205 mov cnt=r0
206 adds src_end=-1,src_end
207 ;;
208(p3) st1 [dst]=t0,1
209(p3) shr.u t0=t0,8
210(p3) adds cnt=1,cnt
211 ;;
212(p4) st2 [dst]=t0,2
213(p4) shr.u t0=t0,16
214(p4) adds cnt=2,cnt
215 ;;
216(p5) st4 [dst]=t0,4
217(p5) adds cnt=4,cnt
218 and src_end=-8,src_end // src_end = last word of source buffer
219 ;;
220
221 // At this point, dst is aligned to 8 bytes and there at least 16-7=9 bytes left to copy:
222
2231:{ add src=cnt,src // make src point to remainder of source buffer
224 sub cnt=in2,cnt // cnt = number of bytes left to copy
225 mov t4=ip
226 } ;;
227 and src2=-8,src // align source pointer
228 adds t4=.memcpy_loops-1b,t4
229 mov ar.ec=N
230
231 and t0=7,src // t0 = src & 7
232 shr.u t2=cnt,3 // t2 = number of 8-byte words left to copy
233 shl cnt=cnt,3 // move bits 0-2 to 3-5
234 ;;
235
236 .rotr val[N+1], w[2]
237 .rotp p[N]
238
239 cmp.ne p6,p0=t0,r0 // is src aligned, too?
240 shl t0=t0,LOG_LOOP_SIZE // t0 = 8*(src & 7)
241 adds t2=-1,t2 // br.ctop is repeat/until
242 ;;
243 add t4=t0,t4
244 mov pr=cnt,0x38 // set (p5,p4,p3) to # of bytes last-word bytes to copy
245 mov ar.lc=t2
246 ;;
247 nop.m 0
248 ;;
249 nop.m 0
250 nop.i 0
251 ;;
252 nop.m 0
253 ;;
254(p6) ld8 val[1]=[src2],8 // prime the pump...
255 mov b6=t4
256 br.sptk.few b6
257 ;;
258
259.memcpy_tail:
260 // At this point, (p5,p4,p3) are set to the number of bytes left to copy (which is
261 // less than 8) and t0 contains the last few bytes of the src buffer:
262(p5) st4 [dst]=t0,4
263(p5) shr.u t0=t0,32
264 mov ar.lc=saved_lc
265 ;;
266(p4) st2 [dst]=t0,2
267(p4) shr.u t0=t0,16
268 mov ar.pfs=saved_pfs
269 ;;
270(p3) st1 [dst]=t0
271 mov pr=saved_pr,-1
272 br.ret.sptk.many rp
273
274///////////////////////////////////////////////////////
275 .align 64
276
277#define COPY(shift,index) \
278 1: { .mib \
279 (p[0]) ld8 val[0]=[src2],8; \
280 (p[MEM_LAT+3]) shrp w[0]=val[MEM_LAT+3],val[MEM_LAT+4-index],shift; \
281 brp.loop.imp 1b, 2f \
282 }; \
283 2: { .mfb \
284 (p[MEM_LAT+4]) st8 [dst]=w[1],8; \
285 nop.f 0; \
286 br.ctop.dptk.few 1b; \
287 }; \
288 ;; \
289 ld8 val[N-1]=[src_end]; /* load last word (may be same as val[N]) */ \
290 ;; \
291 shrp t0=val[N-1],val[N-index],shift; \
292 br .memcpy_tail
293.memcpy_loops:
294 COPY(0, 1) /* no point special casing this---it doesn't go any faster without shrp */
295 COPY(8, 0)
296 COPY(16, 0)
297 COPY(24, 0)
298 COPY(32, 0)
299 COPY(40, 0)
300 COPY(48, 0)
301 COPY(56, 0)
302
303END(memcpy)
304EXPORT_SYMBOL(memcpy)
1/*
2 *
3 * Optimized version of the standard memcpy() function
4 *
5 * Inputs:
6 * in0: destination address
7 * in1: source address
8 * in2: number of bytes to copy
9 * Output:
10 * no return value
11 *
12 * Copyright (C) 2000-2001 Hewlett-Packard Co
13 * Stephane Eranian <eranian@hpl.hp.com>
14 * David Mosberger-Tang <davidm@hpl.hp.com>
15 */
16#include <asm/asmmacro.h>
17
18GLOBAL_ENTRY(memcpy)
19
20# define MEM_LAT 21 /* latency to memory */
21
22# define dst r2
23# define src r3
24# define retval r8
25# define saved_pfs r9
26# define saved_lc r10
27# define saved_pr r11
28# define cnt r16
29# define src2 r17
30# define t0 r18
31# define t1 r19
32# define t2 r20
33# define t3 r21
34# define t4 r22
35# define src_end r23
36
37# define N (MEM_LAT + 4)
38# define Nrot ((N + 7) & ~7)
39
40 /*
41 * First, check if everything (src, dst, len) is a multiple of eight. If
42 * so, we handle everything with no taken branches (other than the loop
43 * itself) and a small icache footprint. Otherwise, we jump off to
44 * the more general copy routine handling arbitrary
45 * sizes/alignment etc.
46 */
47 .prologue
48 .save ar.pfs, saved_pfs
49 alloc saved_pfs=ar.pfs,3,Nrot,0,Nrot
50 .save ar.lc, saved_lc
51 mov saved_lc=ar.lc
52 or t0=in0,in1
53 ;;
54
55 or t0=t0,in2
56 .save pr, saved_pr
57 mov saved_pr=pr
58
59 .body
60
61 cmp.eq p6,p0=in2,r0 // zero length?
62 mov retval=in0 // return dst
63(p6) br.ret.spnt.many rp // zero length, return immediately
64 ;;
65
66 mov dst=in0 // copy because of rotation
67 shr.u cnt=in2,3 // number of 8-byte words to copy
68 mov pr.rot=1<<16
69 ;;
70
71 adds cnt=-1,cnt // br.ctop is repeat/until
72 cmp.gtu p7,p0=16,in2 // copying less than 16 bytes?
73 mov ar.ec=N
74 ;;
75
76 and t0=0x7,t0
77 mov ar.lc=cnt
78 ;;
79 cmp.ne p6,p0=t0,r0
80
81 mov src=in1 // copy because of rotation
82(p7) br.cond.spnt.few .memcpy_short
83(p6) br.cond.spnt.few .memcpy_long
84 ;;
85 nop.m 0
86 ;;
87 nop.m 0
88 nop.i 0
89 ;;
90 nop.m 0
91 ;;
92 .rotr val[N]
93 .rotp p[N]
94 .align 32
951: { .mib
96(p[0]) ld8 val[0]=[src],8
97 nop.i 0
98 brp.loop.imp 1b, 2f
99}
1002: { .mfb
101(p[N-1])st8 [dst]=val[N-1],8
102 nop.f 0
103 br.ctop.dptk.few 1b
104}
105 ;;
106 mov ar.lc=saved_lc
107 mov pr=saved_pr,-1
108 mov ar.pfs=saved_pfs
109 br.ret.sptk.many rp
110
111 /*
112 * Small (<16 bytes) unaligned copying is done via a simple byte-at-the-time
113 * copy loop. This performs relatively poorly on Itanium, but it doesn't
114 * get used very often (gcc inlines small copies) and due to atomicity
115 * issues, we want to avoid read-modify-write of entire words.
116 */
117 .align 32
118.memcpy_short:
119 adds cnt=-1,in2 // br.ctop is repeat/until
120 mov ar.ec=MEM_LAT
121 brp.loop.imp 1f, 2f
122 ;;
123 mov ar.lc=cnt
124 ;;
125 nop.m 0
126 ;;
127 nop.m 0
128 nop.i 0
129 ;;
130 nop.m 0
131 ;;
132 nop.m 0
133 ;;
134 /*
135 * It is faster to put a stop bit in the loop here because it makes
136 * the pipeline shorter (and latency is what matters on short copies).
137 */
138 .align 32
1391: { .mib
140(p[0]) ld1 val[0]=[src],1
141 nop.i 0
142 brp.loop.imp 1b, 2f
143} ;;
1442: { .mfb
145(p[MEM_LAT-1])st1 [dst]=val[MEM_LAT-1],1
146 nop.f 0
147 br.ctop.dptk.few 1b
148} ;;
149 mov ar.lc=saved_lc
150 mov pr=saved_pr,-1
151 mov ar.pfs=saved_pfs
152 br.ret.sptk.many rp
153
154 /*
155 * Large (>= 16 bytes) copying is done in a fancy way. Latency isn't
156 * an overriding concern here, but throughput is. We first do
157 * sub-word copying until the destination is aligned, then we check
158 * if the source is also aligned. If so, we do a simple load/store-loop
159 * until there are less than 8 bytes left over and then we do the tail,
160 * by storing the last few bytes using sub-word copying. If the source
161 * is not aligned, we branch off to the non-congruent loop.
162 *
163 * stage: op:
164 * 0 ld
165 * :
166 * MEM_LAT+3 shrp
167 * MEM_LAT+4 st
168 *
169 * On Itanium, the pipeline itself runs without stalls. However, br.ctop
170 * seems to introduce an unavoidable bubble in the pipeline so the overall
171 * latency is 2 cycles/iteration. This gives us a _copy_ throughput
172 * of 4 byte/cycle. Still not bad.
173 */
174# undef N
175# undef Nrot
176# define N (MEM_LAT + 5) /* number of stages */
177# define Nrot ((N+1 + 2 + 7) & ~7) /* number of rotating regs */
178
179#define LOG_LOOP_SIZE 6
180
181.memcpy_long:
182 alloc t3=ar.pfs,3,Nrot,0,Nrot // resize register frame
183 and t0=-8,src // t0 = src & ~7
184 and t2=7,src // t2 = src & 7
185 ;;
186 ld8 t0=[t0] // t0 = 1st source word
187 adds src2=7,src // src2 = (src + 7)
188 sub t4=r0,dst // t4 = -dst
189 ;;
190 and src2=-8,src2 // src2 = (src + 7) & ~7
191 shl t2=t2,3 // t2 = 8*(src & 7)
192 shl t4=t4,3 // t4 = 8*(dst & 7)
193 ;;
194 ld8 t1=[src2] // t1 = 1st source word if src is 8-byte aligned, 2nd otherwise
195 sub t3=64,t2 // t3 = 64-8*(src & 7)
196 shr.u t0=t0,t2
197 ;;
198 add src_end=src,in2
199 shl t1=t1,t3
200 mov pr=t4,0x38 // (p5,p4,p3)=(dst & 7)
201 ;;
202 or t0=t0,t1
203 mov cnt=r0
204 adds src_end=-1,src_end
205 ;;
206(p3) st1 [dst]=t0,1
207(p3) shr.u t0=t0,8
208(p3) adds cnt=1,cnt
209 ;;
210(p4) st2 [dst]=t0,2
211(p4) shr.u t0=t0,16
212(p4) adds cnt=2,cnt
213 ;;
214(p5) st4 [dst]=t0,4
215(p5) adds cnt=4,cnt
216 and src_end=-8,src_end // src_end = last word of source buffer
217 ;;
218
219 // At this point, dst is aligned to 8 bytes and there at least 16-7=9 bytes left to copy:
220
2211:{ add src=cnt,src // make src point to remainder of source buffer
222 sub cnt=in2,cnt // cnt = number of bytes left to copy
223 mov t4=ip
224 } ;;
225 and src2=-8,src // align source pointer
226 adds t4=.memcpy_loops-1b,t4
227 mov ar.ec=N
228
229 and t0=7,src // t0 = src & 7
230 shr.u t2=cnt,3 // t2 = number of 8-byte words left to copy
231 shl cnt=cnt,3 // move bits 0-2 to 3-5
232 ;;
233
234 .rotr val[N+1], w[2]
235 .rotp p[N]
236
237 cmp.ne p6,p0=t0,r0 // is src aligned, too?
238 shl t0=t0,LOG_LOOP_SIZE // t0 = 8*(src & 7)
239 adds t2=-1,t2 // br.ctop is repeat/until
240 ;;
241 add t4=t0,t4
242 mov pr=cnt,0x38 // set (p5,p4,p3) to # of bytes last-word bytes to copy
243 mov ar.lc=t2
244 ;;
245 nop.m 0
246 ;;
247 nop.m 0
248 nop.i 0
249 ;;
250 nop.m 0
251 ;;
252(p6) ld8 val[1]=[src2],8 // prime the pump...
253 mov b6=t4
254 br.sptk.few b6
255 ;;
256
257.memcpy_tail:
258 // At this point, (p5,p4,p3) are set to the number of bytes left to copy (which is
259 // less than 8) and t0 contains the last few bytes of the src buffer:
260(p5) st4 [dst]=t0,4
261(p5) shr.u t0=t0,32
262 mov ar.lc=saved_lc
263 ;;
264(p4) st2 [dst]=t0,2
265(p4) shr.u t0=t0,16
266 mov ar.pfs=saved_pfs
267 ;;
268(p3) st1 [dst]=t0
269 mov pr=saved_pr,-1
270 br.ret.sptk.many rp
271
272///////////////////////////////////////////////////////
273 .align 64
274
275#define COPY(shift,index) \
276 1: { .mib \
277 (p[0]) ld8 val[0]=[src2],8; \
278 (p[MEM_LAT+3]) shrp w[0]=val[MEM_LAT+3],val[MEM_LAT+4-index],shift; \
279 brp.loop.imp 1b, 2f \
280 }; \
281 2: { .mfb \
282 (p[MEM_LAT+4]) st8 [dst]=w[1],8; \
283 nop.f 0; \
284 br.ctop.dptk.few 1b; \
285 }; \
286 ;; \
287 ld8 val[N-1]=[src_end]; /* load last word (may be same as val[N]) */ \
288 ;; \
289 shrp t0=val[N-1],val[N-index],shift; \
290 br .memcpy_tail
291.memcpy_loops:
292 COPY(0, 1) /* no point special casing this---it doesn't go any faster without shrp */
293 COPY(8, 0)
294 COPY(16, 0)
295 COPY(24, 0)
296 COPY(32, 0)
297 COPY(40, 0)
298 COPY(48, 0)
299 COPY(56, 0)
300
301END(memcpy)