Loading...
1/* NG4memcpy.S: Niagara-4 optimized memcpy.
2 *
3 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
4 */
5
6#ifdef __KERNEL__
7#include <asm/visasm.h>
8#include <asm/asi.h>
9#define GLOBAL_SPARE %g7
10#else
11#define ASI_BLK_INIT_QUAD_LDD_P 0xe2
12#define FPRS_FEF 0x04
13
14/* On T4 it is very expensive to access ASRs like %fprs and
15 * %asi, avoiding a read or a write can save ~50 cycles.
16 */
17#define FPU_ENTER \
18 rd %fprs, %o5; \
19 andcc %o5, FPRS_FEF, %g0; \
20 be,a,pn %icc, 999f; \
21 wr %g0, FPRS_FEF, %fprs; \
22 999:
23
24#ifdef MEMCPY_DEBUG
25#define VISEntryHalf FPU_ENTER; \
26 clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0;
27#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
28#else
29#define VISEntryHalf FPU_ENTER
30#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
31#endif
32
33#define GLOBAL_SPARE %g5
34#endif
35
36#ifndef STORE_ASI
37#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
38#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P
39#else
40#define STORE_ASI 0x80 /* ASI_P */
41#endif
42#endif
43
44#ifndef EX_LD
45#define EX_LD(x) x
46#endif
47
48#ifndef EX_ST
49#define EX_ST(x) x
50#endif
51
52#ifndef EX_RETVAL
53#define EX_RETVAL(x) x
54#endif
55
56#ifndef LOAD
57#define LOAD(type,addr,dest) type [addr], dest
58#endif
59
60#ifndef STORE
61#ifndef MEMCPY_DEBUG
62#define STORE(type,src,addr) type src, [addr]
63#else
64#define STORE(type,src,addr) type##a src, [addr] %asi
65#endif
66#endif
67
68#ifndef STORE_INIT
69#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI
70#endif
71
72#ifndef FUNC_NAME
73#define FUNC_NAME NG4memcpy
74#endif
75#ifndef PREAMBLE
76#define PREAMBLE
77#endif
78
79#ifndef XCC
80#define XCC xcc
81#endif
82
83 .register %g2,#scratch
84 .register %g3,#scratch
85
86 .text
87 .align 64
88
89 .globl FUNC_NAME
90 .type FUNC_NAME,#function
91FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
92#ifdef MEMCPY_DEBUG
93 wr %g0, 0x80, %asi
94#endif
95 srlx %o2, 31, %g2
96 cmp %g2, 0
97 tne %XCC, 5
98 PREAMBLE
99 mov %o0, %o3
100 brz,pn %o2, .Lexit
101 cmp %o2, 3
102 ble,pn %icc, .Ltiny
103 cmp %o2, 19
104 ble,pn %icc, .Lsmall
105 or %o0, %o1, %g2
106 cmp %o2, 128
107 bl,pn %icc, .Lmedium
108 nop
109
110.Llarge:/* len >= 0x80 */
111 /* First get dest 8 byte aligned. */
112 sub %g0, %o0, %g1
113 and %g1, 0x7, %g1
114 brz,pt %g1, 51f
115 sub %o2, %g1, %o2
116
1171: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
118 add %o1, 1, %o1
119 subcc %g1, 1, %g1
120 add %o0, 1, %o0
121 bne,pt %icc, 1b
122 EX_ST(STORE(stb, %g2, %o0 - 0x01))
123
12451: LOAD(prefetch, %o1 + 0x040, #n_reads_strong)
125 LOAD(prefetch, %o1 + 0x080, #n_reads_strong)
126 LOAD(prefetch, %o1 + 0x0c0, #n_reads_strong)
127 LOAD(prefetch, %o1 + 0x100, #n_reads_strong)
128 LOAD(prefetch, %o1 + 0x140, #n_reads_strong)
129 LOAD(prefetch, %o1 + 0x180, #n_reads_strong)
130 LOAD(prefetch, %o1 + 0x1c0, #n_reads_strong)
131 LOAD(prefetch, %o1 + 0x200, #n_reads_strong)
132
133 /* Check if we can use the straight fully aligned
134 * loop, or we require the alignaddr/faligndata variant.
135 */
136 andcc %o1, 0x7, %o5
137 bne,pn %icc, .Llarge_src_unaligned
138 sub %g0, %o0, %g1
139
140 /* Legitimize the use of initializing stores by getting dest
141 * to be 64-byte aligned.
142 */
143 and %g1, 0x3f, %g1
144 brz,pt %g1, .Llarge_aligned
145 sub %o2, %g1, %o2
146
1471: EX_LD(LOAD(ldx, %o1 + 0x00, %g2))
148 add %o1, 8, %o1
149 subcc %g1, 8, %g1
150 add %o0, 8, %o0
151 bne,pt %icc, 1b
152 EX_ST(STORE(stx, %g2, %o0 - 0x08))
153
154.Llarge_aligned:
155 /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */
156 andn %o2, 0x3f, %o4
157 sub %o2, %o4, %o2
158
1591: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
160 add %o1, 0x40, %o1
161 EX_LD(LOAD(ldx, %o1 - 0x38, %g2))
162 subcc %o4, 0x40, %o4
163 EX_LD(LOAD(ldx, %o1 - 0x30, %g3))
164 EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE))
165 EX_LD(LOAD(ldx, %o1 - 0x20, %o5))
166 EX_ST(STORE_INIT(%g1, %o0))
167 add %o0, 0x08, %o0
168 EX_ST(STORE_INIT(%g2, %o0))
169 add %o0, 0x08, %o0
170 EX_LD(LOAD(ldx, %o1 - 0x18, %g2))
171 EX_ST(STORE_INIT(%g3, %o0))
172 add %o0, 0x08, %o0
173 EX_LD(LOAD(ldx, %o1 - 0x10, %g3))
174 EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
175 add %o0, 0x08, %o0
176 EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE))
177 EX_ST(STORE_INIT(%o5, %o0))
178 add %o0, 0x08, %o0
179 EX_ST(STORE_INIT(%g2, %o0))
180 add %o0, 0x08, %o0
181 EX_ST(STORE_INIT(%g3, %o0))
182 add %o0, 0x08, %o0
183 EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
184 add %o0, 0x08, %o0
185 bne,pt %icc, 1b
186 LOAD(prefetch, %o1 + 0x200, #n_reads_strong)
187
188 membar #StoreLoad | #StoreStore
189
190 brz,pn %o2, .Lexit
191 cmp %o2, 19
192 ble,pn %icc, .Lsmall_unaligned
193 nop
194 ba,a,pt %icc, .Lmedium_noprefetch
195
196.Lexit: retl
197 mov EX_RETVAL(%o3), %o0
198
199.Llarge_src_unaligned:
200 andn %o2, 0x3f, %o4
201 sub %o2, %o4, %o2
202 VISEntryHalf
203 alignaddr %o1, %g0, %g1
204 add %o1, %o4, %o1
205 EX_LD(LOAD(ldd, %g1 + 0x00, %f0))
2061: EX_LD(LOAD(ldd, %g1 + 0x08, %f2))
207 subcc %o4, 0x40, %o4
208 EX_LD(LOAD(ldd, %g1 + 0x10, %f4))
209 EX_LD(LOAD(ldd, %g1 + 0x18, %f6))
210 EX_LD(LOAD(ldd, %g1 + 0x20, %f8))
211 EX_LD(LOAD(ldd, %g1 + 0x28, %f10))
212 EX_LD(LOAD(ldd, %g1 + 0x30, %f12))
213 EX_LD(LOAD(ldd, %g1 + 0x38, %f14))
214 faligndata %f0, %f2, %f16
215 EX_LD(LOAD(ldd, %g1 + 0x40, %f0))
216 faligndata %f2, %f4, %f18
217 add %g1, 0x40, %g1
218 faligndata %f4, %f6, %f20
219 faligndata %f6, %f8, %f22
220 faligndata %f8, %f10, %f24
221 faligndata %f10, %f12, %f26
222 faligndata %f12, %f14, %f28
223 faligndata %f14, %f0, %f30
224 EX_ST(STORE(std, %f16, %o0 + 0x00))
225 EX_ST(STORE(std, %f18, %o0 + 0x08))
226 EX_ST(STORE(std, %f20, %o0 + 0x10))
227 EX_ST(STORE(std, %f22, %o0 + 0x18))
228 EX_ST(STORE(std, %f24, %o0 + 0x20))
229 EX_ST(STORE(std, %f26, %o0 + 0x28))
230 EX_ST(STORE(std, %f28, %o0 + 0x30))
231 EX_ST(STORE(std, %f30, %o0 + 0x38))
232 add %o0, 0x40, %o0
233 bne,pt %icc, 1b
234 LOAD(prefetch, %g1 + 0x200, #n_reads_strong)
235 VISExitHalf
236
237 brz,pn %o2, .Lexit
238 cmp %o2, 19
239 ble,pn %icc, .Lsmall_unaligned
240 nop
241 ba,a,pt %icc, .Lmedium_unaligned
242
243.Lmedium:
244 LOAD(prefetch, %o1 + 0x40, #n_reads_strong)
245 andcc %g2, 0x7, %g0
246 bne,pn %icc, .Lmedium_unaligned
247 nop
248.Lmedium_noprefetch:
249 andncc %o2, 0x20 - 1, %o5
250 be,pn %icc, 2f
251 sub %o2, %o5, %o2
2521: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
253 EX_LD(LOAD(ldx, %o1 + 0x08, %g2))
254 EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE))
255 EX_LD(LOAD(ldx, %o1 + 0x18, %o4))
256 add %o1, 0x20, %o1
257 subcc %o5, 0x20, %o5
258 EX_ST(STORE(stx, %g1, %o0 + 0x00))
259 EX_ST(STORE(stx, %g2, %o0 + 0x08))
260 EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10))
261 EX_ST(STORE(stx, %o4, %o0 + 0x18))
262 bne,pt %icc, 1b
263 add %o0, 0x20, %o0
2642: andcc %o2, 0x18, %o5
265 be,pt %icc, 3f
266 sub %o2, %o5, %o2
2671: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
268 add %o1, 0x08, %o1
269 add %o0, 0x08, %o0
270 subcc %o5, 0x08, %o5
271 bne,pt %icc, 1b
272 EX_ST(STORE(stx, %g1, %o0 - 0x08))
2733: brz,pt %o2, .Lexit
274 cmp %o2, 0x04
275 bl,pn %icc, .Ltiny
276 nop
277 EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
278 add %o1, 0x04, %o1
279 add %o0, 0x04, %o0
280 subcc %o2, 0x04, %o2
281 bne,pn %icc, .Ltiny
282 EX_ST(STORE(stw, %g1, %o0 - 0x04))
283 ba,a,pt %icc, .Lexit
284.Lmedium_unaligned:
285 /* First get dest 8 byte aligned. */
286 sub %g0, %o0, %g1
287 and %g1, 0x7, %g1
288 brz,pt %g1, 2f
289 sub %o2, %g1, %o2
290
2911: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
292 add %o1, 1, %o1
293 subcc %g1, 1, %g1
294 add %o0, 1, %o0
295 bne,pt %icc, 1b
296 EX_ST(STORE(stb, %g2, %o0 - 0x01))
2972:
298 and %o1, 0x7, %g1
299 brz,pn %g1, .Lmedium_noprefetch
300 sll %g1, 3, %g1
301 mov 64, %g2
302 sub %g2, %g1, %g2
303 andn %o1, 0x7, %o1
304 EX_LD(LOAD(ldx, %o1 + 0x00, %o4))
305 sllx %o4, %g1, %o4
306 andn %o2, 0x08 - 1, %o5
307 sub %o2, %o5, %o2
3081: EX_LD(LOAD(ldx, %o1 + 0x08, %g3))
309 add %o1, 0x08, %o1
310 subcc %o5, 0x08, %o5
311 srlx %g3, %g2, GLOBAL_SPARE
312 or GLOBAL_SPARE, %o4, GLOBAL_SPARE
313 EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00))
314 add %o0, 0x08, %o0
315 bne,pt %icc, 1b
316 sllx %g3, %g1, %o4
317 srl %g1, 3, %g1
318 add %o1, %g1, %o1
319 brz,pn %o2, .Lexit
320 nop
321 ba,pt %icc, .Lsmall_unaligned
322
323.Ltiny:
324 EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
325 subcc %o2, 1, %o2
326 be,pn %icc, .Lexit
327 EX_ST(STORE(stb, %g1, %o0 + 0x00))
328 EX_LD(LOAD(ldub, %o1 + 0x01, %g1))
329 subcc %o2, 1, %o2
330 be,pn %icc, .Lexit
331 EX_ST(STORE(stb, %g1, %o0 + 0x01))
332 EX_LD(LOAD(ldub, %o1 + 0x02, %g1))
333 ba,pt %icc, .Lexit
334 EX_ST(STORE(stb, %g1, %o0 + 0x02))
335
336.Lsmall:
337 andcc %g2, 0x3, %g0
338 bne,pn %icc, .Lsmall_unaligned
339 andn %o2, 0x4 - 1, %o5
340 sub %o2, %o5, %o2
3411:
342 EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
343 add %o1, 0x04, %o1
344 subcc %o5, 0x04, %o5
345 add %o0, 0x04, %o0
346 bne,pt %icc, 1b
347 EX_ST(STORE(stw, %g1, %o0 - 0x04))
348 brz,pt %o2, .Lexit
349 nop
350 ba,a,pt %icc, .Ltiny
351
352.Lsmall_unaligned:
3531: EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
354 add %o1, 1, %o1
355 add %o0, 1, %o0
356 subcc %o2, 1, %o2
357 bne,pt %icc, 1b
358 EX_ST(STORE(stb, %g1, %o0 - 0x01))
359 ba,a,pt %icc, .Lexit
360 .size FUNC_NAME, .-FUNC_NAME
1/* SPDX-License-Identifier: GPL-2.0 */
2/* NG4memcpy.S: Niagara-4 optimized memcpy.
3 *
4 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
5 */
6
7#ifdef __KERNEL__
8#include <linux/linkage.h>
9#include <asm/visasm.h>
10#include <asm/asi.h>
11#define GLOBAL_SPARE %g7
12#else
13#define ASI_BLK_INIT_QUAD_LDD_P 0xe2
14#define FPRS_FEF 0x04
15
16/* On T4 it is very expensive to access ASRs like %fprs and
17 * %asi, avoiding a read or a write can save ~50 cycles.
18 */
19#define FPU_ENTER \
20 rd %fprs, %o5; \
21 andcc %o5, FPRS_FEF, %g0; \
22 be,a,pn %icc, 999f; \
23 wr %g0, FPRS_FEF, %fprs; \
24 999:
25
26#ifdef MEMCPY_DEBUG
27#define VISEntryHalf FPU_ENTER; \
28 clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0;
29#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
30#else
31#define VISEntryHalf FPU_ENTER
32#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
33#endif
34
35#define GLOBAL_SPARE %g5
36#endif
37
38#ifndef STORE_ASI
39#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
40#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P
41#else
42#define STORE_ASI 0x80 /* ASI_P */
43#endif
44#endif
45
46#if !defined(EX_LD) && !defined(EX_ST)
47#define NON_USER_COPY
48#endif
49
50#ifndef EX_LD
51#define EX_LD(x,y) x
52#endif
53#ifndef EX_LD_FP
54#define EX_LD_FP(x,y) x
55#endif
56
57#ifndef EX_ST
58#define EX_ST(x,y) x
59#endif
60#ifndef EX_ST_FP
61#define EX_ST_FP(x,y) x
62#endif
63
64
65#ifndef LOAD
66#define LOAD(type,addr,dest) type [addr], dest
67#endif
68
69#ifndef STORE
70#ifndef MEMCPY_DEBUG
71#define STORE(type,src,addr) type src, [addr]
72#else
73#define STORE(type,src,addr) type##a src, [addr] %asi
74#endif
75#endif
76
77#ifndef STORE_INIT
78#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI
79#endif
80
81#ifndef FUNC_NAME
82#define FUNC_NAME NG4memcpy
83#endif
84#ifndef PREAMBLE
85#define PREAMBLE
86#endif
87
88#ifndef XCC
89#define XCC xcc
90#endif
91
92 .register %g2,#scratch
93 .register %g3,#scratch
94
95 .text
96#ifndef EX_RETVAL
97#define EX_RETVAL(x) x
98#endif
99 .align 64
100
101 .globl FUNC_NAME
102 .type FUNC_NAME,#function
103FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
104#ifdef MEMCPY_DEBUG
105 wr %g0, 0x80, %asi
106#endif
107 srlx %o2, 31, %g2
108 cmp %g2, 0
109 tne %XCC, 5
110 PREAMBLE
111 mov %o0, %o3
112 brz,pn %o2, .Lexit
113 cmp %o2, 3
114 ble,pn %icc, .Ltiny
115 cmp %o2, 19
116 ble,pn %icc, .Lsmall
117 or %o0, %o1, %g2
118 cmp %o2, 128
119 bl,pn %icc, .Lmedium
120 nop
121
122.Llarge:/* len >= 0x80 */
123 /* First get dest 8 byte aligned. */
124 sub %g0, %o0, %g1
125 and %g1, 0x7, %g1
126 brz,pt %g1, 51f
127 sub %o2, %g1, %o2
128
129
1301: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1)
131 add %o1, 1, %o1
132 subcc %g1, 1, %g1
133 add %o0, 1, %o0
134 bne,pt %icc, 1b
135 EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_g1_plus_1)
136
13751: LOAD(prefetch, %o1 + 0x040, #n_reads_strong)
138 LOAD(prefetch, %o1 + 0x080, #n_reads_strong)
139 LOAD(prefetch, %o1 + 0x0c0, #n_reads_strong)
140 LOAD(prefetch, %o1 + 0x100, #n_reads_strong)
141 LOAD(prefetch, %o1 + 0x140, #n_reads_strong)
142 LOAD(prefetch, %o1 + 0x180, #n_reads_strong)
143 LOAD(prefetch, %o1 + 0x1c0, #n_reads_strong)
144 LOAD(prefetch, %o1 + 0x200, #n_reads_strong)
145
146 /* Check if we can use the straight fully aligned
147 * loop, or we require the alignaddr/faligndata variant.
148 */
149 andcc %o1, 0x7, %o5
150 bne,pn %icc, .Llarge_src_unaligned
151 sub %g0, %o0, %g1
152
153 /* Legitimize the use of initializing stores by getting dest
154 * to be 64-byte aligned.
155 */
156 and %g1, 0x3f, %g1
157 brz,pt %g1, .Llarge_aligned
158 sub %o2, %g1, %o2
159
1601: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1)
161 add %o1, 8, %o1
162 subcc %g1, 8, %g1
163 add %o0, 8, %o0
164 bne,pt %icc, 1b
165 EX_ST(STORE(stx, %g2, %o0 - 0x08), memcpy_retl_o2_plus_g1_plus_8)
166
167.Llarge_aligned:
168 /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */
169 andn %o2, 0x3f, %o4
170 sub %o2, %o4, %o2
171
1721: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o4)
173 add %o1, 0x40, %o1
174 EX_LD(LOAD(ldx, %o1 - 0x38, %g2), memcpy_retl_o2_plus_o4)
175 subcc %o4, 0x40, %o4
176 EX_LD(LOAD(ldx, %o1 - 0x30, %g3), memcpy_retl_o2_plus_o4_plus_64)
177 EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), memcpy_retl_o2_plus_o4_plus_64)
178 EX_LD(LOAD(ldx, %o1 - 0x20, %o5), memcpy_retl_o2_plus_o4_plus_64)
179 EX_ST(STORE_INIT(%g1, %o0), memcpy_retl_o2_plus_o4_plus_64)
180 add %o0, 0x08, %o0
181 EX_ST(STORE_INIT(%g2, %o0), memcpy_retl_o2_plus_o4_plus_56)
182 add %o0, 0x08, %o0
183 EX_LD(LOAD(ldx, %o1 - 0x18, %g2), memcpy_retl_o2_plus_o4_plus_48)
184 EX_ST(STORE_INIT(%g3, %o0), memcpy_retl_o2_plus_o4_plus_48)
185 add %o0, 0x08, %o0
186 EX_LD(LOAD(ldx, %o1 - 0x10, %g3), memcpy_retl_o2_plus_o4_plus_40)
187 EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), memcpy_retl_o2_plus_o4_plus_40)
188 add %o0, 0x08, %o0
189 EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), memcpy_retl_o2_plus_o4_plus_32)
190 EX_ST(STORE_INIT(%o5, %o0), memcpy_retl_o2_plus_o4_plus_32)
191 add %o0, 0x08, %o0
192 EX_ST(STORE_INIT(%g2, %o0), memcpy_retl_o2_plus_o4_plus_24)
193 add %o0, 0x08, %o0
194 EX_ST(STORE_INIT(%g3, %o0), memcpy_retl_o2_plus_o4_plus_16)
195 add %o0, 0x08, %o0
196 EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), memcpy_retl_o2_plus_o4_plus_8)
197 add %o0, 0x08, %o0
198 bne,pt %icc, 1b
199 LOAD(prefetch, %o1 + 0x200, #n_reads_strong)
200
201 membar #StoreLoad | #StoreStore
202
203 brz,pn %o2, .Lexit
204 cmp %o2, 19
205 ble,pn %icc, .Lsmall_unaligned
206 nop
207 ba,a,pt %icc, .Lmedium_noprefetch
208
209.Lexit: retl
210 mov EX_RETVAL(%o3), %o0
211
212.Llarge_src_unaligned:
213#ifdef NON_USER_COPY
214 VISEntryHalfFast(.Lmedium_vis_entry_fail)
215#else
216 VISEntryHalf
217#endif
218 andn %o2, 0x3f, %o4
219 sub %o2, %o4, %o2
220 alignaddr %o1, %g0, %g1
221 add %o1, %o4, %o1
222 EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), memcpy_retl_o2_plus_o4)
2231: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), memcpy_retl_o2_plus_o4)
224 subcc %o4, 0x40, %o4
225 EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), memcpy_retl_o2_plus_o4_plus_64)
226 EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), memcpy_retl_o2_plus_o4_plus_64)
227 EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), memcpy_retl_o2_plus_o4_plus_64)
228 EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), memcpy_retl_o2_plus_o4_plus_64)
229 EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), memcpy_retl_o2_plus_o4_plus_64)
230 EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), memcpy_retl_o2_plus_o4_plus_64)
231 faligndata %f0, %f2, %f16
232 EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), memcpy_retl_o2_plus_o4_plus_64)
233 faligndata %f2, %f4, %f18
234 add %g1, 0x40, %g1
235 faligndata %f4, %f6, %f20
236 faligndata %f6, %f8, %f22
237 faligndata %f8, %f10, %f24
238 faligndata %f10, %f12, %f26
239 faligndata %f12, %f14, %f28
240 faligndata %f14, %f0, %f30
241 EX_ST_FP(STORE(std, %f16, %o0 + 0x00), memcpy_retl_o2_plus_o4_plus_64)
242 EX_ST_FP(STORE(std, %f18, %o0 + 0x08), memcpy_retl_o2_plus_o4_plus_56)
243 EX_ST_FP(STORE(std, %f20, %o0 + 0x10), memcpy_retl_o2_plus_o4_plus_48)
244 EX_ST_FP(STORE(std, %f22, %o0 + 0x18), memcpy_retl_o2_plus_o4_plus_40)
245 EX_ST_FP(STORE(std, %f24, %o0 + 0x20), memcpy_retl_o2_plus_o4_plus_32)
246 EX_ST_FP(STORE(std, %f26, %o0 + 0x28), memcpy_retl_o2_plus_o4_plus_24)
247 EX_ST_FP(STORE(std, %f28, %o0 + 0x30), memcpy_retl_o2_plus_o4_plus_16)
248 EX_ST_FP(STORE(std, %f30, %o0 + 0x38), memcpy_retl_o2_plus_o4_plus_8)
249 add %o0, 0x40, %o0
250 bne,pt %icc, 1b
251 LOAD(prefetch, %g1 + 0x200, #n_reads_strong)
252#ifdef NON_USER_COPY
253 VISExitHalfFast
254#else
255 VISExitHalf
256#endif
257 brz,pn %o2, .Lexit
258 cmp %o2, 19
259 ble,pn %icc, .Lsmall_unaligned
260 nop
261 ba,a,pt %icc, .Lmedium_unaligned
262
263#ifdef NON_USER_COPY
264.Lmedium_vis_entry_fail:
265 or %o0, %o1, %g2
266#endif
267.Lmedium:
268 LOAD(prefetch, %o1 + 0x40, #n_reads_strong)
269 andcc %g2, 0x7, %g0
270 bne,pn %icc, .Lmedium_unaligned
271 nop
272.Lmedium_noprefetch:
273 andncc %o2, 0x20 - 1, %o5
274 be,pn %icc, 2f
275 sub %o2, %o5, %o2
2761: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o5)
277 EX_LD(LOAD(ldx, %o1 + 0x08, %g2), memcpy_retl_o2_plus_o5)
278 EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), memcpy_retl_o2_plus_o5)
279 EX_LD(LOAD(ldx, %o1 + 0x18, %o4), memcpy_retl_o2_plus_o5)
280 add %o1, 0x20, %o1
281 subcc %o5, 0x20, %o5
282 EX_ST(STORE(stx, %g1, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32)
283 EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24)
284 EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_24)
285 EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8)
286 bne,pt %icc, 1b
287 add %o0, 0x20, %o0
2882: andcc %o2, 0x18, %o5
289 be,pt %icc, 3f
290 sub %o2, %o5, %o2
291
2921: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o5)
293 add %o1, 0x08, %o1
294 add %o0, 0x08, %o0
295 subcc %o5, 0x08, %o5
296 bne,pt %icc, 1b
297 EX_ST(STORE(stx, %g1, %o0 - 0x08), memcpy_retl_o2_plus_o5_plus_8)
2983: brz,pt %o2, .Lexit
299 cmp %o2, 0x04
300 bl,pn %icc, .Ltiny
301 nop
302 EX_LD(LOAD(lduw, %o1 + 0x00, %g1), memcpy_retl_o2)
303 add %o1, 0x04, %o1
304 add %o0, 0x04, %o0
305 subcc %o2, 0x04, %o2
306 bne,pn %icc, .Ltiny
307 EX_ST(STORE(stw, %g1, %o0 - 0x04), memcpy_retl_o2_plus_4)
308 ba,a,pt %icc, .Lexit
309.Lmedium_unaligned:
310 /* First get dest 8 byte aligned. */
311 sub %g0, %o0, %g1
312 and %g1, 0x7, %g1
313 brz,pt %g1, 2f
314 sub %o2, %g1, %o2
315
3161: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1)
317 add %o1, 1, %o1
318 subcc %g1, 1, %g1
319 add %o0, 1, %o0
320 bne,pt %icc, 1b
321 EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_g1_plus_1)
3222:
323 and %o1, 0x7, %g1
324 brz,pn %g1, .Lmedium_noprefetch
325 sll %g1, 3, %g1
326 mov 64, %g2
327 sub %g2, %g1, %g2
328 andn %o1, 0x7, %o1
329 EX_LD(LOAD(ldx, %o1 + 0x00, %o4), memcpy_retl_o2)
330 sllx %o4, %g1, %o4
331 andn %o2, 0x08 - 1, %o5
332 sub %o2, %o5, %o2
3331: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), memcpy_retl_o2_plus_o5)
334 add %o1, 0x08, %o1
335 subcc %o5, 0x08, %o5
336 srlx %g3, %g2, GLOBAL_SPARE
337 or GLOBAL_SPARE, %o4, GLOBAL_SPARE
338 EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_8)
339 add %o0, 0x08, %o0
340 bne,pt %icc, 1b
341 sllx %g3, %g1, %o4
342 srl %g1, 3, %g1
343 add %o1, %g1, %o1
344 brz,pn %o2, .Lexit
345 nop
346 ba,pt %icc, .Lsmall_unaligned
347
348.Ltiny:
349 EX_LD(LOAD(ldub, %o1 + 0x00, %g1), memcpy_retl_o2)
350 subcc %o2, 1, %o2
351 be,pn %icc, .Lexit
352 EX_ST(STORE(stb, %g1, %o0 + 0x00), memcpy_retl_o2_plus_1)
353 EX_LD(LOAD(ldub, %o1 + 0x01, %g1), memcpy_retl_o2)
354 subcc %o2, 1, %o2
355 be,pn %icc, .Lexit
356 EX_ST(STORE(stb, %g1, %o0 + 0x01), memcpy_retl_o2_plus_1)
357 EX_LD(LOAD(ldub, %o1 + 0x02, %g1), memcpy_retl_o2)
358 ba,pt %icc, .Lexit
359 EX_ST(STORE(stb, %g1, %o0 + 0x02), memcpy_retl_o2)
360
361.Lsmall:
362 andcc %g2, 0x3, %g0
363 bne,pn %icc, .Lsmall_unaligned
364 andn %o2, 0x4 - 1, %o5
365 sub %o2, %o5, %o2
3661:
367 EX_LD(LOAD(lduw, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o5)
368 add %o1, 0x04, %o1
369 subcc %o5, 0x04, %o5
370 add %o0, 0x04, %o0
371 bne,pt %icc, 1b
372 EX_ST(STORE(stw, %g1, %o0 - 0x04), memcpy_retl_o2_plus_o5_plus_4)
373 brz,pt %o2, .Lexit
374 nop
375 ba,a,pt %icc, .Ltiny
376
377.Lsmall_unaligned:
3781: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), memcpy_retl_o2)
379 add %o1, 1, %o1
380 add %o0, 1, %o0
381 subcc %o2, 1, %o2
382 bne,pt %icc, 1b
383 EX_ST(STORE(stb, %g1, %o0 - 0x01), memcpy_retl_o2_plus_1)
384 ba,a,pt %icc, .Lexit
385 nop
386 .size FUNC_NAME, .-FUNC_NAME