Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *
4 * Optimized version of the standard copy_page() function
5 *
6 * Inputs:
7 * in0: address of target page
8 * in1: address of source page
9 * Output:
10 * no return value
11 *
12 * Copyright (C) 1999, 2001 Hewlett-Packard Co
13 * Stephane Eranian <eranian@hpl.hp.com>
14 * David Mosberger <davidm@hpl.hp.com>
15 *
16 * 4/06/01 davidm Tuned to make it perform well both for cached and uncached copies.
17 */
18#include <asm/asmmacro.h>
19#include <asm/page.h>
20#include <asm/export.h>
21
22#define PIPE_DEPTH 3
23#define EPI p[PIPE_DEPTH-1]
24
25#define lcount r16
26#define saved_pr r17
27#define saved_lc r18
28#define saved_pfs r19
29#define src1 r20
30#define src2 r21
31#define tgt1 r22
32#define tgt2 r23
33#define srcf r24
34#define tgtf r25
35#define tgt_last r26
36
37#define Nrot ((8*PIPE_DEPTH+7)&~7)
38
39GLOBAL_ENTRY(copy_page)
40 .prologue
41 .save ar.pfs, saved_pfs
42 alloc saved_pfs=ar.pfs,3,Nrot-3,0,Nrot
43
44 .rotr t1[PIPE_DEPTH], t2[PIPE_DEPTH], t3[PIPE_DEPTH], t4[PIPE_DEPTH], \
45 t5[PIPE_DEPTH], t6[PIPE_DEPTH], t7[PIPE_DEPTH], t8[PIPE_DEPTH]
46 .rotp p[PIPE_DEPTH]
47
48 .save ar.lc, saved_lc
49 mov saved_lc=ar.lc
50 mov ar.ec=PIPE_DEPTH
51
52 mov lcount=PAGE_SIZE/64-1
53 .save pr, saved_pr
54 mov saved_pr=pr
55 mov pr.rot=1<<16
56
57 .body
58
59 mov src1=in1
60 adds src2=8,in1
61 mov tgt_last = PAGE_SIZE
62 ;;
63 adds tgt2=8,in0
64 add srcf=512,in1
65 mov ar.lc=lcount
66 mov tgt1=in0
67 add tgtf=512,in0
68 add tgt_last = tgt_last, in0
69 ;;
701:
71(p[0]) ld8 t1[0]=[src1],16
72(EPI) st8 [tgt1]=t1[PIPE_DEPTH-1],16
73(p[0]) ld8 t2[0]=[src2],16
74(EPI) st8 [tgt2]=t2[PIPE_DEPTH-1],16
75 cmp.ltu p6,p0 = tgtf, tgt_last
76 ;;
77(p[0]) ld8 t3[0]=[src1],16
78(EPI) st8 [tgt1]=t3[PIPE_DEPTH-1],16
79(p[0]) ld8 t4[0]=[src2],16
80(EPI) st8 [tgt2]=t4[PIPE_DEPTH-1],16
81 ;;
82(p[0]) ld8 t5[0]=[src1],16
83(EPI) st8 [tgt1]=t5[PIPE_DEPTH-1],16
84(p[0]) ld8 t6[0]=[src2],16
85(EPI) st8 [tgt2]=t6[PIPE_DEPTH-1],16
86 ;;
87(p[0]) ld8 t7[0]=[src1],16
88(EPI) st8 [tgt1]=t7[PIPE_DEPTH-1],16
89(p[0]) ld8 t8[0]=[src2],16
90(EPI) st8 [tgt2]=t8[PIPE_DEPTH-1],16
91
92(p6) lfetch [srcf], 64
93(p6) lfetch [tgtf], 64
94 br.ctop.sptk.few 1b
95 ;;
96 mov pr=saved_pr,0xffffffffffff0000 // restore predicates
97 mov ar.pfs=saved_pfs
98 mov ar.lc=saved_lc
99 br.ret.sptk.many rp
100END(copy_page)
101EXPORT_SYMBOL(copy_page)
1/*
2 *
3 * Optimized version of the standard copy_page() function
4 *
5 * Inputs:
6 * in0: address of target page
7 * in1: address of source page
8 * Output:
9 * no return value
10 *
11 * Copyright (C) 1999, 2001 Hewlett-Packard Co
12 * Stephane Eranian <eranian@hpl.hp.com>
13 * David Mosberger <davidm@hpl.hp.com>
14 *
15 * 4/06/01 davidm Tuned to make it perform well both for cached and uncached copies.
16 */
17#include <asm/asmmacro.h>
18#include <asm/page.h>
19
20#define PIPE_DEPTH 3
21#define EPI p[PIPE_DEPTH-1]
22
23#define lcount r16
24#define saved_pr r17
25#define saved_lc r18
26#define saved_pfs r19
27#define src1 r20
28#define src2 r21
29#define tgt1 r22
30#define tgt2 r23
31#define srcf r24
32#define tgtf r25
33#define tgt_last r26
34
35#define Nrot ((8*PIPE_DEPTH+7)&~7)
36
37GLOBAL_ENTRY(copy_page)
38 .prologue
39 .save ar.pfs, saved_pfs
40 alloc saved_pfs=ar.pfs,3,Nrot-3,0,Nrot
41
42 .rotr t1[PIPE_DEPTH], t2[PIPE_DEPTH], t3[PIPE_DEPTH], t4[PIPE_DEPTH], \
43 t5[PIPE_DEPTH], t6[PIPE_DEPTH], t7[PIPE_DEPTH], t8[PIPE_DEPTH]
44 .rotp p[PIPE_DEPTH]
45
46 .save ar.lc, saved_lc
47 mov saved_lc=ar.lc
48 mov ar.ec=PIPE_DEPTH
49
50 mov lcount=PAGE_SIZE/64-1
51 .save pr, saved_pr
52 mov saved_pr=pr
53 mov pr.rot=1<<16
54
55 .body
56
57 mov src1=in1
58 adds src2=8,in1
59 mov tgt_last = PAGE_SIZE
60 ;;
61 adds tgt2=8,in0
62 add srcf=512,in1
63 mov ar.lc=lcount
64 mov tgt1=in0
65 add tgtf=512,in0
66 add tgt_last = tgt_last, in0
67 ;;
681:
69(p[0]) ld8 t1[0]=[src1],16
70(EPI) st8 [tgt1]=t1[PIPE_DEPTH-1],16
71(p[0]) ld8 t2[0]=[src2],16
72(EPI) st8 [tgt2]=t2[PIPE_DEPTH-1],16
73 cmp.ltu p6,p0 = tgtf, tgt_last
74 ;;
75(p[0]) ld8 t3[0]=[src1],16
76(EPI) st8 [tgt1]=t3[PIPE_DEPTH-1],16
77(p[0]) ld8 t4[0]=[src2],16
78(EPI) st8 [tgt2]=t4[PIPE_DEPTH-1],16
79 ;;
80(p[0]) ld8 t5[0]=[src1],16
81(EPI) st8 [tgt1]=t5[PIPE_DEPTH-1],16
82(p[0]) ld8 t6[0]=[src2],16
83(EPI) st8 [tgt2]=t6[PIPE_DEPTH-1],16
84 ;;
85(p[0]) ld8 t7[0]=[src1],16
86(EPI) st8 [tgt1]=t7[PIPE_DEPTH-1],16
87(p[0]) ld8 t8[0]=[src2],16
88(EPI) st8 [tgt2]=t8[PIPE_DEPTH-1],16
89
90(p6) lfetch [srcf], 64
91(p6) lfetch [tgtf], 64
92 br.ctop.sptk.few 1b
93 ;;
94 mov pr=saved_pr,0xffffffffffff0000 // restore predicates
95 mov ar.pfs=saved_pfs
96 mov ar.lc=saved_lc
97 br.ret.sptk.many rp
98END(copy_page)