1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 *
  4 * Optimized version of the standard copy_page() function
  5 *
  6 * Inputs:
  7 *	in0:	address of target page
  8 *	in1:	address of source page
  9 * Output:
 10 *	no return value
 11 *
 12 * Copyright (C) 1999, 2001 Hewlett-Packard Co
 13 *	Stephane Eranian <eranian@hpl.hp.com>
 14 *	David Mosberger <davidm@hpl.hp.com>
 15 *
 16 * 4/06/01 davidm	Tuned to make it perform well both for cached and uncached copies.
 17 */
 18#include <asm/asmmacro.h>
 19#include <asm/page.h>
 20#include <asm/export.h>
 21
 22#define PIPE_DEPTH	3
 23#define EPI		p[PIPE_DEPTH-1]
 24
 25#define lcount		r16
 26#define saved_pr	r17
 27#define saved_lc	r18
 28#define saved_pfs	r19
 29#define src1		r20
 30#define src2		r21
 31#define tgt1		r22
 32#define tgt2		r23
 33#define srcf		r24
 34#define tgtf		r25
 35#define tgt_last	r26
 36
 37#define Nrot		((8*PIPE_DEPTH+7)&~7)
 38
 39GLOBAL_ENTRY(copy_page)
 40	.prologue
 41	.save ar.pfs, saved_pfs
 42	alloc saved_pfs=ar.pfs,3,Nrot-3,0,Nrot
 43
 44	.rotr t1[PIPE_DEPTH], t2[PIPE_DEPTH], t3[PIPE_DEPTH], t4[PIPE_DEPTH], \
 45	      t5[PIPE_DEPTH], t6[PIPE_DEPTH], t7[PIPE_DEPTH], t8[PIPE_DEPTH]
 46	.rotp p[PIPE_DEPTH]
 47
 48	.save ar.lc, saved_lc
 49	mov saved_lc=ar.lc
 50	mov ar.ec=PIPE_DEPTH
 51
 52	mov lcount=PAGE_SIZE/64-1
 53	.save pr, saved_pr
 54	mov saved_pr=pr
 55	mov pr.rot=1<<16
 56
 57	.body
 58
 59	mov src1=in1
 60	adds src2=8,in1
 61	mov tgt_last = PAGE_SIZE
 62	;;
 63	adds tgt2=8,in0
 64	add srcf=512,in1
 65	mov ar.lc=lcount
 66	mov tgt1=in0
 67	add tgtf=512,in0
 68	add tgt_last = tgt_last, in0
 69	;;
 701:
 71(p[0])	ld8 t1[0]=[src1],16
 72(EPI)	st8 [tgt1]=t1[PIPE_DEPTH-1],16
 73(p[0])	ld8 t2[0]=[src2],16
 74(EPI)	st8 [tgt2]=t2[PIPE_DEPTH-1],16
 75	cmp.ltu p6,p0 = tgtf, tgt_last
 76	;;
 77(p[0])	ld8 t3[0]=[src1],16
 78(EPI)	st8 [tgt1]=t3[PIPE_DEPTH-1],16
 79(p[0])	ld8 t4[0]=[src2],16
 80(EPI)	st8 [tgt2]=t4[PIPE_DEPTH-1],16
 81	;;
 82(p[0])	ld8 t5[0]=[src1],16
 83(EPI)	st8 [tgt1]=t5[PIPE_DEPTH-1],16
 84(p[0])	ld8 t6[0]=[src2],16
 85(EPI)	st8 [tgt2]=t6[PIPE_DEPTH-1],16
 86	;;
 87(p[0])	ld8 t7[0]=[src1],16
 88(EPI)	st8 [tgt1]=t7[PIPE_DEPTH-1],16
 89(p[0])	ld8 t8[0]=[src2],16
 90(EPI)	st8 [tgt2]=t8[PIPE_DEPTH-1],16
 91
 92(p6)	lfetch [srcf], 64
 93(p6)	lfetch [tgtf], 64
 94	br.ctop.sptk.few 1b
 95	;;
 96	mov pr=saved_pr,0xffffffffffff0000	// restore predicates
 97	mov ar.pfs=saved_pfs
 98	mov ar.lc=saved_lc
 99	br.ret.sptk.many rp
100END(copy_page)
101EXPORT_SYMBOL(copy_page)