1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * McKinley-optimized version of copy_page().
  4 *
  5 * Copyright (C) 2002 Hewlett-Packard Co
  6 *	David Mosberger <davidm@hpl.hp.com>
  7 *
  8 * Inputs:
  9 *	in0:	address of target page
 10 *	in1:	address of source page
 11 * Output:
 12 *	no return value
 13 *
 14 * General idea:
 15 *	- use regular loads and stores to prefetch data to avoid consuming M-slot just for
 16 *	  lfetches => good for in-cache performance
 17 *	- avoid l2 bank-conflicts by not storing into the same 16-byte bank within a single
 18 *	  cycle
 19 *
 20 * Principle of operation:
 21 *	First, note that L1 has a line-size of 64 bytes and L2 a line-size of 128 bytes.
 22 *	To avoid secondary misses in L2, we prefetch both source and destination with a line-size
 23 *	of 128 bytes.  When both of these lines are in the L2 and the first half of the
 24 *	source line is in L1, we start copying the remaining words.  The second half of the
 25 *	source line is prefetched in an earlier iteration, so that by the time we start
 26 *	accessing it, it's also present in the L1.
 27 *
 28 *	We use a software-pipelined loop to control the overall operation.  The pipeline
 29 *	has 2*PREFETCH_DIST+K stages.  The first PREFETCH_DIST stages are used for prefetching
 30 *	source cache-lines.  The second PREFETCH_DIST stages are used for prefetching destination
 31 *	cache-lines, the last K stages are used to copy the cache-line words not copied by
 32 *	the prefetches.  The four relevant points in the pipelined are called A, B, C, D:
 33 *	p[A] is TRUE if a source-line should be prefetched, p[B] is TRUE if a destination-line
 34 *	should be prefetched, p[C] is TRUE if the second half of an L2 line should be brought
 35 *	into L1D and p[D] is TRUE if a cacheline needs to be copied.
 36 *
 37 *	This all sounds very complicated, but thanks to the modulo-scheduled loop support,
 38 *	the resulting code is very regular and quite easy to follow (once you get the idea).
 39 *
 40 *	As a secondary optimization, the first 2*PREFETCH_DIST iterations are implemented
 41 *	as the separate .prefetch_loop.  Logically, this loop performs exactly like the
 42 *	main-loop (.line_copy), but has all known-to-be-predicated-off instructions removed,
 43 *	so that each loop iteration is faster (again, good for cached case).
 44 *
 45 *	When reading the code, it helps to keep the following picture in mind:
 46 *
 47 *	       word 0 word 1
 48 *            +------+------+---
 49 *	      |	v[x] | 	t1  | ^
 50 *	      |	t2   |	t3  | |
 51 *	      |	t4   |	t5  | |
 52 *	      |	t6   |	t7  | | 128 bytes
 53 *     	      |	n[y] | 	t9  | |	(L2 cache line)
 54 *	      |	t10  | 	t11 | |
 55 *	      |	t12  | 	t13 | |
 56 *	      |	t14  | 	t15 | v
 57 *	      +------+------+---
 58 *
 59 *	Here, v[x] is copied by the (memory) prefetch.  n[y] is loaded at p[C]
 60 *	to fetch the second-half of the L2 cache line into L1, and the tX words are copied in
 61 *	an order that avoids bank conflicts.
 62 */
 63#include <asm/asmmacro.h>
 64#include <asm/page.h>
 65#include <asm/export.h>
 66
 67#define PREFETCH_DIST	8		// McKinley sustains 16 outstanding L2 misses (8 ld, 8 st)
 68
 69#define src0		r2
 70#define src1		r3
 71#define dst0		r9
 72#define dst1		r10
 73#define src_pre_mem	r11
 74#define dst_pre_mem	r14
 75#define src_pre_l2	r15
 76#define dst_pre_l2	r16
 77#define t1		r17
 78#define t2		r18
 79#define t3		r19
 80#define t4		r20
 81#define t5		t1	// alias!
 82#define t6		t2	// alias!
 83#define t7		t3	// alias!
 84#define t9		t5	// alias!
 85#define t10		t4	// alias!
 86#define t11		t7	// alias!
 87#define t12		t6	// alias!
 88#define t14		t10	// alias!
 89#define t13		r21
 90#define t15		r22
 91
 92#define saved_lc	r23
 93#define saved_pr	r24
 94
 95#define	A	0
 96#define B	(PREFETCH_DIST)
 97#define C	(B + PREFETCH_DIST)
 98#define D	(C + 3)
 99#define N	(D + 1)
100#define Nrot	((N + 7) & ~7)
101
102GLOBAL_ENTRY(copy_page)
103	.prologue
104	alloc r8 = ar.pfs, 2, Nrot-2, 0, Nrot
105
106	.rotr v[2*PREFETCH_DIST], n[D-C+1]
107	.rotp p[N]
108
109	.save ar.lc, saved_lc
110	mov saved_lc = ar.lc
111	.save pr, saved_pr
112	mov saved_pr = pr
113	.body
114
115	mov src_pre_mem = in1
116	mov pr.rot = 0x10000
117	mov ar.ec = 1				// special unrolled loop
118
119	mov dst_pre_mem = in0
120	mov ar.lc = 2*PREFETCH_DIST - 1
121
122	add src_pre_l2 = 8*8, in1
123	add dst_pre_l2 = 8*8, in0
124	add src0 = 8, in1			// first t1 src
125	add src1 = 3*8, in1			// first t3 src
126	add dst0 = 8, in0			// first t1 dst
127	add dst1 = 3*8, in0			// first t3 dst
128	mov t1 = (PAGE_SIZE/128) - (2*PREFETCH_DIST) - 1
129	nop.m 0
130	nop.i 0
131	;;
132	// same as .line_copy loop, but with all predicated-off instructions removed:
133.prefetch_loop:
134(p[A])	ld8 v[A] = [src_pre_mem], 128		// M0
135(p[B])	st8 [dst_pre_mem] = v[B], 128		// M2
136	br.ctop.sptk .prefetch_loop
137	;;
138	cmp.eq p16, p0 = r0, r0			// reset p16 to 1 (br.ctop cleared it to zero)
139	mov ar.lc = t1				// with 64KB pages, t1 is too big to fit in 8 bits!
140	mov ar.ec = N				// # of stages in pipeline
141	;;
142.line_copy:
143(p[D])	ld8 t2 = [src0], 3*8			// M0
144(p[D])	ld8 t4 = [src1], 3*8			// M1
145(p[B])	st8 [dst_pre_mem] = v[B], 128		// M2 prefetch dst from memory
146(p[D])	st8 [dst_pre_l2] = n[D-C], 128		// M3 prefetch dst from L2
147	;;
148(p[A])	ld8 v[A] = [src_pre_mem], 128		// M0 prefetch src from memory
149(p[C])	ld8 n[0] = [src_pre_l2], 128		// M1 prefetch src from L2
150(p[D])	st8 [dst0] =  t1, 8			// M2
151(p[D])	st8 [dst1] =  t3, 8			// M3
152	;;
153(p[D])	ld8  t5 = [src0], 8
154(p[D])	ld8  t7 = [src1], 3*8
155(p[D])	st8 [dst0] =  t2, 3*8
156(p[D])	st8 [dst1] =  t4, 3*8
157	;;
158(p[D])	ld8  t6 = [src0], 3*8
159(p[D])	ld8 t10 = [src1], 8
160(p[D])	st8 [dst0] =  t5, 8
161(p[D])	st8 [dst1] =  t7, 3*8
162	;;
163(p[D])	ld8  t9 = [src0], 3*8
164(p[D])	ld8 t11 = [src1], 3*8
165(p[D])	st8 [dst0] =  t6, 3*8
166(p[D])	st8 [dst1] = t10, 8
167	;;
168(p[D])	ld8 t12 = [src0], 8
169(p[D])	ld8 t14 = [src1], 8
170(p[D])	st8 [dst0] =  t9, 3*8
171(p[D])	st8 [dst1] = t11, 3*8
172	;;
173(p[D])	ld8 t13 = [src0], 4*8
174(p[D])	ld8 t15 = [src1], 4*8
175(p[D])	st8 [dst0] = t12, 8
176(p[D])	st8 [dst1] = t14, 8
177	;;
178(p[D-1])ld8  t1 = [src0], 8
179(p[D-1])ld8  t3 = [src1], 8
180(p[D])	st8 [dst0] = t13, 4*8
181(p[D])	st8 [dst1] = t15, 4*8
182	br.ctop.sptk .line_copy
183	;;
184	mov ar.lc = saved_lc
185	mov pr = saved_pr, -1
186	br.ret.sptk.many rp
187END(copy_page)
188EXPORT_SYMBOL(copy_page)