1! SPDX-License-Identifier: GPL-2.0
  2!
  3! Fast SH memcpy
  4!
  5! by Toshiyasu Morita (tm@netcom.com)
  6! hacked by J"orn Rernnecke (joern.rennecke@superh.com) ("o for o-umlaut)
  7! SH5 code Copyright 2002 SuperH Ltd.
  8!
  9! Entry: ARG0: destination pointer
 10!        ARG1: source pointer
 11!        ARG2: byte count
 12!
 13! Exit:  RESULT: destination pointer
 14!        any other registers in the range r0-r7: trashed
 15!
 16! Notes: Usually one wants to do small reads and write a longword, but
 17!        unfortunately it is difficult in some cases to concatanate bytes
 18!        into a longword on the SH, so this does a longword read and small
 19!        writes.
 20!
 21! This implementation makes two assumptions about how it is called:
 22!
 23! 1.: If the byte count is nonzero, the address of the last byte to be
 24!     copied is unsigned greater than the address of the first byte to
 25!     be copied.  This could be easily swapped for a signed comparison,
 26!     but the algorithm used needs some comparison.
 27!
 28! 2.: When there are two or three bytes in the last word of an 11-or-more
 29!     bytes memory chunk to b copied, the rest of the word can be read
 30!     without side effects.
 31!     This could be easily changed by increasing the minimum size of
 32!     a fast memcpy and the amount subtracted from r7 before L_2l_loop be 2,
 33!     however, this would cost a few extra cyles on average.
 34!     For SHmedia, the assumption is that any quadword can be read in its
 35!     enirety if at least one byte is included in the copy.
 36
 37/* Imported into Linux kernel by Richard Curnow.  This is used to implement the
 38   __copy_user function in the general case, so it has to be a distinct
 39   function from intra-kernel memcpy to allow for exception fix-ups in the
 40   event that the user pointer is bad somewhere in the copy (e.g. due to
 41   running off the end of the vma).
 42
 43   Note, this algorithm will be slightly wasteful in the case where the source
 44   and destination pointers are equally aligned, because the stlo/sthi pairs
 45   could then be merged back into single stores.  If there are a lot of cache
 46   misses, this is probably offset by the stall lengths on the preloads.
 47
 48*/
 49
 50/* NOTE : Prefetches removed and allocos guarded by synco to avoid TAKum03020
 51 * erratum.  The first two prefetches are nop-ed out to avoid upsetting the
 52 * instruction counts used in the jump address calculation.
 53 * */
 54
 55	.section .text..SHmedia32,"ax"
 56	.little
 57	.balign 32
 58	.global copy_user_memcpy
 59	.global copy_user_memcpy_end
 60copy_user_memcpy:
 61
 62#define LDUAQ(P,O,D0,D1) ldlo.q P,O,D0; ldhi.q P,O+7,D1
 63#define STUAQ(P,O,D0,D1) stlo.q P,O,D0; sthi.q P,O+7,D1
 64#define LDUAL(P,O,D0,D1) ldlo.l P,O,D0; ldhi.l P,O+3,D1
 65#define STUAL(P,O,D0,D1) stlo.l P,O,D0; sthi.l P,O+3,D1
 66
 67	nop ! ld.b r3,0,r63 ! TAKum03020
 68	pta/l Large,tr0
 69	movi 25,r0
 70	bgeu/u r4,r0,tr0
 71	nsb r4,r0
 72	shlli r0,5,r0
 73	movi (L1-L0+63*32 + 1) & 0xffff,r1
 74	sub r1, r0, r0
 75L0:	ptrel r0,tr0
 76	add r2,r4,r5
 77	ptabs r18,tr1
 78	add r3,r4,r6
 79	blink tr0,r63
 80
 81/* Rearranged to make cut2 safe */
 82	.balign 8
 83L4_7:	/* 4..7 byte memcpy cntd. */
 84	stlo.l r2, 0, r0
 85	or r6, r7, r6
 86	sthi.l r5, -1, r6
 87	stlo.l r5, -4, r6
 88	blink tr1,r63
 89
 90	.balign 8
 91L1:	/* 0 byte memcpy */
 92	nop
 93	blink tr1,r63
 94	nop
 95	nop
 96	nop
 97	nop
 98
 99L2_3:	/* 2 or 3 byte memcpy cntd. */
100	st.b r5,-1,r6
101	blink tr1,r63
102
103	/* 1 byte memcpy */
104	ld.b r3,0,r0
105	st.b r2,0,r0
106	blink tr1,r63
107
108L8_15:	/* 8..15 byte memcpy cntd. */
109	stlo.q r2, 0, r0
110	or r6, r7, r6
111	sthi.q r5, -1, r6
112	stlo.q r5, -8, r6
113	blink tr1,r63
114
115	/* 2 or 3 byte memcpy */
116	ld.b r3,0,r0
117	nop ! ld.b r2,0,r63 ! TAKum03020
118	ld.b r3,1,r1
119	st.b r2,0,r0
120	pta/l L2_3,tr0
121	ld.b r6,-1,r6
122	st.b r2,1,r1
123	blink tr0, r63
124
125	/* 4 .. 7 byte memcpy */
126	LDUAL (r3, 0, r0, r1)
127	pta L4_7, tr0
128	ldlo.l r6, -4, r7
129	or r0, r1, r0
130	sthi.l r2, 3, r0
131	ldhi.l r6, -1, r6
132	blink tr0, r63
133
134	/* 8 .. 15 byte memcpy */
135	LDUAQ (r3, 0, r0, r1)
136	pta L8_15, tr0
137	ldlo.q r6, -8, r7
138	or r0, r1, r0
139	sthi.q r2, 7, r0
140	ldhi.q r6, -1, r6
141	blink tr0, r63
142
143	/* 16 .. 24 byte memcpy */
144	LDUAQ (r3, 0, r0, r1)
145	LDUAQ (r3, 8, r8, r9)
146	or r0, r1, r0
147	sthi.q r2, 7, r0
148	or r8, r9, r8
149	sthi.q r2, 15, r8
150	ldlo.q r6, -8, r7
151	ldhi.q r6, -1, r6
152	stlo.q r2, 8, r8
153	stlo.q r2, 0, r0
154	or r6, r7, r6
155	sthi.q r5, -1, r6
156	stlo.q r5, -8, r6
157	blink tr1,r63
158
159Large:
160	! ld.b r2, 0, r63 ! TAKum03020
161	pta/l  Loop_ua, tr1
162	ori r3, -8, r7
163	sub r2, r7, r22
164	sub r3, r2, r6
165	add r2, r4, r5
166	ldlo.q r3, 0, r0
167	addi r5, -16, r5
168	movi 64+8, r27 ! could subtract r7 from that.
169	stlo.q r2, 0, r0
170	sthi.q r2, 7, r0
171	ldx.q r22, r6, r0
172	bgtu/l r27, r4, tr1
173
174	addi r5, -48, r27
175	pta/l Loop_line, tr0
176	addi r6, 64, r36
177	addi r6, -24, r19
178	addi r6, -16, r20
179	addi r6, -8, r21
180
181Loop_line:
182	! ldx.q r22, r36, r63 ! TAKum03020
183	alloco r22, 32
184	synco
185	addi r22, 32, r22
186	ldx.q r22, r19, r23
187	sthi.q r22, -25, r0
188	ldx.q r22, r20, r24
189	ldx.q r22, r21, r25
190	stlo.q r22, -32, r0
191	ldx.q r22, r6,  r0
192	sthi.q r22, -17, r23
193	sthi.q r22,  -9, r24
194	sthi.q r22,  -1, r25
195	stlo.q r22, -24, r23
196	stlo.q r22, -16, r24
197	stlo.q r22,  -8, r25
198	bgeu r27, r22, tr0
199
200Loop_ua:
201	addi r22, 8, r22
202	sthi.q r22, -1, r0
203	stlo.q r22, -8, r0
204	ldx.q r22, r6, r0
205	bgtu/l r5, r22, tr1
206
207	add r3, r4, r7
208	ldlo.q r7, -8, r1
209	sthi.q r22, 7, r0
210	ldhi.q r7, -1, r7
211	ptabs r18,tr1
212	stlo.q r22, 0, r0
213	or r1, r7, r1
214	sthi.q r5, 15, r1
215	stlo.q r5, 8, r1
216	blink tr1, r63
217copy_user_memcpy_end:
218	nop