1/*
  2 * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com)
  3 *
  4 * This program is free software; you can redistribute it and/or modify
  5 * it under the terms of the GNU General Public License version 2 as
  6 * published by the Free Software Foundation.
  7 */
  8
  9#include <linux/linkage.h>
 10
 11#ifdef __LITTLE_ENDIAN__
 12# define SHIFT_1(RX,RY,IMM)	asl	RX, RY, IMM	; <<
 13# define SHIFT_2(RX,RY,IMM)	lsr	RX, RY, IMM	; >>
 14# define MERGE_1(RX,RY,IMM)	asl	RX, RY, IMM
 15# define MERGE_2(RX,RY,IMM)
 16# define EXTRACT_1(RX,RY,IMM)	and	RX, RY, 0xFFFF
 17# define EXTRACT_2(RX,RY,IMM)	lsr	RX, RY, IMM
 18#else
 19# define SHIFT_1(RX,RY,IMM)	lsr	RX, RY, IMM	; >>
 20# define SHIFT_2(RX,RY,IMM)	asl	RX, RY, IMM	; <<
 21# define MERGE_1(RX,RY,IMM)	asl	RX, RY, IMM	; <<
 22# define MERGE_2(RX,RY,IMM)	asl	RX, RY, IMM	; <<
 23# define EXTRACT_1(RX,RY,IMM)	lsr	RX, RY, IMM
 24# define EXTRACT_2(RX,RY,IMM)	lsr	RX, RY, 0x08
 25#endif
 26
 27#ifdef CONFIG_ARC_HAS_LL64
 28# define PREFETCH_READ(RX)	prefetch    [RX, 56]
 29# define PREFETCH_WRITE(RX)	prefetchw   [RX, 64]
 30# define LOADX(DST,RX)		ldd.ab	DST, [RX, 8]
 31# define STOREX(SRC,RX)		std.ab	SRC, [RX, 8]
 32# define ZOLSHFT		5
 33# define ZOLAND			0x1F
 34#else
 35# define PREFETCH_READ(RX)	prefetch    [RX, 28]
 36# define PREFETCH_WRITE(RX)	prefetchw   [RX, 32]
 37# define LOADX(DST,RX)		ld.ab	DST, [RX, 4]
 38# define STOREX(SRC,RX)		st.ab	SRC, [RX, 4]
 39# define ZOLSHFT		4
 40# define ZOLAND			0xF
 41#endif
 42
 43ENTRY_CFI(memcpy)
 44	prefetch [r1]		; Prefetch the read location
 45	prefetchw [r0]		; Prefetch the write location
 46	mov.f	0, r2
 47;;; if size is zero
 48	jz.d	[blink]
 49	mov	r3, r0		; don;t clobber ret val
 50
 51;;; if size <= 8
 52	cmp	r2, 8
 53	bls.d	@.Lsmallchunk
 54	mov.f	lp_count, r2
 55
 56	and.f	r4, r0, 0x03
 57	rsub	lp_count, r4, 4
 58	lpnz	@.Laligndestination
 59	;; LOOP BEGIN
 60	ldb.ab	r5, [r1,1]
 61	sub	r2, r2, 1
 62	stb.ab	r5, [r3,1]
 63.Laligndestination:
 64
 65;;; Check the alignment of the source
 66	and.f	r4, r1, 0x03
 67	bnz.d	@.Lsourceunaligned
 68
 69;;; CASE 0: Both source and destination are 32bit aligned
 70;;; Convert len to Dwords, unfold x4
 71	lsr.f	lp_count, r2, ZOLSHFT
 72	lpnz	@.Lcopy32_64bytes
 73	;; LOOP START
 74	LOADX (r6, r1)
 75	PREFETCH_READ (r1)
 76	PREFETCH_WRITE (r3)
 77	LOADX (r8, r1)
 78	LOADX (r10, r1)
 79	LOADX (r4, r1)
 80	STOREX (r6, r3)
 81	STOREX (r8, r3)
 82	STOREX (r10, r3)
 83	STOREX (r4, r3)
 84.Lcopy32_64bytes:
 85
 86	and.f	lp_count, r2, ZOLAND ;Last remaining 31 bytes
 87.Lsmallchunk:
 88	lpnz	@.Lcopyremainingbytes
 89	;; LOOP START
 90	ldb.ab	r5, [r1,1]
 91	stb.ab	r5, [r3,1]
 92.Lcopyremainingbytes:
 93
 94	j	[blink]
 95;;; END CASE 0
 96
 97.Lsourceunaligned:
 98	cmp	r4, 2
 99	beq.d	@.LunalignedOffby2
100	sub	r2, r2, 1
101
102	bhi.d	@.LunalignedOffby3
103	ldb.ab	r5, [r1, 1]
104
105;;; CASE 1: The source is unaligned, off by 1
106	;; Hence I need to read 1 byte for a 16bit alignment
107	;; and 2bytes to reach 32bit alignment
108	ldh.ab	r6, [r1, 2]
109	sub	r2, r2, 2
110	;; Convert to words, unfold x2
111	lsr.f	lp_count, r2, 3
112	MERGE_1 (r6, r6, 8)
113	MERGE_2 (r5, r5, 24)
114	or	r5, r5, r6
115
116	;; Both src and dst are aligned
117	lpnz	@.Lcopy8bytes_1
118	;; LOOP START
119	ld.ab	r6, [r1, 4]
120	prefetch [r1, 28]	;Prefetch the next read location
121	ld.ab	r8, [r1,4]
122	prefetchw [r3, 32]	;Prefetch the next write location
123
124	SHIFT_1	(r7, r6, 24)
125	or	r7, r7, r5
126	SHIFT_2	(r5, r6, 8)
127
128	SHIFT_1	(r9, r8, 24)
129	or	r9, r9, r5
130	SHIFT_2	(r5, r8, 8)
131
132	st.ab	r7, [r3, 4]
133	st.ab	r9, [r3, 4]
134.Lcopy8bytes_1:
135
136	;; Write back the remaining 16bits
137	EXTRACT_1 (r6, r5, 16)
138	sth.ab	r6, [r3, 2]
139	;; Write back the remaining 8bits
140	EXTRACT_2 (r5, r5, 16)
141	stb.ab	r5, [r3, 1]
142
143	and.f	lp_count, r2, 0x07 ;Last 8bytes
144	lpnz	@.Lcopybytewise_1
145	;; LOOP START
146	ldb.ab	r6, [r1,1]
147	stb.ab	r6, [r3,1]
148.Lcopybytewise_1:
149	j	[blink]
150
151.LunalignedOffby2:
152;;; CASE 2: The source is unaligned, off by 2
153	ldh.ab	r5, [r1, 2]
154	sub	r2, r2, 1
155
156	;; Both src and dst are aligned
157	;; Convert to words, unfold x2
158	lsr.f	lp_count, r2, 3
159#ifdef __BIG_ENDIAN__
160	asl.nz	r5, r5, 16
161#endif
162	lpnz	@.Lcopy8bytes_2
163	;; LOOP START
164	ld.ab	r6, [r1, 4]
165	prefetch [r1, 28]	;Prefetch the next read location
166	ld.ab	r8, [r1,4]
167	prefetchw [r3, 32]	;Prefetch the next write location
168
169	SHIFT_1	(r7, r6, 16)
170	or	r7, r7, r5
171	SHIFT_2	(r5, r6, 16)
172
173	SHIFT_1	(r9, r8, 16)
174	or	r9, r9, r5
175	SHIFT_2	(r5, r8, 16)
176
177	st.ab	r7, [r3, 4]
178	st.ab	r9, [r3, 4]
179.Lcopy8bytes_2:
180
181#ifdef __BIG_ENDIAN__
182	lsr.nz	r5, r5, 16
183#endif
184	sth.ab	r5, [r3, 2]
185
186	and.f	lp_count, r2, 0x07 ;Last 8bytes
187	lpnz	@.Lcopybytewise_2
188	;; LOOP START
189	ldb.ab	r6, [r1,1]
190	stb.ab	r6, [r3,1]
191.Lcopybytewise_2:
192	j	[blink]
193
194.LunalignedOffby3:
195;;; CASE 3: The source is unaligned, off by 3
196;;; Hence, I need to read 1byte for achieve the 32bit alignment
197
198	;; Both src and dst are aligned
199	;; Convert to words, unfold x2
200	lsr.f	lp_count, r2, 3
201#ifdef __BIG_ENDIAN__
202	asl.ne	r5, r5, 24
203#endif
204	lpnz	@.Lcopy8bytes_3
205	;; LOOP START
206	ld.ab	r6, [r1, 4]
207	prefetch [r1, 28]	;Prefetch the next read location
208	ld.ab	r8, [r1,4]
209	prefetchw [r3, 32]	;Prefetch the next write location
210
211	SHIFT_1	(r7, r6, 8)
212	or	r7, r7, r5
213	SHIFT_2	(r5, r6, 24)
214
215	SHIFT_1	(r9, r8, 8)
216	or	r9, r9, r5
217	SHIFT_2	(r5, r8, 24)
218
219	st.ab	r7, [r3, 4]
220	st.ab	r9, [r3, 4]
221.Lcopy8bytes_3:
222
223#ifdef __BIG_ENDIAN__
224	lsr.nz	r5, r5, 24
225#endif
226	stb.ab	r5, [r3, 1]
227
228	and.f	lp_count, r2, 0x07 ;Last 8bytes
229	lpnz	@.Lcopybytewise_3
230	;; LOOP START
231	ldb.ab	r6, [r1,1]
232	stb.ab	r6, [r3,1]
233.Lcopybytewise_3:
234	j	[blink]
235
236END_CFI(memcpy)