Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com)
4 */
5
6#include <linux/linkage.h>
7
8#ifdef __LITTLE_ENDIAN__
9# define SHIFT_1(RX,RY,IMM) asl RX, RY, IMM ; <<
10# define SHIFT_2(RX,RY,IMM) lsr RX, RY, IMM ; >>
11# define MERGE_1(RX,RY,IMM) asl RX, RY, IMM
12# define MERGE_2(RX,RY,IMM)
13# define EXTRACT_1(RX,RY,IMM) and RX, RY, 0xFFFF
14# define EXTRACT_2(RX,RY,IMM) lsr RX, RY, IMM
15#else
16# define SHIFT_1(RX,RY,IMM) lsr RX, RY, IMM ; >>
17# define SHIFT_2(RX,RY,IMM) asl RX, RY, IMM ; <<
18# define MERGE_1(RX,RY,IMM) asl RX, RY, IMM ; <<
19# define MERGE_2(RX,RY,IMM) asl RX, RY, IMM ; <<
20# define EXTRACT_1(RX,RY,IMM) lsr RX, RY, IMM
21# define EXTRACT_2(RX,RY,IMM) lsr RX, RY, 0x08
22#endif
23
24#ifdef CONFIG_ARC_HAS_LL64
25# define LOADX(DST,RX) ldd.ab DST, [RX, 8]
26# define STOREX(SRC,RX) std.ab SRC, [RX, 8]
27# define ZOLSHFT 5
28# define ZOLAND 0x1F
29#else
30# define LOADX(DST,RX) ld.ab DST, [RX, 4]
31# define STOREX(SRC,RX) st.ab SRC, [RX, 4]
32# define ZOLSHFT 4
33# define ZOLAND 0xF
34#endif
35
36ENTRY_CFI(memcpy)
37 mov.f 0, r2
38;;; if size is zero
39 jz.d [blink]
40 mov r3, r0 ; don;t clobber ret val
41
42;;; if size <= 8
43 cmp r2, 8
44 bls.d @.Lsmallchunk
45 mov.f lp_count, r2
46
47 and.f r4, r0, 0x03
48 rsub lp_count, r4, 4
49 lpnz @.Laligndestination
50 ;; LOOP BEGIN
51 ldb.ab r5, [r1,1]
52 sub r2, r2, 1
53 stb.ab r5, [r3,1]
54.Laligndestination:
55
56;;; Check the alignment of the source
57 and.f r4, r1, 0x03
58 bnz.d @.Lsourceunaligned
59
60;;; CASE 0: Both source and destination are 32bit aligned
61;;; Convert len to Dwords, unfold x4
62 lsr.f lp_count, r2, ZOLSHFT
63 lpnz @.Lcopy32_64bytes
64 ;; LOOP START
65 LOADX (r6, r1)
66 LOADX (r8, r1)
67 LOADX (r10, r1)
68 LOADX (r4, r1)
69 STOREX (r6, r3)
70 STOREX (r8, r3)
71 STOREX (r10, r3)
72 STOREX (r4, r3)
73.Lcopy32_64bytes:
74
75 and.f lp_count, r2, ZOLAND ;Last remaining 31 bytes
76.Lsmallchunk:
77 lpnz @.Lcopyremainingbytes
78 ;; LOOP START
79 ldb.ab r5, [r1,1]
80 stb.ab r5, [r3,1]
81.Lcopyremainingbytes:
82
83 j [blink]
84;;; END CASE 0
85
86.Lsourceunaligned:
87 cmp r4, 2
88 beq.d @.LunalignedOffby2
89 sub r2, r2, 1
90
91 bhi.d @.LunalignedOffby3
92 ldb.ab r5, [r1, 1]
93
94;;; CASE 1: The source is unaligned, off by 1
95 ;; Hence I need to read 1 byte for a 16bit alignment
96 ;; and 2bytes to reach 32bit alignment
97 ldh.ab r6, [r1, 2]
98 sub r2, r2, 2
99 ;; Convert to words, unfold x2
100 lsr.f lp_count, r2, 3
101 MERGE_1 (r6, r6, 8)
102 MERGE_2 (r5, r5, 24)
103 or r5, r5, r6
104
105 ;; Both src and dst are aligned
106 lpnz @.Lcopy8bytes_1
107 ;; LOOP START
108 ld.ab r6, [r1, 4]
109 ld.ab r8, [r1,4]
110
111 SHIFT_1 (r7, r6, 24)
112 or r7, r7, r5
113 SHIFT_2 (r5, r6, 8)
114
115 SHIFT_1 (r9, r8, 24)
116 or r9, r9, r5
117 SHIFT_2 (r5, r8, 8)
118
119 st.ab r7, [r3, 4]
120 st.ab r9, [r3, 4]
121.Lcopy8bytes_1:
122
123 ;; Write back the remaining 16bits
124 EXTRACT_1 (r6, r5, 16)
125 sth.ab r6, [r3, 2]
126 ;; Write back the remaining 8bits
127 EXTRACT_2 (r5, r5, 16)
128 stb.ab r5, [r3, 1]
129
130 and.f lp_count, r2, 0x07 ;Last 8bytes
131 lpnz @.Lcopybytewise_1
132 ;; LOOP START
133 ldb.ab r6, [r1,1]
134 stb.ab r6, [r3,1]
135.Lcopybytewise_1:
136 j [blink]
137
138.LunalignedOffby2:
139;;; CASE 2: The source is unaligned, off by 2
140 ldh.ab r5, [r1, 2]
141 sub r2, r2, 1
142
143 ;; Both src and dst are aligned
144 ;; Convert to words, unfold x2
145 lsr.f lp_count, r2, 3
146#ifdef __BIG_ENDIAN__
147 asl.nz r5, r5, 16
148#endif
149 lpnz @.Lcopy8bytes_2
150 ;; LOOP START
151 ld.ab r6, [r1, 4]
152 ld.ab r8, [r1,4]
153
154 SHIFT_1 (r7, r6, 16)
155 or r7, r7, r5
156 SHIFT_2 (r5, r6, 16)
157
158 SHIFT_1 (r9, r8, 16)
159 or r9, r9, r5
160 SHIFT_2 (r5, r8, 16)
161
162 st.ab r7, [r3, 4]
163 st.ab r9, [r3, 4]
164.Lcopy8bytes_2:
165
166#ifdef __BIG_ENDIAN__
167 lsr.nz r5, r5, 16
168#endif
169 sth.ab r5, [r3, 2]
170
171 and.f lp_count, r2, 0x07 ;Last 8bytes
172 lpnz @.Lcopybytewise_2
173 ;; LOOP START
174 ldb.ab r6, [r1,1]
175 stb.ab r6, [r3,1]
176.Lcopybytewise_2:
177 j [blink]
178
179.LunalignedOffby3:
180;;; CASE 3: The source is unaligned, off by 3
181;;; Hence, I need to read 1byte for achieve the 32bit alignment
182
183 ;; Both src and dst are aligned
184 ;; Convert to words, unfold x2
185 lsr.f lp_count, r2, 3
186#ifdef __BIG_ENDIAN__
187 asl.ne r5, r5, 24
188#endif
189 lpnz @.Lcopy8bytes_3
190 ;; LOOP START
191 ld.ab r6, [r1, 4]
192 ld.ab r8, [r1,4]
193
194 SHIFT_1 (r7, r6, 8)
195 or r7, r7, r5
196 SHIFT_2 (r5, r6, 24)
197
198 SHIFT_1 (r9, r8, 8)
199 or r9, r9, r5
200 SHIFT_2 (r5, r8, 24)
201
202 st.ab r7, [r3, 4]
203 st.ab r9, [r3, 4]
204.Lcopy8bytes_3:
205
206#ifdef __BIG_ENDIAN__
207 lsr.nz r5, r5, 24
208#endif
209 stb.ab r5, [r3, 1]
210
211 and.f lp_count, r2, 0x07 ;Last 8bytes
212 lpnz @.Lcopybytewise_3
213 ;; LOOP START
214 ldb.ab r6, [r1,1]
215 stb.ab r6, [r3,1]
216.Lcopybytewise_3:
217 j [blink]
218
219END_CFI(memcpy)
1/*
2 * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com)
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#include <linux/linkage.h>
10
11#ifdef __LITTLE_ENDIAN__
12# define SHIFT_1(RX,RY,IMM) asl RX, RY, IMM ; <<
13# define SHIFT_2(RX,RY,IMM) lsr RX, RY, IMM ; >>
14# define MERGE_1(RX,RY,IMM) asl RX, RY, IMM
15# define MERGE_2(RX,RY,IMM)
16# define EXTRACT_1(RX,RY,IMM) and RX, RY, 0xFFFF
17# define EXTRACT_2(RX,RY,IMM) lsr RX, RY, IMM
18#else
19# define SHIFT_1(RX,RY,IMM) lsr RX, RY, IMM ; >>
20# define SHIFT_2(RX,RY,IMM) asl RX, RY, IMM ; <<
21# define MERGE_1(RX,RY,IMM) asl RX, RY, IMM ; <<
22# define MERGE_2(RX,RY,IMM) asl RX, RY, IMM ; <<
23# define EXTRACT_1(RX,RY,IMM) lsr RX, RY, IMM
24# define EXTRACT_2(RX,RY,IMM) lsr RX, RY, 0x08
25#endif
26
27#ifdef CONFIG_ARC_HAS_LL64
28# define PREFETCH_READ(RX) prefetch [RX, 56]
29# define PREFETCH_WRITE(RX) prefetchw [RX, 64]
30# define LOADX(DST,RX) ldd.ab DST, [RX, 8]
31# define STOREX(SRC,RX) std.ab SRC, [RX, 8]
32# define ZOLSHFT 5
33# define ZOLAND 0x1F
34#else
35# define PREFETCH_READ(RX) prefetch [RX, 28]
36# define PREFETCH_WRITE(RX) prefetchw [RX, 32]
37# define LOADX(DST,RX) ld.ab DST, [RX, 4]
38# define STOREX(SRC,RX) st.ab SRC, [RX, 4]
39# define ZOLSHFT 4
40# define ZOLAND 0xF
41#endif
42
43ENTRY(memcpy)
44 prefetch [r1] ; Prefetch the read location
45 prefetchw [r0] ; Prefetch the write location
46 mov.f 0, r2
47;;; if size is zero
48 jz.d [blink]
49 mov r3, r0 ; don;t clobber ret val
50
51;;; if size <= 8
52 cmp r2, 8
53 bls.d @.Lsmallchunk
54 mov.f lp_count, r2
55
56 and.f r4, r0, 0x03
57 rsub lp_count, r4, 4
58 lpnz @.Laligndestination
59 ;; LOOP BEGIN
60 ldb.ab r5, [r1,1]
61 sub r2, r2, 1
62 stb.ab r5, [r3,1]
63.Laligndestination:
64
65;;; Check the alignment of the source
66 and.f r4, r1, 0x03
67 bnz.d @.Lsourceunaligned
68
69;;; CASE 0: Both source and destination are 32bit aligned
70;;; Convert len to Dwords, unfold x4
71 lsr.f lp_count, r2, ZOLSHFT
72 lpnz @.Lcopy32_64bytes
73 ;; LOOP START
74 LOADX (r6, r1)
75 PREFETCH_READ (r1)
76 PREFETCH_WRITE (r3)
77 LOADX (r8, r1)
78 LOADX (r10, r1)
79 LOADX (r4, r1)
80 STOREX (r6, r3)
81 STOREX (r8, r3)
82 STOREX (r10, r3)
83 STOREX (r4, r3)
84.Lcopy32_64bytes:
85
86 and.f lp_count, r2, ZOLAND ;Last remaining 31 bytes
87.Lsmallchunk:
88 lpnz @.Lcopyremainingbytes
89 ;; LOOP START
90 ldb.ab r5, [r1,1]
91 stb.ab r5, [r3,1]
92.Lcopyremainingbytes:
93
94 j [blink]
95;;; END CASE 0
96
97.Lsourceunaligned:
98 cmp r4, 2
99 beq.d @.LunalignedOffby2
100 sub r2, r2, 1
101
102 bhi.d @.LunalignedOffby3
103 ldb.ab r5, [r1, 1]
104
105;;; CASE 1: The source is unaligned, off by 1
106 ;; Hence I need to read 1 byte for a 16bit alignment
107 ;; and 2bytes to reach 32bit alignment
108 ldh.ab r6, [r1, 2]
109 sub r2, r2, 2
110 ;; Convert to words, unfold x2
111 lsr.f lp_count, r2, 3
112 MERGE_1 (r6, r6, 8)
113 MERGE_2 (r5, r5, 24)
114 or r5, r5, r6
115
116 ;; Both src and dst are aligned
117 lpnz @.Lcopy8bytes_1
118 ;; LOOP START
119 ld.ab r6, [r1, 4]
120 prefetch [r1, 28] ;Prefetch the next read location
121 ld.ab r8, [r1,4]
122 prefetchw [r3, 32] ;Prefetch the next write location
123
124 SHIFT_1 (r7, r6, 24)
125 or r7, r7, r5
126 SHIFT_2 (r5, r6, 8)
127
128 SHIFT_1 (r9, r8, 24)
129 or r9, r9, r5
130 SHIFT_2 (r5, r8, 8)
131
132 st.ab r7, [r3, 4]
133 st.ab r9, [r3, 4]
134.Lcopy8bytes_1:
135
136 ;; Write back the remaining 16bits
137 EXTRACT_1 (r6, r5, 16)
138 sth.ab r6, [r3, 2]
139 ;; Write back the remaining 8bits
140 EXTRACT_2 (r5, r5, 16)
141 stb.ab r5, [r3, 1]
142
143 and.f lp_count, r2, 0x07 ;Last 8bytes
144 lpnz @.Lcopybytewise_1
145 ;; LOOP START
146 ldb.ab r6, [r1,1]
147 stb.ab r6, [r3,1]
148.Lcopybytewise_1:
149 j [blink]
150
151.LunalignedOffby2:
152;;; CASE 2: The source is unaligned, off by 2
153 ldh.ab r5, [r1, 2]
154 sub r2, r2, 1
155
156 ;; Both src and dst are aligned
157 ;; Convert to words, unfold x2
158 lsr.f lp_count, r2, 3
159#ifdef __BIG_ENDIAN__
160 asl.nz r5, r5, 16
161#endif
162 lpnz @.Lcopy8bytes_2
163 ;; LOOP START
164 ld.ab r6, [r1, 4]
165 prefetch [r1, 28] ;Prefetch the next read location
166 ld.ab r8, [r1,4]
167 prefetchw [r3, 32] ;Prefetch the next write location
168
169 SHIFT_1 (r7, r6, 16)
170 or r7, r7, r5
171 SHIFT_2 (r5, r6, 16)
172
173 SHIFT_1 (r9, r8, 16)
174 or r9, r9, r5
175 SHIFT_2 (r5, r8, 16)
176
177 st.ab r7, [r3, 4]
178 st.ab r9, [r3, 4]
179.Lcopy8bytes_2:
180
181#ifdef __BIG_ENDIAN__
182 lsr.nz r5, r5, 16
183#endif
184 sth.ab r5, [r3, 2]
185
186 and.f lp_count, r2, 0x07 ;Last 8bytes
187 lpnz @.Lcopybytewise_2
188 ;; LOOP START
189 ldb.ab r6, [r1,1]
190 stb.ab r6, [r3,1]
191.Lcopybytewise_2:
192 j [blink]
193
194.LunalignedOffby3:
195;;; CASE 3: The source is unaligned, off by 3
196;;; Hence, I need to read 1byte for achieve the 32bit alignment
197
198 ;; Both src and dst are aligned
199 ;; Convert to words, unfold x2
200 lsr.f lp_count, r2, 3
201#ifdef __BIG_ENDIAN__
202 asl.ne r5, r5, 24
203#endif
204 lpnz @.Lcopy8bytes_3
205 ;; LOOP START
206 ld.ab r6, [r1, 4]
207 prefetch [r1, 28] ;Prefetch the next read location
208 ld.ab r8, [r1,4]
209 prefetchw [r3, 32] ;Prefetch the next write location
210
211 SHIFT_1 (r7, r6, 8)
212 or r7, r7, r5
213 SHIFT_2 (r5, r6, 24)
214
215 SHIFT_1 (r9, r8, 8)
216 or r9, r9, r5
217 SHIFT_2 (r5, r8, 24)
218
219 st.ab r7, [r3, 4]
220 st.ab r9, [r3, 4]
221.Lcopy8bytes_3:
222
223#ifdef __BIG_ENDIAN__
224 lsr.nz r5, r5, 24
225#endif
226 stb.ab r5, [r3, 1]
227
228 and.f lp_count, r2, 0x07 ;Last 8bytes
229 lpnz @.Lcopybytewise_3
230 ;; LOOP START
231 ldb.ab r6, [r1,1]
232 stb.ab r6, [r3,1]
233.Lcopybytewise_3:
234 j [blink]
235
236END(memcpy)