Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * csum_partial_copy - do IP checksumming and copy
4 *
5 * (C) Copyright 1996 Linus Torvalds
6 * accelerated versions (and 21264 assembly versions ) contributed by
7 * Rick Gorton <rick.gorton@alpha-processor.com>
8 *
9 * Don't look at this too closely - you'll go mad. The things
10 * we do for performance..
11 */
12
13#include <linux/types.h>
14#include <linux/string.h>
15#include <linux/uaccess.h>
16
17
18#define ldq_u(x,y) \
19__asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
20
21#define stq_u(x,y) \
22__asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
23
24#define extql(x,y,z) \
25__asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
26
27#define extqh(x,y,z) \
28__asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
29
30#define mskql(x,y,z) \
31__asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
32
33#define mskqh(x,y,z) \
34__asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
35
36#define insql(x,y,z) \
37__asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
38
39#define insqh(x,y,z) \
40__asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
41
42
43#define __get_user_u(x,ptr) \
44({ \
45 long __guu_err; \
46 __asm__ __volatile__( \
47 "1: ldq_u %0,%2\n" \
48 "2:\n" \
49 EXC(1b,2b,%0,%1) \
50 : "=r"(x), "=r"(__guu_err) \
51 : "m"(__m(ptr)), "1"(0)); \
52 __guu_err; \
53})
54
55#define __put_user_u(x,ptr) \
56({ \
57 long __puu_err; \
58 __asm__ __volatile__( \
59 "1: stq_u %2,%1\n" \
60 "2:\n" \
61 EXC(1b,2b,$31,%0) \
62 : "=r"(__puu_err) \
63 : "m"(__m(addr)), "rJ"(x), "0"(0)); \
64 __puu_err; \
65})
66
67
68static inline unsigned short from64to16(unsigned long x)
69{
70 /* Using extract instructions is a bit more efficient
71 than the original shift/bitmask version. */
72
73 union {
74 unsigned long ul;
75 unsigned int ui[2];
76 unsigned short us[4];
77 } in_v, tmp_v, out_v;
78
79 in_v.ul = x;
80 tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
81
82 /* Since the bits of tmp_v.sh[3] are going to always be zero,
83 we don't have to bother to add that in. */
84 out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
85 + (unsigned long) tmp_v.us[2];
86
87 /* Similarly, out_v.us[2] is always zero for the final add. */
88 return out_v.us[0] + out_v.us[1];
89}
90
91
92
93/*
94 * Ok. This isn't fun, but this is the EASY case.
95 */
96static inline unsigned long
97csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
98 long len, unsigned long checksum,
99 int *errp)
100{
101 unsigned long carry = 0;
102 int err = 0;
103
104 while (len >= 0) {
105 unsigned long word;
106 err |= __get_user(word, src);
107 checksum += carry;
108 src++;
109 checksum += word;
110 len -= 8;
111 carry = checksum < word;
112 *dst = word;
113 dst++;
114 }
115 len += 8;
116 checksum += carry;
117 if (len) {
118 unsigned long word, tmp;
119 err |= __get_user(word, src);
120 tmp = *dst;
121 mskql(word, len, word);
122 checksum += word;
123 mskqh(tmp, len, tmp);
124 carry = checksum < word;
125 *dst = word | tmp;
126 checksum += carry;
127 }
128 if (err && errp) *errp = err;
129 return checksum;
130}
131
132/*
133 * This is even less fun, but this is still reasonably
134 * easy.
135 */
136static inline unsigned long
137csum_partial_cfu_dest_aligned(const unsigned long __user *src,
138 unsigned long *dst,
139 unsigned long soff,
140 long len, unsigned long checksum,
141 int *errp)
142{
143 unsigned long first;
144 unsigned long word, carry;
145 unsigned long lastsrc = 7+len+(unsigned long)src;
146 int err = 0;
147
148 err |= __get_user_u(first,src);
149 carry = 0;
150 while (len >= 0) {
151 unsigned long second;
152
153 err |= __get_user_u(second, src+1);
154 extql(first, soff, word);
155 len -= 8;
156 src++;
157 extqh(second, soff, first);
158 checksum += carry;
159 word |= first;
160 first = second;
161 checksum += word;
162 *dst = word;
163 dst++;
164 carry = checksum < word;
165 }
166 len += 8;
167 checksum += carry;
168 if (len) {
169 unsigned long tmp;
170 unsigned long second;
171 err |= __get_user_u(second, lastsrc);
172 tmp = *dst;
173 extql(first, soff, word);
174 extqh(second, soff, first);
175 word |= first;
176 mskql(word, len, word);
177 checksum += word;
178 mskqh(tmp, len, tmp);
179 carry = checksum < word;
180 *dst = word | tmp;
181 checksum += carry;
182 }
183 if (err && errp) *errp = err;
184 return checksum;
185}
186
187/*
188 * This is slightly less fun than the above..
189 */
190static inline unsigned long
191csum_partial_cfu_src_aligned(const unsigned long __user *src,
192 unsigned long *dst,
193 unsigned long doff,
194 long len, unsigned long checksum,
195 unsigned long partial_dest,
196 int *errp)
197{
198 unsigned long carry = 0;
199 unsigned long word;
200 unsigned long second_dest;
201 int err = 0;
202
203 mskql(partial_dest, doff, partial_dest);
204 while (len >= 0) {
205 err |= __get_user(word, src);
206 len -= 8;
207 insql(word, doff, second_dest);
208 checksum += carry;
209 stq_u(partial_dest | second_dest, dst);
210 src++;
211 checksum += word;
212 insqh(word, doff, partial_dest);
213 carry = checksum < word;
214 dst++;
215 }
216 len += 8;
217 if (len) {
218 checksum += carry;
219 err |= __get_user(word, src);
220 mskql(word, len, word);
221 len -= 8;
222 checksum += word;
223 insql(word, doff, second_dest);
224 len += doff;
225 carry = checksum < word;
226 partial_dest |= second_dest;
227 if (len >= 0) {
228 stq_u(partial_dest, dst);
229 if (!len) goto out;
230 dst++;
231 insqh(word, doff, partial_dest);
232 }
233 doff = len;
234 }
235 ldq_u(second_dest, dst);
236 mskqh(second_dest, doff, second_dest);
237 stq_u(partial_dest | second_dest, dst);
238out:
239 checksum += carry;
240 if (err && errp) *errp = err;
241 return checksum;
242}
243
244/*
245 * This is so totally un-fun that it's frightening. Don't
246 * look at this too closely, you'll go blind.
247 */
248static inline unsigned long
249csum_partial_cfu_unaligned(const unsigned long __user * src,
250 unsigned long * dst,
251 unsigned long soff, unsigned long doff,
252 long len, unsigned long checksum,
253 unsigned long partial_dest,
254 int *errp)
255{
256 unsigned long carry = 0;
257 unsigned long first;
258 unsigned long lastsrc;
259 int err = 0;
260
261 err |= __get_user_u(first, src);
262 lastsrc = 7+len+(unsigned long)src;
263 mskql(partial_dest, doff, partial_dest);
264 while (len >= 0) {
265 unsigned long second, word;
266 unsigned long second_dest;
267
268 err |= __get_user_u(second, src+1);
269 extql(first, soff, word);
270 checksum += carry;
271 len -= 8;
272 extqh(second, soff, first);
273 src++;
274 word |= first;
275 first = second;
276 insql(word, doff, second_dest);
277 checksum += word;
278 stq_u(partial_dest | second_dest, dst);
279 carry = checksum < word;
280 insqh(word, doff, partial_dest);
281 dst++;
282 }
283 len += doff;
284 checksum += carry;
285 if (len >= 0) {
286 unsigned long second, word;
287 unsigned long second_dest;
288
289 err |= __get_user_u(second, lastsrc);
290 extql(first, soff, word);
291 extqh(second, soff, first);
292 word |= first;
293 first = second;
294 mskql(word, len-doff, word);
295 checksum += word;
296 insql(word, doff, second_dest);
297 carry = checksum < word;
298 stq_u(partial_dest | second_dest, dst);
299 if (len) {
300 ldq_u(second_dest, dst+1);
301 insqh(word, doff, partial_dest);
302 mskqh(second_dest, len, second_dest);
303 stq_u(partial_dest | second_dest, dst+1);
304 }
305 checksum += carry;
306 } else {
307 unsigned long second, word;
308 unsigned long second_dest;
309
310 err |= __get_user_u(second, lastsrc);
311 extql(first, soff, word);
312 extqh(second, soff, first);
313 word |= first;
314 ldq_u(second_dest, dst);
315 mskql(word, len-doff, word);
316 checksum += word;
317 mskqh(second_dest, len, second_dest);
318 carry = checksum < word;
319 insql(word, doff, word);
320 stq_u(partial_dest | word | second_dest, dst);
321 checksum += carry;
322 }
323 if (err && errp) *errp = err;
324 return checksum;
325}
326
327__wsum
328csum_partial_copy_from_user(const void __user *src, void *dst, int len,
329 __wsum sum, int *errp)
330{
331 unsigned long checksum = (__force u32) sum;
332 unsigned long soff = 7 & (unsigned long) src;
333 unsigned long doff = 7 & (unsigned long) dst;
334
335 if (len) {
336 if (!access_ok(src, len)) {
337 if (errp) *errp = -EFAULT;
338 memset(dst, 0, len);
339 return sum;
340 }
341 if (!doff) {
342 if (!soff)
343 checksum = csum_partial_cfu_aligned(
344 (const unsigned long __user *) src,
345 (unsigned long *) dst,
346 len-8, checksum, errp);
347 else
348 checksum = csum_partial_cfu_dest_aligned(
349 (const unsigned long __user *) src,
350 (unsigned long *) dst,
351 soff, len-8, checksum, errp);
352 } else {
353 unsigned long partial_dest;
354 ldq_u(partial_dest, dst);
355 if (!soff)
356 checksum = csum_partial_cfu_src_aligned(
357 (const unsigned long __user *) src,
358 (unsigned long *) dst,
359 doff, len-8, checksum,
360 partial_dest, errp);
361 else
362 checksum = csum_partial_cfu_unaligned(
363 (const unsigned long __user *) src,
364 (unsigned long *) dst,
365 soff, doff, len-8, checksum,
366 partial_dest, errp);
367 }
368 checksum = from64to16 (checksum);
369 }
370 return (__force __wsum)checksum;
371}
372EXPORT_SYMBOL(csum_partial_copy_from_user);
373
374__wsum
375csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
376{
377 __wsum checksum;
378 mm_segment_t oldfs = get_fs();
379 set_fs(KERNEL_DS);
380 checksum = csum_partial_copy_from_user((__force const void __user *)src,
381 dst, len, sum, NULL);
382 set_fs(oldfs);
383 return checksum;
384}
385EXPORT_SYMBOL(csum_partial_copy_nocheck);