Loading...
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Quick'n'dirty IP checksum ...
7 *
8 * Copyright (C) 1998, 1999 Ralf Baechle
9 * Copyright (C) 1999 Silicon Graphics, Inc.
10 * Copyright (C) 2007 Maciej W. Rozycki
11 */
12#include <linux/errno.h>
13#include <asm/asm.h>
14#include <asm/asm-offsets.h>
15#include <asm/regdef.h>
16
17#ifdef CONFIG_64BIT
18/*
19 * As we are sharing code base with the mips32 tree (which use the o32 ABI
20 * register definitions). We need to redefine the register definitions from
21 * the n64 ABI register naming to the o32 ABI register naming.
22 */
23#undef t0
24#undef t1
25#undef t2
26#undef t3
27#define t0 $8
28#define t1 $9
29#define t2 $10
30#define t3 $11
31#define t4 $12
32#define t5 $13
33#define t6 $14
34#define t7 $15
35
36#define USE_DOUBLE
37#endif
38
39#ifdef USE_DOUBLE
40
41#define LOAD ld
42#define LOAD32 lwu
43#define ADD daddu
44#define NBYTES 8
45
46#else
47
48#define LOAD lw
49#define LOAD32 lw
50#define ADD addu
51#define NBYTES 4
52
53#endif /* USE_DOUBLE */
54
55#define UNIT(unit) ((unit)*NBYTES)
56
57#define ADDC(sum,reg) \
58 ADD sum, reg; \
59 sltu v1, sum, reg; \
60 ADD sum, v1; \
61
62#define ADDC32(sum,reg) \
63 addu sum, reg; \
64 sltu v1, sum, reg; \
65 addu sum, v1; \
66
67#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \
68 LOAD _t0, (offset + UNIT(0))(src); \
69 LOAD _t1, (offset + UNIT(1))(src); \
70 LOAD _t2, (offset + UNIT(2))(src); \
71 LOAD _t3, (offset + UNIT(3))(src); \
72 ADDC(sum, _t0); \
73 ADDC(sum, _t1); \
74 ADDC(sum, _t2); \
75 ADDC(sum, _t3)
76
77#ifdef USE_DOUBLE
78#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
79 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
80#else
81#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
82 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \
83 CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
84#endif
85
86/*
87 * a0: source address
88 * a1: length of the area to checksum
89 * a2: partial checksum
90 */
91
92#define src a0
93#define sum v0
94
95 .text
96 .set noreorder
97 .align 5
98LEAF(csum_partial)
99 move sum, zero
100 move t7, zero
101
102 sltiu t8, a1, 0x8
103 bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */
104 move t2, a1
105
106 andi t7, src, 0x1 /* odd buffer? */
107
108.Lhword_align:
109 beqz t7, .Lword_align
110 andi t8, src, 0x2
111
112 lbu t0, (src)
113 LONG_SUBU a1, a1, 0x1
114#ifdef __MIPSEL__
115 sll t0, t0, 8
116#endif
117 ADDC(sum, t0)
118 PTR_ADDU src, src, 0x1
119 andi t8, src, 0x2
120
121.Lword_align:
122 beqz t8, .Ldword_align
123 sltiu t8, a1, 56
124
125 lhu t0, (src)
126 LONG_SUBU a1, a1, 0x2
127 ADDC(sum, t0)
128 sltiu t8, a1, 56
129 PTR_ADDU src, src, 0x2
130
131.Ldword_align:
132 bnez t8, .Ldo_end_words
133 move t8, a1
134
135 andi t8, src, 0x4
136 beqz t8, .Lqword_align
137 andi t8, src, 0x8
138
139 LOAD32 t0, 0x00(src)
140 LONG_SUBU a1, a1, 0x4
141 ADDC(sum, t0)
142 PTR_ADDU src, src, 0x4
143 andi t8, src, 0x8
144
145.Lqword_align:
146 beqz t8, .Loword_align
147 andi t8, src, 0x10
148
149#ifdef USE_DOUBLE
150 ld t0, 0x00(src)
151 LONG_SUBU a1, a1, 0x8
152 ADDC(sum, t0)
153#else
154 lw t0, 0x00(src)
155 lw t1, 0x04(src)
156 LONG_SUBU a1, a1, 0x8
157 ADDC(sum, t0)
158 ADDC(sum, t1)
159#endif
160 PTR_ADDU src, src, 0x8
161 andi t8, src, 0x10
162
163.Loword_align:
164 beqz t8, .Lbegin_movement
165 LONG_SRL t8, a1, 0x7
166
167#ifdef USE_DOUBLE
168 ld t0, 0x00(src)
169 ld t1, 0x08(src)
170 ADDC(sum, t0)
171 ADDC(sum, t1)
172#else
173 CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
174#endif
175 LONG_SUBU a1, a1, 0x10
176 PTR_ADDU src, src, 0x10
177 LONG_SRL t8, a1, 0x7
178
179.Lbegin_movement:
180 beqz t8, 1f
181 andi t2, a1, 0x40
182
183.Lmove_128bytes:
184 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
185 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
186 CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
187 CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
188 LONG_SUBU t8, t8, 0x01
189 .set reorder /* DADDI_WAR */
190 PTR_ADDU src, src, 0x80
191 bnez t8, .Lmove_128bytes
192 .set noreorder
193
1941:
195 beqz t2, 1f
196 andi t2, a1, 0x20
197
198.Lmove_64bytes:
199 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
200 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
201 PTR_ADDU src, src, 0x40
202
2031:
204 beqz t2, .Ldo_end_words
205 andi t8, a1, 0x1c
206
207.Lmove_32bytes:
208 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
209 andi t8, a1, 0x1c
210 PTR_ADDU src, src, 0x20
211
212.Ldo_end_words:
213 beqz t8, .Lsmall_csumcpy
214 andi t2, a1, 0x3
215 LONG_SRL t8, t8, 0x2
216
217.Lend_words:
218 LOAD32 t0, (src)
219 LONG_SUBU t8, t8, 0x1
220 ADDC(sum, t0)
221 .set reorder /* DADDI_WAR */
222 PTR_ADDU src, src, 0x4
223 bnez t8, .Lend_words
224 .set noreorder
225
226/* unknown src alignment and < 8 bytes to go */
227.Lsmall_csumcpy:
228 move a1, t2
229
230 andi t0, a1, 4
231 beqz t0, 1f
232 andi t0, a1, 2
233
234 /* Still a full word to go */
235 ulw t1, (src)
236 PTR_ADDIU src, 4
237#ifdef USE_DOUBLE
238 dsll t1, t1, 32 /* clear lower 32bit */
239#endif
240 ADDC(sum, t1)
241
2421: move t1, zero
243 beqz t0, 1f
244 andi t0, a1, 1
245
246 /* Still a halfword to go */
247 ulhu t1, (src)
248 PTR_ADDIU src, 2
249
2501: beqz t0, 1f
251 sll t1, t1, 16
252
253 lbu t2, (src)
254 nop
255
256#ifdef __MIPSEB__
257 sll t2, t2, 8
258#endif
259 or t1, t2
260
2611: ADDC(sum, t1)
262
263 /* fold checksum */
264#ifdef USE_DOUBLE
265 dsll32 v1, sum, 0
266 daddu sum, v1
267 sltu v1, sum, v1
268 dsra32 sum, sum, 0
269 addu sum, v1
270#endif
271
272 /* odd buffer alignment? */
273#ifdef CPU_MIPSR2
274 wsbh v1, sum
275 movn sum, v1, t7
276#else
277 beqz t7, 1f /* odd buffer alignment? */
278 lui v1, 0x00ff
279 addu v1, 0x00ff
280 and t0, sum, v1
281 sll t0, t0, 8
282 srl sum, sum, 8
283 and sum, sum, v1
284 or sum, sum, t0
2851:
286#endif
287 .set reorder
288 /* Add the passed partial csum. */
289 ADDC32(sum, a2)
290 jr ra
291 .set noreorder
292 END(csum_partial)
293
294
295/*
296 * checksum and copy routines based on memcpy.S
297 *
298 * csum_partial_copy_nocheck(src, dst, len, sum)
299 * __csum_partial_copy_user(src, dst, len, sum, errp)
300 *
301 * See "Spec" in memcpy.S for details. Unlike __copy_user, all
302 * function in this file use the standard calling convention.
303 */
304
305#define src a0
306#define dst a1
307#define len a2
308#define psum a3
309#define sum v0
310#define odd t8
311#define errptr t9
312
313/*
314 * The exception handler for loads requires that:
315 * 1- AT contain the address of the byte just past the end of the source
316 * of the copy,
317 * 2- src_entry <= src < AT, and
318 * 3- (dst - src) == (dst_entry - src_entry),
319 * The _entry suffix denotes values when __copy_user was called.
320 *
321 * (1) is set up up by __csum_partial_copy_from_user and maintained by
322 * not writing AT in __csum_partial_copy
323 * (2) is met by incrementing src by the number of bytes copied
324 * (3) is met by not doing loads between a pair of increments of dst and src
325 *
326 * The exception handlers for stores stores -EFAULT to errptr and return.
327 * These handlers do not need to overwrite any data.
328 */
329
330#define EXC(inst_reg,addr,handler) \
3319: inst_reg, addr; \
332 .section __ex_table,"a"; \
333 PTR 9b, handler; \
334 .previous
335
336#ifdef USE_DOUBLE
337
338#define LOAD ld
339#define LOADL ldl
340#define LOADR ldr
341#define STOREL sdl
342#define STORER sdr
343#define STORE sd
344#define ADD daddu
345#define SUB dsubu
346#define SRL dsrl
347#define SLL dsll
348#define SLLV dsllv
349#define SRLV dsrlv
350#define NBYTES 8
351#define LOG_NBYTES 3
352
353#else
354
355#define LOAD lw
356#define LOADL lwl
357#define LOADR lwr
358#define STOREL swl
359#define STORER swr
360#define STORE sw
361#define ADD addu
362#define SUB subu
363#define SRL srl
364#define SLL sll
365#define SLLV sllv
366#define SRLV srlv
367#define NBYTES 4
368#define LOG_NBYTES 2
369
370#endif /* USE_DOUBLE */
371
372#ifdef CONFIG_CPU_LITTLE_ENDIAN
373#define LDFIRST LOADR
374#define LDREST LOADL
375#define STFIRST STORER
376#define STREST STOREL
377#define SHIFT_DISCARD SLLV
378#define SHIFT_DISCARD_REVERT SRLV
379#else
380#define LDFIRST LOADL
381#define LDREST LOADR
382#define STFIRST STOREL
383#define STREST STORER
384#define SHIFT_DISCARD SRLV
385#define SHIFT_DISCARD_REVERT SLLV
386#endif
387
388#define FIRST(unit) ((unit)*NBYTES)
389#define REST(unit) (FIRST(unit)+NBYTES-1)
390
391#define ADDRMASK (NBYTES-1)
392
393#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
394 .set noat
395#else
396 .set at=v1
397#endif
398
399LEAF(__csum_partial_copy_user)
400 PTR_ADDU AT, src, len /* See (1) above. */
401#ifdef CONFIG_64BIT
402 move errptr, a4
403#else
404 lw errptr, 16(sp)
405#endif
406FEXPORT(csum_partial_copy_nocheck)
407 move sum, zero
408 move odd, zero
409 /*
410 * Note: dst & src may be unaligned, len may be 0
411 * Temps
412 */
413 /*
414 * The "issue break"s below are very approximate.
415 * Issue delays for dcache fills will perturb the schedule, as will
416 * load queue full replay traps, etc.
417 *
418 * If len < NBYTES use byte operations.
419 */
420 sltu t2, len, NBYTES
421 and t1, dst, ADDRMASK
422 bnez t2, .Lcopy_bytes_checklen
423 and t0, src, ADDRMASK
424 andi odd, dst, 0x1 /* odd buffer? */
425 bnez t1, .Ldst_unaligned
426 nop
427 bnez t0, .Lsrc_unaligned_dst_aligned
428 /*
429 * use delay slot for fall-through
430 * src and dst are aligned; need to compute rem
431 */
432.Lboth_aligned:
433 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
434 beqz t0, .Lcleanup_both_aligned # len < 8*NBYTES
435 nop
436 SUB len, 8*NBYTES # subtract here for bgez loop
437 .align 4
4381:
439EXC( LOAD t0, UNIT(0)(src), .Ll_exc)
440EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy)
441EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy)
442EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy)
443EXC( LOAD t4, UNIT(4)(src), .Ll_exc_copy)
444EXC( LOAD t5, UNIT(5)(src), .Ll_exc_copy)
445EXC( LOAD t6, UNIT(6)(src), .Ll_exc_copy)
446EXC( LOAD t7, UNIT(7)(src), .Ll_exc_copy)
447 SUB len, len, 8*NBYTES
448 ADD src, src, 8*NBYTES
449EXC( STORE t0, UNIT(0)(dst), .Ls_exc)
450 ADDC(sum, t0)
451EXC( STORE t1, UNIT(1)(dst), .Ls_exc)
452 ADDC(sum, t1)
453EXC( STORE t2, UNIT(2)(dst), .Ls_exc)
454 ADDC(sum, t2)
455EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
456 ADDC(sum, t3)
457EXC( STORE t4, UNIT(4)(dst), .Ls_exc)
458 ADDC(sum, t4)
459EXC( STORE t5, UNIT(5)(dst), .Ls_exc)
460 ADDC(sum, t5)
461EXC( STORE t6, UNIT(6)(dst), .Ls_exc)
462 ADDC(sum, t6)
463EXC( STORE t7, UNIT(7)(dst), .Ls_exc)
464 ADDC(sum, t7)
465 .set reorder /* DADDI_WAR */
466 ADD dst, dst, 8*NBYTES
467 bgez len, 1b
468 .set noreorder
469 ADD len, 8*NBYTES # revert len (see above)
470
471 /*
472 * len == the number of bytes left to copy < 8*NBYTES
473 */
474.Lcleanup_both_aligned:
475#define rem t7
476 beqz len, .Ldone
477 sltu t0, len, 4*NBYTES
478 bnez t0, .Lless_than_4units
479 and rem, len, (NBYTES-1) # rem = len % NBYTES
480 /*
481 * len >= 4*NBYTES
482 */
483EXC( LOAD t0, UNIT(0)(src), .Ll_exc)
484EXC( LOAD t1, UNIT(1)(src), .Ll_exc_copy)
485EXC( LOAD t2, UNIT(2)(src), .Ll_exc_copy)
486EXC( LOAD t3, UNIT(3)(src), .Ll_exc_copy)
487 SUB len, len, 4*NBYTES
488 ADD src, src, 4*NBYTES
489EXC( STORE t0, UNIT(0)(dst), .Ls_exc)
490 ADDC(sum, t0)
491EXC( STORE t1, UNIT(1)(dst), .Ls_exc)
492 ADDC(sum, t1)
493EXC( STORE t2, UNIT(2)(dst), .Ls_exc)
494 ADDC(sum, t2)
495EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
496 ADDC(sum, t3)
497 .set reorder /* DADDI_WAR */
498 ADD dst, dst, 4*NBYTES
499 beqz len, .Ldone
500 .set noreorder
501.Lless_than_4units:
502 /*
503 * rem = len % NBYTES
504 */
505 beq rem, len, .Lcopy_bytes
506 nop
5071:
508EXC( LOAD t0, 0(src), .Ll_exc)
509 ADD src, src, NBYTES
510 SUB len, len, NBYTES
511EXC( STORE t0, 0(dst), .Ls_exc)
512 ADDC(sum, t0)
513 .set reorder /* DADDI_WAR */
514 ADD dst, dst, NBYTES
515 bne rem, len, 1b
516 .set noreorder
517
518 /*
519 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
520 * A loop would do only a byte at a time with possible branch
521 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE
522 * because can't assume read-access to dst. Instead, use
523 * STREST dst, which doesn't require read access to dst.
524 *
525 * This code should perform better than a simple loop on modern,
526 * wide-issue mips processors because the code has fewer branches and
527 * more instruction-level parallelism.
528 */
529#define bits t2
530 beqz len, .Ldone
531 ADD t1, dst, len # t1 is just past last byte of dst
532 li bits, 8*NBYTES
533 SLL rem, len, 3 # rem = number of bits to keep
534EXC( LOAD t0, 0(src), .Ll_exc)
535 SUB bits, bits, rem # bits = number of bits to discard
536 SHIFT_DISCARD t0, t0, bits
537EXC( STREST t0, -1(t1), .Ls_exc)
538 SHIFT_DISCARD_REVERT t0, t0, bits
539 .set reorder
540 ADDC(sum, t0)
541 b .Ldone
542 .set noreorder
543.Ldst_unaligned:
544 /*
545 * dst is unaligned
546 * t0 = src & ADDRMASK
547 * t1 = dst & ADDRMASK; T1 > 0
548 * len >= NBYTES
549 *
550 * Copy enough bytes to align dst
551 * Set match = (src and dst have same alignment)
552 */
553#define match rem
554EXC( LDFIRST t3, FIRST(0)(src), .Ll_exc)
555 ADD t2, zero, NBYTES
556EXC( LDREST t3, REST(0)(src), .Ll_exc_copy)
557 SUB t2, t2, t1 # t2 = number of bytes copied
558 xor match, t0, t1
559EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc)
560 SLL t4, t1, 3 # t4 = number of bits to discard
561 SHIFT_DISCARD t3, t3, t4
562 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
563 ADDC(sum, t3)
564 beq len, t2, .Ldone
565 SUB len, len, t2
566 ADD dst, dst, t2
567 beqz match, .Lboth_aligned
568 ADD src, src, t2
569
570.Lsrc_unaligned_dst_aligned:
571 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
572 beqz t0, .Lcleanup_src_unaligned
573 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
5741:
575/*
576 * Avoid consecutive LD*'s to the same register since some mips
577 * implementations can't issue them in the same cycle.
578 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
579 * are to the same unit (unless src is aligned, but it's not).
580 */
581EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc)
582EXC( LDFIRST t1, FIRST(1)(src), .Ll_exc_copy)
583 SUB len, len, 4*NBYTES
584EXC( LDREST t0, REST(0)(src), .Ll_exc_copy)
585EXC( LDREST t1, REST(1)(src), .Ll_exc_copy)
586EXC( LDFIRST t2, FIRST(2)(src), .Ll_exc_copy)
587EXC( LDFIRST t3, FIRST(3)(src), .Ll_exc_copy)
588EXC( LDREST t2, REST(2)(src), .Ll_exc_copy)
589EXC( LDREST t3, REST(3)(src), .Ll_exc_copy)
590 ADD src, src, 4*NBYTES
591#ifdef CONFIG_CPU_SB1
592 nop # improves slotting
593#endif
594EXC( STORE t0, UNIT(0)(dst), .Ls_exc)
595 ADDC(sum, t0)
596EXC( STORE t1, UNIT(1)(dst), .Ls_exc)
597 ADDC(sum, t1)
598EXC( STORE t2, UNIT(2)(dst), .Ls_exc)
599 ADDC(sum, t2)
600EXC( STORE t3, UNIT(3)(dst), .Ls_exc)
601 ADDC(sum, t3)
602 .set reorder /* DADDI_WAR */
603 ADD dst, dst, 4*NBYTES
604 bne len, rem, 1b
605 .set noreorder
606
607.Lcleanup_src_unaligned:
608 beqz len, .Ldone
609 and rem, len, NBYTES-1 # rem = len % NBYTES
610 beq rem, len, .Lcopy_bytes
611 nop
6121:
613EXC( LDFIRST t0, FIRST(0)(src), .Ll_exc)
614EXC( LDREST t0, REST(0)(src), .Ll_exc_copy)
615 ADD src, src, NBYTES
616 SUB len, len, NBYTES
617EXC( STORE t0, 0(dst), .Ls_exc)
618 ADDC(sum, t0)
619 .set reorder /* DADDI_WAR */
620 ADD dst, dst, NBYTES
621 bne len, rem, 1b
622 .set noreorder
623
624.Lcopy_bytes_checklen:
625 beqz len, .Ldone
626 nop
627.Lcopy_bytes:
628 /* 0 < len < NBYTES */
629#ifdef CONFIG_CPU_LITTLE_ENDIAN
630#define SHIFT_START 0
631#define SHIFT_INC 8
632#else
633#define SHIFT_START 8*(NBYTES-1)
634#define SHIFT_INC -8
635#endif
636 move t2, zero # partial word
637 li t3, SHIFT_START # shift
638/* use .Ll_exc_copy here to return correct sum on fault */
639#define COPY_BYTE(N) \
640EXC( lbu t0, N(src), .Ll_exc_copy); \
641 SUB len, len, 1; \
642EXC( sb t0, N(dst), .Ls_exc); \
643 SLLV t0, t0, t3; \
644 addu t3, SHIFT_INC; \
645 beqz len, .Lcopy_bytes_done; \
646 or t2, t0
647
648 COPY_BYTE(0)
649 COPY_BYTE(1)
650#ifdef USE_DOUBLE
651 COPY_BYTE(2)
652 COPY_BYTE(3)
653 COPY_BYTE(4)
654 COPY_BYTE(5)
655#endif
656EXC( lbu t0, NBYTES-2(src), .Ll_exc_copy)
657 SUB len, len, 1
658EXC( sb t0, NBYTES-2(dst), .Ls_exc)
659 SLLV t0, t0, t3
660 or t2, t0
661.Lcopy_bytes_done:
662 ADDC(sum, t2)
663.Ldone:
664 /* fold checksum */
665#ifdef USE_DOUBLE
666 dsll32 v1, sum, 0
667 daddu sum, v1
668 sltu v1, sum, v1
669 dsra32 sum, sum, 0
670 addu sum, v1
671#endif
672
673#ifdef CPU_MIPSR2
674 wsbh v1, sum
675 movn sum, v1, odd
676#else
677 beqz odd, 1f /* odd buffer alignment? */
678 lui v1, 0x00ff
679 addu v1, 0x00ff
680 and t0, sum, v1
681 sll t0, t0, 8
682 srl sum, sum, 8
683 and sum, sum, v1
684 or sum, sum, t0
6851:
686#endif
687 .set reorder
688 ADDC32(sum, psum)
689 jr ra
690 .set noreorder
691
692.Ll_exc_copy:
693 /*
694 * Copy bytes from src until faulting load address (or until a
695 * lb faults)
696 *
697 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
698 * may be more than a byte beyond the last address.
699 * Hence, the lb below may get an exception.
700 *
701 * Assumes src < THREAD_BUADDR($28)
702 */
703 LOAD t0, TI_TASK($28)
704 li t2, SHIFT_START
705 LOAD t0, THREAD_BUADDR(t0)
7061:
707EXC( lbu t1, 0(src), .Ll_exc)
708 ADD src, src, 1
709 sb t1, 0(dst) # can't fault -- we're copy_from_user
710 SLLV t1, t1, t2
711 addu t2, SHIFT_INC
712 ADDC(sum, t1)
713 .set reorder /* DADDI_WAR */
714 ADD dst, dst, 1
715 bne src, t0, 1b
716 .set noreorder
717.Ll_exc:
718 LOAD t0, TI_TASK($28)
719 nop
720 LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address
721 nop
722 SUB len, AT, t0 # len number of uncopied bytes
723 /*
724 * Here's where we rely on src and dst being incremented in tandem,
725 * See (3) above.
726 * dst += (fault addr - src) to put dst at first byte to clear
727 */
728 ADD dst, t0 # compute start address in a1
729 SUB dst, src
730 /*
731 * Clear len bytes starting at dst. Can't call __bzero because it
732 * might modify len. An inefficient loop for these rare times...
733 */
734 .set reorder /* DADDI_WAR */
735 SUB src, len, 1
736 beqz len, .Ldone
737 .set noreorder
7381: sb zero, 0(dst)
739 ADD dst, dst, 1
740 .set push
741 .set noat
742#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
743 bnez src, 1b
744 SUB src, src, 1
745#else
746 li v1, 1
747 bnez src, 1b
748 SUB src, src, v1
749#endif
750 li v1, -EFAULT
751 b .Ldone
752 sw v1, (errptr)
753
754.Ls_exc:
755 li v0, -1 /* invalid checksum */
756 li v1, -EFAULT
757 jr ra
758 sw v1, (errptr)
759 .set pop
760 END(__csum_partial_copy_user)
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Quick'n'dirty IP checksum ...
7 *
8 * Copyright (C) 1998, 1999 Ralf Baechle
9 * Copyright (C) 1999 Silicon Graphics, Inc.
10 * Copyright (C) 2007 Maciej W. Rozycki
11 * Copyright (C) 2014 Imagination Technologies Ltd.
12 */
13#include <linux/errno.h>
14#include <asm/asm.h>
15#include <asm/asm-offsets.h>
16#include <asm/regdef.h>
17
18#ifdef CONFIG_64BIT
19/*
20 * As we are sharing code base with the mips32 tree (which use the o32 ABI
21 * register definitions). We need to redefine the register definitions from
22 * the n64 ABI register naming to the o32 ABI register naming.
23 */
24#undef t0
25#undef t1
26#undef t2
27#undef t3
28#define t0 $8
29#define t1 $9
30#define t2 $10
31#define t3 $11
32#define t4 $12
33#define t5 $13
34#define t6 $14
35#define t7 $15
36
37#define USE_DOUBLE
38#endif
39
40#ifdef USE_DOUBLE
41
42#define LOAD ld
43#define LOAD32 lwu
44#define ADD daddu
45#define NBYTES 8
46
47#else
48
49#define LOAD lw
50#define LOAD32 lw
51#define ADD addu
52#define NBYTES 4
53
54#endif /* USE_DOUBLE */
55
56#define UNIT(unit) ((unit)*NBYTES)
57
58#define ADDC(sum,reg) \
59 .set push; \
60 .set noat; \
61 ADD sum, reg; \
62 sltu v1, sum, reg; \
63 ADD sum, v1; \
64 .set pop
65
66#define ADDC32(sum,reg) \
67 .set push; \
68 .set noat; \
69 addu sum, reg; \
70 sltu v1, sum, reg; \
71 addu sum, v1; \
72 .set pop
73
74#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \
75 LOAD _t0, (offset + UNIT(0))(src); \
76 LOAD _t1, (offset + UNIT(1))(src); \
77 LOAD _t2, (offset + UNIT(2))(src); \
78 LOAD _t3, (offset + UNIT(3))(src); \
79 ADDC(_t0, _t1); \
80 ADDC(_t2, _t3); \
81 ADDC(sum, _t0); \
82 ADDC(sum, _t2)
83
84#ifdef USE_DOUBLE
85#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
86 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
87#else
88#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
89 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \
90 CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
91#endif
92
93/*
94 * a0: source address
95 * a1: length of the area to checksum
96 * a2: partial checksum
97 */
98
99#define src a0
100#define sum v0
101
102 .text
103 .set noreorder
104 .align 5
105LEAF(csum_partial)
106 move sum, zero
107 move t7, zero
108
109 sltiu t8, a1, 0x8
110 bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */
111 move t2, a1
112
113 andi t7, src, 0x1 /* odd buffer? */
114
115.Lhword_align:
116 beqz t7, .Lword_align
117 andi t8, src, 0x2
118
119 lbu t0, (src)
120 LONG_SUBU a1, a1, 0x1
121#ifdef __MIPSEL__
122 sll t0, t0, 8
123#endif
124 ADDC(sum, t0)
125 PTR_ADDU src, src, 0x1
126 andi t8, src, 0x2
127
128.Lword_align:
129 beqz t8, .Ldword_align
130 sltiu t8, a1, 56
131
132 lhu t0, (src)
133 LONG_SUBU a1, a1, 0x2
134 ADDC(sum, t0)
135 sltiu t8, a1, 56
136 PTR_ADDU src, src, 0x2
137
138.Ldword_align:
139 bnez t8, .Ldo_end_words
140 move t8, a1
141
142 andi t8, src, 0x4
143 beqz t8, .Lqword_align
144 andi t8, src, 0x8
145
146 LOAD32 t0, 0x00(src)
147 LONG_SUBU a1, a1, 0x4
148 ADDC(sum, t0)
149 PTR_ADDU src, src, 0x4
150 andi t8, src, 0x8
151
152.Lqword_align:
153 beqz t8, .Loword_align
154 andi t8, src, 0x10
155
156#ifdef USE_DOUBLE
157 ld t0, 0x00(src)
158 LONG_SUBU a1, a1, 0x8
159 ADDC(sum, t0)
160#else
161 lw t0, 0x00(src)
162 lw t1, 0x04(src)
163 LONG_SUBU a1, a1, 0x8
164 ADDC(sum, t0)
165 ADDC(sum, t1)
166#endif
167 PTR_ADDU src, src, 0x8
168 andi t8, src, 0x10
169
170.Loword_align:
171 beqz t8, .Lbegin_movement
172 LONG_SRL t8, a1, 0x7
173
174#ifdef USE_DOUBLE
175 ld t0, 0x00(src)
176 ld t1, 0x08(src)
177 ADDC(sum, t0)
178 ADDC(sum, t1)
179#else
180 CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
181#endif
182 LONG_SUBU a1, a1, 0x10
183 PTR_ADDU src, src, 0x10
184 LONG_SRL t8, a1, 0x7
185
186.Lbegin_movement:
187 beqz t8, 1f
188 andi t2, a1, 0x40
189
190.Lmove_128bytes:
191 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
192 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
193 CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
194 CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
195 LONG_SUBU t8, t8, 0x01
196 .set reorder /* DADDI_WAR */
197 PTR_ADDU src, src, 0x80
198 bnez t8, .Lmove_128bytes
199 .set noreorder
200
2011:
202 beqz t2, 1f
203 andi t2, a1, 0x20
204
205.Lmove_64bytes:
206 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
207 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
208 PTR_ADDU src, src, 0x40
209
2101:
211 beqz t2, .Ldo_end_words
212 andi t8, a1, 0x1c
213
214.Lmove_32bytes:
215 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
216 andi t8, a1, 0x1c
217 PTR_ADDU src, src, 0x20
218
219.Ldo_end_words:
220 beqz t8, .Lsmall_csumcpy
221 andi t2, a1, 0x3
222 LONG_SRL t8, t8, 0x2
223
224.Lend_words:
225 LOAD32 t0, (src)
226 LONG_SUBU t8, t8, 0x1
227 ADDC(sum, t0)
228 .set reorder /* DADDI_WAR */
229 PTR_ADDU src, src, 0x4
230 bnez t8, .Lend_words
231 .set noreorder
232
233/* unknown src alignment and < 8 bytes to go */
234.Lsmall_csumcpy:
235 move a1, t2
236
237 andi t0, a1, 4
238 beqz t0, 1f
239 andi t0, a1, 2
240
241 /* Still a full word to go */
242 ulw t1, (src)
243 PTR_ADDIU src, 4
244#ifdef USE_DOUBLE
245 dsll t1, t1, 32 /* clear lower 32bit */
246#endif
247 ADDC(sum, t1)
248
2491: move t1, zero
250 beqz t0, 1f
251 andi t0, a1, 1
252
253 /* Still a halfword to go */
254 ulhu t1, (src)
255 PTR_ADDIU src, 2
256
2571: beqz t0, 1f
258 sll t1, t1, 16
259
260 lbu t2, (src)
261 nop
262
263#ifdef __MIPSEB__
264 sll t2, t2, 8
265#endif
266 or t1, t2
267
2681: ADDC(sum, t1)
269
270 /* fold checksum */
271#ifdef USE_DOUBLE
272 dsll32 v1, sum, 0
273 daddu sum, v1
274 sltu v1, sum, v1
275 dsra32 sum, sum, 0
276 addu sum, v1
277#endif
278
279 /* odd buffer alignment? */
280#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_LOONGSON3)
281 .set push
282 .set arch=mips32r2
283 wsbh v1, sum
284 movn sum, v1, t7
285 .set pop
286#else
287 beqz t7, 1f /* odd buffer alignment? */
288 lui v1, 0x00ff
289 addu v1, 0x00ff
290 and t0, sum, v1
291 sll t0, t0, 8
292 srl sum, sum, 8
293 and sum, sum, v1
294 or sum, sum, t0
2951:
296#endif
297 .set reorder
298 /* Add the passed partial csum. */
299 ADDC32(sum, a2)
300 jr ra
301 .set noreorder
302 END(csum_partial)
303
304
305/*
306 * checksum and copy routines based on memcpy.S
307 *
308 * csum_partial_copy_nocheck(src, dst, len, sum)
309 * __csum_partial_copy_kernel(src, dst, len, sum, errp)
310 *
311 * See "Spec" in memcpy.S for details. Unlike __copy_user, all
312 * function in this file use the standard calling convention.
313 */
314
315#define src a0
316#define dst a1
317#define len a2
318#define psum a3
319#define sum v0
320#define odd t8
321#define errptr t9
322
323/*
324 * The exception handler for loads requires that:
325 * 1- AT contain the address of the byte just past the end of the source
326 * of the copy,
327 * 2- src_entry <= src < AT, and
328 * 3- (dst - src) == (dst_entry - src_entry),
329 * The _entry suffix denotes values when __copy_user was called.
330 *
331 * (1) is set up up by __csum_partial_copy_from_user and maintained by
332 * not writing AT in __csum_partial_copy
333 * (2) is met by incrementing src by the number of bytes copied
334 * (3) is met by not doing loads between a pair of increments of dst and src
335 *
336 * The exception handlers for stores stores -EFAULT to errptr and return.
337 * These handlers do not need to overwrite any data.
338 */
339
340/* Instruction type */
341#define LD_INSN 1
342#define ST_INSN 2
343#define LEGACY_MODE 1
344#define EVA_MODE 2
345#define USEROP 1
346#define KERNELOP 2
347
348/*
349 * Wrapper to add an entry in the exception table
350 * in case the insn causes a memory exception.
351 * Arguments:
352 * insn : Load/store instruction
353 * type : Instruction type
354 * reg : Register
355 * addr : Address
356 * handler : Exception handler
357 */
358#define EXC(insn, type, reg, addr, handler) \
359 .if \mode == LEGACY_MODE; \
3609: insn reg, addr; \
361 .section __ex_table,"a"; \
362 PTR 9b, handler; \
363 .previous; \
364 /* This is enabled in EVA mode */ \
365 .else; \
366 /* If loading from user or storing to user */ \
367 .if ((\from == USEROP) && (type == LD_INSN)) || \
368 ((\to == USEROP) && (type == ST_INSN)); \
3699: __BUILD_EVA_INSN(insn##e, reg, addr); \
370 .section __ex_table,"a"; \
371 PTR 9b, handler; \
372 .previous; \
373 .else; \
374 /* EVA without exception */ \
375 insn reg, addr; \
376 .endif; \
377 .endif
378
379#undef LOAD
380
381#ifdef USE_DOUBLE
382
383#define LOADK ld /* No exception */
384#define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
385#define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
386#define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
387#define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
388#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
389#define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
390#define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
391#define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
392#define ADD daddu
393#define SUB dsubu
394#define SRL dsrl
395#define SLL dsll
396#define SLLV dsllv
397#define SRLV dsrlv
398#define NBYTES 8
399#define LOG_NBYTES 3
400
401#else
402
403#define LOADK lw /* No exception */
404#define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
405#define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
406#define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
407#define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
408#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
409#define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
410#define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
411#define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
412#define ADD addu
413#define SUB subu
414#define SRL srl
415#define SLL sll
416#define SLLV sllv
417#define SRLV srlv
418#define NBYTES 4
419#define LOG_NBYTES 2
420
421#endif /* USE_DOUBLE */
422
423#ifdef CONFIG_CPU_LITTLE_ENDIAN
424#define LDFIRST LOADR
425#define LDREST LOADL
426#define STFIRST STORER
427#define STREST STOREL
428#define SHIFT_DISCARD SLLV
429#define SHIFT_DISCARD_REVERT SRLV
430#else
431#define LDFIRST LOADL
432#define LDREST LOADR
433#define STFIRST STOREL
434#define STREST STORER
435#define SHIFT_DISCARD SRLV
436#define SHIFT_DISCARD_REVERT SLLV
437#endif
438
439#define FIRST(unit) ((unit)*NBYTES)
440#define REST(unit) (FIRST(unit)+NBYTES-1)
441
442#define ADDRMASK (NBYTES-1)
443
444#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
445 .set noat
446#else
447 .set at=v1
448#endif
449
450 .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to, __nocheck
451
452 PTR_ADDU AT, src, len /* See (1) above. */
453 /* initialize __nocheck if this the first time we execute this
454 * macro
455 */
456#ifdef CONFIG_64BIT
457 move errptr, a4
458#else
459 lw errptr, 16(sp)
460#endif
461 .if \__nocheck == 1
462 FEXPORT(csum_partial_copy_nocheck)
463 .endif
464 move sum, zero
465 move odd, zero
466 /*
467 * Note: dst & src may be unaligned, len may be 0
468 * Temps
469 */
470 /*
471 * The "issue break"s below are very approximate.
472 * Issue delays for dcache fills will perturb the schedule, as will
473 * load queue full replay traps, etc.
474 *
475 * If len < NBYTES use byte operations.
476 */
477 sltu t2, len, NBYTES
478 and t1, dst, ADDRMASK
479 bnez t2, .Lcopy_bytes_checklen\@
480 and t0, src, ADDRMASK
481 andi odd, dst, 0x1 /* odd buffer? */
482 bnez t1, .Ldst_unaligned\@
483 nop
484 bnez t0, .Lsrc_unaligned_dst_aligned\@
485 /*
486 * use delay slot for fall-through
487 * src and dst are aligned; need to compute rem
488 */
489.Lboth_aligned\@:
490 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
491 beqz t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
492 nop
493 SUB len, 8*NBYTES # subtract here for bgez loop
494 .align 4
4951:
496 LOAD(t0, UNIT(0)(src), .Ll_exc\@)
497 LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
498 LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
499 LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
500 LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
501 LOAD(t5, UNIT(5)(src), .Ll_exc_copy\@)
502 LOAD(t6, UNIT(6)(src), .Ll_exc_copy\@)
503 LOAD(t7, UNIT(7)(src), .Ll_exc_copy\@)
504 SUB len, len, 8*NBYTES
505 ADD src, src, 8*NBYTES
506 STORE(t0, UNIT(0)(dst), .Ls_exc\@)
507 ADDC(t0, t1)
508 STORE(t1, UNIT(1)(dst), .Ls_exc\@)
509 ADDC(sum, t0)
510 STORE(t2, UNIT(2)(dst), .Ls_exc\@)
511 ADDC(t2, t3)
512 STORE(t3, UNIT(3)(dst), .Ls_exc\@)
513 ADDC(sum, t2)
514 STORE(t4, UNIT(4)(dst), .Ls_exc\@)
515 ADDC(t4, t5)
516 STORE(t5, UNIT(5)(dst), .Ls_exc\@)
517 ADDC(sum, t4)
518 STORE(t6, UNIT(6)(dst), .Ls_exc\@)
519 ADDC(t6, t7)
520 STORE(t7, UNIT(7)(dst), .Ls_exc\@)
521 ADDC(sum, t6)
522 .set reorder /* DADDI_WAR */
523 ADD dst, dst, 8*NBYTES
524 bgez len, 1b
525 .set noreorder
526 ADD len, 8*NBYTES # revert len (see above)
527
528 /*
529 * len == the number of bytes left to copy < 8*NBYTES
530 */
531.Lcleanup_both_aligned\@:
532#define rem t7
533 beqz len, .Ldone\@
534 sltu t0, len, 4*NBYTES
535 bnez t0, .Lless_than_4units\@
536 and rem, len, (NBYTES-1) # rem = len % NBYTES
537 /*
538 * len >= 4*NBYTES
539 */
540 LOAD(t0, UNIT(0)(src), .Ll_exc\@)
541 LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
542 LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
543 LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
544 SUB len, len, 4*NBYTES
545 ADD src, src, 4*NBYTES
546 STORE(t0, UNIT(0)(dst), .Ls_exc\@)
547 ADDC(t0, t1)
548 STORE(t1, UNIT(1)(dst), .Ls_exc\@)
549 ADDC(sum, t0)
550 STORE(t2, UNIT(2)(dst), .Ls_exc\@)
551 ADDC(t2, t3)
552 STORE(t3, UNIT(3)(dst), .Ls_exc\@)
553 ADDC(sum, t2)
554 .set reorder /* DADDI_WAR */
555 ADD dst, dst, 4*NBYTES
556 beqz len, .Ldone\@
557 .set noreorder
558.Lless_than_4units\@:
559 /*
560 * rem = len % NBYTES
561 */
562 beq rem, len, .Lcopy_bytes\@
563 nop
5641:
565 LOAD(t0, 0(src), .Ll_exc\@)
566 ADD src, src, NBYTES
567 SUB len, len, NBYTES
568 STORE(t0, 0(dst), .Ls_exc\@)
569 ADDC(sum, t0)
570 .set reorder /* DADDI_WAR */
571 ADD dst, dst, NBYTES
572 bne rem, len, 1b
573 .set noreorder
574
575 /*
576 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
577 * A loop would do only a byte at a time with possible branch
578 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE
579 * because can't assume read-access to dst. Instead, use
580 * STREST dst, which doesn't require read access to dst.
581 *
582 * This code should perform better than a simple loop on modern,
583 * wide-issue mips processors because the code has fewer branches and
584 * more instruction-level parallelism.
585 */
586#define bits t2
587 beqz len, .Ldone\@
588 ADD t1, dst, len # t1 is just past last byte of dst
589 li bits, 8*NBYTES
590 SLL rem, len, 3 # rem = number of bits to keep
591 LOAD(t0, 0(src), .Ll_exc\@)
592 SUB bits, bits, rem # bits = number of bits to discard
593 SHIFT_DISCARD t0, t0, bits
594 STREST(t0, -1(t1), .Ls_exc\@)
595 SHIFT_DISCARD_REVERT t0, t0, bits
596 .set reorder
597 ADDC(sum, t0)
598 b .Ldone\@
599 .set noreorder
600.Ldst_unaligned\@:
601 /*
602 * dst is unaligned
603 * t0 = src & ADDRMASK
604 * t1 = dst & ADDRMASK; T1 > 0
605 * len >= NBYTES
606 *
607 * Copy enough bytes to align dst
608 * Set match = (src and dst have same alignment)
609 */
610#define match rem
611 LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
612 ADD t2, zero, NBYTES
613 LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
614 SUB t2, t2, t1 # t2 = number of bytes copied
615 xor match, t0, t1
616 STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
617 SLL t4, t1, 3 # t4 = number of bits to discard
618 SHIFT_DISCARD t3, t3, t4
619 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
620 ADDC(sum, t3)
621 beq len, t2, .Ldone\@
622 SUB len, len, t2
623 ADD dst, dst, t2
624 beqz match, .Lboth_aligned\@
625 ADD src, src, t2
626
627.Lsrc_unaligned_dst_aligned\@:
628 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
629 beqz t0, .Lcleanup_src_unaligned\@
630 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
6311:
632/*
633 * Avoid consecutive LD*'s to the same register since some mips
634 * implementations can't issue them in the same cycle.
635 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
636 * are to the same unit (unless src is aligned, but it's not).
637 */
638 LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
639 LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
640 SUB len, len, 4*NBYTES
641 LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
642 LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
643 LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
644 LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
645 LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
646 LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
647 ADD src, src, 4*NBYTES
648#ifdef CONFIG_CPU_SB1
649 nop # improves slotting
650#endif
651 STORE(t0, UNIT(0)(dst), .Ls_exc\@)
652 ADDC(t0, t1)
653 STORE(t1, UNIT(1)(dst), .Ls_exc\@)
654 ADDC(sum, t0)
655 STORE(t2, UNIT(2)(dst), .Ls_exc\@)
656 ADDC(t2, t3)
657 STORE(t3, UNIT(3)(dst), .Ls_exc\@)
658 ADDC(sum, t2)
659 .set reorder /* DADDI_WAR */
660 ADD dst, dst, 4*NBYTES
661 bne len, rem, 1b
662 .set noreorder
663
664.Lcleanup_src_unaligned\@:
665 beqz len, .Ldone\@
666 and rem, len, NBYTES-1 # rem = len % NBYTES
667 beq rem, len, .Lcopy_bytes\@
668 nop
6691:
670 LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
671 LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
672 ADD src, src, NBYTES
673 SUB len, len, NBYTES
674 STORE(t0, 0(dst), .Ls_exc\@)
675 ADDC(sum, t0)
676 .set reorder /* DADDI_WAR */
677 ADD dst, dst, NBYTES
678 bne len, rem, 1b
679 .set noreorder
680
681.Lcopy_bytes_checklen\@:
682 beqz len, .Ldone\@
683 nop
684.Lcopy_bytes\@:
685 /* 0 < len < NBYTES */
686#ifdef CONFIG_CPU_LITTLE_ENDIAN
687#define SHIFT_START 0
688#define SHIFT_INC 8
689#else
690#define SHIFT_START 8*(NBYTES-1)
691#define SHIFT_INC -8
692#endif
693 move t2, zero # partial word
694 li t3, SHIFT_START # shift
695/* use .Ll_exc_copy here to return correct sum on fault */
696#define COPY_BYTE(N) \
697 LOADBU(t0, N(src), .Ll_exc_copy\@); \
698 SUB len, len, 1; \
699 STOREB(t0, N(dst), .Ls_exc\@); \
700 SLLV t0, t0, t3; \
701 addu t3, SHIFT_INC; \
702 beqz len, .Lcopy_bytes_done\@; \
703 or t2, t0
704
705 COPY_BYTE(0)
706 COPY_BYTE(1)
707#ifdef USE_DOUBLE
708 COPY_BYTE(2)
709 COPY_BYTE(3)
710 COPY_BYTE(4)
711 COPY_BYTE(5)
712#endif
713 LOADBU(t0, NBYTES-2(src), .Ll_exc_copy\@)
714 SUB len, len, 1
715 STOREB(t0, NBYTES-2(dst), .Ls_exc\@)
716 SLLV t0, t0, t3
717 or t2, t0
718.Lcopy_bytes_done\@:
719 ADDC(sum, t2)
720.Ldone\@:
721 /* fold checksum */
722 .set push
723 .set noat
724#ifdef USE_DOUBLE
725 dsll32 v1, sum, 0
726 daddu sum, v1
727 sltu v1, sum, v1
728 dsra32 sum, sum, 0
729 addu sum, v1
730#endif
731
732#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_LOONGSON3)
733 .set push
734 .set arch=mips32r2
735 wsbh v1, sum
736 movn sum, v1, odd
737 .set pop
738#else
739 beqz odd, 1f /* odd buffer alignment? */
740 lui v1, 0x00ff
741 addu v1, 0x00ff
742 and t0, sum, v1
743 sll t0, t0, 8
744 srl sum, sum, 8
745 and sum, sum, v1
746 or sum, sum, t0
7471:
748#endif
749 .set pop
750 .set reorder
751 ADDC32(sum, psum)
752 jr ra
753 .set noreorder
754
755.Ll_exc_copy\@:
756 /*
757 * Copy bytes from src until faulting load address (or until a
758 * lb faults)
759 *
760 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
761 * may be more than a byte beyond the last address.
762 * Hence, the lb below may get an exception.
763 *
764 * Assumes src < THREAD_BUADDR($28)
765 */
766 LOADK t0, TI_TASK($28)
767 li t2, SHIFT_START
768 LOADK t0, THREAD_BUADDR(t0)
7691:
770 LOADBU(t1, 0(src), .Ll_exc\@)
771 ADD src, src, 1
772 sb t1, 0(dst) # can't fault -- we're copy_from_user
773 SLLV t1, t1, t2
774 addu t2, SHIFT_INC
775 ADDC(sum, t1)
776 .set reorder /* DADDI_WAR */
777 ADD dst, dst, 1
778 bne src, t0, 1b
779 .set noreorder
780.Ll_exc\@:
781 LOADK t0, TI_TASK($28)
782 nop
783 LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
784 nop
785 SUB len, AT, t0 # len number of uncopied bytes
786 /*
787 * Here's where we rely on src and dst being incremented in tandem,
788 * See (3) above.
789 * dst += (fault addr - src) to put dst at first byte to clear
790 */
791 ADD dst, t0 # compute start address in a1
792 SUB dst, src
793 /*
794 * Clear len bytes starting at dst. Can't call __bzero because it
795 * might modify len. An inefficient loop for these rare times...
796 */
797 .set reorder /* DADDI_WAR */
798 SUB src, len, 1
799 beqz len, .Ldone\@
800 .set noreorder
8011: sb zero, 0(dst)
802 ADD dst, dst, 1
803 .set push
804 .set noat
805#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
806 bnez src, 1b
807 SUB src, src, 1
808#else
809 li v1, 1
810 bnez src, 1b
811 SUB src, src, v1
812#endif
813 li v1, -EFAULT
814 b .Ldone\@
815 sw v1, (errptr)
816
817.Ls_exc\@:
818 li v0, -1 /* invalid checksum */
819 li v1, -EFAULT
820 jr ra
821 sw v1, (errptr)
822 .set pop
823 .endm
824
825LEAF(__csum_partial_copy_kernel)
826#ifndef CONFIG_EVA
827FEXPORT(__csum_partial_copy_to_user)
828FEXPORT(__csum_partial_copy_from_user)
829#endif
830__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP 1
831END(__csum_partial_copy_kernel)
832
833#ifdef CONFIG_EVA
834LEAF(__csum_partial_copy_to_user)
835__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP 0
836END(__csum_partial_copy_to_user)
837
838LEAF(__csum_partial_copy_from_user)
839__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP 0
840END(__csum_partial_copy_from_user)
841#endif