Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * include/asm-generic/xor.h
4 *
5 * Generic optimized RAID-5 checksumming functions.
6 */
7
8#include <linux/prefetch.h>
9
10static void
11xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1,
12 const unsigned long * __restrict p2)
13{
14 long lines = bytes / (sizeof (long)) / 8;
15
16 do {
17 p1[0] ^= p2[0];
18 p1[1] ^= p2[1];
19 p1[2] ^= p2[2];
20 p1[3] ^= p2[3];
21 p1[4] ^= p2[4];
22 p1[5] ^= p2[5];
23 p1[6] ^= p2[6];
24 p1[7] ^= p2[7];
25 p1 += 8;
26 p2 += 8;
27 } while (--lines > 0);
28}
29
30static void
31xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1,
32 const unsigned long * __restrict p2,
33 const unsigned long * __restrict p3)
34{
35 long lines = bytes / (sizeof (long)) / 8;
36
37 do {
38 p1[0] ^= p2[0] ^ p3[0];
39 p1[1] ^= p2[1] ^ p3[1];
40 p1[2] ^= p2[2] ^ p3[2];
41 p1[3] ^= p2[3] ^ p3[3];
42 p1[4] ^= p2[4] ^ p3[4];
43 p1[5] ^= p2[5] ^ p3[5];
44 p1[6] ^= p2[6] ^ p3[6];
45 p1[7] ^= p2[7] ^ p3[7];
46 p1 += 8;
47 p2 += 8;
48 p3 += 8;
49 } while (--lines > 0);
50}
51
52static void
53xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1,
54 const unsigned long * __restrict p2,
55 const unsigned long * __restrict p3,
56 const unsigned long * __restrict p4)
57{
58 long lines = bytes / (sizeof (long)) / 8;
59
60 do {
61 p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
62 p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
63 p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
64 p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
65 p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
66 p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
67 p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
68 p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
69 p1 += 8;
70 p2 += 8;
71 p3 += 8;
72 p4 += 8;
73 } while (--lines > 0);
74}
75
76static void
77xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1,
78 const unsigned long * __restrict p2,
79 const unsigned long * __restrict p3,
80 const unsigned long * __restrict p4,
81 const unsigned long * __restrict p5)
82{
83 long lines = bytes / (sizeof (long)) / 8;
84
85 do {
86 p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
87 p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
88 p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
89 p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
90 p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
91 p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
92 p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
93 p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
94 p1 += 8;
95 p2 += 8;
96 p3 += 8;
97 p4 += 8;
98 p5 += 8;
99 } while (--lines > 0);
100}
101
102static void
103xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1,
104 const unsigned long * __restrict p2)
105{
106 long lines = bytes / (sizeof (long)) / 8;
107
108 do {
109 register long d0, d1, d2, d3, d4, d5, d6, d7;
110 d0 = p1[0]; /* Pull the stuff into registers */
111 d1 = p1[1]; /* ... in bursts, if possible. */
112 d2 = p1[2];
113 d3 = p1[3];
114 d4 = p1[4];
115 d5 = p1[5];
116 d6 = p1[6];
117 d7 = p1[7];
118 d0 ^= p2[0];
119 d1 ^= p2[1];
120 d2 ^= p2[2];
121 d3 ^= p2[3];
122 d4 ^= p2[4];
123 d5 ^= p2[5];
124 d6 ^= p2[6];
125 d7 ^= p2[7];
126 p1[0] = d0; /* Store the result (in bursts) */
127 p1[1] = d1;
128 p1[2] = d2;
129 p1[3] = d3;
130 p1[4] = d4;
131 p1[5] = d5;
132 p1[6] = d6;
133 p1[7] = d7;
134 p1 += 8;
135 p2 += 8;
136 } while (--lines > 0);
137}
138
139static void
140xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1,
141 const unsigned long * __restrict p2,
142 const unsigned long * __restrict p3)
143{
144 long lines = bytes / (sizeof (long)) / 8;
145
146 do {
147 register long d0, d1, d2, d3, d4, d5, d6, d7;
148 d0 = p1[0]; /* Pull the stuff into registers */
149 d1 = p1[1]; /* ... in bursts, if possible. */
150 d2 = p1[2];
151 d3 = p1[3];
152 d4 = p1[4];
153 d5 = p1[5];
154 d6 = p1[6];
155 d7 = p1[7];
156 d0 ^= p2[0];
157 d1 ^= p2[1];
158 d2 ^= p2[2];
159 d3 ^= p2[3];
160 d4 ^= p2[4];
161 d5 ^= p2[5];
162 d6 ^= p2[6];
163 d7 ^= p2[7];
164 d0 ^= p3[0];
165 d1 ^= p3[1];
166 d2 ^= p3[2];
167 d3 ^= p3[3];
168 d4 ^= p3[4];
169 d5 ^= p3[5];
170 d6 ^= p3[6];
171 d7 ^= p3[7];
172 p1[0] = d0; /* Store the result (in bursts) */
173 p1[1] = d1;
174 p1[2] = d2;
175 p1[3] = d3;
176 p1[4] = d4;
177 p1[5] = d5;
178 p1[6] = d6;
179 p1[7] = d7;
180 p1 += 8;
181 p2 += 8;
182 p3 += 8;
183 } while (--lines > 0);
184}
185
186static void
187xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1,
188 const unsigned long * __restrict p2,
189 const unsigned long * __restrict p3,
190 const unsigned long * __restrict p4)
191{
192 long lines = bytes / (sizeof (long)) / 8;
193
194 do {
195 register long d0, d1, d2, d3, d4, d5, d6, d7;
196 d0 = p1[0]; /* Pull the stuff into registers */
197 d1 = p1[1]; /* ... in bursts, if possible. */
198 d2 = p1[2];
199 d3 = p1[3];
200 d4 = p1[4];
201 d5 = p1[5];
202 d6 = p1[6];
203 d7 = p1[7];
204 d0 ^= p2[0];
205 d1 ^= p2[1];
206 d2 ^= p2[2];
207 d3 ^= p2[3];
208 d4 ^= p2[4];
209 d5 ^= p2[5];
210 d6 ^= p2[6];
211 d7 ^= p2[7];
212 d0 ^= p3[0];
213 d1 ^= p3[1];
214 d2 ^= p3[2];
215 d3 ^= p3[3];
216 d4 ^= p3[4];
217 d5 ^= p3[5];
218 d6 ^= p3[6];
219 d7 ^= p3[7];
220 d0 ^= p4[0];
221 d1 ^= p4[1];
222 d2 ^= p4[2];
223 d3 ^= p4[3];
224 d4 ^= p4[4];
225 d5 ^= p4[5];
226 d6 ^= p4[6];
227 d7 ^= p4[7];
228 p1[0] = d0; /* Store the result (in bursts) */
229 p1[1] = d1;
230 p1[2] = d2;
231 p1[3] = d3;
232 p1[4] = d4;
233 p1[5] = d5;
234 p1[6] = d6;
235 p1[7] = d7;
236 p1 += 8;
237 p2 += 8;
238 p3 += 8;
239 p4 += 8;
240 } while (--lines > 0);
241}
242
243static void
244xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1,
245 const unsigned long * __restrict p2,
246 const unsigned long * __restrict p3,
247 const unsigned long * __restrict p4,
248 const unsigned long * __restrict p5)
249{
250 long lines = bytes / (sizeof (long)) / 8;
251
252 do {
253 register long d0, d1, d2, d3, d4, d5, d6, d7;
254 d0 = p1[0]; /* Pull the stuff into registers */
255 d1 = p1[1]; /* ... in bursts, if possible. */
256 d2 = p1[2];
257 d3 = p1[3];
258 d4 = p1[4];
259 d5 = p1[5];
260 d6 = p1[6];
261 d7 = p1[7];
262 d0 ^= p2[0];
263 d1 ^= p2[1];
264 d2 ^= p2[2];
265 d3 ^= p2[3];
266 d4 ^= p2[4];
267 d5 ^= p2[5];
268 d6 ^= p2[6];
269 d7 ^= p2[7];
270 d0 ^= p3[0];
271 d1 ^= p3[1];
272 d2 ^= p3[2];
273 d3 ^= p3[3];
274 d4 ^= p3[4];
275 d5 ^= p3[5];
276 d6 ^= p3[6];
277 d7 ^= p3[7];
278 d0 ^= p4[0];
279 d1 ^= p4[1];
280 d2 ^= p4[2];
281 d3 ^= p4[3];
282 d4 ^= p4[4];
283 d5 ^= p4[5];
284 d6 ^= p4[6];
285 d7 ^= p4[7];
286 d0 ^= p5[0];
287 d1 ^= p5[1];
288 d2 ^= p5[2];
289 d3 ^= p5[3];
290 d4 ^= p5[4];
291 d5 ^= p5[5];
292 d6 ^= p5[6];
293 d7 ^= p5[7];
294 p1[0] = d0; /* Store the result (in bursts) */
295 p1[1] = d1;
296 p1[2] = d2;
297 p1[3] = d3;
298 p1[4] = d4;
299 p1[5] = d5;
300 p1[6] = d6;
301 p1[7] = d7;
302 p1 += 8;
303 p2 += 8;
304 p3 += 8;
305 p4 += 8;
306 p5 += 8;
307 } while (--lines > 0);
308}
309
310static void
311xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
312 const unsigned long * __restrict p2)
313{
314 long lines = bytes / (sizeof (long)) / 8 - 1;
315 prefetchw(p1);
316 prefetch(p2);
317
318 do {
319 prefetchw(p1+8);
320 prefetch(p2+8);
321 once_more:
322 p1[0] ^= p2[0];
323 p1[1] ^= p2[1];
324 p1[2] ^= p2[2];
325 p1[3] ^= p2[3];
326 p1[4] ^= p2[4];
327 p1[5] ^= p2[5];
328 p1[6] ^= p2[6];
329 p1[7] ^= p2[7];
330 p1 += 8;
331 p2 += 8;
332 } while (--lines > 0);
333 if (lines == 0)
334 goto once_more;
335}
336
337static void
338xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
339 const unsigned long * __restrict p2,
340 const unsigned long * __restrict p3)
341{
342 long lines = bytes / (sizeof (long)) / 8 - 1;
343 prefetchw(p1);
344 prefetch(p2);
345 prefetch(p3);
346
347 do {
348 prefetchw(p1+8);
349 prefetch(p2+8);
350 prefetch(p3+8);
351 once_more:
352 p1[0] ^= p2[0] ^ p3[0];
353 p1[1] ^= p2[1] ^ p3[1];
354 p1[2] ^= p2[2] ^ p3[2];
355 p1[3] ^= p2[3] ^ p3[3];
356 p1[4] ^= p2[4] ^ p3[4];
357 p1[5] ^= p2[5] ^ p3[5];
358 p1[6] ^= p2[6] ^ p3[6];
359 p1[7] ^= p2[7] ^ p3[7];
360 p1 += 8;
361 p2 += 8;
362 p3 += 8;
363 } while (--lines > 0);
364 if (lines == 0)
365 goto once_more;
366}
367
368static void
369xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
370 const unsigned long * __restrict p2,
371 const unsigned long * __restrict p3,
372 const unsigned long * __restrict p4)
373{
374 long lines = bytes / (sizeof (long)) / 8 - 1;
375
376 prefetchw(p1);
377 prefetch(p2);
378 prefetch(p3);
379 prefetch(p4);
380
381 do {
382 prefetchw(p1+8);
383 prefetch(p2+8);
384 prefetch(p3+8);
385 prefetch(p4+8);
386 once_more:
387 p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
388 p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
389 p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
390 p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
391 p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
392 p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
393 p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
394 p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
395 p1 += 8;
396 p2 += 8;
397 p3 += 8;
398 p4 += 8;
399 } while (--lines > 0);
400 if (lines == 0)
401 goto once_more;
402}
403
404static void
405xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
406 const unsigned long * __restrict p2,
407 const unsigned long * __restrict p3,
408 const unsigned long * __restrict p4,
409 const unsigned long * __restrict p5)
410{
411 long lines = bytes / (sizeof (long)) / 8 - 1;
412
413 prefetchw(p1);
414 prefetch(p2);
415 prefetch(p3);
416 prefetch(p4);
417 prefetch(p5);
418
419 do {
420 prefetchw(p1+8);
421 prefetch(p2+8);
422 prefetch(p3+8);
423 prefetch(p4+8);
424 prefetch(p5+8);
425 once_more:
426 p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
427 p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
428 p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
429 p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
430 p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
431 p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
432 p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
433 p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
434 p1 += 8;
435 p2 += 8;
436 p3 += 8;
437 p4 += 8;
438 p5 += 8;
439 } while (--lines > 0);
440 if (lines == 0)
441 goto once_more;
442}
443
444static void
445xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
446 const unsigned long * __restrict p2)
447{
448 long lines = bytes / (sizeof (long)) / 8 - 1;
449
450 prefetchw(p1);
451 prefetch(p2);
452
453 do {
454 register long d0, d1, d2, d3, d4, d5, d6, d7;
455
456 prefetchw(p1+8);
457 prefetch(p2+8);
458 once_more:
459 d0 = p1[0]; /* Pull the stuff into registers */
460 d1 = p1[1]; /* ... in bursts, if possible. */
461 d2 = p1[2];
462 d3 = p1[3];
463 d4 = p1[4];
464 d5 = p1[5];
465 d6 = p1[6];
466 d7 = p1[7];
467 d0 ^= p2[0];
468 d1 ^= p2[1];
469 d2 ^= p2[2];
470 d3 ^= p2[3];
471 d4 ^= p2[4];
472 d5 ^= p2[5];
473 d6 ^= p2[6];
474 d7 ^= p2[7];
475 p1[0] = d0; /* Store the result (in bursts) */
476 p1[1] = d1;
477 p1[2] = d2;
478 p1[3] = d3;
479 p1[4] = d4;
480 p1[5] = d5;
481 p1[6] = d6;
482 p1[7] = d7;
483 p1 += 8;
484 p2 += 8;
485 } while (--lines > 0);
486 if (lines == 0)
487 goto once_more;
488}
489
490static void
491xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
492 const unsigned long * __restrict p2,
493 const unsigned long * __restrict p3)
494{
495 long lines = bytes / (sizeof (long)) / 8 - 1;
496
497 prefetchw(p1);
498 prefetch(p2);
499 prefetch(p3);
500
501 do {
502 register long d0, d1, d2, d3, d4, d5, d6, d7;
503
504 prefetchw(p1+8);
505 prefetch(p2+8);
506 prefetch(p3+8);
507 once_more:
508 d0 = p1[0]; /* Pull the stuff into registers */
509 d1 = p1[1]; /* ... in bursts, if possible. */
510 d2 = p1[2];
511 d3 = p1[3];
512 d4 = p1[4];
513 d5 = p1[5];
514 d6 = p1[6];
515 d7 = p1[7];
516 d0 ^= p2[0];
517 d1 ^= p2[1];
518 d2 ^= p2[2];
519 d3 ^= p2[3];
520 d4 ^= p2[4];
521 d5 ^= p2[5];
522 d6 ^= p2[6];
523 d7 ^= p2[7];
524 d0 ^= p3[0];
525 d1 ^= p3[1];
526 d2 ^= p3[2];
527 d3 ^= p3[3];
528 d4 ^= p3[4];
529 d5 ^= p3[5];
530 d6 ^= p3[6];
531 d7 ^= p3[7];
532 p1[0] = d0; /* Store the result (in bursts) */
533 p1[1] = d1;
534 p1[2] = d2;
535 p1[3] = d3;
536 p1[4] = d4;
537 p1[5] = d5;
538 p1[6] = d6;
539 p1[7] = d7;
540 p1 += 8;
541 p2 += 8;
542 p3 += 8;
543 } while (--lines > 0);
544 if (lines == 0)
545 goto once_more;
546}
547
548static void
549xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
550 const unsigned long * __restrict p2,
551 const unsigned long * __restrict p3,
552 const unsigned long * __restrict p4)
553{
554 long lines = bytes / (sizeof (long)) / 8 - 1;
555
556 prefetchw(p1);
557 prefetch(p2);
558 prefetch(p3);
559 prefetch(p4);
560
561 do {
562 register long d0, d1, d2, d3, d4, d5, d6, d7;
563
564 prefetchw(p1+8);
565 prefetch(p2+8);
566 prefetch(p3+8);
567 prefetch(p4+8);
568 once_more:
569 d0 = p1[0]; /* Pull the stuff into registers */
570 d1 = p1[1]; /* ... in bursts, if possible. */
571 d2 = p1[2];
572 d3 = p1[3];
573 d4 = p1[4];
574 d5 = p1[5];
575 d6 = p1[6];
576 d7 = p1[7];
577 d0 ^= p2[0];
578 d1 ^= p2[1];
579 d2 ^= p2[2];
580 d3 ^= p2[3];
581 d4 ^= p2[4];
582 d5 ^= p2[5];
583 d6 ^= p2[6];
584 d7 ^= p2[7];
585 d0 ^= p3[0];
586 d1 ^= p3[1];
587 d2 ^= p3[2];
588 d3 ^= p3[3];
589 d4 ^= p3[4];
590 d5 ^= p3[5];
591 d6 ^= p3[6];
592 d7 ^= p3[7];
593 d0 ^= p4[0];
594 d1 ^= p4[1];
595 d2 ^= p4[2];
596 d3 ^= p4[3];
597 d4 ^= p4[4];
598 d5 ^= p4[5];
599 d6 ^= p4[6];
600 d7 ^= p4[7];
601 p1[0] = d0; /* Store the result (in bursts) */
602 p1[1] = d1;
603 p1[2] = d2;
604 p1[3] = d3;
605 p1[4] = d4;
606 p1[5] = d5;
607 p1[6] = d6;
608 p1[7] = d7;
609 p1 += 8;
610 p2 += 8;
611 p3 += 8;
612 p4 += 8;
613 } while (--lines > 0);
614 if (lines == 0)
615 goto once_more;
616}
617
618static void
619xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
620 const unsigned long * __restrict p2,
621 const unsigned long * __restrict p3,
622 const unsigned long * __restrict p4,
623 const unsigned long * __restrict p5)
624{
625 long lines = bytes / (sizeof (long)) / 8 - 1;
626
627 prefetchw(p1);
628 prefetch(p2);
629 prefetch(p3);
630 prefetch(p4);
631 prefetch(p5);
632
633 do {
634 register long d0, d1, d2, d3, d4, d5, d6, d7;
635
636 prefetchw(p1+8);
637 prefetch(p2+8);
638 prefetch(p3+8);
639 prefetch(p4+8);
640 prefetch(p5+8);
641 once_more:
642 d0 = p1[0]; /* Pull the stuff into registers */
643 d1 = p1[1]; /* ... in bursts, if possible. */
644 d2 = p1[2];
645 d3 = p1[3];
646 d4 = p1[4];
647 d5 = p1[5];
648 d6 = p1[6];
649 d7 = p1[7];
650 d0 ^= p2[0];
651 d1 ^= p2[1];
652 d2 ^= p2[2];
653 d3 ^= p2[3];
654 d4 ^= p2[4];
655 d5 ^= p2[5];
656 d6 ^= p2[6];
657 d7 ^= p2[7];
658 d0 ^= p3[0];
659 d1 ^= p3[1];
660 d2 ^= p3[2];
661 d3 ^= p3[3];
662 d4 ^= p3[4];
663 d5 ^= p3[5];
664 d6 ^= p3[6];
665 d7 ^= p3[7];
666 d0 ^= p4[0];
667 d1 ^= p4[1];
668 d2 ^= p4[2];
669 d3 ^= p4[3];
670 d4 ^= p4[4];
671 d5 ^= p4[5];
672 d6 ^= p4[6];
673 d7 ^= p4[7];
674 d0 ^= p5[0];
675 d1 ^= p5[1];
676 d2 ^= p5[2];
677 d3 ^= p5[3];
678 d4 ^= p5[4];
679 d5 ^= p5[5];
680 d6 ^= p5[6];
681 d7 ^= p5[7];
682 p1[0] = d0; /* Store the result (in bursts) */
683 p1[1] = d1;
684 p1[2] = d2;
685 p1[3] = d3;
686 p1[4] = d4;
687 p1[5] = d5;
688 p1[6] = d6;
689 p1[7] = d7;
690 p1 += 8;
691 p2 += 8;
692 p3 += 8;
693 p4 += 8;
694 p5 += 8;
695 } while (--lines > 0);
696 if (lines == 0)
697 goto once_more;
698}
699
700static struct xor_block_template xor_block_8regs = {
701 .name = "8regs",
702 .do_2 = xor_8regs_2,
703 .do_3 = xor_8regs_3,
704 .do_4 = xor_8regs_4,
705 .do_5 = xor_8regs_5,
706};
707
708static struct xor_block_template xor_block_32regs = {
709 .name = "32regs",
710 .do_2 = xor_32regs_2,
711 .do_3 = xor_32regs_3,
712 .do_4 = xor_32regs_4,
713 .do_5 = xor_32regs_5,
714};
715
716static struct xor_block_template xor_block_8regs_p __maybe_unused = {
717 .name = "8regs_prefetch",
718 .do_2 = xor_8regs_p_2,
719 .do_3 = xor_8regs_p_3,
720 .do_4 = xor_8regs_p_4,
721 .do_5 = xor_8regs_p_5,
722};
723
724static struct xor_block_template xor_block_32regs_p __maybe_unused = {
725 .name = "32regs_prefetch",
726 .do_2 = xor_32regs_p_2,
727 .do_3 = xor_32regs_p_3,
728 .do_4 = xor_32regs_p_4,
729 .do_5 = xor_32regs_p_5,
730};
731
732#define XOR_TRY_TEMPLATES \
733 do { \
734 xor_speed(&xor_block_8regs); \
735 xor_speed(&xor_block_8regs_p); \
736 xor_speed(&xor_block_32regs); \
737 xor_speed(&xor_block_32regs_p); \
738 } while (0)
1/*
2 * include/asm-generic/xor.h
3 *
4 * Generic optimized RAID-5 checksumming functions.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2, or (at your option)
9 * any later version.
10 *
11 * You should have received a copy of the GNU General Public License
12 * (for example /usr/src/linux/COPYING); if not, write to the Free
13 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
14 */
15
16#include <linux/prefetch.h>
17
18static void
19xor_8regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
20{
21 long lines = bytes / (sizeof (long)) / 8;
22
23 do {
24 p1[0] ^= p2[0];
25 p1[1] ^= p2[1];
26 p1[2] ^= p2[2];
27 p1[3] ^= p2[3];
28 p1[4] ^= p2[4];
29 p1[5] ^= p2[5];
30 p1[6] ^= p2[6];
31 p1[7] ^= p2[7];
32 p1 += 8;
33 p2 += 8;
34 } while (--lines > 0);
35}
36
37static void
38xor_8regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
39 unsigned long *p3)
40{
41 long lines = bytes / (sizeof (long)) / 8;
42
43 do {
44 p1[0] ^= p2[0] ^ p3[0];
45 p1[1] ^= p2[1] ^ p3[1];
46 p1[2] ^= p2[2] ^ p3[2];
47 p1[3] ^= p2[3] ^ p3[3];
48 p1[4] ^= p2[4] ^ p3[4];
49 p1[5] ^= p2[5] ^ p3[5];
50 p1[6] ^= p2[6] ^ p3[6];
51 p1[7] ^= p2[7] ^ p3[7];
52 p1 += 8;
53 p2 += 8;
54 p3 += 8;
55 } while (--lines > 0);
56}
57
58static void
59xor_8regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
60 unsigned long *p3, unsigned long *p4)
61{
62 long lines = bytes / (sizeof (long)) / 8;
63
64 do {
65 p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
66 p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
67 p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
68 p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
69 p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
70 p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
71 p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
72 p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
73 p1 += 8;
74 p2 += 8;
75 p3 += 8;
76 p4 += 8;
77 } while (--lines > 0);
78}
79
80static void
81xor_8regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
82 unsigned long *p3, unsigned long *p4, unsigned long *p5)
83{
84 long lines = bytes / (sizeof (long)) / 8;
85
86 do {
87 p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
88 p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
89 p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
90 p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
91 p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
92 p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
93 p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
94 p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
95 p1 += 8;
96 p2 += 8;
97 p3 += 8;
98 p4 += 8;
99 p5 += 8;
100 } while (--lines > 0);
101}
102
103static void
104xor_32regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
105{
106 long lines = bytes / (sizeof (long)) / 8;
107
108 do {
109 register long d0, d1, d2, d3, d4, d5, d6, d7;
110 d0 = p1[0]; /* Pull the stuff into registers */
111 d1 = p1[1]; /* ... in bursts, if possible. */
112 d2 = p1[2];
113 d3 = p1[3];
114 d4 = p1[4];
115 d5 = p1[5];
116 d6 = p1[6];
117 d7 = p1[7];
118 d0 ^= p2[0];
119 d1 ^= p2[1];
120 d2 ^= p2[2];
121 d3 ^= p2[3];
122 d4 ^= p2[4];
123 d5 ^= p2[5];
124 d6 ^= p2[6];
125 d7 ^= p2[7];
126 p1[0] = d0; /* Store the result (in bursts) */
127 p1[1] = d1;
128 p1[2] = d2;
129 p1[3] = d3;
130 p1[4] = d4;
131 p1[5] = d5;
132 p1[6] = d6;
133 p1[7] = d7;
134 p1 += 8;
135 p2 += 8;
136 } while (--lines > 0);
137}
138
139static void
140xor_32regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
141 unsigned long *p3)
142{
143 long lines = bytes / (sizeof (long)) / 8;
144
145 do {
146 register long d0, d1, d2, d3, d4, d5, d6, d7;
147 d0 = p1[0]; /* Pull the stuff into registers */
148 d1 = p1[1]; /* ... in bursts, if possible. */
149 d2 = p1[2];
150 d3 = p1[3];
151 d4 = p1[4];
152 d5 = p1[5];
153 d6 = p1[6];
154 d7 = p1[7];
155 d0 ^= p2[0];
156 d1 ^= p2[1];
157 d2 ^= p2[2];
158 d3 ^= p2[3];
159 d4 ^= p2[4];
160 d5 ^= p2[5];
161 d6 ^= p2[6];
162 d7 ^= p2[7];
163 d0 ^= p3[0];
164 d1 ^= p3[1];
165 d2 ^= p3[2];
166 d3 ^= p3[3];
167 d4 ^= p3[4];
168 d5 ^= p3[5];
169 d6 ^= p3[6];
170 d7 ^= p3[7];
171 p1[0] = d0; /* Store the result (in bursts) */
172 p1[1] = d1;
173 p1[2] = d2;
174 p1[3] = d3;
175 p1[4] = d4;
176 p1[5] = d5;
177 p1[6] = d6;
178 p1[7] = d7;
179 p1 += 8;
180 p2 += 8;
181 p3 += 8;
182 } while (--lines > 0);
183}
184
185static void
186xor_32regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
187 unsigned long *p3, unsigned long *p4)
188{
189 long lines = bytes / (sizeof (long)) / 8;
190
191 do {
192 register long d0, d1, d2, d3, d4, d5, d6, d7;
193 d0 = p1[0]; /* Pull the stuff into registers */
194 d1 = p1[1]; /* ... in bursts, if possible. */
195 d2 = p1[2];
196 d3 = p1[3];
197 d4 = p1[4];
198 d5 = p1[5];
199 d6 = p1[6];
200 d7 = p1[7];
201 d0 ^= p2[0];
202 d1 ^= p2[1];
203 d2 ^= p2[2];
204 d3 ^= p2[3];
205 d4 ^= p2[4];
206 d5 ^= p2[5];
207 d6 ^= p2[6];
208 d7 ^= p2[7];
209 d0 ^= p3[0];
210 d1 ^= p3[1];
211 d2 ^= p3[2];
212 d3 ^= p3[3];
213 d4 ^= p3[4];
214 d5 ^= p3[5];
215 d6 ^= p3[6];
216 d7 ^= p3[7];
217 d0 ^= p4[0];
218 d1 ^= p4[1];
219 d2 ^= p4[2];
220 d3 ^= p4[3];
221 d4 ^= p4[4];
222 d5 ^= p4[5];
223 d6 ^= p4[6];
224 d7 ^= p4[7];
225 p1[0] = d0; /* Store the result (in bursts) */
226 p1[1] = d1;
227 p1[2] = d2;
228 p1[3] = d3;
229 p1[4] = d4;
230 p1[5] = d5;
231 p1[6] = d6;
232 p1[7] = d7;
233 p1 += 8;
234 p2 += 8;
235 p3 += 8;
236 p4 += 8;
237 } while (--lines > 0);
238}
239
240static void
241xor_32regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
242 unsigned long *p3, unsigned long *p4, unsigned long *p5)
243{
244 long lines = bytes / (sizeof (long)) / 8;
245
246 do {
247 register long d0, d1, d2, d3, d4, d5, d6, d7;
248 d0 = p1[0]; /* Pull the stuff into registers */
249 d1 = p1[1]; /* ... in bursts, if possible. */
250 d2 = p1[2];
251 d3 = p1[3];
252 d4 = p1[4];
253 d5 = p1[5];
254 d6 = p1[6];
255 d7 = p1[7];
256 d0 ^= p2[0];
257 d1 ^= p2[1];
258 d2 ^= p2[2];
259 d3 ^= p2[3];
260 d4 ^= p2[4];
261 d5 ^= p2[5];
262 d6 ^= p2[6];
263 d7 ^= p2[7];
264 d0 ^= p3[0];
265 d1 ^= p3[1];
266 d2 ^= p3[2];
267 d3 ^= p3[3];
268 d4 ^= p3[4];
269 d5 ^= p3[5];
270 d6 ^= p3[6];
271 d7 ^= p3[7];
272 d0 ^= p4[0];
273 d1 ^= p4[1];
274 d2 ^= p4[2];
275 d3 ^= p4[3];
276 d4 ^= p4[4];
277 d5 ^= p4[5];
278 d6 ^= p4[6];
279 d7 ^= p4[7];
280 d0 ^= p5[0];
281 d1 ^= p5[1];
282 d2 ^= p5[2];
283 d3 ^= p5[3];
284 d4 ^= p5[4];
285 d5 ^= p5[5];
286 d6 ^= p5[6];
287 d7 ^= p5[7];
288 p1[0] = d0; /* Store the result (in bursts) */
289 p1[1] = d1;
290 p1[2] = d2;
291 p1[3] = d3;
292 p1[4] = d4;
293 p1[5] = d5;
294 p1[6] = d6;
295 p1[7] = d7;
296 p1 += 8;
297 p2 += 8;
298 p3 += 8;
299 p4 += 8;
300 p5 += 8;
301 } while (--lines > 0);
302}
303
304static void
305xor_8regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
306{
307 long lines = bytes / (sizeof (long)) / 8 - 1;
308 prefetchw(p1);
309 prefetch(p2);
310
311 do {
312 prefetchw(p1+8);
313 prefetch(p2+8);
314 once_more:
315 p1[0] ^= p2[0];
316 p1[1] ^= p2[1];
317 p1[2] ^= p2[2];
318 p1[3] ^= p2[3];
319 p1[4] ^= p2[4];
320 p1[5] ^= p2[5];
321 p1[6] ^= p2[6];
322 p1[7] ^= p2[7];
323 p1 += 8;
324 p2 += 8;
325 } while (--lines > 0);
326 if (lines == 0)
327 goto once_more;
328}
329
330static void
331xor_8regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
332 unsigned long *p3)
333{
334 long lines = bytes / (sizeof (long)) / 8 - 1;
335 prefetchw(p1);
336 prefetch(p2);
337 prefetch(p3);
338
339 do {
340 prefetchw(p1+8);
341 prefetch(p2+8);
342 prefetch(p3+8);
343 once_more:
344 p1[0] ^= p2[0] ^ p3[0];
345 p1[1] ^= p2[1] ^ p3[1];
346 p1[2] ^= p2[2] ^ p3[2];
347 p1[3] ^= p2[3] ^ p3[3];
348 p1[4] ^= p2[4] ^ p3[4];
349 p1[5] ^= p2[5] ^ p3[5];
350 p1[6] ^= p2[6] ^ p3[6];
351 p1[7] ^= p2[7] ^ p3[7];
352 p1 += 8;
353 p2 += 8;
354 p3 += 8;
355 } while (--lines > 0);
356 if (lines == 0)
357 goto once_more;
358}
359
360static void
361xor_8regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
362 unsigned long *p3, unsigned long *p4)
363{
364 long lines = bytes / (sizeof (long)) / 8 - 1;
365
366 prefetchw(p1);
367 prefetch(p2);
368 prefetch(p3);
369 prefetch(p4);
370
371 do {
372 prefetchw(p1+8);
373 prefetch(p2+8);
374 prefetch(p3+8);
375 prefetch(p4+8);
376 once_more:
377 p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
378 p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
379 p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
380 p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
381 p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
382 p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
383 p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
384 p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
385 p1 += 8;
386 p2 += 8;
387 p3 += 8;
388 p4 += 8;
389 } while (--lines > 0);
390 if (lines == 0)
391 goto once_more;
392}
393
394static void
395xor_8regs_p_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
396 unsigned long *p3, unsigned long *p4, unsigned long *p5)
397{
398 long lines = bytes / (sizeof (long)) / 8 - 1;
399
400 prefetchw(p1);
401 prefetch(p2);
402 prefetch(p3);
403 prefetch(p4);
404 prefetch(p5);
405
406 do {
407 prefetchw(p1+8);
408 prefetch(p2+8);
409 prefetch(p3+8);
410 prefetch(p4+8);
411 prefetch(p5+8);
412 once_more:
413 p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
414 p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
415 p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
416 p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
417 p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
418 p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
419 p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
420 p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
421 p1 += 8;
422 p2 += 8;
423 p3 += 8;
424 p4 += 8;
425 p5 += 8;
426 } while (--lines > 0);
427 if (lines == 0)
428 goto once_more;
429}
430
431static void
432xor_32regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
433{
434 long lines = bytes / (sizeof (long)) / 8 - 1;
435
436 prefetchw(p1);
437 prefetch(p2);
438
439 do {
440 register long d0, d1, d2, d3, d4, d5, d6, d7;
441
442 prefetchw(p1+8);
443 prefetch(p2+8);
444 once_more:
445 d0 = p1[0]; /* Pull the stuff into registers */
446 d1 = p1[1]; /* ... in bursts, if possible. */
447 d2 = p1[2];
448 d3 = p1[3];
449 d4 = p1[4];
450 d5 = p1[5];
451 d6 = p1[6];
452 d7 = p1[7];
453 d0 ^= p2[0];
454 d1 ^= p2[1];
455 d2 ^= p2[2];
456 d3 ^= p2[3];
457 d4 ^= p2[4];
458 d5 ^= p2[5];
459 d6 ^= p2[6];
460 d7 ^= p2[7];
461 p1[0] = d0; /* Store the result (in bursts) */
462 p1[1] = d1;
463 p1[2] = d2;
464 p1[3] = d3;
465 p1[4] = d4;
466 p1[5] = d5;
467 p1[6] = d6;
468 p1[7] = d7;
469 p1 += 8;
470 p2 += 8;
471 } while (--lines > 0);
472 if (lines == 0)
473 goto once_more;
474}
475
476static void
477xor_32regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
478 unsigned long *p3)
479{
480 long lines = bytes / (sizeof (long)) / 8 - 1;
481
482 prefetchw(p1);
483 prefetch(p2);
484 prefetch(p3);
485
486 do {
487 register long d0, d1, d2, d3, d4, d5, d6, d7;
488
489 prefetchw(p1+8);
490 prefetch(p2+8);
491 prefetch(p3+8);
492 once_more:
493 d0 = p1[0]; /* Pull the stuff into registers */
494 d1 = p1[1]; /* ... in bursts, if possible. */
495 d2 = p1[2];
496 d3 = p1[3];
497 d4 = p1[4];
498 d5 = p1[5];
499 d6 = p1[6];
500 d7 = p1[7];
501 d0 ^= p2[0];
502 d1 ^= p2[1];
503 d2 ^= p2[2];
504 d3 ^= p2[3];
505 d4 ^= p2[4];
506 d5 ^= p2[5];
507 d6 ^= p2[6];
508 d7 ^= p2[7];
509 d0 ^= p3[0];
510 d1 ^= p3[1];
511 d2 ^= p3[2];
512 d3 ^= p3[3];
513 d4 ^= p3[4];
514 d5 ^= p3[5];
515 d6 ^= p3[6];
516 d7 ^= p3[7];
517 p1[0] = d0; /* Store the result (in bursts) */
518 p1[1] = d1;
519 p1[2] = d2;
520 p1[3] = d3;
521 p1[4] = d4;
522 p1[5] = d5;
523 p1[6] = d6;
524 p1[7] = d7;
525 p1 += 8;
526 p2 += 8;
527 p3 += 8;
528 } while (--lines > 0);
529 if (lines == 0)
530 goto once_more;
531}
532
533static void
534xor_32regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
535 unsigned long *p3, unsigned long *p4)
536{
537 long lines = bytes / (sizeof (long)) / 8 - 1;
538
539 prefetchw(p1);
540 prefetch(p2);
541 prefetch(p3);
542 prefetch(p4);
543
544 do {
545 register long d0, d1, d2, d3, d4, d5, d6, d7;
546
547 prefetchw(p1+8);
548 prefetch(p2+8);
549 prefetch(p3+8);
550 prefetch(p4+8);
551 once_more:
552 d0 = p1[0]; /* Pull the stuff into registers */
553 d1 = p1[1]; /* ... in bursts, if possible. */
554 d2 = p1[2];
555 d3 = p1[3];
556 d4 = p1[4];
557 d5 = p1[5];
558 d6 = p1[6];
559 d7 = p1[7];
560 d0 ^= p2[0];
561 d1 ^= p2[1];
562 d2 ^= p2[2];
563 d3 ^= p2[3];
564 d4 ^= p2[4];
565 d5 ^= p2[5];
566 d6 ^= p2[6];
567 d7 ^= p2[7];
568 d0 ^= p3[0];
569 d1 ^= p3[1];
570 d2 ^= p3[2];
571 d3 ^= p3[3];
572 d4 ^= p3[4];
573 d5 ^= p3[5];
574 d6 ^= p3[6];
575 d7 ^= p3[7];
576 d0 ^= p4[0];
577 d1 ^= p4[1];
578 d2 ^= p4[2];
579 d3 ^= p4[3];
580 d4 ^= p4[4];
581 d5 ^= p4[5];
582 d6 ^= p4[6];
583 d7 ^= p4[7];
584 p1[0] = d0; /* Store the result (in bursts) */
585 p1[1] = d1;
586 p1[2] = d2;
587 p1[3] = d3;
588 p1[4] = d4;
589 p1[5] = d5;
590 p1[6] = d6;
591 p1[7] = d7;
592 p1 += 8;
593 p2 += 8;
594 p3 += 8;
595 p4 += 8;
596 } while (--lines > 0);
597 if (lines == 0)
598 goto once_more;
599}
600
601static void
602xor_32regs_p_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
603 unsigned long *p3, unsigned long *p4, unsigned long *p5)
604{
605 long lines = bytes / (sizeof (long)) / 8 - 1;
606
607 prefetchw(p1);
608 prefetch(p2);
609 prefetch(p3);
610 prefetch(p4);
611 prefetch(p5);
612
613 do {
614 register long d0, d1, d2, d3, d4, d5, d6, d7;
615
616 prefetchw(p1+8);
617 prefetch(p2+8);
618 prefetch(p3+8);
619 prefetch(p4+8);
620 prefetch(p5+8);
621 once_more:
622 d0 = p1[0]; /* Pull the stuff into registers */
623 d1 = p1[1]; /* ... in bursts, if possible. */
624 d2 = p1[2];
625 d3 = p1[3];
626 d4 = p1[4];
627 d5 = p1[5];
628 d6 = p1[6];
629 d7 = p1[7];
630 d0 ^= p2[0];
631 d1 ^= p2[1];
632 d2 ^= p2[2];
633 d3 ^= p2[3];
634 d4 ^= p2[4];
635 d5 ^= p2[5];
636 d6 ^= p2[6];
637 d7 ^= p2[7];
638 d0 ^= p3[0];
639 d1 ^= p3[1];
640 d2 ^= p3[2];
641 d3 ^= p3[3];
642 d4 ^= p3[4];
643 d5 ^= p3[5];
644 d6 ^= p3[6];
645 d7 ^= p3[7];
646 d0 ^= p4[0];
647 d1 ^= p4[1];
648 d2 ^= p4[2];
649 d3 ^= p4[3];
650 d4 ^= p4[4];
651 d5 ^= p4[5];
652 d6 ^= p4[6];
653 d7 ^= p4[7];
654 d0 ^= p5[0];
655 d1 ^= p5[1];
656 d2 ^= p5[2];
657 d3 ^= p5[3];
658 d4 ^= p5[4];
659 d5 ^= p5[5];
660 d6 ^= p5[6];
661 d7 ^= p5[7];
662 p1[0] = d0; /* Store the result (in bursts) */
663 p1[1] = d1;
664 p1[2] = d2;
665 p1[3] = d3;
666 p1[4] = d4;
667 p1[5] = d5;
668 p1[6] = d6;
669 p1[7] = d7;
670 p1 += 8;
671 p2 += 8;
672 p3 += 8;
673 p4 += 8;
674 p5 += 8;
675 } while (--lines > 0);
676 if (lines == 0)
677 goto once_more;
678}
679
680static struct xor_block_template xor_block_8regs = {
681 .name = "8regs",
682 .do_2 = xor_8regs_2,
683 .do_3 = xor_8regs_3,
684 .do_4 = xor_8regs_4,
685 .do_5 = xor_8regs_5,
686};
687
688static struct xor_block_template xor_block_32regs = {
689 .name = "32regs",
690 .do_2 = xor_32regs_2,
691 .do_3 = xor_32regs_3,
692 .do_4 = xor_32regs_4,
693 .do_5 = xor_32regs_5,
694};
695
696static struct xor_block_template xor_block_8regs_p = {
697 .name = "8regs_prefetch",
698 .do_2 = xor_8regs_p_2,
699 .do_3 = xor_8regs_p_3,
700 .do_4 = xor_8regs_p_4,
701 .do_5 = xor_8regs_p_5,
702};
703
704static struct xor_block_template xor_block_32regs_p = {
705 .name = "32regs_prefetch",
706 .do_2 = xor_32regs_p_2,
707 .do_3 = xor_32regs_p_3,
708 .do_4 = xor_32regs_p_4,
709 .do_5 = xor_32regs_p_5,
710};
711
712#define XOR_TRY_TEMPLATES \
713 do { \
714 xor_speed(&xor_block_8regs); \
715 xor_speed(&xor_block_8regs_p); \
716 xor_speed(&xor_block_32regs); \
717 xor_speed(&xor_block_32regs_p); \
718 } while (0)