1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2/*
  3 * include/asm-generic/xor.h
  4 *
  5 * Generic optimized RAID-5 checksumming functions.
  6 */
  7
  8#include <linux/prefetch.h>
  9
 10static void
 11xor_8regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
 12{
 13	long lines = bytes / (sizeof (long)) / 8;
 14
 15	do {
 16		p1[0] ^= p2[0];
 17		p1[1] ^= p2[1];
 18		p1[2] ^= p2[2];
 19		p1[3] ^= p2[3];
 20		p1[4] ^= p2[4];
 21		p1[5] ^= p2[5];
 22		p1[6] ^= p2[6];
 23		p1[7] ^= p2[7];
 24		p1 += 8;
 25		p2 += 8;
 26	} while (--lines > 0);
 27}
 28
 29static void
 30xor_8regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
 31	    unsigned long *p3)
 32{
 33	long lines = bytes / (sizeof (long)) / 8;
 34
 35	do {
 36		p1[0] ^= p2[0] ^ p3[0];
 37		p1[1] ^= p2[1] ^ p3[1];
 38		p1[2] ^= p2[2] ^ p3[2];
 39		p1[3] ^= p2[3] ^ p3[3];
 40		p1[4] ^= p2[4] ^ p3[4];
 41		p1[5] ^= p2[5] ^ p3[5];
 42		p1[6] ^= p2[6] ^ p3[6];
 43		p1[7] ^= p2[7] ^ p3[7];
 44		p1 += 8;
 45		p2 += 8;
 46		p3 += 8;
 47	} while (--lines > 0);
 48}
 49
 50static void
 51xor_8regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
 52	    unsigned long *p3, unsigned long *p4)
 53{
 54	long lines = bytes / (sizeof (long)) / 8;
 55
 56	do {
 57		p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
 58		p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
 59		p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
 60		p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
 61		p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
 62		p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
 63		p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
 64		p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
 65		p1 += 8;
 66		p2 += 8;
 67		p3 += 8;
 68		p4 += 8;
 69	} while (--lines > 0);
 70}
 71
 72static void
 73xor_8regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
 74	    unsigned long *p3, unsigned long *p4, unsigned long *p5)
 75{
 76	long lines = bytes / (sizeof (long)) / 8;
 77
 78	do {
 79		p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
 80		p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
 81		p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
 82		p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
 83		p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
 84		p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
 85		p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
 86		p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
 87		p1 += 8;
 88		p2 += 8;
 89		p3 += 8;
 90		p4 += 8;
 91		p5 += 8;
 92	} while (--lines > 0);
 93}
 94
 95static void
 96xor_32regs_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
 97{
 98	long lines = bytes / (sizeof (long)) / 8;
 99
100	do {
101		register long d0, d1, d2, d3, d4, d5, d6, d7;
102		d0 = p1[0];	/* Pull the stuff into registers	*/
103		d1 = p1[1];	/*  ... in bursts, if possible.		*/
104		d2 = p1[2];
105		d3 = p1[3];
106		d4 = p1[4];
107		d5 = p1[5];
108		d6 = p1[6];
109		d7 = p1[7];
110		d0 ^= p2[0];
111		d1 ^= p2[1];
112		d2 ^= p2[2];
113		d3 ^= p2[3];
114		d4 ^= p2[4];
115		d5 ^= p2[5];
116		d6 ^= p2[6];
117		d7 ^= p2[7];
118		p1[0] = d0;	/* Store the result (in bursts)		*/
119		p1[1] = d1;
120		p1[2] = d2;
121		p1[3] = d3;
122		p1[4] = d4;
123		p1[5] = d5;
124		p1[6] = d6;
125		p1[7] = d7;
126		p1 += 8;
127		p2 += 8;
128	} while (--lines > 0);
129}
130
131static void
132xor_32regs_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
133	    unsigned long *p3)
134{
135	long lines = bytes / (sizeof (long)) / 8;
136
137	do {
138		register long d0, d1, d2, d3, d4, d5, d6, d7;
139		d0 = p1[0];	/* Pull the stuff into registers	*/
140		d1 = p1[1];	/*  ... in bursts, if possible.		*/
141		d2 = p1[2];
142		d3 = p1[3];
143		d4 = p1[4];
144		d5 = p1[5];
145		d6 = p1[6];
146		d7 = p1[7];
147		d0 ^= p2[0];
148		d1 ^= p2[1];
149		d2 ^= p2[2];
150		d3 ^= p2[3];
151		d4 ^= p2[4];
152		d5 ^= p2[5];
153		d6 ^= p2[6];
154		d7 ^= p2[7];
155		d0 ^= p3[0];
156		d1 ^= p3[1];
157		d2 ^= p3[2];
158		d3 ^= p3[3];
159		d4 ^= p3[4];
160		d5 ^= p3[5];
161		d6 ^= p3[6];
162		d7 ^= p3[7];
163		p1[0] = d0;	/* Store the result (in bursts)		*/
164		p1[1] = d1;
165		p1[2] = d2;
166		p1[3] = d3;
167		p1[4] = d4;
168		p1[5] = d5;
169		p1[6] = d6;
170		p1[7] = d7;
171		p1 += 8;
172		p2 += 8;
173		p3 += 8;
174	} while (--lines > 0);
175}
176
177static void
178xor_32regs_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
179	    unsigned long *p3, unsigned long *p4)
180{
181	long lines = bytes / (sizeof (long)) / 8;
182
183	do {
184		register long d0, d1, d2, d3, d4, d5, d6, d7;
185		d0 = p1[0];	/* Pull the stuff into registers	*/
186		d1 = p1[1];	/*  ... in bursts, if possible.		*/
187		d2 = p1[2];
188		d3 = p1[3];
189		d4 = p1[4];
190		d5 = p1[5];
191		d6 = p1[6];
192		d7 = p1[7];
193		d0 ^= p2[0];
194		d1 ^= p2[1];
195		d2 ^= p2[2];
196		d3 ^= p2[3];
197		d4 ^= p2[4];
198		d5 ^= p2[5];
199		d6 ^= p2[6];
200		d7 ^= p2[7];
201		d0 ^= p3[0];
202		d1 ^= p3[1];
203		d2 ^= p3[2];
204		d3 ^= p3[3];
205		d4 ^= p3[4];
206		d5 ^= p3[5];
207		d6 ^= p3[6];
208		d7 ^= p3[7];
209		d0 ^= p4[0];
210		d1 ^= p4[1];
211		d2 ^= p4[2];
212		d3 ^= p4[3];
213		d4 ^= p4[4];
214		d5 ^= p4[5];
215		d6 ^= p4[6];
216		d7 ^= p4[7];
217		p1[0] = d0;	/* Store the result (in bursts)		*/
218		p1[1] = d1;
219		p1[2] = d2;
220		p1[3] = d3;
221		p1[4] = d4;
222		p1[5] = d5;
223		p1[6] = d6;
224		p1[7] = d7;
225		p1 += 8;
226		p2 += 8;
227		p3 += 8;
228		p4 += 8;
229	} while (--lines > 0);
230}
231
232static void
233xor_32regs_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
234	    unsigned long *p3, unsigned long *p4, unsigned long *p5)
235{
236	long lines = bytes / (sizeof (long)) / 8;
237
238	do {
239		register long d0, d1, d2, d3, d4, d5, d6, d7;
240		d0 = p1[0];	/* Pull the stuff into registers	*/
241		d1 = p1[1];	/*  ... in bursts, if possible.		*/
242		d2 = p1[2];
243		d3 = p1[3];
244		d4 = p1[4];
245		d5 = p1[5];
246		d6 = p1[6];
247		d7 = p1[7];
248		d0 ^= p2[0];
249		d1 ^= p2[1];
250		d2 ^= p2[2];
251		d3 ^= p2[3];
252		d4 ^= p2[4];
253		d5 ^= p2[5];
254		d6 ^= p2[6];
255		d7 ^= p2[7];
256		d0 ^= p3[0];
257		d1 ^= p3[1];
258		d2 ^= p3[2];
259		d3 ^= p3[3];
260		d4 ^= p3[4];
261		d5 ^= p3[5];
262		d6 ^= p3[6];
263		d7 ^= p3[7];
264		d0 ^= p4[0];
265		d1 ^= p4[1];
266		d2 ^= p4[2];
267		d3 ^= p4[3];
268		d4 ^= p4[4];
269		d5 ^= p4[5];
270		d6 ^= p4[6];
271		d7 ^= p4[7];
272		d0 ^= p5[0];
273		d1 ^= p5[1];
274		d2 ^= p5[2];
275		d3 ^= p5[3];
276		d4 ^= p5[4];
277		d5 ^= p5[5];
278		d6 ^= p5[6];
279		d7 ^= p5[7];
280		p1[0] = d0;	/* Store the result (in bursts)		*/
281		p1[1] = d1;
282		p1[2] = d2;
283		p1[3] = d3;
284		p1[4] = d4;
285		p1[5] = d5;
286		p1[6] = d6;
287		p1[7] = d7;
288		p1 += 8;
289		p2 += 8;
290		p3 += 8;
291		p4 += 8;
292		p5 += 8;
293	} while (--lines > 0);
294}
295
296static void
297xor_8regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
298{
299	long lines = bytes / (sizeof (long)) / 8 - 1;
300	prefetchw(p1);
301	prefetch(p2);
302
303	do {
304		prefetchw(p1+8);
305		prefetch(p2+8);
306 once_more:
307		p1[0] ^= p2[0];
308		p1[1] ^= p2[1];
309		p1[2] ^= p2[2];
310		p1[3] ^= p2[3];
311		p1[4] ^= p2[4];
312		p1[5] ^= p2[5];
313		p1[6] ^= p2[6];
314		p1[7] ^= p2[7];
315		p1 += 8;
316		p2 += 8;
317	} while (--lines > 0);
318	if (lines == 0)
319		goto once_more;
320}
321
322static void
323xor_8regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
324	    unsigned long *p3)
325{
326	long lines = bytes / (sizeof (long)) / 8 - 1;
327	prefetchw(p1);
328	prefetch(p2);
329	prefetch(p3);
330
331	do {
332		prefetchw(p1+8);
333		prefetch(p2+8);
334		prefetch(p3+8);
335 once_more:
336		p1[0] ^= p2[0] ^ p3[0];
337		p1[1] ^= p2[1] ^ p3[1];
338		p1[2] ^= p2[2] ^ p3[2];
339		p1[3] ^= p2[3] ^ p3[3];
340		p1[4] ^= p2[4] ^ p3[4];
341		p1[5] ^= p2[5] ^ p3[5];
342		p1[6] ^= p2[6] ^ p3[6];
343		p1[7] ^= p2[7] ^ p3[7];
344		p1 += 8;
345		p2 += 8;
346		p3 += 8;
347	} while (--lines > 0);
348	if (lines == 0)
349		goto once_more;
350}
351
352static void
353xor_8regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
354	    unsigned long *p3, unsigned long *p4)
355{
356	long lines = bytes / (sizeof (long)) / 8 - 1;
357
358	prefetchw(p1);
359	prefetch(p2);
360	prefetch(p3);
361	prefetch(p4);
362
363	do {
364		prefetchw(p1+8);
365		prefetch(p2+8);
366		prefetch(p3+8);
367		prefetch(p4+8);
368 once_more:
369		p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
370		p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
371		p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
372		p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
373		p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
374		p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
375		p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
376		p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
377		p1 += 8;
378		p2 += 8;
379		p3 += 8;
380		p4 += 8;
381	} while (--lines > 0);
382	if (lines == 0)
383		goto once_more;
384}
385
386static void
387xor_8regs_p_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
388	    unsigned long *p3, unsigned long *p4, unsigned long *p5)
389{
390	long lines = bytes / (sizeof (long)) / 8 - 1;
391
392	prefetchw(p1);
393	prefetch(p2);
394	prefetch(p3);
395	prefetch(p4);
396	prefetch(p5);
397
398	do {
399		prefetchw(p1+8);
400		prefetch(p2+8);
401		prefetch(p3+8);
402		prefetch(p4+8);
403		prefetch(p5+8);
404 once_more:
405		p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
406		p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
407		p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
408		p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
409		p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
410		p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
411		p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
412		p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
413		p1 += 8;
414		p2 += 8;
415		p3 += 8;
416		p4 += 8;
417		p5 += 8;
418	} while (--lines > 0);
419	if (lines == 0)
420		goto once_more;
421}
422
423static void
424xor_32regs_p_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
425{
426	long lines = bytes / (sizeof (long)) / 8 - 1;
427
428	prefetchw(p1);
429	prefetch(p2);
430
431	do {
432		register long d0, d1, d2, d3, d4, d5, d6, d7;
433
434		prefetchw(p1+8);
435		prefetch(p2+8);
436 once_more:
437		d0 = p1[0];	/* Pull the stuff into registers	*/
438		d1 = p1[1];	/*  ... in bursts, if possible.		*/
439		d2 = p1[2];
440		d3 = p1[3];
441		d4 = p1[4];
442		d5 = p1[5];
443		d6 = p1[6];
444		d7 = p1[7];
445		d0 ^= p2[0];
446		d1 ^= p2[1];
447		d2 ^= p2[2];
448		d3 ^= p2[3];
449		d4 ^= p2[4];
450		d5 ^= p2[5];
451		d6 ^= p2[6];
452		d7 ^= p2[7];
453		p1[0] = d0;	/* Store the result (in bursts)		*/
454		p1[1] = d1;
455		p1[2] = d2;
456		p1[3] = d3;
457		p1[4] = d4;
458		p1[5] = d5;
459		p1[6] = d6;
460		p1[7] = d7;
461		p1 += 8;
462		p2 += 8;
463	} while (--lines > 0);
464	if (lines == 0)
465		goto once_more;
466}
467
468static void
469xor_32regs_p_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
470	    unsigned long *p3)
471{
472	long lines = bytes / (sizeof (long)) / 8 - 1;
473
474	prefetchw(p1);
475	prefetch(p2);
476	prefetch(p3);
477
478	do {
479		register long d0, d1, d2, d3, d4, d5, d6, d7;
480
481		prefetchw(p1+8);
482		prefetch(p2+8);
483		prefetch(p3+8);
484 once_more:
485		d0 = p1[0];	/* Pull the stuff into registers	*/
486		d1 = p1[1];	/*  ... in bursts, if possible.		*/
487		d2 = p1[2];
488		d3 = p1[3];
489		d4 = p1[4];
490		d5 = p1[5];
491		d6 = p1[6];
492		d7 = p1[7];
493		d0 ^= p2[0];
494		d1 ^= p2[1];
495		d2 ^= p2[2];
496		d3 ^= p2[3];
497		d4 ^= p2[4];
498		d5 ^= p2[5];
499		d6 ^= p2[6];
500		d7 ^= p2[7];
501		d0 ^= p3[0];
502		d1 ^= p3[1];
503		d2 ^= p3[2];
504		d3 ^= p3[3];
505		d4 ^= p3[4];
506		d5 ^= p3[5];
507		d6 ^= p3[6];
508		d7 ^= p3[7];
509		p1[0] = d0;	/* Store the result (in bursts)		*/
510		p1[1] = d1;
511		p1[2] = d2;
512		p1[3] = d3;
513		p1[4] = d4;
514		p1[5] = d5;
515		p1[6] = d6;
516		p1[7] = d7;
517		p1 += 8;
518		p2 += 8;
519		p3 += 8;
520	} while (--lines > 0);
521	if (lines == 0)
522		goto once_more;
523}
524
525static void
526xor_32regs_p_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
527	    unsigned long *p3, unsigned long *p4)
528{
529	long lines = bytes / (sizeof (long)) / 8 - 1;
530
531	prefetchw(p1);
532	prefetch(p2);
533	prefetch(p3);
534	prefetch(p4);
535
536	do {
537		register long d0, d1, d2, d3, d4, d5, d6, d7;
538
539		prefetchw(p1+8);
540		prefetch(p2+8);
541		prefetch(p3+8);
542		prefetch(p4+8);
543 once_more:
544		d0 = p1[0];	/* Pull the stuff into registers	*/
545		d1 = p1[1];	/*  ... in bursts, if possible.		*/
546		d2 = p1[2];
547		d3 = p1[3];
548		d4 = p1[4];
549		d5 = p1[5];
550		d6 = p1[6];
551		d7 = p1[7];
552		d0 ^= p2[0];
553		d1 ^= p2[1];
554		d2 ^= p2[2];
555		d3 ^= p2[3];
556		d4 ^= p2[4];
557		d5 ^= p2[5];
558		d6 ^= p2[6];
559		d7 ^= p2[7];
560		d0 ^= p3[0];
561		d1 ^= p3[1];
562		d2 ^= p3[2];
563		d3 ^= p3[3];
564		d4 ^= p3[4];
565		d5 ^= p3[5];
566		d6 ^= p3[6];
567		d7 ^= p3[7];
568		d0 ^= p4[0];
569		d1 ^= p4[1];
570		d2 ^= p4[2];
571		d3 ^= p4[3];
572		d4 ^= p4[4];
573		d5 ^= p4[5];
574		d6 ^= p4[6];
575		d7 ^= p4[7];
576		p1[0] = d0;	/* Store the result (in bursts)		*/
577		p1[1] = d1;
578		p1[2] = d2;
579		p1[3] = d3;
580		p1[4] = d4;
581		p1[5] = d5;
582		p1[6] = d6;
583		p1[7] = d7;
584		p1 += 8;
585		p2 += 8;
586		p3 += 8;
587		p4 += 8;
588	} while (--lines > 0);
589	if (lines == 0)
590		goto once_more;
591}
592
593static void
594xor_32regs_p_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
595	    unsigned long *p3, unsigned long *p4, unsigned long *p5)
596{
597	long lines = bytes / (sizeof (long)) / 8 - 1;
598
599	prefetchw(p1);
600	prefetch(p2);
601	prefetch(p3);
602	prefetch(p4);
603	prefetch(p5);
604
605	do {
606		register long d0, d1, d2, d3, d4, d5, d6, d7;
607
608		prefetchw(p1+8);
609		prefetch(p2+8);
610		prefetch(p3+8);
611		prefetch(p4+8);
612		prefetch(p5+8);
613 once_more:
614		d0 = p1[0];	/* Pull the stuff into registers	*/
615		d1 = p1[1];	/*  ... in bursts, if possible.		*/
616		d2 = p1[2];
617		d3 = p1[3];
618		d4 = p1[4];
619		d5 = p1[5];
620		d6 = p1[6];
621		d7 = p1[7];
622		d0 ^= p2[0];
623		d1 ^= p2[1];
624		d2 ^= p2[2];
625		d3 ^= p2[3];
626		d4 ^= p2[4];
627		d5 ^= p2[5];
628		d6 ^= p2[6];
629		d7 ^= p2[7];
630		d0 ^= p3[0];
631		d1 ^= p3[1];
632		d2 ^= p3[2];
633		d3 ^= p3[3];
634		d4 ^= p3[4];
635		d5 ^= p3[5];
636		d6 ^= p3[6];
637		d7 ^= p3[7];
638		d0 ^= p4[0];
639		d1 ^= p4[1];
640		d2 ^= p4[2];
641		d3 ^= p4[3];
642		d4 ^= p4[4];
643		d5 ^= p4[5];
644		d6 ^= p4[6];
645		d7 ^= p4[7];
646		d0 ^= p5[0];
647		d1 ^= p5[1];
648		d2 ^= p5[2];
649		d3 ^= p5[3];
650		d4 ^= p5[4];
651		d5 ^= p5[5];
652		d6 ^= p5[6];
653		d7 ^= p5[7];
654		p1[0] = d0;	/* Store the result (in bursts)		*/
655		p1[1] = d1;
656		p1[2] = d2;
657		p1[3] = d3;
658		p1[4] = d4;
659		p1[5] = d5;
660		p1[6] = d6;
661		p1[7] = d7;
662		p1 += 8;
663		p2 += 8;
664		p3 += 8;
665		p4 += 8;
666		p5 += 8;
667	} while (--lines > 0);
668	if (lines == 0)
669		goto once_more;
670}
671
672static struct xor_block_template xor_block_8regs = {
673	.name = "8regs",
674	.do_2 = xor_8regs_2,
675	.do_3 = xor_8regs_3,
676	.do_4 = xor_8regs_4,
677	.do_5 = xor_8regs_5,
678};
679
680static struct xor_block_template xor_block_32regs = {
681	.name = "32regs",
682	.do_2 = xor_32regs_2,
683	.do_3 = xor_32regs_3,
684	.do_4 = xor_32regs_4,
685	.do_5 = xor_32regs_5,
686};
687
688static struct xor_block_template xor_block_8regs_p __maybe_unused = {
689	.name = "8regs_prefetch",
690	.do_2 = xor_8regs_p_2,
691	.do_3 = xor_8regs_p_3,
692	.do_4 = xor_8regs_p_4,
693	.do_5 = xor_8regs_p_5,
694};
695
696static struct xor_block_template xor_block_32regs_p __maybe_unused = {
697	.name = "32regs_prefetch",
698	.do_2 = xor_32regs_p_2,
699	.do_3 = xor_32regs_p_3,
700	.do_4 = xor_32regs_p_4,
701	.do_5 = xor_32regs_p_5,
702};
703
704#define XOR_TRY_TEMPLATES			\
705	do {					\
706		xor_speed(&xor_block_8regs);	\
707		xor_speed(&xor_block_8regs_p);	\
708		xor_speed(&xor_block_32regs);	\
709		xor_speed(&xor_block_32regs_p);	\
710	} while (0)