1/* -*- linux-c -*- ------------------------------------------------------- *
  2 *
  3 *   Copyright 2002 H. Peter Anvin - All Rights Reserved
  4 *
  5 *   This program is free software; you can redistribute it and/or modify
  6 *   it under the terms of the GNU General Public License as published by
  7 *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
  8 *   Boston MA 02111-1307, USA; either version 2 of the License, or
  9 *   (at your option) any later version; incorporated herein by reference.
 10 *
 11 * ----------------------------------------------------------------------- */
 12
 13/*
 14 * raid6/mmx.c
 15 *
 16 * MMX implementation of RAID-6 syndrome functions
 17 */
 18
 19#ifdef CONFIG_X86_32
 20
 21#include <linux/raid/pq.h>
 22#include "x86.h"
 23
 24/* Shared with raid6/sse1.c */
 25const struct raid6_mmx_constants {
 26	u64 x1d;
 27} raid6_mmx_constants = {
 28	0x1d1d1d1d1d1d1d1dULL,
 29};
 30
 31static int raid6_have_mmx(void)
 32{
 33	/* Not really "boot_cpu" but "all_cpus" */
 34	return boot_cpu_has(X86_FEATURE_MMX);
 35}
 36
 37/*
 38 * Plain MMX implementation
 39 */
 40static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
 41{
 42	u8 **dptr = (u8 **)ptrs;
 43	u8 *p, *q;
 44	int d, z, z0;
 45
 46	z0 = disks - 3;		/* Highest data disk */
 47	p = dptr[z0+1];		/* XOR parity */
 48	q = dptr[z0+2];		/* RS syndrome */
 49
 50	kernel_fpu_begin();
 51
 52	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
 53	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
 54
 55	for ( d = 0 ; d < bytes ; d += 8 ) {
 56		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
 57		asm volatile("movq %mm2,%mm4");	/* Q[0] */
 58		for ( z = z0-1 ; z >= 0 ; z-- ) {
 59			asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
 60			asm volatile("pcmpgtb %mm4,%mm5");
 61			asm volatile("paddb %mm4,%mm4");
 62			asm volatile("pand %mm0,%mm5");
 63			asm volatile("pxor %mm5,%mm4");
 64			asm volatile("pxor %mm5,%mm5");
 65			asm volatile("pxor %mm6,%mm2");
 66			asm volatile("pxor %mm6,%mm4");
 67		}
 68		asm volatile("movq %%mm2,%0" : "=m" (p[d]));
 69		asm volatile("pxor %mm2,%mm2");
 70		asm volatile("movq %%mm4,%0" : "=m" (q[d]));
 71		asm volatile("pxor %mm4,%mm4");
 72	}
 73
 74	kernel_fpu_end();
 75}
 76
 77const struct raid6_calls raid6_mmxx1 = {
 78	raid6_mmx1_gen_syndrome,
 79	raid6_have_mmx,
 80	"mmxx1",
 81	0
 82};
 83
 84/*
 85 * Unrolled-by-2 MMX implementation
 86 */
 87static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
 88{
 89	u8 **dptr = (u8 **)ptrs;
 90	u8 *p, *q;
 91	int d, z, z0;
 92
 93	z0 = disks - 3;		/* Highest data disk */
 94	p = dptr[z0+1];		/* XOR parity */
 95	q = dptr[z0+2];		/* RS syndrome */
 96
 97	kernel_fpu_begin();
 98
 99	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
100	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
101	asm volatile("pxor %mm7,%mm7"); /* Zero temp */
102
103	for ( d = 0 ; d < bytes ; d += 16 ) {
104		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
105		asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
106		asm volatile("movq %mm2,%mm4"); /* Q[0] */
107		asm volatile("movq %mm3,%mm6"); /* Q[1] */
108		for ( z = z0-1 ; z >= 0 ; z-- ) {
109			asm volatile("pcmpgtb %mm4,%mm5");
110			asm volatile("pcmpgtb %mm6,%mm7");
111			asm volatile("paddb %mm4,%mm4");
112			asm volatile("paddb %mm6,%mm6");
113			asm volatile("pand %mm0,%mm5");
114			asm volatile("pand %mm0,%mm7");
115			asm volatile("pxor %mm5,%mm4");
116			asm volatile("pxor %mm7,%mm6");
117			asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
118			asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
119			asm volatile("pxor %mm5,%mm2");
120			asm volatile("pxor %mm7,%mm3");
121			asm volatile("pxor %mm5,%mm4");
122			asm volatile("pxor %mm7,%mm6");
123			asm volatile("pxor %mm5,%mm5");
124			asm volatile("pxor %mm7,%mm7");
125		}
126		asm volatile("movq %%mm2,%0" : "=m" (p[d]));
127		asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
128		asm volatile("movq %%mm4,%0" : "=m" (q[d]));
129		asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
130	}
131
132	kernel_fpu_end();
133}
134
135const struct raid6_calls raid6_mmxx2 = {
136	raid6_mmx2_gen_syndrome,
137	raid6_have_mmx,
138	"mmxx2",
139	0
140};
141
142#endif