1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * raid6_vx$#.c
  4 *
  5 * $#-way unrolled RAID6 gen/xor functions for s390
  6 * based on the vector facility
  7 *
  8 * Copyright IBM Corp. 2016
  9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
 10 *
 11 * This file is postprocessed using unroll.awk.
 12 */
 13
 14#include <linux/raid/pq.h>
 15#include <asm/fpu/api.h>
 16#include <asm/vx-insn.h>
 17
 18#define NSIZE 16
 19
 20static inline void LOAD_CONST(void)
 21{
 22	asm volatile("VREPIB %v24,7");
 23	asm volatile("VREPIB %v25,0x1d");
 24}
 25
 26/*
 27 * The SHLBYTE() operation shifts each of the 16 bytes in
 28 * vector register y left by 1 bit and stores the result in
 29 * vector register x.
 30 */
 31static inline void SHLBYTE(int x, int y)
 32{
 33	asm volatile ("VAB %0,%1,%1" : : "i" (x), "i" (y));
 34}
 35
 36/*
 37 * For each of the 16 bytes in the vector register y the MASK()
 38 * operation returns 0xFF if the high bit of the byte is 1,
 39 * or 0x00 if the high bit is 0. The result is stored in vector
 40 * register x.
 41 */
 42static inline void MASK(int x, int y)
 43{
 44	asm volatile ("VESRAVB	%0,%1,24" : : "i" (x), "i" (y));
 45}
 46
 47static inline void AND(int x, int y, int z)
 48{
 49	asm volatile ("VN %0,%1,%2" : : "i" (x), "i" (y), "i" (z));
 50}
 51
 52static inline void XOR(int x, int y, int z)
 53{
 54	asm volatile ("VX %0,%1,%2" : : "i" (x), "i" (y), "i" (z));
 55}
 56
 57static inline void LOAD_DATA(int x, u8 *ptr)
 58{
 59	typedef struct { u8 _[16 * $#]; } addrtype;
 60	register addrtype *__ptr asm("1") = (addrtype *) ptr;
 61
 62	asm volatile ("VLM %2,%3,0,%1"
 63		      : : "m" (*__ptr), "a" (__ptr), "i" (x),
 64			  "i" (x + $# - 1));
 65}
 66
 67static inline void STORE_DATA(int x, u8 *ptr)
 68{
 69	typedef struct { u8 _[16 * $#]; } addrtype;
 70	register addrtype *__ptr asm("1") = (addrtype *) ptr;
 71
 72	asm volatile ("VSTM %2,%3,0,1"
 73		      : "=m" (*__ptr) : "a" (__ptr), "i" (x),
 74			"i" (x + $# - 1));
 75}
 76
 77static inline void COPY_VEC(int x, int y)
 78{
 79	asm volatile ("VLR %0,%1" : : "i" (x), "i" (y));
 80}
 81
 82static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
 83{
 84	struct kernel_fpu vxstate;
 85	u8 **dptr, *p, *q;
 86	int d, z, z0;
 87
 88	kernel_fpu_begin(&vxstate, KERNEL_VXR);
 89	LOAD_CONST();
 90
 91	dptr = (u8 **) ptrs;
 92	z0 = disks - 3;		/* Highest data disk */
 93	p = dptr[z0 + 1];	/* XOR parity */
 94	q = dptr[z0 + 2];	/* RS syndrome */
 95
 96	for (d = 0; d < bytes; d += $#*NSIZE) {
 97		LOAD_DATA(0,&dptr[z0][d]);
 98		COPY_VEC(8+$$,0+$$);
 99		for (z = z0 - 1; z >= 0; z--) {
100			MASK(16+$$,8+$$);
101			AND(16+$$,16+$$,25);
102			SHLBYTE(8+$$,8+$$);
103			XOR(8+$$,8+$$,16+$$);
104			LOAD_DATA(16,&dptr[z][d]);
105			XOR(0+$$,0+$$,16+$$);
106			XOR(8+$$,8+$$,16+$$);
107		}
108		STORE_DATA(0,&p[d]);
109		STORE_DATA(8,&q[d]);
110	}
111	kernel_fpu_end(&vxstate, KERNEL_VXR);
112}
113
114static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop,
115					size_t bytes, void **ptrs)
116{
117	struct kernel_fpu vxstate;
118	u8 **dptr, *p, *q;
119	int d, z, z0;
120
121	dptr = (u8 **) ptrs;
122	z0 = stop;		/* P/Q right side optimization */
123	p = dptr[disks - 2];	/* XOR parity */
124	q = dptr[disks - 1];	/* RS syndrome */
125
126	kernel_fpu_begin(&vxstate, KERNEL_VXR);
127	LOAD_CONST();
128
129	for (d = 0; d < bytes; d += $#*NSIZE) {
130		/* P/Q data pages */
131		LOAD_DATA(0,&dptr[z0][d]);
132		COPY_VEC(8+$$,0+$$);
133		for (z = z0 - 1; z >= start; z--) {
134			MASK(16+$$,8+$$);
135			AND(16+$$,16+$$,25);
136			SHLBYTE(8+$$,8+$$);
137			XOR(8+$$,8+$$,16+$$);
138			LOAD_DATA(16,&dptr[z][d]);
139			XOR(0+$$,0+$$,16+$$);
140			XOR(8+$$,8+$$,16+$$);
141		}
142		/* P/Q left side optimization */
143		for (z = start - 1; z >= 0; z--) {
144			MASK(16+$$,8+$$);
145			AND(16+$$,16+$$,25);
146			SHLBYTE(8+$$,8+$$);
147			XOR(8+$$,8+$$,16+$$);
148		}
149		LOAD_DATA(16,&p[d]);
150		XOR(16+$$,16+$$,0+$$);
151		STORE_DATA(16,&p[d]);
152		LOAD_DATA(16,&q[d]);
153		XOR(16+$$,16+$$,8+$$);
154		STORE_DATA(16,&q[d]);
155	}
156	kernel_fpu_end(&vxstate, KERNEL_VXR);
157}
158
159static int raid6_s390vx$#_valid(void)
160{
161	return MACHINE_HAS_VX;
162}
163
164const struct raid6_calls raid6_s390vx$# = {
165	raid6_s390vx$#_gen_syndrome,
166	raid6_s390vx$#_xor_syndrome,
167	raid6_s390vx$#_valid,
168	"vx128x$#",
169	1
170};