1/*
  2 * Copyright (C) 2012 Intel Corporation
  3 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
  4 *
  5 * This program is free software; you can redistribute it and/or
  6 * modify it under the terms of the GNU General Public License
  7 * as published by the Free Software Foundation; version 2
  8 * of the License.
  9 */
 10
 11#include <linux/raid/pq.h>
 12
 13#ifdef __KERNEL__
 14#include <asm/neon.h>
 15#else
 16#define kernel_neon_begin()
 17#define kernel_neon_end()
 18#define cpu_has_neon()		(1)
 19#endif
 20
 21static int raid6_has_neon(void)
 22{
 23	return cpu_has_neon();
 24}
 25
 26void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
 27			      uint8_t *dq, const uint8_t *pbmul,
 28			      const uint8_t *qmul);
 29
 30void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
 31			      const uint8_t *qmul);
 32
 33static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
 34		int failb, void **ptrs)
 35{
 36	u8 *p, *q, *dp, *dq;
 37	const u8 *pbmul;	/* P multiplier table for B data */
 38	const u8 *qmul;		/* Q multiplier table (for both) */
 39
 40	p = (u8 *)ptrs[disks - 2];
 41	q = (u8 *)ptrs[disks - 1];
 42
 43	/*
 44	 * Compute syndrome with zero for the missing data pages
 45	 * Use the dead data pages as temporary storage for
 46	 * delta p and delta q
 47	 */
 48	dp = (u8 *)ptrs[faila];
 49	ptrs[faila] = (void *)raid6_empty_zero_page;
 50	ptrs[disks - 2] = dp;
 51	dq = (u8 *)ptrs[failb];
 52	ptrs[failb] = (void *)raid6_empty_zero_page;
 53	ptrs[disks - 1] = dq;
 54
 55	raid6_call.gen_syndrome(disks, bytes, ptrs);
 56
 57	/* Restore pointer table */
 58	ptrs[faila]     = dp;
 59	ptrs[failb]     = dq;
 60	ptrs[disks - 2] = p;
 61	ptrs[disks - 1] = q;
 62
 63	/* Now, pick the proper data tables */
 64	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
 65	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
 66					 raid6_gfexp[failb]]];
 67
 68	kernel_neon_begin();
 69	__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
 70	kernel_neon_end();
 71}
 72
 73static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
 74		void **ptrs)
 75{
 76	u8 *p, *q, *dq;
 77	const u8 *qmul;		/* Q multiplier table */
 78
 79	p = (u8 *)ptrs[disks - 2];
 80	q = (u8 *)ptrs[disks - 1];
 81
 82	/*
 83	 * Compute syndrome with zero for the missing data page
 84	 * Use the dead data page as temporary storage for delta q
 85	 */
 86	dq = (u8 *)ptrs[faila];
 87	ptrs[faila] = (void *)raid6_empty_zero_page;
 88	ptrs[disks - 1] = dq;
 89
 90	raid6_call.gen_syndrome(disks, bytes, ptrs);
 91
 92	/* Restore pointer table */
 93	ptrs[faila]     = dq;
 94	ptrs[disks - 1] = q;
 95
 96	/* Now, pick the proper data tables */
 97	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
 98
 99	kernel_neon_begin();
100	__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
101	kernel_neon_end();
102}
103
104const struct raid6_recov_calls raid6_recov_neon = {
105	.data2		= raid6_2data_recov_neon,
106	.datap		= raid6_datap_recov_neon,
107	.valid		= raid6_has_neon,
108	.name		= "neon",
109	.priority	= 10,
110};