1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2012 Intel Corporation
  4 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
  5 */
  6
  7#include <linux/raid/pq.h>
  8
  9#ifdef __KERNEL__
 10#include <asm/neon.h>
 11#include "neon.h"
 12#else
 13#define kernel_neon_begin()
 14#define kernel_neon_end()
 15#define cpu_has_neon()		(1)
 16#endif
 17
 18static int raid6_has_neon(void)
 19{
 20	return cpu_has_neon();
 21}
 22
 23static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
 24		int failb, void **ptrs)
 25{
 26	u8 *p, *q, *dp, *dq;
 27	const u8 *pbmul;	/* P multiplier table for B data */
 28	const u8 *qmul;		/* Q multiplier table (for both) */
 29
 30	p = (u8 *)ptrs[disks - 2];
 31	q = (u8 *)ptrs[disks - 1];
 32
 33	/*
 34	 * Compute syndrome with zero for the missing data pages
 35	 * Use the dead data pages as temporary storage for
 36	 * delta p and delta q
 37	 */
 38	dp = (u8 *)ptrs[faila];
 39	ptrs[faila] = (void *)raid6_empty_zero_page;
 40	ptrs[disks - 2] = dp;
 41	dq = (u8 *)ptrs[failb];
 42	ptrs[failb] = (void *)raid6_empty_zero_page;
 43	ptrs[disks - 1] = dq;
 44
 45	raid6_call.gen_syndrome(disks, bytes, ptrs);
 46
 47	/* Restore pointer table */
 48	ptrs[faila]     = dp;
 49	ptrs[failb]     = dq;
 50	ptrs[disks - 2] = p;
 51	ptrs[disks - 1] = q;
 52
 53	/* Now, pick the proper data tables */
 54	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
 55	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
 56					 raid6_gfexp[failb]]];
 57
 58	kernel_neon_begin();
 59	__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
 60	kernel_neon_end();
 61}
 62
 63static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
 64		void **ptrs)
 65{
 66	u8 *p, *q, *dq;
 67	const u8 *qmul;		/* Q multiplier table */
 68
 69	p = (u8 *)ptrs[disks - 2];
 70	q = (u8 *)ptrs[disks - 1];
 71
 72	/*
 73	 * Compute syndrome with zero for the missing data page
 74	 * Use the dead data page as temporary storage for delta q
 75	 */
 76	dq = (u8 *)ptrs[faila];
 77	ptrs[faila] = (void *)raid6_empty_zero_page;
 78	ptrs[disks - 1] = dq;
 79
 80	raid6_call.gen_syndrome(disks, bytes, ptrs);
 81
 82	/* Restore pointer table */
 83	ptrs[faila]     = dq;
 84	ptrs[disks - 1] = q;
 85
 86	/* Now, pick the proper data tables */
 87	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
 88
 89	kernel_neon_begin();
 90	__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
 91	kernel_neon_end();
 92}
 93
 94const struct raid6_recov_calls raid6_recov_neon = {
 95	.data2		= raid6_2data_recov_neon,
 96	.datap		= raid6_datap_recov_neon,
 97	.valid		= raid6_has_neon,
 98	.name		= "neon",
 99	.priority	= 10,
100};