1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2012 Intel Corporation
  4 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
  5 */
  6
  7#include <linux/raid/pq.h>
  8
  9#ifdef __KERNEL__
 10#include <asm/neon.h>
 11#else
 12#define kernel_neon_begin()
 13#define kernel_neon_end()
 14#define cpu_has_neon()		(1)
 15#endif
 16
 17static int raid6_has_neon(void)
 18{
 19	return cpu_has_neon();
 20}
 21
 22void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
 23			      uint8_t *dq, const uint8_t *pbmul,
 24			      const uint8_t *qmul);
 25
 26void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
 27			      const uint8_t *qmul);
 28
 29static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
 30		int failb, void **ptrs)
 31{
 32	u8 *p, *q, *dp, *dq;
 33	const u8 *pbmul;	/* P multiplier table for B data */
 34	const u8 *qmul;		/* Q multiplier table (for both) */
 35
 36	p = (u8 *)ptrs[disks - 2];
 37	q = (u8 *)ptrs[disks - 1];
 38
 39	/*
 40	 * Compute syndrome with zero for the missing data pages
 41	 * Use the dead data pages as temporary storage for
 42	 * delta p and delta q
 43	 */
 44	dp = (u8 *)ptrs[faila];
 45	ptrs[faila] = (void *)raid6_empty_zero_page;
 46	ptrs[disks - 2] = dp;
 47	dq = (u8 *)ptrs[failb];
 48	ptrs[failb] = (void *)raid6_empty_zero_page;
 49	ptrs[disks - 1] = dq;
 50
 51	raid6_call.gen_syndrome(disks, bytes, ptrs);
 52
 53	/* Restore pointer table */
 54	ptrs[faila]     = dp;
 55	ptrs[failb]     = dq;
 56	ptrs[disks - 2] = p;
 57	ptrs[disks - 1] = q;
 58
 59	/* Now, pick the proper data tables */
 60	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
 61	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
 62					 raid6_gfexp[failb]]];
 63
 64	kernel_neon_begin();
 65	__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
 66	kernel_neon_end();
 67}
 68
 69static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
 70		void **ptrs)
 71{
 72	u8 *p, *q, *dq;
 73	const u8 *qmul;		/* Q multiplier table */
 74
 75	p = (u8 *)ptrs[disks - 2];
 76	q = (u8 *)ptrs[disks - 1];
 77
 78	/*
 79	 * Compute syndrome with zero for the missing data page
 80	 * Use the dead data page as temporary storage for delta q
 81	 */
 82	dq = (u8 *)ptrs[faila];
 83	ptrs[faila] = (void *)raid6_empty_zero_page;
 84	ptrs[disks - 1] = dq;
 85
 86	raid6_call.gen_syndrome(disks, bytes, ptrs);
 87
 88	/* Restore pointer table */
 89	ptrs[faila]     = dq;
 90	ptrs[disks - 1] = q;
 91
 92	/* Now, pick the proper data tables */
 93	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
 94
 95	kernel_neon_begin();
 96	__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
 97	kernel_neon_end();
 98}
 99
100const struct raid6_recov_calls raid6_recov_neon = {
101	.data2		= raid6_2data_recov_neon,
102	.datap		= raid6_datap_recov_neon,
103	.valid		= raid6_has_neon,
104	.name		= "neon",
105	.priority	= 10,
106};