Loading...
1/*
2 * Copyright (C) 2012 Intel Corporation
3 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; version 2
8 * of the License.
9 */
10
11#include <linux/raid/pq.h>
12
13#ifdef __KERNEL__
14#include <asm/neon.h>
15#else
16#define kernel_neon_begin()
17#define kernel_neon_end()
18#define cpu_has_neon() (1)
19#endif
20
21static int raid6_has_neon(void)
22{
23 return cpu_has_neon();
24}
25
26void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
27 uint8_t *dq, const uint8_t *pbmul,
28 const uint8_t *qmul);
29
30void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
31 const uint8_t *qmul);
32
33static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
34 int failb, void **ptrs)
35{
36 u8 *p, *q, *dp, *dq;
37 const u8 *pbmul; /* P multiplier table for B data */
38 const u8 *qmul; /* Q multiplier table (for both) */
39
40 p = (u8 *)ptrs[disks - 2];
41 q = (u8 *)ptrs[disks - 1];
42
43 /*
44 * Compute syndrome with zero for the missing data pages
45 * Use the dead data pages as temporary storage for
46 * delta p and delta q
47 */
48 dp = (u8 *)ptrs[faila];
49 ptrs[faila] = (void *)raid6_empty_zero_page;
50 ptrs[disks - 2] = dp;
51 dq = (u8 *)ptrs[failb];
52 ptrs[failb] = (void *)raid6_empty_zero_page;
53 ptrs[disks - 1] = dq;
54
55 raid6_call.gen_syndrome(disks, bytes, ptrs);
56
57 /* Restore pointer table */
58 ptrs[faila] = dp;
59 ptrs[failb] = dq;
60 ptrs[disks - 2] = p;
61 ptrs[disks - 1] = q;
62
63 /* Now, pick the proper data tables */
64 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
65 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
66 raid6_gfexp[failb]]];
67
68 kernel_neon_begin();
69 __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
70 kernel_neon_end();
71}
72
73static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
74 void **ptrs)
75{
76 u8 *p, *q, *dq;
77 const u8 *qmul; /* Q multiplier table */
78
79 p = (u8 *)ptrs[disks - 2];
80 q = (u8 *)ptrs[disks - 1];
81
82 /*
83 * Compute syndrome with zero for the missing data page
84 * Use the dead data page as temporary storage for delta q
85 */
86 dq = (u8 *)ptrs[faila];
87 ptrs[faila] = (void *)raid6_empty_zero_page;
88 ptrs[disks - 1] = dq;
89
90 raid6_call.gen_syndrome(disks, bytes, ptrs);
91
92 /* Restore pointer table */
93 ptrs[faila] = dq;
94 ptrs[disks - 1] = q;
95
96 /* Now, pick the proper data tables */
97 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
98
99 kernel_neon_begin();
100 __raid6_datap_recov_neon(bytes, p, q, dq, qmul);
101 kernel_neon_end();
102}
103
104const struct raid6_recov_calls raid6_recov_neon = {
105 .data2 = raid6_2data_recov_neon,
106 .datap = raid6_datap_recov_neon,
107 .valid = raid6_has_neon,
108 .name = "neon",
109 .priority = 10,
110};
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012 Intel Corporation
4 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 */
6
7#include <linux/raid/pq.h>
8
9#ifdef __KERNEL__
10#include <asm/neon.h>
11#else
12#define kernel_neon_begin()
13#define kernel_neon_end()
14#define cpu_has_neon() (1)
15#endif
16
17static int raid6_has_neon(void)
18{
19 return cpu_has_neon();
20}
21
22void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
23 uint8_t *dq, const uint8_t *pbmul,
24 const uint8_t *qmul);
25
26void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
27 const uint8_t *qmul);
28
29static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
30 int failb, void **ptrs)
31{
32 u8 *p, *q, *dp, *dq;
33 const u8 *pbmul; /* P multiplier table for B data */
34 const u8 *qmul; /* Q multiplier table (for both) */
35
36 p = (u8 *)ptrs[disks - 2];
37 q = (u8 *)ptrs[disks - 1];
38
39 /*
40 * Compute syndrome with zero for the missing data pages
41 * Use the dead data pages as temporary storage for
42 * delta p and delta q
43 */
44 dp = (u8 *)ptrs[faila];
45 ptrs[faila] = (void *)raid6_empty_zero_page;
46 ptrs[disks - 2] = dp;
47 dq = (u8 *)ptrs[failb];
48 ptrs[failb] = (void *)raid6_empty_zero_page;
49 ptrs[disks - 1] = dq;
50
51 raid6_call.gen_syndrome(disks, bytes, ptrs);
52
53 /* Restore pointer table */
54 ptrs[faila] = dp;
55 ptrs[failb] = dq;
56 ptrs[disks - 2] = p;
57 ptrs[disks - 1] = q;
58
59 /* Now, pick the proper data tables */
60 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
61 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
62 raid6_gfexp[failb]]];
63
64 kernel_neon_begin();
65 __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
66 kernel_neon_end();
67}
68
69static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
70 void **ptrs)
71{
72 u8 *p, *q, *dq;
73 const u8 *qmul; /* Q multiplier table */
74
75 p = (u8 *)ptrs[disks - 2];
76 q = (u8 *)ptrs[disks - 1];
77
78 /*
79 * Compute syndrome with zero for the missing data page
80 * Use the dead data page as temporary storage for delta q
81 */
82 dq = (u8 *)ptrs[faila];
83 ptrs[faila] = (void *)raid6_empty_zero_page;
84 ptrs[disks - 1] = dq;
85
86 raid6_call.gen_syndrome(disks, bytes, ptrs);
87
88 /* Restore pointer table */
89 ptrs[faila] = dq;
90 ptrs[disks - 1] = q;
91
92 /* Now, pick the proper data tables */
93 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
94
95 kernel_neon_begin();
96 __raid6_datap_recov_neon(bytes, p, q, dq, qmul);
97 kernel_neon_end();
98}
99
100const struct raid6_recov_calls raid6_recov_neon = {
101 .data2 = raid6_2data_recov_neon,
102 .datap = raid6_datap_recov_neon,
103 .valid = raid6_has_neon,
104 .name = "neon",
105 .priority = 10,
106};