Loading...
1/*
2 * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
3 * Copyright(c) 2009 Intel Corporation
4 *
5 * based on raid6recov.c:
6 * Copyright 2002 H. Peter Anvin
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 51
20 * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
21 *
22 */
23#include <linux/kernel.h>
24#include <linux/interrupt.h>
25#include <linux/module.h>
26#include <linux/dma-mapping.h>
27#include <linux/raid/pq.h>
28#include <linux/async_tx.h>
29#include <linux/dmaengine.h>
30
31static struct dma_async_tx_descriptor *
32async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
33 size_t len, struct async_submit_ctl *submit)
34{
35 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
36 &dest, 1, srcs, 2, len);
37 struct dma_device *dma = chan ? chan->device : NULL;
38 struct dmaengine_unmap_data *unmap = NULL;
39 const u8 *amul, *bmul;
40 u8 ax, bx;
41 u8 *a, *b, *c;
42
43 if (dma)
44 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO);
45
46 if (unmap) {
47 struct device *dev = dma->dev;
48 dma_addr_t pq[2];
49 struct dma_async_tx_descriptor *tx;
50 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
51
52 if (submit->flags & ASYNC_TX_FENCE)
53 dma_flags |= DMA_PREP_FENCE;
54 unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
55 unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
56 unmap->to_cnt = 2;
57
58 unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
59 unmap->bidi_cnt = 1;
60 /* engine only looks at Q, but expects it to follow P */
61 pq[1] = unmap->addr[2];
62
63 unmap->len = len;
64 tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
65 len, dma_flags);
66 if (tx) {
67 dma_set_unmap(tx, unmap);
68 async_tx_submit(chan, tx, submit);
69 dmaengine_unmap_put(unmap);
70 return tx;
71 }
72
73 /* could not get a descriptor, unmap and fall through to
74 * the synchronous path
75 */
76 dmaengine_unmap_put(unmap);
77 }
78
79 /* run the operation synchronously */
80 async_tx_quiesce(&submit->depend_tx);
81 amul = raid6_gfmul[coef[0]];
82 bmul = raid6_gfmul[coef[1]];
83 a = page_address(srcs[0]);
84 b = page_address(srcs[1]);
85 c = page_address(dest);
86
87 while (len--) {
88 ax = amul[*a++];
89 bx = bmul[*b++];
90 *c++ = ax ^ bx;
91 }
92
93 return NULL;
94}
95
96static struct dma_async_tx_descriptor *
97async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
98 struct async_submit_ctl *submit)
99{
100 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
101 &dest, 1, &src, 1, len);
102 struct dma_device *dma = chan ? chan->device : NULL;
103 struct dmaengine_unmap_data *unmap = NULL;
104 const u8 *qmul; /* Q multiplier table */
105 u8 *d, *s;
106
107 if (dma)
108 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO);
109
110 if (unmap) {
111 dma_addr_t dma_dest[2];
112 struct device *dev = dma->dev;
113 struct dma_async_tx_descriptor *tx;
114 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
115
116 if (submit->flags & ASYNC_TX_FENCE)
117 dma_flags |= DMA_PREP_FENCE;
118 unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
119 unmap->to_cnt++;
120 unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
121 dma_dest[1] = unmap->addr[1];
122 unmap->bidi_cnt++;
123 unmap->len = len;
124
125 /* this looks funny, but the engine looks for Q at
126 * dma_dest[1] and ignores dma_dest[0] as a dest
127 * due to DMA_PREP_PQ_DISABLE_P
128 */
129 tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
130 1, &coef, len, dma_flags);
131
132 if (tx) {
133 dma_set_unmap(tx, unmap);
134 dmaengine_unmap_put(unmap);
135 async_tx_submit(chan, tx, submit);
136 return tx;
137 }
138
139 /* could not get a descriptor, unmap and fall through to
140 * the synchronous path
141 */
142 dmaengine_unmap_put(unmap);
143 }
144
145 /* no channel available, or failed to allocate a descriptor, so
146 * perform the operation synchronously
147 */
148 async_tx_quiesce(&submit->depend_tx);
149 qmul = raid6_gfmul[coef];
150 d = page_address(dest);
151 s = page_address(src);
152
153 while (len--)
154 *d++ = qmul[*s++];
155
156 return NULL;
157}
158
159static struct dma_async_tx_descriptor *
160__2data_recov_4(int disks, size_t bytes, int faila, int failb,
161 struct page **blocks, struct async_submit_ctl *submit)
162{
163 struct dma_async_tx_descriptor *tx = NULL;
164 struct page *p, *q, *a, *b;
165 struct page *srcs[2];
166 unsigned char coef[2];
167 enum async_tx_flags flags = submit->flags;
168 dma_async_tx_callback cb_fn = submit->cb_fn;
169 void *cb_param = submit->cb_param;
170 void *scribble = submit->scribble;
171
172 p = blocks[disks-2];
173 q = blocks[disks-1];
174
175 a = blocks[faila];
176 b = blocks[failb];
177
178 /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
179 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
180 srcs[0] = p;
181 srcs[1] = q;
182 coef[0] = raid6_gfexi[failb-faila];
183 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
184 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
185 tx = async_sum_product(b, srcs, coef, bytes, submit);
186
187 /* Dy = P+Pxy+Dx */
188 srcs[0] = p;
189 srcs[1] = b;
190 init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
191 cb_param, scribble);
192 tx = async_xor(a, srcs, 0, 2, bytes, submit);
193
194 return tx;
195
196}
197
198static struct dma_async_tx_descriptor *
199__2data_recov_5(int disks, size_t bytes, int faila, int failb,
200 struct page **blocks, struct async_submit_ctl *submit)
201{
202 struct dma_async_tx_descriptor *tx = NULL;
203 struct page *p, *q, *g, *dp, *dq;
204 struct page *srcs[2];
205 unsigned char coef[2];
206 enum async_tx_flags flags = submit->flags;
207 dma_async_tx_callback cb_fn = submit->cb_fn;
208 void *cb_param = submit->cb_param;
209 void *scribble = submit->scribble;
210 int good_srcs, good, i;
211
212 good_srcs = 0;
213 good = -1;
214 for (i = 0; i < disks-2; i++) {
215 if (blocks[i] == NULL)
216 continue;
217 if (i == faila || i == failb)
218 continue;
219 good = i;
220 good_srcs++;
221 }
222 BUG_ON(good_srcs > 1);
223
224 p = blocks[disks-2];
225 q = blocks[disks-1];
226 g = blocks[good];
227
228 /* Compute syndrome with zero for the missing data pages
229 * Use the dead data pages as temporary storage for delta p and
230 * delta q
231 */
232 dp = blocks[faila];
233 dq = blocks[failb];
234
235 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
236 tx = async_memcpy(dp, g, 0, 0, bytes, submit);
237 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
238 tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
239
240 /* compute P + Pxy */
241 srcs[0] = dp;
242 srcs[1] = p;
243 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
244 NULL, NULL, scribble);
245 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
246
247 /* compute Q + Qxy */
248 srcs[0] = dq;
249 srcs[1] = q;
250 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
251 NULL, NULL, scribble);
252 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
253
254 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
255 srcs[0] = dp;
256 srcs[1] = dq;
257 coef[0] = raid6_gfexi[failb-faila];
258 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
259 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
260 tx = async_sum_product(dq, srcs, coef, bytes, submit);
261
262 /* Dy = P+Pxy+Dx */
263 srcs[0] = dp;
264 srcs[1] = dq;
265 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
266 cb_param, scribble);
267 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
268
269 return tx;
270}
271
272static struct dma_async_tx_descriptor *
273__2data_recov_n(int disks, size_t bytes, int faila, int failb,
274 struct page **blocks, struct async_submit_ctl *submit)
275{
276 struct dma_async_tx_descriptor *tx = NULL;
277 struct page *p, *q, *dp, *dq;
278 struct page *srcs[2];
279 unsigned char coef[2];
280 enum async_tx_flags flags = submit->flags;
281 dma_async_tx_callback cb_fn = submit->cb_fn;
282 void *cb_param = submit->cb_param;
283 void *scribble = submit->scribble;
284
285 p = blocks[disks-2];
286 q = blocks[disks-1];
287
288 /* Compute syndrome with zero for the missing data pages
289 * Use the dead data pages as temporary storage for
290 * delta p and delta q
291 */
292 dp = blocks[faila];
293 blocks[faila] = NULL;
294 blocks[disks-2] = dp;
295 dq = blocks[failb];
296 blocks[failb] = NULL;
297 blocks[disks-1] = dq;
298
299 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
300 tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
301
302 /* Restore pointer table */
303 blocks[faila] = dp;
304 blocks[failb] = dq;
305 blocks[disks-2] = p;
306 blocks[disks-1] = q;
307
308 /* compute P + Pxy */
309 srcs[0] = dp;
310 srcs[1] = p;
311 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
312 NULL, NULL, scribble);
313 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
314
315 /* compute Q + Qxy */
316 srcs[0] = dq;
317 srcs[1] = q;
318 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
319 NULL, NULL, scribble);
320 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
321
322 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
323 srcs[0] = dp;
324 srcs[1] = dq;
325 coef[0] = raid6_gfexi[failb-faila];
326 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
327 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
328 tx = async_sum_product(dq, srcs, coef, bytes, submit);
329
330 /* Dy = P+Pxy+Dx */
331 srcs[0] = dp;
332 srcs[1] = dq;
333 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
334 cb_param, scribble);
335 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
336
337 return tx;
338}
339
340/**
341 * async_raid6_2data_recov - asynchronously calculate two missing data blocks
342 * @disks: number of disks in the RAID-6 array
343 * @bytes: block size
344 * @faila: first failed drive index
345 * @failb: second failed drive index
346 * @blocks: array of source pointers where the last two entries are p and q
347 * @submit: submission/completion modifiers
348 */
349struct dma_async_tx_descriptor *
350async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
351 struct page **blocks, struct async_submit_ctl *submit)
352{
353 void *scribble = submit->scribble;
354 int non_zero_srcs, i;
355
356 BUG_ON(faila == failb);
357 if (failb < faila)
358 swap(faila, failb);
359
360 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
361
362 /* if a dma resource is not available or a scribble buffer is not
363 * available punt to the synchronous path. In the 'dma not
364 * available' case be sure to use the scribble buffer to
365 * preserve the content of 'blocks' as the caller intended.
366 */
367 if (!async_dma_find_channel(DMA_PQ) || !scribble) {
368 void **ptrs = scribble ? scribble : (void **) blocks;
369
370 async_tx_quiesce(&submit->depend_tx);
371 for (i = 0; i < disks; i++)
372 if (blocks[i] == NULL)
373 ptrs[i] = (void *) raid6_empty_zero_page;
374 else
375 ptrs[i] = page_address(blocks[i]);
376
377 raid6_2data_recov(disks, bytes, faila, failb, ptrs);
378
379 async_tx_sync_epilog(submit);
380
381 return NULL;
382 }
383
384 non_zero_srcs = 0;
385 for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
386 if (blocks[i])
387 non_zero_srcs++;
388 switch (non_zero_srcs) {
389 case 0:
390 case 1:
391 /* There must be at least 2 sources - the failed devices. */
392 BUG();
393
394 case 2:
395 /* dma devices do not uniformly understand a zero source pq
396 * operation (in contrast to the synchronous case), so
397 * explicitly handle the special case of a 4 disk array with
398 * both data disks missing.
399 */
400 return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
401 case 3:
402 /* dma devices do not uniformly understand a single
403 * source pq operation (in contrast to the synchronous
404 * case), so explicitly handle the special case of a 5 disk
405 * array with 2 of 3 data disks missing.
406 */
407 return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
408 default:
409 return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
410 }
411}
412EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
413
414/**
415 * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
416 * @disks: number of disks in the RAID-6 array
417 * @bytes: block size
418 * @faila: failed drive index
419 * @blocks: array of source pointers where the last two entries are p and q
420 * @submit: submission/completion modifiers
421 */
422struct dma_async_tx_descriptor *
423async_raid6_datap_recov(int disks, size_t bytes, int faila,
424 struct page **blocks, struct async_submit_ctl *submit)
425{
426 struct dma_async_tx_descriptor *tx = NULL;
427 struct page *p, *q, *dq;
428 u8 coef;
429 enum async_tx_flags flags = submit->flags;
430 dma_async_tx_callback cb_fn = submit->cb_fn;
431 void *cb_param = submit->cb_param;
432 void *scribble = submit->scribble;
433 int good_srcs, good, i;
434 struct page *srcs[2];
435
436 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
437
438 /* if a dma resource is not available or a scribble buffer is not
439 * available punt to the synchronous path. In the 'dma not
440 * available' case be sure to use the scribble buffer to
441 * preserve the content of 'blocks' as the caller intended.
442 */
443 if (!async_dma_find_channel(DMA_PQ) || !scribble) {
444 void **ptrs = scribble ? scribble : (void **) blocks;
445
446 async_tx_quiesce(&submit->depend_tx);
447 for (i = 0; i < disks; i++)
448 if (blocks[i] == NULL)
449 ptrs[i] = (void*)raid6_empty_zero_page;
450 else
451 ptrs[i] = page_address(blocks[i]);
452
453 raid6_datap_recov(disks, bytes, faila, ptrs);
454
455 async_tx_sync_epilog(submit);
456
457 return NULL;
458 }
459
460 good_srcs = 0;
461 good = -1;
462 for (i = 0; i < disks-2; i++) {
463 if (i == faila)
464 continue;
465 if (blocks[i]) {
466 good = i;
467 good_srcs++;
468 if (good_srcs > 1)
469 break;
470 }
471 }
472 BUG_ON(good_srcs == 0);
473
474 p = blocks[disks-2];
475 q = blocks[disks-1];
476
477 /* Compute syndrome with zero for the missing data page
478 * Use the dead data page as temporary storage for delta q
479 */
480 dq = blocks[faila];
481 blocks[faila] = NULL;
482 blocks[disks-1] = dq;
483
484 /* in the 4-disk case we only need to perform a single source
485 * multiplication with the one good data block.
486 */
487 if (good_srcs == 1) {
488 struct page *g = blocks[good];
489
490 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
491 scribble);
492 tx = async_memcpy(p, g, 0, 0, bytes, submit);
493
494 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
495 scribble);
496 tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
497 } else {
498 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
499 scribble);
500 tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
501 }
502
503 /* Restore pointer table */
504 blocks[faila] = dq;
505 blocks[disks-1] = q;
506
507 /* calculate g^{-faila} */
508 coef = raid6_gfinv[raid6_gfexp[faila]];
509
510 srcs[0] = dq;
511 srcs[1] = q;
512 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
513 NULL, NULL, scribble);
514 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
515
516 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
517 tx = async_mult(dq, dq, coef, bytes, submit);
518
519 srcs[0] = p;
520 srcs[1] = dq;
521 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
522 cb_param, scribble);
523 tx = async_xor(p, srcs, 0, 2, bytes, submit);
524
525 return tx;
526}
527EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
528
529MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
530MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
531MODULE_LICENSE("GPL");
1/*
2 * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
3 * Copyright(c) 2009 Intel Corporation
4 *
5 * based on raid6recov.c:
6 * Copyright 2002 H. Peter Anvin
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 51
20 * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
21 *
22 */
23#include <linux/kernel.h>
24#include <linux/interrupt.h>
25#include <linux/module.h>
26#include <linux/dma-mapping.h>
27#include <linux/raid/pq.h>
28#include <linux/async_tx.h>
29
30static struct dma_async_tx_descriptor *
31async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
32 size_t len, struct async_submit_ctl *submit)
33{
34 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
35 &dest, 1, srcs, 2, len);
36 struct dma_device *dma = chan ? chan->device : NULL;
37 const u8 *amul, *bmul;
38 u8 ax, bx;
39 u8 *a, *b, *c;
40
41 if (dma) {
42 dma_addr_t dma_dest[2];
43 dma_addr_t dma_src[2];
44 struct device *dev = dma->dev;
45 struct dma_async_tx_descriptor *tx;
46 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
47
48 if (submit->flags & ASYNC_TX_FENCE)
49 dma_flags |= DMA_PREP_FENCE;
50 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
51 dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
52 dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
53 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef,
54 len, dma_flags);
55 if (tx) {
56 async_tx_submit(chan, tx, submit);
57 return tx;
58 }
59
60 /* could not get a descriptor, unmap and fall through to
61 * the synchronous path
62 */
63 dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
64 dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
65 dma_unmap_page(dev, dma_src[1], len, DMA_TO_DEVICE);
66 }
67
68 /* run the operation synchronously */
69 async_tx_quiesce(&submit->depend_tx);
70 amul = raid6_gfmul[coef[0]];
71 bmul = raid6_gfmul[coef[1]];
72 a = page_address(srcs[0]);
73 b = page_address(srcs[1]);
74 c = page_address(dest);
75
76 while (len--) {
77 ax = amul[*a++];
78 bx = bmul[*b++];
79 *c++ = ax ^ bx;
80 }
81
82 return NULL;
83}
84
85static struct dma_async_tx_descriptor *
86async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
87 struct async_submit_ctl *submit)
88{
89 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
90 &dest, 1, &src, 1, len);
91 struct dma_device *dma = chan ? chan->device : NULL;
92 const u8 *qmul; /* Q multiplier table */
93 u8 *d, *s;
94
95 if (dma) {
96 dma_addr_t dma_dest[2];
97 dma_addr_t dma_src[1];
98 struct device *dev = dma->dev;
99 struct dma_async_tx_descriptor *tx;
100 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
101
102 if (submit->flags & ASYNC_TX_FENCE)
103 dma_flags |= DMA_PREP_FENCE;
104 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
105 dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
106 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef,
107 len, dma_flags);
108 if (tx) {
109 async_tx_submit(chan, tx, submit);
110 return tx;
111 }
112
113 /* could not get a descriptor, unmap and fall through to
114 * the synchronous path
115 */
116 dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
117 dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
118 }
119
120 /* no channel available, or failed to allocate a descriptor, so
121 * perform the operation synchronously
122 */
123 async_tx_quiesce(&submit->depend_tx);
124 qmul = raid6_gfmul[coef];
125 d = page_address(dest);
126 s = page_address(src);
127
128 while (len--)
129 *d++ = qmul[*s++];
130
131 return NULL;
132}
133
134static struct dma_async_tx_descriptor *
135__2data_recov_4(int disks, size_t bytes, int faila, int failb,
136 struct page **blocks, struct async_submit_ctl *submit)
137{
138 struct dma_async_tx_descriptor *tx = NULL;
139 struct page *p, *q, *a, *b;
140 struct page *srcs[2];
141 unsigned char coef[2];
142 enum async_tx_flags flags = submit->flags;
143 dma_async_tx_callback cb_fn = submit->cb_fn;
144 void *cb_param = submit->cb_param;
145 void *scribble = submit->scribble;
146
147 p = blocks[disks-2];
148 q = blocks[disks-1];
149
150 a = blocks[faila];
151 b = blocks[failb];
152
153 /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
154 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
155 srcs[0] = p;
156 srcs[1] = q;
157 coef[0] = raid6_gfexi[failb-faila];
158 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
159 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
160 tx = async_sum_product(b, srcs, coef, bytes, submit);
161
162 /* Dy = P+Pxy+Dx */
163 srcs[0] = p;
164 srcs[1] = b;
165 init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
166 cb_param, scribble);
167 tx = async_xor(a, srcs, 0, 2, bytes, submit);
168
169 return tx;
170
171}
172
173static struct dma_async_tx_descriptor *
174__2data_recov_5(int disks, size_t bytes, int faila, int failb,
175 struct page **blocks, struct async_submit_ctl *submit)
176{
177 struct dma_async_tx_descriptor *tx = NULL;
178 struct page *p, *q, *g, *dp, *dq;
179 struct page *srcs[2];
180 unsigned char coef[2];
181 enum async_tx_flags flags = submit->flags;
182 dma_async_tx_callback cb_fn = submit->cb_fn;
183 void *cb_param = submit->cb_param;
184 void *scribble = submit->scribble;
185 int good_srcs, good, i;
186
187 good_srcs = 0;
188 good = -1;
189 for (i = 0; i < disks-2; i++) {
190 if (blocks[i] == NULL)
191 continue;
192 if (i == faila || i == failb)
193 continue;
194 good = i;
195 good_srcs++;
196 }
197 BUG_ON(good_srcs > 1);
198
199 p = blocks[disks-2];
200 q = blocks[disks-1];
201 g = blocks[good];
202
203 /* Compute syndrome with zero for the missing data pages
204 * Use the dead data pages as temporary storage for delta p and
205 * delta q
206 */
207 dp = blocks[faila];
208 dq = blocks[failb];
209
210 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
211 tx = async_memcpy(dp, g, 0, 0, bytes, submit);
212 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
213 tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
214
215 /* compute P + Pxy */
216 srcs[0] = dp;
217 srcs[1] = p;
218 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
219 NULL, NULL, scribble);
220 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
221
222 /* compute Q + Qxy */
223 srcs[0] = dq;
224 srcs[1] = q;
225 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
226 NULL, NULL, scribble);
227 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
228
229 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
230 srcs[0] = dp;
231 srcs[1] = dq;
232 coef[0] = raid6_gfexi[failb-faila];
233 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
234 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
235 tx = async_sum_product(dq, srcs, coef, bytes, submit);
236
237 /* Dy = P+Pxy+Dx */
238 srcs[0] = dp;
239 srcs[1] = dq;
240 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
241 cb_param, scribble);
242 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
243
244 return tx;
245}
246
247static struct dma_async_tx_descriptor *
248__2data_recov_n(int disks, size_t bytes, int faila, int failb,
249 struct page **blocks, struct async_submit_ctl *submit)
250{
251 struct dma_async_tx_descriptor *tx = NULL;
252 struct page *p, *q, *dp, *dq;
253 struct page *srcs[2];
254 unsigned char coef[2];
255 enum async_tx_flags flags = submit->flags;
256 dma_async_tx_callback cb_fn = submit->cb_fn;
257 void *cb_param = submit->cb_param;
258 void *scribble = submit->scribble;
259
260 p = blocks[disks-2];
261 q = blocks[disks-1];
262
263 /* Compute syndrome with zero for the missing data pages
264 * Use the dead data pages as temporary storage for
265 * delta p and delta q
266 */
267 dp = blocks[faila];
268 blocks[faila] = NULL;
269 blocks[disks-2] = dp;
270 dq = blocks[failb];
271 blocks[failb] = NULL;
272 blocks[disks-1] = dq;
273
274 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
275 tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
276
277 /* Restore pointer table */
278 blocks[faila] = dp;
279 blocks[failb] = dq;
280 blocks[disks-2] = p;
281 blocks[disks-1] = q;
282
283 /* compute P + Pxy */
284 srcs[0] = dp;
285 srcs[1] = p;
286 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
287 NULL, NULL, scribble);
288 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
289
290 /* compute Q + Qxy */
291 srcs[0] = dq;
292 srcs[1] = q;
293 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
294 NULL, NULL, scribble);
295 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
296
297 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
298 srcs[0] = dp;
299 srcs[1] = dq;
300 coef[0] = raid6_gfexi[failb-faila];
301 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
302 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
303 tx = async_sum_product(dq, srcs, coef, bytes, submit);
304
305 /* Dy = P+Pxy+Dx */
306 srcs[0] = dp;
307 srcs[1] = dq;
308 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
309 cb_param, scribble);
310 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
311
312 return tx;
313}
314
315/**
316 * async_raid6_2data_recov - asynchronously calculate two missing data blocks
317 * @disks: number of disks in the RAID-6 array
318 * @bytes: block size
319 * @faila: first failed drive index
320 * @failb: second failed drive index
321 * @blocks: array of source pointers where the last two entries are p and q
322 * @submit: submission/completion modifiers
323 */
324struct dma_async_tx_descriptor *
325async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
326 struct page **blocks, struct async_submit_ctl *submit)
327{
328 void *scribble = submit->scribble;
329 int non_zero_srcs, i;
330
331 BUG_ON(faila == failb);
332 if (failb < faila)
333 swap(faila, failb);
334
335 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
336
337 /* if a dma resource is not available or a scribble buffer is not
338 * available punt to the synchronous path. In the 'dma not
339 * available' case be sure to use the scribble buffer to
340 * preserve the content of 'blocks' as the caller intended.
341 */
342 if (!async_dma_find_channel(DMA_PQ) || !scribble) {
343 void **ptrs = scribble ? scribble : (void **) blocks;
344
345 async_tx_quiesce(&submit->depend_tx);
346 for (i = 0; i < disks; i++)
347 if (blocks[i] == NULL)
348 ptrs[i] = (void *) raid6_empty_zero_page;
349 else
350 ptrs[i] = page_address(blocks[i]);
351
352 raid6_2data_recov(disks, bytes, faila, failb, ptrs);
353
354 async_tx_sync_epilog(submit);
355
356 return NULL;
357 }
358
359 non_zero_srcs = 0;
360 for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
361 if (blocks[i])
362 non_zero_srcs++;
363 switch (non_zero_srcs) {
364 case 0:
365 case 1:
366 /* There must be at least 2 sources - the failed devices. */
367 BUG();
368
369 case 2:
370 /* dma devices do not uniformly understand a zero source pq
371 * operation (in contrast to the synchronous case), so
372 * explicitly handle the special case of a 4 disk array with
373 * both data disks missing.
374 */
375 return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
376 case 3:
377 /* dma devices do not uniformly understand a single
378 * source pq operation (in contrast to the synchronous
379 * case), so explicitly handle the special case of a 5 disk
380 * array with 2 of 3 data disks missing.
381 */
382 return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
383 default:
384 return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
385 }
386}
387EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
388
389/**
390 * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
391 * @disks: number of disks in the RAID-6 array
392 * @bytes: block size
393 * @faila: failed drive index
394 * @blocks: array of source pointers where the last two entries are p and q
395 * @submit: submission/completion modifiers
396 */
397struct dma_async_tx_descriptor *
398async_raid6_datap_recov(int disks, size_t bytes, int faila,
399 struct page **blocks, struct async_submit_ctl *submit)
400{
401 struct dma_async_tx_descriptor *tx = NULL;
402 struct page *p, *q, *dq;
403 u8 coef;
404 enum async_tx_flags flags = submit->flags;
405 dma_async_tx_callback cb_fn = submit->cb_fn;
406 void *cb_param = submit->cb_param;
407 void *scribble = submit->scribble;
408 int good_srcs, good, i;
409 struct page *srcs[2];
410
411 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
412
413 /* if a dma resource is not available or a scribble buffer is not
414 * available punt to the synchronous path. In the 'dma not
415 * available' case be sure to use the scribble buffer to
416 * preserve the content of 'blocks' as the caller intended.
417 */
418 if (!async_dma_find_channel(DMA_PQ) || !scribble) {
419 void **ptrs = scribble ? scribble : (void **) blocks;
420
421 async_tx_quiesce(&submit->depend_tx);
422 for (i = 0; i < disks; i++)
423 if (blocks[i] == NULL)
424 ptrs[i] = (void*)raid6_empty_zero_page;
425 else
426 ptrs[i] = page_address(blocks[i]);
427
428 raid6_datap_recov(disks, bytes, faila, ptrs);
429
430 async_tx_sync_epilog(submit);
431
432 return NULL;
433 }
434
435 good_srcs = 0;
436 good = -1;
437 for (i = 0; i < disks-2; i++) {
438 if (i == faila)
439 continue;
440 if (blocks[i]) {
441 good = i;
442 good_srcs++;
443 if (good_srcs > 1)
444 break;
445 }
446 }
447 BUG_ON(good_srcs == 0);
448
449 p = blocks[disks-2];
450 q = blocks[disks-1];
451
452 /* Compute syndrome with zero for the missing data page
453 * Use the dead data page as temporary storage for delta q
454 */
455 dq = blocks[faila];
456 blocks[faila] = NULL;
457 blocks[disks-1] = dq;
458
459 /* in the 4-disk case we only need to perform a single source
460 * multiplication with the one good data block.
461 */
462 if (good_srcs == 1) {
463 struct page *g = blocks[good];
464
465 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
466 scribble);
467 tx = async_memcpy(p, g, 0, 0, bytes, submit);
468
469 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
470 scribble);
471 tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
472 } else {
473 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
474 scribble);
475 tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
476 }
477
478 /* Restore pointer table */
479 blocks[faila] = dq;
480 blocks[disks-1] = q;
481
482 /* calculate g^{-faila} */
483 coef = raid6_gfinv[raid6_gfexp[faila]];
484
485 srcs[0] = dq;
486 srcs[1] = q;
487 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
488 NULL, NULL, scribble);
489 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
490
491 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
492 tx = async_mult(dq, dq, coef, bytes, submit);
493
494 srcs[0] = p;
495 srcs[1] = dq;
496 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
497 cb_param, scribble);
498 tx = async_xor(p, srcs, 0, 2, bytes, submit);
499
500 return tx;
501}
502EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
503
504MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
505MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
506MODULE_LICENSE("GPL");