10a82a623SDan Williams /* 20a82a623SDan Williams * Asynchronous RAID-6 recovery calculations ASYNC_TX API. 30a82a623SDan Williams * Copyright(c) 2009 Intel Corporation 40a82a623SDan Williams * 50a82a623SDan Williams * based on raid6recov.c: 60a82a623SDan Williams * Copyright 2002 H. Peter Anvin 70a82a623SDan Williams * 80a82a623SDan Williams * This program is free software; you can redistribute it and/or modify it 90a82a623SDan Williams * under the terms of the GNU General Public License as published by the Free 100a82a623SDan Williams * Software Foundation; either version 2 of the License, or (at your option) 110a82a623SDan Williams * any later version. 120a82a623SDan Williams * 130a82a623SDan Williams * This program is distributed in the hope that it will be useful, but WITHOUT 140a82a623SDan Williams * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 150a82a623SDan Williams * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 160a82a623SDan Williams * more details. 170a82a623SDan Williams * 180a82a623SDan Williams * You should have received a copy of the GNU General Public License along with 190a82a623SDan Williams * this program; if not, write to the Free Software Foundation, Inc., 51 200a82a623SDan Williams * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 210a82a623SDan Williams * 220a82a623SDan Williams */ 230a82a623SDan Williams #include <linux/kernel.h> 240a82a623SDan Williams #include <linux/interrupt.h> 250a82a623SDan Williams #include <linux/dma-mapping.h> 260a82a623SDan Williams #include <linux/raid/pq.h> 270a82a623SDan Williams #include <linux/async_tx.h> 280a82a623SDan Williams 290a82a623SDan Williams static struct dma_async_tx_descriptor * 300a82a623SDan Williams async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, 310a82a623SDan Williams size_t len, struct async_submit_ctl *submit) 320a82a623SDan Williams { 330a82a623SDan Williams struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, 340a82a623SDan Williams &dest, 1, srcs, 2, len); 350a82a623SDan Williams struct dma_device *dma = chan ? chan->device : NULL; 360a82a623SDan Williams const u8 *amul, *bmul; 370a82a623SDan Williams u8 ax, bx; 380a82a623SDan Williams u8 *a, *b, *c; 390a82a623SDan Williams 400a82a623SDan Williams if (dma) { 410a82a623SDan Williams dma_addr_t dma_dest[2]; 420a82a623SDan Williams dma_addr_t dma_src[2]; 430a82a623SDan Williams struct device *dev = dma->dev; 440a82a623SDan Williams struct dma_async_tx_descriptor *tx; 450a82a623SDan Williams enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; 460a82a623SDan Williams 470403e382SDan Williams if (submit->flags & ASYNC_TX_FENCE) 480403e382SDan Williams dma_flags |= DMA_PREP_FENCE; 490a82a623SDan Williams dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); 500a82a623SDan Williams dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE); 510a82a623SDan Williams dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE); 520a82a623SDan Williams tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef, 530a82a623SDan Williams len, dma_flags); 540a82a623SDan Williams if (tx) { 550a82a623SDan Williams async_tx_submit(chan, tx, submit); 560a82a623SDan Williams return tx; 570a82a623SDan Williams } 581f6672d4SDan Williams 591f6672d4SDan Williams /* could not get a descriptor, unmap and fall through to 601f6672d4SDan Williams * the synchronous path 611f6672d4SDan Williams */ 621f6672d4SDan Williams dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL); 631f6672d4SDan Williams dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE); 641f6672d4SDan Williams dma_unmap_page(dev, dma_src[1], len, DMA_TO_DEVICE); 650a82a623SDan Williams } 660a82a623SDan Williams 670a82a623SDan Williams /* run the operation synchronously */ 680a82a623SDan Williams async_tx_quiesce(&submit->depend_tx); 690a82a623SDan Williams amul = raid6_gfmul[coef[0]]; 700a82a623SDan Williams bmul = raid6_gfmul[coef[1]]; 710a82a623SDan Williams a = page_address(srcs[0]); 720a82a623SDan Williams b = page_address(srcs[1]); 730a82a623SDan Williams c = page_address(dest); 740a82a623SDan Williams 750a82a623SDan Williams while (len--) { 760a82a623SDan Williams ax = amul[*a++]; 770a82a623SDan Williams bx = bmul[*b++]; 780a82a623SDan Williams *c++ = ax ^ bx; 790a82a623SDan Williams } 800a82a623SDan Williams 810a82a623SDan Williams return NULL; 820a82a623SDan Williams } 830a82a623SDan Williams 840a82a623SDan Williams static struct dma_async_tx_descriptor * 850a82a623SDan Williams async_mult(struct page *dest, struct page *src, u8 coef, size_t len, 860a82a623SDan Williams struct async_submit_ctl *submit) 870a82a623SDan Williams { 880a82a623SDan Williams struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, 890a82a623SDan Williams &dest, 1, &src, 1, len); 900a82a623SDan Williams struct dma_device *dma = chan ? chan->device : NULL; 910a82a623SDan Williams const u8 *qmul; /* Q multiplier table */ 920a82a623SDan Williams u8 *d, *s; 930a82a623SDan Williams 940a82a623SDan Williams if (dma) { 950a82a623SDan Williams dma_addr_t dma_dest[2]; 960a82a623SDan Williams dma_addr_t dma_src[1]; 970a82a623SDan Williams struct device *dev = dma->dev; 980a82a623SDan Williams struct dma_async_tx_descriptor *tx; 990a82a623SDan Williams enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; 1000a82a623SDan Williams 1010403e382SDan Williams if (submit->flags & ASYNC_TX_FENCE) 1020403e382SDan Williams dma_flags |= DMA_PREP_FENCE; 1030a82a623SDan Williams dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); 1040a82a623SDan Williams dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE); 1050a82a623SDan Williams tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef, 1060a82a623SDan Williams len, dma_flags); 1070a82a623SDan Williams if (tx) { 1080a82a623SDan Williams async_tx_submit(chan, tx, submit); 1090a82a623SDan Williams return tx; 1100a82a623SDan Williams } 1111f6672d4SDan Williams 1121f6672d4SDan Williams /* could not get a descriptor, unmap and fall through to 1131f6672d4SDan Williams * the synchronous path 1141f6672d4SDan Williams */ 1151f6672d4SDan Williams dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL); 1161f6672d4SDan Williams dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE); 1170a82a623SDan Williams } 1180a82a623SDan Williams 1190a82a623SDan Williams /* no channel available, or failed to allocate a descriptor, so 1200a82a623SDan Williams * perform the operation synchronously 1210a82a623SDan Williams */ 1220a82a623SDan Williams async_tx_quiesce(&submit->depend_tx); 1230a82a623SDan Williams qmul = raid6_gfmul[coef]; 1240a82a623SDan Williams d = page_address(dest); 1250a82a623SDan Williams s = page_address(src); 1260a82a623SDan Williams 1270a82a623SDan Williams while (len--) 1280a82a623SDan Williams *d++ = qmul[*s++]; 1290a82a623SDan Williams 1300a82a623SDan Williams return NULL; 1310a82a623SDan Williams } 1320a82a623SDan Williams 1330a82a623SDan Williams static struct dma_async_tx_descriptor * 134*da17bf43SDan Williams __2data_recov_4(int disks, size_t bytes, int faila, int failb, 135*da17bf43SDan Williams struct page **blocks, struct async_submit_ctl *submit) 1360a82a623SDan Williams { 1370a82a623SDan Williams struct dma_async_tx_descriptor *tx = NULL; 1380a82a623SDan Williams struct page *p, *q, *a, *b; 1390a82a623SDan Williams struct page *srcs[2]; 1400a82a623SDan Williams unsigned char coef[2]; 1410a82a623SDan Williams enum async_tx_flags flags = submit->flags; 1420a82a623SDan Williams dma_async_tx_callback cb_fn = submit->cb_fn; 1430a82a623SDan Williams void *cb_param = submit->cb_param; 1440a82a623SDan Williams void *scribble = submit->scribble; 1450a82a623SDan Williams 146*da17bf43SDan Williams p = blocks[disks-2]; 147*da17bf43SDan Williams q = blocks[disks-1]; 1480a82a623SDan Williams 1490a82a623SDan Williams a = blocks[faila]; 1500a82a623SDan Williams b = blocks[failb]; 1510a82a623SDan Williams 1520a82a623SDan Williams /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */ 1530a82a623SDan Williams /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ 1540a82a623SDan Williams srcs[0] = p; 1550a82a623SDan Williams srcs[1] = q; 1560a82a623SDan Williams coef[0] = raid6_gfexi[failb-faila]; 1570a82a623SDan Williams coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; 1580403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); 1590a82a623SDan Williams tx = async_sum_product(b, srcs, coef, bytes, submit); 1600a82a623SDan Williams 1610a82a623SDan Williams /* Dy = P+Pxy+Dx */ 1620a82a623SDan Williams srcs[0] = p; 1630a82a623SDan Williams srcs[1] = b; 1640a82a623SDan Williams init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn, 1650a82a623SDan Williams cb_param, scribble); 1660a82a623SDan Williams tx = async_xor(a, srcs, 0, 2, bytes, submit); 1670a82a623SDan Williams 1680a82a623SDan Williams return tx; 1690a82a623SDan Williams 1700a82a623SDan Williams } 1710a82a623SDan Williams 1720a82a623SDan Williams static struct dma_async_tx_descriptor * 173*da17bf43SDan Williams __2data_recov_5(int disks, size_t bytes, int faila, int failb, 174*da17bf43SDan Williams struct page **blocks, struct async_submit_ctl *submit) 1750a82a623SDan Williams { 1760a82a623SDan Williams struct dma_async_tx_descriptor *tx = NULL; 1770a82a623SDan Williams struct page *p, *q, *g, *dp, *dq; 1780a82a623SDan Williams struct page *srcs[2]; 1790a82a623SDan Williams unsigned char coef[2]; 1800a82a623SDan Williams enum async_tx_flags flags = submit->flags; 1810a82a623SDan Williams dma_async_tx_callback cb_fn = submit->cb_fn; 1820a82a623SDan Williams void *cb_param = submit->cb_param; 1830a82a623SDan Williams void *scribble = submit->scribble; 184*da17bf43SDan Williams int good_srcs, good, i; 1850a82a623SDan Williams 186*da17bf43SDan Williams good_srcs = 0; 187*da17bf43SDan Williams good = -1; 188*da17bf43SDan Williams for (i = 0; i < disks-2; i++) { 189*da17bf43SDan Williams if (blocks[i] == NULL) 190*da17bf43SDan Williams continue; 1910a82a623SDan Williams if (i == faila || i == failb) 1920a82a623SDan Williams continue; 1930a82a623SDan Williams good = i; 194*da17bf43SDan Williams good_srcs++; 1950a82a623SDan Williams } 196*da17bf43SDan Williams BUG_ON(good_srcs > 1); 1970a82a623SDan Williams 198*da17bf43SDan Williams p = blocks[disks-2]; 199*da17bf43SDan Williams q = blocks[disks-1]; 2000a82a623SDan Williams g = blocks[good]; 2010a82a623SDan Williams 2020a82a623SDan Williams /* Compute syndrome with zero for the missing data pages 2030a82a623SDan Williams * Use the dead data pages as temporary storage for delta p and 2040a82a623SDan Williams * delta q 2050a82a623SDan Williams */ 2060a82a623SDan Williams dp = blocks[faila]; 2070a82a623SDan Williams dq = blocks[failb]; 2080a82a623SDan Williams 2090403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); 2100a82a623SDan Williams tx = async_memcpy(dp, g, 0, 0, bytes, submit); 2110403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); 2120a82a623SDan Williams tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit); 2130a82a623SDan Williams 2140a82a623SDan Williams /* compute P + Pxy */ 2150a82a623SDan Williams srcs[0] = dp; 2160a82a623SDan Williams srcs[1] = p; 2170403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 2180403e382SDan Williams NULL, NULL, scribble); 2190a82a623SDan Williams tx = async_xor(dp, srcs, 0, 2, bytes, submit); 2200a82a623SDan Williams 2210a82a623SDan Williams /* compute Q + Qxy */ 2220a82a623SDan Williams srcs[0] = dq; 2230a82a623SDan Williams srcs[1] = q; 2240403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 2250403e382SDan Williams NULL, NULL, scribble); 2260a82a623SDan Williams tx = async_xor(dq, srcs, 0, 2, bytes, submit); 2270a82a623SDan Williams 2280a82a623SDan Williams /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ 2290a82a623SDan Williams srcs[0] = dp; 2300a82a623SDan Williams srcs[1] = dq; 2310a82a623SDan Williams coef[0] = raid6_gfexi[failb-faila]; 2320a82a623SDan Williams coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; 2330403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); 2340a82a623SDan Williams tx = async_sum_product(dq, srcs, coef, bytes, submit); 2350a82a623SDan Williams 2360a82a623SDan Williams /* Dy = P+Pxy+Dx */ 2370a82a623SDan Williams srcs[0] = dp; 2380a82a623SDan Williams srcs[1] = dq; 2390a82a623SDan Williams init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, 2400a82a623SDan Williams cb_param, scribble); 2410a82a623SDan Williams tx = async_xor(dp, srcs, 0, 2, bytes, submit); 2420a82a623SDan Williams 2430a82a623SDan Williams return tx; 2440a82a623SDan Williams } 2450a82a623SDan Williams 2460a82a623SDan Williams static struct dma_async_tx_descriptor * 2470a82a623SDan Williams __2data_recov_n(int disks, size_t bytes, int faila, int failb, 2480a82a623SDan Williams struct page **blocks, struct async_submit_ctl *submit) 2490a82a623SDan Williams { 2500a82a623SDan Williams struct dma_async_tx_descriptor *tx = NULL; 2510a82a623SDan Williams struct page *p, *q, *dp, *dq; 2520a82a623SDan Williams struct page *srcs[2]; 2530a82a623SDan Williams unsigned char coef[2]; 2540a82a623SDan Williams enum async_tx_flags flags = submit->flags; 2550a82a623SDan Williams dma_async_tx_callback cb_fn = submit->cb_fn; 2560a82a623SDan Williams void *cb_param = submit->cb_param; 2570a82a623SDan Williams void *scribble = submit->scribble; 2580a82a623SDan Williams 2590a82a623SDan Williams p = blocks[disks-2]; 2600a82a623SDan Williams q = blocks[disks-1]; 2610a82a623SDan Williams 2620a82a623SDan Williams /* Compute syndrome with zero for the missing data pages 2630a82a623SDan Williams * Use the dead data pages as temporary storage for 2640a82a623SDan Williams * delta p and delta q 2650a82a623SDan Williams */ 2660a82a623SDan Williams dp = blocks[faila]; 2675dd33c9aSNeilBrown blocks[faila] = NULL; 2680a82a623SDan Williams blocks[disks-2] = dp; 2690a82a623SDan Williams dq = blocks[failb]; 2705dd33c9aSNeilBrown blocks[failb] = NULL; 2710a82a623SDan Williams blocks[disks-1] = dq; 2720a82a623SDan Williams 2730403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); 2740a82a623SDan Williams tx = async_gen_syndrome(blocks, 0, disks, bytes, submit); 2750a82a623SDan Williams 2760a82a623SDan Williams /* Restore pointer table */ 2770a82a623SDan Williams blocks[faila] = dp; 2780a82a623SDan Williams blocks[failb] = dq; 2790a82a623SDan Williams blocks[disks-2] = p; 2800a82a623SDan Williams blocks[disks-1] = q; 2810a82a623SDan Williams 2820a82a623SDan Williams /* compute P + Pxy */ 2830a82a623SDan Williams srcs[0] = dp; 2840a82a623SDan Williams srcs[1] = p; 2850403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 2860403e382SDan Williams NULL, NULL, scribble); 2870a82a623SDan Williams tx = async_xor(dp, srcs, 0, 2, bytes, submit); 2880a82a623SDan Williams 2890a82a623SDan Williams /* compute Q + Qxy */ 2900a82a623SDan Williams srcs[0] = dq; 2910a82a623SDan Williams srcs[1] = q; 2920403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 2930403e382SDan Williams NULL, NULL, scribble); 2940a82a623SDan Williams tx = async_xor(dq, srcs, 0, 2, bytes, submit); 2950a82a623SDan Williams 2960a82a623SDan Williams /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ 2970a82a623SDan Williams srcs[0] = dp; 2980a82a623SDan Williams srcs[1] = dq; 2990a82a623SDan Williams coef[0] = raid6_gfexi[failb-faila]; 3000a82a623SDan Williams coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; 3010403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); 3020a82a623SDan Williams tx = async_sum_product(dq, srcs, coef, bytes, submit); 3030a82a623SDan Williams 3040a82a623SDan Williams /* Dy = P+Pxy+Dx */ 3050a82a623SDan Williams srcs[0] = dp; 3060a82a623SDan Williams srcs[1] = dq; 3070a82a623SDan Williams init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, 3080a82a623SDan Williams cb_param, scribble); 3090a82a623SDan Williams tx = async_xor(dp, srcs, 0, 2, bytes, submit); 3100a82a623SDan Williams 3110a82a623SDan Williams return tx; 3120a82a623SDan Williams } 3130a82a623SDan Williams 3140a82a623SDan Williams /** 3150a82a623SDan Williams * async_raid6_2data_recov - asynchronously calculate two missing data blocks 3160a82a623SDan Williams * @disks: number of disks in the RAID-6 array 3170a82a623SDan Williams * @bytes: block size 3180a82a623SDan Williams * @faila: first failed drive index 3190a82a623SDan Williams * @failb: second failed drive index 3200a82a623SDan Williams * @blocks: array of source pointers where the last two entries are p and q 3210a82a623SDan Williams * @submit: submission/completion modifiers 3220a82a623SDan Williams */ 3230a82a623SDan Williams struct dma_async_tx_descriptor * 3240a82a623SDan Williams async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, 3250a82a623SDan Williams struct page **blocks, struct async_submit_ctl *submit) 3260a82a623SDan Williams { 327*da17bf43SDan Williams int non_zero_srcs, i; 328*da17bf43SDan Williams 3290a82a623SDan Williams BUG_ON(faila == failb); 3300a82a623SDan Williams if (failb < faila) 3310a82a623SDan Williams swap(faila, failb); 3320a82a623SDan Williams 3330a82a623SDan Williams pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); 3340a82a623SDan Williams 3350a82a623SDan Williams /* we need to preserve the contents of 'blocks' for the async 3360a82a623SDan Williams * case, so punt to synchronous if a scribble buffer is not available 3370a82a623SDan Williams */ 3380a82a623SDan Williams if (!submit->scribble) { 3390a82a623SDan Williams void **ptrs = (void **) blocks; 3400a82a623SDan Williams 3410a82a623SDan Williams async_tx_quiesce(&submit->depend_tx); 3420a82a623SDan Williams for (i = 0; i < disks; i++) 3435dd33c9aSNeilBrown if (blocks[i] == NULL) 3445dd33c9aSNeilBrown ptrs[i] = (void *) raid6_empty_zero_page; 3455dd33c9aSNeilBrown else 3460a82a623SDan Williams ptrs[i] = page_address(blocks[i]); 3470a82a623SDan Williams 3480a82a623SDan Williams raid6_2data_recov(disks, bytes, faila, failb, ptrs); 3490a82a623SDan Williams 3500a82a623SDan Williams async_tx_sync_epilog(submit); 3510a82a623SDan Williams 3520a82a623SDan Williams return NULL; 3530a82a623SDan Williams } 3540a82a623SDan Williams 355*da17bf43SDan Williams non_zero_srcs = 0; 356*da17bf43SDan Williams for (i = 0; i < disks-2 && non_zero_srcs < 4; i++) 357*da17bf43SDan Williams if (blocks[i]) 358*da17bf43SDan Williams non_zero_srcs++; 359*da17bf43SDan Williams switch (non_zero_srcs) { 360*da17bf43SDan Williams case 0: 361*da17bf43SDan Williams case 1: 362*da17bf43SDan Williams /* There must be at least 2 sources - the failed devices. */ 363*da17bf43SDan Williams BUG(); 364*da17bf43SDan Williams 365*da17bf43SDan Williams case 2: 3660a82a623SDan Williams /* dma devices do not uniformly understand a zero source pq 3670a82a623SDan Williams * operation (in contrast to the synchronous case), so 368*da17bf43SDan Williams * explicitly handle the special case of a 4 disk array with 369*da17bf43SDan Williams * both data disks missing. 3700a82a623SDan Williams */ 371*da17bf43SDan Williams return __2data_recov_4(disks, bytes, faila, failb, blocks, submit); 372*da17bf43SDan Williams case 3: 3730a82a623SDan Williams /* dma devices do not uniformly understand a single 3740a82a623SDan Williams * source pq operation (in contrast to the synchronous 375*da17bf43SDan Williams * case), so explicitly handle the special case of a 5 disk 376*da17bf43SDan Williams * array with 2 of 3 data disks missing. 3770a82a623SDan Williams */ 378*da17bf43SDan Williams return __2data_recov_5(disks, bytes, faila, failb, blocks, submit); 3790a82a623SDan Williams default: 3800a82a623SDan Williams return __2data_recov_n(disks, bytes, faila, failb, blocks, submit); 3810a82a623SDan Williams } 3820a82a623SDan Williams } 3830a82a623SDan Williams EXPORT_SYMBOL_GPL(async_raid6_2data_recov); 3840a82a623SDan Williams 3850a82a623SDan Williams /** 3860a82a623SDan Williams * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block 3870a82a623SDan Williams * @disks: number of disks in the RAID-6 array 3880a82a623SDan Williams * @bytes: block size 3890a82a623SDan Williams * @faila: failed drive index 3900a82a623SDan Williams * @blocks: array of source pointers where the last two entries are p and q 3910a82a623SDan Williams * @submit: submission/completion modifiers 3920a82a623SDan Williams */ 3930a82a623SDan Williams struct dma_async_tx_descriptor * 3940a82a623SDan Williams async_raid6_datap_recov(int disks, size_t bytes, int faila, 3950a82a623SDan Williams struct page **blocks, struct async_submit_ctl *submit) 3960a82a623SDan Williams { 3970a82a623SDan Williams struct dma_async_tx_descriptor *tx = NULL; 3980a82a623SDan Williams struct page *p, *q, *dq; 3990a82a623SDan Williams u8 coef; 4000a82a623SDan Williams enum async_tx_flags flags = submit->flags; 4010a82a623SDan Williams dma_async_tx_callback cb_fn = submit->cb_fn; 4020a82a623SDan Williams void *cb_param = submit->cb_param; 4030a82a623SDan Williams void *scribble = submit->scribble; 404*da17bf43SDan Williams int good_srcs, good, i; 4050a82a623SDan Williams struct page *srcs[2]; 4060a82a623SDan Williams 4070a82a623SDan Williams pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); 4080a82a623SDan Williams 4090a82a623SDan Williams /* we need to preserve the contents of 'blocks' for the async 4100a82a623SDan Williams * case, so punt to synchronous if a scribble buffer is not available 4110a82a623SDan Williams */ 4120a82a623SDan Williams if (!scribble) { 4130a82a623SDan Williams void **ptrs = (void **) blocks; 4140a82a623SDan Williams 4150a82a623SDan Williams async_tx_quiesce(&submit->depend_tx); 4160a82a623SDan Williams for (i = 0; i < disks; i++) 4175dd33c9aSNeilBrown if (blocks[i] == NULL) 4185dd33c9aSNeilBrown ptrs[i] = (void*)raid6_empty_zero_page; 4195dd33c9aSNeilBrown else 4200a82a623SDan Williams ptrs[i] = page_address(blocks[i]); 4210a82a623SDan Williams 4220a82a623SDan Williams raid6_datap_recov(disks, bytes, faila, ptrs); 4230a82a623SDan Williams 4240a82a623SDan Williams async_tx_sync_epilog(submit); 4250a82a623SDan Williams 4260a82a623SDan Williams return NULL; 4270a82a623SDan Williams } 4280a82a623SDan Williams 429*da17bf43SDan Williams good_srcs = 0; 430*da17bf43SDan Williams good = -1; 431*da17bf43SDan Williams for (i = 0; i < disks-2; i++) { 432*da17bf43SDan Williams if (i == faila) 433*da17bf43SDan Williams continue; 434*da17bf43SDan Williams if (blocks[i]) { 435*da17bf43SDan Williams good = i; 436*da17bf43SDan Williams good_srcs++; 437*da17bf43SDan Williams if (good_srcs > 1) 438*da17bf43SDan Williams break; 439*da17bf43SDan Williams } 440*da17bf43SDan Williams } 441*da17bf43SDan Williams BUG_ON(good_srcs == 0); 442*da17bf43SDan Williams 4430a82a623SDan Williams p = blocks[disks-2]; 4440a82a623SDan Williams q = blocks[disks-1]; 4450a82a623SDan Williams 4460a82a623SDan Williams /* Compute syndrome with zero for the missing data page 4470a82a623SDan Williams * Use the dead data page as temporary storage for delta q 4480a82a623SDan Williams */ 4490a82a623SDan Williams dq = blocks[faila]; 4505dd33c9aSNeilBrown blocks[faila] = NULL; 4510a82a623SDan Williams blocks[disks-1] = dq; 4520a82a623SDan Williams 453*da17bf43SDan Williams /* in the 4-disk case we only need to perform a single source 454*da17bf43SDan Williams * multiplication with the one good data block. 4550a82a623SDan Williams */ 456*da17bf43SDan Williams if (good_srcs == 1) { 4570a82a623SDan Williams struct page *g = blocks[good]; 4580a82a623SDan Williams 4590403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, 4600403e382SDan Williams scribble); 4610a82a623SDan Williams tx = async_memcpy(p, g, 0, 0, bytes, submit); 4620a82a623SDan Williams 4630403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, 4640403e382SDan Williams scribble); 4650a82a623SDan Williams tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit); 4660a82a623SDan Williams } else { 4670403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, 4680403e382SDan Williams scribble); 4690a82a623SDan Williams tx = async_gen_syndrome(blocks, 0, disks, bytes, submit); 4700a82a623SDan Williams } 4710a82a623SDan Williams 4720a82a623SDan Williams /* Restore pointer table */ 4730a82a623SDan Williams blocks[faila] = dq; 4740a82a623SDan Williams blocks[disks-1] = q; 4750a82a623SDan Williams 4760a82a623SDan Williams /* calculate g^{-faila} */ 4770a82a623SDan Williams coef = raid6_gfinv[raid6_gfexp[faila]]; 4780a82a623SDan Williams 4790a82a623SDan Williams srcs[0] = dq; 4800a82a623SDan Williams srcs[1] = q; 4810403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 4820403e382SDan Williams NULL, NULL, scribble); 4830a82a623SDan Williams tx = async_xor(dq, srcs, 0, 2, bytes, submit); 4840a82a623SDan Williams 4850403e382SDan Williams init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); 4860a82a623SDan Williams tx = async_mult(dq, dq, coef, bytes, submit); 4870a82a623SDan Williams 4880a82a623SDan Williams srcs[0] = p; 4890a82a623SDan Williams srcs[1] = dq; 4900a82a623SDan Williams init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, 4910a82a623SDan Williams cb_param, scribble); 4920a82a623SDan Williams tx = async_xor(p, srcs, 0, 2, bytes, submit); 4930a82a623SDan Williams 4940a82a623SDan Williams return tx; 4950a82a623SDan Williams } 4960a82a623SDan Williams EXPORT_SYMBOL_GPL(async_raid6_datap_recov); 4970a82a623SDan Williams 4980a82a623SDan Williams MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>"); 4990a82a623SDan Williams MODULE_DESCRIPTION("asynchronous RAID-6 recovery api"); 5000a82a623SDan Williams MODULE_LICENSE("GPL"); 501