1599d49deSDave Jiang /* 2599d49deSDave Jiang * Intel I/OAT DMA Linux driver 3599d49deSDave Jiang * Copyright(c) 2004 - 2015 Intel Corporation. 4599d49deSDave Jiang * 5599d49deSDave Jiang * This program is free software; you can redistribute it and/or modify it 6599d49deSDave Jiang * under the terms and conditions of the GNU General Public License, 7599d49deSDave Jiang * version 2, as published by the Free Software Foundation. 8599d49deSDave Jiang * 9599d49deSDave Jiang * This program is distributed in the hope that it will be useful, but WITHOUT 10599d49deSDave Jiang * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11599d49deSDave Jiang * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12599d49deSDave Jiang * more details. 13599d49deSDave Jiang * 14599d49deSDave Jiang * The full GNU General Public License is included in this distribution in 15599d49deSDave Jiang * the file called "COPYING". 16599d49deSDave Jiang * 17599d49deSDave Jiang */ 18599d49deSDave Jiang #include <linux/module.h> 19599d49deSDave Jiang #include <linux/pci.h> 20599d49deSDave Jiang #include <linux/gfp.h> 21599d49deSDave Jiang #include <linux/dmaengine.h> 22599d49deSDave Jiang #include <linux/dma-mapping.h> 23599d49deSDave Jiang #include <linux/prefetch.h> 24599d49deSDave Jiang #include "../dmaengine.h" 25599d49deSDave Jiang #include "registers.h" 26599d49deSDave Jiang #include "hw.h" 27599d49deSDave Jiang #include "dma.h" 28599d49deSDave Jiang 297b7d0ca7SDave Jiang #define MAX_SCF 1024 307b7d0ca7SDave Jiang 31599d49deSDave Jiang /* provide a lookup table for setting the source address in the base or 32599d49deSDave Jiang * extended descriptor of an xor or pq descriptor 33599d49deSDave Jiang */ 34599d49deSDave Jiang static const u8 xor_idx_to_desc = 0xe0; 35599d49deSDave Jiang static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 }; 36599d49deSDave Jiang static const u8 pq_idx_to_desc = 0xf8; 37599d49deSDave Jiang static const u8 pq16_idx_to_desc[] = { 0, 0, 1, 1, 1, 1, 1, 1, 1, 38599d49deSDave Jiang 2, 2, 2, 2, 2, 2, 2 }; 39599d49deSDave Jiang static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 }; 40599d49deSDave Jiang static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7, 41599d49deSDave Jiang 0, 1, 2, 3, 4, 5, 6 }; 42599d49deSDave Jiang 43599d49deSDave Jiang static void xor_set_src(struct ioat_raw_descriptor *descs[2], 44599d49deSDave Jiang dma_addr_t addr, u32 offset, int idx) 45599d49deSDave Jiang { 46599d49deSDave Jiang struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1]; 47599d49deSDave Jiang 48599d49deSDave Jiang raw->field[xor_idx_to_field[idx]] = addr + offset; 49599d49deSDave Jiang } 50599d49deSDave Jiang 51599d49deSDave Jiang static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx) 52599d49deSDave Jiang { 53599d49deSDave Jiang struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1]; 54599d49deSDave Jiang 55599d49deSDave Jiang return raw->field[pq_idx_to_field[idx]]; 56599d49deSDave Jiang } 57599d49deSDave Jiang 58599d49deSDave Jiang static dma_addr_t pq16_get_src(struct ioat_raw_descriptor *desc[3], int idx) 59599d49deSDave Jiang { 60599d49deSDave Jiang struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]]; 61599d49deSDave Jiang 62599d49deSDave Jiang return raw->field[pq16_idx_to_field[idx]]; 63599d49deSDave Jiang } 64599d49deSDave Jiang 65599d49deSDave Jiang static void pq_set_src(struct ioat_raw_descriptor *descs[2], 66599d49deSDave Jiang dma_addr_t addr, u32 offset, u8 coef, int idx) 67599d49deSDave Jiang { 68599d49deSDave Jiang struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0]; 69599d49deSDave Jiang struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1]; 70599d49deSDave Jiang 71599d49deSDave Jiang raw->field[pq_idx_to_field[idx]] = addr + offset; 72599d49deSDave Jiang pq->coef[idx] = coef; 73599d49deSDave Jiang } 74599d49deSDave Jiang 75599d49deSDave Jiang static void pq16_set_src(struct ioat_raw_descriptor *desc[3], 76599d49deSDave Jiang dma_addr_t addr, u32 offset, u8 coef, unsigned idx) 77599d49deSDave Jiang { 78599d49deSDave Jiang struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0]; 79599d49deSDave Jiang struct ioat_pq16a_descriptor *pq16 = 80599d49deSDave Jiang (struct ioat_pq16a_descriptor *)desc[1]; 81599d49deSDave Jiang struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]]; 82599d49deSDave Jiang 83599d49deSDave Jiang raw->field[pq16_idx_to_field[idx]] = addr + offset; 84599d49deSDave Jiang 85599d49deSDave Jiang if (idx < 8) 86599d49deSDave Jiang pq->coef[idx] = coef; 87599d49deSDave Jiang else 88599d49deSDave Jiang pq16->coef[idx - 8] = coef; 89599d49deSDave Jiang } 90599d49deSDave Jiang 91599d49deSDave Jiang static struct ioat_sed_ent * 92599d49deSDave Jiang ioat3_alloc_sed(struct ioatdma_device *ioat_dma, unsigned int hw_pool) 93599d49deSDave Jiang { 94599d49deSDave Jiang struct ioat_sed_ent *sed; 95599d49deSDave Jiang gfp_t flags = __GFP_ZERO | GFP_ATOMIC; 96599d49deSDave Jiang 97599d49deSDave Jiang sed = kmem_cache_alloc(ioat_sed_cache, flags); 98599d49deSDave Jiang if (!sed) 99599d49deSDave Jiang return NULL; 100599d49deSDave Jiang 101599d49deSDave Jiang sed->hw_pool = hw_pool; 102599d49deSDave Jiang sed->hw = dma_pool_alloc(ioat_dma->sed_hw_pool[hw_pool], 103599d49deSDave Jiang flags, &sed->dma); 104599d49deSDave Jiang if (!sed->hw) { 105599d49deSDave Jiang kmem_cache_free(ioat_sed_cache, sed); 106599d49deSDave Jiang return NULL; 107599d49deSDave Jiang } 108599d49deSDave Jiang 109599d49deSDave Jiang return sed; 110599d49deSDave Jiang } 111599d49deSDave Jiang 112599d49deSDave Jiang struct dma_async_tx_descriptor * 113599d49deSDave Jiang ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, 114599d49deSDave Jiang dma_addr_t dma_src, size_t len, unsigned long flags) 115599d49deSDave Jiang { 116599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 117599d49deSDave Jiang struct ioat_dma_descriptor *hw; 118599d49deSDave Jiang struct ioat_ring_ent *desc; 119599d49deSDave Jiang dma_addr_t dst = dma_dest; 120599d49deSDave Jiang dma_addr_t src = dma_src; 121599d49deSDave Jiang size_t total_len = len; 122599d49deSDave Jiang int num_descs, idx, i; 123599d49deSDave Jiang 124599d49deSDave Jiang num_descs = ioat_xferlen_to_descs(ioat_chan, len); 125599d49deSDave Jiang if (likely(num_descs) && 126599d49deSDave Jiang ioat_check_space_lock(ioat_chan, num_descs) == 0) 127599d49deSDave Jiang idx = ioat_chan->head; 128599d49deSDave Jiang else 129599d49deSDave Jiang return NULL; 130599d49deSDave Jiang i = 0; 131599d49deSDave Jiang do { 132599d49deSDave Jiang size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log); 133599d49deSDave Jiang 134599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, idx + i); 135599d49deSDave Jiang hw = desc->hw; 136599d49deSDave Jiang 137599d49deSDave Jiang hw->size = copy; 138599d49deSDave Jiang hw->ctl = 0; 139599d49deSDave Jiang hw->src_addr = src; 140599d49deSDave Jiang hw->dst_addr = dst; 141599d49deSDave Jiang 142599d49deSDave Jiang len -= copy; 143599d49deSDave Jiang dst += copy; 144599d49deSDave Jiang src += copy; 145599d49deSDave Jiang dump_desc_dbg(ioat_chan, desc); 146599d49deSDave Jiang } while (++i < num_descs); 147599d49deSDave Jiang 148599d49deSDave Jiang desc->txd.flags = flags; 149599d49deSDave Jiang desc->len = total_len; 150599d49deSDave Jiang hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 151599d49deSDave Jiang hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 152599d49deSDave Jiang hw->ctl_f.compl_write = 1; 153599d49deSDave Jiang dump_desc_dbg(ioat_chan, desc); 154599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 155599d49deSDave Jiang 156599d49deSDave Jiang return &desc->txd; 157599d49deSDave Jiang } 158599d49deSDave Jiang 159599d49deSDave Jiang 160599d49deSDave Jiang static struct dma_async_tx_descriptor * 161599d49deSDave Jiang __ioat_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, 162599d49deSDave Jiang dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt, 163599d49deSDave Jiang size_t len, unsigned long flags) 164599d49deSDave Jiang { 165599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 166599d49deSDave Jiang struct ioat_ring_ent *compl_desc; 167599d49deSDave Jiang struct ioat_ring_ent *desc; 168599d49deSDave Jiang struct ioat_ring_ent *ext; 169599d49deSDave Jiang size_t total_len = len; 170599d49deSDave Jiang struct ioat_xor_descriptor *xor; 171599d49deSDave Jiang struct ioat_xor_ext_descriptor *xor_ex = NULL; 172599d49deSDave Jiang struct ioat_dma_descriptor *hw; 173599d49deSDave Jiang int num_descs, with_ext, idx, i; 174599d49deSDave Jiang u32 offset = 0; 175599d49deSDave Jiang u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR; 176599d49deSDave Jiang 177599d49deSDave Jiang BUG_ON(src_cnt < 2); 178599d49deSDave Jiang 179599d49deSDave Jiang num_descs = ioat_xferlen_to_descs(ioat_chan, len); 180599d49deSDave Jiang /* we need 2x the number of descriptors to cover greater than 5 181599d49deSDave Jiang * sources 182599d49deSDave Jiang */ 183599d49deSDave Jiang if (src_cnt > 5) { 184599d49deSDave Jiang with_ext = 1; 185599d49deSDave Jiang num_descs *= 2; 186599d49deSDave Jiang } else 187599d49deSDave Jiang with_ext = 0; 188599d49deSDave Jiang 189599d49deSDave Jiang /* completion writes from the raid engine may pass completion 190599d49deSDave Jiang * writes from the legacy engine, so we need one extra null 191599d49deSDave Jiang * (legacy) descriptor to ensure all completion writes arrive in 192599d49deSDave Jiang * order. 193599d49deSDave Jiang */ 194599d49deSDave Jiang if (likely(num_descs) && 195599d49deSDave Jiang ioat_check_space_lock(ioat_chan, num_descs+1) == 0) 196599d49deSDave Jiang idx = ioat_chan->head; 197599d49deSDave Jiang else 198599d49deSDave Jiang return NULL; 199599d49deSDave Jiang i = 0; 200599d49deSDave Jiang do { 201599d49deSDave Jiang struct ioat_raw_descriptor *descs[2]; 202599d49deSDave Jiang size_t xfer_size = min_t(size_t, 203599d49deSDave Jiang len, 1 << ioat_chan->xfercap_log); 204599d49deSDave Jiang int s; 205599d49deSDave Jiang 206599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, idx + i); 207599d49deSDave Jiang xor = desc->xor; 208599d49deSDave Jiang 209599d49deSDave Jiang /* save a branch by unconditionally retrieving the 210599d49deSDave Jiang * extended descriptor xor_set_src() knows to not write 211599d49deSDave Jiang * to it in the single descriptor case 212599d49deSDave Jiang */ 213599d49deSDave Jiang ext = ioat_get_ring_ent(ioat_chan, idx + i + 1); 214599d49deSDave Jiang xor_ex = ext->xor_ex; 215599d49deSDave Jiang 216599d49deSDave Jiang descs[0] = (struct ioat_raw_descriptor *) xor; 217599d49deSDave Jiang descs[1] = (struct ioat_raw_descriptor *) xor_ex; 218599d49deSDave Jiang for (s = 0; s < src_cnt; s++) 219599d49deSDave Jiang xor_set_src(descs, src[s], offset, s); 220599d49deSDave Jiang xor->size = xfer_size; 221599d49deSDave Jiang xor->dst_addr = dest + offset; 222599d49deSDave Jiang xor->ctl = 0; 223599d49deSDave Jiang xor->ctl_f.op = op; 224599d49deSDave Jiang xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt); 225599d49deSDave Jiang 226599d49deSDave Jiang len -= xfer_size; 227599d49deSDave Jiang offset += xfer_size; 228599d49deSDave Jiang dump_desc_dbg(ioat_chan, desc); 229599d49deSDave Jiang } while ((i += 1 + with_ext) < num_descs); 230599d49deSDave Jiang 231599d49deSDave Jiang /* last xor descriptor carries the unmap parameters and fence bit */ 232599d49deSDave Jiang desc->txd.flags = flags; 233599d49deSDave Jiang desc->len = total_len; 234599d49deSDave Jiang if (result) 235599d49deSDave Jiang desc->result = result; 236599d49deSDave Jiang xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 237599d49deSDave Jiang 238599d49deSDave Jiang /* completion descriptor carries interrupt bit */ 239599d49deSDave Jiang compl_desc = ioat_get_ring_ent(ioat_chan, idx + i); 240599d49deSDave Jiang compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; 241599d49deSDave Jiang hw = compl_desc->hw; 242599d49deSDave Jiang hw->ctl = 0; 243599d49deSDave Jiang hw->ctl_f.null = 1; 244599d49deSDave Jiang hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 245599d49deSDave Jiang hw->ctl_f.compl_write = 1; 246599d49deSDave Jiang hw->size = NULL_DESC_BUFFER_SIZE; 247599d49deSDave Jiang dump_desc_dbg(ioat_chan, compl_desc); 248599d49deSDave Jiang 249599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 250599d49deSDave Jiang return &compl_desc->txd; 251599d49deSDave Jiang } 252599d49deSDave Jiang 253599d49deSDave Jiang struct dma_async_tx_descriptor * 254599d49deSDave Jiang ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 255599d49deSDave Jiang unsigned int src_cnt, size_t len, unsigned long flags) 256599d49deSDave Jiang { 257599d49deSDave Jiang return __ioat_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags); 258599d49deSDave Jiang } 259599d49deSDave Jiang 260599d49deSDave Jiang struct dma_async_tx_descriptor * 261599d49deSDave Jiang ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src, 262599d49deSDave Jiang unsigned int src_cnt, size_t len, 263599d49deSDave Jiang enum sum_check_flags *result, unsigned long flags) 264599d49deSDave Jiang { 265599d49deSDave Jiang /* the cleanup routine only sets bits on validate failure, it 266599d49deSDave Jiang * does not clear bits on validate success... so clear it here 267599d49deSDave Jiang */ 268599d49deSDave Jiang *result = 0; 269599d49deSDave Jiang 270599d49deSDave Jiang return __ioat_prep_xor_lock(chan, result, src[0], &src[1], 271599d49deSDave Jiang src_cnt - 1, len, flags); 272599d49deSDave Jiang } 273599d49deSDave Jiang 274599d49deSDave Jiang static void 275599d49deSDave Jiang dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc, 276599d49deSDave Jiang struct ioat_ring_ent *ext) 277599d49deSDave Jiang { 278599d49deSDave Jiang struct device *dev = to_dev(ioat_chan); 279599d49deSDave Jiang struct ioat_pq_descriptor *pq = desc->pq; 280599d49deSDave Jiang struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL; 281599d49deSDave Jiang struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex }; 282599d49deSDave Jiang int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt); 283599d49deSDave Jiang int i; 284599d49deSDave Jiang 285599d49deSDave Jiang dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x" 286599d49deSDave Jiang " sz: %#10.8x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'" 287599d49deSDave Jiang " src_cnt: %d)\n", 288599d49deSDave Jiang desc_id(desc), (unsigned long long) desc->txd.phys, 289599d49deSDave Jiang (unsigned long long) (pq_ex ? pq_ex->next : pq->next), 290599d49deSDave Jiang desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, 291599d49deSDave Jiang pq->ctl_f.int_en, pq->ctl_f.compl_write, 292599d49deSDave Jiang pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q", 293599d49deSDave Jiang pq->ctl_f.src_cnt); 294599d49deSDave Jiang for (i = 0; i < src_cnt; i++) 295599d49deSDave Jiang dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i, 296599d49deSDave Jiang (unsigned long long) pq_get_src(descs, i), pq->coef[i]); 297599d49deSDave Jiang dev_dbg(dev, "\tP: %#llx\n", pq->p_addr); 298599d49deSDave Jiang dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr); 299599d49deSDave Jiang dev_dbg(dev, "\tNEXT: %#llx\n", pq->next); 300599d49deSDave Jiang } 301599d49deSDave Jiang 302599d49deSDave Jiang static void dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan, 303599d49deSDave Jiang struct ioat_ring_ent *desc) 304599d49deSDave Jiang { 305599d49deSDave Jiang struct device *dev = to_dev(ioat_chan); 306599d49deSDave Jiang struct ioat_pq_descriptor *pq = desc->pq; 307599d49deSDave Jiang struct ioat_raw_descriptor *descs[] = { (void *)pq, 308599d49deSDave Jiang (void *)pq, 309599d49deSDave Jiang (void *)pq }; 310599d49deSDave Jiang int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt); 311599d49deSDave Jiang int i; 312599d49deSDave Jiang 313599d49deSDave Jiang if (desc->sed) { 314599d49deSDave Jiang descs[1] = (void *)desc->sed->hw; 315599d49deSDave Jiang descs[2] = (void *)desc->sed->hw + 64; 316599d49deSDave Jiang } 317599d49deSDave Jiang 318599d49deSDave Jiang dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x" 319599d49deSDave Jiang " sz: %#x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'" 320599d49deSDave Jiang " src_cnt: %d)\n", 321599d49deSDave Jiang desc_id(desc), (unsigned long long) desc->txd.phys, 322599d49deSDave Jiang (unsigned long long) pq->next, 323599d49deSDave Jiang desc->txd.flags, pq->size, pq->ctl, 324599d49deSDave Jiang pq->ctl_f.op, pq->ctl_f.int_en, 325599d49deSDave Jiang pq->ctl_f.compl_write, 326599d49deSDave Jiang pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q", 327599d49deSDave Jiang pq->ctl_f.src_cnt); 328599d49deSDave Jiang for (i = 0; i < src_cnt; i++) { 329599d49deSDave Jiang dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i, 330599d49deSDave Jiang (unsigned long long) pq16_get_src(descs, i), 331599d49deSDave Jiang pq->coef[i]); 332599d49deSDave Jiang } 333599d49deSDave Jiang dev_dbg(dev, "\tP: %#llx\n", pq->p_addr); 334599d49deSDave Jiang dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr); 335599d49deSDave Jiang } 336599d49deSDave Jiang 337599d49deSDave Jiang static struct dma_async_tx_descriptor * 338599d49deSDave Jiang __ioat_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, 339599d49deSDave Jiang const dma_addr_t *dst, const dma_addr_t *src, 340599d49deSDave Jiang unsigned int src_cnt, const unsigned char *scf, 341599d49deSDave Jiang size_t len, unsigned long flags) 342599d49deSDave Jiang { 343599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 344599d49deSDave Jiang struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; 345599d49deSDave Jiang struct ioat_ring_ent *compl_desc; 346599d49deSDave Jiang struct ioat_ring_ent *desc; 347599d49deSDave Jiang struct ioat_ring_ent *ext; 348599d49deSDave Jiang size_t total_len = len; 349599d49deSDave Jiang struct ioat_pq_descriptor *pq; 350599d49deSDave Jiang struct ioat_pq_ext_descriptor *pq_ex = NULL; 351599d49deSDave Jiang struct ioat_dma_descriptor *hw; 352599d49deSDave Jiang u32 offset = 0; 353599d49deSDave Jiang u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ; 354599d49deSDave Jiang int i, s, idx, with_ext, num_descs; 355599d49deSDave Jiang int cb32 = (ioat_dma->version < IOAT_VER_3_3) ? 1 : 0; 356599d49deSDave Jiang 357599d49deSDave Jiang dev_dbg(to_dev(ioat_chan), "%s\n", __func__); 358599d49deSDave Jiang /* the engine requires at least two sources (we provide 359599d49deSDave Jiang * at least 1 implied source in the DMA_PREP_CONTINUE case) 360599d49deSDave Jiang */ 361599d49deSDave Jiang BUG_ON(src_cnt + dmaf_continue(flags) < 2); 362599d49deSDave Jiang 363599d49deSDave Jiang num_descs = ioat_xferlen_to_descs(ioat_chan, len); 364599d49deSDave Jiang /* we need 2x the number of descriptors to cover greater than 3 365599d49deSDave Jiang * sources (we need 1 extra source in the q-only continuation 366599d49deSDave Jiang * case and 3 extra sources in the p+q continuation case. 367599d49deSDave Jiang */ 368599d49deSDave Jiang if (src_cnt + dmaf_p_disabled_continue(flags) > 3 || 369599d49deSDave Jiang (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) { 370599d49deSDave Jiang with_ext = 1; 371599d49deSDave Jiang num_descs *= 2; 372599d49deSDave Jiang } else 373599d49deSDave Jiang with_ext = 0; 374599d49deSDave Jiang 375599d49deSDave Jiang /* completion writes from the raid engine may pass completion 376599d49deSDave Jiang * writes from the legacy engine, so we need one extra null 377599d49deSDave Jiang * (legacy) descriptor to ensure all completion writes arrive in 378599d49deSDave Jiang * order. 379599d49deSDave Jiang */ 380599d49deSDave Jiang if (likely(num_descs) && 381599d49deSDave Jiang ioat_check_space_lock(ioat_chan, num_descs + cb32) == 0) 382599d49deSDave Jiang idx = ioat_chan->head; 383599d49deSDave Jiang else 384599d49deSDave Jiang return NULL; 385599d49deSDave Jiang i = 0; 386599d49deSDave Jiang do { 387599d49deSDave Jiang struct ioat_raw_descriptor *descs[2]; 388599d49deSDave Jiang size_t xfer_size = min_t(size_t, len, 389599d49deSDave Jiang 1 << ioat_chan->xfercap_log); 390599d49deSDave Jiang 391599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, idx + i); 392599d49deSDave Jiang pq = desc->pq; 393599d49deSDave Jiang 394599d49deSDave Jiang /* save a branch by unconditionally retrieving the 395599d49deSDave Jiang * extended descriptor pq_set_src() knows to not write 396599d49deSDave Jiang * to it in the single descriptor case 397599d49deSDave Jiang */ 398599d49deSDave Jiang ext = ioat_get_ring_ent(ioat_chan, idx + i + with_ext); 399599d49deSDave Jiang pq_ex = ext->pq_ex; 400599d49deSDave Jiang 401599d49deSDave Jiang descs[0] = (struct ioat_raw_descriptor *) pq; 402599d49deSDave Jiang descs[1] = (struct ioat_raw_descriptor *) pq_ex; 403599d49deSDave Jiang 404599d49deSDave Jiang for (s = 0; s < src_cnt; s++) 405599d49deSDave Jiang pq_set_src(descs, src[s], offset, scf[s], s); 406599d49deSDave Jiang 407599d49deSDave Jiang /* see the comment for dma_maxpq in include/linux/dmaengine.h */ 408599d49deSDave Jiang if (dmaf_p_disabled_continue(flags)) 409599d49deSDave Jiang pq_set_src(descs, dst[1], offset, 1, s++); 410599d49deSDave Jiang else if (dmaf_continue(flags)) { 411599d49deSDave Jiang pq_set_src(descs, dst[0], offset, 0, s++); 412599d49deSDave Jiang pq_set_src(descs, dst[1], offset, 1, s++); 413599d49deSDave Jiang pq_set_src(descs, dst[1], offset, 0, s++); 414599d49deSDave Jiang } 415599d49deSDave Jiang pq->size = xfer_size; 416599d49deSDave Jiang pq->p_addr = dst[0] + offset; 417599d49deSDave Jiang pq->q_addr = dst[1] + offset; 418599d49deSDave Jiang pq->ctl = 0; 419599d49deSDave Jiang pq->ctl_f.op = op; 420599d49deSDave Jiang /* we turn on descriptor write back error status */ 421599d49deSDave Jiang if (ioat_dma->cap & IOAT_CAP_DWBES) 422599d49deSDave Jiang pq->ctl_f.wb_en = result ? 1 : 0; 423599d49deSDave Jiang pq->ctl_f.src_cnt = src_cnt_to_hw(s); 424599d49deSDave Jiang pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P); 425599d49deSDave Jiang pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q); 426599d49deSDave Jiang 427599d49deSDave Jiang len -= xfer_size; 428599d49deSDave Jiang offset += xfer_size; 429599d49deSDave Jiang } while ((i += 1 + with_ext) < num_descs); 430599d49deSDave Jiang 431599d49deSDave Jiang /* last pq descriptor carries the unmap parameters and fence bit */ 432599d49deSDave Jiang desc->txd.flags = flags; 433599d49deSDave Jiang desc->len = total_len; 434599d49deSDave Jiang if (result) 435599d49deSDave Jiang desc->result = result; 436599d49deSDave Jiang pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 437599d49deSDave Jiang dump_pq_desc_dbg(ioat_chan, desc, ext); 438599d49deSDave Jiang 439599d49deSDave Jiang if (!cb32) { 440599d49deSDave Jiang pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 441599d49deSDave Jiang pq->ctl_f.compl_write = 1; 442599d49deSDave Jiang compl_desc = desc; 443599d49deSDave Jiang } else { 444599d49deSDave Jiang /* completion descriptor carries interrupt bit */ 445599d49deSDave Jiang compl_desc = ioat_get_ring_ent(ioat_chan, idx + i); 446599d49deSDave Jiang compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; 447599d49deSDave Jiang hw = compl_desc->hw; 448599d49deSDave Jiang hw->ctl = 0; 449599d49deSDave Jiang hw->ctl_f.null = 1; 450599d49deSDave Jiang hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 451599d49deSDave Jiang hw->ctl_f.compl_write = 1; 452599d49deSDave Jiang hw->size = NULL_DESC_BUFFER_SIZE; 453599d49deSDave Jiang dump_desc_dbg(ioat_chan, compl_desc); 454599d49deSDave Jiang } 455599d49deSDave Jiang 456599d49deSDave Jiang 457599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 458599d49deSDave Jiang return &compl_desc->txd; 459599d49deSDave Jiang } 460599d49deSDave Jiang 461599d49deSDave Jiang static struct dma_async_tx_descriptor * 462599d49deSDave Jiang __ioat_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, 463599d49deSDave Jiang const dma_addr_t *dst, const dma_addr_t *src, 464599d49deSDave Jiang unsigned int src_cnt, const unsigned char *scf, 465599d49deSDave Jiang size_t len, unsigned long flags) 466599d49deSDave Jiang { 467599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 468599d49deSDave Jiang struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; 469599d49deSDave Jiang struct ioat_ring_ent *desc; 470599d49deSDave Jiang size_t total_len = len; 471599d49deSDave Jiang struct ioat_pq_descriptor *pq; 472599d49deSDave Jiang u32 offset = 0; 473599d49deSDave Jiang u8 op; 474599d49deSDave Jiang int i, s, idx, num_descs; 475599d49deSDave Jiang 476599d49deSDave Jiang /* this function is only called with 9-16 sources */ 477599d49deSDave Jiang op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S; 478599d49deSDave Jiang 479599d49deSDave Jiang dev_dbg(to_dev(ioat_chan), "%s\n", __func__); 480599d49deSDave Jiang 481599d49deSDave Jiang num_descs = ioat_xferlen_to_descs(ioat_chan, len); 482599d49deSDave Jiang 483599d49deSDave Jiang /* 484599d49deSDave Jiang * 16 source pq is only available on cb3.3 and has no completion 485599d49deSDave Jiang * write hw bug. 486599d49deSDave Jiang */ 487599d49deSDave Jiang if (num_descs && ioat_check_space_lock(ioat_chan, num_descs) == 0) 488599d49deSDave Jiang idx = ioat_chan->head; 489599d49deSDave Jiang else 490599d49deSDave Jiang return NULL; 491599d49deSDave Jiang 492599d49deSDave Jiang i = 0; 493599d49deSDave Jiang 494599d49deSDave Jiang do { 495599d49deSDave Jiang struct ioat_raw_descriptor *descs[4]; 496599d49deSDave Jiang size_t xfer_size = min_t(size_t, len, 497599d49deSDave Jiang 1 << ioat_chan->xfercap_log); 498599d49deSDave Jiang 499599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, idx + i); 500599d49deSDave Jiang pq = desc->pq; 501599d49deSDave Jiang 502599d49deSDave Jiang descs[0] = (struct ioat_raw_descriptor *) pq; 503599d49deSDave Jiang 504599d49deSDave Jiang desc->sed = ioat3_alloc_sed(ioat_dma, (src_cnt-2) >> 3); 505599d49deSDave Jiang if (!desc->sed) { 506599d49deSDave Jiang dev_err(to_dev(ioat_chan), 507599d49deSDave Jiang "%s: no free sed entries\n", __func__); 508599d49deSDave Jiang return NULL; 509599d49deSDave Jiang } 510599d49deSDave Jiang 511599d49deSDave Jiang pq->sed_addr = desc->sed->dma; 512599d49deSDave Jiang desc->sed->parent = desc; 513599d49deSDave Jiang 514599d49deSDave Jiang descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw; 515599d49deSDave Jiang descs[2] = (void *)descs[1] + 64; 516599d49deSDave Jiang 517599d49deSDave Jiang for (s = 0; s < src_cnt; s++) 518599d49deSDave Jiang pq16_set_src(descs, src[s], offset, scf[s], s); 519599d49deSDave Jiang 520599d49deSDave Jiang /* see the comment for dma_maxpq in include/linux/dmaengine.h */ 521599d49deSDave Jiang if (dmaf_p_disabled_continue(flags)) 522599d49deSDave Jiang pq16_set_src(descs, dst[1], offset, 1, s++); 523599d49deSDave Jiang else if (dmaf_continue(flags)) { 524599d49deSDave Jiang pq16_set_src(descs, dst[0], offset, 0, s++); 525599d49deSDave Jiang pq16_set_src(descs, dst[1], offset, 1, s++); 526599d49deSDave Jiang pq16_set_src(descs, dst[1], offset, 0, s++); 527599d49deSDave Jiang } 528599d49deSDave Jiang 529599d49deSDave Jiang pq->size = xfer_size; 530599d49deSDave Jiang pq->p_addr = dst[0] + offset; 531599d49deSDave Jiang pq->q_addr = dst[1] + offset; 532599d49deSDave Jiang pq->ctl = 0; 533599d49deSDave Jiang pq->ctl_f.op = op; 534599d49deSDave Jiang pq->ctl_f.src_cnt = src16_cnt_to_hw(s); 535599d49deSDave Jiang /* we turn on descriptor write back error status */ 536599d49deSDave Jiang if (ioat_dma->cap & IOAT_CAP_DWBES) 537599d49deSDave Jiang pq->ctl_f.wb_en = result ? 1 : 0; 538599d49deSDave Jiang pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P); 539599d49deSDave Jiang pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q); 540599d49deSDave Jiang 541599d49deSDave Jiang len -= xfer_size; 542599d49deSDave Jiang offset += xfer_size; 543599d49deSDave Jiang } while (++i < num_descs); 544599d49deSDave Jiang 545599d49deSDave Jiang /* last pq descriptor carries the unmap parameters and fence bit */ 546599d49deSDave Jiang desc->txd.flags = flags; 547599d49deSDave Jiang desc->len = total_len; 548599d49deSDave Jiang if (result) 549599d49deSDave Jiang desc->result = result; 550599d49deSDave Jiang pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 551599d49deSDave Jiang 552599d49deSDave Jiang /* with cb3.3 we should be able to do completion w/o a null desc */ 553599d49deSDave Jiang pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 554599d49deSDave Jiang pq->ctl_f.compl_write = 1; 555599d49deSDave Jiang 556599d49deSDave Jiang dump_pq16_desc_dbg(ioat_chan, desc); 557599d49deSDave Jiang 558599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 559599d49deSDave Jiang return &desc->txd; 560599d49deSDave Jiang } 561599d49deSDave Jiang 562599d49deSDave Jiang static int src_cnt_flags(unsigned int src_cnt, unsigned long flags) 563599d49deSDave Jiang { 564599d49deSDave Jiang if (dmaf_p_disabled_continue(flags)) 565599d49deSDave Jiang return src_cnt + 1; 566599d49deSDave Jiang else if (dmaf_continue(flags)) 567599d49deSDave Jiang return src_cnt + 3; 568599d49deSDave Jiang else 569599d49deSDave Jiang return src_cnt; 570599d49deSDave Jiang } 571599d49deSDave Jiang 572599d49deSDave Jiang struct dma_async_tx_descriptor * 573599d49deSDave Jiang ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 574599d49deSDave Jiang unsigned int src_cnt, const unsigned char *scf, size_t len, 575599d49deSDave Jiang unsigned long flags) 576599d49deSDave Jiang { 577599d49deSDave Jiang /* specify valid address for disabled result */ 578599d49deSDave Jiang if (flags & DMA_PREP_PQ_DISABLE_P) 579599d49deSDave Jiang dst[0] = dst[1]; 580599d49deSDave Jiang if (flags & DMA_PREP_PQ_DISABLE_Q) 581599d49deSDave Jiang dst[1] = dst[0]; 582599d49deSDave Jiang 583599d49deSDave Jiang /* handle the single source multiply case from the raid6 584599d49deSDave Jiang * recovery path 585599d49deSDave Jiang */ 586599d49deSDave Jiang if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) { 587599d49deSDave Jiang dma_addr_t single_source[2]; 588599d49deSDave Jiang unsigned char single_source_coef[2]; 589599d49deSDave Jiang 590599d49deSDave Jiang BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q); 591599d49deSDave Jiang single_source[0] = src[0]; 592599d49deSDave Jiang single_source[1] = src[0]; 593599d49deSDave Jiang single_source_coef[0] = scf[0]; 594599d49deSDave Jiang single_source_coef[1] = 0; 595599d49deSDave Jiang 596599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 597599d49deSDave Jiang __ioat_prep_pq16_lock(chan, NULL, dst, single_source, 598599d49deSDave Jiang 2, single_source_coef, len, 599599d49deSDave Jiang flags) : 600599d49deSDave Jiang __ioat_prep_pq_lock(chan, NULL, dst, single_source, 2, 601599d49deSDave Jiang single_source_coef, len, flags); 602599d49deSDave Jiang 603599d49deSDave Jiang } else { 604599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 605599d49deSDave Jiang __ioat_prep_pq16_lock(chan, NULL, dst, src, src_cnt, 606599d49deSDave Jiang scf, len, flags) : 607599d49deSDave Jiang __ioat_prep_pq_lock(chan, NULL, dst, src, src_cnt, 608599d49deSDave Jiang scf, len, flags); 609599d49deSDave Jiang } 610599d49deSDave Jiang } 611599d49deSDave Jiang 612599d49deSDave Jiang struct dma_async_tx_descriptor * 613599d49deSDave Jiang ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 614599d49deSDave Jiang unsigned int src_cnt, const unsigned char *scf, size_t len, 615599d49deSDave Jiang enum sum_check_flags *pqres, unsigned long flags) 616599d49deSDave Jiang { 617599d49deSDave Jiang /* specify valid address for disabled result */ 618599d49deSDave Jiang if (flags & DMA_PREP_PQ_DISABLE_P) 619599d49deSDave Jiang pq[0] = pq[1]; 620599d49deSDave Jiang if (flags & DMA_PREP_PQ_DISABLE_Q) 621599d49deSDave Jiang pq[1] = pq[0]; 622599d49deSDave Jiang 623599d49deSDave Jiang /* the cleanup routine only sets bits on validate failure, it 624599d49deSDave Jiang * does not clear bits on validate success... so clear it here 625599d49deSDave Jiang */ 626599d49deSDave Jiang *pqres = 0; 627599d49deSDave Jiang 628599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 629599d49deSDave Jiang __ioat_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len, 630599d49deSDave Jiang flags) : 631599d49deSDave Jiang __ioat_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len, 632599d49deSDave Jiang flags); 633599d49deSDave Jiang } 634599d49deSDave Jiang 635599d49deSDave Jiang struct dma_async_tx_descriptor * 636599d49deSDave Jiang ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, 637599d49deSDave Jiang unsigned int src_cnt, size_t len, unsigned long flags) 638599d49deSDave Jiang { 6397b7d0ca7SDave Jiang unsigned char scf[MAX_SCF]; 640599d49deSDave Jiang dma_addr_t pq[2]; 641599d49deSDave Jiang 6427b7d0ca7SDave Jiang if (src_cnt > MAX_SCF) 6437b7d0ca7SDave Jiang return NULL; 6447b7d0ca7SDave Jiang 645599d49deSDave Jiang memset(scf, 0, src_cnt); 646599d49deSDave Jiang pq[0] = dst; 647599d49deSDave Jiang flags |= DMA_PREP_PQ_DISABLE_Q; 648599d49deSDave Jiang pq[1] = dst; /* specify valid address for disabled result */ 649599d49deSDave Jiang 650599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 651599d49deSDave Jiang __ioat_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len, 652599d49deSDave Jiang flags) : 653599d49deSDave Jiang __ioat_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, 654599d49deSDave Jiang flags); 655599d49deSDave Jiang } 656599d49deSDave Jiang 657599d49deSDave Jiang struct dma_async_tx_descriptor * 658599d49deSDave Jiang ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, 659599d49deSDave Jiang unsigned int src_cnt, size_t len, 660599d49deSDave Jiang enum sum_check_flags *result, unsigned long flags) 661599d49deSDave Jiang { 6627b7d0ca7SDave Jiang unsigned char scf[MAX_SCF]; 663599d49deSDave Jiang dma_addr_t pq[2]; 664599d49deSDave Jiang 6657b7d0ca7SDave Jiang if (src_cnt > MAX_SCF) 6667b7d0ca7SDave Jiang return NULL; 6677b7d0ca7SDave Jiang 668599d49deSDave Jiang /* the cleanup routine only sets bits on validate failure, it 669599d49deSDave Jiang * does not clear bits on validate success... so clear it here 670599d49deSDave Jiang */ 671599d49deSDave Jiang *result = 0; 672599d49deSDave Jiang 673599d49deSDave Jiang memset(scf, 0, src_cnt); 674599d49deSDave Jiang pq[0] = src[0]; 675599d49deSDave Jiang flags |= DMA_PREP_PQ_DISABLE_Q; 676599d49deSDave Jiang pq[1] = pq[0]; /* specify valid address for disabled result */ 677599d49deSDave Jiang 678599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 679599d49deSDave Jiang __ioat_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1, 680599d49deSDave Jiang scf, len, flags) : 681599d49deSDave Jiang __ioat_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, 682599d49deSDave Jiang scf, len, flags); 683599d49deSDave Jiang } 684599d49deSDave Jiang 685599d49deSDave Jiang struct dma_async_tx_descriptor * 686599d49deSDave Jiang ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags) 687599d49deSDave Jiang { 688599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 689599d49deSDave Jiang struct ioat_ring_ent *desc; 690599d49deSDave Jiang struct ioat_dma_descriptor *hw; 691599d49deSDave Jiang 692599d49deSDave Jiang if (ioat_check_space_lock(ioat_chan, 1) == 0) 693599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head); 694599d49deSDave Jiang else 695599d49deSDave Jiang return NULL; 696599d49deSDave Jiang 697599d49deSDave Jiang hw = desc->hw; 698599d49deSDave Jiang hw->ctl = 0; 699599d49deSDave Jiang hw->ctl_f.null = 1; 700599d49deSDave Jiang hw->ctl_f.int_en = 1; 701599d49deSDave Jiang hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 702599d49deSDave Jiang hw->ctl_f.compl_write = 1; 703599d49deSDave Jiang hw->size = NULL_DESC_BUFFER_SIZE; 704599d49deSDave Jiang hw->src_addr = 0; 705599d49deSDave Jiang hw->dst_addr = 0; 706599d49deSDave Jiang 707599d49deSDave Jiang desc->txd.flags = flags; 708599d49deSDave Jiang desc->len = 1; 709599d49deSDave Jiang 710599d49deSDave Jiang dump_desc_dbg(ioat_chan, desc); 711599d49deSDave Jiang 712599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 713599d49deSDave Jiang return &desc->txd; 714599d49deSDave Jiang } 715599d49deSDave Jiang 716