1599d49deSDave Jiang /* 2599d49deSDave Jiang * Intel I/OAT DMA Linux driver 3599d49deSDave Jiang * Copyright(c) 2004 - 2015 Intel Corporation. 4599d49deSDave Jiang * 5599d49deSDave Jiang * This program is free software; you can redistribute it and/or modify it 6599d49deSDave Jiang * under the terms and conditions of the GNU General Public License, 7599d49deSDave Jiang * version 2, as published by the Free Software Foundation. 8599d49deSDave Jiang * 9599d49deSDave Jiang * This program is distributed in the hope that it will be useful, but WITHOUT 10599d49deSDave Jiang * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11599d49deSDave Jiang * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12599d49deSDave Jiang * more details. 13599d49deSDave Jiang * 14599d49deSDave Jiang * The full GNU General Public License is included in this distribution in 15599d49deSDave Jiang * the file called "COPYING". 16599d49deSDave Jiang * 17599d49deSDave Jiang */ 18599d49deSDave Jiang #include <linux/module.h> 19599d49deSDave Jiang #include <linux/pci.h> 20599d49deSDave Jiang #include <linux/gfp.h> 21599d49deSDave Jiang #include <linux/dmaengine.h> 22599d49deSDave Jiang #include <linux/dma-mapping.h> 23599d49deSDave Jiang #include <linux/prefetch.h> 24599d49deSDave Jiang #include "../dmaengine.h" 25599d49deSDave Jiang #include "registers.h" 26599d49deSDave Jiang #include "hw.h" 27599d49deSDave Jiang #include "dma.h" 28599d49deSDave Jiang 29599d49deSDave Jiang /* provide a lookup table for setting the source address in the base or 30599d49deSDave Jiang * extended descriptor of an xor or pq descriptor 31599d49deSDave Jiang */ 32599d49deSDave Jiang static const u8 xor_idx_to_desc = 0xe0; 33599d49deSDave Jiang static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 }; 34599d49deSDave Jiang static const u8 pq_idx_to_desc = 0xf8; 35599d49deSDave Jiang static const u8 pq16_idx_to_desc[] = { 0, 0, 1, 1, 1, 1, 1, 1, 1, 36599d49deSDave Jiang 2, 2, 2, 2, 2, 2, 2 }; 37599d49deSDave Jiang static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 }; 38599d49deSDave Jiang static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7, 39599d49deSDave Jiang 0, 1, 2, 3, 4, 5, 6 }; 40599d49deSDave Jiang 41599d49deSDave Jiang static void xor_set_src(struct ioat_raw_descriptor *descs[2], 42599d49deSDave Jiang dma_addr_t addr, u32 offset, int idx) 43599d49deSDave Jiang { 44599d49deSDave Jiang struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1]; 45599d49deSDave Jiang 46599d49deSDave Jiang raw->field[xor_idx_to_field[idx]] = addr + offset; 47599d49deSDave Jiang } 48599d49deSDave Jiang 49599d49deSDave Jiang static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx) 50599d49deSDave Jiang { 51599d49deSDave Jiang struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1]; 52599d49deSDave Jiang 53599d49deSDave Jiang return raw->field[pq_idx_to_field[idx]]; 54599d49deSDave Jiang } 55599d49deSDave Jiang 56599d49deSDave Jiang static dma_addr_t pq16_get_src(struct ioat_raw_descriptor *desc[3], int idx) 57599d49deSDave Jiang { 58599d49deSDave Jiang struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]]; 59599d49deSDave Jiang 60599d49deSDave Jiang return raw->field[pq16_idx_to_field[idx]]; 61599d49deSDave Jiang } 62599d49deSDave Jiang 63599d49deSDave Jiang static void pq_set_src(struct ioat_raw_descriptor *descs[2], 64599d49deSDave Jiang dma_addr_t addr, u32 offset, u8 coef, int idx) 65599d49deSDave Jiang { 66599d49deSDave Jiang struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0]; 67599d49deSDave Jiang struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1]; 68599d49deSDave Jiang 69599d49deSDave Jiang raw->field[pq_idx_to_field[idx]] = addr + offset; 70599d49deSDave Jiang pq->coef[idx] = coef; 71599d49deSDave Jiang } 72599d49deSDave Jiang 73599d49deSDave Jiang static void pq16_set_src(struct ioat_raw_descriptor *desc[3], 74599d49deSDave Jiang dma_addr_t addr, u32 offset, u8 coef, unsigned idx) 75599d49deSDave Jiang { 76599d49deSDave Jiang struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0]; 77599d49deSDave Jiang struct ioat_pq16a_descriptor *pq16 = 78599d49deSDave Jiang (struct ioat_pq16a_descriptor *)desc[1]; 79599d49deSDave Jiang struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]]; 80599d49deSDave Jiang 81599d49deSDave Jiang raw->field[pq16_idx_to_field[idx]] = addr + offset; 82599d49deSDave Jiang 83599d49deSDave Jiang if (idx < 8) 84599d49deSDave Jiang pq->coef[idx] = coef; 85599d49deSDave Jiang else 86599d49deSDave Jiang pq16->coef[idx - 8] = coef; 87599d49deSDave Jiang } 88599d49deSDave Jiang 89599d49deSDave Jiang static struct ioat_sed_ent * 90599d49deSDave Jiang ioat3_alloc_sed(struct ioatdma_device *ioat_dma, unsigned int hw_pool) 91599d49deSDave Jiang { 92599d49deSDave Jiang struct ioat_sed_ent *sed; 93599d49deSDave Jiang gfp_t flags = __GFP_ZERO | GFP_ATOMIC; 94599d49deSDave Jiang 95599d49deSDave Jiang sed = kmem_cache_alloc(ioat_sed_cache, flags); 96599d49deSDave Jiang if (!sed) 97599d49deSDave Jiang return NULL; 98599d49deSDave Jiang 99599d49deSDave Jiang sed->hw_pool = hw_pool; 100599d49deSDave Jiang sed->hw = dma_pool_alloc(ioat_dma->sed_hw_pool[hw_pool], 101599d49deSDave Jiang flags, &sed->dma); 102599d49deSDave Jiang if (!sed->hw) { 103599d49deSDave Jiang kmem_cache_free(ioat_sed_cache, sed); 104599d49deSDave Jiang return NULL; 105599d49deSDave Jiang } 106599d49deSDave Jiang 107599d49deSDave Jiang return sed; 108599d49deSDave Jiang } 109599d49deSDave Jiang 110599d49deSDave Jiang struct dma_async_tx_descriptor * 111599d49deSDave Jiang ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, 112599d49deSDave Jiang dma_addr_t dma_src, size_t len, unsigned long flags) 113599d49deSDave Jiang { 114599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 115599d49deSDave Jiang struct ioat_dma_descriptor *hw; 116599d49deSDave Jiang struct ioat_ring_ent *desc; 117599d49deSDave Jiang dma_addr_t dst = dma_dest; 118599d49deSDave Jiang dma_addr_t src = dma_src; 119599d49deSDave Jiang size_t total_len = len; 120599d49deSDave Jiang int num_descs, idx, i; 121599d49deSDave Jiang 122599d49deSDave Jiang num_descs = ioat_xferlen_to_descs(ioat_chan, len); 123599d49deSDave Jiang if (likely(num_descs) && 124599d49deSDave Jiang ioat_check_space_lock(ioat_chan, num_descs) == 0) 125599d49deSDave Jiang idx = ioat_chan->head; 126599d49deSDave Jiang else 127599d49deSDave Jiang return NULL; 128599d49deSDave Jiang i = 0; 129599d49deSDave Jiang do { 130599d49deSDave Jiang size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log); 131599d49deSDave Jiang 132599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, idx + i); 133599d49deSDave Jiang hw = desc->hw; 134599d49deSDave Jiang 135599d49deSDave Jiang hw->size = copy; 136599d49deSDave Jiang hw->ctl = 0; 137599d49deSDave Jiang hw->src_addr = src; 138599d49deSDave Jiang hw->dst_addr = dst; 139599d49deSDave Jiang 140599d49deSDave Jiang len -= copy; 141599d49deSDave Jiang dst += copy; 142599d49deSDave Jiang src += copy; 143599d49deSDave Jiang dump_desc_dbg(ioat_chan, desc); 144599d49deSDave Jiang } while (++i < num_descs); 145599d49deSDave Jiang 146599d49deSDave Jiang desc->txd.flags = flags; 147599d49deSDave Jiang desc->len = total_len; 148599d49deSDave Jiang hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 149599d49deSDave Jiang hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 150599d49deSDave Jiang hw->ctl_f.compl_write = 1; 151599d49deSDave Jiang dump_desc_dbg(ioat_chan, desc); 152599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 153599d49deSDave Jiang 154599d49deSDave Jiang return &desc->txd; 155599d49deSDave Jiang } 156599d49deSDave Jiang 157599d49deSDave Jiang 158599d49deSDave Jiang static struct dma_async_tx_descriptor * 159599d49deSDave Jiang __ioat_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, 160599d49deSDave Jiang dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt, 161599d49deSDave Jiang size_t len, unsigned long flags) 162599d49deSDave Jiang { 163599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 164599d49deSDave Jiang struct ioat_ring_ent *compl_desc; 165599d49deSDave Jiang struct ioat_ring_ent *desc; 166599d49deSDave Jiang struct ioat_ring_ent *ext; 167599d49deSDave Jiang size_t total_len = len; 168599d49deSDave Jiang struct ioat_xor_descriptor *xor; 169599d49deSDave Jiang struct ioat_xor_ext_descriptor *xor_ex = NULL; 170599d49deSDave Jiang struct ioat_dma_descriptor *hw; 171599d49deSDave Jiang int num_descs, with_ext, idx, i; 172599d49deSDave Jiang u32 offset = 0; 173599d49deSDave Jiang u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR; 174599d49deSDave Jiang 175599d49deSDave Jiang BUG_ON(src_cnt < 2); 176599d49deSDave Jiang 177599d49deSDave Jiang num_descs = ioat_xferlen_to_descs(ioat_chan, len); 178599d49deSDave Jiang /* we need 2x the number of descriptors to cover greater than 5 179599d49deSDave Jiang * sources 180599d49deSDave Jiang */ 181599d49deSDave Jiang if (src_cnt > 5) { 182599d49deSDave Jiang with_ext = 1; 183599d49deSDave Jiang num_descs *= 2; 184599d49deSDave Jiang } else 185599d49deSDave Jiang with_ext = 0; 186599d49deSDave Jiang 187599d49deSDave Jiang /* completion writes from the raid engine may pass completion 188599d49deSDave Jiang * writes from the legacy engine, so we need one extra null 189599d49deSDave Jiang * (legacy) descriptor to ensure all completion writes arrive in 190599d49deSDave Jiang * order. 191599d49deSDave Jiang */ 192599d49deSDave Jiang if (likely(num_descs) && 193599d49deSDave Jiang ioat_check_space_lock(ioat_chan, num_descs+1) == 0) 194599d49deSDave Jiang idx = ioat_chan->head; 195599d49deSDave Jiang else 196599d49deSDave Jiang return NULL; 197599d49deSDave Jiang i = 0; 198599d49deSDave Jiang do { 199599d49deSDave Jiang struct ioat_raw_descriptor *descs[2]; 200599d49deSDave Jiang size_t xfer_size = min_t(size_t, 201599d49deSDave Jiang len, 1 << ioat_chan->xfercap_log); 202599d49deSDave Jiang int s; 203599d49deSDave Jiang 204599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, idx + i); 205599d49deSDave Jiang xor = desc->xor; 206599d49deSDave Jiang 207599d49deSDave Jiang /* save a branch by unconditionally retrieving the 208599d49deSDave Jiang * extended descriptor xor_set_src() knows to not write 209599d49deSDave Jiang * to it in the single descriptor case 210599d49deSDave Jiang */ 211599d49deSDave Jiang ext = ioat_get_ring_ent(ioat_chan, idx + i + 1); 212599d49deSDave Jiang xor_ex = ext->xor_ex; 213599d49deSDave Jiang 214599d49deSDave Jiang descs[0] = (struct ioat_raw_descriptor *) xor; 215599d49deSDave Jiang descs[1] = (struct ioat_raw_descriptor *) xor_ex; 216599d49deSDave Jiang for (s = 0; s < src_cnt; s++) 217599d49deSDave Jiang xor_set_src(descs, src[s], offset, s); 218599d49deSDave Jiang xor->size = xfer_size; 219599d49deSDave Jiang xor->dst_addr = dest + offset; 220599d49deSDave Jiang xor->ctl = 0; 221599d49deSDave Jiang xor->ctl_f.op = op; 222599d49deSDave Jiang xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt); 223599d49deSDave Jiang 224599d49deSDave Jiang len -= xfer_size; 225599d49deSDave Jiang offset += xfer_size; 226599d49deSDave Jiang dump_desc_dbg(ioat_chan, desc); 227599d49deSDave Jiang } while ((i += 1 + with_ext) < num_descs); 228599d49deSDave Jiang 229599d49deSDave Jiang /* last xor descriptor carries the unmap parameters and fence bit */ 230599d49deSDave Jiang desc->txd.flags = flags; 231599d49deSDave Jiang desc->len = total_len; 232599d49deSDave Jiang if (result) 233599d49deSDave Jiang desc->result = result; 234599d49deSDave Jiang xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 235599d49deSDave Jiang 236599d49deSDave Jiang /* completion descriptor carries interrupt bit */ 237599d49deSDave Jiang compl_desc = ioat_get_ring_ent(ioat_chan, idx + i); 238599d49deSDave Jiang compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; 239599d49deSDave Jiang hw = compl_desc->hw; 240599d49deSDave Jiang hw->ctl = 0; 241599d49deSDave Jiang hw->ctl_f.null = 1; 242599d49deSDave Jiang hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 243599d49deSDave Jiang hw->ctl_f.compl_write = 1; 244599d49deSDave Jiang hw->size = NULL_DESC_BUFFER_SIZE; 245599d49deSDave Jiang dump_desc_dbg(ioat_chan, compl_desc); 246599d49deSDave Jiang 247599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 248599d49deSDave Jiang return &compl_desc->txd; 249599d49deSDave Jiang } 250599d49deSDave Jiang 251599d49deSDave Jiang struct dma_async_tx_descriptor * 252599d49deSDave Jiang ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 253599d49deSDave Jiang unsigned int src_cnt, size_t len, unsigned long flags) 254599d49deSDave Jiang { 255599d49deSDave Jiang return __ioat_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags); 256599d49deSDave Jiang } 257599d49deSDave Jiang 258599d49deSDave Jiang struct dma_async_tx_descriptor * 259599d49deSDave Jiang ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src, 260599d49deSDave Jiang unsigned int src_cnt, size_t len, 261599d49deSDave Jiang enum sum_check_flags *result, unsigned long flags) 262599d49deSDave Jiang { 263599d49deSDave Jiang /* the cleanup routine only sets bits on validate failure, it 264599d49deSDave Jiang * does not clear bits on validate success... so clear it here 265599d49deSDave Jiang */ 266599d49deSDave Jiang *result = 0; 267599d49deSDave Jiang 268599d49deSDave Jiang return __ioat_prep_xor_lock(chan, result, src[0], &src[1], 269599d49deSDave Jiang src_cnt - 1, len, flags); 270599d49deSDave Jiang } 271599d49deSDave Jiang 272599d49deSDave Jiang static void 273599d49deSDave Jiang dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc, 274599d49deSDave Jiang struct ioat_ring_ent *ext) 275599d49deSDave Jiang { 276599d49deSDave Jiang struct device *dev = to_dev(ioat_chan); 277599d49deSDave Jiang struct ioat_pq_descriptor *pq = desc->pq; 278599d49deSDave Jiang struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL; 279599d49deSDave Jiang struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex }; 280599d49deSDave Jiang int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt); 281599d49deSDave Jiang int i; 282599d49deSDave Jiang 283599d49deSDave Jiang dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x" 284599d49deSDave Jiang " sz: %#10.8x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'" 285599d49deSDave Jiang " src_cnt: %d)\n", 286599d49deSDave Jiang desc_id(desc), (unsigned long long) desc->txd.phys, 287599d49deSDave Jiang (unsigned long long) (pq_ex ? pq_ex->next : pq->next), 288599d49deSDave Jiang desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, 289599d49deSDave Jiang pq->ctl_f.int_en, pq->ctl_f.compl_write, 290599d49deSDave Jiang pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q", 291599d49deSDave Jiang pq->ctl_f.src_cnt); 292599d49deSDave Jiang for (i = 0; i < src_cnt; i++) 293599d49deSDave Jiang dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i, 294599d49deSDave Jiang (unsigned long long) pq_get_src(descs, i), pq->coef[i]); 295599d49deSDave Jiang dev_dbg(dev, "\tP: %#llx\n", pq->p_addr); 296599d49deSDave Jiang dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr); 297599d49deSDave Jiang dev_dbg(dev, "\tNEXT: %#llx\n", pq->next); 298599d49deSDave Jiang } 299599d49deSDave Jiang 300599d49deSDave Jiang static void dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan, 301599d49deSDave Jiang struct ioat_ring_ent *desc) 302599d49deSDave Jiang { 303599d49deSDave Jiang struct device *dev = to_dev(ioat_chan); 304599d49deSDave Jiang struct ioat_pq_descriptor *pq = desc->pq; 305599d49deSDave Jiang struct ioat_raw_descriptor *descs[] = { (void *)pq, 306599d49deSDave Jiang (void *)pq, 307599d49deSDave Jiang (void *)pq }; 308599d49deSDave Jiang int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt); 309599d49deSDave Jiang int i; 310599d49deSDave Jiang 311599d49deSDave Jiang if (desc->sed) { 312599d49deSDave Jiang descs[1] = (void *)desc->sed->hw; 313599d49deSDave Jiang descs[2] = (void *)desc->sed->hw + 64; 314599d49deSDave Jiang } 315599d49deSDave Jiang 316599d49deSDave Jiang dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x" 317599d49deSDave Jiang " sz: %#x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'" 318599d49deSDave Jiang " src_cnt: %d)\n", 319599d49deSDave Jiang desc_id(desc), (unsigned long long) desc->txd.phys, 320599d49deSDave Jiang (unsigned long long) pq->next, 321599d49deSDave Jiang desc->txd.flags, pq->size, pq->ctl, 322599d49deSDave Jiang pq->ctl_f.op, pq->ctl_f.int_en, 323599d49deSDave Jiang pq->ctl_f.compl_write, 324599d49deSDave Jiang pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q", 325599d49deSDave Jiang pq->ctl_f.src_cnt); 326599d49deSDave Jiang for (i = 0; i < src_cnt; i++) { 327599d49deSDave Jiang dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i, 328599d49deSDave Jiang (unsigned long long) pq16_get_src(descs, i), 329599d49deSDave Jiang pq->coef[i]); 330599d49deSDave Jiang } 331599d49deSDave Jiang dev_dbg(dev, "\tP: %#llx\n", pq->p_addr); 332599d49deSDave Jiang dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr); 333599d49deSDave Jiang } 334599d49deSDave Jiang 335599d49deSDave Jiang static struct dma_async_tx_descriptor * 336599d49deSDave Jiang __ioat_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, 337599d49deSDave Jiang const dma_addr_t *dst, const dma_addr_t *src, 338599d49deSDave Jiang unsigned int src_cnt, const unsigned char *scf, 339599d49deSDave Jiang size_t len, unsigned long flags) 340599d49deSDave Jiang { 341599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 342599d49deSDave Jiang struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; 343599d49deSDave Jiang struct ioat_ring_ent *compl_desc; 344599d49deSDave Jiang struct ioat_ring_ent *desc; 345599d49deSDave Jiang struct ioat_ring_ent *ext; 346599d49deSDave Jiang size_t total_len = len; 347599d49deSDave Jiang struct ioat_pq_descriptor *pq; 348599d49deSDave Jiang struct ioat_pq_ext_descriptor *pq_ex = NULL; 349599d49deSDave Jiang struct ioat_dma_descriptor *hw; 350599d49deSDave Jiang u32 offset = 0; 351599d49deSDave Jiang u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ; 352599d49deSDave Jiang int i, s, idx, with_ext, num_descs; 353599d49deSDave Jiang int cb32 = (ioat_dma->version < IOAT_VER_3_3) ? 1 : 0; 354599d49deSDave Jiang 355599d49deSDave Jiang dev_dbg(to_dev(ioat_chan), "%s\n", __func__); 356599d49deSDave Jiang /* the engine requires at least two sources (we provide 357599d49deSDave Jiang * at least 1 implied source in the DMA_PREP_CONTINUE case) 358599d49deSDave Jiang */ 359599d49deSDave Jiang BUG_ON(src_cnt + dmaf_continue(flags) < 2); 360599d49deSDave Jiang 361599d49deSDave Jiang num_descs = ioat_xferlen_to_descs(ioat_chan, len); 362599d49deSDave Jiang /* we need 2x the number of descriptors to cover greater than 3 363599d49deSDave Jiang * sources (we need 1 extra source in the q-only continuation 364599d49deSDave Jiang * case and 3 extra sources in the p+q continuation case. 365599d49deSDave Jiang */ 366599d49deSDave Jiang if (src_cnt + dmaf_p_disabled_continue(flags) > 3 || 367599d49deSDave Jiang (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) { 368599d49deSDave Jiang with_ext = 1; 369599d49deSDave Jiang num_descs *= 2; 370599d49deSDave Jiang } else 371599d49deSDave Jiang with_ext = 0; 372599d49deSDave Jiang 373599d49deSDave Jiang /* completion writes from the raid engine may pass completion 374599d49deSDave Jiang * writes from the legacy engine, so we need one extra null 375599d49deSDave Jiang * (legacy) descriptor to ensure all completion writes arrive in 376599d49deSDave Jiang * order. 377599d49deSDave Jiang */ 378599d49deSDave Jiang if (likely(num_descs) && 379599d49deSDave Jiang ioat_check_space_lock(ioat_chan, num_descs + cb32) == 0) 380599d49deSDave Jiang idx = ioat_chan->head; 381599d49deSDave Jiang else 382599d49deSDave Jiang return NULL; 383599d49deSDave Jiang i = 0; 384599d49deSDave Jiang do { 385599d49deSDave Jiang struct ioat_raw_descriptor *descs[2]; 386599d49deSDave Jiang size_t xfer_size = min_t(size_t, len, 387599d49deSDave Jiang 1 << ioat_chan->xfercap_log); 388599d49deSDave Jiang 389599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, idx + i); 390599d49deSDave Jiang pq = desc->pq; 391599d49deSDave Jiang 392599d49deSDave Jiang /* save a branch by unconditionally retrieving the 393599d49deSDave Jiang * extended descriptor pq_set_src() knows to not write 394599d49deSDave Jiang * to it in the single descriptor case 395599d49deSDave Jiang */ 396599d49deSDave Jiang ext = ioat_get_ring_ent(ioat_chan, idx + i + with_ext); 397599d49deSDave Jiang pq_ex = ext->pq_ex; 398599d49deSDave Jiang 399599d49deSDave Jiang descs[0] = (struct ioat_raw_descriptor *) pq; 400599d49deSDave Jiang descs[1] = (struct ioat_raw_descriptor *) pq_ex; 401599d49deSDave Jiang 402599d49deSDave Jiang for (s = 0; s < src_cnt; s++) 403599d49deSDave Jiang pq_set_src(descs, src[s], offset, scf[s], s); 404599d49deSDave Jiang 405599d49deSDave Jiang /* see the comment for dma_maxpq in include/linux/dmaengine.h */ 406599d49deSDave Jiang if (dmaf_p_disabled_continue(flags)) 407599d49deSDave Jiang pq_set_src(descs, dst[1], offset, 1, s++); 408599d49deSDave Jiang else if (dmaf_continue(flags)) { 409599d49deSDave Jiang pq_set_src(descs, dst[0], offset, 0, s++); 410599d49deSDave Jiang pq_set_src(descs, dst[1], offset, 1, s++); 411599d49deSDave Jiang pq_set_src(descs, dst[1], offset, 0, s++); 412599d49deSDave Jiang } 413599d49deSDave Jiang pq->size = xfer_size; 414599d49deSDave Jiang pq->p_addr = dst[0] + offset; 415599d49deSDave Jiang pq->q_addr = dst[1] + offset; 416599d49deSDave Jiang pq->ctl = 0; 417599d49deSDave Jiang pq->ctl_f.op = op; 418599d49deSDave Jiang /* we turn on descriptor write back error status */ 419599d49deSDave Jiang if (ioat_dma->cap & IOAT_CAP_DWBES) 420599d49deSDave Jiang pq->ctl_f.wb_en = result ? 1 : 0; 421599d49deSDave Jiang pq->ctl_f.src_cnt = src_cnt_to_hw(s); 422599d49deSDave Jiang pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P); 423599d49deSDave Jiang pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q); 424599d49deSDave Jiang 425599d49deSDave Jiang len -= xfer_size; 426599d49deSDave Jiang offset += xfer_size; 427599d49deSDave Jiang } while ((i += 1 + with_ext) < num_descs); 428599d49deSDave Jiang 429599d49deSDave Jiang /* last pq descriptor carries the unmap parameters and fence bit */ 430599d49deSDave Jiang desc->txd.flags = flags; 431599d49deSDave Jiang desc->len = total_len; 432599d49deSDave Jiang if (result) 433599d49deSDave Jiang desc->result = result; 434599d49deSDave Jiang pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 435599d49deSDave Jiang dump_pq_desc_dbg(ioat_chan, desc, ext); 436599d49deSDave Jiang 437599d49deSDave Jiang if (!cb32) { 438599d49deSDave Jiang pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 439599d49deSDave Jiang pq->ctl_f.compl_write = 1; 440599d49deSDave Jiang compl_desc = desc; 441599d49deSDave Jiang } else { 442599d49deSDave Jiang /* completion descriptor carries interrupt bit */ 443599d49deSDave Jiang compl_desc = ioat_get_ring_ent(ioat_chan, idx + i); 444599d49deSDave Jiang compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; 445599d49deSDave Jiang hw = compl_desc->hw; 446599d49deSDave Jiang hw->ctl = 0; 447599d49deSDave Jiang hw->ctl_f.null = 1; 448599d49deSDave Jiang hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 449599d49deSDave Jiang hw->ctl_f.compl_write = 1; 450599d49deSDave Jiang hw->size = NULL_DESC_BUFFER_SIZE; 451599d49deSDave Jiang dump_desc_dbg(ioat_chan, compl_desc); 452599d49deSDave Jiang } 453599d49deSDave Jiang 454599d49deSDave Jiang 455599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 456599d49deSDave Jiang return &compl_desc->txd; 457599d49deSDave Jiang } 458599d49deSDave Jiang 459599d49deSDave Jiang static struct dma_async_tx_descriptor * 460599d49deSDave Jiang __ioat_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, 461599d49deSDave Jiang const dma_addr_t *dst, const dma_addr_t *src, 462599d49deSDave Jiang unsigned int src_cnt, const unsigned char *scf, 463599d49deSDave Jiang size_t len, unsigned long flags) 464599d49deSDave Jiang { 465599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 466599d49deSDave Jiang struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; 467599d49deSDave Jiang struct ioat_ring_ent *desc; 468599d49deSDave Jiang size_t total_len = len; 469599d49deSDave Jiang struct ioat_pq_descriptor *pq; 470599d49deSDave Jiang u32 offset = 0; 471599d49deSDave Jiang u8 op; 472599d49deSDave Jiang int i, s, idx, num_descs; 473599d49deSDave Jiang 474599d49deSDave Jiang /* this function is only called with 9-16 sources */ 475599d49deSDave Jiang op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S; 476599d49deSDave Jiang 477599d49deSDave Jiang dev_dbg(to_dev(ioat_chan), "%s\n", __func__); 478599d49deSDave Jiang 479599d49deSDave Jiang num_descs = ioat_xferlen_to_descs(ioat_chan, len); 480599d49deSDave Jiang 481599d49deSDave Jiang /* 482599d49deSDave Jiang * 16 source pq is only available on cb3.3 and has no completion 483599d49deSDave Jiang * write hw bug. 484599d49deSDave Jiang */ 485599d49deSDave Jiang if (num_descs && ioat_check_space_lock(ioat_chan, num_descs) == 0) 486599d49deSDave Jiang idx = ioat_chan->head; 487599d49deSDave Jiang else 488599d49deSDave Jiang return NULL; 489599d49deSDave Jiang 490599d49deSDave Jiang i = 0; 491599d49deSDave Jiang 492599d49deSDave Jiang do { 493599d49deSDave Jiang struct ioat_raw_descriptor *descs[4]; 494599d49deSDave Jiang size_t xfer_size = min_t(size_t, len, 495599d49deSDave Jiang 1 << ioat_chan->xfercap_log); 496599d49deSDave Jiang 497599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, idx + i); 498599d49deSDave Jiang pq = desc->pq; 499599d49deSDave Jiang 500599d49deSDave Jiang descs[0] = (struct ioat_raw_descriptor *) pq; 501599d49deSDave Jiang 502599d49deSDave Jiang desc->sed = ioat3_alloc_sed(ioat_dma, (src_cnt-2) >> 3); 503599d49deSDave Jiang if (!desc->sed) { 504599d49deSDave Jiang dev_err(to_dev(ioat_chan), 505599d49deSDave Jiang "%s: no free sed entries\n", __func__); 506599d49deSDave Jiang return NULL; 507599d49deSDave Jiang } 508599d49deSDave Jiang 509599d49deSDave Jiang pq->sed_addr = desc->sed->dma; 510599d49deSDave Jiang desc->sed->parent = desc; 511599d49deSDave Jiang 512599d49deSDave Jiang descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw; 513599d49deSDave Jiang descs[2] = (void *)descs[1] + 64; 514599d49deSDave Jiang 515599d49deSDave Jiang for (s = 0; s < src_cnt; s++) 516599d49deSDave Jiang pq16_set_src(descs, src[s], offset, scf[s], s); 517599d49deSDave Jiang 518599d49deSDave Jiang /* see the comment for dma_maxpq in include/linux/dmaengine.h */ 519599d49deSDave Jiang if (dmaf_p_disabled_continue(flags)) 520599d49deSDave Jiang pq16_set_src(descs, dst[1], offset, 1, s++); 521599d49deSDave Jiang else if (dmaf_continue(flags)) { 522599d49deSDave Jiang pq16_set_src(descs, dst[0], offset, 0, s++); 523599d49deSDave Jiang pq16_set_src(descs, dst[1], offset, 1, s++); 524599d49deSDave Jiang pq16_set_src(descs, dst[1], offset, 0, s++); 525599d49deSDave Jiang } 526599d49deSDave Jiang 527599d49deSDave Jiang pq->size = xfer_size; 528599d49deSDave Jiang pq->p_addr = dst[0] + offset; 529599d49deSDave Jiang pq->q_addr = dst[1] + offset; 530599d49deSDave Jiang pq->ctl = 0; 531599d49deSDave Jiang pq->ctl_f.op = op; 532599d49deSDave Jiang pq->ctl_f.src_cnt = src16_cnt_to_hw(s); 533599d49deSDave Jiang /* we turn on descriptor write back error status */ 534599d49deSDave Jiang if (ioat_dma->cap & IOAT_CAP_DWBES) 535599d49deSDave Jiang pq->ctl_f.wb_en = result ? 1 : 0; 536599d49deSDave Jiang pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P); 537599d49deSDave Jiang pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q); 538599d49deSDave Jiang 539599d49deSDave Jiang len -= xfer_size; 540599d49deSDave Jiang offset += xfer_size; 541599d49deSDave Jiang } while (++i < num_descs); 542599d49deSDave Jiang 543599d49deSDave Jiang /* last pq descriptor carries the unmap parameters and fence bit */ 544599d49deSDave Jiang desc->txd.flags = flags; 545599d49deSDave Jiang desc->len = total_len; 546599d49deSDave Jiang if (result) 547599d49deSDave Jiang desc->result = result; 548599d49deSDave Jiang pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 549599d49deSDave Jiang 550599d49deSDave Jiang /* with cb3.3 we should be able to do completion w/o a null desc */ 551599d49deSDave Jiang pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 552599d49deSDave Jiang pq->ctl_f.compl_write = 1; 553599d49deSDave Jiang 554599d49deSDave Jiang dump_pq16_desc_dbg(ioat_chan, desc); 555599d49deSDave Jiang 556599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 557599d49deSDave Jiang return &desc->txd; 558599d49deSDave Jiang } 559599d49deSDave Jiang 560599d49deSDave Jiang static int src_cnt_flags(unsigned int src_cnt, unsigned long flags) 561599d49deSDave Jiang { 562599d49deSDave Jiang if (dmaf_p_disabled_continue(flags)) 563599d49deSDave Jiang return src_cnt + 1; 564599d49deSDave Jiang else if (dmaf_continue(flags)) 565599d49deSDave Jiang return src_cnt + 3; 566599d49deSDave Jiang else 567599d49deSDave Jiang return src_cnt; 568599d49deSDave Jiang } 569599d49deSDave Jiang 570599d49deSDave Jiang struct dma_async_tx_descriptor * 571599d49deSDave Jiang ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 572599d49deSDave Jiang unsigned int src_cnt, const unsigned char *scf, size_t len, 573599d49deSDave Jiang unsigned long flags) 574599d49deSDave Jiang { 575599d49deSDave Jiang /* specify valid address for disabled result */ 576599d49deSDave Jiang if (flags & DMA_PREP_PQ_DISABLE_P) 577599d49deSDave Jiang dst[0] = dst[1]; 578599d49deSDave Jiang if (flags & DMA_PREP_PQ_DISABLE_Q) 579599d49deSDave Jiang dst[1] = dst[0]; 580599d49deSDave Jiang 581599d49deSDave Jiang /* handle the single source multiply case from the raid6 582599d49deSDave Jiang * recovery path 583599d49deSDave Jiang */ 584599d49deSDave Jiang if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) { 585599d49deSDave Jiang dma_addr_t single_source[2]; 586599d49deSDave Jiang unsigned char single_source_coef[2]; 587599d49deSDave Jiang 588599d49deSDave Jiang BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q); 589599d49deSDave Jiang single_source[0] = src[0]; 590599d49deSDave Jiang single_source[1] = src[0]; 591599d49deSDave Jiang single_source_coef[0] = scf[0]; 592599d49deSDave Jiang single_source_coef[1] = 0; 593599d49deSDave Jiang 594599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 595599d49deSDave Jiang __ioat_prep_pq16_lock(chan, NULL, dst, single_source, 596599d49deSDave Jiang 2, single_source_coef, len, 597599d49deSDave Jiang flags) : 598599d49deSDave Jiang __ioat_prep_pq_lock(chan, NULL, dst, single_source, 2, 599599d49deSDave Jiang single_source_coef, len, flags); 600599d49deSDave Jiang 601599d49deSDave Jiang } else { 602599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 603599d49deSDave Jiang __ioat_prep_pq16_lock(chan, NULL, dst, src, src_cnt, 604599d49deSDave Jiang scf, len, flags) : 605599d49deSDave Jiang __ioat_prep_pq_lock(chan, NULL, dst, src, src_cnt, 606599d49deSDave Jiang scf, len, flags); 607599d49deSDave Jiang } 608599d49deSDave Jiang } 609599d49deSDave Jiang 610599d49deSDave Jiang struct dma_async_tx_descriptor * 611599d49deSDave Jiang ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 612599d49deSDave Jiang unsigned int src_cnt, const unsigned char *scf, size_t len, 613599d49deSDave Jiang enum sum_check_flags *pqres, unsigned long flags) 614599d49deSDave Jiang { 615599d49deSDave Jiang /* specify valid address for disabled result */ 616599d49deSDave Jiang if (flags & DMA_PREP_PQ_DISABLE_P) 617599d49deSDave Jiang pq[0] = pq[1]; 618599d49deSDave Jiang if (flags & DMA_PREP_PQ_DISABLE_Q) 619599d49deSDave Jiang pq[1] = pq[0]; 620599d49deSDave Jiang 621599d49deSDave Jiang /* the cleanup routine only sets bits on validate failure, it 622599d49deSDave Jiang * does not clear bits on validate success... so clear it here 623599d49deSDave Jiang */ 624599d49deSDave Jiang *pqres = 0; 625599d49deSDave Jiang 626599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 627599d49deSDave Jiang __ioat_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len, 628599d49deSDave Jiang flags) : 629599d49deSDave Jiang __ioat_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len, 630599d49deSDave Jiang flags); 631599d49deSDave Jiang } 632599d49deSDave Jiang 633599d49deSDave Jiang struct dma_async_tx_descriptor * 634599d49deSDave Jiang ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, 635599d49deSDave Jiang unsigned int src_cnt, size_t len, unsigned long flags) 636599d49deSDave Jiang { 637599d49deSDave Jiang unsigned char scf[src_cnt]; 638599d49deSDave Jiang dma_addr_t pq[2]; 639599d49deSDave Jiang 640599d49deSDave Jiang memset(scf, 0, src_cnt); 641599d49deSDave Jiang pq[0] = dst; 642599d49deSDave Jiang flags |= DMA_PREP_PQ_DISABLE_Q; 643599d49deSDave Jiang pq[1] = dst; /* specify valid address for disabled result */ 644599d49deSDave Jiang 645599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 646599d49deSDave Jiang __ioat_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len, 647599d49deSDave Jiang flags) : 648599d49deSDave Jiang __ioat_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, 649599d49deSDave Jiang flags); 650599d49deSDave Jiang } 651599d49deSDave Jiang 652599d49deSDave Jiang struct dma_async_tx_descriptor * 653599d49deSDave Jiang ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, 654599d49deSDave Jiang unsigned int src_cnt, size_t len, 655599d49deSDave Jiang enum sum_check_flags *result, unsigned long flags) 656599d49deSDave Jiang { 657599d49deSDave Jiang unsigned char scf[src_cnt]; 658599d49deSDave Jiang dma_addr_t pq[2]; 659599d49deSDave Jiang 660599d49deSDave Jiang /* the cleanup routine only sets bits on validate failure, it 661599d49deSDave Jiang * does not clear bits on validate success... so clear it here 662599d49deSDave Jiang */ 663599d49deSDave Jiang *result = 0; 664599d49deSDave Jiang 665599d49deSDave Jiang memset(scf, 0, src_cnt); 666599d49deSDave Jiang pq[0] = src[0]; 667599d49deSDave Jiang flags |= DMA_PREP_PQ_DISABLE_Q; 668599d49deSDave Jiang pq[1] = pq[0]; /* specify valid address for disabled result */ 669599d49deSDave Jiang 670599d49deSDave Jiang return src_cnt_flags(src_cnt, flags) > 8 ? 671599d49deSDave Jiang __ioat_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1, 672599d49deSDave Jiang scf, len, flags) : 673599d49deSDave Jiang __ioat_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, 674599d49deSDave Jiang scf, len, flags); 675599d49deSDave Jiang } 676599d49deSDave Jiang 677599d49deSDave Jiang struct dma_async_tx_descriptor * 678599d49deSDave Jiang ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags) 679599d49deSDave Jiang { 680599d49deSDave Jiang struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 681599d49deSDave Jiang struct ioat_ring_ent *desc; 682599d49deSDave Jiang struct ioat_dma_descriptor *hw; 683599d49deSDave Jiang 684599d49deSDave Jiang if (ioat_check_space_lock(ioat_chan, 1) == 0) 685599d49deSDave Jiang desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head); 686599d49deSDave Jiang else 687599d49deSDave Jiang return NULL; 688599d49deSDave Jiang 689599d49deSDave Jiang hw = desc->hw; 690599d49deSDave Jiang hw->ctl = 0; 691599d49deSDave Jiang hw->ctl_f.null = 1; 692599d49deSDave Jiang hw->ctl_f.int_en = 1; 693599d49deSDave Jiang hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 694599d49deSDave Jiang hw->ctl_f.compl_write = 1; 695599d49deSDave Jiang hw->size = NULL_DESC_BUFFER_SIZE; 696599d49deSDave Jiang hw->src_addr = 0; 697599d49deSDave Jiang hw->dst_addr = 0; 698599d49deSDave Jiang 699599d49deSDave Jiang desc->txd.flags = flags; 700599d49deSDave Jiang desc->len = 1; 701599d49deSDave Jiang 702599d49deSDave Jiang dump_desc_dbg(ioat_chan, desc); 703599d49deSDave Jiang 704599d49deSDave Jiang /* we leave the channel locked to ensure in order submission */ 705599d49deSDave Jiang return &desc->txd; 706599d49deSDave Jiang } 707599d49deSDave Jiang 708