1*8d7f2e76SPhilippe Mathieu-Daudé /* 2*8d7f2e76SPhilippe Mathieu-Daudé * DMA helper functions 3*8d7f2e76SPhilippe Mathieu-Daudé * 4*8d7f2e76SPhilippe Mathieu-Daudé * Copyright (c) 2009,2020 Red Hat 5*8d7f2e76SPhilippe Mathieu-Daudé * 6*8d7f2e76SPhilippe Mathieu-Daudé * This work is licensed under the terms of the GNU General Public License 7*8d7f2e76SPhilippe Mathieu-Daudé * (GNU GPL), version 2 or later. 8*8d7f2e76SPhilippe Mathieu-Daudé */ 9*8d7f2e76SPhilippe Mathieu-Daudé 10*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/osdep.h" 11*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/block-backend.h" 12*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/dma.h" 13*8d7f2e76SPhilippe Mathieu-Daudé #include "trace/trace-root.h" 14*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/thread.h" 15*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/main-loop.h" 16*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/cpu-timers.h" 17*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/range.h" 18*8d7f2e76SPhilippe Mathieu-Daudé 19*8d7f2e76SPhilippe Mathieu-Daudé /* #define DEBUG_IOMMU */ 20*8d7f2e76SPhilippe Mathieu-Daudé 21*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult dma_memory_set(AddressSpace *as, dma_addr_t addr, 22*8d7f2e76SPhilippe Mathieu-Daudé uint8_t c, dma_addr_t len, MemTxAttrs attrs) 23*8d7f2e76SPhilippe Mathieu-Daudé { 24*8d7f2e76SPhilippe Mathieu-Daudé dma_barrier(as, DMA_DIRECTION_FROM_DEVICE); 25*8d7f2e76SPhilippe Mathieu-Daudé 26*8d7f2e76SPhilippe Mathieu-Daudé return address_space_set(as, addr, c, len, attrs); 27*8d7f2e76SPhilippe Mathieu-Daudé } 28*8d7f2e76SPhilippe Mathieu-Daudé 29*8d7f2e76SPhilippe Mathieu-Daudé void qemu_sglist_init(QEMUSGList *qsg, DeviceState *dev, int alloc_hint, 30*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace *as) 31*8d7f2e76SPhilippe Mathieu-Daudé { 32*8d7f2e76SPhilippe Mathieu-Daudé qsg->sg = g_new(ScatterGatherEntry, alloc_hint); 33*8d7f2e76SPhilippe Mathieu-Daudé qsg->nsg = 0; 34*8d7f2e76SPhilippe Mathieu-Daudé qsg->nalloc = alloc_hint; 35*8d7f2e76SPhilippe Mathieu-Daudé qsg->size = 0; 36*8d7f2e76SPhilippe Mathieu-Daudé qsg->as = as; 37*8d7f2e76SPhilippe Mathieu-Daudé qsg->dev = dev; 38*8d7f2e76SPhilippe Mathieu-Daudé object_ref(OBJECT(dev)); 39*8d7f2e76SPhilippe Mathieu-Daudé } 40*8d7f2e76SPhilippe Mathieu-Daudé 41*8d7f2e76SPhilippe Mathieu-Daudé void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len) 42*8d7f2e76SPhilippe Mathieu-Daudé { 43*8d7f2e76SPhilippe Mathieu-Daudé if (qsg->nsg == qsg->nalloc) { 44*8d7f2e76SPhilippe Mathieu-Daudé qsg->nalloc = 2 * qsg->nalloc + 1; 45*8d7f2e76SPhilippe Mathieu-Daudé qsg->sg = g_renew(ScatterGatherEntry, qsg->sg, qsg->nalloc); 46*8d7f2e76SPhilippe Mathieu-Daudé } 47*8d7f2e76SPhilippe Mathieu-Daudé qsg->sg[qsg->nsg].base = base; 48*8d7f2e76SPhilippe Mathieu-Daudé qsg->sg[qsg->nsg].len = len; 49*8d7f2e76SPhilippe Mathieu-Daudé qsg->size += len; 50*8d7f2e76SPhilippe Mathieu-Daudé ++qsg->nsg; 51*8d7f2e76SPhilippe Mathieu-Daudé } 52*8d7f2e76SPhilippe Mathieu-Daudé 53*8d7f2e76SPhilippe Mathieu-Daudé void qemu_sglist_destroy(QEMUSGList *qsg) 54*8d7f2e76SPhilippe Mathieu-Daudé { 55*8d7f2e76SPhilippe Mathieu-Daudé object_unref(OBJECT(qsg->dev)); 56*8d7f2e76SPhilippe Mathieu-Daudé g_free(qsg->sg); 57*8d7f2e76SPhilippe Mathieu-Daudé memset(qsg, 0, sizeof(*qsg)); 58*8d7f2e76SPhilippe Mathieu-Daudé } 59*8d7f2e76SPhilippe Mathieu-Daudé 60*8d7f2e76SPhilippe Mathieu-Daudé typedef struct { 61*8d7f2e76SPhilippe Mathieu-Daudé BlockAIOCB common; 62*8d7f2e76SPhilippe Mathieu-Daudé AioContext *ctx; 63*8d7f2e76SPhilippe Mathieu-Daudé BlockAIOCB *acb; 64*8d7f2e76SPhilippe Mathieu-Daudé QEMUSGList *sg; 65*8d7f2e76SPhilippe Mathieu-Daudé uint32_t align; 66*8d7f2e76SPhilippe Mathieu-Daudé uint64_t offset; 67*8d7f2e76SPhilippe Mathieu-Daudé DMADirection dir; 68*8d7f2e76SPhilippe Mathieu-Daudé int sg_cur_index; 69*8d7f2e76SPhilippe Mathieu-Daudé dma_addr_t sg_cur_byte; 70*8d7f2e76SPhilippe Mathieu-Daudé QEMUIOVector iov; 71*8d7f2e76SPhilippe Mathieu-Daudé QEMUBH *bh; 72*8d7f2e76SPhilippe Mathieu-Daudé DMAIOFunc *io_func; 73*8d7f2e76SPhilippe Mathieu-Daudé void *io_func_opaque; 74*8d7f2e76SPhilippe Mathieu-Daudé } DMAAIOCB; 75*8d7f2e76SPhilippe Mathieu-Daudé 76*8d7f2e76SPhilippe Mathieu-Daudé static void dma_blk_cb(void *opaque, int ret); 77*8d7f2e76SPhilippe Mathieu-Daudé 78*8d7f2e76SPhilippe Mathieu-Daudé static void reschedule_dma(void *opaque) 79*8d7f2e76SPhilippe Mathieu-Daudé { 80*8d7f2e76SPhilippe Mathieu-Daudé DMAAIOCB *dbs = (DMAAIOCB *)opaque; 81*8d7f2e76SPhilippe Mathieu-Daudé 82*8d7f2e76SPhilippe Mathieu-Daudé assert(!dbs->acb && dbs->bh); 83*8d7f2e76SPhilippe Mathieu-Daudé qemu_bh_delete(dbs->bh); 84*8d7f2e76SPhilippe Mathieu-Daudé dbs->bh = NULL; 85*8d7f2e76SPhilippe Mathieu-Daudé dma_blk_cb(dbs, 0); 86*8d7f2e76SPhilippe Mathieu-Daudé } 87*8d7f2e76SPhilippe Mathieu-Daudé 88*8d7f2e76SPhilippe Mathieu-Daudé static void dma_blk_unmap(DMAAIOCB *dbs) 89*8d7f2e76SPhilippe Mathieu-Daudé { 90*8d7f2e76SPhilippe Mathieu-Daudé int i; 91*8d7f2e76SPhilippe Mathieu-Daudé 92*8d7f2e76SPhilippe Mathieu-Daudé for (i = 0; i < dbs->iov.niov; ++i) { 93*8d7f2e76SPhilippe Mathieu-Daudé dma_memory_unmap(dbs->sg->as, dbs->iov.iov[i].iov_base, 94*8d7f2e76SPhilippe Mathieu-Daudé dbs->iov.iov[i].iov_len, dbs->dir, 95*8d7f2e76SPhilippe Mathieu-Daudé dbs->iov.iov[i].iov_len); 96*8d7f2e76SPhilippe Mathieu-Daudé } 97*8d7f2e76SPhilippe Mathieu-Daudé qemu_iovec_reset(&dbs->iov); 98*8d7f2e76SPhilippe Mathieu-Daudé } 99*8d7f2e76SPhilippe Mathieu-Daudé 100*8d7f2e76SPhilippe Mathieu-Daudé static void dma_complete(DMAAIOCB *dbs, int ret) 101*8d7f2e76SPhilippe Mathieu-Daudé { 102*8d7f2e76SPhilippe Mathieu-Daudé trace_dma_complete(dbs, ret, dbs->common.cb); 103*8d7f2e76SPhilippe Mathieu-Daudé 104*8d7f2e76SPhilippe Mathieu-Daudé assert(!dbs->acb && !dbs->bh); 105*8d7f2e76SPhilippe Mathieu-Daudé dma_blk_unmap(dbs); 106*8d7f2e76SPhilippe Mathieu-Daudé if (dbs->common.cb) { 107*8d7f2e76SPhilippe Mathieu-Daudé dbs->common.cb(dbs->common.opaque, ret); 108*8d7f2e76SPhilippe Mathieu-Daudé } 109*8d7f2e76SPhilippe Mathieu-Daudé qemu_iovec_destroy(&dbs->iov); 110*8d7f2e76SPhilippe Mathieu-Daudé qemu_aio_unref(dbs); 111*8d7f2e76SPhilippe Mathieu-Daudé } 112*8d7f2e76SPhilippe Mathieu-Daudé 113*8d7f2e76SPhilippe Mathieu-Daudé static void dma_blk_cb(void *opaque, int ret) 114*8d7f2e76SPhilippe Mathieu-Daudé { 115*8d7f2e76SPhilippe Mathieu-Daudé DMAAIOCB *dbs = (DMAAIOCB *)opaque; 116*8d7f2e76SPhilippe Mathieu-Daudé AioContext *ctx = dbs->ctx; 117*8d7f2e76SPhilippe Mathieu-Daudé dma_addr_t cur_addr, cur_len; 118*8d7f2e76SPhilippe Mathieu-Daudé void *mem; 119*8d7f2e76SPhilippe Mathieu-Daudé 120*8d7f2e76SPhilippe Mathieu-Daudé trace_dma_blk_cb(dbs, ret); 121*8d7f2e76SPhilippe Mathieu-Daudé 122*8d7f2e76SPhilippe Mathieu-Daudé aio_context_acquire(ctx); 123*8d7f2e76SPhilippe Mathieu-Daudé dbs->acb = NULL; 124*8d7f2e76SPhilippe Mathieu-Daudé dbs->offset += dbs->iov.size; 125*8d7f2e76SPhilippe Mathieu-Daudé 126*8d7f2e76SPhilippe Mathieu-Daudé if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) { 127*8d7f2e76SPhilippe Mathieu-Daudé dma_complete(dbs, ret); 128*8d7f2e76SPhilippe Mathieu-Daudé goto out; 129*8d7f2e76SPhilippe Mathieu-Daudé } 130*8d7f2e76SPhilippe Mathieu-Daudé dma_blk_unmap(dbs); 131*8d7f2e76SPhilippe Mathieu-Daudé 132*8d7f2e76SPhilippe Mathieu-Daudé while (dbs->sg_cur_index < dbs->sg->nsg) { 133*8d7f2e76SPhilippe Mathieu-Daudé cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte; 134*8d7f2e76SPhilippe Mathieu-Daudé cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte; 135*8d7f2e76SPhilippe Mathieu-Daudé mem = dma_memory_map(dbs->sg->as, cur_addr, &cur_len, dbs->dir, 136*8d7f2e76SPhilippe Mathieu-Daudé MEMTXATTRS_UNSPECIFIED); 137*8d7f2e76SPhilippe Mathieu-Daudé /* 138*8d7f2e76SPhilippe Mathieu-Daudé * Make reads deterministic in icount mode. Windows sometimes issues 139*8d7f2e76SPhilippe Mathieu-Daudé * disk read requests with overlapping SGs. It leads 140*8d7f2e76SPhilippe Mathieu-Daudé * to non-determinism, because resulting buffer contents may be mixed 141*8d7f2e76SPhilippe Mathieu-Daudé * from several sectors. This code splits all SGs into several 142*8d7f2e76SPhilippe Mathieu-Daudé * groups. SGs in every group do not overlap. 143*8d7f2e76SPhilippe Mathieu-Daudé */ 144*8d7f2e76SPhilippe Mathieu-Daudé if (mem && icount_enabled() && dbs->dir == DMA_DIRECTION_FROM_DEVICE) { 145*8d7f2e76SPhilippe Mathieu-Daudé int i; 146*8d7f2e76SPhilippe Mathieu-Daudé for (i = 0 ; i < dbs->iov.niov ; ++i) { 147*8d7f2e76SPhilippe Mathieu-Daudé if (ranges_overlap((intptr_t)dbs->iov.iov[i].iov_base, 148*8d7f2e76SPhilippe Mathieu-Daudé dbs->iov.iov[i].iov_len, (intptr_t)mem, 149*8d7f2e76SPhilippe Mathieu-Daudé cur_len)) { 150*8d7f2e76SPhilippe Mathieu-Daudé dma_memory_unmap(dbs->sg->as, mem, cur_len, 151*8d7f2e76SPhilippe Mathieu-Daudé dbs->dir, cur_len); 152*8d7f2e76SPhilippe Mathieu-Daudé mem = NULL; 153*8d7f2e76SPhilippe Mathieu-Daudé break; 154*8d7f2e76SPhilippe Mathieu-Daudé } 155*8d7f2e76SPhilippe Mathieu-Daudé } 156*8d7f2e76SPhilippe Mathieu-Daudé } 157*8d7f2e76SPhilippe Mathieu-Daudé if (!mem) 158*8d7f2e76SPhilippe Mathieu-Daudé break; 159*8d7f2e76SPhilippe Mathieu-Daudé qemu_iovec_add(&dbs->iov, mem, cur_len); 160*8d7f2e76SPhilippe Mathieu-Daudé dbs->sg_cur_byte += cur_len; 161*8d7f2e76SPhilippe Mathieu-Daudé if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) { 162*8d7f2e76SPhilippe Mathieu-Daudé dbs->sg_cur_byte = 0; 163*8d7f2e76SPhilippe Mathieu-Daudé ++dbs->sg_cur_index; 164*8d7f2e76SPhilippe Mathieu-Daudé } 165*8d7f2e76SPhilippe Mathieu-Daudé } 166*8d7f2e76SPhilippe Mathieu-Daudé 167*8d7f2e76SPhilippe Mathieu-Daudé if (dbs->iov.size == 0) { 168*8d7f2e76SPhilippe Mathieu-Daudé trace_dma_map_wait(dbs); 169*8d7f2e76SPhilippe Mathieu-Daudé dbs->bh = aio_bh_new(ctx, reschedule_dma, dbs); 170*8d7f2e76SPhilippe Mathieu-Daudé cpu_register_map_client(dbs->bh); 171*8d7f2e76SPhilippe Mathieu-Daudé goto out; 172*8d7f2e76SPhilippe Mathieu-Daudé } 173*8d7f2e76SPhilippe Mathieu-Daudé 174*8d7f2e76SPhilippe Mathieu-Daudé if (!QEMU_IS_ALIGNED(dbs->iov.size, dbs->align)) { 175*8d7f2e76SPhilippe Mathieu-Daudé qemu_iovec_discard_back(&dbs->iov, 176*8d7f2e76SPhilippe Mathieu-Daudé QEMU_ALIGN_DOWN(dbs->iov.size, dbs->align)); 177*8d7f2e76SPhilippe Mathieu-Daudé } 178*8d7f2e76SPhilippe Mathieu-Daudé 179*8d7f2e76SPhilippe Mathieu-Daudé dbs->acb = dbs->io_func(dbs->offset, &dbs->iov, 180*8d7f2e76SPhilippe Mathieu-Daudé dma_blk_cb, dbs, dbs->io_func_opaque); 181*8d7f2e76SPhilippe Mathieu-Daudé assert(dbs->acb); 182*8d7f2e76SPhilippe Mathieu-Daudé out: 183*8d7f2e76SPhilippe Mathieu-Daudé aio_context_release(ctx); 184*8d7f2e76SPhilippe Mathieu-Daudé } 185*8d7f2e76SPhilippe Mathieu-Daudé 186*8d7f2e76SPhilippe Mathieu-Daudé static void dma_aio_cancel(BlockAIOCB *acb) 187*8d7f2e76SPhilippe Mathieu-Daudé { 188*8d7f2e76SPhilippe Mathieu-Daudé DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common); 189*8d7f2e76SPhilippe Mathieu-Daudé 190*8d7f2e76SPhilippe Mathieu-Daudé trace_dma_aio_cancel(dbs); 191*8d7f2e76SPhilippe Mathieu-Daudé 192*8d7f2e76SPhilippe Mathieu-Daudé assert(!(dbs->acb && dbs->bh)); 193*8d7f2e76SPhilippe Mathieu-Daudé if (dbs->acb) { 194*8d7f2e76SPhilippe Mathieu-Daudé /* This will invoke dma_blk_cb. */ 195*8d7f2e76SPhilippe Mathieu-Daudé blk_aio_cancel_async(dbs->acb); 196*8d7f2e76SPhilippe Mathieu-Daudé return; 197*8d7f2e76SPhilippe Mathieu-Daudé } 198*8d7f2e76SPhilippe Mathieu-Daudé 199*8d7f2e76SPhilippe Mathieu-Daudé if (dbs->bh) { 200*8d7f2e76SPhilippe Mathieu-Daudé cpu_unregister_map_client(dbs->bh); 201*8d7f2e76SPhilippe Mathieu-Daudé qemu_bh_delete(dbs->bh); 202*8d7f2e76SPhilippe Mathieu-Daudé dbs->bh = NULL; 203*8d7f2e76SPhilippe Mathieu-Daudé } 204*8d7f2e76SPhilippe Mathieu-Daudé if (dbs->common.cb) { 205*8d7f2e76SPhilippe Mathieu-Daudé dbs->common.cb(dbs->common.opaque, -ECANCELED); 206*8d7f2e76SPhilippe Mathieu-Daudé } 207*8d7f2e76SPhilippe Mathieu-Daudé } 208*8d7f2e76SPhilippe Mathieu-Daudé 209*8d7f2e76SPhilippe Mathieu-Daudé static const AIOCBInfo dma_aiocb_info = { 210*8d7f2e76SPhilippe Mathieu-Daudé .aiocb_size = sizeof(DMAAIOCB), 211*8d7f2e76SPhilippe Mathieu-Daudé .cancel_async = dma_aio_cancel, 212*8d7f2e76SPhilippe Mathieu-Daudé }; 213*8d7f2e76SPhilippe Mathieu-Daudé 214*8d7f2e76SPhilippe Mathieu-Daudé BlockAIOCB *dma_blk_io(AioContext *ctx, 215*8d7f2e76SPhilippe Mathieu-Daudé QEMUSGList *sg, uint64_t offset, uint32_t align, 216*8d7f2e76SPhilippe Mathieu-Daudé DMAIOFunc *io_func, void *io_func_opaque, 217*8d7f2e76SPhilippe Mathieu-Daudé BlockCompletionFunc *cb, 218*8d7f2e76SPhilippe Mathieu-Daudé void *opaque, DMADirection dir) 219*8d7f2e76SPhilippe Mathieu-Daudé { 220*8d7f2e76SPhilippe Mathieu-Daudé DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, NULL, cb, opaque); 221*8d7f2e76SPhilippe Mathieu-Daudé 222*8d7f2e76SPhilippe Mathieu-Daudé trace_dma_blk_io(dbs, io_func_opaque, offset, (dir == DMA_DIRECTION_TO_DEVICE)); 223*8d7f2e76SPhilippe Mathieu-Daudé 224*8d7f2e76SPhilippe Mathieu-Daudé dbs->acb = NULL; 225*8d7f2e76SPhilippe Mathieu-Daudé dbs->sg = sg; 226*8d7f2e76SPhilippe Mathieu-Daudé dbs->ctx = ctx; 227*8d7f2e76SPhilippe Mathieu-Daudé dbs->offset = offset; 228*8d7f2e76SPhilippe Mathieu-Daudé dbs->align = align; 229*8d7f2e76SPhilippe Mathieu-Daudé dbs->sg_cur_index = 0; 230*8d7f2e76SPhilippe Mathieu-Daudé dbs->sg_cur_byte = 0; 231*8d7f2e76SPhilippe Mathieu-Daudé dbs->dir = dir; 232*8d7f2e76SPhilippe Mathieu-Daudé dbs->io_func = io_func; 233*8d7f2e76SPhilippe Mathieu-Daudé dbs->io_func_opaque = io_func_opaque; 234*8d7f2e76SPhilippe Mathieu-Daudé dbs->bh = NULL; 235*8d7f2e76SPhilippe Mathieu-Daudé qemu_iovec_init(&dbs->iov, sg->nsg); 236*8d7f2e76SPhilippe Mathieu-Daudé dma_blk_cb(dbs, 0); 237*8d7f2e76SPhilippe Mathieu-Daudé return &dbs->common; 238*8d7f2e76SPhilippe Mathieu-Daudé } 239*8d7f2e76SPhilippe Mathieu-Daudé 240*8d7f2e76SPhilippe Mathieu-Daudé 241*8d7f2e76SPhilippe Mathieu-Daudé static 242*8d7f2e76SPhilippe Mathieu-Daudé BlockAIOCB *dma_blk_read_io_func(int64_t offset, QEMUIOVector *iov, 243*8d7f2e76SPhilippe Mathieu-Daudé BlockCompletionFunc *cb, void *cb_opaque, 244*8d7f2e76SPhilippe Mathieu-Daudé void *opaque) 245*8d7f2e76SPhilippe Mathieu-Daudé { 246*8d7f2e76SPhilippe Mathieu-Daudé BlockBackend *blk = opaque; 247*8d7f2e76SPhilippe Mathieu-Daudé return blk_aio_preadv(blk, offset, iov, 0, cb, cb_opaque); 248*8d7f2e76SPhilippe Mathieu-Daudé } 249*8d7f2e76SPhilippe Mathieu-Daudé 250*8d7f2e76SPhilippe Mathieu-Daudé BlockAIOCB *dma_blk_read(BlockBackend *blk, 251*8d7f2e76SPhilippe Mathieu-Daudé QEMUSGList *sg, uint64_t offset, uint32_t align, 252*8d7f2e76SPhilippe Mathieu-Daudé void (*cb)(void *opaque, int ret), void *opaque) 253*8d7f2e76SPhilippe Mathieu-Daudé { 254*8d7f2e76SPhilippe Mathieu-Daudé return dma_blk_io(blk_get_aio_context(blk), sg, offset, align, 255*8d7f2e76SPhilippe Mathieu-Daudé dma_blk_read_io_func, blk, cb, opaque, 256*8d7f2e76SPhilippe Mathieu-Daudé DMA_DIRECTION_FROM_DEVICE); 257*8d7f2e76SPhilippe Mathieu-Daudé } 258*8d7f2e76SPhilippe Mathieu-Daudé 259*8d7f2e76SPhilippe Mathieu-Daudé static 260*8d7f2e76SPhilippe Mathieu-Daudé BlockAIOCB *dma_blk_write_io_func(int64_t offset, QEMUIOVector *iov, 261*8d7f2e76SPhilippe Mathieu-Daudé BlockCompletionFunc *cb, void *cb_opaque, 262*8d7f2e76SPhilippe Mathieu-Daudé void *opaque) 263*8d7f2e76SPhilippe Mathieu-Daudé { 264*8d7f2e76SPhilippe Mathieu-Daudé BlockBackend *blk = opaque; 265*8d7f2e76SPhilippe Mathieu-Daudé return blk_aio_pwritev(blk, offset, iov, 0, cb, cb_opaque); 266*8d7f2e76SPhilippe Mathieu-Daudé } 267*8d7f2e76SPhilippe Mathieu-Daudé 268*8d7f2e76SPhilippe Mathieu-Daudé BlockAIOCB *dma_blk_write(BlockBackend *blk, 269*8d7f2e76SPhilippe Mathieu-Daudé QEMUSGList *sg, uint64_t offset, uint32_t align, 270*8d7f2e76SPhilippe Mathieu-Daudé void (*cb)(void *opaque, int ret), void *opaque) 271*8d7f2e76SPhilippe Mathieu-Daudé { 272*8d7f2e76SPhilippe Mathieu-Daudé return dma_blk_io(blk_get_aio_context(blk), sg, offset, align, 273*8d7f2e76SPhilippe Mathieu-Daudé dma_blk_write_io_func, blk, cb, opaque, 274*8d7f2e76SPhilippe Mathieu-Daudé DMA_DIRECTION_TO_DEVICE); 275*8d7f2e76SPhilippe Mathieu-Daudé } 276*8d7f2e76SPhilippe Mathieu-Daudé 277*8d7f2e76SPhilippe Mathieu-Daudé 278*8d7f2e76SPhilippe Mathieu-Daudé static MemTxResult dma_buf_rw(void *buf, dma_addr_t len, dma_addr_t *residual, 279*8d7f2e76SPhilippe Mathieu-Daudé QEMUSGList *sg, DMADirection dir, 280*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs) 281*8d7f2e76SPhilippe Mathieu-Daudé { 282*8d7f2e76SPhilippe Mathieu-Daudé uint8_t *ptr = buf; 283*8d7f2e76SPhilippe Mathieu-Daudé dma_addr_t xresidual; 284*8d7f2e76SPhilippe Mathieu-Daudé int sg_cur_index; 285*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult res = MEMTX_OK; 286*8d7f2e76SPhilippe Mathieu-Daudé 287*8d7f2e76SPhilippe Mathieu-Daudé xresidual = sg->size; 288*8d7f2e76SPhilippe Mathieu-Daudé sg_cur_index = 0; 289*8d7f2e76SPhilippe Mathieu-Daudé len = MIN(len, xresidual); 290*8d7f2e76SPhilippe Mathieu-Daudé while (len > 0) { 291*8d7f2e76SPhilippe Mathieu-Daudé ScatterGatherEntry entry = sg->sg[sg_cur_index++]; 292*8d7f2e76SPhilippe Mathieu-Daudé dma_addr_t xfer = MIN(len, entry.len); 293*8d7f2e76SPhilippe Mathieu-Daudé res |= dma_memory_rw(sg->as, entry.base, ptr, xfer, dir, attrs); 294*8d7f2e76SPhilippe Mathieu-Daudé ptr += xfer; 295*8d7f2e76SPhilippe Mathieu-Daudé len -= xfer; 296*8d7f2e76SPhilippe Mathieu-Daudé xresidual -= xfer; 297*8d7f2e76SPhilippe Mathieu-Daudé } 298*8d7f2e76SPhilippe Mathieu-Daudé 299*8d7f2e76SPhilippe Mathieu-Daudé if (residual) { 300*8d7f2e76SPhilippe Mathieu-Daudé *residual = xresidual; 301*8d7f2e76SPhilippe Mathieu-Daudé } 302*8d7f2e76SPhilippe Mathieu-Daudé return res; 303*8d7f2e76SPhilippe Mathieu-Daudé } 304*8d7f2e76SPhilippe Mathieu-Daudé 305*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult dma_buf_read(void *ptr, dma_addr_t len, dma_addr_t *residual, 306*8d7f2e76SPhilippe Mathieu-Daudé QEMUSGList *sg, MemTxAttrs attrs) 307*8d7f2e76SPhilippe Mathieu-Daudé { 308*8d7f2e76SPhilippe Mathieu-Daudé return dma_buf_rw(ptr, len, residual, sg, DMA_DIRECTION_FROM_DEVICE, attrs); 309*8d7f2e76SPhilippe Mathieu-Daudé } 310*8d7f2e76SPhilippe Mathieu-Daudé 311*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult dma_buf_write(void *ptr, dma_addr_t len, dma_addr_t *residual, 312*8d7f2e76SPhilippe Mathieu-Daudé QEMUSGList *sg, MemTxAttrs attrs) 313*8d7f2e76SPhilippe Mathieu-Daudé { 314*8d7f2e76SPhilippe Mathieu-Daudé return dma_buf_rw(ptr, len, residual, sg, DMA_DIRECTION_TO_DEVICE, attrs); 315*8d7f2e76SPhilippe Mathieu-Daudé } 316*8d7f2e76SPhilippe Mathieu-Daudé 317*8d7f2e76SPhilippe Mathieu-Daudé void dma_acct_start(BlockBackend *blk, BlockAcctCookie *cookie, 318*8d7f2e76SPhilippe Mathieu-Daudé QEMUSGList *sg, enum BlockAcctType type) 319*8d7f2e76SPhilippe Mathieu-Daudé { 320*8d7f2e76SPhilippe Mathieu-Daudé block_acct_start(blk_get_stats(blk), cookie, sg->size, type); 321*8d7f2e76SPhilippe Mathieu-Daudé } 322*8d7f2e76SPhilippe Mathieu-Daudé 323*8d7f2e76SPhilippe Mathieu-Daudé uint64_t dma_aligned_pow2_mask(uint64_t start, uint64_t end, int max_addr_bits) 324*8d7f2e76SPhilippe Mathieu-Daudé { 325*8d7f2e76SPhilippe Mathieu-Daudé uint64_t max_mask = UINT64_MAX, addr_mask = end - start; 326*8d7f2e76SPhilippe Mathieu-Daudé uint64_t alignment_mask, size_mask; 327*8d7f2e76SPhilippe Mathieu-Daudé 328*8d7f2e76SPhilippe Mathieu-Daudé if (max_addr_bits != 64) { 329*8d7f2e76SPhilippe Mathieu-Daudé max_mask = (1ULL << max_addr_bits) - 1; 330*8d7f2e76SPhilippe Mathieu-Daudé } 331*8d7f2e76SPhilippe Mathieu-Daudé 332*8d7f2e76SPhilippe Mathieu-Daudé alignment_mask = start ? (start & -start) - 1 : max_mask; 333*8d7f2e76SPhilippe Mathieu-Daudé alignment_mask = MIN(alignment_mask, max_mask); 334*8d7f2e76SPhilippe Mathieu-Daudé size_mask = MIN(addr_mask, max_mask); 335*8d7f2e76SPhilippe Mathieu-Daudé 336*8d7f2e76SPhilippe Mathieu-Daudé if (alignment_mask <= size_mask) { 337*8d7f2e76SPhilippe Mathieu-Daudé /* Increase the alignment of start */ 338*8d7f2e76SPhilippe Mathieu-Daudé return alignment_mask; 339*8d7f2e76SPhilippe Mathieu-Daudé } else { 340*8d7f2e76SPhilippe Mathieu-Daudé /* Find the largest page mask from size */ 341*8d7f2e76SPhilippe Mathieu-Daudé if (addr_mask == UINT64_MAX) { 342*8d7f2e76SPhilippe Mathieu-Daudé return UINT64_MAX; 343*8d7f2e76SPhilippe Mathieu-Daudé } 344*8d7f2e76SPhilippe Mathieu-Daudé return (1ULL << (63 - clz64(addr_mask + 1))) - 1; 345*8d7f2e76SPhilippe Mathieu-Daudé } 346*8d7f2e76SPhilippe Mathieu-Daudé } 347*8d7f2e76SPhilippe Mathieu-Daudé 348