Lines Matching +full:lock +full:- +full:offset
4 * Copyright 2016 - 2018 Red Hat, Inc.
11 * See the COPYING file in the top-level directory.
19 #include "qemu/defer-call.h"
20 #include "qemu/error-report.h"
21 #include "qemu/host-pci-mmio.h"
22 #include "qemu/main-loop.h"
27 #include "qemu/vfio-helpers.h"
28 #include "block/block-io.h"
30 #include "system/block-backend.h"
45 #define NVME_NUM_REQS (NVME_QUEUE_SIZE - 1)
73 int free_req_next; /* q->reqs[] index of next free req */
77 QemuMutex lock; member
86 /* Fields protected by @lock */
95 /* Thread-safe, no lock necessary */
177 q->head = q->tail = 0; in nvme_init_queue()
178 q->queue = qemu_try_memalign(qemu_real_host_page_size(), bytes); in nvme_init_queue()
179 if (!q->queue) { in nvme_init_queue()
183 memset(q->queue, 0, bytes); in nvme_init_queue()
184 r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova, errp); in nvme_init_queue()
193 qemu_vfree(q->queue); in nvme_free_queue()
198 trace_nvme_free_queue_pair(q->index, q, &q->cq, &q->sq); in nvme_free_queue_pair()
199 if (q->completion_bh) { in nvme_free_queue_pair()
200 qemu_bh_delete(q->completion_bh); in nvme_free_queue_pair()
202 nvme_free_queue(&q->sq); in nvme_free_queue_pair()
203 nvme_free_queue(&q->cq); in nvme_free_queue_pair()
204 qemu_vfree(q->prp_list_pages); in nvme_free_queue_pair()
205 qemu_mutex_destroy(&q->lock); in nvme_free_queue_pair()
213 qemu_mutex_lock(&q->lock); in nvme_free_req_queue_cb()
214 while (q->free_req_head != -1 && in nvme_free_req_queue_cb()
215 qemu_co_enter_next(&q->free_req_queue, &q->lock)) { in nvme_free_req_queue_cb()
218 qemu_mutex_unlock(&q->lock); in nvme_free_req_queue_cb()
238 event_notifier_get_fd(s->irq_notifier)); in nvme_create_queue_pair()
239 bytes = QEMU_ALIGN_UP(s->page_size * NVME_NUM_REQS, in nvme_create_queue_pair()
241 q->prp_list_pages = qemu_try_memalign(qemu_real_host_page_size(), bytes); in nvme_create_queue_pair()
242 if (!q->prp_list_pages) { in nvme_create_queue_pair()
246 memset(q->prp_list_pages, 0, bytes); in nvme_create_queue_pair()
247 qemu_mutex_init(&q->lock); in nvme_create_queue_pair()
248 q->s = s; in nvme_create_queue_pair()
249 q->index = idx; in nvme_create_queue_pair()
250 qemu_co_queue_init(&q->free_req_queue); in nvme_create_queue_pair()
251 q->completion_bh = aio_bh_new(aio_context, nvme_process_completion_bh, q); in nvme_create_queue_pair()
252 r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages, bytes, in nvme_create_queue_pair()
258 q->free_req_head = -1; in nvme_create_queue_pair()
260 NVMeRequest *req = &q->reqs[i]; in nvme_create_queue_pair()
261 req->cid = i + 1; in nvme_create_queue_pair()
262 req->free_req_next = q->free_req_head; in nvme_create_queue_pair()
263 q->free_req_head = i; in nvme_create_queue_pair()
264 req->prp_list_page = q->prp_list_pages + i * s->page_size; in nvme_create_queue_pair()
265 req->prp_list_iova = prp_list_iova + i * s->page_size; in nvme_create_queue_pair()
268 if (!nvme_init_queue(s, &q->sq, size, NVME_SQ_ENTRY_BYTES, errp)) { in nvme_create_queue_pair()
271 q->sq.doorbell = &s->doorbells[idx * s->doorbell_scale].sq_tail; in nvme_create_queue_pair()
273 if (!nvme_init_queue(s, &q->cq, size, NVME_CQ_ENTRY_BYTES, errp)) { in nvme_create_queue_pair()
276 q->cq.doorbell = &s->doorbells[idx * s->doorbell_scale].cq_head; in nvme_create_queue_pair()
284 /* With q->lock */
287 BDRVNVMeState *s = q->s; in nvme_kick()
289 if (!q->need_kick) { in nvme_kick()
292 trace_nvme_kick(s, q->index); in nvme_kick()
293 assert(!(q->sq.tail & 0xFF00)); in nvme_kick()
296 host_pci_stl_le_p(q->sq.doorbell, q->sq.tail); in nvme_kick()
297 q->inflight += q->need_kick; in nvme_kick()
298 q->need_kick = 0; in nvme_kick()
305 req = &q->reqs[q->free_req_head]; in nvme_get_free_req_nofail_locked()
306 q->free_req_head = req->free_req_next; in nvme_get_free_req_nofail_locked()
307 req->free_req_next = -1; in nvme_get_free_req_nofail_locked()
314 QEMU_LOCK_GUARD(&q->lock); in nvme_get_free_req_nowait()
315 if (q->free_req_head == -1) { in nvme_get_free_req_nowait()
327 QEMU_LOCK_GUARD(&q->lock); in nvme_get_free_req()
329 while (q->free_req_head == -1) { in nvme_get_free_req()
330 trace_nvme_free_req_queue_wait(q->s, q->index); in nvme_get_free_req()
331 qemu_co_queue_wait(&q->free_req_queue, &q->lock); in nvme_get_free_req()
337 /* With q->lock */
340 req->free_req_next = q->free_req_head; in nvme_put_free_req_locked()
341 q->free_req_head = req - q->reqs; in nvme_put_free_req_locked()
344 /* With q->lock */
347 if (!qemu_co_queue_empty(&q->free_req_queue)) { in nvme_wake_free_req_locked()
348 replay_bh_schedule_oneshot_event(q->s->aio_context, in nvme_wake_free_req_locked()
356 qemu_mutex_lock(&q->lock); in nvme_put_free_req_and_wake()
359 qemu_mutex_unlock(&q->lock); in nvme_put_free_req_and_wake()
364 uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF; in nvme_translate_error()
366 trace_nvme_error(le32_to_cpu(c->result), in nvme_translate_error()
367 le16_to_cpu(c->sq_head), in nvme_translate_error()
368 le16_to_cpu(c->sq_id), in nvme_translate_error()
369 le16_to_cpu(c->cid), in nvme_translate_error()
376 return -ENOSYS; in nvme_translate_error()
378 return -EINVAL; in nvme_translate_error()
380 return -EIO; in nvme_translate_error()
384 /* With q->lock */
387 BDRVNVMeState *s = q->s; in nvme_process_completion()
393 trace_nvme_process_completion(s, q->index, q->inflight); in nvme_process_completion()
396 * Support re-entrancy when a request cb() function invokes aio_poll(). in nvme_process_completion()
403 qemu_bh_schedule(q->completion_bh); in nvme_process_completion()
405 assert(q->inflight >= 0); in nvme_process_completion()
406 while (q->inflight) { in nvme_process_completion()
410 c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES]; in nvme_process_completion()
411 if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) { in nvme_process_completion()
416 s->stats.completion_errors++; in nvme_process_completion()
418 q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE; in nvme_process_completion()
419 if (!q->cq.head) { in nvme_process_completion()
420 q->cq_phase = !q->cq_phase; in nvme_process_completion()
422 cid = le16_to_cpu(c->cid); in nvme_process_completion()
429 trace_nvme_complete_command(s, q->index, cid); in nvme_process_completion()
430 preq = &q->reqs[cid - 1]; in nvme_process_completion()
435 preq->cb = preq->opaque = NULL; in nvme_process_completion()
436 q->inflight--; in nvme_process_completion()
437 qemu_mutex_unlock(&q->lock); in nvme_process_completion()
439 qemu_mutex_lock(&q->lock); in nvme_process_completion()
445 host_pci_stl_le_p(q->cq.doorbell, q->cq.head); in nvme_process_completion()
449 qemu_bh_cancel(q->completion_bh); in nvme_process_completion()
464 host_pci_stl_le_p(q->cq.doorbell, q->cq.head); in nvme_process_completion_bh()
488 QEMU_LOCK_GUARD(&q->lock); in nvme_kick_and_check_completions()
497 if (qemu_get_current_aio_context() == q->s->aio_context) { in nvme_deferred_fn()
500 aio_bh_schedule_oneshot(q->s->aio_context, in nvme_deferred_fn()
509 assert(!req->cb); in nvme_submit_command()
510 req->cb = cb; in nvme_submit_command()
511 req->opaque = opaque; in nvme_submit_command()
512 cmd->cid = cpu_to_le16(req->cid); in nvme_submit_command()
514 trace_nvme_submit_command(q->s, q->index, req->cid); in nvme_submit_command()
516 qemu_mutex_lock(&q->lock); in nvme_submit_command()
517 memcpy((uint8_t *)q->sq.queue + in nvme_submit_command()
518 q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd)); in nvme_submit_command()
519 q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE; in nvme_submit_command()
520 q->need_kick++; in nvme_submit_command()
521 qemu_mutex_unlock(&q->lock); in nvme_submit_command()
535 BDRVNVMeState *s = bs->opaque; in nvme_admin_cmd_sync()
536 NVMeQueuePair *q = s->queues[INDEX_ADMIN]; in nvme_admin_cmd_sync()
539 int ret = -EINPROGRESS; in nvme_admin_cmd_sync()
542 return -EBUSY; in nvme_admin_cmd_sync()
546 AIO_WAIT_WHILE(aio_context, ret == -EINPROGRESS); in nvme_admin_cmd_sync()
554 BDRVNVMeState *s = bs->opaque; in nvme_identify()
575 r = qemu_vfio_dma_map(s->vfio, id, id_size, true, &iova, errp); in nvme_identify()
588 if (le32_to_cpu(id->ctrl.nn) < namespace) { in nvme_identify()
592 s->write_cache_supported = le32_to_cpu(id->ctrl.vwc) & 0x1; in nvme_identify()
593 s->max_transfer = (id->ctrl.mdts ? 1 << id->ctrl.mdts : 0) * s->page_size; in nvme_identify()
595 * s->page_size / sizeof(uint64_t) entries. */ in nvme_identify()
596 s->max_transfer = MIN_NON_ZERO(s->max_transfer, in nvme_identify()
597 s->page_size / sizeof(uint64_t) * s->page_size); in nvme_identify()
599 oncs = le16_to_cpu(id->ctrl.oncs); in nvme_identify()
600 s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROES); in nvme_identify()
601 s->supports_discard = !!(oncs & NVME_ONCS_DSM); in nvme_identify()
611 s->nsze = le64_to_cpu(id->ns.nsze); in nvme_identify()
612 lbaf = &id->ns.lbaf[NVME_ID_NS_FLBAS_INDEX(id->ns.flbas)]; in nvme_identify()
614 if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(id->ns.dlfeat) && in nvme_identify()
615 NVME_ID_NS_DLFEAT_READ_BEHAVIOR(id->ns.dlfeat) == in nvme_identify()
617 bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP; in nvme_identify()
620 if (lbaf->ms) { in nvme_identify()
625 if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 || in nvme_identify()
626 (1 << lbaf->ds) > s->page_size) in nvme_identify()
629 lbaf->ds); in nvme_identify()
634 s->blkshift = lbaf->ds; in nvme_identify()
636 qemu_vfio_dma_unmap(s->vfio, id); in nvme_identify()
643 const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES; in nvme_poll_queue()
644 NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset]; in nvme_poll_queue()
646 trace_nvme_poll_queue(q->s, q->index); in nvme_poll_queue()
648 * Do an early check for completions. q->lock isn't needed because in nvme_poll_queue()
652 if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) { in nvme_poll_queue()
656 qemu_mutex_lock(&q->lock); in nvme_poll_queue()
660 qemu_mutex_unlock(&q->lock); in nvme_poll_queue()
667 for (i = 0; i < s->queue_count; i++) { in nvme_poll_queues()
668 nvme_poll_queue(s->queues[i]); in nvme_poll_queues()
684 BDRVNVMeState *s = bs->opaque; in nvme_add_io_queue()
685 unsigned n = s->queue_count; in nvme_add_io_queue()
698 .dptr.prp1 = cpu_to_le64(q->cq.iova), in nvme_add_io_queue()
699 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | n), in nvme_add_io_queue()
708 .dptr.prp1 = cpu_to_le64(q->sq.iova), in nvme_add_io_queue()
709 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | n), in nvme_add_io_queue()
716 s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1); in nvme_add_io_queue()
717 s->queues[n] = q; in nvme_add_io_queue()
718 s->queue_count++; in nvme_add_io_queue()
732 for (i = 0; i < s->queue_count; i++) { in nvme_poll_cb()
733 NVMeQueuePair *q = s->queues[i]; in nvme_poll_cb()
734 const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES; in nvme_poll_cb()
735 NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset]; in nvme_poll_cb()
738 * q->lock isn't needed because nvme_process_completion() only runs in in nvme_poll_cb()
741 if ((le16_to_cpu(cqe->status) & 0x1) != q->cq_phase) { in nvme_poll_cb()
759 BDRVNVMeState *s = bs->opaque; in nvme_init()
770 qemu_co_mutex_init(&s->dma_map_lock); in nvme_init()
771 qemu_co_queue_init(&s->dma_flush_queue); in nvme_init()
772 s->device = g_strdup(device); in nvme_init()
773 s->nsid = namespace; in nvme_init()
774 s->aio_context = bdrv_get_aio_context(bs); in nvme_init()
775 ret = event_notifier_init(&s->irq_notifier[MSIX_SHARED_IRQ_IDX], 0); in nvme_init()
781 s->vfio = qemu_vfio_open_pci(device, errp); in nvme_init()
782 if (!s->vfio) { in nvme_init()
783 ret = -EINVAL; in nvme_init()
787 regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, sizeof(NvmeBar), in nvme_init()
790 ret = -EINVAL; in nvme_init()
796 cap = host_pci_ldq_le_p(®s->cap); in nvme_init()
812 ret = -EINVAL; in nvme_init()
816 s->page_size = 1u << (12 + NVME_CAP_MPSMIN(cap)); in nvme_init()
817 s->doorbell_scale = (4 << NVME_CAP_DSTRD(cap)) / sizeof(uint32_t); in nvme_init()
818 bs->bl.opt_mem_alignment = s->page_size; in nvme_init()
819 bs->bl.request_alignment = s->page_size; in nvme_init()
822 ver = host_pci_ldl_le_p(®s->vs); in nvme_init()
828 cc = host_pci_ldl_le_p(®s->cc); in nvme_init()
829 host_pci_stl_le_p(®s->cc, cc & 0xFE); in nvme_init()
832 while (NVME_CSTS_RDY(host_pci_ldl_le_p(®s->csts))) { in nvme_init()
837 ret = -ETIMEDOUT; in nvme_init()
842 s->bar0_wo_map = qemu_vfio_pci_map_bar(s->vfio, 0, 0, in nvme_init()
845 s->doorbells = (void *)((uintptr_t)s->bar0_wo_map + sizeof(NvmeBar)); in nvme_init()
846 if (!s->doorbells) { in nvme_init()
847 ret = -EINVAL; in nvme_init()
852 s->queues = g_new(NVMeQueuePair *, 1); in nvme_init()
855 ret = -EINVAL; in nvme_init()
858 s->queues[INDEX_ADMIN] = q; in nvme_init()
859 s->queue_count = 1; in nvme_init()
860 QEMU_BUILD_BUG_ON((NVME_QUEUE_SIZE - 1) & 0xF000); in nvme_init()
861 host_pci_stl_le_p(®s->aqa, in nvme_init()
862 ((NVME_QUEUE_SIZE - 1) << AQA_ACQS_SHIFT) | in nvme_init()
863 ((NVME_QUEUE_SIZE - 1) << AQA_ASQS_SHIFT)); in nvme_init()
864 host_pci_stq_le_p(®s->asq, q->sq.iova); in nvme_init()
865 host_pci_stq_le_p(®s->acq, q->cq.iova); in nvme_init()
868 host_pci_stl_le_p(®s->cc, in nvme_init()
875 while (!NVME_CSTS_RDY(host_pci_ldl_le_p(®s->csts))) { in nvme_init()
880 ret = -ETIMEDOUT; in nvme_init()
885 ret = qemu_vfio_pci_init_irq(s->vfio, s->irq_notifier, in nvme_init()
891 &s->irq_notifier[MSIX_SHARED_IRQ_IDX], in nvme_init()
896 ret = -EIO; in nvme_init()
902 ret = -EIO; in nvme_init()
906 qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)regs, 0, sizeof(NvmeBar)); in nvme_init()
935 device = g_strndup(tmp, slash - tmp); in nvme_parse_filename()
953 BDRVNVMeState *s = bs->opaque; in nvme_enable_disable_write_cache()
956 .nsid = cpu_to_le32(s->nsid), in nvme_enable_disable_write_cache()
970 BDRVNVMeState *s = bs->opaque; in nvme_close()
972 for (unsigned i = 0; i < s->queue_count; ++i) { in nvme_close()
973 nvme_free_queue_pair(s->queues[i]); in nvme_close()
975 g_free(s->queues); in nvme_close()
977 &s->irq_notifier[MSIX_SHARED_IRQ_IDX], in nvme_close()
979 event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]); in nvme_close()
980 qemu_vfio_pci_unmap_bar(s->vfio, 0, s->bar0_wo_map, in nvme_close()
982 qemu_vfio_close(s->vfio); in nvme_close()
984 g_free(s->device); in nvme_close()
994 BDRVNVMeState *s = bs->opaque; in nvme_open()
996 bs->supported_write_flags = BDRV_REQ_FUA; in nvme_open()
1004 return -EINVAL; in nvme_open()
1014 if (!s->write_cache_supported) { in nvme_open()
1017 ret = -EINVAL; in nvme_open()
1034 BDRVNVMeState *s = bs->opaque; in nvme_co_getlength()
1035 return s->nsze << s->blkshift; in nvme_co_getlength()
1040 BDRVNVMeState *s = bs->opaque; in nvme_get_blocksize()
1041 assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12); in nvme_get_blocksize()
1042 return UINT32_C(1) << s->blkshift; in nvme_get_blocksize()
1048 bsz->phys = blocksize; in nvme_probe_blocksizes()
1049 bsz->log = blocksize; in nvme_probe_blocksizes()
1053 /* Called with s->dma_map_lock */
1058 BDRVNVMeState *s = bs->opaque; in nvme_cmd_unmap_qiov()
1060 s->dma_map_count -= qiov->size; in nvme_cmd_unmap_qiov()
1061 if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) { in nvme_cmd_unmap_qiov()
1062 r = qemu_vfio_dma_reset_temporary(s->vfio); in nvme_cmd_unmap_qiov()
1064 qemu_co_queue_restart_all(&s->dma_flush_queue); in nvme_cmd_unmap_qiov()
1070 /* Called with s->dma_map_lock */
1074 BDRVNVMeState *s = bs->opaque; in nvme_cmd_map_qiov()
1075 uint64_t *pagelist = req->prp_list_page; in nvme_cmd_map_qiov()
1080 assert(qiov->size); in nvme_cmd_map_qiov()
1081 assert(QEMU_IS_ALIGNED(qiov->size, s->page_size)); in nvme_cmd_map_qiov()
1082 assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t)); in nvme_cmd_map_qiov()
1083 for (i = 0; i < qiov->niov; ++i) { in nvme_cmd_map_qiov()
1086 size_t len = QEMU_ALIGN_UP(qiov->iov[i].iov_len, in nvme_cmd_map_qiov()
1089 r = qemu_vfio_dma_map(s->vfio, in nvme_cmd_map_qiov()
1090 qiov->iov[i].iov_base, in nvme_cmd_map_qiov()
1092 if (r == -ENOSPC) { in nvme_cmd_map_qiov()
1094 * In addition to the -ENOMEM error, the VFIO_IOMMU_MAP_DMA in nvme_cmd_map_qiov()
1095 * ioctl returns -ENOSPC to signal the user exhausted the DMA in nvme_cmd_map_qiov()
1098 * April 2019, see CVE-2019-3882). in nvme_cmd_map_qiov()
1101 * for the -ENOMEM error, so we directly replace -ENOSPC by in nvme_cmd_map_qiov()
1102 * -ENOMEM. Beside, -ENOSPC has a specific meaning for blockdev in nvme_cmd_map_qiov()
1108 r = -ENOMEM; in nvme_cmd_map_qiov()
1110 if (r == -ENOMEM && retry) { in nvme_cmd_map_qiov()
1117 if (s->dma_map_count) { in nvme_cmd_map_qiov()
1119 qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock); in nvme_cmd_map_qiov()
1121 r = qemu_vfio_dma_reset_temporary(s->vfio); in nvme_cmd_map_qiov()
1134 for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) { in nvme_cmd_map_qiov()
1135 pagelist[entries++] = cpu_to_le64(iova + j * s->page_size); in nvme_cmd_map_qiov()
1137 trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base, in nvme_cmd_map_qiov()
1138 qiov->iov[i].iov_len / s->page_size); in nvme_cmd_map_qiov()
1141 s->dma_map_count += qiov->size; in nvme_cmd_map_qiov()
1143 assert(entries <= s->page_size / sizeof(uint64_t)); in nvme_cmd_map_qiov()
1148 cmd->dptr.prp1 = pagelist[0]; in nvme_cmd_map_qiov()
1149 cmd->dptr.prp2 = 0; in nvme_cmd_map_qiov()
1152 cmd->dptr.prp1 = pagelist[0]; in nvme_cmd_map_qiov()
1153 cmd->dptr.prp2 = pagelist[1]; in nvme_cmd_map_qiov()
1156 cmd->dptr.prp1 = pagelist[0]; in nvme_cmd_map_qiov()
1157 cmd->dptr.prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t)); in nvme_cmd_map_qiov()
1166 /* No need to unmap [0 - i) iovs even if we've failed, since we don't in nvme_cmd_map_qiov()
1167 * increment s->dma_map_count. This is okay for fixed mapping memory areas in nvme_cmd_map_qiov()
1187 data->ret = ret; in nvme_rw_cb()
1189 if (data->co == qemu_coroutine_self()) { in nvme_rw_cb()
1193 * We can set data->skip_yield here to keep the coroutine from in nvme_rw_cb()
1196 data->skip_yield = true; in nvme_rw_cb()
1202 * No need to release NVMeQueuePair.lock (we are called without it in nvme_rw_cb()
1206 aio_co_wake(data->co); in nvme_rw_cb()
1211 uint64_t offset, uint64_t bytes, in nvme_co_prw_aligned() argument
1217 BDRVNVMeState *s = bs->opaque; in nvme_co_prw_aligned()
1218 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)]; in nvme_co_prw_aligned()
1221 uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) | in nvme_co_prw_aligned()
1225 .nsid = cpu_to_le32(s->nsid), in nvme_co_prw_aligned()
1226 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF), in nvme_co_prw_aligned()
1227 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF), in nvme_co_prw_aligned()
1232 .ret = -EINPROGRESS, in nvme_co_prw_aligned()
1235 trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov); in nvme_co_prw_aligned()
1236 assert(s->queue_count > 1); in nvme_co_prw_aligned()
1240 qemu_co_mutex_lock(&s->dma_map_lock); in nvme_co_prw_aligned()
1242 qemu_co_mutex_unlock(&s->dma_map_lock); in nvme_co_prw_aligned()
1252 qemu_co_mutex_lock(&s->dma_map_lock); in nvme_co_prw_aligned()
1254 qemu_co_mutex_unlock(&s->dma_map_lock); in nvme_co_prw_aligned()
1259 trace_nvme_rw_done(s, is_write, offset, bytes, data.ret); in nvme_co_prw_aligned()
1267 BDRVNVMeState *s = bs->opaque; in nvme_qiov_aligned()
1269 for (i = 0; i < qiov->niov; ++i) { in nvme_qiov_aligned()
1270 if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base, in nvme_qiov_aligned()
1272 !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, qemu_real_host_page_size())) { in nvme_qiov_aligned()
1273 trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base, in nvme_qiov_aligned()
1274 qiov->iov[i].iov_len, s->page_size); in nvme_qiov_aligned()
1282 uint64_t offset, uint64_t bytes, in nvme_co_prw() argument
1286 BDRVNVMeState *s = bs->opaque; in nvme_co_prw()
1291 assert(QEMU_IS_ALIGNED(offset, s->page_size)); in nvme_co_prw()
1292 assert(QEMU_IS_ALIGNED(bytes, s->page_size)); in nvme_co_prw()
1293 assert(bytes <= s->max_transfer); in nvme_co_prw()
1295 s->stats.aligned_accesses++; in nvme_co_prw()
1296 return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags); in nvme_co_prw()
1298 s->stats.unaligned_accesses++; in nvme_co_prw()
1299 trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write); in nvme_co_prw()
1303 return -ENOMEM; in nvme_co_prw()
1310 r = nvme_co_prw_aligned(bs, offset, bytes, &local_qiov, is_write, flags); in nvme_co_prw()
1319 int64_t offset, int64_t bytes, in nvme_co_preadv() argument
1323 return nvme_co_prw(bs, offset, bytes, qiov, false, flags); in nvme_co_preadv()
1327 int64_t offset, int64_t bytes, in nvme_co_pwritev() argument
1331 return nvme_co_prw(bs, offset, bytes, qiov, true, flags); in nvme_co_pwritev()
1336 BDRVNVMeState *s = bs->opaque; in nvme_co_flush()
1337 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)]; in nvme_co_flush()
1341 .nsid = cpu_to_le32(s->nsid), in nvme_co_flush()
1345 .ret = -EINPROGRESS, in nvme_co_flush()
1348 assert(s->queue_count > 1); in nvme_co_flush()
1361 int64_t offset, in nvme_co_pwrite_zeroes() argument
1365 BDRVNVMeState *s = bs->opaque; in nvme_co_pwrite_zeroes()
1366 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)]; in nvme_co_pwrite_zeroes()
1370 if (!s->supports_write_zeroes) { in nvme_co_pwrite_zeroes()
1371 return -ENOTSUP; in nvme_co_pwrite_zeroes()
1378 cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF; in nvme_co_pwrite_zeroes()
1383 assert(((cdw12 + 1) << s->blkshift) == bytes); in nvme_co_pwrite_zeroes()
1387 .nsid = cpu_to_le32(s->nsid), in nvme_co_pwrite_zeroes()
1388 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF), in nvme_co_pwrite_zeroes()
1389 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF), in nvme_co_pwrite_zeroes()
1394 .ret = -EINPROGRESS, in nvme_co_pwrite_zeroes()
1407 trace_nvme_write_zeroes(s, offset, bytes, flags); in nvme_co_pwrite_zeroes()
1408 assert(s->queue_count > 1); in nvme_co_pwrite_zeroes()
1417 trace_nvme_rw_done(s, true, offset, bytes, data.ret); in nvme_co_pwrite_zeroes()
1423 int64_t offset, in nvme_co_pdiscard() argument
1426 BDRVNVMeState *s = bs->opaque; in nvme_co_pdiscard()
1427 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)]; in nvme_co_pdiscard()
1435 .nsid = cpu_to_le32(s->nsid), in nvme_co_pdiscard()
1436 .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/ in nvme_co_pdiscard()
1442 .ret = -EINPROGRESS, in nvme_co_pdiscard()
1445 if (!s->supports_discard) { in nvme_co_pdiscard()
1446 return -ENOTSUP; in nvme_co_pdiscard()
1449 assert(s->queue_count > 1); in nvme_co_pdiscard()
1452 * Filling the @buf requires @offset and @bytes to satisfy restrictions in nvme_co_pdiscard()
1455 assert(QEMU_IS_ALIGNED(bytes, 1UL << s->blkshift)); in nvme_co_pdiscard()
1456 assert(QEMU_IS_ALIGNED(offset, 1UL << s->blkshift)); in nvme_co_pdiscard()
1457 assert((bytes >> s->blkshift) <= UINT32_MAX); in nvme_co_pdiscard()
1459 buf = qemu_try_memalign(s->page_size, s->page_size); in nvme_co_pdiscard()
1461 return -ENOMEM; in nvme_co_pdiscard()
1463 memset(buf, 0, s->page_size); in nvme_co_pdiscard()
1464 buf->nlb = cpu_to_le32(bytes >> s->blkshift); in nvme_co_pdiscard()
1465 buf->slba = cpu_to_le64(offset >> s->blkshift); in nvme_co_pdiscard()
1466 buf->cattr = 0; in nvme_co_pdiscard()
1474 qemu_co_mutex_lock(&s->dma_map_lock); in nvme_co_pdiscard()
1476 qemu_co_mutex_unlock(&s->dma_map_lock); in nvme_co_pdiscard()
1483 trace_nvme_dsm(s, offset, bytes); in nvme_co_pdiscard()
1490 qemu_co_mutex_lock(&s->dma_map_lock); in nvme_co_pdiscard()
1492 qemu_co_mutex_unlock(&s->dma_map_lock); in nvme_co_pdiscard()
1499 trace_nvme_dsm_done(s, offset, bytes, ret); in nvme_co_pdiscard()
1506 static int coroutine_fn nvme_co_truncate(BlockDriverState *bs, int64_t offset, in nvme_co_truncate() argument
1515 return -ENOTSUP; in nvme_co_truncate()
1519 if (offset != cur_length && exact) { in nvme_co_truncate()
1521 return -ENOTSUP; in nvme_co_truncate()
1522 } else if (offset > cur_length) { in nvme_co_truncate()
1524 return -EINVAL; in nvme_co_truncate()
1538 BDRVNVMeState *s = bs->opaque; in nvme_refresh_filename()
1540 snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i", in nvme_refresh_filename()
1541 s->device, s->nsid); in nvme_refresh_filename()
1546 BDRVNVMeState *s = bs->opaque; in nvme_refresh_limits()
1548 bs->bl.opt_mem_alignment = s->page_size; in nvme_refresh_limits()
1549 bs->bl.request_alignment = s->page_size; in nvme_refresh_limits()
1550 bs->bl.max_transfer = s->max_transfer; in nvme_refresh_limits()
1556 bs->bl.max_pwrite_zeroes = 1ULL << (s->blkshift + 16); in nvme_refresh_limits()
1557 bs->bl.pwrite_zeroes_alignment = MAX(bs->bl.request_alignment, in nvme_refresh_limits()
1558 1UL << s->blkshift); in nvme_refresh_limits()
1560 bs->bl.max_pdiscard = (uint64_t)UINT32_MAX << s->blkshift; in nvme_refresh_limits()
1561 bs->bl.pdiscard_alignment = MAX(bs->bl.request_alignment, in nvme_refresh_limits()
1562 1UL << s->blkshift); in nvme_refresh_limits()
1567 BDRVNVMeState *s = bs->opaque; in nvme_detach_aio_context()
1569 for (unsigned i = 0; i < s->queue_count; i++) { in nvme_detach_aio_context()
1570 NVMeQueuePair *q = s->queues[i]; in nvme_detach_aio_context()
1572 qemu_bh_delete(q->completion_bh); in nvme_detach_aio_context()
1573 q->completion_bh = NULL; in nvme_detach_aio_context()
1577 &s->irq_notifier[MSIX_SHARED_IRQ_IDX], in nvme_detach_aio_context()
1584 BDRVNVMeState *s = bs->opaque; in nvme_attach_aio_context()
1586 s->aio_context = new_context; in nvme_attach_aio_context()
1587 aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX], in nvme_attach_aio_context()
1591 for (unsigned i = 0; i < s->queue_count; i++) { in nvme_attach_aio_context()
1592 NVMeQueuePair *q = s->queues[i]; in nvme_attach_aio_context()
1594 q->completion_bh = in nvme_attach_aio_context()
1603 BDRVNVMeState *s = bs->opaque; in nvme_register_buf()
1610 ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL, errp); in nvme_register_buf()
1616 BDRVNVMeState *s = bs->opaque; in nvme_unregister_buf()
1618 qemu_vfio_dma_unmap(s->vfio, host); in nvme_unregister_buf()
1624 BDRVNVMeState *s = bs->opaque; in nvme_get_specific_stats()
1626 stats->driver = BLOCKDEV_DRIVER_NVME; in nvme_get_specific_stats()
1627 stats->u.nvme = (BlockStatsSpecificNvme) { in nvme_get_specific_stats()
1628 .completion_errors = s->stats.completion_errors, in nvme_get_specific_stats()
1629 .aligned_accesses = s->stats.aligned_accesses, in nvme_get_specific_stats()
1630 .unaligned_accesses = s->stats.unaligned_accesses, in nvme_get_specific_stats()