xref: /openbmc/qemu/block/nvme.c (revision 62a4db55)
1 /*
2  * NVMe block driver based on vfio
3  *
4  * Copyright 2016 - 2018 Red Hat, Inc.
5  *
6  * Authors:
7  *   Fam Zheng <famz@redhat.com>
8  *   Paolo Bonzini <pbonzini@redhat.com>
9  *
10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
11  * See the COPYING file in the top-level directory.
12  */
13 
14 #include "qemu/osdep.h"
15 #include <linux/vfio.h>
16 #include "qapi/error.h"
17 #include "qapi/qmp/qdict.h"
18 #include "qapi/qmp/qstring.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "qemu/cutils.h"
23 #include "qemu/option.h"
24 #include "qemu/vfio-helpers.h"
25 #include "block/block_int.h"
26 #include "sysemu/replay.h"
27 #include "trace.h"
28 
29 #include "block/nvme.h"
30 
31 #define NVME_SQ_ENTRY_BYTES 64
32 #define NVME_CQ_ENTRY_BYTES 16
33 #define NVME_QUEUE_SIZE 128
34 #define NVME_DOORBELL_SIZE 4096
35 
36 /*
37  * We have to leave one slot empty as that is the full queue case where
38  * head == tail + 1.
39  */
40 #define NVME_NUM_REQS (NVME_QUEUE_SIZE - 1)
41 
42 typedef struct BDRVNVMeState BDRVNVMeState;
43 
44 /* Same index is used for queues and IRQs */
45 #define INDEX_ADMIN     0
46 #define INDEX_IO(n)     (1 + n)
47 
48 /* This driver shares a single MSIX IRQ for the admin and I/O queues */
49 enum {
50     MSIX_SHARED_IRQ_IDX = 0,
51     MSIX_IRQ_COUNT = 1
52 };
53 
54 typedef struct {
55     int32_t  head, tail;
56     uint8_t  *queue;
57     uint64_t iova;
58     /* Hardware MMIO register */
59     volatile uint32_t *doorbell;
60 } NVMeQueue;
61 
62 typedef struct {
63     BlockCompletionFunc *cb;
64     void *opaque;
65     int cid;
66     void *prp_list_page;
67     uint64_t prp_list_iova;
68     int free_req_next; /* q->reqs[] index of next free req */
69 } NVMeRequest;
70 
71 typedef struct {
72     QemuMutex   lock;
73 
74     /* Read from I/O code path, initialized under BQL */
75     BDRVNVMeState   *s;
76     int             index;
77 
78     /* Fields protected by BQL */
79     uint8_t     *prp_list_pages;
80 
81     /* Fields protected by @lock */
82     CoQueue     free_req_queue;
83     NVMeQueue   sq, cq;
84     int         cq_phase;
85     int         free_req_head;
86     NVMeRequest reqs[NVME_NUM_REQS];
87     int         need_kick;
88     int         inflight;
89 
90     /* Thread-safe, no lock necessary */
91     QEMUBH      *completion_bh;
92 } NVMeQueuePair;
93 
94 struct BDRVNVMeState {
95     AioContext *aio_context;
96     QEMUVFIOState *vfio;
97     void *bar0_wo_map;
98     /* Memory mapped registers */
99     volatile struct {
100         uint32_t sq_tail;
101         uint32_t cq_head;
102     } *doorbells;
103     /* The submission/completion queue pairs.
104      * [0]: admin queue.
105      * [1..]: io queues.
106      */
107     NVMeQueuePair **queues;
108     unsigned queue_count;
109     size_t page_size;
110     /* How many uint32_t elements does each doorbell entry take. */
111     size_t doorbell_scale;
112     bool write_cache_supported;
113     EventNotifier irq_notifier[MSIX_IRQ_COUNT];
114 
115     uint64_t nsze; /* Namespace size reported by identify command */
116     int nsid;      /* The namespace id to read/write data. */
117     int blkshift;
118 
119     uint64_t max_transfer;
120     bool plugged;
121 
122     bool supports_write_zeroes;
123     bool supports_discard;
124 
125     CoMutex dma_map_lock;
126     CoQueue dma_flush_queue;
127 
128     /* Total size of mapped qiov, accessed under dma_map_lock */
129     int dma_map_count;
130 
131     /* PCI address (required for nvme_refresh_filename()) */
132     char *device;
133 
134     struct {
135         uint64_t completion_errors;
136         uint64_t aligned_accesses;
137         uint64_t unaligned_accesses;
138     } stats;
139 };
140 
141 #define NVME_BLOCK_OPT_DEVICE "device"
142 #define NVME_BLOCK_OPT_NAMESPACE "namespace"
143 
144 static void nvme_process_completion_bh(void *opaque);
145 
146 static QemuOptsList runtime_opts = {
147     .name = "nvme",
148     .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
149     .desc = {
150         {
151             .name = NVME_BLOCK_OPT_DEVICE,
152             .type = QEMU_OPT_STRING,
153             .help = "NVMe PCI device address",
154         },
155         {
156             .name = NVME_BLOCK_OPT_NAMESPACE,
157             .type = QEMU_OPT_NUMBER,
158             .help = "NVMe namespace",
159         },
160         { /* end of list */ }
161     },
162 };
163 
164 /* Returns true on success, false on failure. */
165 static bool nvme_init_queue(BDRVNVMeState *s, NVMeQueue *q,
166                             unsigned nentries, size_t entry_bytes, Error **errp)
167 {
168     size_t bytes;
169     int r;
170 
171     bytes = ROUND_UP(nentries * entry_bytes, qemu_real_host_page_size);
172     q->head = q->tail = 0;
173     q->queue = qemu_try_memalign(qemu_real_host_page_size, bytes);
174     if (!q->queue) {
175         error_setg(errp, "Cannot allocate queue");
176         return false;
177     }
178     memset(q->queue, 0, bytes);
179     r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova);
180     if (r) {
181         error_setg(errp, "Cannot map queue");
182         return false;
183     }
184     return true;
185 }
186 
187 static void nvme_free_queue_pair(NVMeQueuePair *q)
188 {
189     trace_nvme_free_queue_pair(q->index, q);
190     if (q->completion_bh) {
191         qemu_bh_delete(q->completion_bh);
192     }
193     qemu_vfree(q->prp_list_pages);
194     qemu_vfree(q->sq.queue);
195     qemu_vfree(q->cq.queue);
196     qemu_mutex_destroy(&q->lock);
197     g_free(q);
198 }
199 
200 static void nvme_free_req_queue_cb(void *opaque)
201 {
202     NVMeQueuePair *q = opaque;
203 
204     qemu_mutex_lock(&q->lock);
205     while (qemu_co_enter_next(&q->free_req_queue, &q->lock)) {
206         /* Retry all pending requests */
207     }
208     qemu_mutex_unlock(&q->lock);
209 }
210 
211 static NVMeQueuePair *nvme_create_queue_pair(BDRVNVMeState *s,
212                                              AioContext *aio_context,
213                                              unsigned idx, size_t size,
214                                              Error **errp)
215 {
216     int i, r;
217     NVMeQueuePair *q;
218     uint64_t prp_list_iova;
219     size_t bytes;
220 
221     q = g_try_new0(NVMeQueuePair, 1);
222     if (!q) {
223         return NULL;
224     }
225     trace_nvme_create_queue_pair(idx, q, size, aio_context,
226                                  event_notifier_get_fd(s->irq_notifier));
227     bytes = QEMU_ALIGN_UP(s->page_size * NVME_NUM_REQS,
228                           qemu_real_host_page_size);
229     q->prp_list_pages = qemu_try_memalign(qemu_real_host_page_size, bytes);
230     if (!q->prp_list_pages) {
231         goto fail;
232     }
233     memset(q->prp_list_pages, 0, bytes);
234     qemu_mutex_init(&q->lock);
235     q->s = s;
236     q->index = idx;
237     qemu_co_queue_init(&q->free_req_queue);
238     q->completion_bh = aio_bh_new(aio_context, nvme_process_completion_bh, q);
239     r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages, bytes,
240                           false, &prp_list_iova);
241     if (r) {
242         goto fail;
243     }
244     q->free_req_head = -1;
245     for (i = 0; i < NVME_NUM_REQS; i++) {
246         NVMeRequest *req = &q->reqs[i];
247         req->cid = i + 1;
248         req->free_req_next = q->free_req_head;
249         q->free_req_head = i;
250         req->prp_list_page = q->prp_list_pages + i * s->page_size;
251         req->prp_list_iova = prp_list_iova + i * s->page_size;
252     }
253 
254     if (!nvme_init_queue(s, &q->sq, size, NVME_SQ_ENTRY_BYTES, errp)) {
255         goto fail;
256     }
257     q->sq.doorbell = &s->doorbells[idx * s->doorbell_scale].sq_tail;
258 
259     if (!nvme_init_queue(s, &q->cq, size, NVME_CQ_ENTRY_BYTES, errp)) {
260         goto fail;
261     }
262     q->cq.doorbell = &s->doorbells[idx * s->doorbell_scale].cq_head;
263 
264     return q;
265 fail:
266     nvme_free_queue_pair(q);
267     return NULL;
268 }
269 
270 /* With q->lock */
271 static void nvme_kick(NVMeQueuePair *q)
272 {
273     BDRVNVMeState *s = q->s;
274 
275     if (s->plugged || !q->need_kick) {
276         return;
277     }
278     trace_nvme_kick(s, q->index);
279     assert(!(q->sq.tail & 0xFF00));
280     /* Fence the write to submission queue entry before notifying the device. */
281     smp_wmb();
282     *q->sq.doorbell = cpu_to_le32(q->sq.tail);
283     q->inflight += q->need_kick;
284     q->need_kick = 0;
285 }
286 
287 /* Find a free request element if any, otherwise:
288  * a) if in coroutine context, try to wait for one to become available;
289  * b) if not in coroutine, return NULL;
290  */
291 static NVMeRequest *nvme_get_free_req(NVMeQueuePair *q)
292 {
293     NVMeRequest *req;
294 
295     qemu_mutex_lock(&q->lock);
296 
297     while (q->free_req_head == -1) {
298         if (qemu_in_coroutine()) {
299             trace_nvme_free_req_queue_wait(q->s, q->index);
300             qemu_co_queue_wait(&q->free_req_queue, &q->lock);
301         } else {
302             qemu_mutex_unlock(&q->lock);
303             return NULL;
304         }
305     }
306 
307     req = &q->reqs[q->free_req_head];
308     q->free_req_head = req->free_req_next;
309     req->free_req_next = -1;
310 
311     qemu_mutex_unlock(&q->lock);
312     return req;
313 }
314 
315 /* With q->lock */
316 static void nvme_put_free_req_locked(NVMeQueuePair *q, NVMeRequest *req)
317 {
318     req->free_req_next = q->free_req_head;
319     q->free_req_head = req - q->reqs;
320 }
321 
322 /* With q->lock */
323 static void nvme_wake_free_req_locked(NVMeQueuePair *q)
324 {
325     if (!qemu_co_queue_empty(&q->free_req_queue)) {
326         replay_bh_schedule_oneshot_event(q->s->aio_context,
327                 nvme_free_req_queue_cb, q);
328     }
329 }
330 
331 /* Insert a request in the freelist and wake waiters */
332 static void nvme_put_free_req_and_wake(NVMeQueuePair *q, NVMeRequest *req)
333 {
334     qemu_mutex_lock(&q->lock);
335     nvme_put_free_req_locked(q, req);
336     nvme_wake_free_req_locked(q);
337     qemu_mutex_unlock(&q->lock);
338 }
339 
340 static inline int nvme_translate_error(const NvmeCqe *c)
341 {
342     uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF;
343     if (status) {
344         trace_nvme_error(le32_to_cpu(c->result),
345                          le16_to_cpu(c->sq_head),
346                          le16_to_cpu(c->sq_id),
347                          le16_to_cpu(c->cid),
348                          le16_to_cpu(status));
349     }
350     switch (status) {
351     case 0:
352         return 0;
353     case 1:
354         return -ENOSYS;
355     case 2:
356         return -EINVAL;
357     default:
358         return -EIO;
359     }
360 }
361 
362 /* With q->lock */
363 static bool nvme_process_completion(NVMeQueuePair *q)
364 {
365     BDRVNVMeState *s = q->s;
366     bool progress = false;
367     NVMeRequest *preq;
368     NVMeRequest req;
369     NvmeCqe *c;
370 
371     trace_nvme_process_completion(s, q->index, q->inflight);
372     if (s->plugged) {
373         trace_nvme_process_completion_queue_plugged(s, q->index);
374         return false;
375     }
376 
377     /*
378      * Support re-entrancy when a request cb() function invokes aio_poll().
379      * Pending completions must be visible to aio_poll() so that a cb()
380      * function can wait for the completion of another request.
381      *
382      * The aio_poll() loop will execute our BH and we'll resume completion
383      * processing there.
384      */
385     qemu_bh_schedule(q->completion_bh);
386 
387     assert(q->inflight >= 0);
388     while (q->inflight) {
389         int ret;
390         int16_t cid;
391 
392         c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES];
393         if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) {
394             break;
395         }
396         ret = nvme_translate_error(c);
397         if (ret) {
398             s->stats.completion_errors++;
399         }
400         q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE;
401         if (!q->cq.head) {
402             q->cq_phase = !q->cq_phase;
403         }
404         cid = le16_to_cpu(c->cid);
405         if (cid == 0 || cid > NVME_QUEUE_SIZE) {
406             warn_report("NVMe: Unexpected CID in completion queue: %"PRIu32", "
407                         "queue size: %u", cid, NVME_QUEUE_SIZE);
408             continue;
409         }
410         trace_nvme_complete_command(s, q->index, cid);
411         preq = &q->reqs[cid - 1];
412         req = *preq;
413         assert(req.cid == cid);
414         assert(req.cb);
415         nvme_put_free_req_locked(q, preq);
416         preq->cb = preq->opaque = NULL;
417         q->inflight--;
418         qemu_mutex_unlock(&q->lock);
419         req.cb(req.opaque, ret);
420         qemu_mutex_lock(&q->lock);
421         progress = true;
422     }
423     if (progress) {
424         /* Notify the device so it can post more completions. */
425         smp_mb_release();
426         *q->cq.doorbell = cpu_to_le32(q->cq.head);
427         nvme_wake_free_req_locked(q);
428     }
429 
430     qemu_bh_cancel(q->completion_bh);
431 
432     return progress;
433 }
434 
435 static void nvme_process_completion_bh(void *opaque)
436 {
437     NVMeQueuePair *q = opaque;
438 
439     /*
440      * We're being invoked because a nvme_process_completion() cb() function
441      * called aio_poll(). The callback may be waiting for further completions
442      * so notify the device that it has space to fill in more completions now.
443      */
444     smp_mb_release();
445     *q->cq.doorbell = cpu_to_le32(q->cq.head);
446     nvme_wake_free_req_locked(q);
447 
448     nvme_process_completion(q);
449 }
450 
451 static void nvme_trace_command(const NvmeCmd *cmd)
452 {
453     int i;
454 
455     if (!trace_event_get_state_backends(TRACE_NVME_SUBMIT_COMMAND_RAW)) {
456         return;
457     }
458     for (i = 0; i < 8; ++i) {
459         uint8_t *cmdp = (uint8_t *)cmd + i * 8;
460         trace_nvme_submit_command_raw(cmdp[0], cmdp[1], cmdp[2], cmdp[3],
461                                       cmdp[4], cmdp[5], cmdp[6], cmdp[7]);
462     }
463 }
464 
465 static void nvme_submit_command(NVMeQueuePair *q, NVMeRequest *req,
466                                 NvmeCmd *cmd, BlockCompletionFunc cb,
467                                 void *opaque)
468 {
469     assert(!req->cb);
470     req->cb = cb;
471     req->opaque = opaque;
472     cmd->cid = cpu_to_le16(req->cid);
473 
474     trace_nvme_submit_command(q->s, q->index, req->cid);
475     nvme_trace_command(cmd);
476     qemu_mutex_lock(&q->lock);
477     memcpy((uint8_t *)q->sq.queue +
478            q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd));
479     q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE;
480     q->need_kick++;
481     nvme_kick(q);
482     nvme_process_completion(q);
483     qemu_mutex_unlock(&q->lock);
484 }
485 
486 static void nvme_admin_cmd_sync_cb(void *opaque, int ret)
487 {
488     int *pret = opaque;
489     *pret = ret;
490     aio_wait_kick();
491 }
492 
493 static int nvme_admin_cmd_sync(BlockDriverState *bs, NvmeCmd *cmd)
494 {
495     BDRVNVMeState *s = bs->opaque;
496     NVMeQueuePair *q = s->queues[INDEX_ADMIN];
497     AioContext *aio_context = bdrv_get_aio_context(bs);
498     NVMeRequest *req;
499     int ret = -EINPROGRESS;
500     req = nvme_get_free_req(q);
501     if (!req) {
502         return -EBUSY;
503     }
504     nvme_submit_command(q, req, cmd, nvme_admin_cmd_sync_cb, &ret);
505 
506     AIO_WAIT_WHILE(aio_context, ret == -EINPROGRESS);
507     return ret;
508 }
509 
510 /* Returns true on success, false on failure. */
511 static bool nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
512 {
513     BDRVNVMeState *s = bs->opaque;
514     bool ret = false;
515     union {
516         NvmeIdCtrl ctrl;
517         NvmeIdNs ns;
518     } *id;
519     NvmeLBAF *lbaf;
520     uint16_t oncs;
521     int r;
522     uint64_t iova;
523     NvmeCmd cmd = {
524         .opcode = NVME_ADM_CMD_IDENTIFY,
525         .cdw10 = cpu_to_le32(0x1),
526     };
527     size_t id_size = QEMU_ALIGN_UP(sizeof(*id), qemu_real_host_page_size);
528 
529     id = qemu_try_memalign(qemu_real_host_page_size, id_size);
530     if (!id) {
531         error_setg(errp, "Cannot allocate buffer for identify response");
532         goto out;
533     }
534     r = qemu_vfio_dma_map(s->vfio, id, id_size, true, &iova);
535     if (r) {
536         error_setg(errp, "Cannot map buffer for DMA");
537         goto out;
538     }
539 
540     memset(id, 0, id_size);
541     cmd.dptr.prp1 = cpu_to_le64(iova);
542     if (nvme_admin_cmd_sync(bs, &cmd)) {
543         error_setg(errp, "Failed to identify controller");
544         goto out;
545     }
546 
547     if (le32_to_cpu(id->ctrl.nn) < namespace) {
548         error_setg(errp, "Invalid namespace");
549         goto out;
550     }
551     s->write_cache_supported = le32_to_cpu(id->ctrl.vwc) & 0x1;
552     s->max_transfer = (id->ctrl.mdts ? 1 << id->ctrl.mdts : 0) * s->page_size;
553     /* For now the page list buffer per command is one page, to hold at most
554      * s->page_size / sizeof(uint64_t) entries. */
555     s->max_transfer = MIN_NON_ZERO(s->max_transfer,
556                           s->page_size / sizeof(uint64_t) * s->page_size);
557 
558     oncs = le16_to_cpu(id->ctrl.oncs);
559     s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROES);
560     s->supports_discard = !!(oncs & NVME_ONCS_DSM);
561 
562     memset(id, 0, id_size);
563     cmd.cdw10 = 0;
564     cmd.nsid = cpu_to_le32(namespace);
565     if (nvme_admin_cmd_sync(bs, &cmd)) {
566         error_setg(errp, "Failed to identify namespace");
567         goto out;
568     }
569 
570     s->nsze = le64_to_cpu(id->ns.nsze);
571     lbaf = &id->ns.lbaf[NVME_ID_NS_FLBAS_INDEX(id->ns.flbas)];
572 
573     if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(id->ns.dlfeat) &&
574             NVME_ID_NS_DLFEAT_READ_BEHAVIOR(id->ns.dlfeat) ==
575                     NVME_ID_NS_DLFEAT_READ_BEHAVIOR_ZEROES) {
576         bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP;
577     }
578 
579     if (lbaf->ms) {
580         error_setg(errp, "Namespaces with metadata are not yet supported");
581         goto out;
582     }
583 
584     if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 ||
585         (1 << lbaf->ds) > s->page_size)
586     {
587         error_setg(errp, "Namespace has unsupported block size (2^%d)",
588                    lbaf->ds);
589         goto out;
590     }
591 
592     ret = true;
593     s->blkshift = lbaf->ds;
594 out:
595     qemu_vfio_dma_unmap(s->vfio, id);
596     qemu_vfree(id);
597 
598     return ret;
599 }
600 
601 static bool nvme_poll_queue(NVMeQueuePair *q)
602 {
603     bool progress = false;
604 
605     const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES;
606     NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset];
607 
608     trace_nvme_poll_queue(q->s, q->index);
609     /*
610      * Do an early check for completions. q->lock isn't needed because
611      * nvme_process_completion() only runs in the event loop thread and
612      * cannot race with itself.
613      */
614     if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) {
615         return false;
616     }
617 
618     qemu_mutex_lock(&q->lock);
619     while (nvme_process_completion(q)) {
620         /* Keep polling */
621         progress = true;
622     }
623     qemu_mutex_unlock(&q->lock);
624 
625     return progress;
626 }
627 
628 static bool nvme_poll_queues(BDRVNVMeState *s)
629 {
630     bool progress = false;
631     int i;
632 
633     for (i = 0; i < s->queue_count; i++) {
634         if (nvme_poll_queue(s->queues[i])) {
635             progress = true;
636         }
637     }
638     return progress;
639 }
640 
641 static void nvme_handle_event(EventNotifier *n)
642 {
643     BDRVNVMeState *s = container_of(n, BDRVNVMeState,
644                                     irq_notifier[MSIX_SHARED_IRQ_IDX]);
645 
646     trace_nvme_handle_event(s);
647     event_notifier_test_and_clear(n);
648     nvme_poll_queues(s);
649 }
650 
651 static bool nvme_add_io_queue(BlockDriverState *bs, Error **errp)
652 {
653     BDRVNVMeState *s = bs->opaque;
654     unsigned n = s->queue_count;
655     NVMeQueuePair *q;
656     NvmeCmd cmd;
657     unsigned queue_size = NVME_QUEUE_SIZE;
658 
659     assert(n <= UINT16_MAX);
660     q = nvme_create_queue_pair(s, bdrv_get_aio_context(bs),
661                                n, queue_size, errp);
662     if (!q) {
663         return false;
664     }
665     cmd = (NvmeCmd) {
666         .opcode = NVME_ADM_CMD_CREATE_CQ,
667         .dptr.prp1 = cpu_to_le64(q->cq.iova),
668         .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | n),
669         .cdw11 = cpu_to_le32(NVME_CQ_IEN | NVME_CQ_PC),
670     };
671     if (nvme_admin_cmd_sync(bs, &cmd)) {
672         error_setg(errp, "Failed to create CQ io queue [%u]", n);
673         goto out_error;
674     }
675     cmd = (NvmeCmd) {
676         .opcode = NVME_ADM_CMD_CREATE_SQ,
677         .dptr.prp1 = cpu_to_le64(q->sq.iova),
678         .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | n),
679         .cdw11 = cpu_to_le32(NVME_SQ_PC | (n << 16)),
680     };
681     if (nvme_admin_cmd_sync(bs, &cmd)) {
682         error_setg(errp, "Failed to create SQ io queue [%u]", n);
683         goto out_error;
684     }
685     s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1);
686     s->queues[n] = q;
687     s->queue_count++;
688     return true;
689 out_error:
690     nvme_free_queue_pair(q);
691     return false;
692 }
693 
694 static bool nvme_poll_cb(void *opaque)
695 {
696     EventNotifier *e = opaque;
697     BDRVNVMeState *s = container_of(e, BDRVNVMeState,
698                                     irq_notifier[MSIX_SHARED_IRQ_IDX]);
699 
700     return nvme_poll_queues(s);
701 }
702 
703 static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
704                      Error **errp)
705 {
706     BDRVNVMeState *s = bs->opaque;
707     NVMeQueuePair *q;
708     AioContext *aio_context = bdrv_get_aio_context(bs);
709     int ret;
710     uint64_t cap;
711     uint32_t ver;
712     uint64_t timeout_ms;
713     uint64_t deadline, now;
714     volatile NvmeBar *regs = NULL;
715 
716     qemu_co_mutex_init(&s->dma_map_lock);
717     qemu_co_queue_init(&s->dma_flush_queue);
718     s->device = g_strdup(device);
719     s->nsid = namespace;
720     s->aio_context = bdrv_get_aio_context(bs);
721     ret = event_notifier_init(&s->irq_notifier[MSIX_SHARED_IRQ_IDX], 0);
722     if (ret) {
723         error_setg(errp, "Failed to init event notifier");
724         return ret;
725     }
726 
727     s->vfio = qemu_vfio_open_pci(device, errp);
728     if (!s->vfio) {
729         ret = -EINVAL;
730         goto out;
731     }
732 
733     regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, sizeof(NvmeBar),
734                                  PROT_READ | PROT_WRITE, errp);
735     if (!regs) {
736         ret = -EINVAL;
737         goto out;
738     }
739     /* Perform initialize sequence as described in NVMe spec "7.6.1
740      * Initialization". */
741 
742     cap = le64_to_cpu(regs->cap);
743     trace_nvme_controller_capability_raw(cap);
744     trace_nvme_controller_capability("Maximum Queue Entries Supported",
745                                      1 + NVME_CAP_MQES(cap));
746     trace_nvme_controller_capability("Contiguous Queues Required",
747                                      NVME_CAP_CQR(cap));
748     trace_nvme_controller_capability("Doorbell Stride",
749                                      1 << (2 + NVME_CAP_DSTRD(cap)));
750     trace_nvme_controller_capability("Subsystem Reset Supported",
751                                      NVME_CAP_NSSRS(cap));
752     trace_nvme_controller_capability("Memory Page Size Minimum",
753                                      1 << (12 + NVME_CAP_MPSMIN(cap)));
754     trace_nvme_controller_capability("Memory Page Size Maximum",
755                                      1 << (12 + NVME_CAP_MPSMAX(cap)));
756     if (!NVME_CAP_CSS(cap)) {
757         error_setg(errp, "Device doesn't support NVMe command set");
758         ret = -EINVAL;
759         goto out;
760     }
761 
762     s->page_size = 1u << (12 + NVME_CAP_MPSMIN(cap));
763     s->doorbell_scale = (4 << NVME_CAP_DSTRD(cap)) / sizeof(uint32_t);
764     bs->bl.opt_mem_alignment = s->page_size;
765     bs->bl.request_alignment = s->page_size;
766     timeout_ms = MIN(500 * NVME_CAP_TO(cap), 30000);
767 
768     ver = le32_to_cpu(regs->vs);
769     trace_nvme_controller_spec_version(extract32(ver, 16, 16),
770                                        extract32(ver, 8, 8),
771                                        extract32(ver, 0, 8));
772 
773     /* Reset device to get a clean state. */
774     regs->cc = cpu_to_le32(le32_to_cpu(regs->cc) & 0xFE);
775     /* Wait for CSTS.RDY = 0. */
776     deadline = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ms * SCALE_MS;
777     while (NVME_CSTS_RDY(le32_to_cpu(regs->csts))) {
778         if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
779             error_setg(errp, "Timeout while waiting for device to reset (%"
780                              PRId64 " ms)",
781                        timeout_ms);
782             ret = -ETIMEDOUT;
783             goto out;
784         }
785     }
786 
787     s->bar0_wo_map = qemu_vfio_pci_map_bar(s->vfio, 0, 0,
788                                            sizeof(NvmeBar) + NVME_DOORBELL_SIZE,
789                                            PROT_WRITE, errp);
790     s->doorbells = (void *)((uintptr_t)s->bar0_wo_map + sizeof(NvmeBar));
791     if (!s->doorbells) {
792         ret = -EINVAL;
793         goto out;
794     }
795 
796     /* Set up admin queue. */
797     s->queues = g_new(NVMeQueuePair *, 1);
798     q = nvme_create_queue_pair(s, aio_context, 0, NVME_QUEUE_SIZE, errp);
799     if (!q) {
800         ret = -EINVAL;
801         goto out;
802     }
803     s->queues[INDEX_ADMIN] = q;
804     s->queue_count = 1;
805     QEMU_BUILD_BUG_ON((NVME_QUEUE_SIZE - 1) & 0xF000);
806     regs->aqa = cpu_to_le32(((NVME_QUEUE_SIZE - 1) << AQA_ACQS_SHIFT) |
807                             ((NVME_QUEUE_SIZE - 1) << AQA_ASQS_SHIFT));
808     regs->asq = cpu_to_le64(q->sq.iova);
809     regs->acq = cpu_to_le64(q->cq.iova);
810 
811     /* After setting up all control registers we can enable device now. */
812     regs->cc = cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES) << CC_IOCQES_SHIFT) |
813                            (ctz32(NVME_SQ_ENTRY_BYTES) << CC_IOSQES_SHIFT) |
814                            CC_EN_MASK);
815     /* Wait for CSTS.RDY = 1. */
816     now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
817     deadline = now + timeout_ms * SCALE_MS;
818     while (!NVME_CSTS_RDY(le32_to_cpu(regs->csts))) {
819         if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
820             error_setg(errp, "Timeout while waiting for device to start (%"
821                              PRId64 " ms)",
822                        timeout_ms);
823             ret = -ETIMEDOUT;
824             goto out;
825         }
826     }
827 
828     ret = qemu_vfio_pci_init_irq(s->vfio, s->irq_notifier,
829                                  VFIO_PCI_MSIX_IRQ_INDEX, errp);
830     if (ret) {
831         goto out;
832     }
833     aio_set_event_notifier(bdrv_get_aio_context(bs),
834                            &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
835                            false, nvme_handle_event, nvme_poll_cb);
836 
837     if (!nvme_identify(bs, namespace, errp)) {
838         ret = -EIO;
839         goto out;
840     }
841 
842     /* Set up command queues. */
843     if (!nvme_add_io_queue(bs, errp)) {
844         ret = -EIO;
845     }
846 out:
847     if (regs) {
848         qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)regs, 0, sizeof(NvmeBar));
849     }
850 
851     /* Cleaning up is done in nvme_file_open() upon error. */
852     return ret;
853 }
854 
855 /* Parse a filename in the format of nvme://XXXX:XX:XX.X/X. Example:
856  *
857  *     nvme://0000:44:00.0/1
858  *
859  * where the "nvme://" is a fixed form of the protocol prefix, the middle part
860  * is the PCI address, and the last part is the namespace number starting from
861  * 1 according to the NVMe spec. */
862 static void nvme_parse_filename(const char *filename, QDict *options,
863                                 Error **errp)
864 {
865     int pref = strlen("nvme://");
866 
867     if (strlen(filename) > pref && !strncmp(filename, "nvme://", pref)) {
868         const char *tmp = filename + pref;
869         char *device;
870         const char *namespace;
871         unsigned long ns;
872         const char *slash = strchr(tmp, '/');
873         if (!slash) {
874             qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, tmp);
875             return;
876         }
877         device = g_strndup(tmp, slash - tmp);
878         qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, device);
879         g_free(device);
880         namespace = slash + 1;
881         if (*namespace && qemu_strtoul(namespace, NULL, 10, &ns)) {
882             error_setg(errp, "Invalid namespace '%s', positive number expected",
883                        namespace);
884             return;
885         }
886         qdict_put_str(options, NVME_BLOCK_OPT_NAMESPACE,
887                       *namespace ? namespace : "1");
888     }
889 }
890 
891 static int nvme_enable_disable_write_cache(BlockDriverState *bs, bool enable,
892                                            Error **errp)
893 {
894     int ret;
895     BDRVNVMeState *s = bs->opaque;
896     NvmeCmd cmd = {
897         .opcode = NVME_ADM_CMD_SET_FEATURES,
898         .nsid = cpu_to_le32(s->nsid),
899         .cdw10 = cpu_to_le32(0x06),
900         .cdw11 = cpu_to_le32(enable ? 0x01 : 0x00),
901     };
902 
903     ret = nvme_admin_cmd_sync(bs, &cmd);
904     if (ret) {
905         error_setg(errp, "Failed to configure NVMe write cache");
906     }
907     return ret;
908 }
909 
910 static void nvme_close(BlockDriverState *bs)
911 {
912     BDRVNVMeState *s = bs->opaque;
913 
914     for (unsigned i = 0; i < s->queue_count; ++i) {
915         nvme_free_queue_pair(s->queues[i]);
916     }
917     g_free(s->queues);
918     aio_set_event_notifier(bdrv_get_aio_context(bs),
919                            &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
920                            false, NULL, NULL);
921     event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]);
922     qemu_vfio_pci_unmap_bar(s->vfio, 0, s->bar0_wo_map,
923                             0, sizeof(NvmeBar) + NVME_DOORBELL_SIZE);
924     qemu_vfio_close(s->vfio);
925 
926     g_free(s->device);
927 }
928 
929 static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
930                           Error **errp)
931 {
932     const char *device;
933     QemuOpts *opts;
934     int namespace;
935     int ret;
936     BDRVNVMeState *s = bs->opaque;
937 
938     bs->supported_write_flags = BDRV_REQ_FUA;
939 
940     opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
941     qemu_opts_absorb_qdict(opts, options, &error_abort);
942     device = qemu_opt_get(opts, NVME_BLOCK_OPT_DEVICE);
943     if (!device) {
944         error_setg(errp, "'" NVME_BLOCK_OPT_DEVICE "' option is required");
945         qemu_opts_del(opts);
946         return -EINVAL;
947     }
948 
949     namespace = qemu_opt_get_number(opts, NVME_BLOCK_OPT_NAMESPACE, 1);
950     ret = nvme_init(bs, device, namespace, errp);
951     qemu_opts_del(opts);
952     if (ret) {
953         goto fail;
954     }
955     if (flags & BDRV_O_NOCACHE) {
956         if (!s->write_cache_supported) {
957             error_setg(errp,
958                        "NVMe controller doesn't support write cache configuration");
959             ret = -EINVAL;
960         } else {
961             ret = nvme_enable_disable_write_cache(bs, !(flags & BDRV_O_NOCACHE),
962                                                   errp);
963         }
964         if (ret) {
965             goto fail;
966         }
967     }
968     return 0;
969 fail:
970     nvme_close(bs);
971     return ret;
972 }
973 
974 static int64_t nvme_getlength(BlockDriverState *bs)
975 {
976     BDRVNVMeState *s = bs->opaque;
977     return s->nsze << s->blkshift;
978 }
979 
980 static uint32_t nvme_get_blocksize(BlockDriverState *bs)
981 {
982     BDRVNVMeState *s = bs->opaque;
983     assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12);
984     return UINT32_C(1) << s->blkshift;
985 }
986 
987 static int nvme_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
988 {
989     uint32_t blocksize = nvme_get_blocksize(bs);
990     bsz->phys = blocksize;
991     bsz->log = blocksize;
992     return 0;
993 }
994 
995 /* Called with s->dma_map_lock */
996 static coroutine_fn int nvme_cmd_unmap_qiov(BlockDriverState *bs,
997                                             QEMUIOVector *qiov)
998 {
999     int r = 0;
1000     BDRVNVMeState *s = bs->opaque;
1001 
1002     s->dma_map_count -= qiov->size;
1003     if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) {
1004         r = qemu_vfio_dma_reset_temporary(s->vfio);
1005         if (!r) {
1006             qemu_co_queue_restart_all(&s->dma_flush_queue);
1007         }
1008     }
1009     return r;
1010 }
1011 
1012 /* Called with s->dma_map_lock */
1013 static coroutine_fn int nvme_cmd_map_qiov(BlockDriverState *bs, NvmeCmd *cmd,
1014                                           NVMeRequest *req, QEMUIOVector *qiov)
1015 {
1016     BDRVNVMeState *s = bs->opaque;
1017     uint64_t *pagelist = req->prp_list_page;
1018     int i, j, r;
1019     int entries = 0;
1020 
1021     assert(qiov->size);
1022     assert(QEMU_IS_ALIGNED(qiov->size, s->page_size));
1023     assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t));
1024     for (i = 0; i < qiov->niov; ++i) {
1025         bool retry = true;
1026         uint64_t iova;
1027         size_t len = QEMU_ALIGN_UP(qiov->iov[i].iov_len,
1028                                    qemu_real_host_page_size);
1029 try_map:
1030         r = qemu_vfio_dma_map(s->vfio,
1031                               qiov->iov[i].iov_base,
1032                               len, true, &iova);
1033         if (r == -ENOSPC) {
1034             /*
1035              * In addition to the -ENOMEM error, the VFIO_IOMMU_MAP_DMA
1036              * ioctl returns -ENOSPC to signal the user exhausted the DMA
1037              * mappings available for a container since Linux kernel commit
1038              * 492855939bdb ("vfio/type1: Limit DMA mappings per container",
1039              * April 2019, see CVE-2019-3882).
1040              *
1041              * This block driver already handles this error path by checking
1042              * for the -ENOMEM error, so we directly replace -ENOSPC by
1043              * -ENOMEM. Beside, -ENOSPC has a specific meaning for blockdev
1044              * coroutines: it triggers BLOCKDEV_ON_ERROR_ENOSPC and
1045              * BLOCK_ERROR_ACTION_STOP which stops the VM, asking the operator
1046              * to add more storage to the blockdev. Not something we can do
1047              * easily with an IOMMU :)
1048              */
1049             r = -ENOMEM;
1050         }
1051         if (r == -ENOMEM && retry) {
1052             /*
1053              * We exhausted the DMA mappings available for our container:
1054              * recycle the volatile IOVA mappings.
1055              */
1056             retry = false;
1057             trace_nvme_dma_flush_queue_wait(s);
1058             if (s->dma_map_count) {
1059                 trace_nvme_dma_map_flush(s);
1060                 qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock);
1061             } else {
1062                 r = qemu_vfio_dma_reset_temporary(s->vfio);
1063                 if (r) {
1064                     goto fail;
1065                 }
1066             }
1067             goto try_map;
1068         }
1069         if (r) {
1070             goto fail;
1071         }
1072 
1073         for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) {
1074             pagelist[entries++] = cpu_to_le64(iova + j * s->page_size);
1075         }
1076         trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base,
1077                                     qiov->iov[i].iov_len / s->page_size);
1078     }
1079 
1080     s->dma_map_count += qiov->size;
1081 
1082     assert(entries <= s->page_size / sizeof(uint64_t));
1083     switch (entries) {
1084     case 0:
1085         abort();
1086     case 1:
1087         cmd->dptr.prp1 = pagelist[0];
1088         cmd->dptr.prp2 = 0;
1089         break;
1090     case 2:
1091         cmd->dptr.prp1 = pagelist[0];
1092         cmd->dptr.prp2 = pagelist[1];
1093         break;
1094     default:
1095         cmd->dptr.prp1 = pagelist[0];
1096         cmd->dptr.prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t));
1097         break;
1098     }
1099     trace_nvme_cmd_map_qiov(s, cmd, req, qiov, entries);
1100     for (i = 0; i < entries; ++i) {
1101         trace_nvme_cmd_map_qiov_pages(s, i, pagelist[i]);
1102     }
1103     return 0;
1104 fail:
1105     /* No need to unmap [0 - i) iovs even if we've failed, since we don't
1106      * increment s->dma_map_count. This is okay for fixed mapping memory areas
1107      * because they are already mapped before calling this function; for
1108      * temporary mappings, a later nvme_cmd_(un)map_qiov will reclaim by
1109      * calling qemu_vfio_dma_reset_temporary when necessary. */
1110     return r;
1111 }
1112 
1113 typedef struct {
1114     Coroutine *co;
1115     int ret;
1116     AioContext *ctx;
1117 } NVMeCoData;
1118 
1119 static void nvme_rw_cb_bh(void *opaque)
1120 {
1121     NVMeCoData *data = opaque;
1122     qemu_coroutine_enter(data->co);
1123 }
1124 
1125 static void nvme_rw_cb(void *opaque, int ret)
1126 {
1127     NVMeCoData *data = opaque;
1128     data->ret = ret;
1129     if (!data->co) {
1130         /* The rw coroutine hasn't yielded, don't try to enter. */
1131         return;
1132     }
1133     replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data);
1134 }
1135 
1136 static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs,
1137                                             uint64_t offset, uint64_t bytes,
1138                                             QEMUIOVector *qiov,
1139                                             bool is_write,
1140                                             int flags)
1141 {
1142     int r;
1143     BDRVNVMeState *s = bs->opaque;
1144     NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1145     NVMeRequest *req;
1146 
1147     uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) |
1148                        (flags & BDRV_REQ_FUA ? 1 << 30 : 0);
1149     NvmeCmd cmd = {
1150         .opcode = is_write ? NVME_CMD_WRITE : NVME_CMD_READ,
1151         .nsid = cpu_to_le32(s->nsid),
1152         .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1153         .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1154         .cdw12 = cpu_to_le32(cdw12),
1155     };
1156     NVMeCoData data = {
1157         .ctx = bdrv_get_aio_context(bs),
1158         .ret = -EINPROGRESS,
1159     };
1160 
1161     trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov);
1162     assert(s->queue_count > 1);
1163     req = nvme_get_free_req(ioq);
1164     assert(req);
1165 
1166     qemu_co_mutex_lock(&s->dma_map_lock);
1167     r = nvme_cmd_map_qiov(bs, &cmd, req, qiov);
1168     qemu_co_mutex_unlock(&s->dma_map_lock);
1169     if (r) {
1170         nvme_put_free_req_and_wake(ioq, req);
1171         return r;
1172     }
1173     nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1174 
1175     data.co = qemu_coroutine_self();
1176     while (data.ret == -EINPROGRESS) {
1177         qemu_coroutine_yield();
1178     }
1179 
1180     qemu_co_mutex_lock(&s->dma_map_lock);
1181     r = nvme_cmd_unmap_qiov(bs, qiov);
1182     qemu_co_mutex_unlock(&s->dma_map_lock);
1183     if (r) {
1184         return r;
1185     }
1186 
1187     trace_nvme_rw_done(s, is_write, offset, bytes, data.ret);
1188     return data.ret;
1189 }
1190 
1191 static inline bool nvme_qiov_aligned(BlockDriverState *bs,
1192                                      const QEMUIOVector *qiov)
1193 {
1194     int i;
1195     BDRVNVMeState *s = bs->opaque;
1196 
1197     for (i = 0; i < qiov->niov; ++i) {
1198         if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base,
1199                                  qemu_real_host_page_size) ||
1200             !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, qemu_real_host_page_size)) {
1201             trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base,
1202                                       qiov->iov[i].iov_len, s->page_size);
1203             return false;
1204         }
1205     }
1206     return true;
1207 }
1208 
1209 static int nvme_co_prw(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
1210                        QEMUIOVector *qiov, bool is_write, int flags)
1211 {
1212     BDRVNVMeState *s = bs->opaque;
1213     int r;
1214     uint8_t *buf = NULL;
1215     QEMUIOVector local_qiov;
1216     size_t len = QEMU_ALIGN_UP(bytes, qemu_real_host_page_size);
1217     assert(QEMU_IS_ALIGNED(offset, s->page_size));
1218     assert(QEMU_IS_ALIGNED(bytes, s->page_size));
1219     assert(bytes <= s->max_transfer);
1220     if (nvme_qiov_aligned(bs, qiov)) {
1221         s->stats.aligned_accesses++;
1222         return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags);
1223     }
1224     s->stats.unaligned_accesses++;
1225     trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write);
1226     buf = qemu_try_memalign(qemu_real_host_page_size, len);
1227 
1228     if (!buf) {
1229         return -ENOMEM;
1230     }
1231     qemu_iovec_init(&local_qiov, 1);
1232     if (is_write) {
1233         qemu_iovec_to_buf(qiov, 0, buf, bytes);
1234     }
1235     qemu_iovec_add(&local_qiov, buf, bytes);
1236     r = nvme_co_prw_aligned(bs, offset, bytes, &local_qiov, is_write, flags);
1237     qemu_iovec_destroy(&local_qiov);
1238     if (!r && !is_write) {
1239         qemu_iovec_from_buf(qiov, 0, buf, bytes);
1240     }
1241     qemu_vfree(buf);
1242     return r;
1243 }
1244 
1245 static coroutine_fn int nvme_co_preadv(BlockDriverState *bs,
1246                                        uint64_t offset, uint64_t bytes,
1247                                        QEMUIOVector *qiov, int flags)
1248 {
1249     return nvme_co_prw(bs, offset, bytes, qiov, false, flags);
1250 }
1251 
1252 static coroutine_fn int nvme_co_pwritev(BlockDriverState *bs,
1253                                         uint64_t offset, uint64_t bytes,
1254                                         QEMUIOVector *qiov, int flags)
1255 {
1256     return nvme_co_prw(bs, offset, bytes, qiov, true, flags);
1257 }
1258 
1259 static coroutine_fn int nvme_co_flush(BlockDriverState *bs)
1260 {
1261     BDRVNVMeState *s = bs->opaque;
1262     NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1263     NVMeRequest *req;
1264     NvmeCmd cmd = {
1265         .opcode = NVME_CMD_FLUSH,
1266         .nsid = cpu_to_le32(s->nsid),
1267     };
1268     NVMeCoData data = {
1269         .ctx = bdrv_get_aio_context(bs),
1270         .ret = -EINPROGRESS,
1271     };
1272 
1273     assert(s->queue_count > 1);
1274     req = nvme_get_free_req(ioq);
1275     assert(req);
1276     nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1277 
1278     data.co = qemu_coroutine_self();
1279     if (data.ret == -EINPROGRESS) {
1280         qemu_coroutine_yield();
1281     }
1282 
1283     return data.ret;
1284 }
1285 
1286 
1287 static coroutine_fn int nvme_co_pwrite_zeroes(BlockDriverState *bs,
1288                                               int64_t offset,
1289                                               int bytes,
1290                                               BdrvRequestFlags flags)
1291 {
1292     BDRVNVMeState *s = bs->opaque;
1293     NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1294     NVMeRequest *req;
1295 
1296     uint32_t cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF;
1297 
1298     if (!s->supports_write_zeroes) {
1299         return -ENOTSUP;
1300     }
1301 
1302     NvmeCmd cmd = {
1303         .opcode = NVME_CMD_WRITE_ZEROES,
1304         .nsid = cpu_to_le32(s->nsid),
1305         .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1306         .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1307     };
1308 
1309     NVMeCoData data = {
1310         .ctx = bdrv_get_aio_context(bs),
1311         .ret = -EINPROGRESS,
1312     };
1313 
1314     if (flags & BDRV_REQ_MAY_UNMAP) {
1315         cdw12 |= (1 << 25);
1316     }
1317 
1318     if (flags & BDRV_REQ_FUA) {
1319         cdw12 |= (1 << 30);
1320     }
1321 
1322     cmd.cdw12 = cpu_to_le32(cdw12);
1323 
1324     trace_nvme_write_zeroes(s, offset, bytes, flags);
1325     assert(s->queue_count > 1);
1326     req = nvme_get_free_req(ioq);
1327     assert(req);
1328 
1329     nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1330 
1331     data.co = qemu_coroutine_self();
1332     while (data.ret == -EINPROGRESS) {
1333         qemu_coroutine_yield();
1334     }
1335 
1336     trace_nvme_rw_done(s, true, offset, bytes, data.ret);
1337     return data.ret;
1338 }
1339 
1340 
1341 static int coroutine_fn nvme_co_pdiscard(BlockDriverState *bs,
1342                                          int64_t offset,
1343                                          int bytes)
1344 {
1345     BDRVNVMeState *s = bs->opaque;
1346     NVMeQueuePair *ioq = s->queues[INDEX_IO(0)];
1347     NVMeRequest *req;
1348     NvmeDsmRange *buf;
1349     QEMUIOVector local_qiov;
1350     int ret;
1351 
1352     NvmeCmd cmd = {
1353         .opcode = NVME_CMD_DSM,
1354         .nsid = cpu_to_le32(s->nsid),
1355         .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/
1356         .cdw11 = cpu_to_le32(1 << 2), /*deallocate bit*/
1357     };
1358 
1359     NVMeCoData data = {
1360         .ctx = bdrv_get_aio_context(bs),
1361         .ret = -EINPROGRESS,
1362     };
1363 
1364     if (!s->supports_discard) {
1365         return -ENOTSUP;
1366     }
1367 
1368     assert(s->queue_count > 1);
1369 
1370     buf = qemu_try_memalign(s->page_size, s->page_size);
1371     if (!buf) {
1372         return -ENOMEM;
1373     }
1374     memset(buf, 0, s->page_size);
1375     buf->nlb = cpu_to_le32(bytes >> s->blkshift);
1376     buf->slba = cpu_to_le64(offset >> s->blkshift);
1377     buf->cattr = 0;
1378 
1379     qemu_iovec_init(&local_qiov, 1);
1380     qemu_iovec_add(&local_qiov, buf, 4096);
1381 
1382     req = nvme_get_free_req(ioq);
1383     assert(req);
1384 
1385     qemu_co_mutex_lock(&s->dma_map_lock);
1386     ret = nvme_cmd_map_qiov(bs, &cmd, req, &local_qiov);
1387     qemu_co_mutex_unlock(&s->dma_map_lock);
1388 
1389     if (ret) {
1390         nvme_put_free_req_and_wake(ioq, req);
1391         goto out;
1392     }
1393 
1394     trace_nvme_dsm(s, offset, bytes);
1395 
1396     nvme_submit_command(ioq, req, &cmd, nvme_rw_cb, &data);
1397 
1398     data.co = qemu_coroutine_self();
1399     while (data.ret == -EINPROGRESS) {
1400         qemu_coroutine_yield();
1401     }
1402 
1403     qemu_co_mutex_lock(&s->dma_map_lock);
1404     ret = nvme_cmd_unmap_qiov(bs, &local_qiov);
1405     qemu_co_mutex_unlock(&s->dma_map_lock);
1406 
1407     if (ret) {
1408         goto out;
1409     }
1410 
1411     ret = data.ret;
1412     trace_nvme_dsm_done(s, offset, bytes, ret);
1413 out:
1414     qemu_iovec_destroy(&local_qiov);
1415     qemu_vfree(buf);
1416     return ret;
1417 
1418 }
1419 
1420 static int coroutine_fn nvme_co_truncate(BlockDriverState *bs, int64_t offset,
1421                                          bool exact, PreallocMode prealloc,
1422                                          BdrvRequestFlags flags, Error **errp)
1423 {
1424     int64_t cur_length;
1425 
1426     if (prealloc != PREALLOC_MODE_OFF) {
1427         error_setg(errp, "Unsupported preallocation mode '%s'",
1428                    PreallocMode_str(prealloc));
1429         return -ENOTSUP;
1430     }
1431 
1432     cur_length = nvme_getlength(bs);
1433     if (offset != cur_length && exact) {
1434         error_setg(errp, "Cannot resize NVMe devices");
1435         return -ENOTSUP;
1436     } else if (offset > cur_length) {
1437         error_setg(errp, "Cannot grow NVMe devices");
1438         return -EINVAL;
1439     }
1440 
1441     return 0;
1442 }
1443 
1444 static int nvme_reopen_prepare(BDRVReopenState *reopen_state,
1445                                BlockReopenQueue *queue, Error **errp)
1446 {
1447     return 0;
1448 }
1449 
1450 static void nvme_refresh_filename(BlockDriverState *bs)
1451 {
1452     BDRVNVMeState *s = bs->opaque;
1453 
1454     snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i",
1455              s->device, s->nsid);
1456 }
1457 
1458 static void nvme_refresh_limits(BlockDriverState *bs, Error **errp)
1459 {
1460     BDRVNVMeState *s = bs->opaque;
1461 
1462     bs->bl.opt_mem_alignment = s->page_size;
1463     bs->bl.request_alignment = s->page_size;
1464     bs->bl.max_transfer = s->max_transfer;
1465 }
1466 
1467 static void nvme_detach_aio_context(BlockDriverState *bs)
1468 {
1469     BDRVNVMeState *s = bs->opaque;
1470 
1471     for (unsigned i = 0; i < s->queue_count; i++) {
1472         NVMeQueuePair *q = s->queues[i];
1473 
1474         qemu_bh_delete(q->completion_bh);
1475         q->completion_bh = NULL;
1476     }
1477 
1478     aio_set_event_notifier(bdrv_get_aio_context(bs),
1479                            &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1480                            false, NULL, NULL);
1481 }
1482 
1483 static void nvme_attach_aio_context(BlockDriverState *bs,
1484                                     AioContext *new_context)
1485 {
1486     BDRVNVMeState *s = bs->opaque;
1487 
1488     s->aio_context = new_context;
1489     aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX],
1490                            false, nvme_handle_event, nvme_poll_cb);
1491 
1492     for (unsigned i = 0; i < s->queue_count; i++) {
1493         NVMeQueuePair *q = s->queues[i];
1494 
1495         q->completion_bh =
1496             aio_bh_new(new_context, nvme_process_completion_bh, q);
1497     }
1498 }
1499 
1500 static void nvme_aio_plug(BlockDriverState *bs)
1501 {
1502     BDRVNVMeState *s = bs->opaque;
1503     assert(!s->plugged);
1504     s->plugged = true;
1505 }
1506 
1507 static void nvme_aio_unplug(BlockDriverState *bs)
1508 {
1509     BDRVNVMeState *s = bs->opaque;
1510     assert(s->plugged);
1511     s->plugged = false;
1512     for (unsigned i = INDEX_IO(0); i < s->queue_count; i++) {
1513         NVMeQueuePair *q = s->queues[i];
1514         qemu_mutex_lock(&q->lock);
1515         nvme_kick(q);
1516         nvme_process_completion(q);
1517         qemu_mutex_unlock(&q->lock);
1518     }
1519 }
1520 
1521 static void nvme_register_buf(BlockDriverState *bs, void *host, size_t size)
1522 {
1523     int ret;
1524     BDRVNVMeState *s = bs->opaque;
1525 
1526     ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL);
1527     if (ret) {
1528         /* FIXME: we may run out of IOVA addresses after repeated
1529          * bdrv_register_buf/bdrv_unregister_buf, because nvme_vfio_dma_unmap
1530          * doesn't reclaim addresses for fixed mappings. */
1531         error_report("nvme_register_buf failed: %s", strerror(-ret));
1532     }
1533 }
1534 
1535 static void nvme_unregister_buf(BlockDriverState *bs, void *host)
1536 {
1537     BDRVNVMeState *s = bs->opaque;
1538 
1539     qemu_vfio_dma_unmap(s->vfio, host);
1540 }
1541 
1542 static BlockStatsSpecific *nvme_get_specific_stats(BlockDriverState *bs)
1543 {
1544     BlockStatsSpecific *stats = g_new(BlockStatsSpecific, 1);
1545     BDRVNVMeState *s = bs->opaque;
1546 
1547     stats->driver = BLOCKDEV_DRIVER_NVME;
1548     stats->u.nvme = (BlockStatsSpecificNvme) {
1549         .completion_errors = s->stats.completion_errors,
1550         .aligned_accesses = s->stats.aligned_accesses,
1551         .unaligned_accesses = s->stats.unaligned_accesses,
1552     };
1553 
1554     return stats;
1555 }
1556 
1557 static const char *const nvme_strong_runtime_opts[] = {
1558     NVME_BLOCK_OPT_DEVICE,
1559     NVME_BLOCK_OPT_NAMESPACE,
1560 
1561     NULL
1562 };
1563 
1564 static BlockDriver bdrv_nvme = {
1565     .format_name              = "nvme",
1566     .protocol_name            = "nvme",
1567     .instance_size            = sizeof(BDRVNVMeState),
1568 
1569     .bdrv_co_create_opts      = bdrv_co_create_opts_simple,
1570     .create_opts              = &bdrv_create_opts_simple,
1571 
1572     .bdrv_parse_filename      = nvme_parse_filename,
1573     .bdrv_file_open           = nvme_file_open,
1574     .bdrv_close               = nvme_close,
1575     .bdrv_getlength           = nvme_getlength,
1576     .bdrv_probe_blocksizes    = nvme_probe_blocksizes,
1577     .bdrv_co_truncate         = nvme_co_truncate,
1578 
1579     .bdrv_co_preadv           = nvme_co_preadv,
1580     .bdrv_co_pwritev          = nvme_co_pwritev,
1581 
1582     .bdrv_co_pwrite_zeroes    = nvme_co_pwrite_zeroes,
1583     .bdrv_co_pdiscard         = nvme_co_pdiscard,
1584 
1585     .bdrv_co_flush_to_disk    = nvme_co_flush,
1586     .bdrv_reopen_prepare      = nvme_reopen_prepare,
1587 
1588     .bdrv_refresh_filename    = nvme_refresh_filename,
1589     .bdrv_refresh_limits      = nvme_refresh_limits,
1590     .strong_runtime_opts      = nvme_strong_runtime_opts,
1591     .bdrv_get_specific_stats  = nvme_get_specific_stats,
1592 
1593     .bdrv_detach_aio_context  = nvme_detach_aio_context,
1594     .bdrv_attach_aio_context  = nvme_attach_aio_context,
1595 
1596     .bdrv_io_plug             = nvme_aio_plug,
1597     .bdrv_io_unplug           = nvme_aio_unplug,
1598 
1599     .bdrv_register_buf        = nvme_register_buf,
1600     .bdrv_unregister_buf      = nvme_unregister_buf,
1601 };
1602 
1603 static void bdrv_nvme_init(void)
1604 {
1605     bdrv_register(&bdrv_nvme);
1606 }
1607 
1608 block_init(bdrv_nvme_init);
1609