xref: /openbmc/qemu/hw/nvme/ctrl.c (revision f14eced5)
1 /*
2  * QEMU NVM Express Controller
3  *
4  * Copyright (c) 2012, Intel Corporation
5  *
6  * Written by Keith Busch <keith.busch@intel.com>
7  *
8  * This code is licensed under the GNU GPL v2 or later.
9  */
10 
11 /**
12  * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e
13  *
14  *  https://nvmexpress.org/developers/nvme-specification/
15  *
16  *
17  * Notes on coding style
18  * ---------------------
19  * While QEMU coding style prefers lowercase hexadecimals in constants, the
20  * NVMe subsystem use this format from the NVMe specifications in the comments
21  * (i.e. 'h' suffix instead of '0x' prefix).
22  *
23  * Usage
24  * -----
25  * See docs/system/nvme.rst for extensive documentation.
26  *
27  * Add options:
28  *      -drive file=<file>,if=none,id=<drive_id>
29  *      -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id>
30  *      -device nvme,serial=<serial>,id=<bus_name>, \
31  *              cmb_size_mb=<cmb_size_mb[optional]>, \
32  *              [pmrdev=<mem_backend_file_id>,] \
33  *              max_ioqpairs=<N[optional]>, \
34  *              aerl=<N[optional]>,aer_max_queued=<N[optional]>, \
35  *              mdts=<N[optional]>,vsl=<N[optional]>, \
36  *              zoned.zasl=<N[optional]>, \
37  *              zoned.auto_transition=<on|off[optional]>, \
38  *              sriov_max_vfs=<N[optional]> \
39  *              sriov_vq_flexible=<N[optional]> \
40  *              sriov_vi_flexible=<N[optional]> \
41  *              sriov_max_vi_per_vf=<N[optional]> \
42  *              sriov_max_vq_per_vf=<N[optional]> \
43  *              subsys=<subsys_id>
44  *      -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
45  *              zoned=<true|false[optional]>, \
46  *              subsys=<subsys_id>,shared=<true|false[optional]>, \
47  *              detached=<true|false[optional]>, \
48  *              zoned.zone_size=<N[optional]>, \
49  *              zoned.zone_capacity=<N[optional]>, \
50  *              zoned.descr_ext_size=<N[optional]>, \
51  *              zoned.max_active=<N[optional]>, \
52  *              zoned.max_open=<N[optional]>, \
53  *              zoned.cross_read=<true|false[optional]>
54  *
55  * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
56  * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
57  * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to
58  * always enable the CMBLOC and CMBSZ registers (v1.3 behavior).
59  *
60  * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
61  * For example:
62  * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
63  *  size=<size> .... -device nvme,...,pmrdev=<mem_id>
64  *
65  * The PMR will use BAR 4/5 exclusively.
66  *
67  * To place controller(s) and namespace(s) to a subsystem, then provide
68  * nvme-subsys device as above.
69  *
70  * nvme subsystem device parameters
71  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
72  * - `nqn`
73  *   This parameter provides the `<nqn_id>` part of the string
74  *   `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field
75  *   of subsystem controllers. Note that `<nqn_id>` should be unique per
76  *   subsystem, but this is not enforced by QEMU. If not specified, it will
77  *   default to the value of the `id` parameter (`<subsys_id>`).
78  *
79  * nvme device parameters
80  * ~~~~~~~~~~~~~~~~~~~~~~
81  * - `subsys`
82  *   Specifying this parameter attaches the controller to the subsystem and
83  *   the SUBNQN field in the controller will report the NQN of the subsystem
84  *   device. This also enables multi controller capability represented in
85  *   Identify Controller data structure in CMIC (Controller Multi-path I/O and
86  *   Namespace Sharing Capabilities).
87  *
88  * - `aerl`
89  *   The Asynchronous Event Request Limit (AERL). Indicates the maximum number
90  *   of concurrently outstanding Asynchronous Event Request commands support
91  *   by the controller. This is a 0's based value.
92  *
93  * - `aer_max_queued`
94  *   This is the maximum number of events that the device will enqueue for
95  *   completion when there are no outstanding AERs. When the maximum number of
96  *   enqueued events are reached, subsequent events will be dropped.
97  *
98  * - `mdts`
99  *   Indicates the maximum data transfer size for a command that transfers data
100  *   between host-accessible memory and the controller. The value is specified
101  *   as a power of two (2^n) and is in units of the minimum memory page size
102  *   (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB).
103  *
104  * - `vsl`
105  *   Indicates the maximum data size limit for the Verify command. Like `mdts`,
106  *   this value is specified as a power of two (2^n) and is in units of the
107  *   minimum memory page size (CAP.MPSMIN). The default value is 7 (i.e. 512
108  *   KiB).
109  *
110  * - `zoned.zasl`
111  *   Indicates the maximum data transfer size for the Zone Append command. Like
112  *   `mdts`, the value is specified as a power of two (2^n) and is in units of
113  *   the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e.
114  *   defaulting to the value of `mdts`).
115  *
116  * - `zoned.auto_transition`
117  *   Indicates if zones in zone state implicitly opened can be automatically
118  *   transitioned to zone state closed for resource management purposes.
119  *   Defaults to 'on'.
120  *
121  * - `sriov_max_vfs`
122  *   Indicates the maximum number of PCIe virtual functions supported
123  *   by the controller. The default value is 0. Specifying a non-zero value
124  *   enables reporting of both SR-IOV and ARI capabilities by the NVMe device.
125  *   Virtual function controllers will not report SR-IOV capability.
126  *
127  *   NOTE: Single Root I/O Virtualization support is experimental.
128  *   All the related parameters may be subject to change.
129  *
130  * - `sriov_vq_flexible`
131  *   Indicates the total number of flexible queue resources assignable to all
132  *   the secondary controllers. Implicitly sets the number of primary
133  *   controller's private resources to `(max_ioqpairs - sriov_vq_flexible)`.
134  *
135  * - `sriov_vi_flexible`
136  *   Indicates the total number of flexible interrupt resources assignable to
137  *   all the secondary controllers. Implicitly sets the number of primary
138  *   controller's private resources to `(msix_qsize - sriov_vi_flexible)`.
139  *
140  * - `sriov_max_vi_per_vf`
141  *   Indicates the maximum number of virtual interrupt resources assignable
142  *   to a secondary controller. The default 0 resolves to
143  *   `(sriov_vi_flexible / sriov_max_vfs)`.
144  *
145  * - `sriov_max_vq_per_vf`
146  *   Indicates the maximum number of virtual queue resources assignable to
147  *   a secondary controller. The default 0 resolves to
148  *   `(sriov_vq_flexible / sriov_max_vfs)`.
149  *
150  * nvme namespace device parameters
151  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
152  * - `shared`
153  *   When the parent nvme device (as defined explicitly by the 'bus' parameter
154  *   or implicitly by the most recently defined NvmeBus) is linked to an
155  *   nvme-subsys device, the namespace will be attached to all controllers in
156  *   the subsystem. If set to 'off' (the default), the namespace will remain a
157  *   private namespace and may only be attached to a single controller at a
158  *   time.
159  *
160  * - `detached`
161  *   This parameter is only valid together with the `subsys` parameter. If left
162  *   at the default value (`false/off`), the namespace will be attached to all
163  *   controllers in the NVMe subsystem at boot-up. If set to `true/on`, the
164  *   namespace will be available in the subsystem but not attached to any
165  *   controllers.
166  *
167  * Setting `zoned` to true selects Zoned Command Set at the namespace.
168  * In this case, the following namespace properties are available to configure
169  * zoned operation:
170  *     zoned.zone_size=<zone size in bytes, default: 128MiB>
171  *         The number may be followed by K, M, G as in kilo-, mega- or giga-.
172  *
173  *     zoned.zone_capacity=<zone capacity in bytes, default: zone size>
174  *         The value 0 (default) forces zone capacity to be the same as zone
175  *         size. The value of this property may not exceed zone size.
176  *
177  *     zoned.descr_ext_size=<zone descriptor extension size, default 0>
178  *         This value needs to be specified in 64B units. If it is zero,
179  *         namespace(s) will not support zone descriptor extensions.
180  *
181  *     zoned.max_active=<Maximum Active Resources (zones), default: 0>
182  *         The default value means there is no limit to the number of
183  *         concurrently active zones.
184  *
185  *     zoned.max_open=<Maximum Open Resources (zones), default: 0>
186  *         The default value means there is no limit to the number of
187  *         concurrently open zones.
188  *
189  *     zoned.cross_read=<enable RAZB, default: false>
190  *         Setting this property to true enables Read Across Zone Boundaries.
191  */
192 
193 #include "qemu/osdep.h"
194 #include "qemu/cutils.h"
195 #include "qemu/error-report.h"
196 #include "qemu/log.h"
197 #include "qemu/units.h"
198 #include "qemu/range.h"
199 #include "qapi/error.h"
200 #include "qapi/visitor.h"
201 #include "sysemu/sysemu.h"
202 #include "sysemu/block-backend.h"
203 #include "sysemu/hostmem.h"
204 #include "hw/pci/msix.h"
205 #include "hw/pci/pcie_sriov.h"
206 #include "migration/vmstate.h"
207 
208 #include "nvme.h"
209 #include "dif.h"
210 #include "trace.h"
211 
212 #define NVME_MAX_IOQPAIRS 0xffff
213 #define NVME_DB_SIZE  4
214 #define NVME_SPEC_VER 0x00010400
215 #define NVME_CMB_BIR 2
216 #define NVME_PMR_BIR 4
217 #define NVME_TEMPERATURE 0x143
218 #define NVME_TEMPERATURE_WARNING 0x157
219 #define NVME_TEMPERATURE_CRITICAL 0x175
220 #define NVME_NUM_FW_SLOTS 1
221 #define NVME_DEFAULT_MAX_ZA_SIZE (128 * KiB)
222 #define NVME_MAX_VFS 127
223 #define NVME_VF_RES_GRANULARITY 1
224 #define NVME_VF_OFFSET 0x1
225 #define NVME_VF_STRIDE 1
226 
227 #define NVME_GUEST_ERR(trace, fmt, ...) \
228     do { \
229         (trace_##trace)(__VA_ARGS__); \
230         qemu_log_mask(LOG_GUEST_ERROR, #trace \
231             " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
232     } while (0)
233 
234 static const bool nvme_feature_support[NVME_FID_MAX] = {
235     [NVME_ARBITRATION]              = true,
236     [NVME_POWER_MANAGEMENT]         = true,
237     [NVME_TEMPERATURE_THRESHOLD]    = true,
238     [NVME_ERROR_RECOVERY]           = true,
239     [NVME_VOLATILE_WRITE_CACHE]     = true,
240     [NVME_NUMBER_OF_QUEUES]         = true,
241     [NVME_INTERRUPT_COALESCING]     = true,
242     [NVME_INTERRUPT_VECTOR_CONF]    = true,
243     [NVME_WRITE_ATOMICITY]          = true,
244     [NVME_ASYNCHRONOUS_EVENT_CONF]  = true,
245     [NVME_TIMESTAMP]                = true,
246     [NVME_HOST_BEHAVIOR_SUPPORT]    = true,
247     [NVME_COMMAND_SET_PROFILE]      = true,
248     [NVME_FDP_MODE]                 = true,
249     [NVME_FDP_EVENTS]               = true,
250 };
251 
252 static const uint32_t nvme_feature_cap[NVME_FID_MAX] = {
253     [NVME_TEMPERATURE_THRESHOLD]    = NVME_FEAT_CAP_CHANGE,
254     [NVME_ERROR_RECOVERY]           = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
255     [NVME_VOLATILE_WRITE_CACHE]     = NVME_FEAT_CAP_CHANGE,
256     [NVME_NUMBER_OF_QUEUES]         = NVME_FEAT_CAP_CHANGE,
257     [NVME_ASYNCHRONOUS_EVENT_CONF]  = NVME_FEAT_CAP_CHANGE,
258     [NVME_TIMESTAMP]                = NVME_FEAT_CAP_CHANGE,
259     [NVME_HOST_BEHAVIOR_SUPPORT]    = NVME_FEAT_CAP_CHANGE,
260     [NVME_COMMAND_SET_PROFILE]      = NVME_FEAT_CAP_CHANGE,
261     [NVME_FDP_MODE]                 = NVME_FEAT_CAP_CHANGE,
262     [NVME_FDP_EVENTS]               = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
263 };
264 
265 static const uint32_t nvme_cse_acs[256] = {
266     [NVME_ADM_CMD_DELETE_SQ]        = NVME_CMD_EFF_CSUPP,
267     [NVME_ADM_CMD_CREATE_SQ]        = NVME_CMD_EFF_CSUPP,
268     [NVME_ADM_CMD_GET_LOG_PAGE]     = NVME_CMD_EFF_CSUPP,
269     [NVME_ADM_CMD_DELETE_CQ]        = NVME_CMD_EFF_CSUPP,
270     [NVME_ADM_CMD_CREATE_CQ]        = NVME_CMD_EFF_CSUPP,
271     [NVME_ADM_CMD_IDENTIFY]         = NVME_CMD_EFF_CSUPP,
272     [NVME_ADM_CMD_ABORT]            = NVME_CMD_EFF_CSUPP,
273     [NVME_ADM_CMD_SET_FEATURES]     = NVME_CMD_EFF_CSUPP,
274     [NVME_ADM_CMD_GET_FEATURES]     = NVME_CMD_EFF_CSUPP,
275     [NVME_ADM_CMD_ASYNC_EV_REQ]     = NVME_CMD_EFF_CSUPP,
276     [NVME_ADM_CMD_NS_ATTACHMENT]    = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_NIC,
277     [NVME_ADM_CMD_VIRT_MNGMT]       = NVME_CMD_EFF_CSUPP,
278     [NVME_ADM_CMD_DBBUF_CONFIG]     = NVME_CMD_EFF_CSUPP,
279     [NVME_ADM_CMD_FORMAT_NVM]       = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
280     [NVME_ADM_CMD_DIRECTIVE_RECV]   = NVME_CMD_EFF_CSUPP,
281     [NVME_ADM_CMD_DIRECTIVE_SEND]   = NVME_CMD_EFF_CSUPP,
282 };
283 
284 static const uint32_t nvme_cse_iocs_none[256];
285 
286 static const uint32_t nvme_cse_iocs_nvm[256] = {
287     [NVME_CMD_FLUSH]                = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
288     [NVME_CMD_WRITE_ZEROES]         = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
289     [NVME_CMD_WRITE]                = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
290     [NVME_CMD_READ]                 = NVME_CMD_EFF_CSUPP,
291     [NVME_CMD_DSM]                  = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
292     [NVME_CMD_VERIFY]               = NVME_CMD_EFF_CSUPP,
293     [NVME_CMD_COPY]                 = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
294     [NVME_CMD_COMPARE]              = NVME_CMD_EFF_CSUPP,
295     [NVME_CMD_IO_MGMT_RECV]         = NVME_CMD_EFF_CSUPP,
296     [NVME_CMD_IO_MGMT_SEND]         = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
297 };
298 
299 static const uint32_t nvme_cse_iocs_zoned[256] = {
300     [NVME_CMD_FLUSH]                = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
301     [NVME_CMD_WRITE_ZEROES]         = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
302     [NVME_CMD_WRITE]                = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
303     [NVME_CMD_READ]                 = NVME_CMD_EFF_CSUPP,
304     [NVME_CMD_DSM]                  = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
305     [NVME_CMD_VERIFY]               = NVME_CMD_EFF_CSUPP,
306     [NVME_CMD_COPY]                 = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
307     [NVME_CMD_COMPARE]              = NVME_CMD_EFF_CSUPP,
308     [NVME_CMD_ZONE_APPEND]          = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
309     [NVME_CMD_ZONE_MGMT_SEND]       = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
310     [NVME_CMD_ZONE_MGMT_RECV]       = NVME_CMD_EFF_CSUPP,
311 };
312 
313 static void nvme_process_sq(void *opaque);
314 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst);
315 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n);
316 
317 static uint16_t nvme_sqid(NvmeRequest *req)
318 {
319     return le16_to_cpu(req->sq->sqid);
320 }
321 
322 static inline uint16_t nvme_make_pid(NvmeNamespace *ns, uint16_t rg,
323                                      uint16_t ph)
324 {
325     uint16_t rgif = ns->endgrp->fdp.rgif;
326 
327     if (!rgif) {
328         return ph;
329     }
330 
331     return (rg << (16 - rgif)) | ph;
332 }
333 
334 static inline bool nvme_ph_valid(NvmeNamespace *ns, uint16_t ph)
335 {
336     return ph < ns->fdp.nphs;
337 }
338 
339 static inline bool nvme_rg_valid(NvmeEnduranceGroup *endgrp, uint16_t rg)
340 {
341     return rg < endgrp->fdp.nrg;
342 }
343 
344 static inline uint16_t nvme_pid2ph(NvmeNamespace *ns, uint16_t pid)
345 {
346     uint16_t rgif = ns->endgrp->fdp.rgif;
347 
348     if (!rgif) {
349         return pid;
350     }
351 
352     return pid & ((1 << (15 - rgif)) - 1);
353 }
354 
355 static inline uint16_t nvme_pid2rg(NvmeNamespace *ns, uint16_t pid)
356 {
357     uint16_t rgif = ns->endgrp->fdp.rgif;
358 
359     if (!rgif) {
360         return 0;
361     }
362 
363     return pid >> (16 - rgif);
364 }
365 
366 static inline bool nvme_parse_pid(NvmeNamespace *ns, uint16_t pid,
367                                   uint16_t *ph, uint16_t *rg)
368 {
369     *rg = nvme_pid2rg(ns, pid);
370     *ph = nvme_pid2ph(ns, pid);
371 
372     return nvme_ph_valid(ns, *ph) && nvme_rg_valid(ns->endgrp, *rg);
373 }
374 
375 static void nvme_assign_zone_state(NvmeNamespace *ns, NvmeZone *zone,
376                                    NvmeZoneState state)
377 {
378     if (QTAILQ_IN_USE(zone, entry)) {
379         switch (nvme_get_zone_state(zone)) {
380         case NVME_ZONE_STATE_EXPLICITLY_OPEN:
381             QTAILQ_REMOVE(&ns->exp_open_zones, zone, entry);
382             break;
383         case NVME_ZONE_STATE_IMPLICITLY_OPEN:
384             QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
385             break;
386         case NVME_ZONE_STATE_CLOSED:
387             QTAILQ_REMOVE(&ns->closed_zones, zone, entry);
388             break;
389         case NVME_ZONE_STATE_FULL:
390             QTAILQ_REMOVE(&ns->full_zones, zone, entry);
391         default:
392             ;
393         }
394     }
395 
396     nvme_set_zone_state(zone, state);
397 
398     switch (state) {
399     case NVME_ZONE_STATE_EXPLICITLY_OPEN:
400         QTAILQ_INSERT_TAIL(&ns->exp_open_zones, zone, entry);
401         break;
402     case NVME_ZONE_STATE_IMPLICITLY_OPEN:
403         QTAILQ_INSERT_TAIL(&ns->imp_open_zones, zone, entry);
404         break;
405     case NVME_ZONE_STATE_CLOSED:
406         QTAILQ_INSERT_TAIL(&ns->closed_zones, zone, entry);
407         break;
408     case NVME_ZONE_STATE_FULL:
409         QTAILQ_INSERT_TAIL(&ns->full_zones, zone, entry);
410     case NVME_ZONE_STATE_READ_ONLY:
411         break;
412     default:
413         zone->d.za = 0;
414     }
415 }
416 
417 static uint16_t nvme_zns_check_resources(NvmeNamespace *ns, uint32_t act,
418                                          uint32_t opn, uint32_t zrwa)
419 {
420     if (ns->params.max_active_zones != 0 &&
421         ns->nr_active_zones + act > ns->params.max_active_zones) {
422         trace_pci_nvme_err_insuff_active_res(ns->params.max_active_zones);
423         return NVME_ZONE_TOO_MANY_ACTIVE | NVME_DNR;
424     }
425 
426     if (ns->params.max_open_zones != 0 &&
427         ns->nr_open_zones + opn > ns->params.max_open_zones) {
428         trace_pci_nvme_err_insuff_open_res(ns->params.max_open_zones);
429         return NVME_ZONE_TOO_MANY_OPEN | NVME_DNR;
430     }
431 
432     if (zrwa > ns->zns.numzrwa) {
433         return NVME_NOZRWA | NVME_DNR;
434     }
435 
436     return NVME_SUCCESS;
437 }
438 
439 /*
440  * Check if we can open a zone without exceeding open/active limits.
441  * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5).
442  */
443 static uint16_t nvme_aor_check(NvmeNamespace *ns, uint32_t act, uint32_t opn)
444 {
445     return nvme_zns_check_resources(ns, act, opn, 0);
446 }
447 
448 static NvmeFdpEvent *nvme_fdp_alloc_event(NvmeCtrl *n, NvmeFdpEventBuffer *ebuf)
449 {
450     NvmeFdpEvent *ret = NULL;
451     bool is_full = ebuf->next == ebuf->start && ebuf->nelems;
452 
453     ret = &ebuf->events[ebuf->next++];
454     if (unlikely(ebuf->next == NVME_FDP_MAX_EVENTS)) {
455         ebuf->next = 0;
456     }
457     if (is_full) {
458         ebuf->start = ebuf->next;
459     } else {
460         ebuf->nelems++;
461     }
462 
463     memset(ret, 0, sizeof(NvmeFdpEvent));
464     ret->timestamp = nvme_get_timestamp(n);
465 
466     return ret;
467 }
468 
469 static inline int log_event(NvmeRuHandle *ruh, uint8_t event_type)
470 {
471     return (ruh->event_filter >> nvme_fdp_evf_shifts[event_type]) & 0x1;
472 }
473 
474 static bool nvme_update_ruh(NvmeCtrl *n, NvmeNamespace *ns, uint16_t pid)
475 {
476     NvmeEnduranceGroup *endgrp = ns->endgrp;
477     NvmeRuHandle *ruh;
478     NvmeReclaimUnit *ru;
479     NvmeFdpEvent *e = NULL;
480     uint16_t ph, rg, ruhid;
481 
482     if (!nvme_parse_pid(ns, pid, &ph, &rg)) {
483         return false;
484     }
485 
486     ruhid = ns->fdp.phs[ph];
487 
488     ruh = &endgrp->fdp.ruhs[ruhid];
489     ru = &ruh->rus[rg];
490 
491     if (ru->ruamw) {
492         if (log_event(ruh, FDP_EVT_RU_NOT_FULLY_WRITTEN)) {
493             e = nvme_fdp_alloc_event(n, &endgrp->fdp.host_events);
494             e->type = FDP_EVT_RU_NOT_FULLY_WRITTEN;
495             e->flags = FDPEF_PIV | FDPEF_NSIDV | FDPEF_LV;
496             e->pid = cpu_to_le16(pid);
497             e->nsid = cpu_to_le32(ns->params.nsid);
498             e->rgid = cpu_to_le16(rg);
499             e->ruhid = cpu_to_le16(ruhid);
500         }
501 
502         /* log (eventual) GC overhead of prematurely swapping the RU */
503         nvme_fdp_stat_inc(&endgrp->fdp.mbmw, nvme_l2b(ns, ru->ruamw));
504     }
505 
506     ru->ruamw = ruh->ruamw;
507 
508     return true;
509 }
510 
511 static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
512 {
513     hwaddr hi, lo;
514 
515     if (!n->cmb.cmse) {
516         return false;
517     }
518 
519     lo = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
520     hi = lo + int128_get64(n->cmb.mem.size);
521 
522     return addr >= lo && addr < hi;
523 }
524 
525 static inline void *nvme_addr_to_cmb(NvmeCtrl *n, hwaddr addr)
526 {
527     hwaddr base = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
528     return &n->cmb.buf[addr - base];
529 }
530 
531 static bool nvme_addr_is_pmr(NvmeCtrl *n, hwaddr addr)
532 {
533     hwaddr hi;
534 
535     if (!n->pmr.cmse) {
536         return false;
537     }
538 
539     hi = n->pmr.cba + int128_get64(n->pmr.dev->mr.size);
540 
541     return addr >= n->pmr.cba && addr < hi;
542 }
543 
544 static inline void *nvme_addr_to_pmr(NvmeCtrl *n, hwaddr addr)
545 {
546     return memory_region_get_ram_ptr(&n->pmr.dev->mr) + (addr - n->pmr.cba);
547 }
548 
549 static inline bool nvme_addr_is_iomem(NvmeCtrl *n, hwaddr addr)
550 {
551     hwaddr hi, lo;
552 
553     /*
554      * The purpose of this check is to guard against invalid "local" access to
555      * the iomem (i.e. controller registers). Thus, we check against the range
556      * covered by the 'bar0' MemoryRegion since that is currently composed of
557      * two subregions (the NVMe "MBAR" and the MSI-X table/pba). Note, however,
558      * that if the device model is ever changed to allow the CMB to be located
559      * in BAR0 as well, then this must be changed.
560      */
561     lo = n->bar0.addr;
562     hi = lo + int128_get64(n->bar0.size);
563 
564     return addr >= lo && addr < hi;
565 }
566 
567 static int nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
568 {
569     hwaddr hi = addr + size - 1;
570     if (hi < addr) {
571         return 1;
572     }
573 
574     if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
575         memcpy(buf, nvme_addr_to_cmb(n, addr), size);
576         return 0;
577     }
578 
579     if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
580         memcpy(buf, nvme_addr_to_pmr(n, addr), size);
581         return 0;
582     }
583 
584     return pci_dma_read(PCI_DEVICE(n), addr, buf, size);
585 }
586 
587 static int nvme_addr_write(NvmeCtrl *n, hwaddr addr, const void *buf, int size)
588 {
589     hwaddr hi = addr + size - 1;
590     if (hi < addr) {
591         return 1;
592     }
593 
594     if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
595         memcpy(nvme_addr_to_cmb(n, addr), buf, size);
596         return 0;
597     }
598 
599     if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
600         memcpy(nvme_addr_to_pmr(n, addr), buf, size);
601         return 0;
602     }
603 
604     return pci_dma_write(PCI_DEVICE(n), addr, buf, size);
605 }
606 
607 static bool nvme_nsid_valid(NvmeCtrl *n, uint32_t nsid)
608 {
609     return nsid &&
610         (nsid == NVME_NSID_BROADCAST || nsid <= NVME_MAX_NAMESPACES);
611 }
612 
613 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
614 {
615     return sqid < n->conf_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1;
616 }
617 
618 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
619 {
620     return cqid < n->conf_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1;
621 }
622 
623 static void nvme_inc_cq_tail(NvmeCQueue *cq)
624 {
625     cq->tail++;
626     if (cq->tail >= cq->size) {
627         cq->tail = 0;
628         cq->phase = !cq->phase;
629     }
630 }
631 
632 static void nvme_inc_sq_head(NvmeSQueue *sq)
633 {
634     sq->head = (sq->head + 1) % sq->size;
635 }
636 
637 static uint8_t nvme_cq_full(NvmeCQueue *cq)
638 {
639     return (cq->tail + 1) % cq->size == cq->head;
640 }
641 
642 static uint8_t nvme_sq_empty(NvmeSQueue *sq)
643 {
644     return sq->head == sq->tail;
645 }
646 
647 static void nvme_irq_check(NvmeCtrl *n)
648 {
649     PCIDevice *pci = PCI_DEVICE(n);
650     uint32_t intms = ldl_le_p(&n->bar.intms);
651 
652     if (msix_enabled(pci)) {
653         return;
654     }
655     if (~intms & n->irq_status) {
656         pci_irq_assert(pci);
657     } else {
658         pci_irq_deassert(pci);
659     }
660 }
661 
662 static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
663 {
664     PCIDevice *pci = PCI_DEVICE(n);
665 
666     if (cq->irq_enabled) {
667         if (msix_enabled(pci)) {
668             trace_pci_nvme_irq_msix(cq->vector);
669             msix_notify(pci, cq->vector);
670         } else {
671             trace_pci_nvme_irq_pin();
672             assert(cq->vector < 32);
673             n->irq_status |= 1 << cq->vector;
674             nvme_irq_check(n);
675         }
676     } else {
677         trace_pci_nvme_irq_masked();
678     }
679 }
680 
681 static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
682 {
683     if (cq->irq_enabled) {
684         if (msix_enabled(PCI_DEVICE(n))) {
685             return;
686         } else {
687             assert(cq->vector < 32);
688             if (!n->cq_pending) {
689                 n->irq_status &= ~(1 << cq->vector);
690             }
691             nvme_irq_check(n);
692         }
693     }
694 }
695 
696 static void nvme_req_clear(NvmeRequest *req)
697 {
698     req->ns = NULL;
699     req->opaque = NULL;
700     req->aiocb = NULL;
701     memset(&req->cqe, 0x0, sizeof(req->cqe));
702     req->status = NVME_SUCCESS;
703 }
704 
705 static inline void nvme_sg_init(NvmeCtrl *n, NvmeSg *sg, bool dma)
706 {
707     if (dma) {
708         pci_dma_sglist_init(&sg->qsg, PCI_DEVICE(n), 0);
709         sg->flags = NVME_SG_DMA;
710     } else {
711         qemu_iovec_init(&sg->iov, 0);
712     }
713 
714     sg->flags |= NVME_SG_ALLOC;
715 }
716 
717 static inline void nvme_sg_unmap(NvmeSg *sg)
718 {
719     if (!(sg->flags & NVME_SG_ALLOC)) {
720         return;
721     }
722 
723     if (sg->flags & NVME_SG_DMA) {
724         qemu_sglist_destroy(&sg->qsg);
725     } else {
726         qemu_iovec_destroy(&sg->iov);
727     }
728 
729     memset(sg, 0x0, sizeof(*sg));
730 }
731 
732 /*
733  * When metadata is transferred as extended LBAs, the DPTR mapped into `sg`
734  * holds both data and metadata. This function splits the data and metadata
735  * into two separate QSG/IOVs.
736  */
737 static void nvme_sg_split(NvmeSg *sg, NvmeNamespace *ns, NvmeSg *data,
738                           NvmeSg *mdata)
739 {
740     NvmeSg *dst = data;
741     uint32_t trans_len, count = ns->lbasz;
742     uint64_t offset = 0;
743     bool dma = sg->flags & NVME_SG_DMA;
744     size_t sge_len;
745     size_t sg_len = dma ? sg->qsg.size : sg->iov.size;
746     int sg_idx = 0;
747 
748     assert(sg->flags & NVME_SG_ALLOC);
749 
750     while (sg_len) {
751         sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
752 
753         trans_len = MIN(sg_len, count);
754         trans_len = MIN(trans_len, sge_len - offset);
755 
756         if (dst) {
757             if (dma) {
758                 qemu_sglist_add(&dst->qsg, sg->qsg.sg[sg_idx].base + offset,
759                                 trans_len);
760             } else {
761                 qemu_iovec_add(&dst->iov,
762                                sg->iov.iov[sg_idx].iov_base + offset,
763                                trans_len);
764             }
765         }
766 
767         sg_len -= trans_len;
768         count -= trans_len;
769         offset += trans_len;
770 
771         if (count == 0) {
772             dst = (dst == data) ? mdata : data;
773             count = (dst == data) ? ns->lbasz : ns->lbaf.ms;
774         }
775 
776         if (sge_len == offset) {
777             offset = 0;
778             sg_idx++;
779         }
780     }
781 }
782 
783 static uint16_t nvme_map_addr_cmb(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
784                                   size_t len)
785 {
786     if (!len) {
787         return NVME_SUCCESS;
788     }
789 
790     trace_pci_nvme_map_addr_cmb(addr, len);
791 
792     if (!nvme_addr_is_cmb(n, addr) || !nvme_addr_is_cmb(n, addr + len - 1)) {
793         return NVME_DATA_TRAS_ERROR;
794     }
795 
796     qemu_iovec_add(iov, nvme_addr_to_cmb(n, addr), len);
797 
798     return NVME_SUCCESS;
799 }
800 
801 static uint16_t nvme_map_addr_pmr(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
802                                   size_t len)
803 {
804     if (!len) {
805         return NVME_SUCCESS;
806     }
807 
808     if (!nvme_addr_is_pmr(n, addr) || !nvme_addr_is_pmr(n, addr + len - 1)) {
809         return NVME_DATA_TRAS_ERROR;
810     }
811 
812     qemu_iovec_add(iov, nvme_addr_to_pmr(n, addr), len);
813 
814     return NVME_SUCCESS;
815 }
816 
817 static uint16_t nvme_map_addr(NvmeCtrl *n, NvmeSg *sg, hwaddr addr, size_t len)
818 {
819     bool cmb = false, pmr = false;
820 
821     if (!len) {
822         return NVME_SUCCESS;
823     }
824 
825     trace_pci_nvme_map_addr(addr, len);
826 
827     if (nvme_addr_is_iomem(n, addr)) {
828         return NVME_DATA_TRAS_ERROR;
829     }
830 
831     if (nvme_addr_is_cmb(n, addr)) {
832         cmb = true;
833     } else if (nvme_addr_is_pmr(n, addr)) {
834         pmr = true;
835     }
836 
837     if (cmb || pmr) {
838         if (sg->flags & NVME_SG_DMA) {
839             return NVME_INVALID_USE_OF_CMB | NVME_DNR;
840         }
841 
842         if (sg->iov.niov + 1 > IOV_MAX) {
843             goto max_mappings_exceeded;
844         }
845 
846         if (cmb) {
847             return nvme_map_addr_cmb(n, &sg->iov, addr, len);
848         } else {
849             return nvme_map_addr_pmr(n, &sg->iov, addr, len);
850         }
851     }
852 
853     if (!(sg->flags & NVME_SG_DMA)) {
854         return NVME_INVALID_USE_OF_CMB | NVME_DNR;
855     }
856 
857     if (sg->qsg.nsg + 1 > IOV_MAX) {
858         goto max_mappings_exceeded;
859     }
860 
861     qemu_sglist_add(&sg->qsg, addr, len);
862 
863     return NVME_SUCCESS;
864 
865 max_mappings_exceeded:
866     NVME_GUEST_ERR(pci_nvme_ub_too_many_mappings,
867                    "number of mappings exceed 1024");
868     return NVME_INTERNAL_DEV_ERROR | NVME_DNR;
869 }
870 
871 static inline bool nvme_addr_is_dma(NvmeCtrl *n, hwaddr addr)
872 {
873     return !(nvme_addr_is_cmb(n, addr) || nvme_addr_is_pmr(n, addr));
874 }
875 
876 static uint16_t nvme_map_prp(NvmeCtrl *n, NvmeSg *sg, uint64_t prp1,
877                              uint64_t prp2, uint32_t len)
878 {
879     hwaddr trans_len = n->page_size - (prp1 % n->page_size);
880     trans_len = MIN(len, trans_len);
881     int num_prps = (len >> n->page_bits) + 1;
882     uint16_t status;
883     int ret;
884 
885     trace_pci_nvme_map_prp(trans_len, len, prp1, prp2, num_prps);
886 
887     nvme_sg_init(n, sg, nvme_addr_is_dma(n, prp1));
888 
889     status = nvme_map_addr(n, sg, prp1, trans_len);
890     if (status) {
891         goto unmap;
892     }
893 
894     len -= trans_len;
895     if (len) {
896         if (len > n->page_size) {
897             g_autofree uint64_t *prp_list = g_new(uint64_t, n->max_prp_ents);
898             uint32_t nents, prp_trans;
899             int i = 0;
900 
901             /*
902              * The first PRP list entry, pointed to by PRP2 may contain offset.
903              * Hence, we need to calculate the number of entries in based on
904              * that offset.
905              */
906             nents = (n->page_size - (prp2 & (n->page_size - 1))) >> 3;
907             prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
908             ret = nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
909             if (ret) {
910                 trace_pci_nvme_err_addr_read(prp2);
911                 status = NVME_DATA_TRAS_ERROR;
912                 goto unmap;
913             }
914             while (len != 0) {
915                 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
916 
917                 if (i == nents - 1 && len > n->page_size) {
918                     if (unlikely(prp_ent & (n->page_size - 1))) {
919                         trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
920                         status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
921                         goto unmap;
922                     }
923 
924                     i = 0;
925                     nents = (len + n->page_size - 1) >> n->page_bits;
926                     nents = MIN(nents, n->max_prp_ents);
927                     prp_trans = nents * sizeof(uint64_t);
928                     ret = nvme_addr_read(n, prp_ent, (void *)prp_list,
929                                          prp_trans);
930                     if (ret) {
931                         trace_pci_nvme_err_addr_read(prp_ent);
932                         status = NVME_DATA_TRAS_ERROR;
933                         goto unmap;
934                     }
935                     prp_ent = le64_to_cpu(prp_list[i]);
936                 }
937 
938                 if (unlikely(prp_ent & (n->page_size - 1))) {
939                     trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
940                     status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
941                     goto unmap;
942                 }
943 
944                 trans_len = MIN(len, n->page_size);
945                 status = nvme_map_addr(n, sg, prp_ent, trans_len);
946                 if (status) {
947                     goto unmap;
948                 }
949 
950                 len -= trans_len;
951                 i++;
952             }
953         } else {
954             if (unlikely(prp2 & (n->page_size - 1))) {
955                 trace_pci_nvme_err_invalid_prp2_align(prp2);
956                 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
957                 goto unmap;
958             }
959             status = nvme_map_addr(n, sg, prp2, len);
960             if (status) {
961                 goto unmap;
962             }
963         }
964     }
965 
966     return NVME_SUCCESS;
967 
968 unmap:
969     nvme_sg_unmap(sg);
970     return status;
971 }
972 
973 /*
974  * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
975  * number of bytes mapped in len.
976  */
977 static uint16_t nvme_map_sgl_data(NvmeCtrl *n, NvmeSg *sg,
978                                   NvmeSglDescriptor *segment, uint64_t nsgld,
979                                   size_t *len, NvmeCmd *cmd)
980 {
981     dma_addr_t addr, trans_len;
982     uint32_t dlen;
983     uint16_t status;
984 
985     for (int i = 0; i < nsgld; i++) {
986         uint8_t type = NVME_SGL_TYPE(segment[i].type);
987 
988         switch (type) {
989         case NVME_SGL_DESCR_TYPE_DATA_BLOCK:
990             break;
991         case NVME_SGL_DESCR_TYPE_SEGMENT:
992         case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
993             return NVME_INVALID_NUM_SGL_DESCRS | NVME_DNR;
994         default:
995             return NVME_SGL_DESCR_TYPE_INVALID | NVME_DNR;
996         }
997 
998         dlen = le32_to_cpu(segment[i].len);
999 
1000         if (!dlen) {
1001             continue;
1002         }
1003 
1004         if (*len == 0) {
1005             /*
1006              * All data has been mapped, but the SGL contains additional
1007              * segments and/or descriptors. The controller might accept
1008              * ignoring the rest of the SGL.
1009              */
1010             uint32_t sgls = le32_to_cpu(n->id_ctrl.sgls);
1011             if (sgls & NVME_CTRL_SGLS_EXCESS_LENGTH) {
1012                 break;
1013             }
1014 
1015             trace_pci_nvme_err_invalid_sgl_excess_length(dlen);
1016             return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1017         }
1018 
1019         trans_len = MIN(*len, dlen);
1020 
1021         addr = le64_to_cpu(segment[i].addr);
1022 
1023         if (UINT64_MAX - addr < dlen) {
1024             return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1025         }
1026 
1027         status = nvme_map_addr(n, sg, addr, trans_len);
1028         if (status) {
1029             return status;
1030         }
1031 
1032         *len -= trans_len;
1033     }
1034 
1035     return NVME_SUCCESS;
1036 }
1037 
1038 static uint16_t nvme_map_sgl(NvmeCtrl *n, NvmeSg *sg, NvmeSglDescriptor sgl,
1039                              size_t len, NvmeCmd *cmd)
1040 {
1041     /*
1042      * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
1043      * dynamically allocating a potentially huge SGL. The spec allows the SGL
1044      * to be larger (as in number of bytes required to describe the SGL
1045      * descriptors and segment chain) than the command transfer size, so it is
1046      * not bounded by MDTS.
1047      */
1048 #define SEG_CHUNK_SIZE 256
1049 
1050     NvmeSglDescriptor segment[SEG_CHUNK_SIZE], *sgld, *last_sgld;
1051     uint64_t nsgld;
1052     uint32_t seg_len;
1053     uint16_t status;
1054     hwaddr addr;
1055     int ret;
1056 
1057     sgld = &sgl;
1058     addr = le64_to_cpu(sgl.addr);
1059 
1060     trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl.type), len);
1061 
1062     nvme_sg_init(n, sg, nvme_addr_is_dma(n, addr));
1063 
1064     /*
1065      * If the entire transfer can be described with a single data block it can
1066      * be mapped directly.
1067      */
1068     if (NVME_SGL_TYPE(sgl.type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1069         status = nvme_map_sgl_data(n, sg, sgld, 1, &len, cmd);
1070         if (status) {
1071             goto unmap;
1072         }
1073 
1074         goto out;
1075     }
1076 
1077     for (;;) {
1078         switch (NVME_SGL_TYPE(sgld->type)) {
1079         case NVME_SGL_DESCR_TYPE_SEGMENT:
1080         case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
1081             break;
1082         default:
1083             return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1084         }
1085 
1086         seg_len = le32_to_cpu(sgld->len);
1087 
1088         /* check the length of the (Last) Segment descriptor */
1089         if (!seg_len || seg_len & 0xf) {
1090             return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1091         }
1092 
1093         if (UINT64_MAX - addr < seg_len) {
1094             return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1095         }
1096 
1097         nsgld = seg_len / sizeof(NvmeSglDescriptor);
1098 
1099         while (nsgld > SEG_CHUNK_SIZE) {
1100             if (nvme_addr_read(n, addr, segment, sizeof(segment))) {
1101                 trace_pci_nvme_err_addr_read(addr);
1102                 status = NVME_DATA_TRAS_ERROR;
1103                 goto unmap;
1104             }
1105 
1106             status = nvme_map_sgl_data(n, sg, segment, SEG_CHUNK_SIZE,
1107                                        &len, cmd);
1108             if (status) {
1109                 goto unmap;
1110             }
1111 
1112             nsgld -= SEG_CHUNK_SIZE;
1113             addr += SEG_CHUNK_SIZE * sizeof(NvmeSglDescriptor);
1114         }
1115 
1116         ret = nvme_addr_read(n, addr, segment, nsgld *
1117                              sizeof(NvmeSglDescriptor));
1118         if (ret) {
1119             trace_pci_nvme_err_addr_read(addr);
1120             status = NVME_DATA_TRAS_ERROR;
1121             goto unmap;
1122         }
1123 
1124         last_sgld = &segment[nsgld - 1];
1125 
1126         /*
1127          * If the segment ends with a Data Block, then we are done.
1128          */
1129         if (NVME_SGL_TYPE(last_sgld->type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1130             status = nvme_map_sgl_data(n, sg, segment, nsgld, &len, cmd);
1131             if (status) {
1132                 goto unmap;
1133             }
1134 
1135             goto out;
1136         }
1137 
1138         /*
1139          * If the last descriptor was not a Data Block, then the current
1140          * segment must not be a Last Segment.
1141          */
1142         if (NVME_SGL_TYPE(sgld->type) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT) {
1143             status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1144             goto unmap;
1145         }
1146 
1147         sgld = last_sgld;
1148         addr = le64_to_cpu(sgld->addr);
1149 
1150         /*
1151          * Do not map the last descriptor; it will be a Segment or Last Segment
1152          * descriptor and is handled by the next iteration.
1153          */
1154         status = nvme_map_sgl_data(n, sg, segment, nsgld - 1, &len, cmd);
1155         if (status) {
1156             goto unmap;
1157         }
1158     }
1159 
1160 out:
1161     /* if there is any residual left in len, the SGL was too short */
1162     if (len) {
1163         status = NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1164         goto unmap;
1165     }
1166 
1167     return NVME_SUCCESS;
1168 
1169 unmap:
1170     nvme_sg_unmap(sg);
1171     return status;
1172 }
1173 
1174 uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1175                        NvmeCmd *cmd)
1176 {
1177     uint64_t prp1, prp2;
1178 
1179     switch (NVME_CMD_FLAGS_PSDT(cmd->flags)) {
1180     case NVME_PSDT_PRP:
1181         prp1 = le64_to_cpu(cmd->dptr.prp1);
1182         prp2 = le64_to_cpu(cmd->dptr.prp2);
1183 
1184         return nvme_map_prp(n, sg, prp1, prp2, len);
1185     case NVME_PSDT_SGL_MPTR_CONTIGUOUS:
1186     case NVME_PSDT_SGL_MPTR_SGL:
1187         return nvme_map_sgl(n, sg, cmd->dptr.sgl, len, cmd);
1188     default:
1189         return NVME_INVALID_FIELD;
1190     }
1191 }
1192 
1193 static uint16_t nvme_map_mptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1194                               NvmeCmd *cmd)
1195 {
1196     int psdt = NVME_CMD_FLAGS_PSDT(cmd->flags);
1197     hwaddr mptr = le64_to_cpu(cmd->mptr);
1198     uint16_t status;
1199 
1200     if (psdt == NVME_PSDT_SGL_MPTR_SGL) {
1201         NvmeSglDescriptor sgl;
1202 
1203         if (nvme_addr_read(n, mptr, &sgl, sizeof(sgl))) {
1204             return NVME_DATA_TRAS_ERROR;
1205         }
1206 
1207         status = nvme_map_sgl(n, sg, sgl, len, cmd);
1208         if (status && (status & 0x7ff) == NVME_DATA_SGL_LEN_INVALID) {
1209             status = NVME_MD_SGL_LEN_INVALID | NVME_DNR;
1210         }
1211 
1212         return status;
1213     }
1214 
1215     nvme_sg_init(n, sg, nvme_addr_is_dma(n, mptr));
1216     status = nvme_map_addr(n, sg, mptr, len);
1217     if (status) {
1218         nvme_sg_unmap(sg);
1219     }
1220 
1221     return status;
1222 }
1223 
1224 static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1225 {
1226     NvmeNamespace *ns = req->ns;
1227     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1228     bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1229     bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1230     size_t len = nvme_l2b(ns, nlb);
1231     uint16_t status;
1232 
1233     if (nvme_ns_ext(ns) &&
1234         !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1235         NvmeSg sg;
1236 
1237         len += nvme_m2b(ns, nlb);
1238 
1239         status = nvme_map_dptr(n, &sg, len, &req->cmd);
1240         if (status) {
1241             return status;
1242         }
1243 
1244         nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1245         nvme_sg_split(&sg, ns, &req->sg, NULL);
1246         nvme_sg_unmap(&sg);
1247 
1248         return NVME_SUCCESS;
1249     }
1250 
1251     return nvme_map_dptr(n, &req->sg, len, &req->cmd);
1252 }
1253 
1254 static uint16_t nvme_map_mdata(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1255 {
1256     NvmeNamespace *ns = req->ns;
1257     size_t len = nvme_m2b(ns, nlb);
1258     uint16_t status;
1259 
1260     if (nvme_ns_ext(ns)) {
1261         NvmeSg sg;
1262 
1263         len += nvme_l2b(ns, nlb);
1264 
1265         status = nvme_map_dptr(n, &sg, len, &req->cmd);
1266         if (status) {
1267             return status;
1268         }
1269 
1270         nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1271         nvme_sg_split(&sg, ns, NULL, &req->sg);
1272         nvme_sg_unmap(&sg);
1273 
1274         return NVME_SUCCESS;
1275     }
1276 
1277     return nvme_map_mptr(n, &req->sg, len, &req->cmd);
1278 }
1279 
1280 static uint16_t nvme_tx_interleaved(NvmeCtrl *n, NvmeSg *sg, uint8_t *ptr,
1281                                     uint32_t len, uint32_t bytes,
1282                                     int32_t skip_bytes, int64_t offset,
1283                                     NvmeTxDirection dir)
1284 {
1285     hwaddr addr;
1286     uint32_t trans_len, count = bytes;
1287     bool dma = sg->flags & NVME_SG_DMA;
1288     int64_t sge_len;
1289     int sg_idx = 0;
1290     int ret;
1291 
1292     assert(sg->flags & NVME_SG_ALLOC);
1293 
1294     while (len) {
1295         sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
1296 
1297         if (sge_len - offset < 0) {
1298             offset -= sge_len;
1299             sg_idx++;
1300             continue;
1301         }
1302 
1303         if (sge_len == offset) {
1304             offset = 0;
1305             sg_idx++;
1306             continue;
1307         }
1308 
1309         trans_len = MIN(len, count);
1310         trans_len = MIN(trans_len, sge_len - offset);
1311 
1312         if (dma) {
1313             addr = sg->qsg.sg[sg_idx].base + offset;
1314         } else {
1315             addr = (hwaddr)(uintptr_t)sg->iov.iov[sg_idx].iov_base + offset;
1316         }
1317 
1318         if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1319             ret = nvme_addr_read(n, addr, ptr, trans_len);
1320         } else {
1321             ret = nvme_addr_write(n, addr, ptr, trans_len);
1322         }
1323 
1324         if (ret) {
1325             return NVME_DATA_TRAS_ERROR;
1326         }
1327 
1328         ptr += trans_len;
1329         len -= trans_len;
1330         count -= trans_len;
1331         offset += trans_len;
1332 
1333         if (count == 0) {
1334             count = bytes;
1335             offset += skip_bytes;
1336         }
1337     }
1338 
1339     return NVME_SUCCESS;
1340 }
1341 
1342 static uint16_t nvme_tx(NvmeCtrl *n, NvmeSg *sg, void *ptr, uint32_t len,
1343                         NvmeTxDirection dir)
1344 {
1345     assert(sg->flags & NVME_SG_ALLOC);
1346 
1347     if (sg->flags & NVME_SG_DMA) {
1348         const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
1349         dma_addr_t residual;
1350 
1351         if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1352             dma_buf_write(ptr, len, &residual, &sg->qsg, attrs);
1353         } else {
1354             dma_buf_read(ptr, len, &residual, &sg->qsg, attrs);
1355         }
1356 
1357         if (unlikely(residual)) {
1358             trace_pci_nvme_err_invalid_dma();
1359             return NVME_INVALID_FIELD | NVME_DNR;
1360         }
1361     } else {
1362         size_t bytes;
1363 
1364         if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1365             bytes = qemu_iovec_to_buf(&sg->iov, 0, ptr, len);
1366         } else {
1367             bytes = qemu_iovec_from_buf(&sg->iov, 0, ptr, len);
1368         }
1369 
1370         if (unlikely(bytes != len)) {
1371             trace_pci_nvme_err_invalid_dma();
1372             return NVME_INVALID_FIELD | NVME_DNR;
1373         }
1374     }
1375 
1376     return NVME_SUCCESS;
1377 }
1378 
1379 static inline uint16_t nvme_c2h(NvmeCtrl *n, void *ptr, uint32_t len,
1380                                 NvmeRequest *req)
1381 {
1382     uint16_t status;
1383 
1384     status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1385     if (status) {
1386         return status;
1387     }
1388 
1389     return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_FROM_DEVICE);
1390 }
1391 
1392 static inline uint16_t nvme_h2c(NvmeCtrl *n, void *ptr, uint32_t len,
1393                                 NvmeRequest *req)
1394 {
1395     uint16_t status;
1396 
1397     status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1398     if (status) {
1399         return status;
1400     }
1401 
1402     return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_TO_DEVICE);
1403 }
1404 
1405 uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
1406                           NvmeTxDirection dir, NvmeRequest *req)
1407 {
1408     NvmeNamespace *ns = req->ns;
1409     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1410     bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1411     bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1412 
1413     if (nvme_ns_ext(ns) &&
1414         !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1415         return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbasz,
1416                                    ns->lbaf.ms, 0, dir);
1417     }
1418 
1419     return nvme_tx(n, &req->sg, ptr, len, dir);
1420 }
1421 
1422 uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
1423                            NvmeTxDirection dir, NvmeRequest *req)
1424 {
1425     NvmeNamespace *ns = req->ns;
1426     uint16_t status;
1427 
1428     if (nvme_ns_ext(ns)) {
1429         return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbaf.ms,
1430                                    ns->lbasz, ns->lbasz, dir);
1431     }
1432 
1433     nvme_sg_unmap(&req->sg);
1434 
1435     status = nvme_map_mptr(n, &req->sg, len, &req->cmd);
1436     if (status) {
1437         return status;
1438     }
1439 
1440     return nvme_tx(n, &req->sg, ptr, len, dir);
1441 }
1442 
1443 static inline void nvme_blk_read(BlockBackend *blk, int64_t offset,
1444                                  uint32_t align, BlockCompletionFunc *cb,
1445                                  NvmeRequest *req)
1446 {
1447     assert(req->sg.flags & NVME_SG_ALLOC);
1448 
1449     if (req->sg.flags & NVME_SG_DMA) {
1450         req->aiocb = dma_blk_read(blk, &req->sg.qsg, offset, align, cb, req);
1451     } else {
1452         req->aiocb = blk_aio_preadv(blk, offset, &req->sg.iov, 0, cb, req);
1453     }
1454 }
1455 
1456 static inline void nvme_blk_write(BlockBackend *blk, int64_t offset,
1457                                   uint32_t align, BlockCompletionFunc *cb,
1458                                   NvmeRequest *req)
1459 {
1460     assert(req->sg.flags & NVME_SG_ALLOC);
1461 
1462     if (req->sg.flags & NVME_SG_DMA) {
1463         req->aiocb = dma_blk_write(blk, &req->sg.qsg, offset, align, cb, req);
1464     } else {
1465         req->aiocb = blk_aio_pwritev(blk, offset, &req->sg.iov, 0, cb, req);
1466     }
1467 }
1468 
1469 static void nvme_update_cq_eventidx(const NvmeCQueue *cq)
1470 {
1471     trace_pci_nvme_update_cq_eventidx(cq->cqid, cq->head);
1472 
1473     stl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->ei_addr, cq->head,
1474                    MEMTXATTRS_UNSPECIFIED);
1475 }
1476 
1477 static void nvme_update_cq_head(NvmeCQueue *cq)
1478 {
1479     ldl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->db_addr, &cq->head,
1480                    MEMTXATTRS_UNSPECIFIED);
1481 
1482     trace_pci_nvme_update_cq_head(cq->cqid, cq->head);
1483 }
1484 
1485 static void nvme_post_cqes(void *opaque)
1486 {
1487     NvmeCQueue *cq = opaque;
1488     NvmeCtrl *n = cq->ctrl;
1489     NvmeRequest *req, *next;
1490     bool pending = cq->head != cq->tail;
1491     int ret;
1492 
1493     QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
1494         NvmeSQueue *sq;
1495         hwaddr addr;
1496 
1497         if (n->dbbuf_enabled) {
1498             nvme_update_cq_eventidx(cq);
1499             nvme_update_cq_head(cq);
1500         }
1501 
1502         if (nvme_cq_full(cq)) {
1503             break;
1504         }
1505 
1506         sq = req->sq;
1507         req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
1508         req->cqe.sq_id = cpu_to_le16(sq->sqid);
1509         req->cqe.sq_head = cpu_to_le16(sq->head);
1510         addr = cq->dma_addr + (cq->tail << NVME_CQES);
1511         ret = pci_dma_write(PCI_DEVICE(n), addr, (void *)&req->cqe,
1512                             sizeof(req->cqe));
1513         if (ret) {
1514             trace_pci_nvme_err_addr_write(addr);
1515             trace_pci_nvme_err_cfs();
1516             stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
1517             break;
1518         }
1519         QTAILQ_REMOVE(&cq->req_list, req, entry);
1520         nvme_inc_cq_tail(cq);
1521         nvme_sg_unmap(&req->sg);
1522         QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
1523     }
1524     if (cq->tail != cq->head) {
1525         if (cq->irq_enabled && !pending) {
1526             n->cq_pending++;
1527         }
1528 
1529         nvme_irq_assert(n, cq);
1530     }
1531 }
1532 
1533 static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
1534 {
1535     assert(cq->cqid == req->sq->cqid);
1536     trace_pci_nvme_enqueue_req_completion(nvme_cid(req), cq->cqid,
1537                                           le32_to_cpu(req->cqe.result),
1538                                           le32_to_cpu(req->cqe.dw1),
1539                                           req->status);
1540 
1541     if (req->status) {
1542         trace_pci_nvme_err_req_status(nvme_cid(req), nvme_nsid(req->ns),
1543                                       req->status, req->cmd.opcode);
1544     }
1545 
1546     QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
1547     QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
1548 
1549     qemu_bh_schedule(cq->bh);
1550 }
1551 
1552 static void nvme_process_aers(void *opaque)
1553 {
1554     NvmeCtrl *n = opaque;
1555     NvmeAsyncEvent *event, *next;
1556 
1557     trace_pci_nvme_process_aers(n->aer_queued);
1558 
1559     QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
1560         NvmeRequest *req;
1561         NvmeAerResult *result;
1562 
1563         /* can't post cqe if there is nothing to complete */
1564         if (!n->outstanding_aers) {
1565             trace_pci_nvme_no_outstanding_aers();
1566             break;
1567         }
1568 
1569         /* ignore if masked (cqe posted, but event not cleared) */
1570         if (n->aer_mask & (1 << event->result.event_type)) {
1571             trace_pci_nvme_aer_masked(event->result.event_type, n->aer_mask);
1572             continue;
1573         }
1574 
1575         QTAILQ_REMOVE(&n->aer_queue, event, entry);
1576         n->aer_queued--;
1577 
1578         n->aer_mask |= 1 << event->result.event_type;
1579         n->outstanding_aers--;
1580 
1581         req = n->aer_reqs[n->outstanding_aers];
1582 
1583         result = (NvmeAerResult *) &req->cqe.result;
1584         result->event_type = event->result.event_type;
1585         result->event_info = event->result.event_info;
1586         result->log_page = event->result.log_page;
1587         g_free(event);
1588 
1589         trace_pci_nvme_aer_post_cqe(result->event_type, result->event_info,
1590                                     result->log_page);
1591 
1592         nvme_enqueue_req_completion(&n->admin_cq, req);
1593     }
1594 }
1595 
1596 static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type,
1597                                uint8_t event_info, uint8_t log_page)
1598 {
1599     NvmeAsyncEvent *event;
1600 
1601     trace_pci_nvme_enqueue_event(event_type, event_info, log_page);
1602 
1603     if (n->aer_queued == n->params.aer_max_queued) {
1604         trace_pci_nvme_enqueue_event_noqueue(n->aer_queued);
1605         return;
1606     }
1607 
1608     event = g_new(NvmeAsyncEvent, 1);
1609     event->result = (NvmeAerResult) {
1610         .event_type = event_type,
1611         .event_info = event_info,
1612         .log_page   = log_page,
1613     };
1614 
1615     QTAILQ_INSERT_TAIL(&n->aer_queue, event, entry);
1616     n->aer_queued++;
1617 
1618     nvme_process_aers(n);
1619 }
1620 
1621 static void nvme_smart_event(NvmeCtrl *n, uint8_t event)
1622 {
1623     uint8_t aer_info;
1624 
1625     /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */
1626     if (!(NVME_AEC_SMART(n->features.async_config) & event)) {
1627         return;
1628     }
1629 
1630     switch (event) {
1631     case NVME_SMART_SPARE:
1632         aer_info = NVME_AER_INFO_SMART_SPARE_THRESH;
1633         break;
1634     case NVME_SMART_TEMPERATURE:
1635         aer_info = NVME_AER_INFO_SMART_TEMP_THRESH;
1636         break;
1637     case NVME_SMART_RELIABILITY:
1638     case NVME_SMART_MEDIA_READ_ONLY:
1639     case NVME_SMART_FAILED_VOLATILE_MEDIA:
1640     case NVME_SMART_PMR_UNRELIABLE:
1641         aer_info = NVME_AER_INFO_SMART_RELIABILITY;
1642         break;
1643     default:
1644         return;
1645     }
1646 
1647     nvme_enqueue_event(n, NVME_AER_TYPE_SMART, aer_info, NVME_LOG_SMART_INFO);
1648 }
1649 
1650 static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type)
1651 {
1652     n->aer_mask &= ~(1 << event_type);
1653     if (!QTAILQ_EMPTY(&n->aer_queue)) {
1654         nvme_process_aers(n);
1655     }
1656 }
1657 
1658 static inline uint16_t nvme_check_mdts(NvmeCtrl *n, size_t len)
1659 {
1660     uint8_t mdts = n->params.mdts;
1661 
1662     if (mdts && len > n->page_size << mdts) {
1663         trace_pci_nvme_err_mdts(len);
1664         return NVME_INVALID_FIELD | NVME_DNR;
1665     }
1666 
1667     return NVME_SUCCESS;
1668 }
1669 
1670 static inline uint16_t nvme_check_bounds(NvmeNamespace *ns, uint64_t slba,
1671                                          uint32_t nlb)
1672 {
1673     uint64_t nsze = le64_to_cpu(ns->id_ns.nsze);
1674 
1675     if (unlikely(UINT64_MAX - slba < nlb || slba + nlb > nsze)) {
1676         trace_pci_nvme_err_invalid_lba_range(slba, nlb, nsze);
1677         return NVME_LBA_RANGE | NVME_DNR;
1678     }
1679 
1680     return NVME_SUCCESS;
1681 }
1682 
1683 static int nvme_block_status_all(NvmeNamespace *ns, uint64_t slba,
1684                                  uint32_t nlb, int flags)
1685 {
1686     BlockDriverState *bs = blk_bs(ns->blkconf.blk);
1687 
1688     int64_t pnum = 0, bytes = nvme_l2b(ns, nlb);
1689     int64_t offset = nvme_l2b(ns, slba);
1690     int ret;
1691 
1692     /*
1693      * `pnum` holds the number of bytes after offset that shares the same
1694      * allocation status as the byte at offset. If `pnum` is different from
1695      * `bytes`, we should check the allocation status of the next range and
1696      * continue this until all bytes have been checked.
1697      */
1698     do {
1699         bytes -= pnum;
1700 
1701         ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL);
1702         if (ret < 0) {
1703             return ret;
1704         }
1705 
1706 
1707         trace_pci_nvme_block_status(offset, bytes, pnum, ret,
1708                                     !!(ret & BDRV_BLOCK_ZERO));
1709 
1710         if (!(ret & flags)) {
1711             return 1;
1712         }
1713 
1714         offset += pnum;
1715     } while (pnum != bytes);
1716 
1717     return 0;
1718 }
1719 
1720 static uint16_t nvme_check_dulbe(NvmeNamespace *ns, uint64_t slba,
1721                                  uint32_t nlb)
1722 {
1723     int ret;
1724     Error *err = NULL;
1725 
1726     ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_DATA);
1727     if (ret) {
1728         if (ret < 0) {
1729             error_setg_errno(&err, -ret, "unable to get block status");
1730             error_report_err(err);
1731 
1732             return NVME_INTERNAL_DEV_ERROR;
1733         }
1734 
1735         return NVME_DULB;
1736     }
1737 
1738     return NVME_SUCCESS;
1739 }
1740 
1741 static void nvme_aio_err(NvmeRequest *req, int ret)
1742 {
1743     uint16_t status = NVME_SUCCESS;
1744     Error *local_err = NULL;
1745 
1746     switch (req->cmd.opcode) {
1747     case NVME_CMD_READ:
1748         status = NVME_UNRECOVERED_READ;
1749         break;
1750     case NVME_CMD_FLUSH:
1751     case NVME_CMD_WRITE:
1752     case NVME_CMD_WRITE_ZEROES:
1753     case NVME_CMD_ZONE_APPEND:
1754     case NVME_CMD_COPY:
1755         status = NVME_WRITE_FAULT;
1756         break;
1757     default:
1758         status = NVME_INTERNAL_DEV_ERROR;
1759         break;
1760     }
1761 
1762     trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), status);
1763 
1764     error_setg_errno(&local_err, -ret, "aio failed");
1765     error_report_err(local_err);
1766 
1767     /*
1768      * Set the command status code to the first encountered error but allow a
1769      * subsequent Internal Device Error to trump it.
1770      */
1771     if (req->status && status != NVME_INTERNAL_DEV_ERROR) {
1772         return;
1773     }
1774 
1775     req->status = status;
1776 }
1777 
1778 static inline uint32_t nvme_zone_idx(NvmeNamespace *ns, uint64_t slba)
1779 {
1780     return ns->zone_size_log2 > 0 ? slba >> ns->zone_size_log2 :
1781                                     slba / ns->zone_size;
1782 }
1783 
1784 static inline NvmeZone *nvme_get_zone_by_slba(NvmeNamespace *ns, uint64_t slba)
1785 {
1786     uint32_t zone_idx = nvme_zone_idx(ns, slba);
1787 
1788     if (zone_idx >= ns->num_zones) {
1789         return NULL;
1790     }
1791 
1792     return &ns->zone_array[zone_idx];
1793 }
1794 
1795 static uint16_t nvme_check_zone_state_for_write(NvmeZone *zone)
1796 {
1797     uint64_t zslba = zone->d.zslba;
1798 
1799     switch (nvme_get_zone_state(zone)) {
1800     case NVME_ZONE_STATE_EMPTY:
1801     case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1802     case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1803     case NVME_ZONE_STATE_CLOSED:
1804         return NVME_SUCCESS;
1805     case NVME_ZONE_STATE_FULL:
1806         trace_pci_nvme_err_zone_is_full(zslba);
1807         return NVME_ZONE_FULL;
1808     case NVME_ZONE_STATE_OFFLINE:
1809         trace_pci_nvme_err_zone_is_offline(zslba);
1810         return NVME_ZONE_OFFLINE;
1811     case NVME_ZONE_STATE_READ_ONLY:
1812         trace_pci_nvme_err_zone_is_read_only(zslba);
1813         return NVME_ZONE_READ_ONLY;
1814     default:
1815         assert(false);
1816     }
1817 
1818     return NVME_INTERNAL_DEV_ERROR;
1819 }
1820 
1821 static uint16_t nvme_check_zone_write(NvmeNamespace *ns, NvmeZone *zone,
1822                                       uint64_t slba, uint32_t nlb)
1823 {
1824     uint64_t zcap = nvme_zone_wr_boundary(zone);
1825     uint16_t status;
1826 
1827     status = nvme_check_zone_state_for_write(zone);
1828     if (status) {
1829         return status;
1830     }
1831 
1832     if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1833         uint64_t ezrwa = zone->w_ptr + 2 * ns->zns.zrwas;
1834 
1835         if (slba < zone->w_ptr || slba + nlb > ezrwa) {
1836             trace_pci_nvme_err_zone_invalid_write(slba, zone->w_ptr);
1837             return NVME_ZONE_INVALID_WRITE;
1838         }
1839     } else {
1840         if (unlikely(slba != zone->w_ptr)) {
1841             trace_pci_nvme_err_write_not_at_wp(slba, zone->d.zslba,
1842                                                zone->w_ptr);
1843             return NVME_ZONE_INVALID_WRITE;
1844         }
1845     }
1846 
1847     if (unlikely((slba + nlb) > zcap)) {
1848         trace_pci_nvme_err_zone_boundary(slba, nlb, zcap);
1849         return NVME_ZONE_BOUNDARY_ERROR;
1850     }
1851 
1852     return NVME_SUCCESS;
1853 }
1854 
1855 static uint16_t nvme_check_zone_state_for_read(NvmeZone *zone)
1856 {
1857     switch (nvme_get_zone_state(zone)) {
1858     case NVME_ZONE_STATE_EMPTY:
1859     case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1860     case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1861     case NVME_ZONE_STATE_FULL:
1862     case NVME_ZONE_STATE_CLOSED:
1863     case NVME_ZONE_STATE_READ_ONLY:
1864         return NVME_SUCCESS;
1865     case NVME_ZONE_STATE_OFFLINE:
1866         trace_pci_nvme_err_zone_is_offline(zone->d.zslba);
1867         return NVME_ZONE_OFFLINE;
1868     default:
1869         assert(false);
1870     }
1871 
1872     return NVME_INTERNAL_DEV_ERROR;
1873 }
1874 
1875 static uint16_t nvme_check_zone_read(NvmeNamespace *ns, uint64_t slba,
1876                                      uint32_t nlb)
1877 {
1878     NvmeZone *zone;
1879     uint64_t bndry, end;
1880     uint16_t status;
1881 
1882     zone = nvme_get_zone_by_slba(ns, slba);
1883     assert(zone);
1884 
1885     bndry = nvme_zone_rd_boundary(ns, zone);
1886     end = slba + nlb;
1887 
1888     status = nvme_check_zone_state_for_read(zone);
1889     if (status) {
1890         ;
1891     } else if (unlikely(end > bndry)) {
1892         if (!ns->params.cross_zone_read) {
1893             status = NVME_ZONE_BOUNDARY_ERROR;
1894         } else {
1895             /*
1896              * Read across zone boundary - check that all subsequent
1897              * zones that are being read have an appropriate state.
1898              */
1899             do {
1900                 zone++;
1901                 status = nvme_check_zone_state_for_read(zone);
1902                 if (status) {
1903                     break;
1904                 }
1905             } while (end > nvme_zone_rd_boundary(ns, zone));
1906         }
1907     }
1908 
1909     return status;
1910 }
1911 
1912 static uint16_t nvme_zrm_finish(NvmeNamespace *ns, NvmeZone *zone)
1913 {
1914     switch (nvme_get_zone_state(zone)) {
1915     case NVME_ZONE_STATE_FULL:
1916         return NVME_SUCCESS;
1917 
1918     case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1919     case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1920         nvme_aor_dec_open(ns);
1921         /* fallthrough */
1922     case NVME_ZONE_STATE_CLOSED:
1923         nvme_aor_dec_active(ns);
1924 
1925         if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1926             zone->d.za &= ~NVME_ZA_ZRWA_VALID;
1927             if (ns->params.numzrwa) {
1928                 ns->zns.numzrwa++;
1929             }
1930         }
1931 
1932         /* fallthrough */
1933     case NVME_ZONE_STATE_EMPTY:
1934         nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_FULL);
1935         return NVME_SUCCESS;
1936 
1937     default:
1938         return NVME_ZONE_INVAL_TRANSITION;
1939     }
1940 }
1941 
1942 static uint16_t nvme_zrm_close(NvmeNamespace *ns, NvmeZone *zone)
1943 {
1944     switch (nvme_get_zone_state(zone)) {
1945     case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1946     case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1947         nvme_aor_dec_open(ns);
1948         nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
1949         /* fall through */
1950     case NVME_ZONE_STATE_CLOSED:
1951         return NVME_SUCCESS;
1952 
1953     default:
1954         return NVME_ZONE_INVAL_TRANSITION;
1955     }
1956 }
1957 
1958 static uint16_t nvme_zrm_reset(NvmeNamespace *ns, NvmeZone *zone)
1959 {
1960     switch (nvme_get_zone_state(zone)) {
1961     case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1962     case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1963         nvme_aor_dec_open(ns);
1964         /* fallthrough */
1965     case NVME_ZONE_STATE_CLOSED:
1966         nvme_aor_dec_active(ns);
1967 
1968         if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1969             if (ns->params.numzrwa) {
1970                 ns->zns.numzrwa++;
1971             }
1972         }
1973 
1974         /* fallthrough */
1975     case NVME_ZONE_STATE_FULL:
1976         zone->w_ptr = zone->d.zslba;
1977         zone->d.wp = zone->w_ptr;
1978         nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EMPTY);
1979         /* fallthrough */
1980     case NVME_ZONE_STATE_EMPTY:
1981         return NVME_SUCCESS;
1982 
1983     default:
1984         return NVME_ZONE_INVAL_TRANSITION;
1985     }
1986 }
1987 
1988 static void nvme_zrm_auto_transition_zone(NvmeNamespace *ns)
1989 {
1990     NvmeZone *zone;
1991 
1992     if (ns->params.max_open_zones &&
1993         ns->nr_open_zones == ns->params.max_open_zones) {
1994         zone = QTAILQ_FIRST(&ns->imp_open_zones);
1995         if (zone) {
1996             /*
1997              * Automatically close this implicitly open zone.
1998              */
1999             QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
2000             nvme_zrm_close(ns, zone);
2001         }
2002     }
2003 }
2004 
2005 enum {
2006     NVME_ZRM_AUTO = 1 << 0,
2007     NVME_ZRM_ZRWA = 1 << 1,
2008 };
2009 
2010 static uint16_t nvme_zrm_open_flags(NvmeCtrl *n, NvmeNamespace *ns,
2011                                     NvmeZone *zone, int flags)
2012 {
2013     int act = 0;
2014     uint16_t status;
2015 
2016     switch (nvme_get_zone_state(zone)) {
2017     case NVME_ZONE_STATE_EMPTY:
2018         act = 1;
2019 
2020         /* fallthrough */
2021 
2022     case NVME_ZONE_STATE_CLOSED:
2023         if (n->params.auto_transition_zones) {
2024             nvme_zrm_auto_transition_zone(ns);
2025         }
2026         status = nvme_zns_check_resources(ns, act, 1,
2027                                           (flags & NVME_ZRM_ZRWA) ? 1 : 0);
2028         if (status) {
2029             return status;
2030         }
2031 
2032         if (act) {
2033             nvme_aor_inc_active(ns);
2034         }
2035 
2036         nvme_aor_inc_open(ns);
2037 
2038         if (flags & NVME_ZRM_AUTO) {
2039             nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_IMPLICITLY_OPEN);
2040             return NVME_SUCCESS;
2041         }
2042 
2043         /* fallthrough */
2044 
2045     case NVME_ZONE_STATE_IMPLICITLY_OPEN:
2046         if (flags & NVME_ZRM_AUTO) {
2047             return NVME_SUCCESS;
2048         }
2049 
2050         nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EXPLICITLY_OPEN);
2051 
2052         /* fallthrough */
2053 
2054     case NVME_ZONE_STATE_EXPLICITLY_OPEN:
2055         if (flags & NVME_ZRM_ZRWA) {
2056             ns->zns.numzrwa--;
2057 
2058             zone->d.za |= NVME_ZA_ZRWA_VALID;
2059         }
2060 
2061         return NVME_SUCCESS;
2062 
2063     default:
2064         return NVME_ZONE_INVAL_TRANSITION;
2065     }
2066 }
2067 
2068 static inline uint16_t nvme_zrm_auto(NvmeCtrl *n, NvmeNamespace *ns,
2069                                      NvmeZone *zone)
2070 {
2071     return nvme_zrm_open_flags(n, ns, zone, NVME_ZRM_AUTO);
2072 }
2073 
2074 static void nvme_advance_zone_wp(NvmeNamespace *ns, NvmeZone *zone,
2075                                  uint32_t nlb)
2076 {
2077     zone->d.wp += nlb;
2078 
2079     if (zone->d.wp == nvme_zone_wr_boundary(zone)) {
2080         nvme_zrm_finish(ns, zone);
2081     }
2082 }
2083 
2084 static void nvme_zoned_zrwa_implicit_flush(NvmeNamespace *ns, NvmeZone *zone,
2085                                            uint32_t nlbc)
2086 {
2087     uint16_t nzrwafgs = DIV_ROUND_UP(nlbc, ns->zns.zrwafg);
2088 
2089     nlbc = nzrwafgs * ns->zns.zrwafg;
2090 
2091     trace_pci_nvme_zoned_zrwa_implicit_flush(zone->d.zslba, nlbc);
2092 
2093     zone->w_ptr += nlbc;
2094 
2095     nvme_advance_zone_wp(ns, zone, nlbc);
2096 }
2097 
2098 static void nvme_finalize_zoned_write(NvmeNamespace *ns, NvmeRequest *req)
2099 {
2100     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2101     NvmeZone *zone;
2102     uint64_t slba;
2103     uint32_t nlb;
2104 
2105     slba = le64_to_cpu(rw->slba);
2106     nlb = le16_to_cpu(rw->nlb) + 1;
2107     zone = nvme_get_zone_by_slba(ns, slba);
2108     assert(zone);
2109 
2110     if (zone->d.za & NVME_ZA_ZRWA_VALID) {
2111         uint64_t ezrwa = zone->w_ptr + ns->zns.zrwas - 1;
2112         uint64_t elba = slba + nlb - 1;
2113 
2114         if (elba > ezrwa) {
2115             nvme_zoned_zrwa_implicit_flush(ns, zone, elba - ezrwa);
2116         }
2117 
2118         return;
2119     }
2120 
2121     nvme_advance_zone_wp(ns, zone, nlb);
2122 }
2123 
2124 static inline bool nvme_is_write(NvmeRequest *req)
2125 {
2126     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2127 
2128     return rw->opcode == NVME_CMD_WRITE ||
2129            rw->opcode == NVME_CMD_ZONE_APPEND ||
2130            rw->opcode == NVME_CMD_WRITE_ZEROES;
2131 }
2132 
2133 static void nvme_misc_cb(void *opaque, int ret)
2134 {
2135     NvmeRequest *req = opaque;
2136 
2137     trace_pci_nvme_misc_cb(nvme_cid(req));
2138 
2139     if (ret) {
2140         nvme_aio_err(req, ret);
2141     }
2142 
2143     nvme_enqueue_req_completion(nvme_cq(req), req);
2144 }
2145 
2146 void nvme_rw_complete_cb(void *opaque, int ret)
2147 {
2148     NvmeRequest *req = opaque;
2149     NvmeNamespace *ns = req->ns;
2150     BlockBackend *blk = ns->blkconf.blk;
2151     BlockAcctCookie *acct = &req->acct;
2152     BlockAcctStats *stats = blk_get_stats(blk);
2153 
2154     trace_pci_nvme_rw_complete_cb(nvme_cid(req), blk_name(blk));
2155 
2156     if (ret) {
2157         block_acct_failed(stats, acct);
2158         nvme_aio_err(req, ret);
2159     } else {
2160         block_acct_done(stats, acct);
2161     }
2162 
2163     if (ns->params.zoned && nvme_is_write(req)) {
2164         nvme_finalize_zoned_write(ns, req);
2165     }
2166 
2167     nvme_enqueue_req_completion(nvme_cq(req), req);
2168 }
2169 
2170 static void nvme_rw_cb(void *opaque, int ret)
2171 {
2172     NvmeRequest *req = opaque;
2173     NvmeNamespace *ns = req->ns;
2174 
2175     BlockBackend *blk = ns->blkconf.blk;
2176 
2177     trace_pci_nvme_rw_cb(nvme_cid(req), blk_name(blk));
2178 
2179     if (ret) {
2180         goto out;
2181     }
2182 
2183     if (ns->lbaf.ms) {
2184         NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2185         uint64_t slba = le64_to_cpu(rw->slba);
2186         uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
2187         uint64_t offset = nvme_moff(ns, slba);
2188 
2189         if (req->cmd.opcode == NVME_CMD_WRITE_ZEROES) {
2190             size_t mlen = nvme_m2b(ns, nlb);
2191 
2192             req->aiocb = blk_aio_pwrite_zeroes(blk, offset, mlen,
2193                                                BDRV_REQ_MAY_UNMAP,
2194                                                nvme_rw_complete_cb, req);
2195             return;
2196         }
2197 
2198         if (nvme_ns_ext(ns) || req->cmd.mptr) {
2199             uint16_t status;
2200 
2201             nvme_sg_unmap(&req->sg);
2202             status = nvme_map_mdata(nvme_ctrl(req), nlb, req);
2203             if (status) {
2204                 ret = -EFAULT;
2205                 goto out;
2206             }
2207 
2208             if (req->cmd.opcode == NVME_CMD_READ) {
2209                 return nvme_blk_read(blk, offset, 1, nvme_rw_complete_cb, req);
2210             }
2211 
2212             return nvme_blk_write(blk, offset, 1, nvme_rw_complete_cb, req);
2213         }
2214     }
2215 
2216 out:
2217     nvme_rw_complete_cb(req, ret);
2218 }
2219 
2220 static void nvme_verify_cb(void *opaque, int ret)
2221 {
2222     NvmeBounceContext *ctx = opaque;
2223     NvmeRequest *req = ctx->req;
2224     NvmeNamespace *ns = req->ns;
2225     BlockBackend *blk = ns->blkconf.blk;
2226     BlockAcctCookie *acct = &req->acct;
2227     BlockAcctStats *stats = blk_get_stats(blk);
2228     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2229     uint64_t slba = le64_to_cpu(rw->slba);
2230     uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2231     uint16_t apptag = le16_to_cpu(rw->apptag);
2232     uint16_t appmask = le16_to_cpu(rw->appmask);
2233     uint64_t reftag = le32_to_cpu(rw->reftag);
2234     uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2235     uint16_t status;
2236 
2237     reftag |= cdw3 << 32;
2238 
2239     trace_pci_nvme_verify_cb(nvme_cid(req), prinfo, apptag, appmask, reftag);
2240 
2241     if (ret) {
2242         block_acct_failed(stats, acct);
2243         nvme_aio_err(req, ret);
2244         goto out;
2245     }
2246 
2247     block_acct_done(stats, acct);
2248 
2249     if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2250         status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce,
2251                                        ctx->mdata.iov.size, slba);
2252         if (status) {
2253             req->status = status;
2254             goto out;
2255         }
2256 
2257         req->status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2258                                      ctx->mdata.bounce, ctx->mdata.iov.size,
2259                                      prinfo, slba, apptag, appmask, &reftag);
2260     }
2261 
2262 out:
2263     qemu_iovec_destroy(&ctx->data.iov);
2264     g_free(ctx->data.bounce);
2265 
2266     qemu_iovec_destroy(&ctx->mdata.iov);
2267     g_free(ctx->mdata.bounce);
2268 
2269     g_free(ctx);
2270 
2271     nvme_enqueue_req_completion(nvme_cq(req), req);
2272 }
2273 
2274 
2275 static void nvme_verify_mdata_in_cb(void *opaque, int ret)
2276 {
2277     NvmeBounceContext *ctx = opaque;
2278     NvmeRequest *req = ctx->req;
2279     NvmeNamespace *ns = req->ns;
2280     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2281     uint64_t slba = le64_to_cpu(rw->slba);
2282     uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2283     size_t mlen = nvme_m2b(ns, nlb);
2284     uint64_t offset = nvme_moff(ns, slba);
2285     BlockBackend *blk = ns->blkconf.blk;
2286 
2287     trace_pci_nvme_verify_mdata_in_cb(nvme_cid(req), blk_name(blk));
2288 
2289     if (ret) {
2290         goto out;
2291     }
2292 
2293     ctx->mdata.bounce = g_malloc(mlen);
2294 
2295     qemu_iovec_reset(&ctx->mdata.iov);
2296     qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2297 
2298     req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2299                                 nvme_verify_cb, ctx);
2300     return;
2301 
2302 out:
2303     nvme_verify_cb(ctx, ret);
2304 }
2305 
2306 struct nvme_compare_ctx {
2307     struct {
2308         QEMUIOVector iov;
2309         uint8_t *bounce;
2310     } data;
2311 
2312     struct {
2313         QEMUIOVector iov;
2314         uint8_t *bounce;
2315     } mdata;
2316 };
2317 
2318 static void nvme_compare_mdata_cb(void *opaque, int ret)
2319 {
2320     NvmeRequest *req = opaque;
2321     NvmeNamespace *ns = req->ns;
2322     NvmeCtrl *n = nvme_ctrl(req);
2323     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2324     uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2325     uint16_t apptag = le16_to_cpu(rw->apptag);
2326     uint16_t appmask = le16_to_cpu(rw->appmask);
2327     uint64_t reftag = le32_to_cpu(rw->reftag);
2328     uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2329     struct nvme_compare_ctx *ctx = req->opaque;
2330     g_autofree uint8_t *buf = NULL;
2331     BlockBackend *blk = ns->blkconf.blk;
2332     BlockAcctCookie *acct = &req->acct;
2333     BlockAcctStats *stats = blk_get_stats(blk);
2334     uint16_t status = NVME_SUCCESS;
2335 
2336     reftag |= cdw3 << 32;
2337 
2338     trace_pci_nvme_compare_mdata_cb(nvme_cid(req));
2339 
2340     if (ret) {
2341         block_acct_failed(stats, acct);
2342         nvme_aio_err(req, ret);
2343         goto out;
2344     }
2345 
2346     buf = g_malloc(ctx->mdata.iov.size);
2347 
2348     status = nvme_bounce_mdata(n, buf, ctx->mdata.iov.size,
2349                                NVME_TX_DIRECTION_TO_DEVICE, req);
2350     if (status) {
2351         req->status = status;
2352         goto out;
2353     }
2354 
2355     if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2356         uint64_t slba = le64_to_cpu(rw->slba);
2357         uint8_t *bufp;
2358         uint8_t *mbufp = ctx->mdata.bounce;
2359         uint8_t *end = mbufp + ctx->mdata.iov.size;
2360         int16_t pil = 0;
2361 
2362         status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2363                                 ctx->mdata.bounce, ctx->mdata.iov.size, prinfo,
2364                                 slba, apptag, appmask, &reftag);
2365         if (status) {
2366             req->status = status;
2367             goto out;
2368         }
2369 
2370         /*
2371          * When formatted with protection information, do not compare the DIF
2372          * tuple.
2373          */
2374         if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
2375             pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
2376         }
2377 
2378         for (bufp = buf; mbufp < end; bufp += ns->lbaf.ms, mbufp += ns->lbaf.ms) {
2379             if (memcmp(bufp + pil, mbufp + pil, ns->lbaf.ms - pil)) {
2380                 req->status = NVME_CMP_FAILURE | NVME_DNR;
2381                 goto out;
2382             }
2383         }
2384 
2385         goto out;
2386     }
2387 
2388     if (memcmp(buf, ctx->mdata.bounce, ctx->mdata.iov.size)) {
2389         req->status = NVME_CMP_FAILURE | NVME_DNR;
2390         goto out;
2391     }
2392 
2393     block_acct_done(stats, acct);
2394 
2395 out:
2396     qemu_iovec_destroy(&ctx->data.iov);
2397     g_free(ctx->data.bounce);
2398 
2399     qemu_iovec_destroy(&ctx->mdata.iov);
2400     g_free(ctx->mdata.bounce);
2401 
2402     g_free(ctx);
2403 
2404     nvme_enqueue_req_completion(nvme_cq(req), req);
2405 }
2406 
2407 static void nvme_compare_data_cb(void *opaque, int ret)
2408 {
2409     NvmeRequest *req = opaque;
2410     NvmeCtrl *n = nvme_ctrl(req);
2411     NvmeNamespace *ns = req->ns;
2412     BlockBackend *blk = ns->blkconf.blk;
2413     BlockAcctCookie *acct = &req->acct;
2414     BlockAcctStats *stats = blk_get_stats(blk);
2415 
2416     struct nvme_compare_ctx *ctx = req->opaque;
2417     g_autofree uint8_t *buf = NULL;
2418     uint16_t status;
2419 
2420     trace_pci_nvme_compare_data_cb(nvme_cid(req));
2421 
2422     if (ret) {
2423         block_acct_failed(stats, acct);
2424         nvme_aio_err(req, ret);
2425         goto out;
2426     }
2427 
2428     buf = g_malloc(ctx->data.iov.size);
2429 
2430     status = nvme_bounce_data(n, buf, ctx->data.iov.size,
2431                               NVME_TX_DIRECTION_TO_DEVICE, req);
2432     if (status) {
2433         req->status = status;
2434         goto out;
2435     }
2436 
2437     if (memcmp(buf, ctx->data.bounce, ctx->data.iov.size)) {
2438         req->status = NVME_CMP_FAILURE | NVME_DNR;
2439         goto out;
2440     }
2441 
2442     if (ns->lbaf.ms) {
2443         NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2444         uint64_t slba = le64_to_cpu(rw->slba);
2445         uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2446         size_t mlen = nvme_m2b(ns, nlb);
2447         uint64_t offset = nvme_moff(ns, slba);
2448 
2449         ctx->mdata.bounce = g_malloc(mlen);
2450 
2451         qemu_iovec_init(&ctx->mdata.iov, 1);
2452         qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2453 
2454         req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2455                                     nvme_compare_mdata_cb, req);
2456         return;
2457     }
2458 
2459     block_acct_done(stats, acct);
2460 
2461 out:
2462     qemu_iovec_destroy(&ctx->data.iov);
2463     g_free(ctx->data.bounce);
2464     g_free(ctx);
2465 
2466     nvme_enqueue_req_completion(nvme_cq(req), req);
2467 }
2468 
2469 typedef struct NvmeDSMAIOCB {
2470     BlockAIOCB common;
2471     BlockAIOCB *aiocb;
2472     NvmeRequest *req;
2473     int ret;
2474 
2475     NvmeDsmRange *range;
2476     unsigned int nr;
2477     unsigned int idx;
2478 } NvmeDSMAIOCB;
2479 
2480 static void nvme_dsm_cancel(BlockAIOCB *aiocb)
2481 {
2482     NvmeDSMAIOCB *iocb = container_of(aiocb, NvmeDSMAIOCB, common);
2483 
2484     /* break nvme_dsm_cb loop */
2485     iocb->idx = iocb->nr;
2486     iocb->ret = -ECANCELED;
2487 
2488     if (iocb->aiocb) {
2489         blk_aio_cancel_async(iocb->aiocb);
2490         iocb->aiocb = NULL;
2491     } else {
2492         /*
2493          * We only reach this if nvme_dsm_cancel() has already been called or
2494          * the command ran to completion.
2495          */
2496         assert(iocb->idx == iocb->nr);
2497     }
2498 }
2499 
2500 static const AIOCBInfo nvme_dsm_aiocb_info = {
2501     .aiocb_size   = sizeof(NvmeDSMAIOCB),
2502     .cancel_async = nvme_dsm_cancel,
2503 };
2504 
2505 static void nvme_dsm_cb(void *opaque, int ret);
2506 
2507 static void nvme_dsm_md_cb(void *opaque, int ret)
2508 {
2509     NvmeDSMAIOCB *iocb = opaque;
2510     NvmeRequest *req = iocb->req;
2511     NvmeNamespace *ns = req->ns;
2512     NvmeDsmRange *range;
2513     uint64_t slba;
2514     uint32_t nlb;
2515 
2516     if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2517         goto done;
2518     }
2519 
2520     range = &iocb->range[iocb->idx - 1];
2521     slba = le64_to_cpu(range->slba);
2522     nlb = le32_to_cpu(range->nlb);
2523 
2524     /*
2525      * Check that all block were discarded (zeroed); otherwise we do not zero
2526      * the metadata.
2527      */
2528 
2529     ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_ZERO);
2530     if (ret) {
2531         if (ret < 0) {
2532             goto done;
2533         }
2534 
2535         nvme_dsm_cb(iocb, 0);
2536         return;
2537     }
2538 
2539     iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, nvme_moff(ns, slba),
2540                                         nvme_m2b(ns, nlb), BDRV_REQ_MAY_UNMAP,
2541                                         nvme_dsm_cb, iocb);
2542     return;
2543 
2544 done:
2545     nvme_dsm_cb(iocb, ret);
2546 }
2547 
2548 static void nvme_dsm_cb(void *opaque, int ret)
2549 {
2550     NvmeDSMAIOCB *iocb = opaque;
2551     NvmeRequest *req = iocb->req;
2552     NvmeCtrl *n = nvme_ctrl(req);
2553     NvmeNamespace *ns = req->ns;
2554     NvmeDsmRange *range;
2555     uint64_t slba;
2556     uint32_t nlb;
2557 
2558     if (iocb->ret < 0) {
2559         goto done;
2560     } else if (ret < 0) {
2561         iocb->ret = ret;
2562         goto done;
2563     }
2564 
2565 next:
2566     if (iocb->idx == iocb->nr) {
2567         goto done;
2568     }
2569 
2570     range = &iocb->range[iocb->idx++];
2571     slba = le64_to_cpu(range->slba);
2572     nlb = le32_to_cpu(range->nlb);
2573 
2574     trace_pci_nvme_dsm_deallocate(slba, nlb);
2575 
2576     if (nlb > n->dmrsl) {
2577         trace_pci_nvme_dsm_single_range_limit_exceeded(nlb, n->dmrsl);
2578         goto next;
2579     }
2580 
2581     if (nvme_check_bounds(ns, slba, nlb)) {
2582         trace_pci_nvme_err_invalid_lba_range(slba, nlb,
2583                                              ns->id_ns.nsze);
2584         goto next;
2585     }
2586 
2587     iocb->aiocb = blk_aio_pdiscard(ns->blkconf.blk, nvme_l2b(ns, slba),
2588                                    nvme_l2b(ns, nlb),
2589                                    nvme_dsm_md_cb, iocb);
2590     return;
2591 
2592 done:
2593     iocb->aiocb = NULL;
2594     iocb->common.cb(iocb->common.opaque, iocb->ret);
2595     qemu_aio_unref(iocb);
2596 }
2597 
2598 static uint16_t nvme_dsm(NvmeCtrl *n, NvmeRequest *req)
2599 {
2600     NvmeNamespace *ns = req->ns;
2601     NvmeDsmCmd *dsm = (NvmeDsmCmd *) &req->cmd;
2602     uint32_t attr = le32_to_cpu(dsm->attributes);
2603     uint32_t nr = (le32_to_cpu(dsm->nr) & 0xff) + 1;
2604     uint16_t status = NVME_SUCCESS;
2605 
2606     trace_pci_nvme_dsm(nr, attr);
2607 
2608     if (attr & NVME_DSMGMT_AD) {
2609         NvmeDSMAIOCB *iocb = blk_aio_get(&nvme_dsm_aiocb_info, ns->blkconf.blk,
2610                                          nvme_misc_cb, req);
2611 
2612         iocb->req = req;
2613         iocb->ret = 0;
2614         iocb->range = g_new(NvmeDsmRange, nr);
2615         iocb->nr = nr;
2616         iocb->idx = 0;
2617 
2618         status = nvme_h2c(n, (uint8_t *)iocb->range, sizeof(NvmeDsmRange) * nr,
2619                           req);
2620         if (status) {
2621             g_free(iocb->range);
2622             qemu_aio_unref(iocb);
2623 
2624             return status;
2625         }
2626 
2627         req->aiocb = &iocb->common;
2628         nvme_dsm_cb(iocb, 0);
2629 
2630         return NVME_NO_COMPLETE;
2631     }
2632 
2633     return status;
2634 }
2635 
2636 static uint16_t nvme_verify(NvmeCtrl *n, NvmeRequest *req)
2637 {
2638     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2639     NvmeNamespace *ns = req->ns;
2640     BlockBackend *blk = ns->blkconf.blk;
2641     uint64_t slba = le64_to_cpu(rw->slba);
2642     uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2643     size_t len = nvme_l2b(ns, nlb);
2644     int64_t offset = nvme_l2b(ns, slba);
2645     uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2646     uint32_t reftag = le32_to_cpu(rw->reftag);
2647     NvmeBounceContext *ctx = NULL;
2648     uint16_t status;
2649 
2650     trace_pci_nvme_verify(nvme_cid(req), nvme_nsid(ns), slba, nlb);
2651 
2652     if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2653         status = nvme_check_prinfo(ns, prinfo, slba, reftag);
2654         if (status) {
2655             return status;
2656         }
2657 
2658         if (prinfo & NVME_PRINFO_PRACT) {
2659             return NVME_INVALID_PROT_INFO | NVME_DNR;
2660         }
2661     }
2662 
2663     if (len > n->page_size << n->params.vsl) {
2664         return NVME_INVALID_FIELD | NVME_DNR;
2665     }
2666 
2667     status = nvme_check_bounds(ns, slba, nlb);
2668     if (status) {
2669         return status;
2670     }
2671 
2672     if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
2673         status = nvme_check_dulbe(ns, slba, nlb);
2674         if (status) {
2675             return status;
2676         }
2677     }
2678 
2679     ctx = g_new0(NvmeBounceContext, 1);
2680     ctx->req = req;
2681 
2682     ctx->data.bounce = g_malloc(len);
2683 
2684     qemu_iovec_init(&ctx->data.iov, 1);
2685     qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len);
2686 
2687     block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size,
2688                      BLOCK_ACCT_READ);
2689 
2690     req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0,
2691                                 nvme_verify_mdata_in_cb, ctx);
2692     return NVME_NO_COMPLETE;
2693 }
2694 
2695 typedef struct NvmeCopyAIOCB {
2696     BlockAIOCB common;
2697     BlockAIOCB *aiocb;
2698     NvmeRequest *req;
2699     int ret;
2700 
2701     void *ranges;
2702     unsigned int format;
2703     int nr;
2704     int idx;
2705 
2706     uint8_t *bounce;
2707     QEMUIOVector iov;
2708     struct {
2709         BlockAcctCookie read;
2710         BlockAcctCookie write;
2711     } acct;
2712 
2713     uint64_t reftag;
2714     uint64_t slba;
2715 
2716     NvmeZone *zone;
2717 } NvmeCopyAIOCB;
2718 
2719 static void nvme_copy_cancel(BlockAIOCB *aiocb)
2720 {
2721     NvmeCopyAIOCB *iocb = container_of(aiocb, NvmeCopyAIOCB, common);
2722 
2723     iocb->ret = -ECANCELED;
2724 
2725     if (iocb->aiocb) {
2726         blk_aio_cancel_async(iocb->aiocb);
2727         iocb->aiocb = NULL;
2728     }
2729 }
2730 
2731 static const AIOCBInfo nvme_copy_aiocb_info = {
2732     .aiocb_size   = sizeof(NvmeCopyAIOCB),
2733     .cancel_async = nvme_copy_cancel,
2734 };
2735 
2736 static void nvme_copy_done(NvmeCopyAIOCB *iocb)
2737 {
2738     NvmeRequest *req = iocb->req;
2739     NvmeNamespace *ns = req->ns;
2740     BlockAcctStats *stats = blk_get_stats(ns->blkconf.blk);
2741 
2742     if (iocb->idx != iocb->nr) {
2743         req->cqe.result = cpu_to_le32(iocb->idx);
2744     }
2745 
2746     qemu_iovec_destroy(&iocb->iov);
2747     g_free(iocb->bounce);
2748 
2749     if (iocb->ret < 0) {
2750         block_acct_failed(stats, &iocb->acct.read);
2751         block_acct_failed(stats, &iocb->acct.write);
2752     } else {
2753         block_acct_done(stats, &iocb->acct.read);
2754         block_acct_done(stats, &iocb->acct.write);
2755     }
2756 
2757     iocb->common.cb(iocb->common.opaque, iocb->ret);
2758     qemu_aio_unref(iocb);
2759 }
2760 
2761 static void nvme_do_copy(NvmeCopyAIOCB *iocb);
2762 
2763 static void nvme_copy_source_range_parse_format0(void *ranges, int idx,
2764                                                  uint64_t *slba, uint32_t *nlb,
2765                                                  uint16_t *apptag,
2766                                                  uint16_t *appmask,
2767                                                  uint64_t *reftag)
2768 {
2769     NvmeCopySourceRangeFormat0 *_ranges = ranges;
2770 
2771     if (slba) {
2772         *slba = le64_to_cpu(_ranges[idx].slba);
2773     }
2774 
2775     if (nlb) {
2776         *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2777     }
2778 
2779     if (apptag) {
2780         *apptag = le16_to_cpu(_ranges[idx].apptag);
2781     }
2782 
2783     if (appmask) {
2784         *appmask = le16_to_cpu(_ranges[idx].appmask);
2785     }
2786 
2787     if (reftag) {
2788         *reftag = le32_to_cpu(_ranges[idx].reftag);
2789     }
2790 }
2791 
2792 static void nvme_copy_source_range_parse_format1(void *ranges, int idx,
2793                                                  uint64_t *slba, uint32_t *nlb,
2794                                                  uint16_t *apptag,
2795                                                  uint16_t *appmask,
2796                                                  uint64_t *reftag)
2797 {
2798     NvmeCopySourceRangeFormat1 *_ranges = ranges;
2799 
2800     if (slba) {
2801         *slba = le64_to_cpu(_ranges[idx].slba);
2802     }
2803 
2804     if (nlb) {
2805         *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2806     }
2807 
2808     if (apptag) {
2809         *apptag = le16_to_cpu(_ranges[idx].apptag);
2810     }
2811 
2812     if (appmask) {
2813         *appmask = le16_to_cpu(_ranges[idx].appmask);
2814     }
2815 
2816     if (reftag) {
2817         *reftag = 0;
2818 
2819         *reftag |= (uint64_t)_ranges[idx].sr[4] << 40;
2820         *reftag |= (uint64_t)_ranges[idx].sr[5] << 32;
2821         *reftag |= (uint64_t)_ranges[idx].sr[6] << 24;
2822         *reftag |= (uint64_t)_ranges[idx].sr[7] << 16;
2823         *reftag |= (uint64_t)_ranges[idx].sr[8] << 8;
2824         *reftag |= (uint64_t)_ranges[idx].sr[9];
2825     }
2826 }
2827 
2828 static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format,
2829                                          uint64_t *slba, uint32_t *nlb,
2830                                          uint16_t *apptag, uint16_t *appmask,
2831                                          uint64_t *reftag)
2832 {
2833     switch (format) {
2834     case NVME_COPY_FORMAT_0:
2835         nvme_copy_source_range_parse_format0(ranges, idx, slba, nlb, apptag,
2836                                              appmask, reftag);
2837         break;
2838 
2839     case NVME_COPY_FORMAT_1:
2840         nvme_copy_source_range_parse_format1(ranges, idx, slba, nlb, apptag,
2841                                              appmask, reftag);
2842         break;
2843 
2844     default:
2845         abort();
2846     }
2847 }
2848 
2849 static inline uint16_t nvme_check_copy_mcl(NvmeNamespace *ns,
2850                                            NvmeCopyAIOCB *iocb, uint16_t nr)
2851 {
2852     uint32_t copy_len = 0;
2853 
2854     for (int idx = 0; idx < nr; idx++) {
2855         uint32_t nlb;
2856         nvme_copy_source_range_parse(iocb->ranges, idx, iocb->format, NULL,
2857                                      &nlb, NULL, NULL, NULL);
2858         copy_len += nlb + 1;
2859     }
2860 
2861     if (copy_len > ns->id_ns.mcl) {
2862         return NVME_CMD_SIZE_LIMIT | NVME_DNR;
2863     }
2864 
2865     return NVME_SUCCESS;
2866 }
2867 
2868 static void nvme_copy_out_completed_cb(void *opaque, int ret)
2869 {
2870     NvmeCopyAIOCB *iocb = opaque;
2871     NvmeRequest *req = iocb->req;
2872     NvmeNamespace *ns = req->ns;
2873     uint32_t nlb;
2874 
2875     nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2876                                  &nlb, NULL, NULL, NULL);
2877 
2878     if (ret < 0) {
2879         iocb->ret = ret;
2880         goto out;
2881     } else if (iocb->ret < 0) {
2882         goto out;
2883     }
2884 
2885     if (ns->params.zoned) {
2886         nvme_advance_zone_wp(ns, iocb->zone, nlb);
2887     }
2888 
2889     iocb->idx++;
2890     iocb->slba += nlb;
2891 out:
2892     nvme_do_copy(iocb);
2893 }
2894 
2895 static void nvme_copy_out_cb(void *opaque, int ret)
2896 {
2897     NvmeCopyAIOCB *iocb = opaque;
2898     NvmeRequest *req = iocb->req;
2899     NvmeNamespace *ns = req->ns;
2900     uint32_t nlb;
2901     size_t mlen;
2902     uint8_t *mbounce;
2903 
2904     if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2905         goto out;
2906     }
2907 
2908     nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2909                                  &nlb, NULL, NULL, NULL);
2910 
2911     mlen = nvme_m2b(ns, nlb);
2912     mbounce = iocb->bounce + nvme_l2b(ns, nlb);
2913 
2914     qemu_iovec_reset(&iocb->iov);
2915     qemu_iovec_add(&iocb->iov, mbounce, mlen);
2916 
2917     iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_moff(ns, iocb->slba),
2918                                   &iocb->iov, 0, nvme_copy_out_completed_cb,
2919                                   iocb);
2920 
2921     return;
2922 
2923 out:
2924     nvme_copy_out_completed_cb(iocb, ret);
2925 }
2926 
2927 static void nvme_copy_in_completed_cb(void *opaque, int ret)
2928 {
2929     NvmeCopyAIOCB *iocb = opaque;
2930     NvmeRequest *req = iocb->req;
2931     NvmeNamespace *ns = req->ns;
2932     uint32_t nlb;
2933     uint64_t slba;
2934     uint16_t apptag, appmask;
2935     uint64_t reftag;
2936     size_t len;
2937     uint16_t status;
2938 
2939     if (ret < 0) {
2940         iocb->ret = ret;
2941         goto out;
2942     } else if (iocb->ret < 0) {
2943         goto out;
2944     }
2945 
2946     nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
2947                                  &nlb, &apptag, &appmask, &reftag);
2948     len = nvme_l2b(ns, nlb);
2949 
2950     trace_pci_nvme_copy_out(iocb->slba, nlb);
2951 
2952     if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2953         NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
2954 
2955         uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
2956         uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
2957 
2958         size_t mlen = nvme_m2b(ns, nlb);
2959         uint8_t *mbounce = iocb->bounce + nvme_l2b(ns, nlb);
2960 
2961         status = nvme_dif_mangle_mdata(ns, mbounce, mlen, slba);
2962         if (status) {
2963             goto invalid;
2964         }
2965         status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen, prinfor,
2966                                 slba, apptag, appmask, &reftag);
2967         if (status) {
2968             goto invalid;
2969         }
2970 
2971         apptag = le16_to_cpu(copy->apptag);
2972         appmask = le16_to_cpu(copy->appmask);
2973 
2974         if (prinfow & NVME_PRINFO_PRACT) {
2975             status = nvme_check_prinfo(ns, prinfow, iocb->slba, iocb->reftag);
2976             if (status) {
2977                 goto invalid;
2978             }
2979 
2980             nvme_dif_pract_generate_dif(ns, iocb->bounce, len, mbounce, mlen,
2981                                         apptag, &iocb->reftag);
2982         } else {
2983             status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen,
2984                                     prinfow, iocb->slba, apptag, appmask,
2985                                     &iocb->reftag);
2986             if (status) {
2987                 goto invalid;
2988             }
2989         }
2990     }
2991 
2992     status = nvme_check_bounds(ns, iocb->slba, nlb);
2993     if (status) {
2994         goto invalid;
2995     }
2996 
2997     if (ns->params.zoned) {
2998         status = nvme_check_zone_write(ns, iocb->zone, iocb->slba, nlb);
2999         if (status) {
3000             goto invalid;
3001         }
3002 
3003         if (!(iocb->zone->d.za & NVME_ZA_ZRWA_VALID)) {
3004             iocb->zone->w_ptr += nlb;
3005         }
3006     }
3007 
3008     qemu_iovec_reset(&iocb->iov);
3009     qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3010 
3011     iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_l2b(ns, iocb->slba),
3012                                   &iocb->iov, 0, nvme_copy_out_cb, iocb);
3013 
3014     return;
3015 
3016 invalid:
3017     req->status = status;
3018     iocb->ret = -1;
3019 out:
3020     nvme_do_copy(iocb);
3021 }
3022 
3023 static void nvme_copy_in_cb(void *opaque, int ret)
3024 {
3025     NvmeCopyAIOCB *iocb = opaque;
3026     NvmeRequest *req = iocb->req;
3027     NvmeNamespace *ns = req->ns;
3028     uint64_t slba;
3029     uint32_t nlb;
3030 
3031     if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
3032         goto out;
3033     }
3034 
3035     nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3036                                  &nlb, NULL, NULL, NULL);
3037 
3038     qemu_iovec_reset(&iocb->iov);
3039     qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(ns, nlb),
3040                    nvme_m2b(ns, nlb));
3041 
3042     iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_moff(ns, slba),
3043                                  &iocb->iov, 0, nvme_copy_in_completed_cb,
3044                                  iocb);
3045     return;
3046 
3047 out:
3048     nvme_copy_in_completed_cb(iocb, ret);
3049 }
3050 
3051 static void nvme_do_copy(NvmeCopyAIOCB *iocb)
3052 {
3053     NvmeRequest *req = iocb->req;
3054     NvmeNamespace *ns = req->ns;
3055     uint64_t slba;
3056     uint32_t nlb;
3057     size_t len;
3058     uint16_t status;
3059 
3060     if (iocb->ret < 0) {
3061         goto done;
3062     }
3063 
3064     if (iocb->idx == iocb->nr) {
3065         goto done;
3066     }
3067 
3068     nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3069                                  &nlb, NULL, NULL, NULL);
3070     len = nvme_l2b(ns, nlb);
3071 
3072     trace_pci_nvme_copy_source_range(slba, nlb);
3073 
3074     if (nlb > le16_to_cpu(ns->id_ns.mssrl)) {
3075         status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3076         goto invalid;
3077     }
3078 
3079     status = nvme_check_bounds(ns, slba, nlb);
3080     if (status) {
3081         goto invalid;
3082     }
3083 
3084     if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3085         status = nvme_check_dulbe(ns, slba, nlb);
3086         if (status) {
3087             goto invalid;
3088         }
3089     }
3090 
3091     if (ns->params.zoned) {
3092         status = nvme_check_zone_read(ns, slba, nlb);
3093         if (status) {
3094             goto invalid;
3095         }
3096     }
3097 
3098     qemu_iovec_reset(&iocb->iov);
3099     qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3100 
3101     iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_l2b(ns, slba),
3102                                  &iocb->iov, 0, nvme_copy_in_cb, iocb);
3103     return;
3104 
3105 invalid:
3106     req->status = status;
3107     iocb->ret = -1;
3108 done:
3109     nvme_copy_done(iocb);
3110 }
3111 
3112 static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req)
3113 {
3114     NvmeNamespace *ns = req->ns;
3115     NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3116     NvmeCopyAIOCB *iocb = blk_aio_get(&nvme_copy_aiocb_info, ns->blkconf.blk,
3117                                       nvme_misc_cb, req);
3118     uint16_t nr = copy->nr + 1;
3119     uint8_t format = copy->control[0] & 0xf;
3120     uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
3121     uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
3122     size_t len = sizeof(NvmeCopySourceRangeFormat0);
3123 
3124     uint16_t status;
3125 
3126     trace_pci_nvme_copy(nvme_cid(req), nvme_nsid(ns), nr, format);
3127 
3128     iocb->ranges = NULL;
3129     iocb->zone = NULL;
3130 
3131     if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) &&
3132         ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) {
3133         status = NVME_INVALID_FIELD | NVME_DNR;
3134         goto invalid;
3135     }
3136 
3137     if (!(n->id_ctrl.ocfs & (1 << format))) {
3138         trace_pci_nvme_err_copy_invalid_format(format);
3139         status = NVME_INVALID_FIELD | NVME_DNR;
3140         goto invalid;
3141     }
3142 
3143     if (nr > ns->id_ns.msrc + 1) {
3144         status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3145         goto invalid;
3146     }
3147 
3148     if ((ns->pif == 0x0 && format != 0x0) ||
3149         (ns->pif != 0x0 && format != 0x1)) {
3150         status = NVME_INVALID_FORMAT | NVME_DNR;
3151         goto invalid;
3152     }
3153 
3154     if (ns->pif) {
3155         len = sizeof(NvmeCopySourceRangeFormat1);
3156     }
3157 
3158     iocb->format = format;
3159     iocb->ranges = g_malloc_n(nr, len);
3160     status = nvme_h2c(n, (uint8_t *)iocb->ranges, len * nr, req);
3161     if (status) {
3162         goto invalid;
3163     }
3164 
3165     iocb->slba = le64_to_cpu(copy->sdlba);
3166 
3167     if (ns->params.zoned) {
3168         iocb->zone = nvme_get_zone_by_slba(ns, iocb->slba);
3169         if (!iocb->zone) {
3170             status = NVME_LBA_RANGE | NVME_DNR;
3171             goto invalid;
3172         }
3173 
3174         status = nvme_zrm_auto(n, ns, iocb->zone);
3175         if (status) {
3176             goto invalid;
3177         }
3178     }
3179 
3180     status = nvme_check_copy_mcl(ns, iocb, nr);
3181     if (status) {
3182         goto invalid;
3183     }
3184 
3185     iocb->req = req;
3186     iocb->ret = 0;
3187     iocb->nr = nr;
3188     iocb->idx = 0;
3189     iocb->reftag = le32_to_cpu(copy->reftag);
3190     iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32;
3191     iocb->bounce = g_malloc_n(le16_to_cpu(ns->id_ns.mssrl),
3192                               ns->lbasz + ns->lbaf.ms);
3193 
3194     qemu_iovec_init(&iocb->iov, 1);
3195 
3196     block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.read, 0,
3197                      BLOCK_ACCT_READ);
3198     block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.write, 0,
3199                      BLOCK_ACCT_WRITE);
3200 
3201     req->aiocb = &iocb->common;
3202     nvme_do_copy(iocb);
3203 
3204     return NVME_NO_COMPLETE;
3205 
3206 invalid:
3207     g_free(iocb->ranges);
3208     qemu_aio_unref(iocb);
3209     return status;
3210 }
3211 
3212 static uint16_t nvme_compare(NvmeCtrl *n, NvmeRequest *req)
3213 {
3214     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3215     NvmeNamespace *ns = req->ns;
3216     BlockBackend *blk = ns->blkconf.blk;
3217     uint64_t slba = le64_to_cpu(rw->slba);
3218     uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
3219     uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3220     size_t data_len = nvme_l2b(ns, nlb);
3221     size_t len = data_len;
3222     int64_t offset = nvme_l2b(ns, slba);
3223     struct nvme_compare_ctx *ctx = NULL;
3224     uint16_t status;
3225 
3226     trace_pci_nvme_compare(nvme_cid(req), nvme_nsid(ns), slba, nlb);
3227 
3228     if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) && (prinfo & NVME_PRINFO_PRACT)) {
3229         return NVME_INVALID_PROT_INFO | NVME_DNR;
3230     }
3231 
3232     if (nvme_ns_ext(ns)) {
3233         len += nvme_m2b(ns, nlb);
3234     }
3235 
3236     status = nvme_check_mdts(n, len);
3237     if (status) {
3238         return status;
3239     }
3240 
3241     status = nvme_check_bounds(ns, slba, nlb);
3242     if (status) {
3243         return status;
3244     }
3245 
3246     if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3247         status = nvme_check_dulbe(ns, slba, nlb);
3248         if (status) {
3249             return status;
3250         }
3251     }
3252 
3253     status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
3254     if (status) {
3255         return status;
3256     }
3257 
3258     ctx = g_new(struct nvme_compare_ctx, 1);
3259     ctx->data.bounce = g_malloc(data_len);
3260 
3261     req->opaque = ctx;
3262 
3263     qemu_iovec_init(&ctx->data.iov, 1);
3264     qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, data_len);
3265 
3266     block_acct_start(blk_get_stats(blk), &req->acct, data_len,
3267                      BLOCK_ACCT_READ);
3268     req->aiocb = blk_aio_preadv(blk, offset, &ctx->data.iov, 0,
3269                                 nvme_compare_data_cb, req);
3270 
3271     return NVME_NO_COMPLETE;
3272 }
3273 
3274 typedef struct NvmeFlushAIOCB {
3275     BlockAIOCB common;
3276     BlockAIOCB *aiocb;
3277     NvmeRequest *req;
3278     int ret;
3279 
3280     NvmeNamespace *ns;
3281     uint32_t nsid;
3282     bool broadcast;
3283 } NvmeFlushAIOCB;
3284 
3285 static void nvme_flush_cancel(BlockAIOCB *acb)
3286 {
3287     NvmeFlushAIOCB *iocb = container_of(acb, NvmeFlushAIOCB, common);
3288 
3289     iocb->ret = -ECANCELED;
3290 
3291     if (iocb->aiocb) {
3292         blk_aio_cancel_async(iocb->aiocb);
3293         iocb->aiocb = NULL;
3294     }
3295 }
3296 
3297 static const AIOCBInfo nvme_flush_aiocb_info = {
3298     .aiocb_size = sizeof(NvmeFlushAIOCB),
3299     .cancel_async = nvme_flush_cancel,
3300 };
3301 
3302 static void nvme_do_flush(NvmeFlushAIOCB *iocb);
3303 
3304 static void nvme_flush_ns_cb(void *opaque, int ret)
3305 {
3306     NvmeFlushAIOCB *iocb = opaque;
3307     NvmeNamespace *ns = iocb->ns;
3308 
3309     if (ret < 0) {
3310         iocb->ret = ret;
3311         goto out;
3312     } else if (iocb->ret < 0) {
3313         goto out;
3314     }
3315 
3316     if (ns) {
3317         trace_pci_nvme_flush_ns(iocb->nsid);
3318 
3319         iocb->ns = NULL;
3320         iocb->aiocb = blk_aio_flush(ns->blkconf.blk, nvme_flush_ns_cb, iocb);
3321         return;
3322     }
3323 
3324 out:
3325     nvme_do_flush(iocb);
3326 }
3327 
3328 static void nvme_do_flush(NvmeFlushAIOCB *iocb)
3329 {
3330     NvmeRequest *req = iocb->req;
3331     NvmeCtrl *n = nvme_ctrl(req);
3332     int i;
3333 
3334     if (iocb->ret < 0) {
3335         goto done;
3336     }
3337 
3338     if (iocb->broadcast) {
3339         for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
3340             iocb->ns = nvme_ns(n, i);
3341             if (iocb->ns) {
3342                 iocb->nsid = i;
3343                 break;
3344             }
3345         }
3346     }
3347 
3348     if (!iocb->ns) {
3349         goto done;
3350     }
3351 
3352     nvme_flush_ns_cb(iocb, 0);
3353     return;
3354 
3355 done:
3356     iocb->common.cb(iocb->common.opaque, iocb->ret);
3357     qemu_aio_unref(iocb);
3358 }
3359 
3360 static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req)
3361 {
3362     NvmeFlushAIOCB *iocb;
3363     uint32_t nsid = le32_to_cpu(req->cmd.nsid);
3364     uint16_t status;
3365 
3366     iocb = qemu_aio_get(&nvme_flush_aiocb_info, NULL, nvme_misc_cb, req);
3367 
3368     iocb->req = req;
3369     iocb->ret = 0;
3370     iocb->ns = NULL;
3371     iocb->nsid = 0;
3372     iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
3373 
3374     if (!iocb->broadcast) {
3375         if (!nvme_nsid_valid(n, nsid)) {
3376             status = NVME_INVALID_NSID | NVME_DNR;
3377             goto out;
3378         }
3379 
3380         iocb->ns = nvme_ns(n, nsid);
3381         if (!iocb->ns) {
3382             status = NVME_INVALID_FIELD | NVME_DNR;
3383             goto out;
3384         }
3385 
3386         iocb->nsid = nsid;
3387     }
3388 
3389     req->aiocb = &iocb->common;
3390     nvme_do_flush(iocb);
3391 
3392     return NVME_NO_COMPLETE;
3393 
3394 out:
3395     qemu_aio_unref(iocb);
3396 
3397     return status;
3398 }
3399 
3400 static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req)
3401 {
3402     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3403     NvmeNamespace *ns = req->ns;
3404     uint64_t slba = le64_to_cpu(rw->slba);
3405     uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3406     uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3407     uint64_t data_size = nvme_l2b(ns, nlb);
3408     uint64_t mapped_size = data_size;
3409     uint64_t data_offset;
3410     BlockBackend *blk = ns->blkconf.blk;
3411     uint16_t status;
3412 
3413     if (nvme_ns_ext(ns)) {
3414         mapped_size += nvme_m2b(ns, nlb);
3415 
3416         if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3417             bool pract = prinfo & NVME_PRINFO_PRACT;
3418 
3419             if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3420                 mapped_size = data_size;
3421             }
3422         }
3423     }
3424 
3425     trace_pci_nvme_read(nvme_cid(req), nvme_nsid(ns), nlb, mapped_size, slba);
3426 
3427     status = nvme_check_mdts(n, mapped_size);
3428     if (status) {
3429         goto invalid;
3430     }
3431 
3432     status = nvme_check_bounds(ns, slba, nlb);
3433     if (status) {
3434         goto invalid;
3435     }
3436 
3437     if (ns->params.zoned) {
3438         status = nvme_check_zone_read(ns, slba, nlb);
3439         if (status) {
3440             trace_pci_nvme_err_zone_read_not_ok(slba, nlb, status);
3441             goto invalid;
3442         }
3443     }
3444 
3445     if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3446         status = nvme_check_dulbe(ns, slba, nlb);
3447         if (status) {
3448             goto invalid;
3449         }
3450     }
3451 
3452     if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3453         return nvme_dif_rw(n, req);
3454     }
3455 
3456     status = nvme_map_data(n, nlb, req);
3457     if (status) {
3458         goto invalid;
3459     }
3460 
3461     data_offset = nvme_l2b(ns, slba);
3462 
3463     block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3464                      BLOCK_ACCT_READ);
3465     nvme_blk_read(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3466     return NVME_NO_COMPLETE;
3467 
3468 invalid:
3469     block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
3470     return status | NVME_DNR;
3471 }
3472 
3473 static void nvme_do_write_fdp(NvmeCtrl *n, NvmeRequest *req, uint64_t slba,
3474                               uint32_t nlb)
3475 {
3476     NvmeNamespace *ns = req->ns;
3477     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3478     uint64_t data_size = nvme_l2b(ns, nlb);
3479     uint32_t dw12 = le32_to_cpu(req->cmd.cdw12);
3480     uint8_t dtype = (dw12 >> 20) & 0xf;
3481     uint16_t pid = le16_to_cpu(rw->dspec);
3482     uint16_t ph, rg, ruhid;
3483     NvmeReclaimUnit *ru;
3484 
3485     if (dtype != NVME_DIRECTIVE_DATA_PLACEMENT ||
3486         !nvme_parse_pid(ns, pid, &ph, &rg)) {
3487         ph = 0;
3488         rg = 0;
3489     }
3490 
3491     ruhid = ns->fdp.phs[ph];
3492     ru = &ns->endgrp->fdp.ruhs[ruhid].rus[rg];
3493 
3494     nvme_fdp_stat_inc(&ns->endgrp->fdp.hbmw, data_size);
3495     nvme_fdp_stat_inc(&ns->endgrp->fdp.mbmw, data_size);
3496 
3497     while (nlb) {
3498         if (nlb < ru->ruamw) {
3499             ru->ruamw -= nlb;
3500             break;
3501         }
3502 
3503         nlb -= ru->ruamw;
3504         nvme_update_ruh(n, ns, pid);
3505     }
3506 }
3507 
3508 static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append,
3509                               bool wrz)
3510 {
3511     NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3512     NvmeNamespace *ns = req->ns;
3513     uint64_t slba = le64_to_cpu(rw->slba);
3514     uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3515     uint16_t ctrl = le16_to_cpu(rw->control);
3516     uint8_t prinfo = NVME_RW_PRINFO(ctrl);
3517     uint64_t data_size = nvme_l2b(ns, nlb);
3518     uint64_t mapped_size = data_size;
3519     uint64_t data_offset;
3520     NvmeZone *zone;
3521     NvmeZonedResult *res = (NvmeZonedResult *)&req->cqe;
3522     BlockBackend *blk = ns->blkconf.blk;
3523     uint16_t status;
3524 
3525     if (nvme_ns_ext(ns)) {
3526         mapped_size += nvme_m2b(ns, nlb);
3527 
3528         if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3529             bool pract = prinfo & NVME_PRINFO_PRACT;
3530 
3531             if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3532                 mapped_size -= nvme_m2b(ns, nlb);
3533             }
3534         }
3535     }
3536 
3537     trace_pci_nvme_write(nvme_cid(req), nvme_io_opc_str(rw->opcode),
3538                          nvme_nsid(ns), nlb, mapped_size, slba);
3539 
3540     if (!wrz) {
3541         status = nvme_check_mdts(n, mapped_size);
3542         if (status) {
3543             goto invalid;
3544         }
3545     }
3546 
3547     status = nvme_check_bounds(ns, slba, nlb);
3548     if (status) {
3549         goto invalid;
3550     }
3551 
3552     if (ns->params.zoned) {
3553         zone = nvme_get_zone_by_slba(ns, slba);
3554         assert(zone);
3555 
3556         if (append) {
3557             bool piremap = !!(ctrl & NVME_RW_PIREMAP);
3558 
3559             if (unlikely(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3560                 return NVME_INVALID_ZONE_OP | NVME_DNR;
3561             }
3562 
3563             if (unlikely(slba != zone->d.zslba)) {
3564                 trace_pci_nvme_err_append_not_at_start(slba, zone->d.zslba);
3565                 status = NVME_INVALID_FIELD;
3566                 goto invalid;
3567             }
3568 
3569             if (n->params.zasl &&
3570                 data_size > (uint64_t)n->page_size << n->params.zasl) {
3571                 trace_pci_nvme_err_zasl(data_size);
3572                 return NVME_INVALID_FIELD | NVME_DNR;
3573             }
3574 
3575             slba = zone->w_ptr;
3576             rw->slba = cpu_to_le64(slba);
3577             res->slba = cpu_to_le64(slba);
3578 
3579             switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3580             case NVME_ID_NS_DPS_TYPE_1:
3581                 if (!piremap) {
3582                     return NVME_INVALID_PROT_INFO | NVME_DNR;
3583                 }
3584 
3585                 /* fallthrough */
3586 
3587             case NVME_ID_NS_DPS_TYPE_2:
3588                 if (piremap) {
3589                     uint32_t reftag = le32_to_cpu(rw->reftag);
3590                     rw->reftag = cpu_to_le32(reftag + (slba - zone->d.zslba));
3591                 }
3592 
3593                 break;
3594 
3595             case NVME_ID_NS_DPS_TYPE_3:
3596                 if (piremap) {
3597                     return NVME_INVALID_PROT_INFO | NVME_DNR;
3598                 }
3599 
3600                 break;
3601             }
3602         }
3603 
3604         status = nvme_check_zone_write(ns, zone, slba, nlb);
3605         if (status) {
3606             goto invalid;
3607         }
3608 
3609         status = nvme_zrm_auto(n, ns, zone);
3610         if (status) {
3611             goto invalid;
3612         }
3613 
3614         if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3615             zone->w_ptr += nlb;
3616         }
3617     } else if (ns->endgrp && ns->endgrp->fdp.enabled) {
3618         nvme_do_write_fdp(n, req, slba, nlb);
3619     }
3620 
3621     data_offset = nvme_l2b(ns, slba);
3622 
3623     if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3624         return nvme_dif_rw(n, req);
3625     }
3626 
3627     if (!wrz) {
3628         status = nvme_map_data(n, nlb, req);
3629         if (status) {
3630             goto invalid;
3631         }
3632 
3633         block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3634                          BLOCK_ACCT_WRITE);
3635         nvme_blk_write(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3636     } else {
3637         req->aiocb = blk_aio_pwrite_zeroes(blk, data_offset, data_size,
3638                                            BDRV_REQ_MAY_UNMAP, nvme_rw_cb,
3639                                            req);
3640     }
3641 
3642     return NVME_NO_COMPLETE;
3643 
3644 invalid:
3645     block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
3646     return status | NVME_DNR;
3647 }
3648 
3649 static inline uint16_t nvme_write(NvmeCtrl *n, NvmeRequest *req)
3650 {
3651     return nvme_do_write(n, req, false, false);
3652 }
3653 
3654 static inline uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req)
3655 {
3656     return nvme_do_write(n, req, false, true);
3657 }
3658 
3659 static inline uint16_t nvme_zone_append(NvmeCtrl *n, NvmeRequest *req)
3660 {
3661     return nvme_do_write(n, req, true, false);
3662 }
3663 
3664 static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace *ns, NvmeCmd *c,
3665                                             uint64_t *slba, uint32_t *zone_idx)
3666 {
3667     uint32_t dw10 = le32_to_cpu(c->cdw10);
3668     uint32_t dw11 = le32_to_cpu(c->cdw11);
3669 
3670     if (!ns->params.zoned) {
3671         trace_pci_nvme_err_invalid_opc(c->opcode);
3672         return NVME_INVALID_OPCODE | NVME_DNR;
3673     }
3674 
3675     *slba = ((uint64_t)dw11) << 32 | dw10;
3676     if (unlikely(*slba >= ns->id_ns.nsze)) {
3677         trace_pci_nvme_err_invalid_lba_range(*slba, 0, ns->id_ns.nsze);
3678         *slba = 0;
3679         return NVME_LBA_RANGE | NVME_DNR;
3680     }
3681 
3682     *zone_idx = nvme_zone_idx(ns, *slba);
3683     assert(*zone_idx < ns->num_zones);
3684 
3685     return NVME_SUCCESS;
3686 }
3687 
3688 typedef uint16_t (*op_handler_t)(NvmeNamespace *, NvmeZone *, NvmeZoneState,
3689                                  NvmeRequest *);
3690 
3691 enum NvmeZoneProcessingMask {
3692     NVME_PROC_CURRENT_ZONE    = 0,
3693     NVME_PROC_OPENED_ZONES    = 1 << 0,
3694     NVME_PROC_CLOSED_ZONES    = 1 << 1,
3695     NVME_PROC_READ_ONLY_ZONES = 1 << 2,
3696     NVME_PROC_FULL_ZONES      = 1 << 3,
3697 };
3698 
3699 static uint16_t nvme_open_zone(NvmeNamespace *ns, NvmeZone *zone,
3700                                NvmeZoneState state, NvmeRequest *req)
3701 {
3702     NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
3703     int flags = 0;
3704 
3705     if (cmd->zsflags & NVME_ZSFLAG_ZRWA_ALLOC) {
3706         uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3707 
3708         if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3709             return NVME_INVALID_ZONE_OP | NVME_DNR;
3710         }
3711 
3712         if (zone->w_ptr % ns->zns.zrwafg) {
3713             return NVME_NOZRWA | NVME_DNR;
3714         }
3715 
3716         flags = NVME_ZRM_ZRWA;
3717     }
3718 
3719     return nvme_zrm_open_flags(nvme_ctrl(req), ns, zone, flags);
3720 }
3721 
3722 static uint16_t nvme_close_zone(NvmeNamespace *ns, NvmeZone *zone,
3723                                 NvmeZoneState state, NvmeRequest *req)
3724 {
3725     return nvme_zrm_close(ns, zone);
3726 }
3727 
3728 static uint16_t nvme_finish_zone(NvmeNamespace *ns, NvmeZone *zone,
3729                                  NvmeZoneState state, NvmeRequest *req)
3730 {
3731     return nvme_zrm_finish(ns, zone);
3732 }
3733 
3734 static uint16_t nvme_offline_zone(NvmeNamespace *ns, NvmeZone *zone,
3735                                   NvmeZoneState state, NvmeRequest *req)
3736 {
3737     switch (state) {
3738     case NVME_ZONE_STATE_READ_ONLY:
3739         nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_OFFLINE);
3740         /* fall through */
3741     case NVME_ZONE_STATE_OFFLINE:
3742         return NVME_SUCCESS;
3743     default:
3744         return NVME_ZONE_INVAL_TRANSITION;
3745     }
3746 }
3747 
3748 static uint16_t nvme_set_zd_ext(NvmeNamespace *ns, NvmeZone *zone)
3749 {
3750     uint16_t status;
3751     uint8_t state = nvme_get_zone_state(zone);
3752 
3753     if (state == NVME_ZONE_STATE_EMPTY) {
3754         status = nvme_aor_check(ns, 1, 0);
3755         if (status) {
3756             return status;
3757         }
3758         nvme_aor_inc_active(ns);
3759         zone->d.za |= NVME_ZA_ZD_EXT_VALID;
3760         nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
3761         return NVME_SUCCESS;
3762     }
3763 
3764     return NVME_ZONE_INVAL_TRANSITION;
3765 }
3766 
3767 static uint16_t nvme_bulk_proc_zone(NvmeNamespace *ns, NvmeZone *zone,
3768                                     enum NvmeZoneProcessingMask proc_mask,
3769                                     op_handler_t op_hndlr, NvmeRequest *req)
3770 {
3771     uint16_t status = NVME_SUCCESS;
3772     NvmeZoneState zs = nvme_get_zone_state(zone);
3773     bool proc_zone;
3774 
3775     switch (zs) {
3776     case NVME_ZONE_STATE_IMPLICITLY_OPEN:
3777     case NVME_ZONE_STATE_EXPLICITLY_OPEN:
3778         proc_zone = proc_mask & NVME_PROC_OPENED_ZONES;
3779         break;
3780     case NVME_ZONE_STATE_CLOSED:
3781         proc_zone = proc_mask & NVME_PROC_CLOSED_ZONES;
3782         break;
3783     case NVME_ZONE_STATE_READ_ONLY:
3784         proc_zone = proc_mask & NVME_PROC_READ_ONLY_ZONES;
3785         break;
3786     case NVME_ZONE_STATE_FULL:
3787         proc_zone = proc_mask & NVME_PROC_FULL_ZONES;
3788         break;
3789     default:
3790         proc_zone = false;
3791     }
3792 
3793     if (proc_zone) {
3794         status = op_hndlr(ns, zone, zs, req);
3795     }
3796 
3797     return status;
3798 }
3799 
3800 static uint16_t nvme_do_zone_op(NvmeNamespace *ns, NvmeZone *zone,
3801                                 enum NvmeZoneProcessingMask proc_mask,
3802                                 op_handler_t op_hndlr, NvmeRequest *req)
3803 {
3804     NvmeZone *next;
3805     uint16_t status = NVME_SUCCESS;
3806     int i;
3807 
3808     if (!proc_mask) {
3809         status = op_hndlr(ns, zone, nvme_get_zone_state(zone), req);
3810     } else {
3811         if (proc_mask & NVME_PROC_CLOSED_ZONES) {
3812             QTAILQ_FOREACH_SAFE(zone, &ns->closed_zones, entry, next) {
3813                 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3814                                              req);
3815                 if (status && status != NVME_NO_COMPLETE) {
3816                     goto out;
3817                 }
3818             }
3819         }
3820         if (proc_mask & NVME_PROC_OPENED_ZONES) {
3821             QTAILQ_FOREACH_SAFE(zone, &ns->imp_open_zones, entry, next) {
3822                 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3823                                              req);
3824                 if (status && status != NVME_NO_COMPLETE) {
3825                     goto out;
3826                 }
3827             }
3828 
3829             QTAILQ_FOREACH_SAFE(zone, &ns->exp_open_zones, entry, next) {
3830                 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3831                                              req);
3832                 if (status && status != NVME_NO_COMPLETE) {
3833                     goto out;
3834                 }
3835             }
3836         }
3837         if (proc_mask & NVME_PROC_FULL_ZONES) {
3838             QTAILQ_FOREACH_SAFE(zone, &ns->full_zones, entry, next) {
3839                 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3840                                              req);
3841                 if (status && status != NVME_NO_COMPLETE) {
3842                     goto out;
3843                 }
3844             }
3845         }
3846 
3847         if (proc_mask & NVME_PROC_READ_ONLY_ZONES) {
3848             for (i = 0; i < ns->num_zones; i++, zone++) {
3849                 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3850                                              req);
3851                 if (status && status != NVME_NO_COMPLETE) {
3852                     goto out;
3853                 }
3854             }
3855         }
3856     }
3857 
3858 out:
3859     return status;
3860 }
3861 
3862 typedef struct NvmeZoneResetAIOCB {
3863     BlockAIOCB common;
3864     BlockAIOCB *aiocb;
3865     NvmeRequest *req;
3866     int ret;
3867 
3868     bool all;
3869     int idx;
3870     NvmeZone *zone;
3871 } NvmeZoneResetAIOCB;
3872 
3873 static void nvme_zone_reset_cancel(BlockAIOCB *aiocb)
3874 {
3875     NvmeZoneResetAIOCB *iocb = container_of(aiocb, NvmeZoneResetAIOCB, common);
3876     NvmeRequest *req = iocb->req;
3877     NvmeNamespace *ns = req->ns;
3878 
3879     iocb->idx = ns->num_zones;
3880 
3881     iocb->ret = -ECANCELED;
3882 
3883     if (iocb->aiocb) {
3884         blk_aio_cancel_async(iocb->aiocb);
3885         iocb->aiocb = NULL;
3886     }
3887 }
3888 
3889 static const AIOCBInfo nvme_zone_reset_aiocb_info = {
3890     .aiocb_size = sizeof(NvmeZoneResetAIOCB),
3891     .cancel_async = nvme_zone_reset_cancel,
3892 };
3893 
3894 static void nvme_zone_reset_cb(void *opaque, int ret);
3895 
3896 static void nvme_zone_reset_epilogue_cb(void *opaque, int ret)
3897 {
3898     NvmeZoneResetAIOCB *iocb = opaque;
3899     NvmeRequest *req = iocb->req;
3900     NvmeNamespace *ns = req->ns;
3901     int64_t moff;
3902     int count;
3903 
3904     if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
3905         goto out;
3906     }
3907 
3908     moff = nvme_moff(ns, iocb->zone->d.zslba);
3909     count = nvme_m2b(ns, ns->zone_size);
3910 
3911     iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, moff, count,
3912                                         BDRV_REQ_MAY_UNMAP,
3913                                         nvme_zone_reset_cb, iocb);
3914     return;
3915 
3916 out:
3917     nvme_zone_reset_cb(iocb, ret);
3918 }
3919 
3920 static void nvme_zone_reset_cb(void *opaque, int ret)
3921 {
3922     NvmeZoneResetAIOCB *iocb = opaque;
3923     NvmeRequest *req = iocb->req;
3924     NvmeNamespace *ns = req->ns;
3925 
3926     if (iocb->ret < 0) {
3927         goto done;
3928     } else if (ret < 0) {
3929         iocb->ret = ret;
3930         goto done;
3931     }
3932 
3933     if (iocb->zone) {
3934         nvme_zrm_reset(ns, iocb->zone);
3935 
3936         if (!iocb->all) {
3937             goto done;
3938         }
3939     }
3940 
3941     while (iocb->idx < ns->num_zones) {
3942         NvmeZone *zone = &ns->zone_array[iocb->idx++];
3943 
3944         switch (nvme_get_zone_state(zone)) {
3945         case NVME_ZONE_STATE_EMPTY:
3946             if (!iocb->all) {
3947                 goto done;
3948             }
3949 
3950             continue;
3951 
3952         case NVME_ZONE_STATE_EXPLICITLY_OPEN:
3953         case NVME_ZONE_STATE_IMPLICITLY_OPEN:
3954         case NVME_ZONE_STATE_CLOSED:
3955         case NVME_ZONE_STATE_FULL:
3956             iocb->zone = zone;
3957             break;
3958 
3959         default:
3960             continue;
3961         }
3962 
3963         trace_pci_nvme_zns_zone_reset(zone->d.zslba);
3964 
3965         iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk,
3966                                             nvme_l2b(ns, zone->d.zslba),
3967                                             nvme_l2b(ns, ns->zone_size),
3968                                             BDRV_REQ_MAY_UNMAP,
3969                                             nvme_zone_reset_epilogue_cb,
3970                                             iocb);
3971         return;
3972     }
3973 
3974 done:
3975     iocb->aiocb = NULL;
3976 
3977     iocb->common.cb(iocb->common.opaque, iocb->ret);
3978     qemu_aio_unref(iocb);
3979 }
3980 
3981 static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl *n, NvmeZone *zone,
3982                                                uint64_t elba, NvmeRequest *req)
3983 {
3984     NvmeNamespace *ns = req->ns;
3985     uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3986     uint64_t wp = zone->d.wp;
3987     uint32_t nlb = elba - wp + 1;
3988     uint16_t status;
3989 
3990 
3991     if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3992         return NVME_INVALID_ZONE_OP | NVME_DNR;
3993     }
3994 
3995     if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3996         return NVME_INVALID_FIELD | NVME_DNR;
3997     }
3998 
3999     if (elba < wp || elba > wp + ns->zns.zrwas) {
4000         return NVME_ZONE_BOUNDARY_ERROR | NVME_DNR;
4001     }
4002 
4003     if (nlb % ns->zns.zrwafg) {
4004         return NVME_INVALID_FIELD | NVME_DNR;
4005     }
4006 
4007     status = nvme_zrm_auto(n, ns, zone);
4008     if (status) {
4009         return status;
4010     }
4011 
4012     zone->w_ptr += nlb;
4013 
4014     nvme_advance_zone_wp(ns, zone, nlb);
4015 
4016     return NVME_SUCCESS;
4017 }
4018 
4019 static uint16_t nvme_zone_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4020 {
4021     NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
4022     NvmeNamespace *ns = req->ns;
4023     NvmeZone *zone;
4024     NvmeZoneResetAIOCB *iocb;
4025     uint8_t *zd_ext;
4026     uint64_t slba = 0;
4027     uint32_t zone_idx = 0;
4028     uint16_t status;
4029     uint8_t action = cmd->zsa;
4030     bool all;
4031     enum NvmeZoneProcessingMask proc_mask = NVME_PROC_CURRENT_ZONE;
4032 
4033     all = cmd->zsflags & NVME_ZSFLAG_SELECT_ALL;
4034 
4035     req->status = NVME_SUCCESS;
4036 
4037     if (!all) {
4038         status = nvme_get_mgmt_zone_slba_idx(ns, &req->cmd, &slba, &zone_idx);
4039         if (status) {
4040             return status;
4041         }
4042     }
4043 
4044     zone = &ns->zone_array[zone_idx];
4045     if (slba != zone->d.zslba && action != NVME_ZONE_ACTION_ZRWA_FLUSH) {
4046         trace_pci_nvme_err_unaligned_zone_cmd(action, slba, zone->d.zslba);
4047         return NVME_INVALID_FIELD | NVME_DNR;
4048     }
4049 
4050     switch (action) {
4051 
4052     case NVME_ZONE_ACTION_OPEN:
4053         if (all) {
4054             proc_mask = NVME_PROC_CLOSED_ZONES;
4055         }
4056         trace_pci_nvme_open_zone(slba, zone_idx, all);
4057         status = nvme_do_zone_op(ns, zone, proc_mask, nvme_open_zone, req);
4058         break;
4059 
4060     case NVME_ZONE_ACTION_CLOSE:
4061         if (all) {
4062             proc_mask = NVME_PROC_OPENED_ZONES;
4063         }
4064         trace_pci_nvme_close_zone(slba, zone_idx, all);
4065         status = nvme_do_zone_op(ns, zone, proc_mask, nvme_close_zone, req);
4066         break;
4067 
4068     case NVME_ZONE_ACTION_FINISH:
4069         if (all) {
4070             proc_mask = NVME_PROC_OPENED_ZONES | NVME_PROC_CLOSED_ZONES;
4071         }
4072         trace_pci_nvme_finish_zone(slba, zone_idx, all);
4073         status = nvme_do_zone_op(ns, zone, proc_mask, nvme_finish_zone, req);
4074         break;
4075 
4076     case NVME_ZONE_ACTION_RESET:
4077         trace_pci_nvme_reset_zone(slba, zone_idx, all);
4078 
4079         iocb = blk_aio_get(&nvme_zone_reset_aiocb_info, ns->blkconf.blk,
4080                            nvme_misc_cb, req);
4081 
4082         iocb->req = req;
4083         iocb->ret = 0;
4084         iocb->all = all;
4085         iocb->idx = zone_idx;
4086         iocb->zone = NULL;
4087 
4088         req->aiocb = &iocb->common;
4089         nvme_zone_reset_cb(iocb, 0);
4090 
4091         return NVME_NO_COMPLETE;
4092 
4093     case NVME_ZONE_ACTION_OFFLINE:
4094         if (all) {
4095             proc_mask = NVME_PROC_READ_ONLY_ZONES;
4096         }
4097         trace_pci_nvme_offline_zone(slba, zone_idx, all);
4098         status = nvme_do_zone_op(ns, zone, proc_mask, nvme_offline_zone, req);
4099         break;
4100 
4101     case NVME_ZONE_ACTION_SET_ZD_EXT:
4102         trace_pci_nvme_set_descriptor_extension(slba, zone_idx);
4103         if (all || !ns->params.zd_extension_size) {
4104             return NVME_INVALID_FIELD | NVME_DNR;
4105         }
4106         zd_ext = nvme_get_zd_extension(ns, zone_idx);
4107         status = nvme_h2c(n, zd_ext, ns->params.zd_extension_size, req);
4108         if (status) {
4109             trace_pci_nvme_err_zd_extension_map_error(zone_idx);
4110             return status;
4111         }
4112 
4113         status = nvme_set_zd_ext(ns, zone);
4114         if (status == NVME_SUCCESS) {
4115             trace_pci_nvme_zd_extension_set(zone_idx);
4116             return status;
4117         }
4118         break;
4119 
4120     case NVME_ZONE_ACTION_ZRWA_FLUSH:
4121         if (all) {
4122             return NVME_INVALID_FIELD | NVME_DNR;
4123         }
4124 
4125         return nvme_zone_mgmt_send_zrwa_flush(n, zone, slba, req);
4126 
4127     default:
4128         trace_pci_nvme_err_invalid_mgmt_action(action);
4129         status = NVME_INVALID_FIELD;
4130     }
4131 
4132     if (status == NVME_ZONE_INVAL_TRANSITION) {
4133         trace_pci_nvme_err_invalid_zone_state_transition(action, slba,
4134                                                          zone->d.za);
4135     }
4136     if (status) {
4137         status |= NVME_DNR;
4138     }
4139 
4140     return status;
4141 }
4142 
4143 static bool nvme_zone_matches_filter(uint32_t zafs, NvmeZone *zl)
4144 {
4145     NvmeZoneState zs = nvme_get_zone_state(zl);
4146 
4147     switch (zafs) {
4148     case NVME_ZONE_REPORT_ALL:
4149         return true;
4150     case NVME_ZONE_REPORT_EMPTY:
4151         return zs == NVME_ZONE_STATE_EMPTY;
4152     case NVME_ZONE_REPORT_IMPLICITLY_OPEN:
4153         return zs == NVME_ZONE_STATE_IMPLICITLY_OPEN;
4154     case NVME_ZONE_REPORT_EXPLICITLY_OPEN:
4155         return zs == NVME_ZONE_STATE_EXPLICITLY_OPEN;
4156     case NVME_ZONE_REPORT_CLOSED:
4157         return zs == NVME_ZONE_STATE_CLOSED;
4158     case NVME_ZONE_REPORT_FULL:
4159         return zs == NVME_ZONE_STATE_FULL;
4160     case NVME_ZONE_REPORT_READ_ONLY:
4161         return zs == NVME_ZONE_STATE_READ_ONLY;
4162     case NVME_ZONE_REPORT_OFFLINE:
4163         return zs == NVME_ZONE_STATE_OFFLINE;
4164     default:
4165         return false;
4166     }
4167 }
4168 
4169 static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4170 {
4171     NvmeCmd *cmd = (NvmeCmd *)&req->cmd;
4172     NvmeNamespace *ns = req->ns;
4173     /* cdw12 is zero-based number of dwords to return. Convert to bytes */
4174     uint32_t data_size = (le32_to_cpu(cmd->cdw12) + 1) << 2;
4175     uint32_t dw13 = le32_to_cpu(cmd->cdw13);
4176     uint32_t zone_idx, zra, zrasf, partial;
4177     uint64_t max_zones, nr_zones = 0;
4178     uint16_t status;
4179     uint64_t slba;
4180     NvmeZoneDescr *z;
4181     NvmeZone *zone;
4182     NvmeZoneReportHeader *header;
4183     void *buf, *buf_p;
4184     size_t zone_entry_sz;
4185     int i;
4186 
4187     req->status = NVME_SUCCESS;
4188 
4189     status = nvme_get_mgmt_zone_slba_idx(ns, cmd, &slba, &zone_idx);
4190     if (status) {
4191         return status;
4192     }
4193 
4194     zra = dw13 & 0xff;
4195     if (zra != NVME_ZONE_REPORT && zra != NVME_ZONE_REPORT_EXTENDED) {
4196         return NVME_INVALID_FIELD | NVME_DNR;
4197     }
4198     if (zra == NVME_ZONE_REPORT_EXTENDED && !ns->params.zd_extension_size) {
4199         return NVME_INVALID_FIELD | NVME_DNR;
4200     }
4201 
4202     zrasf = (dw13 >> 8) & 0xff;
4203     if (zrasf > NVME_ZONE_REPORT_OFFLINE) {
4204         return NVME_INVALID_FIELD | NVME_DNR;
4205     }
4206 
4207     if (data_size < sizeof(NvmeZoneReportHeader)) {
4208         return NVME_INVALID_FIELD | NVME_DNR;
4209     }
4210 
4211     status = nvme_check_mdts(n, data_size);
4212     if (status) {
4213         return status;
4214     }
4215 
4216     partial = (dw13 >> 16) & 0x01;
4217 
4218     zone_entry_sz = sizeof(NvmeZoneDescr);
4219     if (zra == NVME_ZONE_REPORT_EXTENDED) {
4220         zone_entry_sz += ns->params.zd_extension_size;
4221     }
4222 
4223     max_zones = (data_size - sizeof(NvmeZoneReportHeader)) / zone_entry_sz;
4224     buf = g_malloc0(data_size);
4225 
4226     zone = &ns->zone_array[zone_idx];
4227     for (i = zone_idx; i < ns->num_zones; i++) {
4228         if (partial && nr_zones >= max_zones) {
4229             break;
4230         }
4231         if (nvme_zone_matches_filter(zrasf, zone++)) {
4232             nr_zones++;
4233         }
4234     }
4235     header = buf;
4236     header->nr_zones = cpu_to_le64(nr_zones);
4237 
4238     buf_p = buf + sizeof(NvmeZoneReportHeader);
4239     for (; zone_idx < ns->num_zones && max_zones > 0; zone_idx++) {
4240         zone = &ns->zone_array[zone_idx];
4241         if (nvme_zone_matches_filter(zrasf, zone)) {
4242             z = buf_p;
4243             buf_p += sizeof(NvmeZoneDescr);
4244 
4245             z->zt = zone->d.zt;
4246             z->zs = zone->d.zs;
4247             z->zcap = cpu_to_le64(zone->d.zcap);
4248             z->zslba = cpu_to_le64(zone->d.zslba);
4249             z->za = zone->d.za;
4250 
4251             if (nvme_wp_is_valid(zone)) {
4252                 z->wp = cpu_to_le64(zone->d.wp);
4253             } else {
4254                 z->wp = cpu_to_le64(~0ULL);
4255             }
4256 
4257             if (zra == NVME_ZONE_REPORT_EXTENDED) {
4258                 if (zone->d.za & NVME_ZA_ZD_EXT_VALID) {
4259                     memcpy(buf_p, nvme_get_zd_extension(ns, zone_idx),
4260                            ns->params.zd_extension_size);
4261                 }
4262                 buf_p += ns->params.zd_extension_size;
4263             }
4264 
4265             max_zones--;
4266         }
4267     }
4268 
4269     status = nvme_c2h(n, (uint8_t *)buf, data_size, req);
4270 
4271     g_free(buf);
4272 
4273     return status;
4274 }
4275 
4276 static uint16_t nvme_io_mgmt_recv_ruhs(NvmeCtrl *n, NvmeRequest *req,
4277                                        size_t len)
4278 {
4279     NvmeNamespace *ns = req->ns;
4280     NvmeEnduranceGroup *endgrp;
4281     NvmeRuhStatus *hdr;
4282     NvmeRuhStatusDescr *ruhsd;
4283     unsigned int nruhsd;
4284     uint16_t rg, ph, *ruhid;
4285     size_t trans_len;
4286     g_autofree uint8_t *buf = NULL;
4287 
4288     if (!n->subsys) {
4289         return NVME_INVALID_FIELD | NVME_DNR;
4290     }
4291 
4292     if (ns->params.nsid == 0 || ns->params.nsid == 0xffffffff) {
4293         return NVME_INVALID_NSID | NVME_DNR;
4294     }
4295 
4296     if (!n->subsys->endgrp.fdp.enabled) {
4297         return NVME_FDP_DISABLED | NVME_DNR;
4298     }
4299 
4300     endgrp = ns->endgrp;
4301 
4302     nruhsd = ns->fdp.nphs * endgrp->fdp.nrg;
4303     trans_len = sizeof(NvmeRuhStatus) + nruhsd * sizeof(NvmeRuhStatusDescr);
4304     buf = g_malloc(trans_len);
4305 
4306     trans_len = MIN(trans_len, len);
4307 
4308     hdr = (NvmeRuhStatus *)buf;
4309     ruhsd = (NvmeRuhStatusDescr *)(buf + sizeof(NvmeRuhStatus));
4310 
4311     hdr->nruhsd = cpu_to_le16(nruhsd);
4312 
4313     ruhid = ns->fdp.phs;
4314 
4315     for (ph = 0; ph < ns->fdp.nphs; ph++, ruhid++) {
4316         NvmeRuHandle *ruh = &endgrp->fdp.ruhs[*ruhid];
4317 
4318         for (rg = 0; rg < endgrp->fdp.nrg; rg++, ruhsd++) {
4319             uint16_t pid = nvme_make_pid(ns, rg, ph);
4320 
4321             ruhsd->pid = cpu_to_le16(pid);
4322             ruhsd->ruhid = *ruhid;
4323             ruhsd->earutr = 0;
4324             ruhsd->ruamw = cpu_to_le64(ruh->rus[rg].ruamw);
4325         }
4326     }
4327 
4328     return nvme_c2h(n, buf, trans_len, req);
4329 }
4330 
4331 static uint16_t nvme_io_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4332 {
4333     NvmeCmd *cmd = &req->cmd;
4334     uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4335     uint32_t numd = le32_to_cpu(cmd->cdw11);
4336     uint8_t mo = (cdw10 & 0xff);
4337     size_t len = (numd + 1) << 2;
4338 
4339     switch (mo) {
4340     case NVME_IOMR_MO_NOP:
4341         return 0;
4342     case NVME_IOMR_MO_RUH_STATUS:
4343         return nvme_io_mgmt_recv_ruhs(n, req, len);
4344     default:
4345         return NVME_INVALID_FIELD | NVME_DNR;
4346     };
4347 }
4348 
4349 static uint16_t nvme_io_mgmt_send_ruh_update(NvmeCtrl *n, NvmeRequest *req)
4350 {
4351     NvmeCmd *cmd = &req->cmd;
4352     NvmeNamespace *ns = req->ns;
4353     uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4354     uint16_t ret = NVME_SUCCESS;
4355     uint32_t npid = (cdw10 >> 1) + 1;
4356     unsigned int i = 0;
4357     g_autofree uint16_t *pids = NULL;
4358     uint32_t maxnpid;
4359 
4360     if (!ns->endgrp || !ns->endgrp->fdp.enabled) {
4361         return NVME_FDP_DISABLED | NVME_DNR;
4362     }
4363 
4364     maxnpid = n->subsys->endgrp.fdp.nrg * n->subsys->endgrp.fdp.nruh;
4365 
4366     if (unlikely(npid >= MIN(NVME_FDP_MAXPIDS, maxnpid))) {
4367         return NVME_INVALID_FIELD | NVME_DNR;
4368     }
4369 
4370     pids = g_new(uint16_t, npid);
4371 
4372     ret = nvme_h2c(n, pids, npid * sizeof(uint16_t), req);
4373     if (ret) {
4374         return ret;
4375     }
4376 
4377     for (; i < npid; i++) {
4378         if (!nvme_update_ruh(n, ns, pids[i])) {
4379             return NVME_INVALID_FIELD | NVME_DNR;
4380         }
4381     }
4382 
4383     return ret;
4384 }
4385 
4386 static uint16_t nvme_io_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4387 {
4388     NvmeCmd *cmd = &req->cmd;
4389     uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4390     uint8_t mo = (cdw10 & 0xff);
4391 
4392     switch (mo) {
4393     case NVME_IOMS_MO_NOP:
4394         return 0;
4395     case NVME_IOMS_MO_RUH_UPDATE:
4396         return nvme_io_mgmt_send_ruh_update(n, req);
4397     default:
4398         return NVME_INVALID_FIELD | NVME_DNR;
4399     };
4400 }
4401 
4402 static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req)
4403 {
4404     NvmeNamespace *ns;
4405     uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4406 
4407     trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req),
4408                           req->cmd.opcode, nvme_io_opc_str(req->cmd.opcode));
4409 
4410     if (!nvme_nsid_valid(n, nsid)) {
4411         return NVME_INVALID_NSID | NVME_DNR;
4412     }
4413 
4414     /*
4415      * In the base NVM command set, Flush may apply to all namespaces
4416      * (indicated by NSID being set to FFFFFFFFh). But if that feature is used
4417      * along with TP 4056 (Namespace Types), it may be pretty screwed up.
4418      *
4419      * If NSID is indeed set to FFFFFFFFh, we simply cannot associate the
4420      * opcode with a specific command since we cannot determine a unique I/O
4421      * command set. Opcode 0h could have any other meaning than something
4422      * equivalent to flushing and say it DOES have completely different
4423      * semantics in some other command set - does an NSID of FFFFFFFFh then
4424      * mean "for all namespaces, apply whatever command set specific command
4425      * that uses the 0h opcode?" Or does it mean "for all namespaces, apply
4426      * whatever command that uses the 0h opcode if, and only if, it allows NSID
4427      * to be FFFFFFFFh"?
4428      *
4429      * Anyway (and luckily), for now, we do not care about this since the
4430      * device only supports namespace types that includes the NVM Flush command
4431      * (NVM and Zoned), so always do an NVM Flush.
4432      */
4433     if (req->cmd.opcode == NVME_CMD_FLUSH) {
4434         return nvme_flush(n, req);
4435     }
4436 
4437     ns = nvme_ns(n, nsid);
4438     if (unlikely(!ns)) {
4439         return NVME_INVALID_FIELD | NVME_DNR;
4440     }
4441 
4442     if (!(ns->iocs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
4443         trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
4444         return NVME_INVALID_OPCODE | NVME_DNR;
4445     }
4446 
4447     if (ns->status) {
4448         return ns->status;
4449     }
4450 
4451     if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
4452         return NVME_INVALID_FIELD;
4453     }
4454 
4455     req->ns = ns;
4456 
4457     switch (req->cmd.opcode) {
4458     case NVME_CMD_WRITE_ZEROES:
4459         return nvme_write_zeroes(n, req);
4460     case NVME_CMD_ZONE_APPEND:
4461         return nvme_zone_append(n, req);
4462     case NVME_CMD_WRITE:
4463         return nvme_write(n, req);
4464     case NVME_CMD_READ:
4465         return nvme_read(n, req);
4466     case NVME_CMD_COMPARE:
4467         return nvme_compare(n, req);
4468     case NVME_CMD_DSM:
4469         return nvme_dsm(n, req);
4470     case NVME_CMD_VERIFY:
4471         return nvme_verify(n, req);
4472     case NVME_CMD_COPY:
4473         return nvme_copy(n, req);
4474     case NVME_CMD_ZONE_MGMT_SEND:
4475         return nvme_zone_mgmt_send(n, req);
4476     case NVME_CMD_ZONE_MGMT_RECV:
4477         return nvme_zone_mgmt_recv(n, req);
4478     case NVME_CMD_IO_MGMT_RECV:
4479         return nvme_io_mgmt_recv(n, req);
4480     case NVME_CMD_IO_MGMT_SEND:
4481         return nvme_io_mgmt_send(n, req);
4482     default:
4483         assert(false);
4484     }
4485 
4486     return NVME_INVALID_OPCODE | NVME_DNR;
4487 }
4488 
4489 static void nvme_cq_notifier(EventNotifier *e)
4490 {
4491     NvmeCQueue *cq = container_of(e, NvmeCQueue, notifier);
4492     NvmeCtrl *n = cq->ctrl;
4493 
4494     if (!event_notifier_test_and_clear(e)) {
4495         return;
4496     }
4497 
4498     nvme_update_cq_head(cq);
4499 
4500     if (cq->tail == cq->head) {
4501         if (cq->irq_enabled) {
4502             n->cq_pending--;
4503         }
4504 
4505         nvme_irq_deassert(n, cq);
4506     }
4507 
4508     qemu_bh_schedule(cq->bh);
4509 }
4510 
4511 static int nvme_init_cq_ioeventfd(NvmeCQueue *cq)
4512 {
4513     NvmeCtrl *n = cq->ctrl;
4514     uint16_t offset = (cq->cqid << 3) + (1 << 2);
4515     int ret;
4516 
4517     ret = event_notifier_init(&cq->notifier, 0);
4518     if (ret < 0) {
4519         return ret;
4520     }
4521 
4522     event_notifier_set_handler(&cq->notifier, nvme_cq_notifier);
4523     memory_region_add_eventfd(&n->iomem,
4524                               0x1000 + offset, 4, false, 0, &cq->notifier);
4525 
4526     return 0;
4527 }
4528 
4529 static void nvme_sq_notifier(EventNotifier *e)
4530 {
4531     NvmeSQueue *sq = container_of(e, NvmeSQueue, notifier);
4532 
4533     if (!event_notifier_test_and_clear(e)) {
4534         return;
4535     }
4536 
4537     nvme_process_sq(sq);
4538 }
4539 
4540 static int nvme_init_sq_ioeventfd(NvmeSQueue *sq)
4541 {
4542     NvmeCtrl *n = sq->ctrl;
4543     uint16_t offset = sq->sqid << 3;
4544     int ret;
4545 
4546     ret = event_notifier_init(&sq->notifier, 0);
4547     if (ret < 0) {
4548         return ret;
4549     }
4550 
4551     event_notifier_set_handler(&sq->notifier, nvme_sq_notifier);
4552     memory_region_add_eventfd(&n->iomem,
4553                               0x1000 + offset, 4, false, 0, &sq->notifier);
4554 
4555     return 0;
4556 }
4557 
4558 static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
4559 {
4560     uint16_t offset = sq->sqid << 3;
4561 
4562     n->sq[sq->sqid] = NULL;
4563     qemu_bh_delete(sq->bh);
4564     if (sq->ioeventfd_enabled) {
4565         memory_region_del_eventfd(&n->iomem,
4566                                   0x1000 + offset, 4, false, 0, &sq->notifier);
4567         event_notifier_set_handler(&sq->notifier, NULL);
4568         event_notifier_cleanup(&sq->notifier);
4569     }
4570     g_free(sq->io_req);
4571     if (sq->sqid) {
4572         g_free(sq);
4573     }
4574 }
4575 
4576 static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req)
4577 {
4578     NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
4579     NvmeRequest *r, *next;
4580     NvmeSQueue *sq;
4581     NvmeCQueue *cq;
4582     uint16_t qid = le16_to_cpu(c->qid);
4583 
4584     if (unlikely(!qid || nvme_check_sqid(n, qid))) {
4585         trace_pci_nvme_err_invalid_del_sq(qid);
4586         return NVME_INVALID_QID | NVME_DNR;
4587     }
4588 
4589     trace_pci_nvme_del_sq(qid);
4590 
4591     sq = n->sq[qid];
4592     while (!QTAILQ_EMPTY(&sq->out_req_list)) {
4593         r = QTAILQ_FIRST(&sq->out_req_list);
4594         assert(r->aiocb);
4595         blk_aio_cancel(r->aiocb);
4596     }
4597 
4598     assert(QTAILQ_EMPTY(&sq->out_req_list));
4599 
4600     if (!nvme_check_cqid(n, sq->cqid)) {
4601         cq = n->cq[sq->cqid];
4602         QTAILQ_REMOVE(&cq->sq_list, sq, entry);
4603 
4604         nvme_post_cqes(cq);
4605         QTAILQ_FOREACH_SAFE(r, &cq->req_list, entry, next) {
4606             if (r->sq == sq) {
4607                 QTAILQ_REMOVE(&cq->req_list, r, entry);
4608                 QTAILQ_INSERT_TAIL(&sq->req_list, r, entry);
4609             }
4610         }
4611     }
4612 
4613     nvme_free_sq(sq, n);
4614     return NVME_SUCCESS;
4615 }
4616 
4617 static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
4618                          uint16_t sqid, uint16_t cqid, uint16_t size)
4619 {
4620     int i;
4621     NvmeCQueue *cq;
4622 
4623     sq->ctrl = n;
4624     sq->dma_addr = dma_addr;
4625     sq->sqid = sqid;
4626     sq->size = size;
4627     sq->cqid = cqid;
4628     sq->head = sq->tail = 0;
4629     sq->io_req = g_new0(NvmeRequest, sq->size);
4630 
4631     QTAILQ_INIT(&sq->req_list);
4632     QTAILQ_INIT(&sq->out_req_list);
4633     for (i = 0; i < sq->size; i++) {
4634         sq->io_req[i].sq = sq;
4635         QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
4636     }
4637 
4638     sq->bh = qemu_bh_new_guarded(nvme_process_sq, sq,
4639                                  &DEVICE(sq->ctrl)->mem_reentrancy_guard);
4640 
4641     if (n->dbbuf_enabled) {
4642         sq->db_addr = n->dbbuf_dbs + (sqid << 3);
4643         sq->ei_addr = n->dbbuf_eis + (sqid << 3);
4644 
4645         if (n->params.ioeventfd && sq->sqid != 0) {
4646             if (!nvme_init_sq_ioeventfd(sq)) {
4647                 sq->ioeventfd_enabled = true;
4648             }
4649         }
4650     }
4651 
4652     assert(n->cq[cqid]);
4653     cq = n->cq[cqid];
4654     QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
4655     n->sq[sqid] = sq;
4656 }
4657 
4658 static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req)
4659 {
4660     NvmeSQueue *sq;
4661     NvmeCreateSq *c = (NvmeCreateSq *)&req->cmd;
4662 
4663     uint16_t cqid = le16_to_cpu(c->cqid);
4664     uint16_t sqid = le16_to_cpu(c->sqid);
4665     uint16_t qsize = le16_to_cpu(c->qsize);
4666     uint16_t qflags = le16_to_cpu(c->sq_flags);
4667     uint64_t prp1 = le64_to_cpu(c->prp1);
4668 
4669     trace_pci_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
4670 
4671     if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
4672         trace_pci_nvme_err_invalid_create_sq_cqid(cqid);
4673         return NVME_INVALID_CQID | NVME_DNR;
4674     }
4675     if (unlikely(!sqid || sqid > n->conf_ioqpairs || n->sq[sqid] != NULL)) {
4676         trace_pci_nvme_err_invalid_create_sq_sqid(sqid);
4677         return NVME_INVALID_QID | NVME_DNR;
4678     }
4679     if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
4680         trace_pci_nvme_err_invalid_create_sq_size(qsize);
4681         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
4682     }
4683     if (unlikely(prp1 & (n->page_size - 1))) {
4684         trace_pci_nvme_err_invalid_create_sq_addr(prp1);
4685         return NVME_INVALID_PRP_OFFSET | NVME_DNR;
4686     }
4687     if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
4688         trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
4689         return NVME_INVALID_FIELD | NVME_DNR;
4690     }
4691     sq = g_malloc0(sizeof(*sq));
4692     nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
4693     return NVME_SUCCESS;
4694 }
4695 
4696 struct nvme_stats {
4697     uint64_t units_read;
4698     uint64_t units_written;
4699     uint64_t read_commands;
4700     uint64_t write_commands;
4701 };
4702 
4703 static void nvme_set_blk_stats(NvmeNamespace *ns, struct nvme_stats *stats)
4704 {
4705     BlockAcctStats *s = blk_get_stats(ns->blkconf.blk);
4706 
4707     stats->units_read += s->nr_bytes[BLOCK_ACCT_READ];
4708     stats->units_written += s->nr_bytes[BLOCK_ACCT_WRITE];
4709     stats->read_commands += s->nr_ops[BLOCK_ACCT_READ];
4710     stats->write_commands += s->nr_ops[BLOCK_ACCT_WRITE];
4711 }
4712 
4713 static uint16_t nvme_smart_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4714                                 uint64_t off, NvmeRequest *req)
4715 {
4716     uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4717     struct nvme_stats stats = { 0 };
4718     NvmeSmartLog smart = { 0 };
4719     uint32_t trans_len;
4720     NvmeNamespace *ns;
4721     time_t current_ms;
4722     uint64_t u_read, u_written;
4723 
4724     if (off >= sizeof(smart)) {
4725         return NVME_INVALID_FIELD | NVME_DNR;
4726     }
4727 
4728     if (nsid != 0xffffffff) {
4729         ns = nvme_ns(n, nsid);
4730         if (!ns) {
4731             return NVME_INVALID_NSID | NVME_DNR;
4732         }
4733         nvme_set_blk_stats(ns, &stats);
4734     } else {
4735         int i;
4736 
4737         for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
4738             ns = nvme_ns(n, i);
4739             if (!ns) {
4740                 continue;
4741             }
4742             nvme_set_blk_stats(ns, &stats);
4743         }
4744     }
4745 
4746     trans_len = MIN(sizeof(smart) - off, buf_len);
4747     smart.critical_warning = n->smart_critical_warning;
4748 
4749     u_read = DIV_ROUND_UP(stats.units_read >> BDRV_SECTOR_BITS, 1000);
4750     u_written = DIV_ROUND_UP(stats.units_written >> BDRV_SECTOR_BITS, 1000);
4751 
4752     smart.data_units_read[0] = cpu_to_le64(u_read);
4753     smart.data_units_written[0] = cpu_to_le64(u_written);
4754     smart.host_read_commands[0] = cpu_to_le64(stats.read_commands);
4755     smart.host_write_commands[0] = cpu_to_le64(stats.write_commands);
4756 
4757     smart.temperature = cpu_to_le16(n->temperature);
4758 
4759     if ((n->temperature >= n->features.temp_thresh_hi) ||
4760         (n->temperature <= n->features.temp_thresh_low)) {
4761         smart.critical_warning |= NVME_SMART_TEMPERATURE;
4762     }
4763 
4764     current_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
4765     smart.power_on_hours[0] =
4766         cpu_to_le64((((current_ms - n->starttime_ms) / 1000) / 60) / 60);
4767 
4768     if (!rae) {
4769         nvme_clear_events(n, NVME_AER_TYPE_SMART);
4770     }
4771 
4772     return nvme_c2h(n, (uint8_t *) &smart + off, trans_len, req);
4773 }
4774 
4775 static uint16_t nvme_endgrp_info(NvmeCtrl *n,  uint8_t rae, uint32_t buf_len,
4776                                  uint64_t off, NvmeRequest *req)
4777 {
4778     uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
4779     uint16_t endgrpid = (dw11 >> 16) & 0xffff;
4780     struct nvme_stats stats = {};
4781     NvmeEndGrpLog info = {};
4782     int i;
4783 
4784     if (!n->subsys || endgrpid != 0x1) {
4785         return NVME_INVALID_FIELD | NVME_DNR;
4786     }
4787 
4788     if (off >= sizeof(info)) {
4789         return NVME_INVALID_FIELD | NVME_DNR;
4790     }
4791 
4792     for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
4793         NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
4794         if (!ns) {
4795             continue;
4796         }
4797 
4798         nvme_set_blk_stats(ns, &stats);
4799     }
4800 
4801     info.data_units_read[0] =
4802         cpu_to_le64(DIV_ROUND_UP(stats.units_read / 1000000000, 1000000000));
4803     info.data_units_written[0] =
4804         cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4805     info.media_units_written[0] =
4806         cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4807 
4808     info.host_read_commands[0] = cpu_to_le64(stats.read_commands);
4809     info.host_write_commands[0] = cpu_to_le64(stats.write_commands);
4810 
4811     buf_len = MIN(sizeof(info) - off, buf_len);
4812 
4813     return nvme_c2h(n, (uint8_t *)&info + off, buf_len, req);
4814 }
4815 
4816 
4817 static uint16_t nvme_fw_log_info(NvmeCtrl *n, uint32_t buf_len, uint64_t off,
4818                                  NvmeRequest *req)
4819 {
4820     uint32_t trans_len;
4821     NvmeFwSlotInfoLog fw_log = {
4822         .afi = 0x1,
4823     };
4824 
4825     if (off >= sizeof(fw_log)) {
4826         return NVME_INVALID_FIELD | NVME_DNR;
4827     }
4828 
4829     strpadcpy((char *)&fw_log.frs1, sizeof(fw_log.frs1), "1.0", ' ');
4830     trans_len = MIN(sizeof(fw_log) - off, buf_len);
4831 
4832     return nvme_c2h(n, (uint8_t *) &fw_log + off, trans_len, req);
4833 }
4834 
4835 static uint16_t nvme_error_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4836                                 uint64_t off, NvmeRequest *req)
4837 {
4838     uint32_t trans_len;
4839     NvmeErrorLog errlog;
4840 
4841     if (off >= sizeof(errlog)) {
4842         return NVME_INVALID_FIELD | NVME_DNR;
4843     }
4844 
4845     if (!rae) {
4846         nvme_clear_events(n, NVME_AER_TYPE_ERROR);
4847     }
4848 
4849     memset(&errlog, 0x0, sizeof(errlog));
4850     trans_len = MIN(sizeof(errlog) - off, buf_len);
4851 
4852     return nvme_c2h(n, (uint8_t *)&errlog, trans_len, req);
4853 }
4854 
4855 static uint16_t nvme_changed_nslist(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4856                                     uint64_t off, NvmeRequest *req)
4857 {
4858     uint32_t nslist[1024];
4859     uint32_t trans_len;
4860     int i = 0;
4861     uint32_t nsid;
4862 
4863     if (off >= sizeof(nslist)) {
4864         trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(nslist));
4865         return NVME_INVALID_FIELD | NVME_DNR;
4866     }
4867 
4868     memset(nslist, 0x0, sizeof(nslist));
4869     trans_len = MIN(sizeof(nslist) - off, buf_len);
4870 
4871     while ((nsid = find_first_bit(n->changed_nsids, NVME_CHANGED_NSID_SIZE)) !=
4872             NVME_CHANGED_NSID_SIZE) {
4873         /*
4874          * If more than 1024 namespaces, the first entry in the log page should
4875          * be set to FFFFFFFFh and the others to 0 as spec.
4876          */
4877         if (i == ARRAY_SIZE(nslist)) {
4878             memset(nslist, 0x0, sizeof(nslist));
4879             nslist[0] = 0xffffffff;
4880             break;
4881         }
4882 
4883         nslist[i++] = nsid;
4884         clear_bit(nsid, n->changed_nsids);
4885     }
4886 
4887     /*
4888      * Remove all the remaining list entries in case returns directly due to
4889      * more than 1024 namespaces.
4890      */
4891     if (nslist[0] == 0xffffffff) {
4892         bitmap_zero(n->changed_nsids, NVME_CHANGED_NSID_SIZE);
4893     }
4894 
4895     if (!rae) {
4896         nvme_clear_events(n, NVME_AER_TYPE_NOTICE);
4897     }
4898 
4899     return nvme_c2h(n, ((uint8_t *)nslist) + off, trans_len, req);
4900 }
4901 
4902 static uint16_t nvme_cmd_effects(NvmeCtrl *n, uint8_t csi, uint32_t buf_len,
4903                                  uint64_t off, NvmeRequest *req)
4904 {
4905     NvmeEffectsLog log = {};
4906     const uint32_t *src_iocs = NULL;
4907     uint32_t trans_len;
4908 
4909     if (off >= sizeof(log)) {
4910         trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(log));
4911         return NVME_INVALID_FIELD | NVME_DNR;
4912     }
4913 
4914     switch (NVME_CC_CSS(ldl_le_p(&n->bar.cc))) {
4915     case NVME_CC_CSS_NVM:
4916         src_iocs = nvme_cse_iocs_nvm;
4917         /* fall through */
4918     case NVME_CC_CSS_ADMIN_ONLY:
4919         break;
4920     case NVME_CC_CSS_CSI:
4921         switch (csi) {
4922         case NVME_CSI_NVM:
4923             src_iocs = nvme_cse_iocs_nvm;
4924             break;
4925         case NVME_CSI_ZONED:
4926             src_iocs = nvme_cse_iocs_zoned;
4927             break;
4928         }
4929     }
4930 
4931     memcpy(log.acs, nvme_cse_acs, sizeof(nvme_cse_acs));
4932 
4933     if (src_iocs) {
4934         memcpy(log.iocs, src_iocs, sizeof(log.iocs));
4935     }
4936 
4937     trans_len = MIN(sizeof(log) - off, buf_len);
4938 
4939     return nvme_c2h(n, ((uint8_t *)&log) + off, trans_len, req);
4940 }
4941 
4942 static size_t sizeof_fdp_conf_descr(size_t nruh, size_t vss)
4943 {
4944     size_t entry_siz = sizeof(NvmeFdpDescrHdr) + nruh * sizeof(NvmeRuhDescr)
4945                        + vss;
4946     return ROUND_UP(entry_siz, 8);
4947 }
4948 
4949 static uint16_t nvme_fdp_confs(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
4950                                uint64_t off, NvmeRequest *req)
4951 {
4952     uint32_t log_size, trans_len;
4953     g_autofree uint8_t *buf = NULL;
4954     NvmeFdpDescrHdr *hdr;
4955     NvmeRuhDescr *ruhd;
4956     NvmeEnduranceGroup *endgrp;
4957     NvmeFdpConfsHdr *log;
4958     size_t nruh, fdp_descr_size;
4959     int i;
4960 
4961     if (endgrpid != 1 || !n->subsys) {
4962         return NVME_INVALID_FIELD | NVME_DNR;
4963     }
4964 
4965     endgrp = &n->subsys->endgrp;
4966 
4967     if (endgrp->fdp.enabled) {
4968         nruh = endgrp->fdp.nruh;
4969     } else {
4970         nruh = 1;
4971     }
4972 
4973     fdp_descr_size = sizeof_fdp_conf_descr(nruh, FDPVSS);
4974     log_size = sizeof(NvmeFdpConfsHdr) + fdp_descr_size;
4975 
4976     if (off >= log_size) {
4977         return NVME_INVALID_FIELD | NVME_DNR;
4978     }
4979 
4980     trans_len = MIN(log_size - off, buf_len);
4981 
4982     buf = g_malloc0(log_size);
4983     log = (NvmeFdpConfsHdr *)buf;
4984     hdr = (NvmeFdpDescrHdr *)(log + 1);
4985     ruhd = (NvmeRuhDescr *)(buf + sizeof(*log) + sizeof(*hdr));
4986 
4987     log->num_confs = cpu_to_le16(0);
4988     log->size = cpu_to_le32(log_size);
4989 
4990     hdr->descr_size = cpu_to_le16(fdp_descr_size);
4991     if (endgrp->fdp.enabled) {
4992         hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, VALID, 1);
4993         hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, RGIF, endgrp->fdp.rgif);
4994         hdr->nrg = cpu_to_le16(endgrp->fdp.nrg);
4995         hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
4996         hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
4997         hdr->nnss = cpu_to_le32(NVME_MAX_NAMESPACES);
4998         hdr->runs = cpu_to_le64(endgrp->fdp.runs);
4999 
5000         for (i = 0; i < nruh; i++) {
5001             ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
5002             ruhd++;
5003         }
5004     } else {
5005         /* 1 bit for RUH in PIF -> 2 RUHs max. */
5006         hdr->nrg = cpu_to_le16(1);
5007         hdr->nruh = cpu_to_le16(1);
5008         hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
5009         hdr->nnss = cpu_to_le32(1);
5010         hdr->runs = cpu_to_le64(96 * MiB);
5011 
5012         ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
5013     }
5014 
5015     return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
5016 }
5017 
5018 static uint16_t nvme_fdp_ruh_usage(NvmeCtrl *n, uint32_t endgrpid,
5019                                    uint32_t dw10, uint32_t dw12,
5020                                    uint32_t buf_len, uint64_t off,
5021                                    NvmeRequest *req)
5022 {
5023     NvmeRuHandle *ruh;
5024     NvmeRuhuLog *hdr;
5025     NvmeRuhuDescr *ruhud;
5026     NvmeEnduranceGroup *endgrp;
5027     g_autofree uint8_t *buf = NULL;
5028     uint32_t log_size, trans_len;
5029     uint16_t i;
5030 
5031     if (endgrpid != 1 || !n->subsys) {
5032         return NVME_INVALID_FIELD | NVME_DNR;
5033     }
5034 
5035     endgrp = &n->subsys->endgrp;
5036 
5037     if (!endgrp->fdp.enabled) {
5038         return NVME_FDP_DISABLED | NVME_DNR;
5039     }
5040 
5041     log_size = sizeof(NvmeRuhuLog) + endgrp->fdp.nruh * sizeof(NvmeRuhuDescr);
5042 
5043     if (off >= log_size) {
5044         return NVME_INVALID_FIELD | NVME_DNR;
5045     }
5046 
5047     trans_len = MIN(log_size - off, buf_len);
5048 
5049     buf = g_malloc0(log_size);
5050     hdr = (NvmeRuhuLog *)buf;
5051     ruhud = (NvmeRuhuDescr *)(hdr + 1);
5052 
5053     ruh = endgrp->fdp.ruhs;
5054     hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
5055 
5056     for (i = 0; i < endgrp->fdp.nruh; i++, ruhud++, ruh++) {
5057         ruhud->ruha = ruh->ruha;
5058     }
5059 
5060     return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
5061 }
5062 
5063 static uint16_t nvme_fdp_stats(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
5064                                uint64_t off, NvmeRequest *req)
5065 {
5066     NvmeEnduranceGroup *endgrp;
5067     NvmeFdpStatsLog log = {};
5068     uint32_t trans_len;
5069 
5070     if (off >= sizeof(NvmeFdpStatsLog)) {
5071         return NVME_INVALID_FIELD | NVME_DNR;
5072     }
5073 
5074     if (endgrpid != 1 || !n->subsys) {
5075         return NVME_INVALID_FIELD | NVME_DNR;
5076     }
5077 
5078     if (!n->subsys->endgrp.fdp.enabled) {
5079         return NVME_FDP_DISABLED | NVME_DNR;
5080     }
5081 
5082     endgrp = &n->subsys->endgrp;
5083 
5084     trans_len = MIN(sizeof(log) - off, buf_len);
5085 
5086     /* spec value is 128 bit, we only use 64 bit */
5087     log.hbmw[0] = cpu_to_le64(endgrp->fdp.hbmw);
5088     log.mbmw[0] = cpu_to_le64(endgrp->fdp.mbmw);
5089     log.mbe[0] = cpu_to_le64(endgrp->fdp.mbe);
5090 
5091     return nvme_c2h(n, (uint8_t *)&log + off, trans_len, req);
5092 }
5093 
5094 static uint16_t nvme_fdp_events(NvmeCtrl *n, uint32_t endgrpid,
5095                                 uint32_t buf_len, uint64_t off,
5096                                 NvmeRequest *req)
5097 {
5098     NvmeEnduranceGroup *endgrp;
5099     NvmeCmd *cmd = &req->cmd;
5100     bool host_events = (cmd->cdw10 >> 8) & 0x1;
5101     uint32_t log_size, trans_len;
5102     NvmeFdpEventBuffer *ebuf;
5103     g_autofree NvmeFdpEventsLog *elog = NULL;
5104     NvmeFdpEvent *event;
5105 
5106     if (endgrpid != 1 || !n->subsys) {
5107         return NVME_INVALID_FIELD | NVME_DNR;
5108     }
5109 
5110     endgrp = &n->subsys->endgrp;
5111 
5112     if (!endgrp->fdp.enabled) {
5113         return NVME_FDP_DISABLED | NVME_DNR;
5114     }
5115 
5116     if (host_events) {
5117         ebuf = &endgrp->fdp.host_events;
5118     } else {
5119         ebuf = &endgrp->fdp.ctrl_events;
5120     }
5121 
5122     log_size = sizeof(NvmeFdpEventsLog) + ebuf->nelems * sizeof(NvmeFdpEvent);
5123 
5124     if (off >= log_size) {
5125         return NVME_INVALID_FIELD | NVME_DNR;
5126     }
5127 
5128     trans_len = MIN(log_size - off, buf_len);
5129     elog = g_malloc0(log_size);
5130     elog->num_events = cpu_to_le32(ebuf->nelems);
5131     event = (NvmeFdpEvent *)(elog + 1);
5132 
5133     if (ebuf->nelems && ebuf->start == ebuf->next) {
5134         unsigned int nelems = (NVME_FDP_MAX_EVENTS - ebuf->start);
5135         /* wrap over, copy [start;NVME_FDP_MAX_EVENTS[ and [0; next[ */
5136         memcpy(event, &ebuf->events[ebuf->start],
5137                sizeof(NvmeFdpEvent) * nelems);
5138         memcpy(event + nelems, ebuf->events,
5139                sizeof(NvmeFdpEvent) * ebuf->next);
5140     } else if (ebuf->start < ebuf->next) {
5141         memcpy(event, &ebuf->events[ebuf->start],
5142                sizeof(NvmeFdpEvent) * (ebuf->next - ebuf->start));
5143     }
5144 
5145     return nvme_c2h(n, (uint8_t *)elog + off, trans_len, req);
5146 }
5147 
5148 static uint16_t nvme_get_log(NvmeCtrl *n, NvmeRequest *req)
5149 {
5150     NvmeCmd *cmd = &req->cmd;
5151 
5152     uint32_t dw10 = le32_to_cpu(cmd->cdw10);
5153     uint32_t dw11 = le32_to_cpu(cmd->cdw11);
5154     uint32_t dw12 = le32_to_cpu(cmd->cdw12);
5155     uint32_t dw13 = le32_to_cpu(cmd->cdw13);
5156     uint8_t  lid = dw10 & 0xff;
5157     uint8_t  lsp = (dw10 >> 8) & 0xf;
5158     uint8_t  rae = (dw10 >> 15) & 0x1;
5159     uint8_t  csi = le32_to_cpu(cmd->cdw14) >> 24;
5160     uint32_t numdl, numdu, lspi;
5161     uint64_t off, lpol, lpou;
5162     size_t   len;
5163     uint16_t status;
5164 
5165     numdl = (dw10 >> 16);
5166     numdu = (dw11 & 0xffff);
5167     lspi = (dw11 >> 16);
5168     lpol = dw12;
5169     lpou = dw13;
5170 
5171     len = (((numdu << 16) | numdl) + 1) << 2;
5172     off = (lpou << 32ULL) | lpol;
5173 
5174     if (off & 0x3) {
5175         return NVME_INVALID_FIELD | NVME_DNR;
5176     }
5177 
5178     trace_pci_nvme_get_log(nvme_cid(req), lid, lsp, rae, len, off);
5179 
5180     status = nvme_check_mdts(n, len);
5181     if (status) {
5182         return status;
5183     }
5184 
5185     switch (lid) {
5186     case NVME_LOG_ERROR_INFO:
5187         return nvme_error_info(n, rae, len, off, req);
5188     case NVME_LOG_SMART_INFO:
5189         return nvme_smart_info(n, rae, len, off, req);
5190     case NVME_LOG_FW_SLOT_INFO:
5191         return nvme_fw_log_info(n, len, off, req);
5192     case NVME_LOG_CHANGED_NSLIST:
5193         return nvme_changed_nslist(n, rae, len, off, req);
5194     case NVME_LOG_CMD_EFFECTS:
5195         return nvme_cmd_effects(n, csi, len, off, req);
5196     case NVME_LOG_ENDGRP:
5197         return nvme_endgrp_info(n, rae, len, off, req);
5198     case NVME_LOG_FDP_CONFS:
5199         return nvme_fdp_confs(n, lspi, len, off, req);
5200     case NVME_LOG_FDP_RUH_USAGE:
5201         return nvme_fdp_ruh_usage(n, lspi, dw10, dw12, len, off, req);
5202     case NVME_LOG_FDP_STATS:
5203         return nvme_fdp_stats(n, lspi, len, off, req);
5204     case NVME_LOG_FDP_EVENTS:
5205         return nvme_fdp_events(n, lspi, len, off, req);
5206     default:
5207         trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid);
5208         return NVME_INVALID_FIELD | NVME_DNR;
5209     }
5210 }
5211 
5212 static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
5213 {
5214     PCIDevice *pci = PCI_DEVICE(n);
5215     uint16_t offset = (cq->cqid << 3) + (1 << 2);
5216 
5217     n->cq[cq->cqid] = NULL;
5218     qemu_bh_delete(cq->bh);
5219     if (cq->ioeventfd_enabled) {
5220         memory_region_del_eventfd(&n->iomem,
5221                                   0x1000 + offset, 4, false, 0, &cq->notifier);
5222         event_notifier_set_handler(&cq->notifier, NULL);
5223         event_notifier_cleanup(&cq->notifier);
5224     }
5225     if (msix_enabled(pci)) {
5226         msix_vector_unuse(pci, cq->vector);
5227     }
5228     if (cq->cqid) {
5229         g_free(cq);
5230     }
5231 }
5232 
5233 static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
5234 {
5235     NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
5236     NvmeCQueue *cq;
5237     uint16_t qid = le16_to_cpu(c->qid);
5238 
5239     if (unlikely(!qid || nvme_check_cqid(n, qid))) {
5240         trace_pci_nvme_err_invalid_del_cq_cqid(qid);
5241         return NVME_INVALID_CQID | NVME_DNR;
5242     }
5243 
5244     cq = n->cq[qid];
5245     if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
5246         trace_pci_nvme_err_invalid_del_cq_notempty(qid);
5247         return NVME_INVALID_QUEUE_DEL;
5248     }
5249 
5250     if (cq->irq_enabled && cq->tail != cq->head) {
5251         n->cq_pending--;
5252     }
5253 
5254     nvme_irq_deassert(n, cq);
5255     trace_pci_nvme_del_cq(qid);
5256     nvme_free_cq(cq, n);
5257     return NVME_SUCCESS;
5258 }
5259 
5260 static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
5261                          uint16_t cqid, uint16_t vector, uint16_t size,
5262                          uint16_t irq_enabled)
5263 {
5264     PCIDevice *pci = PCI_DEVICE(n);
5265 
5266     if (msix_enabled(pci)) {
5267         msix_vector_use(pci, vector);
5268     }
5269     cq->ctrl = n;
5270     cq->cqid = cqid;
5271     cq->size = size;
5272     cq->dma_addr = dma_addr;
5273     cq->phase = 1;
5274     cq->irq_enabled = irq_enabled;
5275     cq->vector = vector;
5276     cq->head = cq->tail = 0;
5277     QTAILQ_INIT(&cq->req_list);
5278     QTAILQ_INIT(&cq->sq_list);
5279     if (n->dbbuf_enabled) {
5280         cq->db_addr = n->dbbuf_dbs + (cqid << 3) + (1 << 2);
5281         cq->ei_addr = n->dbbuf_eis + (cqid << 3) + (1 << 2);
5282 
5283         if (n->params.ioeventfd && cqid != 0) {
5284             if (!nvme_init_cq_ioeventfd(cq)) {
5285                 cq->ioeventfd_enabled = true;
5286             }
5287         }
5288     }
5289     n->cq[cqid] = cq;
5290     cq->bh = qemu_bh_new_guarded(nvme_post_cqes, cq,
5291                                  &DEVICE(cq->ctrl)->mem_reentrancy_guard);
5292 }
5293 
5294 static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeRequest *req)
5295 {
5296     NvmeCQueue *cq;
5297     NvmeCreateCq *c = (NvmeCreateCq *)&req->cmd;
5298     uint16_t cqid = le16_to_cpu(c->cqid);
5299     uint16_t vector = le16_to_cpu(c->irq_vector);
5300     uint16_t qsize = le16_to_cpu(c->qsize);
5301     uint16_t qflags = le16_to_cpu(c->cq_flags);
5302     uint64_t prp1 = le64_to_cpu(c->prp1);
5303     uint32_t cc = ldq_le_p(&n->bar.cc);
5304     uint8_t iocqes = NVME_CC_IOCQES(cc);
5305     uint8_t iosqes = NVME_CC_IOSQES(cc);
5306 
5307     trace_pci_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
5308                              NVME_CQ_FLAGS_IEN(qflags) != 0);
5309 
5310     if (iosqes != NVME_SQES || iocqes != NVME_CQES) {
5311         trace_pci_nvme_err_invalid_create_cq_entry_size(iosqes, iocqes);
5312         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
5313     }
5314 
5315     if (unlikely(!cqid || cqid > n->conf_ioqpairs || n->cq[cqid] != NULL)) {
5316         trace_pci_nvme_err_invalid_create_cq_cqid(cqid);
5317         return NVME_INVALID_QID | NVME_DNR;
5318     }
5319     if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
5320         trace_pci_nvme_err_invalid_create_cq_size(qsize);
5321         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
5322     }
5323     if (unlikely(prp1 & (n->page_size - 1))) {
5324         trace_pci_nvme_err_invalid_create_cq_addr(prp1);
5325         return NVME_INVALID_PRP_OFFSET | NVME_DNR;
5326     }
5327     if (unlikely(!msix_enabled(PCI_DEVICE(n)) && vector)) {
5328         trace_pci_nvme_err_invalid_create_cq_vector(vector);
5329         return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5330     }
5331     if (unlikely(vector >= n->conf_msix_qsize)) {
5332         trace_pci_nvme_err_invalid_create_cq_vector(vector);
5333         return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5334     }
5335     if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
5336         trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
5337         return NVME_INVALID_FIELD | NVME_DNR;
5338     }
5339 
5340     cq = g_malloc0(sizeof(*cq));
5341     nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
5342                  NVME_CQ_FLAGS_IEN(qflags));
5343 
5344     /*
5345      * It is only required to set qs_created when creating a completion queue;
5346      * creating a submission queue without a matching completion queue will
5347      * fail.
5348      */
5349     n->qs_created = true;
5350     return NVME_SUCCESS;
5351 }
5352 
5353 static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl *n, NvmeRequest *req)
5354 {
5355     uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5356 
5357     return nvme_c2h(n, id, sizeof(id), req);
5358 }
5359 
5360 static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeRequest *req)
5361 {
5362     trace_pci_nvme_identify_ctrl();
5363 
5364     return nvme_c2h(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl), req);
5365 }
5366 
5367 static uint16_t nvme_identify_ctrl_csi(NvmeCtrl *n, NvmeRequest *req)
5368 {
5369     NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5370     uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5371     NvmeIdCtrlNvm *id_nvm = (NvmeIdCtrlNvm *)&id;
5372 
5373     trace_pci_nvme_identify_ctrl_csi(c->csi);
5374 
5375     switch (c->csi) {
5376     case NVME_CSI_NVM:
5377         id_nvm->vsl = n->params.vsl;
5378         id_nvm->dmrsl = cpu_to_le32(n->dmrsl);
5379         break;
5380 
5381     case NVME_CSI_ZONED:
5382         ((NvmeIdCtrlZoned *)&id)->zasl = n->params.zasl;
5383         break;
5384 
5385     default:
5386         return NVME_INVALID_FIELD | NVME_DNR;
5387     }
5388 
5389     return nvme_c2h(n, id, sizeof(id), req);
5390 }
5391 
5392 static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeRequest *req, bool active)
5393 {
5394     NvmeNamespace *ns;
5395     NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5396     uint32_t nsid = le32_to_cpu(c->nsid);
5397 
5398     trace_pci_nvme_identify_ns(nsid);
5399 
5400     if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5401         return NVME_INVALID_NSID | NVME_DNR;
5402     }
5403 
5404     ns = nvme_ns(n, nsid);
5405     if (unlikely(!ns)) {
5406         if (!active) {
5407             ns = nvme_subsys_ns(n->subsys, nsid);
5408             if (!ns) {
5409                 return nvme_rpt_empty_id_struct(n, req);
5410             }
5411         } else {
5412             return nvme_rpt_empty_id_struct(n, req);
5413         }
5414     }
5415 
5416     if (active || ns->csi == NVME_CSI_NVM) {
5417         return nvme_c2h(n, (uint8_t *)&ns->id_ns, sizeof(NvmeIdNs), req);
5418     }
5419 
5420     return NVME_INVALID_CMD_SET | NVME_DNR;
5421 }
5422 
5423 static uint16_t nvme_identify_ctrl_list(NvmeCtrl *n, NvmeRequest *req,
5424                                         bool attached)
5425 {
5426     NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5427     uint32_t nsid = le32_to_cpu(c->nsid);
5428     uint16_t min_id = le16_to_cpu(c->ctrlid);
5429     uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
5430     uint16_t *ids = &list[1];
5431     NvmeNamespace *ns;
5432     NvmeCtrl *ctrl;
5433     int cntlid, nr_ids = 0;
5434 
5435     trace_pci_nvme_identify_ctrl_list(c->cns, min_id);
5436 
5437     if (!n->subsys) {
5438         return NVME_INVALID_FIELD | NVME_DNR;
5439     }
5440 
5441     if (attached) {
5442         if (nsid == NVME_NSID_BROADCAST) {
5443             return NVME_INVALID_FIELD | NVME_DNR;
5444         }
5445 
5446         ns = nvme_subsys_ns(n->subsys, nsid);
5447         if (!ns) {
5448             return NVME_INVALID_FIELD | NVME_DNR;
5449         }
5450     }
5451 
5452     for (cntlid = min_id; cntlid < ARRAY_SIZE(n->subsys->ctrls); cntlid++) {
5453         ctrl = nvme_subsys_ctrl(n->subsys, cntlid);
5454         if (!ctrl) {
5455             continue;
5456         }
5457 
5458         if (attached && !nvme_ns(ctrl, nsid)) {
5459             continue;
5460         }
5461 
5462         ids[nr_ids++] = cntlid;
5463     }
5464 
5465     list[0] = nr_ids;
5466 
5467     return nvme_c2h(n, (uint8_t *)list, sizeof(list), req);
5468 }
5469 
5470 static uint16_t nvme_identify_pri_ctrl_cap(NvmeCtrl *n, NvmeRequest *req)
5471 {
5472     trace_pci_nvme_identify_pri_ctrl_cap(le16_to_cpu(n->pri_ctrl_cap.cntlid));
5473 
5474     return nvme_c2h(n, (uint8_t *)&n->pri_ctrl_cap,
5475                     sizeof(NvmePriCtrlCap), req);
5476 }
5477 
5478 static uint16_t nvme_identify_sec_ctrl_list(NvmeCtrl *n, NvmeRequest *req)
5479 {
5480     NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5481     uint16_t pri_ctrl_id = le16_to_cpu(n->pri_ctrl_cap.cntlid);
5482     uint16_t min_id = le16_to_cpu(c->ctrlid);
5483     uint8_t num_sec_ctrl = n->sec_ctrl_list.numcntl;
5484     NvmeSecCtrlList list = {0};
5485     uint8_t i;
5486 
5487     for (i = 0; i < num_sec_ctrl; i++) {
5488         if (n->sec_ctrl_list.sec[i].scid >= min_id) {
5489             list.numcntl = num_sec_ctrl - i;
5490             memcpy(&list.sec, n->sec_ctrl_list.sec + i,
5491                    list.numcntl * sizeof(NvmeSecCtrlEntry));
5492             break;
5493         }
5494     }
5495 
5496     trace_pci_nvme_identify_sec_ctrl_list(pri_ctrl_id, list.numcntl);
5497 
5498     return nvme_c2h(n, (uint8_t *)&list, sizeof(list), req);
5499 }
5500 
5501 static uint16_t nvme_identify_ns_csi(NvmeCtrl *n, NvmeRequest *req,
5502                                      bool active)
5503 {
5504     NvmeNamespace *ns;
5505     NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5506     uint32_t nsid = le32_to_cpu(c->nsid);
5507 
5508     trace_pci_nvme_identify_ns_csi(nsid, c->csi);
5509 
5510     if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5511         return NVME_INVALID_NSID | NVME_DNR;
5512     }
5513 
5514     ns = nvme_ns(n, nsid);
5515     if (unlikely(!ns)) {
5516         if (!active) {
5517             ns = nvme_subsys_ns(n->subsys, nsid);
5518             if (!ns) {
5519                 return nvme_rpt_empty_id_struct(n, req);
5520             }
5521         } else {
5522             return nvme_rpt_empty_id_struct(n, req);
5523         }
5524     }
5525 
5526     if (c->csi == NVME_CSI_NVM) {
5527         return nvme_c2h(n, (uint8_t *)&ns->id_ns_nvm, sizeof(NvmeIdNsNvm),
5528                         req);
5529     } else if (c->csi == NVME_CSI_ZONED && ns->csi == NVME_CSI_ZONED) {
5530         return nvme_c2h(n, (uint8_t *)ns->id_ns_zoned, sizeof(NvmeIdNsZoned),
5531                         req);
5532     }
5533 
5534     return NVME_INVALID_FIELD | NVME_DNR;
5535 }
5536 
5537 static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req,
5538                                      bool active)
5539 {
5540     NvmeNamespace *ns;
5541     NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5542     uint32_t min_nsid = le32_to_cpu(c->nsid);
5543     uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5544     static const int data_len = sizeof(list);
5545     uint32_t *list_ptr = (uint32_t *)list;
5546     int i, j = 0;
5547 
5548     trace_pci_nvme_identify_nslist(min_nsid);
5549 
5550     /*
5551      * Both FFFFFFFFh (NVME_NSID_BROADCAST) and FFFFFFFFEh are invalid values
5552      * since the Active Namespace ID List should return namespaces with ids
5553      * *higher* than the NSID specified in the command. This is also specified
5554      * in the spec (NVM Express v1.3d, Section 5.15.4).
5555      */
5556     if (min_nsid >= NVME_NSID_BROADCAST - 1) {
5557         return NVME_INVALID_NSID | NVME_DNR;
5558     }
5559 
5560     for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5561         ns = nvme_ns(n, i);
5562         if (!ns) {
5563             if (!active) {
5564                 ns = nvme_subsys_ns(n->subsys, i);
5565                 if (!ns) {
5566                     continue;
5567                 }
5568             } else {
5569                 continue;
5570             }
5571         }
5572         if (ns->params.nsid <= min_nsid) {
5573             continue;
5574         }
5575         list_ptr[j++] = cpu_to_le32(ns->params.nsid);
5576         if (j == data_len / sizeof(uint32_t)) {
5577             break;
5578         }
5579     }
5580 
5581     return nvme_c2h(n, list, data_len, req);
5582 }
5583 
5584 static uint16_t nvme_identify_nslist_csi(NvmeCtrl *n, NvmeRequest *req,
5585                                          bool active)
5586 {
5587     NvmeNamespace *ns;
5588     NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5589     uint32_t min_nsid = le32_to_cpu(c->nsid);
5590     uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5591     static const int data_len = sizeof(list);
5592     uint32_t *list_ptr = (uint32_t *)list;
5593     int i, j = 0;
5594 
5595     trace_pci_nvme_identify_nslist_csi(min_nsid, c->csi);
5596 
5597     /*
5598      * Same as in nvme_identify_nslist(), FFFFFFFFh/FFFFFFFFEh are invalid.
5599      */
5600     if (min_nsid >= NVME_NSID_BROADCAST - 1) {
5601         return NVME_INVALID_NSID | NVME_DNR;
5602     }
5603 
5604     if (c->csi != NVME_CSI_NVM && c->csi != NVME_CSI_ZONED) {
5605         return NVME_INVALID_FIELD | NVME_DNR;
5606     }
5607 
5608     for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5609         ns = nvme_ns(n, i);
5610         if (!ns) {
5611             if (!active) {
5612                 ns = nvme_subsys_ns(n->subsys, i);
5613                 if (!ns) {
5614                     continue;
5615                 }
5616             } else {
5617                 continue;
5618             }
5619         }
5620         if (ns->params.nsid <= min_nsid || c->csi != ns->csi) {
5621             continue;
5622         }
5623         list_ptr[j++] = cpu_to_le32(ns->params.nsid);
5624         if (j == data_len / sizeof(uint32_t)) {
5625             break;
5626         }
5627     }
5628 
5629     return nvme_c2h(n, list, data_len, req);
5630 }
5631 
5632 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeRequest *req)
5633 {
5634     NvmeNamespace *ns;
5635     NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5636     uint32_t nsid = le32_to_cpu(c->nsid);
5637     uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5638     uint8_t *pos = list;
5639     struct {
5640         NvmeIdNsDescr hdr;
5641         uint8_t v[NVME_NIDL_UUID];
5642     } QEMU_PACKED uuid = {};
5643     struct {
5644         NvmeIdNsDescr hdr;
5645         uint64_t v;
5646     } QEMU_PACKED eui64 = {};
5647     struct {
5648         NvmeIdNsDescr hdr;
5649         uint8_t v;
5650     } QEMU_PACKED csi = {};
5651 
5652     trace_pci_nvme_identify_ns_descr_list(nsid);
5653 
5654     if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5655         return NVME_INVALID_NSID | NVME_DNR;
5656     }
5657 
5658     ns = nvme_ns(n, nsid);
5659     if (unlikely(!ns)) {
5660         return NVME_INVALID_FIELD | NVME_DNR;
5661     }
5662 
5663     if (!qemu_uuid_is_null(&ns->params.uuid)) {
5664         uuid.hdr.nidt = NVME_NIDT_UUID;
5665         uuid.hdr.nidl = NVME_NIDL_UUID;
5666         memcpy(uuid.v, ns->params.uuid.data, NVME_NIDL_UUID);
5667         memcpy(pos, &uuid, sizeof(uuid));
5668         pos += sizeof(uuid);
5669     }
5670 
5671     if (ns->params.eui64) {
5672         eui64.hdr.nidt = NVME_NIDT_EUI64;
5673         eui64.hdr.nidl = NVME_NIDL_EUI64;
5674         eui64.v = cpu_to_be64(ns->params.eui64);
5675         memcpy(pos, &eui64, sizeof(eui64));
5676         pos += sizeof(eui64);
5677     }
5678 
5679     csi.hdr.nidt = NVME_NIDT_CSI;
5680     csi.hdr.nidl = NVME_NIDL_CSI;
5681     csi.v = ns->csi;
5682     memcpy(pos, &csi, sizeof(csi));
5683     pos += sizeof(csi);
5684 
5685     return nvme_c2h(n, list, sizeof(list), req);
5686 }
5687 
5688 static uint16_t nvme_identify_cmd_set(NvmeCtrl *n, NvmeRequest *req)
5689 {
5690     uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5691     static const int data_len = sizeof(list);
5692 
5693     trace_pci_nvme_identify_cmd_set();
5694 
5695     NVME_SET_CSI(*list, NVME_CSI_NVM);
5696     NVME_SET_CSI(*list, NVME_CSI_ZONED);
5697 
5698     return nvme_c2h(n, list, data_len, req);
5699 }
5700 
5701 static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req)
5702 {
5703     NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5704 
5705     trace_pci_nvme_identify(nvme_cid(req), c->cns, le16_to_cpu(c->ctrlid),
5706                             c->csi);
5707 
5708     switch (c->cns) {
5709     case NVME_ID_CNS_NS:
5710         return nvme_identify_ns(n, req, true);
5711     case NVME_ID_CNS_NS_PRESENT:
5712         return nvme_identify_ns(n, req, false);
5713     case NVME_ID_CNS_NS_ATTACHED_CTRL_LIST:
5714         return nvme_identify_ctrl_list(n, req, true);
5715     case NVME_ID_CNS_CTRL_LIST:
5716         return nvme_identify_ctrl_list(n, req, false);
5717     case NVME_ID_CNS_PRIMARY_CTRL_CAP:
5718         return nvme_identify_pri_ctrl_cap(n, req);
5719     case NVME_ID_CNS_SECONDARY_CTRL_LIST:
5720         return nvme_identify_sec_ctrl_list(n, req);
5721     case NVME_ID_CNS_CS_NS:
5722         return nvme_identify_ns_csi(n, req, true);
5723     case NVME_ID_CNS_CS_NS_PRESENT:
5724         return nvme_identify_ns_csi(n, req, false);
5725     case NVME_ID_CNS_CTRL:
5726         return nvme_identify_ctrl(n, req);
5727     case NVME_ID_CNS_CS_CTRL:
5728         return nvme_identify_ctrl_csi(n, req);
5729     case NVME_ID_CNS_NS_ACTIVE_LIST:
5730         return nvme_identify_nslist(n, req, true);
5731     case NVME_ID_CNS_NS_PRESENT_LIST:
5732         return nvme_identify_nslist(n, req, false);
5733     case NVME_ID_CNS_CS_NS_ACTIVE_LIST:
5734         return nvme_identify_nslist_csi(n, req, true);
5735     case NVME_ID_CNS_CS_NS_PRESENT_LIST:
5736         return nvme_identify_nslist_csi(n, req, false);
5737     case NVME_ID_CNS_NS_DESCR_LIST:
5738         return nvme_identify_ns_descr_list(n, req);
5739     case NVME_ID_CNS_IO_COMMAND_SET:
5740         return nvme_identify_cmd_set(n, req);
5741     default:
5742         trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
5743         return NVME_INVALID_FIELD | NVME_DNR;
5744     }
5745 }
5746 
5747 static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req)
5748 {
5749     uint16_t sqid = le32_to_cpu(req->cmd.cdw10) & 0xffff;
5750 
5751     req->cqe.result = 1;
5752     if (nvme_check_sqid(n, sqid)) {
5753         return NVME_INVALID_FIELD | NVME_DNR;
5754     }
5755 
5756     return NVME_SUCCESS;
5757 }
5758 
5759 static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts)
5760 {
5761     trace_pci_nvme_setfeat_timestamp(ts);
5762 
5763     n->host_timestamp = le64_to_cpu(ts);
5764     n->timestamp_set_qemu_clock_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
5765 }
5766 
5767 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)
5768 {
5769     uint64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
5770     uint64_t elapsed_time = current_time - n->timestamp_set_qemu_clock_ms;
5771 
5772     union nvme_timestamp {
5773         struct {
5774             uint64_t timestamp:48;
5775             uint64_t sync:1;
5776             uint64_t origin:3;
5777             uint64_t rsvd1:12;
5778         };
5779         uint64_t all;
5780     };
5781 
5782     union nvme_timestamp ts;
5783     ts.all = 0;
5784     ts.timestamp = n->host_timestamp + elapsed_time;
5785 
5786     /* If the host timestamp is non-zero, set the timestamp origin */
5787     ts.origin = n->host_timestamp ? 0x01 : 0x00;
5788 
5789     trace_pci_nvme_getfeat_timestamp(ts.all);
5790 
5791     return cpu_to_le64(ts.all);
5792 }
5793 
5794 static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
5795 {
5796     uint64_t timestamp = nvme_get_timestamp(n);
5797 
5798     return nvme_c2h(n, (uint8_t *)&timestamp, sizeof(timestamp), req);
5799 }
5800 
5801 static int nvme_get_feature_fdp(NvmeCtrl *n, uint32_t endgrpid,
5802                                 uint32_t *result)
5803 {
5804     *result = 0;
5805 
5806     if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
5807         return NVME_INVALID_FIELD | NVME_DNR;
5808     }
5809 
5810     *result = FIELD_DP16(0, FEAT_FDP, FDPE, 1);
5811     *result = FIELD_DP16(*result, FEAT_FDP, CONF_NDX, 0);
5812 
5813     return NVME_SUCCESS;
5814 }
5815 
5816 static uint16_t nvme_get_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
5817                                             NvmeRequest *req, uint32_t *result)
5818 {
5819     NvmeCmd *cmd = &req->cmd;
5820     uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
5821     uint16_t ph = cdw11 & 0xffff;
5822     uint8_t noet = (cdw11 >> 16) & 0xff;
5823     uint16_t ruhid, ret;
5824     uint32_t nentries = 0;
5825     uint8_t s_events_ndx = 0;
5826     size_t s_events_siz = sizeof(NvmeFdpEventDescr) * noet;
5827     g_autofree NvmeFdpEventDescr *s_events = g_malloc0(s_events_siz);
5828     NvmeRuHandle *ruh;
5829     NvmeFdpEventDescr *s_event;
5830 
5831     if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
5832         return NVME_FDP_DISABLED | NVME_DNR;
5833     }
5834 
5835     if (!nvme_ph_valid(ns, ph)) {
5836         return NVME_INVALID_FIELD | NVME_DNR;
5837     }
5838 
5839     ruhid = ns->fdp.phs[ph];
5840     ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
5841 
5842     assert(ruh);
5843 
5844     if (unlikely(noet == 0)) {
5845         return NVME_INVALID_FIELD | NVME_DNR;
5846     }
5847 
5848     for (uint8_t event_type = 0; event_type < FDP_EVT_MAX; event_type++) {
5849         uint8_t shift = nvme_fdp_evf_shifts[event_type];
5850         if (!shift && event_type) {
5851             /*
5852              * only first entry (event_type == 0) has a shift value of 0
5853              * other entries are simply unpopulated.
5854              */
5855             continue;
5856         }
5857 
5858         nentries++;
5859 
5860         s_event = &s_events[s_events_ndx];
5861         s_event->evt = event_type;
5862         s_event->evta = (ruh->event_filter >> shift) & 0x1;
5863 
5864         /* break if all `noet` entries are filled */
5865         if ((++s_events_ndx) == noet) {
5866             break;
5867         }
5868     }
5869 
5870     ret = nvme_c2h(n, s_events, s_events_siz, req);
5871     if (ret) {
5872         return ret;
5873     }
5874 
5875     *result = nentries;
5876     return NVME_SUCCESS;
5877 }
5878 
5879 static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req)
5880 {
5881     NvmeCmd *cmd = &req->cmd;
5882     uint32_t dw10 = le32_to_cpu(cmd->cdw10);
5883     uint32_t dw11 = le32_to_cpu(cmd->cdw11);
5884     uint32_t nsid = le32_to_cpu(cmd->nsid);
5885     uint32_t result;
5886     uint8_t fid = NVME_GETSETFEAT_FID(dw10);
5887     NvmeGetFeatureSelect sel = NVME_GETFEAT_SELECT(dw10);
5888     uint16_t iv;
5889     NvmeNamespace *ns;
5890     int i;
5891     uint16_t endgrpid = 0, ret = NVME_SUCCESS;
5892 
5893     static const uint32_t nvme_feature_default[NVME_FID_MAX] = {
5894         [NVME_ARBITRATION] = NVME_ARB_AB_NOLIMIT,
5895     };
5896 
5897     trace_pci_nvme_getfeat(nvme_cid(req), nsid, fid, sel, dw11);
5898 
5899     if (!nvme_feature_support[fid]) {
5900         return NVME_INVALID_FIELD | NVME_DNR;
5901     }
5902 
5903     if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
5904         if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5905             /*
5906              * The Reservation Notification Mask and Reservation Persistence
5907              * features require a status code of Invalid Field in Command when
5908              * NSID is FFFFFFFFh. Since the device does not support those
5909              * features we can always return Invalid Namespace or Format as we
5910              * should do for all other features.
5911              */
5912             return NVME_INVALID_NSID | NVME_DNR;
5913         }
5914 
5915         if (!nvme_ns(n, nsid)) {
5916             return NVME_INVALID_FIELD | NVME_DNR;
5917         }
5918     }
5919 
5920     switch (sel) {
5921     case NVME_GETFEAT_SELECT_CURRENT:
5922         break;
5923     case NVME_GETFEAT_SELECT_SAVED:
5924         /* no features are saveable by the controller; fallthrough */
5925     case NVME_GETFEAT_SELECT_DEFAULT:
5926         goto defaults;
5927     case NVME_GETFEAT_SELECT_CAP:
5928         result = nvme_feature_cap[fid];
5929         goto out;
5930     }
5931 
5932     switch (fid) {
5933     case NVME_TEMPERATURE_THRESHOLD:
5934         result = 0;
5935 
5936         /*
5937          * The controller only implements the Composite Temperature sensor, so
5938          * return 0 for all other sensors.
5939          */
5940         if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
5941             goto out;
5942         }
5943 
5944         switch (NVME_TEMP_THSEL(dw11)) {
5945         case NVME_TEMP_THSEL_OVER:
5946             result = n->features.temp_thresh_hi;
5947             goto out;
5948         case NVME_TEMP_THSEL_UNDER:
5949             result = n->features.temp_thresh_low;
5950             goto out;
5951         }
5952 
5953         return NVME_INVALID_FIELD | NVME_DNR;
5954     case NVME_ERROR_RECOVERY:
5955         if (!nvme_nsid_valid(n, nsid)) {
5956             return NVME_INVALID_NSID | NVME_DNR;
5957         }
5958 
5959         ns = nvme_ns(n, nsid);
5960         if (unlikely(!ns)) {
5961             return NVME_INVALID_FIELD | NVME_DNR;
5962         }
5963 
5964         result = ns->features.err_rec;
5965         goto out;
5966     case NVME_VOLATILE_WRITE_CACHE:
5967         result = 0;
5968         for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5969             ns = nvme_ns(n, i);
5970             if (!ns) {
5971                 continue;
5972             }
5973 
5974             result = blk_enable_write_cache(ns->blkconf.blk);
5975             if (result) {
5976                 break;
5977             }
5978         }
5979         trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
5980         goto out;
5981     case NVME_ASYNCHRONOUS_EVENT_CONF:
5982         result = n->features.async_config;
5983         goto out;
5984     case NVME_TIMESTAMP:
5985         return nvme_get_feature_timestamp(n, req);
5986     case NVME_HOST_BEHAVIOR_SUPPORT:
5987         return nvme_c2h(n, (uint8_t *)&n->features.hbs,
5988                         sizeof(n->features.hbs), req);
5989     case NVME_FDP_MODE:
5990         endgrpid = dw11 & 0xff;
5991 
5992         if (endgrpid != 0x1) {
5993             return NVME_INVALID_FIELD | NVME_DNR;
5994         }
5995 
5996         ret = nvme_get_feature_fdp(n, endgrpid, &result);
5997         if (ret) {
5998             return ret;
5999         }
6000         goto out;
6001     case NVME_FDP_EVENTS:
6002         if (!nvme_nsid_valid(n, nsid)) {
6003             return NVME_INVALID_NSID | NVME_DNR;
6004         }
6005 
6006         ns = nvme_ns(n, nsid);
6007         if (unlikely(!ns)) {
6008             return NVME_INVALID_FIELD | NVME_DNR;
6009         }
6010 
6011         ret = nvme_get_feature_fdp_events(n, ns, req, &result);
6012         if (ret) {
6013             return ret;
6014         }
6015         goto out;
6016     default:
6017         break;
6018     }
6019 
6020 defaults:
6021     switch (fid) {
6022     case NVME_TEMPERATURE_THRESHOLD:
6023         result = 0;
6024 
6025         if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6026             break;
6027         }
6028 
6029         if (NVME_TEMP_THSEL(dw11) == NVME_TEMP_THSEL_OVER) {
6030             result = NVME_TEMPERATURE_WARNING;
6031         }
6032 
6033         break;
6034     case NVME_NUMBER_OF_QUEUES:
6035         result = (n->conf_ioqpairs - 1) | ((n->conf_ioqpairs - 1) << 16);
6036         trace_pci_nvme_getfeat_numq(result);
6037         break;
6038     case NVME_INTERRUPT_VECTOR_CONF:
6039         iv = dw11 & 0xffff;
6040         if (iv >= n->conf_ioqpairs + 1) {
6041             return NVME_INVALID_FIELD | NVME_DNR;
6042         }
6043 
6044         result = iv;
6045         if (iv == n->admin_cq.vector) {
6046             result |= NVME_INTVC_NOCOALESCING;
6047         }
6048         break;
6049     case NVME_FDP_MODE:
6050         endgrpid = dw11 & 0xff;
6051 
6052         if (endgrpid != 0x1) {
6053             return NVME_INVALID_FIELD | NVME_DNR;
6054         }
6055 
6056         ret = nvme_get_feature_fdp(n, endgrpid, &result);
6057         if (ret) {
6058             return ret;
6059         }
6060         goto out;
6061 
6062         break;
6063     default:
6064         result = nvme_feature_default[fid];
6065         break;
6066     }
6067 
6068 out:
6069     req->cqe.result = cpu_to_le32(result);
6070     return ret;
6071 }
6072 
6073 static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
6074 {
6075     uint16_t ret;
6076     uint64_t timestamp;
6077 
6078     ret = nvme_h2c(n, (uint8_t *)&timestamp, sizeof(timestamp), req);
6079     if (ret) {
6080         return ret;
6081     }
6082 
6083     nvme_set_timestamp(n, timestamp);
6084 
6085     return NVME_SUCCESS;
6086 }
6087 
6088 static uint16_t nvme_set_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
6089                                             NvmeRequest *req)
6090 {
6091     NvmeCmd *cmd = &req->cmd;
6092     uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
6093     uint16_t ph = cdw11 & 0xffff;
6094     uint8_t noet = (cdw11 >> 16) & 0xff;
6095     uint16_t ret, ruhid;
6096     uint8_t enable = le32_to_cpu(cmd->cdw12) & 0x1;
6097     uint8_t event_mask = 0;
6098     unsigned int i;
6099     g_autofree uint8_t *events = g_malloc0(noet);
6100     NvmeRuHandle *ruh = NULL;
6101 
6102     assert(ns);
6103 
6104     if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
6105         return NVME_FDP_DISABLED | NVME_DNR;
6106     }
6107 
6108     if (!nvme_ph_valid(ns, ph)) {
6109         return NVME_INVALID_FIELD | NVME_DNR;
6110     }
6111 
6112     ruhid = ns->fdp.phs[ph];
6113     ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
6114 
6115     ret = nvme_h2c(n, events, noet, req);
6116     if (ret) {
6117         return ret;
6118     }
6119 
6120     for (i = 0; i < noet; i++) {
6121         event_mask |= (1 << nvme_fdp_evf_shifts[events[i]]);
6122     }
6123 
6124     if (enable) {
6125         ruh->event_filter |= event_mask;
6126     } else {
6127         ruh->event_filter = ruh->event_filter & ~event_mask;
6128     }
6129 
6130     return NVME_SUCCESS;
6131 }
6132 
6133 static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req)
6134 {
6135     NvmeNamespace *ns = NULL;
6136 
6137     NvmeCmd *cmd = &req->cmd;
6138     uint32_t dw10 = le32_to_cpu(cmd->cdw10);
6139     uint32_t dw11 = le32_to_cpu(cmd->cdw11);
6140     uint32_t nsid = le32_to_cpu(cmd->nsid);
6141     uint8_t fid = NVME_GETSETFEAT_FID(dw10);
6142     uint8_t save = NVME_SETFEAT_SAVE(dw10);
6143     uint16_t status;
6144     int i;
6145 
6146     trace_pci_nvme_setfeat(nvme_cid(req), nsid, fid, save, dw11);
6147 
6148     if (save && !(nvme_feature_cap[fid] & NVME_FEAT_CAP_SAVE)) {
6149         return NVME_FID_NOT_SAVEABLE | NVME_DNR;
6150     }
6151 
6152     if (!nvme_feature_support[fid]) {
6153         return NVME_INVALID_FIELD | NVME_DNR;
6154     }
6155 
6156     if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
6157         if (nsid != NVME_NSID_BROADCAST) {
6158             if (!nvme_nsid_valid(n, nsid)) {
6159                 return NVME_INVALID_NSID | NVME_DNR;
6160             }
6161 
6162             ns = nvme_ns(n, nsid);
6163             if (unlikely(!ns)) {
6164                 return NVME_INVALID_FIELD | NVME_DNR;
6165             }
6166         }
6167     } else if (nsid && nsid != NVME_NSID_BROADCAST) {
6168         if (!nvme_nsid_valid(n, nsid)) {
6169             return NVME_INVALID_NSID | NVME_DNR;
6170         }
6171 
6172         return NVME_FEAT_NOT_NS_SPEC | NVME_DNR;
6173     }
6174 
6175     if (!(nvme_feature_cap[fid] & NVME_FEAT_CAP_CHANGE)) {
6176         return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6177     }
6178 
6179     switch (fid) {
6180     case NVME_TEMPERATURE_THRESHOLD:
6181         if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6182             break;
6183         }
6184 
6185         switch (NVME_TEMP_THSEL(dw11)) {
6186         case NVME_TEMP_THSEL_OVER:
6187             n->features.temp_thresh_hi = NVME_TEMP_TMPTH(dw11);
6188             break;
6189         case NVME_TEMP_THSEL_UNDER:
6190             n->features.temp_thresh_low = NVME_TEMP_TMPTH(dw11);
6191             break;
6192         default:
6193             return NVME_INVALID_FIELD | NVME_DNR;
6194         }
6195 
6196         if ((n->temperature >= n->features.temp_thresh_hi) ||
6197             (n->temperature <= n->features.temp_thresh_low)) {
6198             nvme_smart_event(n, NVME_SMART_TEMPERATURE);
6199         }
6200 
6201         break;
6202     case NVME_ERROR_RECOVERY:
6203         if (nsid == NVME_NSID_BROADCAST) {
6204             for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6205                 ns = nvme_ns(n, i);
6206 
6207                 if (!ns) {
6208                     continue;
6209                 }
6210 
6211                 if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
6212                     ns->features.err_rec = dw11;
6213                 }
6214             }
6215 
6216             break;
6217         }
6218 
6219         assert(ns);
6220         if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat))  {
6221             ns->features.err_rec = dw11;
6222         }
6223         break;
6224     case NVME_VOLATILE_WRITE_CACHE:
6225         for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6226             ns = nvme_ns(n, i);
6227             if (!ns) {
6228                 continue;
6229             }
6230 
6231             if (!(dw11 & 0x1) && blk_enable_write_cache(ns->blkconf.blk)) {
6232                 blk_flush(ns->blkconf.blk);
6233             }
6234 
6235             blk_set_enable_write_cache(ns->blkconf.blk, dw11 & 1);
6236         }
6237 
6238         break;
6239 
6240     case NVME_NUMBER_OF_QUEUES:
6241         if (n->qs_created) {
6242             return NVME_CMD_SEQ_ERROR | NVME_DNR;
6243         }
6244 
6245         /*
6246          * NVMe v1.3, Section 5.21.1.7: FFFFh is not an allowed value for NCQR
6247          * and NSQR.
6248          */
6249         if ((dw11 & 0xffff) == 0xffff || ((dw11 >> 16) & 0xffff) == 0xffff) {
6250             return NVME_INVALID_FIELD | NVME_DNR;
6251         }
6252 
6253         trace_pci_nvme_setfeat_numq((dw11 & 0xffff) + 1,
6254                                     ((dw11 >> 16) & 0xffff) + 1,
6255                                     n->conf_ioqpairs,
6256                                     n->conf_ioqpairs);
6257         req->cqe.result = cpu_to_le32((n->conf_ioqpairs - 1) |
6258                                       ((n->conf_ioqpairs - 1) << 16));
6259         break;
6260     case NVME_ASYNCHRONOUS_EVENT_CONF:
6261         n->features.async_config = dw11;
6262         break;
6263     case NVME_TIMESTAMP:
6264         return nvme_set_feature_timestamp(n, req);
6265     case NVME_HOST_BEHAVIOR_SUPPORT:
6266         status = nvme_h2c(n, (uint8_t *)&n->features.hbs,
6267                           sizeof(n->features.hbs), req);
6268         if (status) {
6269             return status;
6270         }
6271 
6272         for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6273             ns = nvme_ns(n, i);
6274 
6275             if (!ns) {
6276                 continue;
6277             }
6278 
6279             ns->id_ns.nlbaf = ns->nlbaf - 1;
6280             if (!n->features.hbs.lbafee) {
6281                 ns->id_ns.nlbaf = MIN(ns->id_ns.nlbaf, 15);
6282             }
6283         }
6284 
6285         return status;
6286     case NVME_COMMAND_SET_PROFILE:
6287         if (dw11 & 0x1ff) {
6288             trace_pci_nvme_err_invalid_iocsci(dw11 & 0x1ff);
6289             return NVME_CMD_SET_CMB_REJECTED | NVME_DNR;
6290         }
6291         break;
6292     case NVME_FDP_MODE:
6293         /* spec: abort with cmd seq err if there's one or more NS' in endgrp */
6294         return NVME_CMD_SEQ_ERROR | NVME_DNR;
6295     case NVME_FDP_EVENTS:
6296         return nvme_set_feature_fdp_events(n, ns, req);
6297     default:
6298         return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6299     }
6300     return NVME_SUCCESS;
6301 }
6302 
6303 static uint16_t nvme_aer(NvmeCtrl *n, NvmeRequest *req)
6304 {
6305     trace_pci_nvme_aer(nvme_cid(req));
6306 
6307     if (n->outstanding_aers > n->params.aerl) {
6308         trace_pci_nvme_aer_aerl_exceeded();
6309         return NVME_AER_LIMIT_EXCEEDED;
6310     }
6311 
6312     n->aer_reqs[n->outstanding_aers] = req;
6313     n->outstanding_aers++;
6314 
6315     if (!QTAILQ_EMPTY(&n->aer_queue)) {
6316         nvme_process_aers(n);
6317     }
6318 
6319     return NVME_NO_COMPLETE;
6320 }
6321 
6322 static void nvme_update_dmrsl(NvmeCtrl *n)
6323 {
6324     int nsid;
6325 
6326     for (nsid = 1; nsid <= NVME_MAX_NAMESPACES; nsid++) {
6327         NvmeNamespace *ns = nvme_ns(n, nsid);
6328         if (!ns) {
6329             continue;
6330         }
6331 
6332         n->dmrsl = MIN_NON_ZERO(n->dmrsl,
6333                                 BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
6334     }
6335 }
6336 
6337 static void nvme_select_iocs_ns(NvmeCtrl *n, NvmeNamespace *ns)
6338 {
6339     uint32_t cc = ldl_le_p(&n->bar.cc);
6340 
6341     ns->iocs = nvme_cse_iocs_none;
6342     switch (ns->csi) {
6343     case NVME_CSI_NVM:
6344         if (NVME_CC_CSS(cc) != NVME_CC_CSS_ADMIN_ONLY) {
6345             ns->iocs = nvme_cse_iocs_nvm;
6346         }
6347         break;
6348     case NVME_CSI_ZONED:
6349         if (NVME_CC_CSS(cc) == NVME_CC_CSS_CSI) {
6350             ns->iocs = nvme_cse_iocs_zoned;
6351         } else if (NVME_CC_CSS(cc) == NVME_CC_CSS_NVM) {
6352             ns->iocs = nvme_cse_iocs_nvm;
6353         }
6354         break;
6355     }
6356 }
6357 
6358 static uint16_t nvme_ns_attachment(NvmeCtrl *n, NvmeRequest *req)
6359 {
6360     NvmeNamespace *ns;
6361     NvmeCtrl *ctrl;
6362     uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
6363     uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6364     uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6365     uint8_t sel = dw10 & 0xf;
6366     uint16_t *nr_ids = &list[0];
6367     uint16_t *ids = &list[1];
6368     uint16_t ret;
6369     int i;
6370 
6371     trace_pci_nvme_ns_attachment(nvme_cid(req), dw10 & 0xf);
6372 
6373     if (!nvme_nsid_valid(n, nsid)) {
6374         return NVME_INVALID_NSID | NVME_DNR;
6375     }
6376 
6377     ns = nvme_subsys_ns(n->subsys, nsid);
6378     if (!ns) {
6379         return NVME_INVALID_FIELD | NVME_DNR;
6380     }
6381 
6382     ret = nvme_h2c(n, (uint8_t *)list, 4096, req);
6383     if (ret) {
6384         return ret;
6385     }
6386 
6387     if (!*nr_ids) {
6388         return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
6389     }
6390 
6391     *nr_ids = MIN(*nr_ids, NVME_CONTROLLER_LIST_SIZE - 1);
6392     for (i = 0; i < *nr_ids; i++) {
6393         ctrl = nvme_subsys_ctrl(n->subsys, ids[i]);
6394         if (!ctrl) {
6395             return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
6396         }
6397 
6398         switch (sel) {
6399         case NVME_NS_ATTACHMENT_ATTACH:
6400             if (nvme_ns(ctrl, nsid)) {
6401                 return NVME_NS_ALREADY_ATTACHED | NVME_DNR;
6402             }
6403 
6404             if (ns->attached && !ns->params.shared) {
6405                 return NVME_NS_PRIVATE | NVME_DNR;
6406             }
6407 
6408             nvme_attach_ns(ctrl, ns);
6409             nvme_select_iocs_ns(ctrl, ns);
6410 
6411             break;
6412 
6413         case NVME_NS_ATTACHMENT_DETACH:
6414             if (!nvme_ns(ctrl, nsid)) {
6415                 return NVME_NS_NOT_ATTACHED | NVME_DNR;
6416             }
6417 
6418             ctrl->namespaces[nsid] = NULL;
6419             ns->attached--;
6420 
6421             nvme_update_dmrsl(ctrl);
6422 
6423             break;
6424 
6425         default:
6426             return NVME_INVALID_FIELD | NVME_DNR;
6427         }
6428 
6429         /*
6430          * Add namespace id to the changed namespace id list for event clearing
6431          * via Get Log Page command.
6432          */
6433         if (!test_and_set_bit(nsid, ctrl->changed_nsids)) {
6434             nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE,
6435                                NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED,
6436                                NVME_LOG_CHANGED_NSLIST);
6437         }
6438     }
6439 
6440     return NVME_SUCCESS;
6441 }
6442 
6443 typedef struct NvmeFormatAIOCB {
6444     BlockAIOCB common;
6445     BlockAIOCB *aiocb;
6446     NvmeRequest *req;
6447     int ret;
6448 
6449     NvmeNamespace *ns;
6450     uint32_t nsid;
6451     bool broadcast;
6452     int64_t offset;
6453 
6454     uint8_t lbaf;
6455     uint8_t mset;
6456     uint8_t pi;
6457     uint8_t pil;
6458 } NvmeFormatAIOCB;
6459 
6460 static void nvme_format_cancel(BlockAIOCB *aiocb)
6461 {
6462     NvmeFormatAIOCB *iocb = container_of(aiocb, NvmeFormatAIOCB, common);
6463 
6464     iocb->ret = -ECANCELED;
6465 
6466     if (iocb->aiocb) {
6467         blk_aio_cancel_async(iocb->aiocb);
6468         iocb->aiocb = NULL;
6469     }
6470 }
6471 
6472 static const AIOCBInfo nvme_format_aiocb_info = {
6473     .aiocb_size = sizeof(NvmeFormatAIOCB),
6474     .cancel_async = nvme_format_cancel,
6475 };
6476 
6477 static void nvme_format_set(NvmeNamespace *ns, uint8_t lbaf, uint8_t mset,
6478                             uint8_t pi, uint8_t pil)
6479 {
6480     uint8_t lbafl = lbaf & 0xf;
6481     uint8_t lbafu = lbaf >> 4;
6482 
6483     trace_pci_nvme_format_set(ns->params.nsid, lbaf, mset, pi, pil);
6484 
6485     ns->id_ns.dps = (pil << 3) | pi;
6486     ns->id_ns.flbas = (lbafu << 5) | (mset << 4) | lbafl;
6487 
6488     nvme_ns_init_format(ns);
6489 }
6490 
6491 static void nvme_do_format(NvmeFormatAIOCB *iocb);
6492 
6493 static void nvme_format_ns_cb(void *opaque, int ret)
6494 {
6495     NvmeFormatAIOCB *iocb = opaque;
6496     NvmeNamespace *ns = iocb->ns;
6497     int bytes;
6498 
6499     if (iocb->ret < 0) {
6500         goto done;
6501     } else if (ret < 0) {
6502         iocb->ret = ret;
6503         goto done;
6504     }
6505 
6506     assert(ns);
6507 
6508     if (iocb->offset < ns->size) {
6509         bytes = MIN(BDRV_REQUEST_MAX_BYTES, ns->size - iocb->offset);
6510 
6511         iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, iocb->offset,
6512                                             bytes, BDRV_REQ_MAY_UNMAP,
6513                                             nvme_format_ns_cb, iocb);
6514 
6515         iocb->offset += bytes;
6516         return;
6517     }
6518 
6519     nvme_format_set(ns, iocb->lbaf, iocb->mset, iocb->pi, iocb->pil);
6520     ns->status = 0x0;
6521     iocb->ns = NULL;
6522     iocb->offset = 0;
6523 
6524 done:
6525     nvme_do_format(iocb);
6526 }
6527 
6528 static uint16_t nvme_format_check(NvmeNamespace *ns, uint8_t lbaf, uint8_t pi)
6529 {
6530     if (ns->params.zoned) {
6531         return NVME_INVALID_FORMAT | NVME_DNR;
6532     }
6533 
6534     if (lbaf > ns->id_ns.nlbaf) {
6535         return NVME_INVALID_FORMAT | NVME_DNR;
6536     }
6537 
6538     if (pi && (ns->id_ns.lbaf[lbaf].ms < nvme_pi_tuple_size(ns))) {
6539         return NVME_INVALID_FORMAT | NVME_DNR;
6540     }
6541 
6542     if (pi && pi > NVME_ID_NS_DPS_TYPE_3) {
6543         return NVME_INVALID_FIELD | NVME_DNR;
6544     }
6545 
6546     return NVME_SUCCESS;
6547 }
6548 
6549 static void nvme_do_format(NvmeFormatAIOCB *iocb)
6550 {
6551     NvmeRequest *req = iocb->req;
6552     NvmeCtrl *n = nvme_ctrl(req);
6553     uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6554     uint8_t lbaf = dw10 & 0xf;
6555     uint8_t pi = (dw10 >> 5) & 0x7;
6556     uint16_t status;
6557     int i;
6558 
6559     if (iocb->ret < 0) {
6560         goto done;
6561     }
6562 
6563     if (iocb->broadcast) {
6564         for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
6565             iocb->ns = nvme_ns(n, i);
6566             if (iocb->ns) {
6567                 iocb->nsid = i;
6568                 break;
6569             }
6570         }
6571     }
6572 
6573     if (!iocb->ns) {
6574         goto done;
6575     }
6576 
6577     status = nvme_format_check(iocb->ns, lbaf, pi);
6578     if (status) {
6579         req->status = status;
6580         goto done;
6581     }
6582 
6583     iocb->ns->status = NVME_FORMAT_IN_PROGRESS;
6584     nvme_format_ns_cb(iocb, 0);
6585     return;
6586 
6587 done:
6588     iocb->common.cb(iocb->common.opaque, iocb->ret);
6589     qemu_aio_unref(iocb);
6590 }
6591 
6592 static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req)
6593 {
6594     NvmeFormatAIOCB *iocb;
6595     uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6596     uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6597     uint8_t lbaf = dw10 & 0xf;
6598     uint8_t mset = (dw10 >> 4) & 0x1;
6599     uint8_t pi = (dw10 >> 5) & 0x7;
6600     uint8_t pil = (dw10 >> 8) & 0x1;
6601     uint8_t lbafu = (dw10 >> 12) & 0x3;
6602     uint16_t status;
6603 
6604     iocb = qemu_aio_get(&nvme_format_aiocb_info, NULL, nvme_misc_cb, req);
6605 
6606     iocb->req = req;
6607     iocb->ret = 0;
6608     iocb->ns = NULL;
6609     iocb->nsid = 0;
6610     iocb->lbaf = lbaf;
6611     iocb->mset = mset;
6612     iocb->pi = pi;
6613     iocb->pil = pil;
6614     iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
6615     iocb->offset = 0;
6616 
6617     if (n->features.hbs.lbafee) {
6618         iocb->lbaf |= lbafu << 4;
6619     }
6620 
6621     if (!iocb->broadcast) {
6622         if (!nvme_nsid_valid(n, nsid)) {
6623             status = NVME_INVALID_NSID | NVME_DNR;
6624             goto out;
6625         }
6626 
6627         iocb->ns = nvme_ns(n, nsid);
6628         if (!iocb->ns) {
6629             status = NVME_INVALID_FIELD | NVME_DNR;
6630             goto out;
6631         }
6632     }
6633 
6634     req->aiocb = &iocb->common;
6635     nvme_do_format(iocb);
6636 
6637     return NVME_NO_COMPLETE;
6638 
6639 out:
6640     qemu_aio_unref(iocb);
6641 
6642     return status;
6643 }
6644 
6645 static void nvme_get_virt_res_num(NvmeCtrl *n, uint8_t rt, int *num_total,
6646                                   int *num_prim, int *num_sec)
6647 {
6648     *num_total = le32_to_cpu(rt ?
6649                              n->pri_ctrl_cap.vifrt : n->pri_ctrl_cap.vqfrt);
6650     *num_prim = le16_to_cpu(rt ?
6651                             n->pri_ctrl_cap.virfap : n->pri_ctrl_cap.vqrfap);
6652     *num_sec = le16_to_cpu(rt ? n->pri_ctrl_cap.virfa : n->pri_ctrl_cap.vqrfa);
6653 }
6654 
6655 static uint16_t nvme_assign_virt_res_to_prim(NvmeCtrl *n, NvmeRequest *req,
6656                                              uint16_t cntlid, uint8_t rt,
6657                                              int nr)
6658 {
6659     int num_total, num_prim, num_sec;
6660 
6661     if (cntlid != n->cntlid) {
6662         return NVME_INVALID_CTRL_ID | NVME_DNR;
6663     }
6664 
6665     nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
6666 
6667     if (nr > num_total) {
6668         return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
6669     }
6670 
6671     if (nr > num_total - num_sec) {
6672         return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6673     }
6674 
6675     if (rt) {
6676         n->next_pri_ctrl_cap.virfap = cpu_to_le16(nr);
6677     } else {
6678         n->next_pri_ctrl_cap.vqrfap = cpu_to_le16(nr);
6679     }
6680 
6681     req->cqe.result = cpu_to_le32(nr);
6682     return req->status;
6683 }
6684 
6685 static void nvme_update_virt_res(NvmeCtrl *n, NvmeSecCtrlEntry *sctrl,
6686                                  uint8_t rt, int nr)
6687 {
6688     int prev_nr, prev_total;
6689 
6690     if (rt) {
6691         prev_nr = le16_to_cpu(sctrl->nvi);
6692         prev_total = le32_to_cpu(n->pri_ctrl_cap.virfa);
6693         sctrl->nvi = cpu_to_le16(nr);
6694         n->pri_ctrl_cap.virfa = cpu_to_le32(prev_total + nr - prev_nr);
6695     } else {
6696         prev_nr = le16_to_cpu(sctrl->nvq);
6697         prev_total = le32_to_cpu(n->pri_ctrl_cap.vqrfa);
6698         sctrl->nvq = cpu_to_le16(nr);
6699         n->pri_ctrl_cap.vqrfa = cpu_to_le32(prev_total + nr - prev_nr);
6700     }
6701 }
6702 
6703 static uint16_t nvme_assign_virt_res_to_sec(NvmeCtrl *n, NvmeRequest *req,
6704                                             uint16_t cntlid, uint8_t rt, int nr)
6705 {
6706     int num_total, num_prim, num_sec, num_free, diff, limit;
6707     NvmeSecCtrlEntry *sctrl;
6708 
6709     sctrl = nvme_sctrl_for_cntlid(n, cntlid);
6710     if (!sctrl) {
6711         return NVME_INVALID_CTRL_ID | NVME_DNR;
6712     }
6713 
6714     if (sctrl->scs) {
6715         return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
6716     }
6717 
6718     limit = le16_to_cpu(rt ? n->pri_ctrl_cap.vifrsm : n->pri_ctrl_cap.vqfrsm);
6719     if (nr > limit) {
6720         return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
6721     }
6722 
6723     nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
6724     num_free = num_total - num_prim - num_sec;
6725     diff = nr - le16_to_cpu(rt ? sctrl->nvi : sctrl->nvq);
6726 
6727     if (diff > num_free) {
6728         return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6729     }
6730 
6731     nvme_update_virt_res(n, sctrl, rt, nr);
6732     req->cqe.result = cpu_to_le32(nr);
6733 
6734     return req->status;
6735 }
6736 
6737 static uint16_t nvme_virt_set_state(NvmeCtrl *n, uint16_t cntlid, bool online)
6738 {
6739     PCIDevice *pci = PCI_DEVICE(n);
6740     NvmeCtrl *sn = NULL;
6741     NvmeSecCtrlEntry *sctrl;
6742     int vf_index;
6743 
6744     sctrl = nvme_sctrl_for_cntlid(n, cntlid);
6745     if (!sctrl) {
6746         return NVME_INVALID_CTRL_ID | NVME_DNR;
6747     }
6748 
6749     if (!pci_is_vf(pci)) {
6750         vf_index = le16_to_cpu(sctrl->vfn) - 1;
6751         sn = NVME(pcie_sriov_get_vf_at_index(pci, vf_index));
6752     }
6753 
6754     if (online) {
6755         if (!sctrl->nvi || (le16_to_cpu(sctrl->nvq) < 2) || !sn) {
6756             return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
6757         }
6758 
6759         if (!sctrl->scs) {
6760             sctrl->scs = 0x1;
6761             nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
6762         }
6763     } else {
6764         nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_INTERRUPT, 0);
6765         nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_QUEUE, 0);
6766 
6767         if (sctrl->scs) {
6768             sctrl->scs = 0x0;
6769             if (sn) {
6770                 nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
6771             }
6772         }
6773     }
6774 
6775     return NVME_SUCCESS;
6776 }
6777 
6778 static uint16_t nvme_virt_mngmt(NvmeCtrl *n, NvmeRequest *req)
6779 {
6780     uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6781     uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
6782     uint8_t act = dw10 & 0xf;
6783     uint8_t rt = (dw10 >> 8) & 0x7;
6784     uint16_t cntlid = (dw10 >> 16) & 0xffff;
6785     int nr = dw11 & 0xffff;
6786 
6787     trace_pci_nvme_virt_mngmt(nvme_cid(req), act, cntlid, rt ? "VI" : "VQ", nr);
6788 
6789     if (rt != NVME_VIRT_RES_QUEUE && rt != NVME_VIRT_RES_INTERRUPT) {
6790         return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6791     }
6792 
6793     switch (act) {
6794     case NVME_VIRT_MNGMT_ACTION_SEC_ASSIGN:
6795         return nvme_assign_virt_res_to_sec(n, req, cntlid, rt, nr);
6796     case NVME_VIRT_MNGMT_ACTION_PRM_ALLOC:
6797         return nvme_assign_virt_res_to_prim(n, req, cntlid, rt, nr);
6798     case NVME_VIRT_MNGMT_ACTION_SEC_ONLINE:
6799         return nvme_virt_set_state(n, cntlid, true);
6800     case NVME_VIRT_MNGMT_ACTION_SEC_OFFLINE:
6801         return nvme_virt_set_state(n, cntlid, false);
6802     default:
6803         return NVME_INVALID_FIELD | NVME_DNR;
6804     }
6805 }
6806 
6807 static uint16_t nvme_dbbuf_config(NvmeCtrl *n, const NvmeRequest *req)
6808 {
6809     PCIDevice *pci = PCI_DEVICE(n);
6810     uint64_t dbs_addr = le64_to_cpu(req->cmd.dptr.prp1);
6811     uint64_t eis_addr = le64_to_cpu(req->cmd.dptr.prp2);
6812     int i;
6813 
6814     /* Address should be page aligned */
6815     if (dbs_addr & (n->page_size - 1) || eis_addr & (n->page_size - 1)) {
6816         return NVME_INVALID_FIELD | NVME_DNR;
6817     }
6818 
6819     /* Save shadow buffer base addr for use during queue creation */
6820     n->dbbuf_dbs = dbs_addr;
6821     n->dbbuf_eis = eis_addr;
6822     n->dbbuf_enabled = true;
6823 
6824     for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
6825         NvmeSQueue *sq = n->sq[i];
6826         NvmeCQueue *cq = n->cq[i];
6827 
6828         if (sq) {
6829             /*
6830              * CAP.DSTRD is 0, so offset of ith sq db_addr is (i<<3)
6831              * nvme_process_db() uses this hard-coded way to calculate
6832              * doorbell offsets. Be consistent with that here.
6833              */
6834             sq->db_addr = dbs_addr + (i << 3);
6835             sq->ei_addr = eis_addr + (i << 3);
6836             stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED);
6837 
6838             if (n->params.ioeventfd && sq->sqid != 0) {
6839                 if (!nvme_init_sq_ioeventfd(sq)) {
6840                     sq->ioeventfd_enabled = true;
6841                 }
6842             }
6843         }
6844 
6845         if (cq) {
6846             /* CAP.DSTRD is 0, so offset of ith cq db_addr is (i<<3)+(1<<2) */
6847             cq->db_addr = dbs_addr + (i << 3) + (1 << 2);
6848             cq->ei_addr = eis_addr + (i << 3) + (1 << 2);
6849             stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED);
6850 
6851             if (n->params.ioeventfd && cq->cqid != 0) {
6852                 if (!nvme_init_cq_ioeventfd(cq)) {
6853                     cq->ioeventfd_enabled = true;
6854                 }
6855             }
6856         }
6857     }
6858 
6859     trace_pci_nvme_dbbuf_config(dbs_addr, eis_addr);
6860 
6861     return NVME_SUCCESS;
6862 }
6863 
6864 static uint16_t nvme_directive_send(NvmeCtrl *n, NvmeRequest *req)
6865 {
6866     return NVME_INVALID_FIELD | NVME_DNR;
6867 }
6868 
6869 static uint16_t nvme_directive_receive(NvmeCtrl *n, NvmeRequest *req)
6870 {
6871     NvmeNamespace *ns;
6872     uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6873     uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
6874     uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6875     uint8_t doper, dtype;
6876     uint32_t numd, trans_len;
6877     NvmeDirectiveIdentify id = {
6878         .supported = 1 << NVME_DIRECTIVE_IDENTIFY,
6879         .enabled = 1 << NVME_DIRECTIVE_IDENTIFY,
6880     };
6881 
6882     numd = dw10 + 1;
6883     doper = dw11 & 0xff;
6884     dtype = (dw11 >> 8) & 0xff;
6885 
6886     trans_len = MIN(sizeof(NvmeDirectiveIdentify), numd << 2);
6887 
6888     if (nsid == NVME_NSID_BROADCAST || dtype != NVME_DIRECTIVE_IDENTIFY ||
6889         doper != NVME_DIRECTIVE_RETURN_PARAMS) {
6890         return NVME_INVALID_FIELD | NVME_DNR;
6891     }
6892 
6893     ns = nvme_ns(n, nsid);
6894     if (!ns) {
6895         return NVME_INVALID_FIELD | NVME_DNR;
6896     }
6897 
6898     switch (dtype) {
6899     case NVME_DIRECTIVE_IDENTIFY:
6900         switch (doper) {
6901         case NVME_DIRECTIVE_RETURN_PARAMS:
6902             if (ns->endgrp && ns->endgrp->fdp.enabled) {
6903                 id.supported |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
6904                 id.enabled |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
6905                 id.persistent |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
6906             }
6907 
6908             return nvme_c2h(n, (uint8_t *)&id, trans_len, req);
6909 
6910         default:
6911             return NVME_INVALID_FIELD | NVME_DNR;
6912         }
6913 
6914     default:
6915         return NVME_INVALID_FIELD;
6916     }
6917 }
6918 
6919 static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeRequest *req)
6920 {
6921     trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), req->cmd.opcode,
6922                              nvme_adm_opc_str(req->cmd.opcode));
6923 
6924     if (!(nvme_cse_acs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
6925         trace_pci_nvme_err_invalid_admin_opc(req->cmd.opcode);
6926         return NVME_INVALID_OPCODE | NVME_DNR;
6927     }
6928 
6929     /* SGLs shall not be used for Admin commands in NVMe over PCIe */
6930     if (NVME_CMD_FLAGS_PSDT(req->cmd.flags) != NVME_PSDT_PRP) {
6931         return NVME_INVALID_FIELD | NVME_DNR;
6932     }
6933 
6934     if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
6935         return NVME_INVALID_FIELD;
6936     }
6937 
6938     switch (req->cmd.opcode) {
6939     case NVME_ADM_CMD_DELETE_SQ:
6940         return nvme_del_sq(n, req);
6941     case NVME_ADM_CMD_CREATE_SQ:
6942         return nvme_create_sq(n, req);
6943     case NVME_ADM_CMD_GET_LOG_PAGE:
6944         return nvme_get_log(n, req);
6945     case NVME_ADM_CMD_DELETE_CQ:
6946         return nvme_del_cq(n, req);
6947     case NVME_ADM_CMD_CREATE_CQ:
6948         return nvme_create_cq(n, req);
6949     case NVME_ADM_CMD_IDENTIFY:
6950         return nvme_identify(n, req);
6951     case NVME_ADM_CMD_ABORT:
6952         return nvme_abort(n, req);
6953     case NVME_ADM_CMD_SET_FEATURES:
6954         return nvme_set_feature(n, req);
6955     case NVME_ADM_CMD_GET_FEATURES:
6956         return nvme_get_feature(n, req);
6957     case NVME_ADM_CMD_ASYNC_EV_REQ:
6958         return nvme_aer(n, req);
6959     case NVME_ADM_CMD_NS_ATTACHMENT:
6960         return nvme_ns_attachment(n, req);
6961     case NVME_ADM_CMD_VIRT_MNGMT:
6962         return nvme_virt_mngmt(n, req);
6963     case NVME_ADM_CMD_DBBUF_CONFIG:
6964         return nvme_dbbuf_config(n, req);
6965     case NVME_ADM_CMD_FORMAT_NVM:
6966         return nvme_format(n, req);
6967     case NVME_ADM_CMD_DIRECTIVE_SEND:
6968         return nvme_directive_send(n, req);
6969     case NVME_ADM_CMD_DIRECTIVE_RECV:
6970         return nvme_directive_receive(n, req);
6971     default:
6972         assert(false);
6973     }
6974 
6975     return NVME_INVALID_OPCODE | NVME_DNR;
6976 }
6977 
6978 static void nvme_update_sq_eventidx(const NvmeSQueue *sq)
6979 {
6980     trace_pci_nvme_update_sq_eventidx(sq->sqid, sq->tail);
6981 
6982     stl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->ei_addr, sq->tail,
6983                    MEMTXATTRS_UNSPECIFIED);
6984 }
6985 
6986 static void nvme_update_sq_tail(NvmeSQueue *sq)
6987 {
6988     ldl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->db_addr, &sq->tail,
6989                    MEMTXATTRS_UNSPECIFIED);
6990 
6991     trace_pci_nvme_update_sq_tail(sq->sqid, sq->tail);
6992 }
6993 
6994 static void nvme_process_sq(void *opaque)
6995 {
6996     NvmeSQueue *sq = opaque;
6997     NvmeCtrl *n = sq->ctrl;
6998     NvmeCQueue *cq = n->cq[sq->cqid];
6999 
7000     uint16_t status;
7001     hwaddr addr;
7002     NvmeCmd cmd;
7003     NvmeRequest *req;
7004 
7005     if (n->dbbuf_enabled) {
7006         nvme_update_sq_tail(sq);
7007     }
7008 
7009     while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
7010         addr = sq->dma_addr + (sq->head << NVME_SQES);
7011         if (nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd))) {
7012             trace_pci_nvme_err_addr_read(addr);
7013             trace_pci_nvme_err_cfs();
7014             stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
7015             break;
7016         }
7017         nvme_inc_sq_head(sq);
7018 
7019         req = QTAILQ_FIRST(&sq->req_list);
7020         QTAILQ_REMOVE(&sq->req_list, req, entry);
7021         QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
7022         nvme_req_clear(req);
7023         req->cqe.cid = cmd.cid;
7024         memcpy(&req->cmd, &cmd, sizeof(NvmeCmd));
7025 
7026         status = sq->sqid ? nvme_io_cmd(n, req) :
7027             nvme_admin_cmd(n, req);
7028         if (status != NVME_NO_COMPLETE) {
7029             req->status = status;
7030             nvme_enqueue_req_completion(cq, req);
7031         }
7032 
7033         if (n->dbbuf_enabled) {
7034             nvme_update_sq_eventidx(sq);
7035             nvme_update_sq_tail(sq);
7036         }
7037     }
7038 }
7039 
7040 static void nvme_update_msixcap_ts(PCIDevice *pci_dev, uint32_t table_size)
7041 {
7042     uint8_t *config;
7043 
7044     if (!msix_present(pci_dev)) {
7045         return;
7046     }
7047 
7048     assert(table_size > 0 && table_size <= pci_dev->msix_entries_nr);
7049 
7050     config = pci_dev->config + pci_dev->msix_cap;
7051     pci_set_word_by_mask(config + PCI_MSIX_FLAGS, PCI_MSIX_FLAGS_QSIZE,
7052                          table_size - 1);
7053 }
7054 
7055 static void nvme_activate_virt_res(NvmeCtrl *n)
7056 {
7057     PCIDevice *pci_dev = PCI_DEVICE(n);
7058     NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
7059     NvmeSecCtrlEntry *sctrl;
7060 
7061     /* -1 to account for the admin queue */
7062     if (pci_is_vf(pci_dev)) {
7063         sctrl = nvme_sctrl(n);
7064         cap->vqprt = sctrl->nvq;
7065         cap->viprt = sctrl->nvi;
7066         n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
7067         n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
7068     } else {
7069         cap->vqrfap = n->next_pri_ctrl_cap.vqrfap;
7070         cap->virfap = n->next_pri_ctrl_cap.virfap;
7071         n->conf_ioqpairs = le16_to_cpu(cap->vqprt) +
7072                            le16_to_cpu(cap->vqrfap) - 1;
7073         n->conf_msix_qsize = le16_to_cpu(cap->viprt) +
7074                              le16_to_cpu(cap->virfap);
7075     }
7076 }
7077 
7078 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst)
7079 {
7080     PCIDevice *pci_dev = PCI_DEVICE(n);
7081     NvmeSecCtrlEntry *sctrl;
7082     NvmeNamespace *ns;
7083     int i;
7084 
7085     for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7086         ns = nvme_ns(n, i);
7087         if (!ns) {
7088             continue;
7089         }
7090 
7091         nvme_ns_drain(ns);
7092     }
7093 
7094     for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
7095         if (n->sq[i] != NULL) {
7096             nvme_free_sq(n->sq[i], n);
7097         }
7098     }
7099     for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
7100         if (n->cq[i] != NULL) {
7101             nvme_free_cq(n->cq[i], n);
7102         }
7103     }
7104 
7105     while (!QTAILQ_EMPTY(&n->aer_queue)) {
7106         NvmeAsyncEvent *event = QTAILQ_FIRST(&n->aer_queue);
7107         QTAILQ_REMOVE(&n->aer_queue, event, entry);
7108         g_free(event);
7109     }
7110 
7111     if (n->params.sriov_max_vfs) {
7112         if (!pci_is_vf(pci_dev)) {
7113             for (i = 0; i < n->sec_ctrl_list.numcntl; i++) {
7114                 sctrl = &n->sec_ctrl_list.sec[i];
7115                 nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
7116             }
7117 
7118             if (rst != NVME_RESET_CONTROLLER) {
7119                 pcie_sriov_pf_disable_vfs(pci_dev);
7120             }
7121         }
7122 
7123         if (rst != NVME_RESET_CONTROLLER) {
7124             nvme_activate_virt_res(n);
7125         }
7126     }
7127 
7128     n->aer_queued = 0;
7129     n->aer_mask = 0;
7130     n->outstanding_aers = 0;
7131     n->qs_created = false;
7132 
7133     nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
7134 
7135     if (pci_is_vf(pci_dev)) {
7136         sctrl = nvme_sctrl(n);
7137 
7138         stl_le_p(&n->bar.csts, sctrl->scs ? 0 : NVME_CSTS_FAILED);
7139     } else {
7140         stl_le_p(&n->bar.csts, 0);
7141     }
7142 
7143     stl_le_p(&n->bar.intms, 0);
7144     stl_le_p(&n->bar.intmc, 0);
7145     stl_le_p(&n->bar.cc, 0);
7146 
7147     n->dbbuf_dbs = 0;
7148     n->dbbuf_eis = 0;
7149     n->dbbuf_enabled = false;
7150 }
7151 
7152 static void nvme_ctrl_shutdown(NvmeCtrl *n)
7153 {
7154     NvmeNamespace *ns;
7155     int i;
7156 
7157     if (n->pmr.dev) {
7158         memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
7159     }
7160 
7161     for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7162         ns = nvme_ns(n, i);
7163         if (!ns) {
7164             continue;
7165         }
7166 
7167         nvme_ns_shutdown(ns);
7168     }
7169 }
7170 
7171 static void nvme_select_iocs(NvmeCtrl *n)
7172 {
7173     NvmeNamespace *ns;
7174     int i;
7175 
7176     for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7177         ns = nvme_ns(n, i);
7178         if (!ns) {
7179             continue;
7180         }
7181 
7182         nvme_select_iocs_ns(n, ns);
7183     }
7184 }
7185 
7186 static int nvme_start_ctrl(NvmeCtrl *n)
7187 {
7188     uint64_t cap = ldq_le_p(&n->bar.cap);
7189     uint32_t cc = ldl_le_p(&n->bar.cc);
7190     uint32_t aqa = ldl_le_p(&n->bar.aqa);
7191     uint64_t asq = ldq_le_p(&n->bar.asq);
7192     uint64_t acq = ldq_le_p(&n->bar.acq);
7193     uint32_t page_bits = NVME_CC_MPS(cc) + 12;
7194     uint32_t page_size = 1 << page_bits;
7195     NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
7196 
7197     if (pci_is_vf(PCI_DEVICE(n)) && !sctrl->scs) {
7198         trace_pci_nvme_err_startfail_virt_state(le16_to_cpu(sctrl->nvi),
7199                                                 le16_to_cpu(sctrl->nvq));
7200         return -1;
7201     }
7202     if (unlikely(n->cq[0])) {
7203         trace_pci_nvme_err_startfail_cq();
7204         return -1;
7205     }
7206     if (unlikely(n->sq[0])) {
7207         trace_pci_nvme_err_startfail_sq();
7208         return -1;
7209     }
7210     if (unlikely(asq & (page_size - 1))) {
7211         trace_pci_nvme_err_startfail_asq_misaligned(asq);
7212         return -1;
7213     }
7214     if (unlikely(acq & (page_size - 1))) {
7215         trace_pci_nvme_err_startfail_acq_misaligned(acq);
7216         return -1;
7217     }
7218     if (unlikely(!(NVME_CAP_CSS(cap) & (1 << NVME_CC_CSS(cc))))) {
7219         trace_pci_nvme_err_startfail_css(NVME_CC_CSS(cc));
7220         return -1;
7221     }
7222     if (unlikely(NVME_CC_MPS(cc) < NVME_CAP_MPSMIN(cap))) {
7223         trace_pci_nvme_err_startfail_page_too_small(
7224                     NVME_CC_MPS(cc),
7225                     NVME_CAP_MPSMIN(cap));
7226         return -1;
7227     }
7228     if (unlikely(NVME_CC_MPS(cc) >
7229                  NVME_CAP_MPSMAX(cap))) {
7230         trace_pci_nvme_err_startfail_page_too_large(
7231                     NVME_CC_MPS(cc),
7232                     NVME_CAP_MPSMAX(cap));
7233         return -1;
7234     }
7235     if (unlikely(!NVME_AQA_ASQS(aqa))) {
7236         trace_pci_nvme_err_startfail_asqent_sz_zero();
7237         return -1;
7238     }
7239     if (unlikely(!NVME_AQA_ACQS(aqa))) {
7240         trace_pci_nvme_err_startfail_acqent_sz_zero();
7241         return -1;
7242     }
7243 
7244     n->page_bits = page_bits;
7245     n->page_size = page_size;
7246     n->max_prp_ents = n->page_size / sizeof(uint64_t);
7247     nvme_init_cq(&n->admin_cq, n, acq, 0, 0, NVME_AQA_ACQS(aqa) + 1, 1);
7248     nvme_init_sq(&n->admin_sq, n, asq, 0, 0, NVME_AQA_ASQS(aqa) + 1);
7249 
7250     nvme_set_timestamp(n, 0ULL);
7251 
7252     nvme_select_iocs(n);
7253 
7254     return 0;
7255 }
7256 
7257 static void nvme_cmb_enable_regs(NvmeCtrl *n)
7258 {
7259     uint32_t cmbloc = ldl_le_p(&n->bar.cmbloc);
7260     uint32_t cmbsz = ldl_le_p(&n->bar.cmbsz);
7261 
7262     NVME_CMBLOC_SET_CDPCILS(cmbloc, 1);
7263     NVME_CMBLOC_SET_CDPMLS(cmbloc, 1);
7264     NVME_CMBLOC_SET_BIR(cmbloc, NVME_CMB_BIR);
7265     stl_le_p(&n->bar.cmbloc, cmbloc);
7266 
7267     NVME_CMBSZ_SET_SQS(cmbsz, 1);
7268     NVME_CMBSZ_SET_CQS(cmbsz, 0);
7269     NVME_CMBSZ_SET_LISTS(cmbsz, 1);
7270     NVME_CMBSZ_SET_RDS(cmbsz, 1);
7271     NVME_CMBSZ_SET_WDS(cmbsz, 1);
7272     NVME_CMBSZ_SET_SZU(cmbsz, 2); /* MBs */
7273     NVME_CMBSZ_SET_SZ(cmbsz, n->params.cmb_size_mb);
7274     stl_le_p(&n->bar.cmbsz, cmbsz);
7275 }
7276 
7277 static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
7278                            unsigned size)
7279 {
7280     PCIDevice *pci = PCI_DEVICE(n);
7281     uint64_t cap = ldq_le_p(&n->bar.cap);
7282     uint32_t cc = ldl_le_p(&n->bar.cc);
7283     uint32_t intms = ldl_le_p(&n->bar.intms);
7284     uint32_t csts = ldl_le_p(&n->bar.csts);
7285     uint32_t pmrsts = ldl_le_p(&n->bar.pmrsts);
7286 
7287     if (unlikely(offset & (sizeof(uint32_t) - 1))) {
7288         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32,
7289                        "MMIO write not 32-bit aligned,"
7290                        " offset=0x%"PRIx64"", offset);
7291         /* should be ignored, fall through for now */
7292     }
7293 
7294     if (unlikely(size < sizeof(uint32_t))) {
7295         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall,
7296                        "MMIO write smaller than 32-bits,"
7297                        " offset=0x%"PRIx64", size=%u",
7298                        offset, size);
7299         /* should be ignored, fall through for now */
7300     }
7301 
7302     switch (offset) {
7303     case NVME_REG_INTMS:
7304         if (unlikely(msix_enabled(pci))) {
7305             NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
7306                            "undefined access to interrupt mask set"
7307                            " when MSI-X is enabled");
7308             /* should be ignored, fall through for now */
7309         }
7310         intms |= data;
7311         stl_le_p(&n->bar.intms, intms);
7312         n->bar.intmc = n->bar.intms;
7313         trace_pci_nvme_mmio_intm_set(data & 0xffffffff, intms);
7314         nvme_irq_check(n);
7315         break;
7316     case NVME_REG_INTMC:
7317         if (unlikely(msix_enabled(pci))) {
7318             NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
7319                            "undefined access to interrupt mask clr"
7320                            " when MSI-X is enabled");
7321             /* should be ignored, fall through for now */
7322         }
7323         intms &= ~data;
7324         stl_le_p(&n->bar.intms, intms);
7325         n->bar.intmc = n->bar.intms;
7326         trace_pci_nvme_mmio_intm_clr(data & 0xffffffff, intms);
7327         nvme_irq_check(n);
7328         break;
7329     case NVME_REG_CC:
7330         stl_le_p(&n->bar.cc, data);
7331 
7332         trace_pci_nvme_mmio_cfg(data & 0xffffffff);
7333 
7334         if (NVME_CC_SHN(data) && !(NVME_CC_SHN(cc))) {
7335             trace_pci_nvme_mmio_shutdown_set();
7336             nvme_ctrl_shutdown(n);
7337             csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
7338             csts |= NVME_CSTS_SHST_COMPLETE;
7339         } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(cc)) {
7340             trace_pci_nvme_mmio_shutdown_cleared();
7341             csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
7342         }
7343 
7344         if (NVME_CC_EN(data) && !NVME_CC_EN(cc)) {
7345             if (unlikely(nvme_start_ctrl(n))) {
7346                 trace_pci_nvme_err_startfail();
7347                 csts = NVME_CSTS_FAILED;
7348             } else {
7349                 trace_pci_nvme_mmio_start_success();
7350                 csts = NVME_CSTS_READY;
7351             }
7352         } else if (!NVME_CC_EN(data) && NVME_CC_EN(cc)) {
7353             trace_pci_nvme_mmio_stopped();
7354             nvme_ctrl_reset(n, NVME_RESET_CONTROLLER);
7355 
7356             break;
7357         }
7358 
7359         stl_le_p(&n->bar.csts, csts);
7360 
7361         break;
7362     case NVME_REG_CSTS:
7363         if (data & (1 << 4)) {
7364             NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported,
7365                            "attempted to W1C CSTS.NSSRO"
7366                            " but CAP.NSSRS is zero (not supported)");
7367         } else if (data != 0) {
7368             NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts,
7369                            "attempted to set a read only bit"
7370                            " of controller status");
7371         }
7372         break;
7373     case NVME_REG_NSSR:
7374         if (data == 0x4e564d65) {
7375             trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
7376         } else {
7377             /* The spec says that writes of other values have no effect */
7378             return;
7379         }
7380         break;
7381     case NVME_REG_AQA:
7382         stl_le_p(&n->bar.aqa, data);
7383         trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
7384         break;
7385     case NVME_REG_ASQ:
7386         stn_le_p(&n->bar.asq, size, data);
7387         trace_pci_nvme_mmio_asqaddr(data);
7388         break;
7389     case NVME_REG_ASQ + 4:
7390         stl_le_p((uint8_t *)&n->bar.asq + 4, data);
7391         trace_pci_nvme_mmio_asqaddr_hi(data, ldq_le_p(&n->bar.asq));
7392         break;
7393     case NVME_REG_ACQ:
7394         trace_pci_nvme_mmio_acqaddr(data);
7395         stn_le_p(&n->bar.acq, size, data);
7396         break;
7397     case NVME_REG_ACQ + 4:
7398         stl_le_p((uint8_t *)&n->bar.acq + 4, data);
7399         trace_pci_nvme_mmio_acqaddr_hi(data, ldq_le_p(&n->bar.acq));
7400         break;
7401     case NVME_REG_CMBLOC:
7402         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved,
7403                        "invalid write to reserved CMBLOC"
7404                        " when CMBSZ is zero, ignored");
7405         return;
7406     case NVME_REG_CMBSZ:
7407         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly,
7408                        "invalid write to read only CMBSZ, ignored");
7409         return;
7410     case NVME_REG_CMBMSC:
7411         if (!NVME_CAP_CMBS(cap)) {
7412             return;
7413         }
7414 
7415         stn_le_p(&n->bar.cmbmsc, size, data);
7416         n->cmb.cmse = false;
7417 
7418         if (NVME_CMBMSC_CRE(data)) {
7419             nvme_cmb_enable_regs(n);
7420 
7421             if (NVME_CMBMSC_CMSE(data)) {
7422                 uint64_t cmbmsc = ldq_le_p(&n->bar.cmbmsc);
7423                 hwaddr cba = NVME_CMBMSC_CBA(cmbmsc) << CMBMSC_CBA_SHIFT;
7424                 if (cba + int128_get64(n->cmb.mem.size) < cba) {
7425                     uint32_t cmbsts = ldl_le_p(&n->bar.cmbsts);
7426                     NVME_CMBSTS_SET_CBAI(cmbsts, 1);
7427                     stl_le_p(&n->bar.cmbsts, cmbsts);
7428                     return;
7429                 }
7430 
7431                 n->cmb.cba = cba;
7432                 n->cmb.cmse = true;
7433             }
7434         } else {
7435             n->bar.cmbsz = 0;
7436             n->bar.cmbloc = 0;
7437         }
7438 
7439         return;
7440     case NVME_REG_CMBMSC + 4:
7441         stl_le_p((uint8_t *)&n->bar.cmbmsc + 4, data);
7442         return;
7443 
7444     case NVME_REG_PMRCAP:
7445         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly,
7446                        "invalid write to PMRCAP register, ignored");
7447         return;
7448     case NVME_REG_PMRCTL:
7449         if (!NVME_CAP_PMRS(cap)) {
7450             return;
7451         }
7452 
7453         stl_le_p(&n->bar.pmrctl, data);
7454         if (NVME_PMRCTL_EN(data)) {
7455             memory_region_set_enabled(&n->pmr.dev->mr, true);
7456             pmrsts = 0;
7457         } else {
7458             memory_region_set_enabled(&n->pmr.dev->mr, false);
7459             NVME_PMRSTS_SET_NRDY(pmrsts, 1);
7460             n->pmr.cmse = false;
7461         }
7462         stl_le_p(&n->bar.pmrsts, pmrsts);
7463         return;
7464     case NVME_REG_PMRSTS:
7465         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly,
7466                        "invalid write to PMRSTS register, ignored");
7467         return;
7468     case NVME_REG_PMREBS:
7469         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly,
7470                        "invalid write to PMREBS register, ignored");
7471         return;
7472     case NVME_REG_PMRSWTP:
7473         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly,
7474                        "invalid write to PMRSWTP register, ignored");
7475         return;
7476     case NVME_REG_PMRMSCL:
7477         if (!NVME_CAP_PMRS(cap)) {
7478             return;
7479         }
7480 
7481         stl_le_p(&n->bar.pmrmscl, data);
7482         n->pmr.cmse = false;
7483 
7484         if (NVME_PMRMSCL_CMSE(data)) {
7485             uint64_t pmrmscu = ldl_le_p(&n->bar.pmrmscu);
7486             hwaddr cba = pmrmscu << 32 |
7487                 (NVME_PMRMSCL_CBA(data) << PMRMSCL_CBA_SHIFT);
7488             if (cba + int128_get64(n->pmr.dev->mr.size) < cba) {
7489                 NVME_PMRSTS_SET_CBAI(pmrsts, 1);
7490                 stl_le_p(&n->bar.pmrsts, pmrsts);
7491                 return;
7492             }
7493 
7494             n->pmr.cmse = true;
7495             n->pmr.cba = cba;
7496         }
7497 
7498         return;
7499     case NVME_REG_PMRMSCU:
7500         if (!NVME_CAP_PMRS(cap)) {
7501             return;
7502         }
7503 
7504         stl_le_p(&n->bar.pmrmscu, data);
7505         return;
7506     default:
7507         NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid,
7508                        "invalid MMIO write,"
7509                        " offset=0x%"PRIx64", data=%"PRIx64"",
7510                        offset, data);
7511         break;
7512     }
7513 }
7514 
7515 static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
7516 {
7517     NvmeCtrl *n = (NvmeCtrl *)opaque;
7518     uint8_t *ptr = (uint8_t *)&n->bar;
7519 
7520     trace_pci_nvme_mmio_read(addr, size);
7521 
7522     if (unlikely(addr & (sizeof(uint32_t) - 1))) {
7523         NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32,
7524                        "MMIO read not 32-bit aligned,"
7525                        " offset=0x%"PRIx64"", addr);
7526         /* should RAZ, fall through for now */
7527     } else if (unlikely(size < sizeof(uint32_t))) {
7528         NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall,
7529                        "MMIO read smaller than 32-bits,"
7530                        " offset=0x%"PRIx64"", addr);
7531         /* should RAZ, fall through for now */
7532     }
7533 
7534     if (addr > sizeof(n->bar) - size) {
7535         NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs,
7536                        "MMIO read beyond last register,"
7537                        " offset=0x%"PRIx64", returning 0", addr);
7538 
7539         return 0;
7540     }
7541 
7542     if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
7543         addr != NVME_REG_CSTS) {
7544         trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
7545         return 0;
7546     }
7547 
7548     /*
7549      * When PMRWBM bit 1 is set then read from
7550      * from PMRSTS should ensure prior writes
7551      * made it to persistent media
7552      */
7553     if (addr == NVME_REG_PMRSTS &&
7554         (NVME_PMRCAP_PMRWBM(ldl_le_p(&n->bar.pmrcap)) & 0x02)) {
7555         memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
7556     }
7557 
7558     return ldn_le_p(ptr + addr, size);
7559 }
7560 
7561 static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
7562 {
7563     PCIDevice *pci = PCI_DEVICE(n);
7564     uint32_t qid;
7565 
7566     if (unlikely(addr & ((1 << 2) - 1))) {
7567         NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned,
7568                        "doorbell write not 32-bit aligned,"
7569                        " offset=0x%"PRIx64", ignoring", addr);
7570         return;
7571     }
7572 
7573     if (((addr - 0x1000) >> 2) & 1) {
7574         /* Completion queue doorbell write */
7575 
7576         uint16_t new_head = val & 0xffff;
7577         int start_sqs;
7578         NvmeCQueue *cq;
7579 
7580         qid = (addr - (0x1000 + (1 << 2))) >> 3;
7581         if (unlikely(nvme_check_cqid(n, qid))) {
7582             NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq,
7583                            "completion queue doorbell write"
7584                            " for nonexistent queue,"
7585                            " sqid=%"PRIu32", ignoring", qid);
7586 
7587             /*
7588              * NVM Express v1.3d, Section 4.1 state: "If host software writes
7589              * an invalid value to the Submission Queue Tail Doorbell or
7590              * Completion Queue Head Doorbell register and an Asynchronous Event
7591              * Request command is outstanding, then an asynchronous event is
7592              * posted to the Admin Completion Queue with a status code of
7593              * Invalid Doorbell Write Value."
7594              *
7595              * Also note that the spec includes the "Invalid Doorbell Register"
7596              * status code, but nowhere does it specify when to use it.
7597              * However, it seems reasonable to use it here in a similar
7598              * fashion.
7599              */
7600             if (n->outstanding_aers) {
7601                 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7602                                    NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
7603                                    NVME_LOG_ERROR_INFO);
7604             }
7605 
7606             return;
7607         }
7608 
7609         cq = n->cq[qid];
7610         if (unlikely(new_head >= cq->size)) {
7611             NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead,
7612                            "completion queue doorbell write value"
7613                            " beyond queue size, sqid=%"PRIu32","
7614                            " new_head=%"PRIu16", ignoring",
7615                            qid, new_head);
7616 
7617             if (n->outstanding_aers) {
7618                 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7619                                    NVME_AER_INFO_ERR_INVALID_DB_VALUE,
7620                                    NVME_LOG_ERROR_INFO);
7621             }
7622 
7623             return;
7624         }
7625 
7626         trace_pci_nvme_mmio_doorbell_cq(cq->cqid, new_head);
7627 
7628         start_sqs = nvme_cq_full(cq) ? 1 : 0;
7629         cq->head = new_head;
7630         if (!qid && n->dbbuf_enabled) {
7631             stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED);
7632         }
7633         if (start_sqs) {
7634             NvmeSQueue *sq;
7635             QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
7636                 qemu_bh_schedule(sq->bh);
7637             }
7638             qemu_bh_schedule(cq->bh);
7639         }
7640 
7641         if (cq->tail == cq->head) {
7642             if (cq->irq_enabled) {
7643                 n->cq_pending--;
7644             }
7645 
7646             nvme_irq_deassert(n, cq);
7647         }
7648     } else {
7649         /* Submission queue doorbell write */
7650 
7651         uint16_t new_tail = val & 0xffff;
7652         NvmeSQueue *sq;
7653 
7654         qid = (addr - 0x1000) >> 3;
7655         if (unlikely(nvme_check_sqid(n, qid))) {
7656             NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq,
7657                            "submission queue doorbell write"
7658                            " for nonexistent queue,"
7659                            " sqid=%"PRIu32", ignoring", qid);
7660 
7661             if (n->outstanding_aers) {
7662                 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7663                                    NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
7664                                    NVME_LOG_ERROR_INFO);
7665             }
7666 
7667             return;
7668         }
7669 
7670         sq = n->sq[qid];
7671         if (unlikely(new_tail >= sq->size)) {
7672             NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail,
7673                            "submission queue doorbell write value"
7674                            " beyond queue size, sqid=%"PRIu32","
7675                            " new_tail=%"PRIu16", ignoring",
7676                            qid, new_tail);
7677 
7678             if (n->outstanding_aers) {
7679                 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7680                                    NVME_AER_INFO_ERR_INVALID_DB_VALUE,
7681                                    NVME_LOG_ERROR_INFO);
7682             }
7683 
7684             return;
7685         }
7686 
7687         trace_pci_nvme_mmio_doorbell_sq(sq->sqid, new_tail);
7688 
7689         sq->tail = new_tail;
7690         if (!qid && n->dbbuf_enabled) {
7691             /*
7692              * The spec states "the host shall also update the controller's
7693              * corresponding doorbell property to match the value of that entry
7694              * in the Shadow Doorbell buffer."
7695              *
7696              * Since this context is currently a VM trap, we can safely enforce
7697              * the requirement from the device side in case the host is
7698              * misbehaving.
7699              *
7700              * Note, we shouldn't have to do this, but various drivers
7701              * including ones that run on Linux, are not updating Admin Queues,
7702              * so we can't trust reading it for an appropriate sq tail.
7703              */
7704             stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED);
7705         }
7706 
7707         qemu_bh_schedule(sq->bh);
7708     }
7709 }
7710 
7711 static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
7712                             unsigned size)
7713 {
7714     NvmeCtrl *n = (NvmeCtrl *)opaque;
7715 
7716     trace_pci_nvme_mmio_write(addr, data, size);
7717 
7718     if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
7719         addr != NVME_REG_CSTS) {
7720         trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
7721         return;
7722     }
7723 
7724     if (addr < sizeof(n->bar)) {
7725         nvme_write_bar(n, addr, data, size);
7726     } else {
7727         nvme_process_db(n, addr, data);
7728     }
7729 }
7730 
7731 static const MemoryRegionOps nvme_mmio_ops = {
7732     .read = nvme_mmio_read,
7733     .write = nvme_mmio_write,
7734     .endianness = DEVICE_LITTLE_ENDIAN,
7735     .impl = {
7736         .min_access_size = 2,
7737         .max_access_size = 8,
7738     },
7739 };
7740 
7741 static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data,
7742                            unsigned size)
7743 {
7744     NvmeCtrl *n = (NvmeCtrl *)opaque;
7745     stn_le_p(&n->cmb.buf[addr], size, data);
7746 }
7747 
7748 static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size)
7749 {
7750     NvmeCtrl *n = (NvmeCtrl *)opaque;
7751     return ldn_le_p(&n->cmb.buf[addr], size);
7752 }
7753 
7754 static const MemoryRegionOps nvme_cmb_ops = {
7755     .read = nvme_cmb_read,
7756     .write = nvme_cmb_write,
7757     .endianness = DEVICE_LITTLE_ENDIAN,
7758     .impl = {
7759         .min_access_size = 1,
7760         .max_access_size = 8,
7761     },
7762 };
7763 
7764 static bool nvme_check_params(NvmeCtrl *n, Error **errp)
7765 {
7766     NvmeParams *params = &n->params;
7767 
7768     if (params->num_queues) {
7769         warn_report("num_queues is deprecated; please use max_ioqpairs "
7770                     "instead");
7771 
7772         params->max_ioqpairs = params->num_queues - 1;
7773     }
7774 
7775     if (n->namespace.blkconf.blk && n->subsys) {
7776         error_setg(errp, "subsystem support is unavailable with legacy "
7777                    "namespace ('drive' property)");
7778         return false;
7779     }
7780 
7781     if (params->max_ioqpairs < 1 ||
7782         params->max_ioqpairs > NVME_MAX_IOQPAIRS) {
7783         error_setg(errp, "max_ioqpairs must be between 1 and %d",
7784                    NVME_MAX_IOQPAIRS);
7785         return false;
7786     }
7787 
7788     if (params->msix_qsize < 1 ||
7789         params->msix_qsize > PCI_MSIX_FLAGS_QSIZE + 1) {
7790         error_setg(errp, "msix_qsize must be between 1 and %d",
7791                    PCI_MSIX_FLAGS_QSIZE + 1);
7792         return false;
7793     }
7794 
7795     if (!params->serial) {
7796         error_setg(errp, "serial property not set");
7797         return false;
7798     }
7799 
7800     if (n->pmr.dev) {
7801         if (host_memory_backend_is_mapped(n->pmr.dev)) {
7802             error_setg(errp, "can't use already busy memdev: %s",
7803                        object_get_canonical_path_component(OBJECT(n->pmr.dev)));
7804             return false;
7805         }
7806 
7807         if (!is_power_of_2(n->pmr.dev->size)) {
7808             error_setg(errp, "pmr backend size needs to be power of 2 in size");
7809             return false;
7810         }
7811 
7812         host_memory_backend_set_mapped(n->pmr.dev, true);
7813     }
7814 
7815     if (n->params.zasl > n->params.mdts) {
7816         error_setg(errp, "zoned.zasl (Zone Append Size Limit) must be less "
7817                    "than or equal to mdts (Maximum Data Transfer Size)");
7818         return false;
7819     }
7820 
7821     if (!n->params.vsl) {
7822         error_setg(errp, "vsl must be non-zero");
7823         return false;
7824     }
7825 
7826     if (params->sriov_max_vfs) {
7827         if (!n->subsys) {
7828             error_setg(errp, "subsystem is required for the use of SR-IOV");
7829             return false;
7830         }
7831 
7832         if (params->sriov_max_vfs > NVME_MAX_VFS) {
7833             error_setg(errp, "sriov_max_vfs must be between 0 and %d",
7834                        NVME_MAX_VFS);
7835             return false;
7836         }
7837 
7838         if (params->cmb_size_mb) {
7839             error_setg(errp, "CMB is not supported with SR-IOV");
7840             return false;
7841         }
7842 
7843         if (n->pmr.dev) {
7844             error_setg(errp, "PMR is not supported with SR-IOV");
7845             return false;
7846         }
7847 
7848         if (!params->sriov_vq_flexible || !params->sriov_vi_flexible) {
7849             error_setg(errp, "both sriov_vq_flexible and sriov_vi_flexible"
7850                        " must be set for the use of SR-IOV");
7851             return false;
7852         }
7853 
7854         if (params->sriov_vq_flexible < params->sriov_max_vfs * 2) {
7855             error_setg(errp, "sriov_vq_flexible must be greater than or equal"
7856                        " to %d (sriov_max_vfs * 2)", params->sriov_max_vfs * 2);
7857             return false;
7858         }
7859 
7860         if (params->max_ioqpairs < params->sriov_vq_flexible + 2) {
7861             error_setg(errp, "(max_ioqpairs - sriov_vq_flexible) must be"
7862                        " greater than or equal to 2");
7863             return false;
7864         }
7865 
7866         if (params->sriov_vi_flexible < params->sriov_max_vfs) {
7867             error_setg(errp, "sriov_vi_flexible must be greater than or equal"
7868                        " to %d (sriov_max_vfs)", params->sriov_max_vfs);
7869             return false;
7870         }
7871 
7872         if (params->msix_qsize < params->sriov_vi_flexible + 1) {
7873             error_setg(errp, "(msix_qsize - sriov_vi_flexible) must be"
7874                        " greater than or equal to 1");
7875             return false;
7876         }
7877 
7878         if (params->sriov_max_vi_per_vf &&
7879             (params->sriov_max_vi_per_vf - 1) % NVME_VF_RES_GRANULARITY) {
7880             error_setg(errp, "sriov_max_vi_per_vf must meet:"
7881                        " (sriov_max_vi_per_vf - 1) %% %d == 0 and"
7882                        " sriov_max_vi_per_vf >= 1", NVME_VF_RES_GRANULARITY);
7883             return false;
7884         }
7885 
7886         if (params->sriov_max_vq_per_vf &&
7887             (params->sriov_max_vq_per_vf < 2 ||
7888              (params->sriov_max_vq_per_vf - 1) % NVME_VF_RES_GRANULARITY)) {
7889             error_setg(errp, "sriov_max_vq_per_vf must meet:"
7890                        " (sriov_max_vq_per_vf - 1) %% %d == 0 and"
7891                        " sriov_max_vq_per_vf >= 2", NVME_VF_RES_GRANULARITY);
7892             return false;
7893         }
7894     }
7895 
7896     return true;
7897 }
7898 
7899 static void nvme_init_state(NvmeCtrl *n)
7900 {
7901     NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
7902     NvmeSecCtrlList *list = &n->sec_ctrl_list;
7903     NvmeSecCtrlEntry *sctrl;
7904     PCIDevice *pci = PCI_DEVICE(n);
7905     uint8_t max_vfs;
7906     int i;
7907 
7908     if (pci_is_vf(pci)) {
7909         sctrl = nvme_sctrl(n);
7910         max_vfs = 0;
7911         n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
7912         n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
7913     } else {
7914         max_vfs = n->params.sriov_max_vfs;
7915         n->conf_ioqpairs = n->params.max_ioqpairs;
7916         n->conf_msix_qsize = n->params.msix_qsize;
7917     }
7918 
7919     n->sq = g_new0(NvmeSQueue *, n->params.max_ioqpairs + 1);
7920     n->cq = g_new0(NvmeCQueue *, n->params.max_ioqpairs + 1);
7921     n->temperature = NVME_TEMPERATURE;
7922     n->features.temp_thresh_hi = NVME_TEMPERATURE_WARNING;
7923     n->starttime_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
7924     n->aer_reqs = g_new0(NvmeRequest *, n->params.aerl + 1);
7925     QTAILQ_INIT(&n->aer_queue);
7926 
7927     list->numcntl = cpu_to_le16(max_vfs);
7928     for (i = 0; i < max_vfs; i++) {
7929         sctrl = &list->sec[i];
7930         sctrl->pcid = cpu_to_le16(n->cntlid);
7931         sctrl->vfn = cpu_to_le16(i + 1);
7932     }
7933 
7934     cap->cntlid = cpu_to_le16(n->cntlid);
7935     cap->crt = NVME_CRT_VQ | NVME_CRT_VI;
7936 
7937     if (pci_is_vf(pci)) {
7938         cap->vqprt = cpu_to_le16(1 + n->conf_ioqpairs);
7939     } else {
7940         cap->vqprt = cpu_to_le16(1 + n->params.max_ioqpairs -
7941                                  n->params.sriov_vq_flexible);
7942         cap->vqfrt = cpu_to_le32(n->params.sriov_vq_flexible);
7943         cap->vqrfap = cap->vqfrt;
7944         cap->vqgran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
7945         cap->vqfrsm = n->params.sriov_max_vq_per_vf ?
7946                         cpu_to_le16(n->params.sriov_max_vq_per_vf) :
7947                         cap->vqfrt / MAX(max_vfs, 1);
7948     }
7949 
7950     if (pci_is_vf(pci)) {
7951         cap->viprt = cpu_to_le16(n->conf_msix_qsize);
7952     } else {
7953         cap->viprt = cpu_to_le16(n->params.msix_qsize -
7954                                  n->params.sriov_vi_flexible);
7955         cap->vifrt = cpu_to_le32(n->params.sriov_vi_flexible);
7956         cap->virfap = cap->vifrt;
7957         cap->vigran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
7958         cap->vifrsm = n->params.sriov_max_vi_per_vf ?
7959                         cpu_to_le16(n->params.sriov_max_vi_per_vf) :
7960                         cap->vifrt / MAX(max_vfs, 1);
7961     }
7962 }
7963 
7964 static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
7965 {
7966     uint64_t cmb_size = n->params.cmb_size_mb * MiB;
7967     uint64_t cap = ldq_le_p(&n->bar.cap);
7968 
7969     n->cmb.buf = g_malloc0(cmb_size);
7970     memory_region_init_io(&n->cmb.mem, OBJECT(n), &nvme_cmb_ops, n,
7971                           "nvme-cmb", cmb_size);
7972     pci_register_bar(pci_dev, NVME_CMB_BIR,
7973                      PCI_BASE_ADDRESS_SPACE_MEMORY |
7974                      PCI_BASE_ADDRESS_MEM_TYPE_64 |
7975                      PCI_BASE_ADDRESS_MEM_PREFETCH, &n->cmb.mem);
7976 
7977     NVME_CAP_SET_CMBS(cap, 1);
7978     stq_le_p(&n->bar.cap, cap);
7979 
7980     if (n->params.legacy_cmb) {
7981         nvme_cmb_enable_regs(n);
7982         n->cmb.cmse = true;
7983     }
7984 }
7985 
7986 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
7987 {
7988     uint32_t pmrcap = ldl_le_p(&n->bar.pmrcap);
7989 
7990     NVME_PMRCAP_SET_RDS(pmrcap, 1);
7991     NVME_PMRCAP_SET_WDS(pmrcap, 1);
7992     NVME_PMRCAP_SET_BIR(pmrcap, NVME_PMR_BIR);
7993     /* Turn on bit 1 support */
7994     NVME_PMRCAP_SET_PMRWBM(pmrcap, 0x02);
7995     NVME_PMRCAP_SET_CMSS(pmrcap, 1);
7996     stl_le_p(&n->bar.pmrcap, pmrcap);
7997 
7998     pci_register_bar(pci_dev, NVME_PMR_BIR,
7999                      PCI_BASE_ADDRESS_SPACE_MEMORY |
8000                      PCI_BASE_ADDRESS_MEM_TYPE_64 |
8001                      PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmr.dev->mr);
8002 
8003     memory_region_set_enabled(&n->pmr.dev->mr, false);
8004 }
8005 
8006 static uint64_t nvme_bar_size(unsigned total_queues, unsigned total_irqs,
8007                               unsigned *msix_table_offset,
8008                               unsigned *msix_pba_offset)
8009 {
8010     uint64_t bar_size, msix_table_size, msix_pba_size;
8011 
8012     bar_size = sizeof(NvmeBar) + 2 * total_queues * NVME_DB_SIZE;
8013     bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8014 
8015     if (msix_table_offset) {
8016         *msix_table_offset = bar_size;
8017     }
8018 
8019     msix_table_size = PCI_MSIX_ENTRY_SIZE * total_irqs;
8020     bar_size += msix_table_size;
8021     bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8022 
8023     if (msix_pba_offset) {
8024         *msix_pba_offset = bar_size;
8025     }
8026 
8027     msix_pba_size = QEMU_ALIGN_UP(total_irqs, 64) / 8;
8028     bar_size += msix_pba_size;
8029 
8030     bar_size = pow2ceil(bar_size);
8031     return bar_size;
8032 }
8033 
8034 static void nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset)
8035 {
8036     uint16_t vf_dev_id = n->params.use_intel_id ?
8037                          PCI_DEVICE_ID_INTEL_NVME : PCI_DEVICE_ID_REDHAT_NVME;
8038     NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
8039     uint64_t bar_size = nvme_bar_size(le16_to_cpu(cap->vqfrsm),
8040                                       le16_to_cpu(cap->vifrsm),
8041                                       NULL, NULL);
8042 
8043     pcie_sriov_pf_init(pci_dev, offset, "nvme", vf_dev_id,
8044                        n->params.sriov_max_vfs, n->params.sriov_max_vfs,
8045                        NVME_VF_OFFSET, NVME_VF_STRIDE);
8046 
8047     pcie_sriov_pf_init_vf_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
8048                               PCI_BASE_ADDRESS_MEM_TYPE_64, bar_size);
8049 }
8050 
8051 static int nvme_add_pm_capability(PCIDevice *pci_dev, uint8_t offset)
8052 {
8053     Error *err = NULL;
8054     int ret;
8055 
8056     ret = pci_add_capability(pci_dev, PCI_CAP_ID_PM, offset,
8057                              PCI_PM_SIZEOF, &err);
8058     if (err) {
8059         error_report_err(err);
8060         return ret;
8061     }
8062 
8063     pci_set_word(pci_dev->config + offset + PCI_PM_PMC,
8064                  PCI_PM_CAP_VER_1_2);
8065     pci_set_word(pci_dev->config + offset + PCI_PM_CTRL,
8066                  PCI_PM_CTRL_NO_SOFT_RESET);
8067     pci_set_word(pci_dev->wmask + offset + PCI_PM_CTRL,
8068                  PCI_PM_CTRL_STATE_MASK);
8069 
8070     return 0;
8071 }
8072 
8073 static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
8074 {
8075     ERRP_GUARD();
8076     uint8_t *pci_conf = pci_dev->config;
8077     uint64_t bar_size;
8078     unsigned msix_table_offset, msix_pba_offset;
8079     int ret;
8080 
8081     pci_conf[PCI_INTERRUPT_PIN] = 1;
8082     pci_config_set_prog_interface(pci_conf, 0x2);
8083 
8084     if (n->params.use_intel_id) {
8085         pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL);
8086         pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_INTEL_NVME);
8087     } else {
8088         pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_REDHAT);
8089         pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_REDHAT_NVME);
8090     }
8091 
8092     pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
8093     nvme_add_pm_capability(pci_dev, 0x60);
8094     pcie_endpoint_cap_init(pci_dev, 0x80);
8095     pcie_cap_flr_init(pci_dev);
8096     if (n->params.sriov_max_vfs) {
8097         pcie_ari_init(pci_dev, 0x100);
8098     }
8099 
8100     /* add one to max_ioqpairs to account for the admin queue pair */
8101     bar_size = nvme_bar_size(n->params.max_ioqpairs + 1, n->params.msix_qsize,
8102                              &msix_table_offset, &msix_pba_offset);
8103 
8104     memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
8105     memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
8106                           msix_table_offset);
8107     memory_region_add_subregion(&n->bar0, 0, &n->iomem);
8108 
8109     if (pci_is_vf(pci_dev)) {
8110         pcie_sriov_vf_register_bar(pci_dev, 0, &n->bar0);
8111     } else {
8112         pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
8113                          PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
8114     }
8115     ret = msix_init(pci_dev, n->params.msix_qsize,
8116                     &n->bar0, 0, msix_table_offset,
8117                     &n->bar0, 0, msix_pba_offset, 0, errp);
8118     if (ret == -ENOTSUP) {
8119         /* report that msix is not supported, but do not error out */
8120         warn_report_err(*errp);
8121         *errp = NULL;
8122     } else if (ret < 0) {
8123         /* propagate error to caller */
8124         return false;
8125     }
8126 
8127     nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
8128 
8129     if (n->params.cmb_size_mb) {
8130         nvme_init_cmb(n, pci_dev);
8131     }
8132 
8133     if (n->pmr.dev) {
8134         nvme_init_pmr(n, pci_dev);
8135     }
8136 
8137     if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
8138         nvme_init_sriov(n, pci_dev, 0x120);
8139     }
8140 
8141     return true;
8142 }
8143 
8144 static void nvme_init_subnqn(NvmeCtrl *n)
8145 {
8146     NvmeSubsystem *subsys = n->subsys;
8147     NvmeIdCtrl *id = &n->id_ctrl;
8148 
8149     if (!subsys) {
8150         snprintf((char *)id->subnqn, sizeof(id->subnqn),
8151                  "nqn.2019-08.org.qemu:%s", n->params.serial);
8152     } else {
8153         pstrcpy((char *)id->subnqn, sizeof(id->subnqn), (char*)subsys->subnqn);
8154     }
8155 }
8156 
8157 static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
8158 {
8159     NvmeIdCtrl *id = &n->id_ctrl;
8160     uint8_t *pci_conf = pci_dev->config;
8161     uint64_t cap = ldq_le_p(&n->bar.cap);
8162     NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
8163     uint32_t ctratt;
8164 
8165     id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
8166     id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
8167     strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
8168     strpadcpy((char *)id->fr, sizeof(id->fr), QEMU_VERSION, ' ');
8169     strpadcpy((char *)id->sn, sizeof(id->sn), n->params.serial, ' ');
8170 
8171     id->cntlid = cpu_to_le16(n->cntlid);
8172 
8173     id->oaes = cpu_to_le32(NVME_OAES_NS_ATTR);
8174     ctratt = NVME_CTRATT_ELBAS;
8175 
8176     id->rab = 6;
8177 
8178     if (n->params.use_intel_id) {
8179         id->ieee[0] = 0xb3;
8180         id->ieee[1] = 0x02;
8181         id->ieee[2] = 0x00;
8182     } else {
8183         id->ieee[0] = 0x00;
8184         id->ieee[1] = 0x54;
8185         id->ieee[2] = 0x52;
8186     }
8187 
8188     id->mdts = n->params.mdts;
8189     id->ver = cpu_to_le32(NVME_SPEC_VER);
8190     id->oacs =
8191         cpu_to_le16(NVME_OACS_NS_MGMT | NVME_OACS_FORMAT | NVME_OACS_DBBUF |
8192                     NVME_OACS_DIRECTIVES);
8193     id->cntrltype = 0x1;
8194 
8195     /*
8196      * Because the controller always completes the Abort command immediately,
8197      * there can never be more than one concurrently executing Abort command,
8198      * so this value is never used for anything. Note that there can easily be
8199      * many Abort commands in the queues, but they are not considered
8200      * "executing" until processed by nvme_abort.
8201      *
8202      * The specification recommends a value of 3 for Abort Command Limit (four
8203      * concurrently outstanding Abort commands), so lets use that though it is
8204      * inconsequential.
8205      */
8206     id->acl = 3;
8207     id->aerl = n->params.aerl;
8208     id->frmw = (NVME_NUM_FW_SLOTS << 1) | NVME_FRMW_SLOT1_RO;
8209     id->lpa = NVME_LPA_NS_SMART | NVME_LPA_CSE | NVME_LPA_EXTENDED;
8210 
8211     /* recommended default value (~70 C) */
8212     id->wctemp = cpu_to_le16(NVME_TEMPERATURE_WARNING);
8213     id->cctemp = cpu_to_le16(NVME_TEMPERATURE_CRITICAL);
8214 
8215     id->sqes = (NVME_SQES << 4) | NVME_SQES;
8216     id->cqes = (NVME_CQES << 4) | NVME_CQES;
8217     id->nn = cpu_to_le32(NVME_MAX_NAMESPACES);
8218     id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROES | NVME_ONCS_TIMESTAMP |
8219                            NVME_ONCS_FEATURES | NVME_ONCS_DSM |
8220                            NVME_ONCS_COMPARE | NVME_ONCS_COPY);
8221 
8222     /*
8223      * NOTE: If this device ever supports a command set that does NOT use 0x0
8224      * as a Flush-equivalent operation, support for the broadcast NSID in Flush
8225      * should probably be removed.
8226      *
8227      * See comment in nvme_io_cmd.
8228      */
8229     id->vwc = NVME_VWC_NSID_BROADCAST_SUPPORT | NVME_VWC_PRESENT;
8230 
8231     id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1);
8232     id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN);
8233 
8234     nvme_init_subnqn(n);
8235 
8236     id->psd[0].mp = cpu_to_le16(0x9c4);
8237     id->psd[0].enlat = cpu_to_le32(0x10);
8238     id->psd[0].exlat = cpu_to_le32(0x4);
8239 
8240     if (n->subsys) {
8241         id->cmic |= NVME_CMIC_MULTI_CTRL;
8242         ctratt |= NVME_CTRATT_ENDGRPS;
8243 
8244         id->endgidmax = cpu_to_le16(0x1);
8245 
8246         if (n->subsys->endgrp.fdp.enabled) {
8247             ctratt |= NVME_CTRATT_FDPS;
8248         }
8249     }
8250 
8251     id->ctratt = cpu_to_le32(ctratt);
8252 
8253     NVME_CAP_SET_MQES(cap, 0x7ff);
8254     NVME_CAP_SET_CQR(cap, 1);
8255     NVME_CAP_SET_TO(cap, 0xf);
8256     NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_NVM);
8257     NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_CSI_SUPP);
8258     NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_ADMIN_ONLY);
8259     NVME_CAP_SET_MPSMAX(cap, 4);
8260     NVME_CAP_SET_CMBS(cap, n->params.cmb_size_mb ? 1 : 0);
8261     NVME_CAP_SET_PMRS(cap, n->pmr.dev ? 1 : 0);
8262     stq_le_p(&n->bar.cap, cap);
8263 
8264     stl_le_p(&n->bar.vs, NVME_SPEC_VER);
8265     n->bar.intmc = n->bar.intms = 0;
8266 
8267     if (pci_is_vf(pci_dev) && !sctrl->scs) {
8268         stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
8269     }
8270 }
8271 
8272 static int nvme_init_subsys(NvmeCtrl *n, Error **errp)
8273 {
8274     int cntlid;
8275 
8276     if (!n->subsys) {
8277         return 0;
8278     }
8279 
8280     cntlid = nvme_subsys_register_ctrl(n, errp);
8281     if (cntlid < 0) {
8282         return -1;
8283     }
8284 
8285     n->cntlid = cntlid;
8286 
8287     return 0;
8288 }
8289 
8290 void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns)
8291 {
8292     uint32_t nsid = ns->params.nsid;
8293     assert(nsid && nsid <= NVME_MAX_NAMESPACES);
8294 
8295     n->namespaces[nsid] = ns;
8296     ns->attached++;
8297 
8298     n->dmrsl = MIN_NON_ZERO(n->dmrsl,
8299                             BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
8300 }
8301 
8302 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
8303 {
8304     NvmeCtrl *n = NVME(pci_dev);
8305     DeviceState *dev = DEVICE(pci_dev);
8306     NvmeNamespace *ns;
8307     NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));
8308 
8309     if (pci_is_vf(pci_dev)) {
8310         /*
8311          * VFs derive settings from the parent. PF's lifespan exceeds
8312          * that of VF's, so it's safe to share params.serial.
8313          */
8314         memcpy(&n->params, &pn->params, sizeof(NvmeParams));
8315         n->subsys = pn->subsys;
8316     }
8317 
8318     if (!nvme_check_params(n, errp)) {
8319         return;
8320     }
8321 
8322     qbus_init(&n->bus, sizeof(NvmeBus), TYPE_NVME_BUS, dev, dev->id);
8323 
8324     if (nvme_init_subsys(n, errp)) {
8325         return;
8326     }
8327     nvme_init_state(n);
8328     if (!nvme_init_pci(n, pci_dev, errp)) {
8329         return;
8330     }
8331     nvme_init_ctrl(n, pci_dev);
8332 
8333     /* setup a namespace if the controller drive property was given */
8334     if (n->namespace.blkconf.blk) {
8335         ns = &n->namespace;
8336         ns->params.nsid = 1;
8337 
8338         if (nvme_ns_setup(ns, errp)) {
8339             return;
8340         }
8341 
8342         nvme_attach_ns(n, ns);
8343     }
8344 }
8345 
8346 static void nvme_exit(PCIDevice *pci_dev)
8347 {
8348     NvmeCtrl *n = NVME(pci_dev);
8349     NvmeNamespace *ns;
8350     int i;
8351 
8352     nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
8353 
8354     if (n->subsys) {
8355         for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
8356             ns = nvme_ns(n, i);
8357             if (ns) {
8358                 ns->attached--;
8359             }
8360         }
8361 
8362         nvme_subsys_unregister_ctrl(n->subsys, n);
8363     }
8364 
8365     g_free(n->cq);
8366     g_free(n->sq);
8367     g_free(n->aer_reqs);
8368 
8369     if (n->params.cmb_size_mb) {
8370         g_free(n->cmb.buf);
8371     }
8372 
8373     if (n->pmr.dev) {
8374         host_memory_backend_set_mapped(n->pmr.dev, false);
8375     }
8376 
8377     if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
8378         pcie_sriov_pf_exit(pci_dev);
8379     }
8380 
8381     msix_uninit(pci_dev, &n->bar0, &n->bar0);
8382     memory_region_del_subregion(&n->bar0, &n->iomem);
8383 }
8384 
8385 static Property nvme_props[] = {
8386     DEFINE_BLOCK_PROPERTIES(NvmeCtrl, namespace.blkconf),
8387     DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmr.dev, TYPE_MEMORY_BACKEND,
8388                      HostMemoryBackend *),
8389     DEFINE_PROP_LINK("subsys", NvmeCtrl, subsys, TYPE_NVME_SUBSYS,
8390                      NvmeSubsystem *),
8391     DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial),
8392     DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0),
8393     DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0),
8394     DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl, params.max_ioqpairs, 64),
8395     DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl, params.msix_qsize, 65),
8396     DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
8397     DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64),
8398     DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7),
8399     DEFINE_PROP_UINT8("vsl", NvmeCtrl, params.vsl, 7),
8400     DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl, params.use_intel_id, false),
8401     DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl, params.legacy_cmb, false),
8402     DEFINE_PROP_BOOL("ioeventfd", NvmeCtrl, params.ioeventfd, false),
8403     DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl, params.zasl, 0),
8404     DEFINE_PROP_BOOL("zoned.auto_transition", NvmeCtrl,
8405                      params.auto_transition_zones, true),
8406     DEFINE_PROP_UINT8("sriov_max_vfs", NvmeCtrl, params.sriov_max_vfs, 0),
8407     DEFINE_PROP_UINT16("sriov_vq_flexible", NvmeCtrl,
8408                        params.sriov_vq_flexible, 0),
8409     DEFINE_PROP_UINT16("sriov_vi_flexible", NvmeCtrl,
8410                        params.sriov_vi_flexible, 0),
8411     DEFINE_PROP_UINT8("sriov_max_vi_per_vf", NvmeCtrl,
8412                       params.sriov_max_vi_per_vf, 0),
8413     DEFINE_PROP_UINT8("sriov_max_vq_per_vf", NvmeCtrl,
8414                       params.sriov_max_vq_per_vf, 0),
8415     DEFINE_PROP_END_OF_LIST(),
8416 };
8417 
8418 static void nvme_get_smart_warning(Object *obj, Visitor *v, const char *name,
8419                                    void *opaque, Error **errp)
8420 {
8421     NvmeCtrl *n = NVME(obj);
8422     uint8_t value = n->smart_critical_warning;
8423 
8424     visit_type_uint8(v, name, &value, errp);
8425 }
8426 
8427 static void nvme_set_smart_warning(Object *obj, Visitor *v, const char *name,
8428                                    void *opaque, Error **errp)
8429 {
8430     NvmeCtrl *n = NVME(obj);
8431     uint8_t value, old_value, cap = 0, index, event;
8432 
8433     if (!visit_type_uint8(v, name, &value, errp)) {
8434         return;
8435     }
8436 
8437     cap = NVME_SMART_SPARE | NVME_SMART_TEMPERATURE | NVME_SMART_RELIABILITY
8438           | NVME_SMART_MEDIA_READ_ONLY | NVME_SMART_FAILED_VOLATILE_MEDIA;
8439     if (NVME_CAP_PMRS(ldq_le_p(&n->bar.cap))) {
8440         cap |= NVME_SMART_PMR_UNRELIABLE;
8441     }
8442 
8443     if ((value & cap) != value) {
8444         error_setg(errp, "unsupported smart critical warning bits: 0x%x",
8445                    value & ~cap);
8446         return;
8447     }
8448 
8449     old_value = n->smart_critical_warning;
8450     n->smart_critical_warning = value;
8451 
8452     /* only inject new bits of smart critical warning */
8453     for (index = 0; index < NVME_SMART_WARN_MAX; index++) {
8454         event = 1 << index;
8455         if (value & ~old_value & event)
8456             nvme_smart_event(n, event);
8457     }
8458 }
8459 
8460 static void nvme_pci_reset(DeviceState *qdev)
8461 {
8462     PCIDevice *pci_dev = PCI_DEVICE(qdev);
8463     NvmeCtrl *n = NVME(pci_dev);
8464 
8465     trace_pci_nvme_pci_reset();
8466     nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
8467 }
8468 
8469 static void nvme_sriov_pre_write_ctrl(PCIDevice *dev, uint32_t address,
8470                                       uint32_t val, int len)
8471 {
8472     NvmeCtrl *n = NVME(dev);
8473     NvmeSecCtrlEntry *sctrl;
8474     uint16_t sriov_cap = dev->exp.sriov_cap;
8475     uint32_t off = address - sriov_cap;
8476     int i, num_vfs;
8477 
8478     if (!sriov_cap) {
8479         return;
8480     }
8481 
8482     if (range_covers_byte(off, len, PCI_SRIOV_CTRL)) {
8483         if (!(val & PCI_SRIOV_CTRL_VFE)) {
8484             num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
8485             for (i = 0; i < num_vfs; i++) {
8486                 sctrl = &n->sec_ctrl_list.sec[i];
8487                 nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
8488             }
8489         }
8490     }
8491 }
8492 
8493 static void nvme_pci_write_config(PCIDevice *dev, uint32_t address,
8494                                   uint32_t val, int len)
8495 {
8496     nvme_sriov_pre_write_ctrl(dev, address, val, len);
8497     pci_default_write_config(dev, address, val, len);
8498     pcie_cap_flr_write_config(dev, address, val, len);
8499 }
8500 
8501 static const VMStateDescription nvme_vmstate = {
8502     .name = "nvme",
8503     .unmigratable = 1,
8504 };
8505 
8506 static void nvme_class_init(ObjectClass *oc, void *data)
8507 {
8508     DeviceClass *dc = DEVICE_CLASS(oc);
8509     PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
8510 
8511     pc->realize = nvme_realize;
8512     pc->config_write = nvme_pci_write_config;
8513     pc->exit = nvme_exit;
8514     pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
8515     pc->revision = 2;
8516 
8517     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
8518     dc->desc = "Non-Volatile Memory Express";
8519     device_class_set_props(dc, nvme_props);
8520     dc->vmsd = &nvme_vmstate;
8521     dc->reset = nvme_pci_reset;
8522 }
8523 
8524 static void nvme_instance_init(Object *obj)
8525 {
8526     NvmeCtrl *n = NVME(obj);
8527 
8528     device_add_bootindex_property(obj, &n->namespace.blkconf.bootindex,
8529                                   "bootindex", "/namespace@1,0",
8530                                   DEVICE(obj));
8531 
8532     object_property_add(obj, "smart_critical_warning", "uint8",
8533                         nvme_get_smart_warning,
8534                         nvme_set_smart_warning, NULL, NULL);
8535 }
8536 
8537 static const TypeInfo nvme_info = {
8538     .name          = TYPE_NVME,
8539     .parent        = TYPE_PCI_DEVICE,
8540     .instance_size = sizeof(NvmeCtrl),
8541     .instance_init = nvme_instance_init,
8542     .class_init    = nvme_class_init,
8543     .interfaces = (InterfaceInfo[]) {
8544         { INTERFACE_PCIE_DEVICE },
8545         { }
8546     },
8547 };
8548 
8549 static const TypeInfo nvme_bus_info = {
8550     .name = TYPE_NVME_BUS,
8551     .parent = TYPE_BUS,
8552     .instance_size = sizeof(NvmeBus),
8553 };
8554 
8555 static void nvme_register_types(void)
8556 {
8557     type_register_static(&nvme_info);
8558     type_register_static(&nvme_bus_info);
8559 }
8560 
8561 type_init(nvme_register_types)
8562