1 /*
2 * QEMU NVM Express Controller
3 *
4 * Copyright (c) 2012, Intel Corporation
5 *
6 * Written by Keith Busch <keith.busch@intel.com>
7 *
8 * This code is licensed under the GNU GPL v2 or later.
9 */
10
11 /**
12 * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e
13 *
14 * https://nvmexpress.org/developers/nvme-specification/
15 *
16 *
17 * Notes on coding style
18 * ---------------------
19 * While QEMU coding style prefers lowercase hexadecimals in constants, the
20 * NVMe subsystem use this format from the NVMe specifications in the comments
21 * (i.e. 'h' suffix instead of '0x' prefix).
22 *
23 * Usage
24 * -----
25 * See docs/system/nvme.rst for extensive documentation.
26 *
27 * Add options:
28 * -drive file=<file>,if=none,id=<drive_id>
29 * -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id>
30 * -device nvme,serial=<serial>,id=<bus_name>, \
31 * cmb_size_mb=<cmb_size_mb[optional]>, \
32 * [pmrdev=<mem_backend_file_id>,] \
33 * max_ioqpairs=<N[optional]>, \
34 * aerl=<N[optional]>,aer_max_queued=<N[optional]>, \
35 * mdts=<N[optional]>,vsl=<N[optional]>, \
36 * zoned.zasl=<N[optional]>, \
37 * zoned.auto_transition=<on|off[optional]>, \
38 * sriov_max_vfs=<N[optional]> \
39 * sriov_vq_flexible=<N[optional]> \
40 * sriov_vi_flexible=<N[optional]> \
41 * sriov_max_vi_per_vf=<N[optional]> \
42 * sriov_max_vq_per_vf=<N[optional]> \
43 * subsys=<subsys_id>
44 * -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
45 * zoned=<true|false[optional]>, \
46 * subsys=<subsys_id>,shared=<true|false[optional]>, \
47 * detached=<true|false[optional]>, \
48 * zoned.zone_size=<N[optional]>, \
49 * zoned.zone_capacity=<N[optional]>, \
50 * zoned.descr_ext_size=<N[optional]>, \
51 * zoned.max_active=<N[optional]>, \
52 * zoned.max_open=<N[optional]>, \
53 * zoned.cross_read=<true|false[optional]>
54 *
55 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
56 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
57 * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to
58 * always enable the CMBLOC and CMBSZ registers (v1.3 behavior).
59 *
60 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
61 * For example:
62 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
63 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
64 *
65 * The PMR will use BAR 4/5 exclusively.
66 *
67 * To place controller(s) and namespace(s) to a subsystem, then provide
68 * nvme-subsys device as above.
69 *
70 * nvme subsystem device parameters
71 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
72 * - `nqn`
73 * This parameter provides the `<nqn_id>` part of the string
74 * `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field
75 * of subsystem controllers. Note that `<nqn_id>` should be unique per
76 * subsystem, but this is not enforced by QEMU. If not specified, it will
77 * default to the value of the `id` parameter (`<subsys_id>`).
78 *
79 * nvme device parameters
80 * ~~~~~~~~~~~~~~~~~~~~~~
81 * - `subsys`
82 * Specifying this parameter attaches the controller to the subsystem and
83 * the SUBNQN field in the controller will report the NQN of the subsystem
84 * device. This also enables multi controller capability represented in
85 * Identify Controller data structure in CMIC (Controller Multi-path I/O and
86 * Namespace Sharing Capabilities).
87 *
88 * - `aerl`
89 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
90 * of concurrently outstanding Asynchronous Event Request commands support
91 * by the controller. This is a 0's based value.
92 *
93 * - `aer_max_queued`
94 * This is the maximum number of events that the device will enqueue for
95 * completion when there are no outstanding AERs. When the maximum number of
96 * enqueued events are reached, subsequent events will be dropped.
97 *
98 * - `mdts`
99 * Indicates the maximum data transfer size for a command that transfers data
100 * between host-accessible memory and the controller. The value is specified
101 * as a power of two (2^n) and is in units of the minimum memory page size
102 * (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB).
103 *
104 * - `vsl`
105 * Indicates the maximum data size limit for the Verify command. Like `mdts`,
106 * this value is specified as a power of two (2^n) and is in units of the
107 * minimum memory page size (CAP.MPSMIN). The default value is 7 (i.e. 512
108 * KiB).
109 *
110 * - `zoned.zasl`
111 * Indicates the maximum data transfer size for the Zone Append command. Like
112 * `mdts`, the value is specified as a power of two (2^n) and is in units of
113 * the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e.
114 * defaulting to the value of `mdts`).
115 *
116 * - `zoned.auto_transition`
117 * Indicates if zones in zone state implicitly opened can be automatically
118 * transitioned to zone state closed for resource management purposes.
119 * Defaults to 'on'.
120 *
121 * - `sriov_max_vfs`
122 * Indicates the maximum number of PCIe virtual functions supported
123 * by the controller. The default value is 0. Specifying a non-zero value
124 * enables reporting of both SR-IOV and ARI capabilities by the NVMe device.
125 * Virtual function controllers will not report SR-IOV capability.
126 *
127 * NOTE: Single Root I/O Virtualization support is experimental.
128 * All the related parameters may be subject to change.
129 *
130 * - `sriov_vq_flexible`
131 * Indicates the total number of flexible queue resources assignable to all
132 * the secondary controllers. Implicitly sets the number of primary
133 * controller's private resources to `(max_ioqpairs - sriov_vq_flexible)`.
134 *
135 * - `sriov_vi_flexible`
136 * Indicates the total number of flexible interrupt resources assignable to
137 * all the secondary controllers. Implicitly sets the number of primary
138 * controller's private resources to `(msix_qsize - sriov_vi_flexible)`.
139 *
140 * - `sriov_max_vi_per_vf`
141 * Indicates the maximum number of virtual interrupt resources assignable
142 * to a secondary controller. The default 0 resolves to
143 * `(sriov_vi_flexible / sriov_max_vfs)`.
144 *
145 * - `sriov_max_vq_per_vf`
146 * Indicates the maximum number of virtual queue resources assignable to
147 * a secondary controller. The default 0 resolves to
148 * `(sriov_vq_flexible / sriov_max_vfs)`.
149 *
150 * nvme namespace device parameters
151 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
152 * - `shared`
153 * When the parent nvme device (as defined explicitly by the 'bus' parameter
154 * or implicitly by the most recently defined NvmeBus) is linked to an
155 * nvme-subsys device, the namespace will be attached to all controllers in
156 * the subsystem. If set to 'off' (the default), the namespace will remain a
157 * private namespace and may only be attached to a single controller at a
158 * time.
159 *
160 * - `detached`
161 * This parameter is only valid together with the `subsys` parameter. If left
162 * at the default value (`false/off`), the namespace will be attached to all
163 * controllers in the NVMe subsystem at boot-up. If set to `true/on`, the
164 * namespace will be available in the subsystem but not attached to any
165 * controllers.
166 *
167 * Setting `zoned` to true selects Zoned Command Set at the namespace.
168 * In this case, the following namespace properties are available to configure
169 * zoned operation:
170 * zoned.zone_size=<zone size in bytes, default: 128MiB>
171 * The number may be followed by K, M, G as in kilo-, mega- or giga-.
172 *
173 * zoned.zone_capacity=<zone capacity in bytes, default: zone size>
174 * The value 0 (default) forces zone capacity to be the same as zone
175 * size. The value of this property may not exceed zone size.
176 *
177 * zoned.descr_ext_size=<zone descriptor extension size, default 0>
178 * This value needs to be specified in 64B units. If it is zero,
179 * namespace(s) will not support zone descriptor extensions.
180 *
181 * zoned.max_active=<Maximum Active Resources (zones), default: 0>
182 * The default value means there is no limit to the number of
183 * concurrently active zones.
184 *
185 * zoned.max_open=<Maximum Open Resources (zones), default: 0>
186 * The default value means there is no limit to the number of
187 * concurrently open zones.
188 *
189 * zoned.cross_read=<enable RAZB, default: false>
190 * Setting this property to true enables Read Across Zone Boundaries.
191 */
192
193 #include "qemu/osdep.h"
194 #include "qemu/cutils.h"
195 #include "qemu/error-report.h"
196 #include "qemu/log.h"
197 #include "qemu/units.h"
198 #include "qemu/range.h"
199 #include "qapi/error.h"
200 #include "qapi/visitor.h"
201 #include "sysemu/sysemu.h"
202 #include "sysemu/block-backend.h"
203 #include "sysemu/hostmem.h"
204 #include "hw/pci/msix.h"
205 #include "hw/pci/pcie_sriov.h"
206 #include "sysemu/spdm-socket.h"
207 #include "migration/vmstate.h"
208
209 #include "nvme.h"
210 #include "dif.h"
211 #include "trace.h"
212
213 #define NVME_MAX_IOQPAIRS 0xffff
214 #define NVME_DB_SIZE 4
215 #define NVME_SPEC_VER 0x00010400
216 #define NVME_CMB_BIR 2
217 #define NVME_PMR_BIR 4
218 #define NVME_TEMPERATURE 0x143
219 #define NVME_TEMPERATURE_WARNING 0x157
220 #define NVME_TEMPERATURE_CRITICAL 0x175
221 #define NVME_NUM_FW_SLOTS 1
222 #define NVME_DEFAULT_MAX_ZA_SIZE (128 * KiB)
223 #define NVME_VF_RES_GRANULARITY 1
224 #define NVME_VF_OFFSET 0x1
225 #define NVME_VF_STRIDE 1
226
227 #define NVME_GUEST_ERR(trace, fmt, ...) \
228 do { \
229 (trace_##trace)(__VA_ARGS__); \
230 qemu_log_mask(LOG_GUEST_ERROR, #trace \
231 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
232 } while (0)
233
234 static const bool nvme_feature_support[NVME_FID_MAX] = {
235 [NVME_ARBITRATION] = true,
236 [NVME_POWER_MANAGEMENT] = true,
237 [NVME_TEMPERATURE_THRESHOLD] = true,
238 [NVME_ERROR_RECOVERY] = true,
239 [NVME_VOLATILE_WRITE_CACHE] = true,
240 [NVME_NUMBER_OF_QUEUES] = true,
241 [NVME_INTERRUPT_COALESCING] = true,
242 [NVME_INTERRUPT_VECTOR_CONF] = true,
243 [NVME_WRITE_ATOMICITY] = true,
244 [NVME_ASYNCHRONOUS_EVENT_CONF] = true,
245 [NVME_TIMESTAMP] = true,
246 [NVME_HOST_BEHAVIOR_SUPPORT] = true,
247 [NVME_COMMAND_SET_PROFILE] = true,
248 [NVME_FDP_MODE] = true,
249 [NVME_FDP_EVENTS] = true,
250 };
251
252 static const uint32_t nvme_feature_cap[NVME_FID_MAX] = {
253 [NVME_TEMPERATURE_THRESHOLD] = NVME_FEAT_CAP_CHANGE,
254 [NVME_ERROR_RECOVERY] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
255 [NVME_VOLATILE_WRITE_CACHE] = NVME_FEAT_CAP_CHANGE,
256 [NVME_NUMBER_OF_QUEUES] = NVME_FEAT_CAP_CHANGE,
257 [NVME_ASYNCHRONOUS_EVENT_CONF] = NVME_FEAT_CAP_CHANGE,
258 [NVME_TIMESTAMP] = NVME_FEAT_CAP_CHANGE,
259 [NVME_HOST_BEHAVIOR_SUPPORT] = NVME_FEAT_CAP_CHANGE,
260 [NVME_COMMAND_SET_PROFILE] = NVME_FEAT_CAP_CHANGE,
261 [NVME_FDP_MODE] = NVME_FEAT_CAP_CHANGE,
262 [NVME_FDP_EVENTS] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
263 };
264
265 static const uint32_t nvme_cse_acs[256] = {
266 [NVME_ADM_CMD_DELETE_SQ] = NVME_CMD_EFF_CSUPP,
267 [NVME_ADM_CMD_CREATE_SQ] = NVME_CMD_EFF_CSUPP,
268 [NVME_ADM_CMD_GET_LOG_PAGE] = NVME_CMD_EFF_CSUPP,
269 [NVME_ADM_CMD_DELETE_CQ] = NVME_CMD_EFF_CSUPP,
270 [NVME_ADM_CMD_CREATE_CQ] = NVME_CMD_EFF_CSUPP,
271 [NVME_ADM_CMD_IDENTIFY] = NVME_CMD_EFF_CSUPP,
272 [NVME_ADM_CMD_ABORT] = NVME_CMD_EFF_CSUPP,
273 [NVME_ADM_CMD_SET_FEATURES] = NVME_CMD_EFF_CSUPP,
274 [NVME_ADM_CMD_GET_FEATURES] = NVME_CMD_EFF_CSUPP,
275 [NVME_ADM_CMD_ASYNC_EV_REQ] = NVME_CMD_EFF_CSUPP,
276 [NVME_ADM_CMD_NS_ATTACHMENT] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_NIC,
277 [NVME_ADM_CMD_VIRT_MNGMT] = NVME_CMD_EFF_CSUPP,
278 [NVME_ADM_CMD_DBBUF_CONFIG] = NVME_CMD_EFF_CSUPP,
279 [NVME_ADM_CMD_FORMAT_NVM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
280 [NVME_ADM_CMD_DIRECTIVE_RECV] = NVME_CMD_EFF_CSUPP,
281 [NVME_ADM_CMD_DIRECTIVE_SEND] = NVME_CMD_EFF_CSUPP,
282 };
283
284 static const uint32_t nvme_cse_iocs_none[256];
285
286 static const uint32_t nvme_cse_iocs_nvm[256] = {
287 [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
288 [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
289 [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
290 [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
291 [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
292 [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
293 [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
294 [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
295 [NVME_CMD_IO_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
296 [NVME_CMD_IO_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
297 };
298
299 static const uint32_t nvme_cse_iocs_zoned[256] = {
300 [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
301 [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
302 [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
303 [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
304 [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
305 [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
306 [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
307 [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
308 [NVME_CMD_ZONE_APPEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
309 [NVME_CMD_ZONE_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
310 [NVME_CMD_ZONE_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
311 };
312
313 static void nvme_process_sq(void *opaque);
314 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst);
315 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n);
316
nvme_sqid(NvmeRequest * req)317 static uint16_t nvme_sqid(NvmeRequest *req)
318 {
319 return le16_to_cpu(req->sq->sqid);
320 }
321
nvme_make_pid(NvmeNamespace * ns,uint16_t rg,uint16_t ph)322 static inline uint16_t nvme_make_pid(NvmeNamespace *ns, uint16_t rg,
323 uint16_t ph)
324 {
325 uint16_t rgif = ns->endgrp->fdp.rgif;
326
327 if (!rgif) {
328 return ph;
329 }
330
331 return (rg << (16 - rgif)) | ph;
332 }
333
nvme_ph_valid(NvmeNamespace * ns,uint16_t ph)334 static inline bool nvme_ph_valid(NvmeNamespace *ns, uint16_t ph)
335 {
336 return ph < ns->fdp.nphs;
337 }
338
nvme_rg_valid(NvmeEnduranceGroup * endgrp,uint16_t rg)339 static inline bool nvme_rg_valid(NvmeEnduranceGroup *endgrp, uint16_t rg)
340 {
341 return rg < endgrp->fdp.nrg;
342 }
343
nvme_pid2ph(NvmeNamespace * ns,uint16_t pid)344 static inline uint16_t nvme_pid2ph(NvmeNamespace *ns, uint16_t pid)
345 {
346 uint16_t rgif = ns->endgrp->fdp.rgif;
347
348 if (!rgif) {
349 return pid;
350 }
351
352 return pid & ((1 << (15 - rgif)) - 1);
353 }
354
nvme_pid2rg(NvmeNamespace * ns,uint16_t pid)355 static inline uint16_t nvme_pid2rg(NvmeNamespace *ns, uint16_t pid)
356 {
357 uint16_t rgif = ns->endgrp->fdp.rgif;
358
359 if (!rgif) {
360 return 0;
361 }
362
363 return pid >> (16 - rgif);
364 }
365
nvme_parse_pid(NvmeNamespace * ns,uint16_t pid,uint16_t * ph,uint16_t * rg)366 static inline bool nvme_parse_pid(NvmeNamespace *ns, uint16_t pid,
367 uint16_t *ph, uint16_t *rg)
368 {
369 *rg = nvme_pid2rg(ns, pid);
370 *ph = nvme_pid2ph(ns, pid);
371
372 return nvme_ph_valid(ns, *ph) && nvme_rg_valid(ns->endgrp, *rg);
373 }
374
nvme_assign_zone_state(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state)375 static void nvme_assign_zone_state(NvmeNamespace *ns, NvmeZone *zone,
376 NvmeZoneState state)
377 {
378 if (QTAILQ_IN_USE(zone, entry)) {
379 switch (nvme_get_zone_state(zone)) {
380 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
381 QTAILQ_REMOVE(&ns->exp_open_zones, zone, entry);
382 break;
383 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
384 QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
385 break;
386 case NVME_ZONE_STATE_CLOSED:
387 QTAILQ_REMOVE(&ns->closed_zones, zone, entry);
388 break;
389 case NVME_ZONE_STATE_FULL:
390 QTAILQ_REMOVE(&ns->full_zones, zone, entry);
391 default:
392 ;
393 }
394 }
395
396 nvme_set_zone_state(zone, state);
397
398 switch (state) {
399 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
400 QTAILQ_INSERT_TAIL(&ns->exp_open_zones, zone, entry);
401 break;
402 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
403 QTAILQ_INSERT_TAIL(&ns->imp_open_zones, zone, entry);
404 break;
405 case NVME_ZONE_STATE_CLOSED:
406 QTAILQ_INSERT_TAIL(&ns->closed_zones, zone, entry);
407 break;
408 case NVME_ZONE_STATE_FULL:
409 QTAILQ_INSERT_TAIL(&ns->full_zones, zone, entry);
410 case NVME_ZONE_STATE_READ_ONLY:
411 break;
412 default:
413 zone->d.za = 0;
414 }
415 }
416
nvme_zns_check_resources(NvmeNamespace * ns,uint32_t act,uint32_t opn,uint32_t zrwa)417 static uint16_t nvme_zns_check_resources(NvmeNamespace *ns, uint32_t act,
418 uint32_t opn, uint32_t zrwa)
419 {
420 if (ns->params.max_active_zones != 0 &&
421 ns->nr_active_zones + act > ns->params.max_active_zones) {
422 trace_pci_nvme_err_insuff_active_res(ns->params.max_active_zones);
423 return NVME_ZONE_TOO_MANY_ACTIVE | NVME_DNR;
424 }
425
426 if (ns->params.max_open_zones != 0 &&
427 ns->nr_open_zones + opn > ns->params.max_open_zones) {
428 trace_pci_nvme_err_insuff_open_res(ns->params.max_open_zones);
429 return NVME_ZONE_TOO_MANY_OPEN | NVME_DNR;
430 }
431
432 if (zrwa > ns->zns.numzrwa) {
433 return NVME_NOZRWA | NVME_DNR;
434 }
435
436 return NVME_SUCCESS;
437 }
438
439 /*
440 * Check if we can open a zone without exceeding open/active limits.
441 * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5).
442 */
nvme_aor_check(NvmeNamespace * ns,uint32_t act,uint32_t opn)443 static uint16_t nvme_aor_check(NvmeNamespace *ns, uint32_t act, uint32_t opn)
444 {
445 return nvme_zns_check_resources(ns, act, opn, 0);
446 }
447
nvme_fdp_alloc_event(NvmeCtrl * n,NvmeFdpEventBuffer * ebuf)448 static NvmeFdpEvent *nvme_fdp_alloc_event(NvmeCtrl *n, NvmeFdpEventBuffer *ebuf)
449 {
450 NvmeFdpEvent *ret = NULL;
451 bool is_full = ebuf->next == ebuf->start && ebuf->nelems;
452
453 ret = &ebuf->events[ebuf->next++];
454 if (unlikely(ebuf->next == NVME_FDP_MAX_EVENTS)) {
455 ebuf->next = 0;
456 }
457 if (is_full) {
458 ebuf->start = ebuf->next;
459 } else {
460 ebuf->nelems++;
461 }
462
463 memset(ret, 0, sizeof(NvmeFdpEvent));
464 ret->timestamp = nvme_get_timestamp(n);
465
466 return ret;
467 }
468
log_event(NvmeRuHandle * ruh,uint8_t event_type)469 static inline int log_event(NvmeRuHandle *ruh, uint8_t event_type)
470 {
471 return (ruh->event_filter >> nvme_fdp_evf_shifts[event_type]) & 0x1;
472 }
473
nvme_update_ruh(NvmeCtrl * n,NvmeNamespace * ns,uint16_t pid)474 static bool nvme_update_ruh(NvmeCtrl *n, NvmeNamespace *ns, uint16_t pid)
475 {
476 NvmeEnduranceGroup *endgrp = ns->endgrp;
477 NvmeRuHandle *ruh;
478 NvmeReclaimUnit *ru;
479 NvmeFdpEvent *e = NULL;
480 uint16_t ph, rg, ruhid;
481
482 if (!nvme_parse_pid(ns, pid, &ph, &rg)) {
483 return false;
484 }
485
486 ruhid = ns->fdp.phs[ph];
487
488 ruh = &endgrp->fdp.ruhs[ruhid];
489 ru = &ruh->rus[rg];
490
491 if (ru->ruamw) {
492 if (log_event(ruh, FDP_EVT_RU_NOT_FULLY_WRITTEN)) {
493 e = nvme_fdp_alloc_event(n, &endgrp->fdp.host_events);
494 e->type = FDP_EVT_RU_NOT_FULLY_WRITTEN;
495 e->flags = FDPEF_PIV | FDPEF_NSIDV | FDPEF_LV;
496 e->pid = cpu_to_le16(pid);
497 e->nsid = cpu_to_le32(ns->params.nsid);
498 e->rgid = cpu_to_le16(rg);
499 e->ruhid = cpu_to_le16(ruhid);
500 }
501
502 /* log (eventual) GC overhead of prematurely swapping the RU */
503 nvme_fdp_stat_inc(&endgrp->fdp.mbmw, nvme_l2b(ns, ru->ruamw));
504 }
505
506 ru->ruamw = ruh->ruamw;
507
508 return true;
509 }
510
nvme_addr_is_cmb(NvmeCtrl * n,hwaddr addr)511 static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
512 {
513 hwaddr hi, lo;
514
515 if (!n->cmb.cmse) {
516 return false;
517 }
518
519 lo = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
520 hi = lo + int128_get64(n->cmb.mem.size);
521
522 return addr >= lo && addr < hi;
523 }
524
nvme_addr_to_cmb(NvmeCtrl * n,hwaddr addr)525 static inline void *nvme_addr_to_cmb(NvmeCtrl *n, hwaddr addr)
526 {
527 hwaddr base = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
528 return &n->cmb.buf[addr - base];
529 }
530
nvme_addr_is_pmr(NvmeCtrl * n,hwaddr addr)531 static bool nvme_addr_is_pmr(NvmeCtrl *n, hwaddr addr)
532 {
533 hwaddr hi;
534
535 if (!n->pmr.cmse) {
536 return false;
537 }
538
539 hi = n->pmr.cba + int128_get64(n->pmr.dev->mr.size);
540
541 return addr >= n->pmr.cba && addr < hi;
542 }
543
nvme_addr_to_pmr(NvmeCtrl * n,hwaddr addr)544 static inline void *nvme_addr_to_pmr(NvmeCtrl *n, hwaddr addr)
545 {
546 return memory_region_get_ram_ptr(&n->pmr.dev->mr) + (addr - n->pmr.cba);
547 }
548
nvme_addr_is_iomem(NvmeCtrl * n,hwaddr addr)549 static inline bool nvme_addr_is_iomem(NvmeCtrl *n, hwaddr addr)
550 {
551 hwaddr hi, lo;
552
553 /*
554 * The purpose of this check is to guard against invalid "local" access to
555 * the iomem (i.e. controller registers). Thus, we check against the range
556 * covered by the 'bar0' MemoryRegion since that is currently composed of
557 * two subregions (the NVMe "MBAR" and the MSI-X table/pba). Note, however,
558 * that if the device model is ever changed to allow the CMB to be located
559 * in BAR0 as well, then this must be changed.
560 */
561 lo = n->bar0.addr;
562 hi = lo + int128_get64(n->bar0.size);
563
564 return addr >= lo && addr < hi;
565 }
566
nvme_addr_read(NvmeCtrl * n,hwaddr addr,void * buf,int size)567 static int nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
568 {
569 hwaddr hi = addr + size - 1;
570 if (hi < addr) {
571 return 1;
572 }
573
574 if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
575 memcpy(buf, nvme_addr_to_cmb(n, addr), size);
576 return 0;
577 }
578
579 if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
580 memcpy(buf, nvme_addr_to_pmr(n, addr), size);
581 return 0;
582 }
583
584 return pci_dma_read(PCI_DEVICE(n), addr, buf, size);
585 }
586
nvme_addr_write(NvmeCtrl * n,hwaddr addr,const void * buf,int size)587 static int nvme_addr_write(NvmeCtrl *n, hwaddr addr, const void *buf, int size)
588 {
589 hwaddr hi = addr + size - 1;
590 if (hi < addr) {
591 return 1;
592 }
593
594 if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
595 memcpy(nvme_addr_to_cmb(n, addr), buf, size);
596 return 0;
597 }
598
599 if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
600 memcpy(nvme_addr_to_pmr(n, addr), buf, size);
601 return 0;
602 }
603
604 return pci_dma_write(PCI_DEVICE(n), addr, buf, size);
605 }
606
nvme_nsid_valid(NvmeCtrl * n,uint32_t nsid)607 static bool nvme_nsid_valid(NvmeCtrl *n, uint32_t nsid)
608 {
609 return nsid &&
610 (nsid == NVME_NSID_BROADCAST || nsid <= NVME_MAX_NAMESPACES);
611 }
612
nvme_check_sqid(NvmeCtrl * n,uint16_t sqid)613 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
614 {
615 return sqid < n->conf_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1;
616 }
617
nvme_check_cqid(NvmeCtrl * n,uint16_t cqid)618 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
619 {
620 return cqid < n->conf_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1;
621 }
622
nvme_inc_cq_tail(NvmeCQueue * cq)623 static void nvme_inc_cq_tail(NvmeCQueue *cq)
624 {
625 cq->tail++;
626 if (cq->tail >= cq->size) {
627 cq->tail = 0;
628 cq->phase = !cq->phase;
629 }
630 }
631
nvme_inc_sq_head(NvmeSQueue * sq)632 static void nvme_inc_sq_head(NvmeSQueue *sq)
633 {
634 sq->head = (sq->head + 1) % sq->size;
635 }
636
nvme_cq_full(NvmeCQueue * cq)637 static uint8_t nvme_cq_full(NvmeCQueue *cq)
638 {
639 return (cq->tail + 1) % cq->size == cq->head;
640 }
641
nvme_sq_empty(NvmeSQueue * sq)642 static uint8_t nvme_sq_empty(NvmeSQueue *sq)
643 {
644 return sq->head == sq->tail;
645 }
646
nvme_irq_check(NvmeCtrl * n)647 static void nvme_irq_check(NvmeCtrl *n)
648 {
649 PCIDevice *pci = PCI_DEVICE(n);
650 uint32_t intms = ldl_le_p(&n->bar.intms);
651
652 if (msix_enabled(pci)) {
653 return;
654 }
655 if (~intms & n->irq_status) {
656 pci_irq_assert(pci);
657 } else {
658 pci_irq_deassert(pci);
659 }
660 }
661
nvme_irq_assert(NvmeCtrl * n,NvmeCQueue * cq)662 static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
663 {
664 PCIDevice *pci = PCI_DEVICE(n);
665
666 if (cq->irq_enabled) {
667 if (msix_enabled(pci)) {
668 trace_pci_nvme_irq_msix(cq->vector);
669 msix_notify(pci, cq->vector);
670 } else {
671 trace_pci_nvme_irq_pin();
672 assert(cq->vector < 32);
673 n->irq_status |= 1 << cq->vector;
674 nvme_irq_check(n);
675 }
676 } else {
677 trace_pci_nvme_irq_masked();
678 }
679 }
680
nvme_irq_deassert(NvmeCtrl * n,NvmeCQueue * cq)681 static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
682 {
683 if (cq->irq_enabled) {
684 if (msix_enabled(PCI_DEVICE(n))) {
685 return;
686 } else {
687 assert(cq->vector < 32);
688 if (!n->cq_pending) {
689 n->irq_status &= ~(1 << cq->vector);
690 }
691 nvme_irq_check(n);
692 }
693 }
694 }
695
nvme_req_clear(NvmeRequest * req)696 static void nvme_req_clear(NvmeRequest *req)
697 {
698 req->ns = NULL;
699 req->opaque = NULL;
700 req->aiocb = NULL;
701 memset(&req->cqe, 0x0, sizeof(req->cqe));
702 req->status = NVME_SUCCESS;
703 }
704
nvme_sg_init(NvmeCtrl * n,NvmeSg * sg,bool dma)705 static inline void nvme_sg_init(NvmeCtrl *n, NvmeSg *sg, bool dma)
706 {
707 if (dma) {
708 pci_dma_sglist_init(&sg->qsg, PCI_DEVICE(n), 0);
709 sg->flags = NVME_SG_DMA;
710 } else {
711 qemu_iovec_init(&sg->iov, 0);
712 }
713
714 sg->flags |= NVME_SG_ALLOC;
715 }
716
nvme_sg_unmap(NvmeSg * sg)717 static inline void nvme_sg_unmap(NvmeSg *sg)
718 {
719 if (!(sg->flags & NVME_SG_ALLOC)) {
720 return;
721 }
722
723 if (sg->flags & NVME_SG_DMA) {
724 qemu_sglist_destroy(&sg->qsg);
725 } else {
726 qemu_iovec_destroy(&sg->iov);
727 }
728
729 memset(sg, 0x0, sizeof(*sg));
730 }
731
732 /*
733 * When metadata is transferred as extended LBAs, the DPTR mapped into `sg`
734 * holds both data and metadata. This function splits the data and metadata
735 * into two separate QSG/IOVs.
736 */
nvme_sg_split(NvmeSg * sg,NvmeNamespace * ns,NvmeSg * data,NvmeSg * mdata)737 static void nvme_sg_split(NvmeSg *sg, NvmeNamespace *ns, NvmeSg *data,
738 NvmeSg *mdata)
739 {
740 NvmeSg *dst = data;
741 uint32_t trans_len, count = ns->lbasz;
742 uint64_t offset = 0;
743 bool dma = sg->flags & NVME_SG_DMA;
744 size_t sge_len;
745 size_t sg_len = dma ? sg->qsg.size : sg->iov.size;
746 int sg_idx = 0;
747
748 assert(sg->flags & NVME_SG_ALLOC);
749
750 while (sg_len) {
751 sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
752
753 trans_len = MIN(sg_len, count);
754 trans_len = MIN(trans_len, sge_len - offset);
755
756 if (dst) {
757 if (dma) {
758 qemu_sglist_add(&dst->qsg, sg->qsg.sg[sg_idx].base + offset,
759 trans_len);
760 } else {
761 qemu_iovec_add(&dst->iov,
762 sg->iov.iov[sg_idx].iov_base + offset,
763 trans_len);
764 }
765 }
766
767 sg_len -= trans_len;
768 count -= trans_len;
769 offset += trans_len;
770
771 if (count == 0) {
772 dst = (dst == data) ? mdata : data;
773 count = (dst == data) ? ns->lbasz : ns->lbaf.ms;
774 }
775
776 if (sge_len == offset) {
777 offset = 0;
778 sg_idx++;
779 }
780 }
781 }
782
nvme_map_addr_cmb(NvmeCtrl * n,QEMUIOVector * iov,hwaddr addr,size_t len)783 static uint16_t nvme_map_addr_cmb(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
784 size_t len)
785 {
786 if (!len) {
787 return NVME_SUCCESS;
788 }
789
790 trace_pci_nvme_map_addr_cmb(addr, len);
791
792 if (!nvme_addr_is_cmb(n, addr) || !nvme_addr_is_cmb(n, addr + len - 1)) {
793 return NVME_DATA_TRAS_ERROR;
794 }
795
796 qemu_iovec_add(iov, nvme_addr_to_cmb(n, addr), len);
797
798 return NVME_SUCCESS;
799 }
800
nvme_map_addr_pmr(NvmeCtrl * n,QEMUIOVector * iov,hwaddr addr,size_t len)801 static uint16_t nvme_map_addr_pmr(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
802 size_t len)
803 {
804 if (!len) {
805 return NVME_SUCCESS;
806 }
807
808 if (!nvme_addr_is_pmr(n, addr) || !nvme_addr_is_pmr(n, addr + len - 1)) {
809 return NVME_DATA_TRAS_ERROR;
810 }
811
812 qemu_iovec_add(iov, nvme_addr_to_pmr(n, addr), len);
813
814 return NVME_SUCCESS;
815 }
816
nvme_map_addr(NvmeCtrl * n,NvmeSg * sg,hwaddr addr,size_t len)817 static uint16_t nvme_map_addr(NvmeCtrl *n, NvmeSg *sg, hwaddr addr, size_t len)
818 {
819 bool cmb = false, pmr = false;
820
821 if (!len) {
822 return NVME_SUCCESS;
823 }
824
825 trace_pci_nvme_map_addr(addr, len);
826
827 if (nvme_addr_is_iomem(n, addr)) {
828 return NVME_DATA_TRAS_ERROR;
829 }
830
831 if (nvme_addr_is_cmb(n, addr)) {
832 cmb = true;
833 } else if (nvme_addr_is_pmr(n, addr)) {
834 pmr = true;
835 }
836
837 if (cmb || pmr) {
838 if (sg->flags & NVME_SG_DMA) {
839 return NVME_INVALID_USE_OF_CMB | NVME_DNR;
840 }
841
842 if (sg->iov.niov + 1 > IOV_MAX) {
843 goto max_mappings_exceeded;
844 }
845
846 if (cmb) {
847 return nvme_map_addr_cmb(n, &sg->iov, addr, len);
848 } else {
849 return nvme_map_addr_pmr(n, &sg->iov, addr, len);
850 }
851 }
852
853 if (!(sg->flags & NVME_SG_DMA)) {
854 return NVME_INVALID_USE_OF_CMB | NVME_DNR;
855 }
856
857 if (sg->qsg.nsg + 1 > IOV_MAX) {
858 goto max_mappings_exceeded;
859 }
860
861 qemu_sglist_add(&sg->qsg, addr, len);
862
863 return NVME_SUCCESS;
864
865 max_mappings_exceeded:
866 NVME_GUEST_ERR(pci_nvme_ub_too_many_mappings,
867 "number of mappings exceed 1024");
868 return NVME_INTERNAL_DEV_ERROR | NVME_DNR;
869 }
870
nvme_addr_is_dma(NvmeCtrl * n,hwaddr addr)871 static inline bool nvme_addr_is_dma(NvmeCtrl *n, hwaddr addr)
872 {
873 return !(nvme_addr_is_cmb(n, addr) || nvme_addr_is_pmr(n, addr));
874 }
875
nvme_map_prp(NvmeCtrl * n,NvmeSg * sg,uint64_t prp1,uint64_t prp2,uint32_t len)876 static uint16_t nvme_map_prp(NvmeCtrl *n, NvmeSg *sg, uint64_t prp1,
877 uint64_t prp2, uint32_t len)
878 {
879 hwaddr trans_len = n->page_size - (prp1 % n->page_size);
880 trans_len = MIN(len, trans_len);
881 int num_prps = (len >> n->page_bits) + 1;
882 uint16_t status;
883 int ret;
884
885 trace_pci_nvme_map_prp(trans_len, len, prp1, prp2, num_prps);
886
887 nvme_sg_init(n, sg, nvme_addr_is_dma(n, prp1));
888
889 status = nvme_map_addr(n, sg, prp1, trans_len);
890 if (status) {
891 goto unmap;
892 }
893
894 len -= trans_len;
895 if (len) {
896 if (len > n->page_size) {
897 g_autofree uint64_t *prp_list = g_new(uint64_t, n->max_prp_ents);
898 uint32_t nents, prp_trans;
899 int i = 0;
900
901 /*
902 * The first PRP list entry, pointed to by PRP2 may contain offset.
903 * Hence, we need to calculate the number of entries in based on
904 * that offset.
905 */
906 nents = (n->page_size - (prp2 & (n->page_size - 1))) >> 3;
907 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
908 ret = nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
909 if (ret) {
910 trace_pci_nvme_err_addr_read(prp2);
911 status = NVME_DATA_TRAS_ERROR;
912 goto unmap;
913 }
914 while (len != 0) {
915 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
916
917 if (i == nents - 1 && len > n->page_size) {
918 if (unlikely(prp_ent & (n->page_size - 1))) {
919 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
920 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
921 goto unmap;
922 }
923
924 i = 0;
925 nents = (len + n->page_size - 1) >> n->page_bits;
926 nents = MIN(nents, n->max_prp_ents);
927 prp_trans = nents * sizeof(uint64_t);
928 ret = nvme_addr_read(n, prp_ent, (void *)prp_list,
929 prp_trans);
930 if (ret) {
931 trace_pci_nvme_err_addr_read(prp_ent);
932 status = NVME_DATA_TRAS_ERROR;
933 goto unmap;
934 }
935 prp_ent = le64_to_cpu(prp_list[i]);
936 }
937
938 if (unlikely(prp_ent & (n->page_size - 1))) {
939 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
940 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
941 goto unmap;
942 }
943
944 trans_len = MIN(len, n->page_size);
945 status = nvme_map_addr(n, sg, prp_ent, trans_len);
946 if (status) {
947 goto unmap;
948 }
949
950 len -= trans_len;
951 i++;
952 }
953 } else {
954 if (unlikely(prp2 & (n->page_size - 1))) {
955 trace_pci_nvme_err_invalid_prp2_align(prp2);
956 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
957 goto unmap;
958 }
959 status = nvme_map_addr(n, sg, prp2, len);
960 if (status) {
961 goto unmap;
962 }
963 }
964 }
965
966 return NVME_SUCCESS;
967
968 unmap:
969 nvme_sg_unmap(sg);
970 return status;
971 }
972
973 /*
974 * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
975 * number of bytes mapped in len.
976 */
nvme_map_sgl_data(NvmeCtrl * n,NvmeSg * sg,NvmeSglDescriptor * segment,uint64_t nsgld,size_t * len,NvmeCmd * cmd)977 static uint16_t nvme_map_sgl_data(NvmeCtrl *n, NvmeSg *sg,
978 NvmeSglDescriptor *segment, uint64_t nsgld,
979 size_t *len, NvmeCmd *cmd)
980 {
981 dma_addr_t addr, trans_len;
982 uint32_t dlen;
983 uint16_t status;
984
985 for (int i = 0; i < nsgld; i++) {
986 uint8_t type = NVME_SGL_TYPE(segment[i].type);
987
988 switch (type) {
989 case NVME_SGL_DESCR_TYPE_DATA_BLOCK:
990 break;
991 case NVME_SGL_DESCR_TYPE_SEGMENT:
992 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
993 return NVME_INVALID_NUM_SGL_DESCRS | NVME_DNR;
994 default:
995 return NVME_SGL_DESCR_TYPE_INVALID | NVME_DNR;
996 }
997
998 dlen = le32_to_cpu(segment[i].len);
999
1000 if (!dlen) {
1001 continue;
1002 }
1003
1004 if (*len == 0) {
1005 /*
1006 * All data has been mapped, but the SGL contains additional
1007 * segments and/or descriptors. The controller might accept
1008 * ignoring the rest of the SGL.
1009 */
1010 uint32_t sgls = le32_to_cpu(n->id_ctrl.sgls);
1011 if (sgls & NVME_CTRL_SGLS_EXCESS_LENGTH) {
1012 break;
1013 }
1014
1015 trace_pci_nvme_err_invalid_sgl_excess_length(dlen);
1016 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1017 }
1018
1019 trans_len = MIN(*len, dlen);
1020
1021 addr = le64_to_cpu(segment[i].addr);
1022
1023 if (UINT64_MAX - addr < dlen) {
1024 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1025 }
1026
1027 status = nvme_map_addr(n, sg, addr, trans_len);
1028 if (status) {
1029 return status;
1030 }
1031
1032 *len -= trans_len;
1033 }
1034
1035 return NVME_SUCCESS;
1036 }
1037
nvme_map_sgl(NvmeCtrl * n,NvmeSg * sg,NvmeSglDescriptor sgl,size_t len,NvmeCmd * cmd)1038 static uint16_t nvme_map_sgl(NvmeCtrl *n, NvmeSg *sg, NvmeSglDescriptor sgl,
1039 size_t len, NvmeCmd *cmd)
1040 {
1041 /*
1042 * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
1043 * dynamically allocating a potentially huge SGL. The spec allows the SGL
1044 * to be larger (as in number of bytes required to describe the SGL
1045 * descriptors and segment chain) than the command transfer size, so it is
1046 * not bounded by MDTS.
1047 */
1048 #define SEG_CHUNK_SIZE 256
1049
1050 NvmeSglDescriptor segment[SEG_CHUNK_SIZE], *sgld, *last_sgld;
1051 uint64_t nsgld;
1052 uint32_t seg_len;
1053 uint16_t status;
1054 hwaddr addr;
1055 int ret;
1056
1057 sgld = &sgl;
1058 addr = le64_to_cpu(sgl.addr);
1059
1060 trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl.type), len);
1061
1062 nvme_sg_init(n, sg, nvme_addr_is_dma(n, addr));
1063
1064 /*
1065 * If the entire transfer can be described with a single data block it can
1066 * be mapped directly.
1067 */
1068 if (NVME_SGL_TYPE(sgl.type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1069 status = nvme_map_sgl_data(n, sg, sgld, 1, &len, cmd);
1070 if (status) {
1071 goto unmap;
1072 }
1073
1074 goto out;
1075 }
1076
1077 for (;;) {
1078 switch (NVME_SGL_TYPE(sgld->type)) {
1079 case NVME_SGL_DESCR_TYPE_SEGMENT:
1080 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
1081 break;
1082 default:
1083 return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1084 }
1085
1086 seg_len = le32_to_cpu(sgld->len);
1087
1088 /* check the length of the (Last) Segment descriptor */
1089 if (!seg_len || seg_len & 0xf) {
1090 return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1091 }
1092
1093 if (UINT64_MAX - addr < seg_len) {
1094 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1095 }
1096
1097 nsgld = seg_len / sizeof(NvmeSglDescriptor);
1098
1099 while (nsgld > SEG_CHUNK_SIZE) {
1100 if (nvme_addr_read(n, addr, segment, sizeof(segment))) {
1101 trace_pci_nvme_err_addr_read(addr);
1102 status = NVME_DATA_TRAS_ERROR;
1103 goto unmap;
1104 }
1105
1106 status = nvme_map_sgl_data(n, sg, segment, SEG_CHUNK_SIZE,
1107 &len, cmd);
1108 if (status) {
1109 goto unmap;
1110 }
1111
1112 nsgld -= SEG_CHUNK_SIZE;
1113 addr += SEG_CHUNK_SIZE * sizeof(NvmeSglDescriptor);
1114 }
1115
1116 ret = nvme_addr_read(n, addr, segment, nsgld *
1117 sizeof(NvmeSglDescriptor));
1118 if (ret) {
1119 trace_pci_nvme_err_addr_read(addr);
1120 status = NVME_DATA_TRAS_ERROR;
1121 goto unmap;
1122 }
1123
1124 last_sgld = &segment[nsgld - 1];
1125
1126 /*
1127 * If the segment ends with a Data Block, then we are done.
1128 */
1129 if (NVME_SGL_TYPE(last_sgld->type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1130 status = nvme_map_sgl_data(n, sg, segment, nsgld, &len, cmd);
1131 if (status) {
1132 goto unmap;
1133 }
1134
1135 goto out;
1136 }
1137
1138 /*
1139 * If the last descriptor was not a Data Block, then the current
1140 * segment must not be a Last Segment.
1141 */
1142 if (NVME_SGL_TYPE(sgld->type) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT) {
1143 status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1144 goto unmap;
1145 }
1146
1147 sgld = last_sgld;
1148 addr = le64_to_cpu(sgld->addr);
1149
1150 /*
1151 * Do not map the last descriptor; it will be a Segment or Last Segment
1152 * descriptor and is handled by the next iteration.
1153 */
1154 status = nvme_map_sgl_data(n, sg, segment, nsgld - 1, &len, cmd);
1155 if (status) {
1156 goto unmap;
1157 }
1158 }
1159
1160 out:
1161 /* if there is any residual left in len, the SGL was too short */
1162 if (len) {
1163 status = NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1164 goto unmap;
1165 }
1166
1167 return NVME_SUCCESS;
1168
1169 unmap:
1170 nvme_sg_unmap(sg);
1171 return status;
1172 }
1173
nvme_map_dptr(NvmeCtrl * n,NvmeSg * sg,size_t len,NvmeCmd * cmd)1174 uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1175 NvmeCmd *cmd)
1176 {
1177 uint64_t prp1, prp2;
1178
1179 switch (NVME_CMD_FLAGS_PSDT(cmd->flags)) {
1180 case NVME_PSDT_PRP:
1181 prp1 = le64_to_cpu(cmd->dptr.prp1);
1182 prp2 = le64_to_cpu(cmd->dptr.prp2);
1183
1184 return nvme_map_prp(n, sg, prp1, prp2, len);
1185 case NVME_PSDT_SGL_MPTR_CONTIGUOUS:
1186 case NVME_PSDT_SGL_MPTR_SGL:
1187 return nvme_map_sgl(n, sg, cmd->dptr.sgl, len, cmd);
1188 default:
1189 return NVME_INVALID_FIELD;
1190 }
1191 }
1192
nvme_map_mptr(NvmeCtrl * n,NvmeSg * sg,size_t len,NvmeCmd * cmd)1193 static uint16_t nvme_map_mptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1194 NvmeCmd *cmd)
1195 {
1196 int psdt = NVME_CMD_FLAGS_PSDT(cmd->flags);
1197 hwaddr mptr = le64_to_cpu(cmd->mptr);
1198 uint16_t status;
1199
1200 if (psdt == NVME_PSDT_SGL_MPTR_SGL) {
1201 NvmeSglDescriptor sgl;
1202
1203 if (nvme_addr_read(n, mptr, &sgl, sizeof(sgl))) {
1204 return NVME_DATA_TRAS_ERROR;
1205 }
1206
1207 status = nvme_map_sgl(n, sg, sgl, len, cmd);
1208 if (status && (status & 0x7ff) == NVME_DATA_SGL_LEN_INVALID) {
1209 status = NVME_MD_SGL_LEN_INVALID | NVME_DNR;
1210 }
1211
1212 return status;
1213 }
1214
1215 nvme_sg_init(n, sg, nvme_addr_is_dma(n, mptr));
1216 status = nvme_map_addr(n, sg, mptr, len);
1217 if (status) {
1218 nvme_sg_unmap(sg);
1219 }
1220
1221 return status;
1222 }
1223
nvme_map_data(NvmeCtrl * n,uint32_t nlb,NvmeRequest * req)1224 static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1225 {
1226 NvmeNamespace *ns = req->ns;
1227 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1228 bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1229 bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1230 size_t len = nvme_l2b(ns, nlb);
1231 uint16_t status;
1232
1233 if (nvme_ns_ext(ns) &&
1234 !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1235 NvmeSg sg;
1236
1237 len += nvme_m2b(ns, nlb);
1238
1239 status = nvme_map_dptr(n, &sg, len, &req->cmd);
1240 if (status) {
1241 return status;
1242 }
1243
1244 nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1245 nvme_sg_split(&sg, ns, &req->sg, NULL);
1246 nvme_sg_unmap(&sg);
1247
1248 return NVME_SUCCESS;
1249 }
1250
1251 return nvme_map_dptr(n, &req->sg, len, &req->cmd);
1252 }
1253
nvme_map_mdata(NvmeCtrl * n,uint32_t nlb,NvmeRequest * req)1254 static uint16_t nvme_map_mdata(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1255 {
1256 NvmeNamespace *ns = req->ns;
1257 size_t len = nvme_m2b(ns, nlb);
1258 uint16_t status;
1259
1260 if (nvme_ns_ext(ns)) {
1261 NvmeSg sg;
1262
1263 len += nvme_l2b(ns, nlb);
1264
1265 status = nvme_map_dptr(n, &sg, len, &req->cmd);
1266 if (status) {
1267 return status;
1268 }
1269
1270 nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1271 nvme_sg_split(&sg, ns, NULL, &req->sg);
1272 nvme_sg_unmap(&sg);
1273
1274 return NVME_SUCCESS;
1275 }
1276
1277 return nvme_map_mptr(n, &req->sg, len, &req->cmd);
1278 }
1279
nvme_tx_interleaved(NvmeCtrl * n,NvmeSg * sg,uint8_t * ptr,uint32_t len,uint32_t bytes,int32_t skip_bytes,int64_t offset,NvmeTxDirection dir)1280 static uint16_t nvme_tx_interleaved(NvmeCtrl *n, NvmeSg *sg, uint8_t *ptr,
1281 uint32_t len, uint32_t bytes,
1282 int32_t skip_bytes, int64_t offset,
1283 NvmeTxDirection dir)
1284 {
1285 hwaddr addr;
1286 uint32_t trans_len, count = bytes;
1287 bool dma = sg->flags & NVME_SG_DMA;
1288 int64_t sge_len;
1289 int sg_idx = 0;
1290 int ret;
1291
1292 assert(sg->flags & NVME_SG_ALLOC);
1293
1294 while (len) {
1295 sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
1296
1297 if (sge_len - offset < 0) {
1298 offset -= sge_len;
1299 sg_idx++;
1300 continue;
1301 }
1302
1303 if (sge_len == offset) {
1304 offset = 0;
1305 sg_idx++;
1306 continue;
1307 }
1308
1309 trans_len = MIN(len, count);
1310 trans_len = MIN(trans_len, sge_len - offset);
1311
1312 if (dma) {
1313 addr = sg->qsg.sg[sg_idx].base + offset;
1314 } else {
1315 addr = (hwaddr)(uintptr_t)sg->iov.iov[sg_idx].iov_base + offset;
1316 }
1317
1318 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1319 ret = nvme_addr_read(n, addr, ptr, trans_len);
1320 } else {
1321 ret = nvme_addr_write(n, addr, ptr, trans_len);
1322 }
1323
1324 if (ret) {
1325 return NVME_DATA_TRAS_ERROR;
1326 }
1327
1328 ptr += trans_len;
1329 len -= trans_len;
1330 count -= trans_len;
1331 offset += trans_len;
1332
1333 if (count == 0) {
1334 count = bytes;
1335 offset += skip_bytes;
1336 }
1337 }
1338
1339 return NVME_SUCCESS;
1340 }
1341
nvme_tx(NvmeCtrl * n,NvmeSg * sg,void * ptr,uint32_t len,NvmeTxDirection dir)1342 static uint16_t nvme_tx(NvmeCtrl *n, NvmeSg *sg, void *ptr, uint32_t len,
1343 NvmeTxDirection dir)
1344 {
1345 assert(sg->flags & NVME_SG_ALLOC);
1346
1347 if (sg->flags & NVME_SG_DMA) {
1348 const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
1349 dma_addr_t residual;
1350
1351 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1352 dma_buf_write(ptr, len, &residual, &sg->qsg, attrs);
1353 } else {
1354 dma_buf_read(ptr, len, &residual, &sg->qsg, attrs);
1355 }
1356
1357 if (unlikely(residual)) {
1358 trace_pci_nvme_err_invalid_dma();
1359 return NVME_INVALID_FIELD | NVME_DNR;
1360 }
1361 } else {
1362 size_t bytes;
1363
1364 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1365 bytes = qemu_iovec_to_buf(&sg->iov, 0, ptr, len);
1366 } else {
1367 bytes = qemu_iovec_from_buf(&sg->iov, 0, ptr, len);
1368 }
1369
1370 if (unlikely(bytes != len)) {
1371 trace_pci_nvme_err_invalid_dma();
1372 return NVME_INVALID_FIELD | NVME_DNR;
1373 }
1374 }
1375
1376 return NVME_SUCCESS;
1377 }
1378
nvme_c2h(NvmeCtrl * n,void * ptr,uint32_t len,NvmeRequest * req)1379 static inline uint16_t nvme_c2h(NvmeCtrl *n, void *ptr, uint32_t len,
1380 NvmeRequest *req)
1381 {
1382 uint16_t status;
1383
1384 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1385 if (status) {
1386 return status;
1387 }
1388
1389 return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_FROM_DEVICE);
1390 }
1391
nvme_h2c(NvmeCtrl * n,void * ptr,uint32_t len,NvmeRequest * req)1392 static inline uint16_t nvme_h2c(NvmeCtrl *n, void *ptr, uint32_t len,
1393 NvmeRequest *req)
1394 {
1395 uint16_t status;
1396
1397 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1398 if (status) {
1399 return status;
1400 }
1401
1402 return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_TO_DEVICE);
1403 }
1404
nvme_bounce_data(NvmeCtrl * n,void * ptr,uint32_t len,NvmeTxDirection dir,NvmeRequest * req)1405 uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
1406 NvmeTxDirection dir, NvmeRequest *req)
1407 {
1408 NvmeNamespace *ns = req->ns;
1409 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1410 bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1411 bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1412
1413 if (nvme_ns_ext(ns) &&
1414 !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1415 return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbasz,
1416 ns->lbaf.ms, 0, dir);
1417 }
1418
1419 return nvme_tx(n, &req->sg, ptr, len, dir);
1420 }
1421
nvme_bounce_mdata(NvmeCtrl * n,void * ptr,uint32_t len,NvmeTxDirection dir,NvmeRequest * req)1422 uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
1423 NvmeTxDirection dir, NvmeRequest *req)
1424 {
1425 NvmeNamespace *ns = req->ns;
1426 uint16_t status;
1427
1428 if (nvme_ns_ext(ns)) {
1429 return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbaf.ms,
1430 ns->lbasz, ns->lbasz, dir);
1431 }
1432
1433 nvme_sg_unmap(&req->sg);
1434
1435 status = nvme_map_mptr(n, &req->sg, len, &req->cmd);
1436 if (status) {
1437 return status;
1438 }
1439
1440 return nvme_tx(n, &req->sg, ptr, len, dir);
1441 }
1442
nvme_blk_read(BlockBackend * blk,int64_t offset,uint32_t align,BlockCompletionFunc * cb,NvmeRequest * req)1443 static inline void nvme_blk_read(BlockBackend *blk, int64_t offset,
1444 uint32_t align, BlockCompletionFunc *cb,
1445 NvmeRequest *req)
1446 {
1447 assert(req->sg.flags & NVME_SG_ALLOC);
1448
1449 if (req->sg.flags & NVME_SG_DMA) {
1450 req->aiocb = dma_blk_read(blk, &req->sg.qsg, offset, align, cb, req);
1451 } else {
1452 req->aiocb = blk_aio_preadv(blk, offset, &req->sg.iov, 0, cb, req);
1453 }
1454 }
1455
nvme_blk_write(BlockBackend * blk,int64_t offset,uint32_t align,BlockCompletionFunc * cb,NvmeRequest * req)1456 static inline void nvme_blk_write(BlockBackend *blk, int64_t offset,
1457 uint32_t align, BlockCompletionFunc *cb,
1458 NvmeRequest *req)
1459 {
1460 assert(req->sg.flags & NVME_SG_ALLOC);
1461
1462 if (req->sg.flags & NVME_SG_DMA) {
1463 req->aiocb = dma_blk_write(blk, &req->sg.qsg, offset, align, cb, req);
1464 } else {
1465 req->aiocb = blk_aio_pwritev(blk, offset, &req->sg.iov, 0, cb, req);
1466 }
1467 }
1468
nvme_update_cq_eventidx(const NvmeCQueue * cq)1469 static void nvme_update_cq_eventidx(const NvmeCQueue *cq)
1470 {
1471 trace_pci_nvme_update_cq_eventidx(cq->cqid, cq->head);
1472
1473 stl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->ei_addr, cq->head,
1474 MEMTXATTRS_UNSPECIFIED);
1475 }
1476
nvme_update_cq_head(NvmeCQueue * cq)1477 static void nvme_update_cq_head(NvmeCQueue *cq)
1478 {
1479 ldl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->db_addr, &cq->head,
1480 MEMTXATTRS_UNSPECIFIED);
1481
1482 trace_pci_nvme_update_cq_head(cq->cqid, cq->head);
1483 }
1484
nvme_post_cqes(void * opaque)1485 static void nvme_post_cqes(void *opaque)
1486 {
1487 NvmeCQueue *cq = opaque;
1488 NvmeCtrl *n = cq->ctrl;
1489 NvmeRequest *req, *next;
1490 bool pending = cq->head != cq->tail;
1491 int ret;
1492
1493 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
1494 NvmeSQueue *sq;
1495 hwaddr addr;
1496
1497 if (n->dbbuf_enabled) {
1498 nvme_update_cq_eventidx(cq);
1499 nvme_update_cq_head(cq);
1500 }
1501
1502 if (nvme_cq_full(cq)) {
1503 break;
1504 }
1505
1506 sq = req->sq;
1507 req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
1508 req->cqe.sq_id = cpu_to_le16(sq->sqid);
1509 req->cqe.sq_head = cpu_to_le16(sq->head);
1510 addr = cq->dma_addr + (cq->tail << NVME_CQES);
1511 ret = pci_dma_write(PCI_DEVICE(n), addr, (void *)&req->cqe,
1512 sizeof(req->cqe));
1513 if (ret) {
1514 trace_pci_nvme_err_addr_write(addr);
1515 trace_pci_nvme_err_cfs();
1516 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
1517 break;
1518 }
1519 QTAILQ_REMOVE(&cq->req_list, req, entry);
1520 nvme_inc_cq_tail(cq);
1521 nvme_sg_unmap(&req->sg);
1522 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
1523 }
1524 if (cq->tail != cq->head) {
1525 if (cq->irq_enabled && !pending) {
1526 n->cq_pending++;
1527 }
1528
1529 nvme_irq_assert(n, cq);
1530 }
1531 }
1532
nvme_enqueue_req_completion(NvmeCQueue * cq,NvmeRequest * req)1533 static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
1534 {
1535 assert(cq->cqid == req->sq->cqid);
1536 trace_pci_nvme_enqueue_req_completion(nvme_cid(req), cq->cqid,
1537 le32_to_cpu(req->cqe.result),
1538 le32_to_cpu(req->cqe.dw1),
1539 req->status);
1540
1541 if (req->status) {
1542 trace_pci_nvme_err_req_status(nvme_cid(req), nvme_nsid(req->ns),
1543 req->status, req->cmd.opcode);
1544 }
1545
1546 QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
1547 QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
1548
1549 qemu_bh_schedule(cq->bh);
1550 }
1551
nvme_process_aers(void * opaque)1552 static void nvme_process_aers(void *opaque)
1553 {
1554 NvmeCtrl *n = opaque;
1555 NvmeAsyncEvent *event, *next;
1556
1557 trace_pci_nvme_process_aers(n->aer_queued);
1558
1559 QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
1560 NvmeRequest *req;
1561 NvmeAerResult *result;
1562
1563 /* can't post cqe if there is nothing to complete */
1564 if (!n->outstanding_aers) {
1565 trace_pci_nvme_no_outstanding_aers();
1566 break;
1567 }
1568
1569 /* ignore if masked (cqe posted, but event not cleared) */
1570 if (n->aer_mask & (1 << event->result.event_type)) {
1571 trace_pci_nvme_aer_masked(event->result.event_type, n->aer_mask);
1572 continue;
1573 }
1574
1575 QTAILQ_REMOVE(&n->aer_queue, event, entry);
1576 n->aer_queued--;
1577
1578 n->aer_mask |= 1 << event->result.event_type;
1579 n->outstanding_aers--;
1580
1581 req = n->aer_reqs[n->outstanding_aers];
1582
1583 result = (NvmeAerResult *) &req->cqe.result;
1584 result->event_type = event->result.event_type;
1585 result->event_info = event->result.event_info;
1586 result->log_page = event->result.log_page;
1587 g_free(event);
1588
1589 trace_pci_nvme_aer_post_cqe(result->event_type, result->event_info,
1590 result->log_page);
1591
1592 nvme_enqueue_req_completion(&n->admin_cq, req);
1593 }
1594 }
1595
nvme_enqueue_event(NvmeCtrl * n,uint8_t event_type,uint8_t event_info,uint8_t log_page)1596 static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type,
1597 uint8_t event_info, uint8_t log_page)
1598 {
1599 NvmeAsyncEvent *event;
1600
1601 trace_pci_nvme_enqueue_event(event_type, event_info, log_page);
1602
1603 if (n->aer_queued == n->params.aer_max_queued) {
1604 trace_pci_nvme_enqueue_event_noqueue(n->aer_queued);
1605 return;
1606 }
1607
1608 event = g_new(NvmeAsyncEvent, 1);
1609 event->result = (NvmeAerResult) {
1610 .event_type = event_type,
1611 .event_info = event_info,
1612 .log_page = log_page,
1613 };
1614
1615 QTAILQ_INSERT_TAIL(&n->aer_queue, event, entry);
1616 n->aer_queued++;
1617
1618 nvme_process_aers(n);
1619 }
1620
nvme_smart_event(NvmeCtrl * n,uint8_t event)1621 static void nvme_smart_event(NvmeCtrl *n, uint8_t event)
1622 {
1623 uint8_t aer_info;
1624
1625 /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */
1626 if (!(NVME_AEC_SMART(n->features.async_config) & event)) {
1627 return;
1628 }
1629
1630 switch (event) {
1631 case NVME_SMART_SPARE:
1632 aer_info = NVME_AER_INFO_SMART_SPARE_THRESH;
1633 break;
1634 case NVME_SMART_TEMPERATURE:
1635 aer_info = NVME_AER_INFO_SMART_TEMP_THRESH;
1636 break;
1637 case NVME_SMART_RELIABILITY:
1638 case NVME_SMART_MEDIA_READ_ONLY:
1639 case NVME_SMART_FAILED_VOLATILE_MEDIA:
1640 case NVME_SMART_PMR_UNRELIABLE:
1641 aer_info = NVME_AER_INFO_SMART_RELIABILITY;
1642 break;
1643 default:
1644 return;
1645 }
1646
1647 nvme_enqueue_event(n, NVME_AER_TYPE_SMART, aer_info, NVME_LOG_SMART_INFO);
1648 }
1649
nvme_clear_events(NvmeCtrl * n,uint8_t event_type)1650 static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type)
1651 {
1652 n->aer_mask &= ~(1 << event_type);
1653 if (!QTAILQ_EMPTY(&n->aer_queue)) {
1654 nvme_process_aers(n);
1655 }
1656 }
1657
nvme_check_mdts(NvmeCtrl * n,size_t len)1658 static inline uint16_t nvme_check_mdts(NvmeCtrl *n, size_t len)
1659 {
1660 uint8_t mdts = n->params.mdts;
1661
1662 if (mdts && len > n->page_size << mdts) {
1663 trace_pci_nvme_err_mdts(len);
1664 return NVME_INVALID_FIELD | NVME_DNR;
1665 }
1666
1667 return NVME_SUCCESS;
1668 }
1669
nvme_check_bounds(NvmeNamespace * ns,uint64_t slba,uint32_t nlb)1670 static inline uint16_t nvme_check_bounds(NvmeNamespace *ns, uint64_t slba,
1671 uint32_t nlb)
1672 {
1673 uint64_t nsze = le64_to_cpu(ns->id_ns.nsze);
1674
1675 if (unlikely(UINT64_MAX - slba < nlb || slba + nlb > nsze)) {
1676 trace_pci_nvme_err_invalid_lba_range(slba, nlb, nsze);
1677 return NVME_LBA_RANGE | NVME_DNR;
1678 }
1679
1680 return NVME_SUCCESS;
1681 }
1682
nvme_block_status_all(NvmeNamespace * ns,uint64_t slba,uint32_t nlb,int flags)1683 static int nvme_block_status_all(NvmeNamespace *ns, uint64_t slba,
1684 uint32_t nlb, int flags)
1685 {
1686 BlockDriverState *bs = blk_bs(ns->blkconf.blk);
1687
1688 int64_t pnum = 0, bytes = nvme_l2b(ns, nlb);
1689 int64_t offset = nvme_l2b(ns, slba);
1690 int ret;
1691
1692 /*
1693 * `pnum` holds the number of bytes after offset that shares the same
1694 * allocation status as the byte at offset. If `pnum` is different from
1695 * `bytes`, we should check the allocation status of the next range and
1696 * continue this until all bytes have been checked.
1697 */
1698 do {
1699 bytes -= pnum;
1700
1701 ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL);
1702 if (ret < 0) {
1703 return ret;
1704 }
1705
1706
1707 trace_pci_nvme_block_status(offset, bytes, pnum, ret,
1708 !!(ret & BDRV_BLOCK_ZERO));
1709
1710 if (!(ret & flags)) {
1711 return 1;
1712 }
1713
1714 offset += pnum;
1715 } while (pnum != bytes);
1716
1717 return 0;
1718 }
1719
nvme_check_dulbe(NvmeNamespace * ns,uint64_t slba,uint32_t nlb)1720 static uint16_t nvme_check_dulbe(NvmeNamespace *ns, uint64_t slba,
1721 uint32_t nlb)
1722 {
1723 int ret;
1724 Error *err = NULL;
1725
1726 ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_DATA);
1727 if (ret) {
1728 if (ret < 0) {
1729 error_setg_errno(&err, -ret, "unable to get block status");
1730 error_report_err(err);
1731
1732 return NVME_INTERNAL_DEV_ERROR;
1733 }
1734
1735 return NVME_DULB;
1736 }
1737
1738 return NVME_SUCCESS;
1739 }
1740
nvme_aio_err(NvmeRequest * req,int ret)1741 static void nvme_aio_err(NvmeRequest *req, int ret)
1742 {
1743 uint16_t status = NVME_SUCCESS;
1744 Error *local_err = NULL;
1745
1746 switch (req->cmd.opcode) {
1747 case NVME_CMD_READ:
1748 status = NVME_UNRECOVERED_READ;
1749 break;
1750 case NVME_CMD_FLUSH:
1751 case NVME_CMD_WRITE:
1752 case NVME_CMD_WRITE_ZEROES:
1753 case NVME_CMD_ZONE_APPEND:
1754 case NVME_CMD_COPY:
1755 status = NVME_WRITE_FAULT;
1756 break;
1757 default:
1758 status = NVME_INTERNAL_DEV_ERROR;
1759 break;
1760 }
1761
1762 if (ret == -ECANCELED) {
1763 status = NVME_CMD_ABORT_REQ;
1764 }
1765
1766 trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), status);
1767
1768 error_setg_errno(&local_err, -ret, "aio failed");
1769 error_report_err(local_err);
1770
1771 /*
1772 * Set the command status code to the first encountered error but allow a
1773 * subsequent Internal Device Error to trump it.
1774 */
1775 if (req->status && status != NVME_INTERNAL_DEV_ERROR) {
1776 return;
1777 }
1778
1779 req->status = status;
1780 }
1781
nvme_zone_idx(NvmeNamespace * ns,uint64_t slba)1782 static inline uint32_t nvme_zone_idx(NvmeNamespace *ns, uint64_t slba)
1783 {
1784 return ns->zone_size_log2 > 0 ? slba >> ns->zone_size_log2 :
1785 slba / ns->zone_size;
1786 }
1787
nvme_get_zone_by_slba(NvmeNamespace * ns,uint64_t slba)1788 static inline NvmeZone *nvme_get_zone_by_slba(NvmeNamespace *ns, uint64_t slba)
1789 {
1790 uint32_t zone_idx = nvme_zone_idx(ns, slba);
1791
1792 if (zone_idx >= ns->num_zones) {
1793 return NULL;
1794 }
1795
1796 return &ns->zone_array[zone_idx];
1797 }
1798
nvme_check_zone_state_for_write(NvmeZone * zone)1799 static uint16_t nvme_check_zone_state_for_write(NvmeZone *zone)
1800 {
1801 uint64_t zslba = zone->d.zslba;
1802
1803 switch (nvme_get_zone_state(zone)) {
1804 case NVME_ZONE_STATE_EMPTY:
1805 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1806 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1807 case NVME_ZONE_STATE_CLOSED:
1808 return NVME_SUCCESS;
1809 case NVME_ZONE_STATE_FULL:
1810 trace_pci_nvme_err_zone_is_full(zslba);
1811 return NVME_ZONE_FULL;
1812 case NVME_ZONE_STATE_OFFLINE:
1813 trace_pci_nvme_err_zone_is_offline(zslba);
1814 return NVME_ZONE_OFFLINE;
1815 case NVME_ZONE_STATE_READ_ONLY:
1816 trace_pci_nvme_err_zone_is_read_only(zslba);
1817 return NVME_ZONE_READ_ONLY;
1818 default:
1819 assert(false);
1820 }
1821
1822 return NVME_INTERNAL_DEV_ERROR;
1823 }
1824
nvme_check_zone_write(NvmeNamespace * ns,NvmeZone * zone,uint64_t slba,uint32_t nlb)1825 static uint16_t nvme_check_zone_write(NvmeNamespace *ns, NvmeZone *zone,
1826 uint64_t slba, uint32_t nlb)
1827 {
1828 uint64_t zcap = nvme_zone_wr_boundary(zone);
1829 uint16_t status;
1830
1831 status = nvme_check_zone_state_for_write(zone);
1832 if (status) {
1833 return status;
1834 }
1835
1836 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1837 uint64_t ezrwa = zone->w_ptr + 2 * ns->zns.zrwas;
1838
1839 if (slba < zone->w_ptr || slba + nlb > ezrwa) {
1840 trace_pci_nvme_err_zone_invalid_write(slba, zone->w_ptr);
1841 return NVME_ZONE_INVALID_WRITE;
1842 }
1843 } else {
1844 if (unlikely(slba != zone->w_ptr)) {
1845 trace_pci_nvme_err_write_not_at_wp(slba, zone->d.zslba,
1846 zone->w_ptr);
1847 return NVME_ZONE_INVALID_WRITE;
1848 }
1849 }
1850
1851 if (unlikely((slba + nlb) > zcap)) {
1852 trace_pci_nvme_err_zone_boundary(slba, nlb, zcap);
1853 return NVME_ZONE_BOUNDARY_ERROR;
1854 }
1855
1856 return NVME_SUCCESS;
1857 }
1858
nvme_check_zone_state_for_read(NvmeZone * zone)1859 static uint16_t nvme_check_zone_state_for_read(NvmeZone *zone)
1860 {
1861 switch (nvme_get_zone_state(zone)) {
1862 case NVME_ZONE_STATE_EMPTY:
1863 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1864 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1865 case NVME_ZONE_STATE_FULL:
1866 case NVME_ZONE_STATE_CLOSED:
1867 case NVME_ZONE_STATE_READ_ONLY:
1868 return NVME_SUCCESS;
1869 case NVME_ZONE_STATE_OFFLINE:
1870 trace_pci_nvme_err_zone_is_offline(zone->d.zslba);
1871 return NVME_ZONE_OFFLINE;
1872 default:
1873 assert(false);
1874 }
1875
1876 return NVME_INTERNAL_DEV_ERROR;
1877 }
1878
nvme_check_zone_read(NvmeNamespace * ns,uint64_t slba,uint32_t nlb)1879 static uint16_t nvme_check_zone_read(NvmeNamespace *ns, uint64_t slba,
1880 uint32_t nlb)
1881 {
1882 NvmeZone *zone;
1883 uint64_t bndry, end;
1884 uint16_t status;
1885
1886 zone = nvme_get_zone_by_slba(ns, slba);
1887 assert(zone);
1888
1889 bndry = nvme_zone_rd_boundary(ns, zone);
1890 end = slba + nlb;
1891
1892 status = nvme_check_zone_state_for_read(zone);
1893 if (status) {
1894 ;
1895 } else if (unlikely(end > bndry)) {
1896 if (!ns->params.cross_zone_read) {
1897 status = NVME_ZONE_BOUNDARY_ERROR;
1898 } else {
1899 /*
1900 * Read across zone boundary - check that all subsequent
1901 * zones that are being read have an appropriate state.
1902 */
1903 do {
1904 zone++;
1905 status = nvme_check_zone_state_for_read(zone);
1906 if (status) {
1907 break;
1908 }
1909 } while (end > nvme_zone_rd_boundary(ns, zone));
1910 }
1911 }
1912
1913 return status;
1914 }
1915
nvme_zrm_finish(NvmeNamespace * ns,NvmeZone * zone)1916 static uint16_t nvme_zrm_finish(NvmeNamespace *ns, NvmeZone *zone)
1917 {
1918 switch (nvme_get_zone_state(zone)) {
1919 case NVME_ZONE_STATE_FULL:
1920 return NVME_SUCCESS;
1921
1922 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1923 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1924 nvme_aor_dec_open(ns);
1925 /* fallthrough */
1926 case NVME_ZONE_STATE_CLOSED:
1927 nvme_aor_dec_active(ns);
1928
1929 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1930 zone->d.za &= ~NVME_ZA_ZRWA_VALID;
1931 if (ns->params.numzrwa) {
1932 ns->zns.numzrwa++;
1933 }
1934 }
1935
1936 /* fallthrough */
1937 case NVME_ZONE_STATE_EMPTY:
1938 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_FULL);
1939 return NVME_SUCCESS;
1940
1941 default:
1942 return NVME_ZONE_INVAL_TRANSITION;
1943 }
1944 }
1945
nvme_zrm_close(NvmeNamespace * ns,NvmeZone * zone)1946 static uint16_t nvme_zrm_close(NvmeNamespace *ns, NvmeZone *zone)
1947 {
1948 switch (nvme_get_zone_state(zone)) {
1949 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1950 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1951 nvme_aor_dec_open(ns);
1952 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
1953 /* fall through */
1954 case NVME_ZONE_STATE_CLOSED:
1955 return NVME_SUCCESS;
1956
1957 default:
1958 return NVME_ZONE_INVAL_TRANSITION;
1959 }
1960 }
1961
nvme_zrm_reset(NvmeNamespace * ns,NvmeZone * zone)1962 static uint16_t nvme_zrm_reset(NvmeNamespace *ns, NvmeZone *zone)
1963 {
1964 switch (nvme_get_zone_state(zone)) {
1965 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1966 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1967 nvme_aor_dec_open(ns);
1968 /* fallthrough */
1969 case NVME_ZONE_STATE_CLOSED:
1970 nvme_aor_dec_active(ns);
1971
1972 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1973 if (ns->params.numzrwa) {
1974 ns->zns.numzrwa++;
1975 }
1976 }
1977
1978 /* fallthrough */
1979 case NVME_ZONE_STATE_FULL:
1980 zone->w_ptr = zone->d.zslba;
1981 zone->d.wp = zone->w_ptr;
1982 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EMPTY);
1983 /* fallthrough */
1984 case NVME_ZONE_STATE_EMPTY:
1985 return NVME_SUCCESS;
1986
1987 default:
1988 return NVME_ZONE_INVAL_TRANSITION;
1989 }
1990 }
1991
nvme_zrm_auto_transition_zone(NvmeNamespace * ns)1992 static void nvme_zrm_auto_transition_zone(NvmeNamespace *ns)
1993 {
1994 NvmeZone *zone;
1995
1996 if (ns->params.max_open_zones &&
1997 ns->nr_open_zones == ns->params.max_open_zones) {
1998 zone = QTAILQ_FIRST(&ns->imp_open_zones);
1999 if (zone) {
2000 /*
2001 * Automatically close this implicitly open zone.
2002 */
2003 QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
2004 nvme_zrm_close(ns, zone);
2005 }
2006 }
2007 }
2008
2009 enum {
2010 NVME_ZRM_AUTO = 1 << 0,
2011 NVME_ZRM_ZRWA = 1 << 1,
2012 };
2013
nvme_zrm_open_flags(NvmeCtrl * n,NvmeNamespace * ns,NvmeZone * zone,int flags)2014 static uint16_t nvme_zrm_open_flags(NvmeCtrl *n, NvmeNamespace *ns,
2015 NvmeZone *zone, int flags)
2016 {
2017 int act = 0;
2018 uint16_t status;
2019
2020 switch (nvme_get_zone_state(zone)) {
2021 case NVME_ZONE_STATE_EMPTY:
2022 act = 1;
2023
2024 /* fallthrough */
2025
2026 case NVME_ZONE_STATE_CLOSED:
2027 if (n->params.auto_transition_zones) {
2028 nvme_zrm_auto_transition_zone(ns);
2029 }
2030 status = nvme_zns_check_resources(ns, act, 1,
2031 (flags & NVME_ZRM_ZRWA) ? 1 : 0);
2032 if (status) {
2033 return status;
2034 }
2035
2036 if (act) {
2037 nvme_aor_inc_active(ns);
2038 }
2039
2040 nvme_aor_inc_open(ns);
2041
2042 if (flags & NVME_ZRM_AUTO) {
2043 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_IMPLICITLY_OPEN);
2044 return NVME_SUCCESS;
2045 }
2046
2047 /* fallthrough */
2048
2049 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
2050 if (flags & NVME_ZRM_AUTO) {
2051 return NVME_SUCCESS;
2052 }
2053
2054 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EXPLICITLY_OPEN);
2055
2056 /* fallthrough */
2057
2058 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
2059 if (flags & NVME_ZRM_ZRWA) {
2060 ns->zns.numzrwa--;
2061
2062 zone->d.za |= NVME_ZA_ZRWA_VALID;
2063 }
2064
2065 return NVME_SUCCESS;
2066
2067 default:
2068 return NVME_ZONE_INVAL_TRANSITION;
2069 }
2070 }
2071
nvme_zrm_auto(NvmeCtrl * n,NvmeNamespace * ns,NvmeZone * zone)2072 static inline uint16_t nvme_zrm_auto(NvmeCtrl *n, NvmeNamespace *ns,
2073 NvmeZone *zone)
2074 {
2075 return nvme_zrm_open_flags(n, ns, zone, NVME_ZRM_AUTO);
2076 }
2077
nvme_advance_zone_wp(NvmeNamespace * ns,NvmeZone * zone,uint32_t nlb)2078 static void nvme_advance_zone_wp(NvmeNamespace *ns, NvmeZone *zone,
2079 uint32_t nlb)
2080 {
2081 zone->d.wp += nlb;
2082
2083 if (zone->d.wp == nvme_zone_wr_boundary(zone)) {
2084 nvme_zrm_finish(ns, zone);
2085 }
2086 }
2087
nvme_zoned_zrwa_implicit_flush(NvmeNamespace * ns,NvmeZone * zone,uint32_t nlbc)2088 static void nvme_zoned_zrwa_implicit_flush(NvmeNamespace *ns, NvmeZone *zone,
2089 uint32_t nlbc)
2090 {
2091 uint16_t nzrwafgs = DIV_ROUND_UP(nlbc, ns->zns.zrwafg);
2092
2093 nlbc = nzrwafgs * ns->zns.zrwafg;
2094
2095 trace_pci_nvme_zoned_zrwa_implicit_flush(zone->d.zslba, nlbc);
2096
2097 zone->w_ptr += nlbc;
2098
2099 nvme_advance_zone_wp(ns, zone, nlbc);
2100 }
2101
nvme_finalize_zoned_write(NvmeNamespace * ns,NvmeRequest * req)2102 static void nvme_finalize_zoned_write(NvmeNamespace *ns, NvmeRequest *req)
2103 {
2104 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2105 NvmeZone *zone;
2106 uint64_t slba;
2107 uint32_t nlb;
2108
2109 slba = le64_to_cpu(rw->slba);
2110 nlb = le16_to_cpu(rw->nlb) + 1;
2111 zone = nvme_get_zone_by_slba(ns, slba);
2112 assert(zone);
2113
2114 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
2115 uint64_t ezrwa = zone->w_ptr + ns->zns.zrwas - 1;
2116 uint64_t elba = slba + nlb - 1;
2117
2118 if (elba > ezrwa) {
2119 nvme_zoned_zrwa_implicit_flush(ns, zone, elba - ezrwa);
2120 }
2121
2122 return;
2123 }
2124
2125 nvme_advance_zone_wp(ns, zone, nlb);
2126 }
2127
nvme_is_write(NvmeRequest * req)2128 static inline bool nvme_is_write(NvmeRequest *req)
2129 {
2130 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2131
2132 return rw->opcode == NVME_CMD_WRITE ||
2133 rw->opcode == NVME_CMD_ZONE_APPEND ||
2134 rw->opcode == NVME_CMD_WRITE_ZEROES;
2135 }
2136
nvme_misc_cb(void * opaque,int ret)2137 static void nvme_misc_cb(void *opaque, int ret)
2138 {
2139 NvmeRequest *req = opaque;
2140
2141 trace_pci_nvme_misc_cb(nvme_cid(req));
2142
2143 if (ret) {
2144 nvme_aio_err(req, ret);
2145 }
2146
2147 nvme_enqueue_req_completion(nvme_cq(req), req);
2148 }
2149
nvme_rw_complete_cb(void * opaque,int ret)2150 void nvme_rw_complete_cb(void *opaque, int ret)
2151 {
2152 NvmeRequest *req = opaque;
2153 NvmeNamespace *ns = req->ns;
2154 BlockBackend *blk = ns->blkconf.blk;
2155 BlockAcctCookie *acct = &req->acct;
2156 BlockAcctStats *stats = blk_get_stats(blk);
2157
2158 trace_pci_nvme_rw_complete_cb(nvme_cid(req), blk_name(blk));
2159
2160 if (ret) {
2161 block_acct_failed(stats, acct);
2162 nvme_aio_err(req, ret);
2163 } else {
2164 block_acct_done(stats, acct);
2165 }
2166
2167 if (ns->params.zoned && nvme_is_write(req)) {
2168 nvme_finalize_zoned_write(ns, req);
2169 }
2170
2171 nvme_enqueue_req_completion(nvme_cq(req), req);
2172 }
2173
nvme_rw_cb(void * opaque,int ret)2174 static void nvme_rw_cb(void *opaque, int ret)
2175 {
2176 NvmeRequest *req = opaque;
2177 NvmeNamespace *ns = req->ns;
2178
2179 BlockBackend *blk = ns->blkconf.blk;
2180
2181 trace_pci_nvme_rw_cb(nvme_cid(req), blk_name(blk));
2182
2183 if (ret) {
2184 goto out;
2185 }
2186
2187 if (ns->lbaf.ms) {
2188 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2189 uint64_t slba = le64_to_cpu(rw->slba);
2190 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
2191 uint64_t offset = nvme_moff(ns, slba);
2192
2193 if (req->cmd.opcode == NVME_CMD_WRITE_ZEROES) {
2194 size_t mlen = nvme_m2b(ns, nlb);
2195
2196 req->aiocb = blk_aio_pwrite_zeroes(blk, offset, mlen,
2197 BDRV_REQ_MAY_UNMAP,
2198 nvme_rw_complete_cb, req);
2199 return;
2200 }
2201
2202 if (nvme_ns_ext(ns) || req->cmd.mptr) {
2203 uint16_t status;
2204
2205 nvme_sg_unmap(&req->sg);
2206 status = nvme_map_mdata(nvme_ctrl(req), nlb, req);
2207 if (status) {
2208 ret = -EFAULT;
2209 goto out;
2210 }
2211
2212 if (req->cmd.opcode == NVME_CMD_READ) {
2213 return nvme_blk_read(blk, offset, 1, nvme_rw_complete_cb, req);
2214 }
2215
2216 return nvme_blk_write(blk, offset, 1, nvme_rw_complete_cb, req);
2217 }
2218 }
2219
2220 out:
2221 nvme_rw_complete_cb(req, ret);
2222 }
2223
nvme_verify_cb(void * opaque,int ret)2224 static void nvme_verify_cb(void *opaque, int ret)
2225 {
2226 NvmeBounceContext *ctx = opaque;
2227 NvmeRequest *req = ctx->req;
2228 NvmeNamespace *ns = req->ns;
2229 BlockBackend *blk = ns->blkconf.blk;
2230 BlockAcctCookie *acct = &req->acct;
2231 BlockAcctStats *stats = blk_get_stats(blk);
2232 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2233 uint64_t slba = le64_to_cpu(rw->slba);
2234 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2235 uint16_t apptag = le16_to_cpu(rw->apptag);
2236 uint16_t appmask = le16_to_cpu(rw->appmask);
2237 uint64_t reftag = le32_to_cpu(rw->reftag);
2238 uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2239 uint16_t status;
2240
2241 reftag |= cdw3 << 32;
2242
2243 trace_pci_nvme_verify_cb(nvme_cid(req), prinfo, apptag, appmask, reftag);
2244
2245 if (ret) {
2246 block_acct_failed(stats, acct);
2247 nvme_aio_err(req, ret);
2248 goto out;
2249 }
2250
2251 block_acct_done(stats, acct);
2252
2253 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2254 status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce,
2255 ctx->mdata.iov.size, slba);
2256 if (status) {
2257 req->status = status;
2258 goto out;
2259 }
2260
2261 req->status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2262 ctx->mdata.bounce, ctx->mdata.iov.size,
2263 prinfo, slba, apptag, appmask, &reftag);
2264 }
2265
2266 out:
2267 qemu_iovec_destroy(&ctx->data.iov);
2268 g_free(ctx->data.bounce);
2269
2270 qemu_iovec_destroy(&ctx->mdata.iov);
2271 g_free(ctx->mdata.bounce);
2272
2273 g_free(ctx);
2274
2275 nvme_enqueue_req_completion(nvme_cq(req), req);
2276 }
2277
2278
nvme_verify_mdata_in_cb(void * opaque,int ret)2279 static void nvme_verify_mdata_in_cb(void *opaque, int ret)
2280 {
2281 NvmeBounceContext *ctx = opaque;
2282 NvmeRequest *req = ctx->req;
2283 NvmeNamespace *ns = req->ns;
2284 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2285 uint64_t slba = le64_to_cpu(rw->slba);
2286 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2287 size_t mlen = nvme_m2b(ns, nlb);
2288 uint64_t offset = nvme_moff(ns, slba);
2289 BlockBackend *blk = ns->blkconf.blk;
2290
2291 trace_pci_nvme_verify_mdata_in_cb(nvme_cid(req), blk_name(blk));
2292
2293 if (ret) {
2294 goto out;
2295 }
2296
2297 ctx->mdata.bounce = g_malloc(mlen);
2298
2299 qemu_iovec_reset(&ctx->mdata.iov);
2300 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2301
2302 req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2303 nvme_verify_cb, ctx);
2304 return;
2305
2306 out:
2307 nvme_verify_cb(ctx, ret);
2308 }
2309
2310 struct nvme_compare_ctx {
2311 struct {
2312 QEMUIOVector iov;
2313 uint8_t *bounce;
2314 } data;
2315
2316 struct {
2317 QEMUIOVector iov;
2318 uint8_t *bounce;
2319 } mdata;
2320 };
2321
nvme_compare_mdata_cb(void * opaque,int ret)2322 static void nvme_compare_mdata_cb(void *opaque, int ret)
2323 {
2324 NvmeRequest *req = opaque;
2325 NvmeNamespace *ns = req->ns;
2326 NvmeCtrl *n = nvme_ctrl(req);
2327 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2328 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2329 uint16_t apptag = le16_to_cpu(rw->apptag);
2330 uint16_t appmask = le16_to_cpu(rw->appmask);
2331 uint64_t reftag = le32_to_cpu(rw->reftag);
2332 uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2333 struct nvme_compare_ctx *ctx = req->opaque;
2334 g_autofree uint8_t *buf = NULL;
2335 BlockBackend *blk = ns->blkconf.blk;
2336 BlockAcctCookie *acct = &req->acct;
2337 BlockAcctStats *stats = blk_get_stats(blk);
2338 uint16_t status = NVME_SUCCESS;
2339
2340 reftag |= cdw3 << 32;
2341
2342 trace_pci_nvme_compare_mdata_cb(nvme_cid(req));
2343
2344 if (ret) {
2345 block_acct_failed(stats, acct);
2346 nvme_aio_err(req, ret);
2347 goto out;
2348 }
2349
2350 buf = g_malloc(ctx->mdata.iov.size);
2351
2352 status = nvme_bounce_mdata(n, buf, ctx->mdata.iov.size,
2353 NVME_TX_DIRECTION_TO_DEVICE, req);
2354 if (status) {
2355 req->status = status;
2356 goto out;
2357 }
2358
2359 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2360 uint64_t slba = le64_to_cpu(rw->slba);
2361 uint8_t *bufp;
2362 uint8_t *mbufp = ctx->mdata.bounce;
2363 uint8_t *end = mbufp + ctx->mdata.iov.size;
2364 int16_t pil = 0;
2365
2366 status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2367 ctx->mdata.bounce, ctx->mdata.iov.size, prinfo,
2368 slba, apptag, appmask, &reftag);
2369 if (status) {
2370 req->status = status;
2371 goto out;
2372 }
2373
2374 /*
2375 * When formatted with protection information, do not compare the DIF
2376 * tuple.
2377 */
2378 if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
2379 pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
2380 }
2381
2382 for (bufp = buf; mbufp < end; bufp += ns->lbaf.ms, mbufp += ns->lbaf.ms) {
2383 if (memcmp(bufp + pil, mbufp + pil, ns->lbaf.ms - pil)) {
2384 req->status = NVME_CMP_FAILURE | NVME_DNR;
2385 goto out;
2386 }
2387 }
2388
2389 goto out;
2390 }
2391
2392 if (memcmp(buf, ctx->mdata.bounce, ctx->mdata.iov.size)) {
2393 req->status = NVME_CMP_FAILURE | NVME_DNR;
2394 goto out;
2395 }
2396
2397 block_acct_done(stats, acct);
2398
2399 out:
2400 qemu_iovec_destroy(&ctx->data.iov);
2401 g_free(ctx->data.bounce);
2402
2403 qemu_iovec_destroy(&ctx->mdata.iov);
2404 g_free(ctx->mdata.bounce);
2405
2406 g_free(ctx);
2407
2408 nvme_enqueue_req_completion(nvme_cq(req), req);
2409 }
2410
nvme_compare_data_cb(void * opaque,int ret)2411 static void nvme_compare_data_cb(void *opaque, int ret)
2412 {
2413 NvmeRequest *req = opaque;
2414 NvmeCtrl *n = nvme_ctrl(req);
2415 NvmeNamespace *ns = req->ns;
2416 BlockBackend *blk = ns->blkconf.blk;
2417 BlockAcctCookie *acct = &req->acct;
2418 BlockAcctStats *stats = blk_get_stats(blk);
2419
2420 struct nvme_compare_ctx *ctx = req->opaque;
2421 g_autofree uint8_t *buf = NULL;
2422 uint16_t status;
2423
2424 trace_pci_nvme_compare_data_cb(nvme_cid(req));
2425
2426 if (ret) {
2427 block_acct_failed(stats, acct);
2428 nvme_aio_err(req, ret);
2429 goto out;
2430 }
2431
2432 buf = g_malloc(ctx->data.iov.size);
2433
2434 status = nvme_bounce_data(n, buf, ctx->data.iov.size,
2435 NVME_TX_DIRECTION_TO_DEVICE, req);
2436 if (status) {
2437 req->status = status;
2438 goto out;
2439 }
2440
2441 if (memcmp(buf, ctx->data.bounce, ctx->data.iov.size)) {
2442 req->status = NVME_CMP_FAILURE | NVME_DNR;
2443 goto out;
2444 }
2445
2446 if (ns->lbaf.ms) {
2447 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2448 uint64_t slba = le64_to_cpu(rw->slba);
2449 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2450 size_t mlen = nvme_m2b(ns, nlb);
2451 uint64_t offset = nvme_moff(ns, slba);
2452
2453 ctx->mdata.bounce = g_malloc(mlen);
2454
2455 qemu_iovec_init(&ctx->mdata.iov, 1);
2456 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2457
2458 req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2459 nvme_compare_mdata_cb, req);
2460 return;
2461 }
2462
2463 block_acct_done(stats, acct);
2464
2465 out:
2466 qemu_iovec_destroy(&ctx->data.iov);
2467 g_free(ctx->data.bounce);
2468 g_free(ctx);
2469
2470 nvme_enqueue_req_completion(nvme_cq(req), req);
2471 }
2472
2473 typedef struct NvmeDSMAIOCB {
2474 BlockAIOCB common;
2475 BlockAIOCB *aiocb;
2476 NvmeRequest *req;
2477 int ret;
2478
2479 NvmeDsmRange *range;
2480 unsigned int nr;
2481 unsigned int idx;
2482 } NvmeDSMAIOCB;
2483
nvme_dsm_cancel(BlockAIOCB * aiocb)2484 static void nvme_dsm_cancel(BlockAIOCB *aiocb)
2485 {
2486 NvmeDSMAIOCB *iocb = container_of(aiocb, NvmeDSMAIOCB, common);
2487
2488 /* break nvme_dsm_cb loop */
2489 iocb->idx = iocb->nr;
2490 iocb->ret = -ECANCELED;
2491
2492 if (iocb->aiocb) {
2493 blk_aio_cancel_async(iocb->aiocb);
2494 iocb->aiocb = NULL;
2495 } else {
2496 /*
2497 * We only reach this if nvme_dsm_cancel() has already been called or
2498 * the command ran to completion.
2499 */
2500 assert(iocb->idx == iocb->nr);
2501 }
2502 }
2503
2504 static const AIOCBInfo nvme_dsm_aiocb_info = {
2505 .aiocb_size = sizeof(NvmeDSMAIOCB),
2506 .cancel_async = nvme_dsm_cancel,
2507 };
2508
2509 static void nvme_dsm_cb(void *opaque, int ret);
2510
nvme_dsm_md_cb(void * opaque,int ret)2511 static void nvme_dsm_md_cb(void *opaque, int ret)
2512 {
2513 NvmeDSMAIOCB *iocb = opaque;
2514 NvmeRequest *req = iocb->req;
2515 NvmeNamespace *ns = req->ns;
2516 NvmeDsmRange *range;
2517 uint64_t slba;
2518 uint32_t nlb;
2519
2520 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2521 goto done;
2522 }
2523
2524 range = &iocb->range[iocb->idx - 1];
2525 slba = le64_to_cpu(range->slba);
2526 nlb = le32_to_cpu(range->nlb);
2527
2528 /*
2529 * Check that all block were discarded (zeroed); otherwise we do not zero
2530 * the metadata.
2531 */
2532
2533 ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_ZERO);
2534 if (ret) {
2535 if (ret < 0) {
2536 goto done;
2537 }
2538
2539 nvme_dsm_cb(iocb, 0);
2540 return;
2541 }
2542
2543 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, nvme_moff(ns, slba),
2544 nvme_m2b(ns, nlb), BDRV_REQ_MAY_UNMAP,
2545 nvme_dsm_cb, iocb);
2546 return;
2547
2548 done:
2549 nvme_dsm_cb(iocb, ret);
2550 }
2551
nvme_dsm_cb(void * opaque,int ret)2552 static void nvme_dsm_cb(void *opaque, int ret)
2553 {
2554 NvmeDSMAIOCB *iocb = opaque;
2555 NvmeRequest *req = iocb->req;
2556 NvmeCtrl *n = nvme_ctrl(req);
2557 NvmeNamespace *ns = req->ns;
2558 NvmeDsmRange *range;
2559 uint64_t slba;
2560 uint32_t nlb;
2561
2562 if (iocb->ret < 0) {
2563 goto done;
2564 } else if (ret < 0) {
2565 iocb->ret = ret;
2566 goto done;
2567 }
2568
2569 next:
2570 if (iocb->idx == iocb->nr) {
2571 goto done;
2572 }
2573
2574 range = &iocb->range[iocb->idx++];
2575 slba = le64_to_cpu(range->slba);
2576 nlb = le32_to_cpu(range->nlb);
2577
2578 trace_pci_nvme_dsm_deallocate(slba, nlb);
2579
2580 if (nlb > n->dmrsl) {
2581 trace_pci_nvme_dsm_single_range_limit_exceeded(nlb, n->dmrsl);
2582 goto next;
2583 }
2584
2585 if (nvme_check_bounds(ns, slba, nlb)) {
2586 trace_pci_nvme_err_invalid_lba_range(slba, nlb,
2587 ns->id_ns.nsze);
2588 goto next;
2589 }
2590
2591 iocb->aiocb = blk_aio_pdiscard(ns->blkconf.blk, nvme_l2b(ns, slba),
2592 nvme_l2b(ns, nlb),
2593 nvme_dsm_md_cb, iocb);
2594 return;
2595
2596 done:
2597 iocb->aiocb = NULL;
2598 iocb->common.cb(iocb->common.opaque, iocb->ret);
2599 g_free(iocb->range);
2600 qemu_aio_unref(iocb);
2601 }
2602
nvme_dsm(NvmeCtrl * n,NvmeRequest * req)2603 static uint16_t nvme_dsm(NvmeCtrl *n, NvmeRequest *req)
2604 {
2605 NvmeNamespace *ns = req->ns;
2606 NvmeDsmCmd *dsm = (NvmeDsmCmd *) &req->cmd;
2607 uint32_t attr = le32_to_cpu(dsm->attributes);
2608 uint32_t nr = (le32_to_cpu(dsm->nr) & 0xff) + 1;
2609 uint16_t status = NVME_SUCCESS;
2610
2611 trace_pci_nvme_dsm(nr, attr);
2612
2613 if (attr & NVME_DSMGMT_AD) {
2614 NvmeDSMAIOCB *iocb = blk_aio_get(&nvme_dsm_aiocb_info, ns->blkconf.blk,
2615 nvme_misc_cb, req);
2616
2617 iocb->req = req;
2618 iocb->ret = 0;
2619 iocb->range = g_new(NvmeDsmRange, nr);
2620 iocb->nr = nr;
2621 iocb->idx = 0;
2622
2623 status = nvme_h2c(n, (uint8_t *)iocb->range, sizeof(NvmeDsmRange) * nr,
2624 req);
2625 if (status) {
2626 g_free(iocb->range);
2627 qemu_aio_unref(iocb);
2628
2629 return status;
2630 }
2631
2632 req->aiocb = &iocb->common;
2633 nvme_dsm_cb(iocb, 0);
2634
2635 return NVME_NO_COMPLETE;
2636 }
2637
2638 return status;
2639 }
2640
nvme_verify(NvmeCtrl * n,NvmeRequest * req)2641 static uint16_t nvme_verify(NvmeCtrl *n, NvmeRequest *req)
2642 {
2643 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2644 NvmeNamespace *ns = req->ns;
2645 BlockBackend *blk = ns->blkconf.blk;
2646 uint64_t slba = le64_to_cpu(rw->slba);
2647 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2648 size_t len = nvme_l2b(ns, nlb);
2649 int64_t offset = nvme_l2b(ns, slba);
2650 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2651 uint32_t reftag = le32_to_cpu(rw->reftag);
2652 NvmeBounceContext *ctx = NULL;
2653 uint16_t status;
2654
2655 trace_pci_nvme_verify(nvme_cid(req), nvme_nsid(ns), slba, nlb);
2656
2657 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2658 status = nvme_check_prinfo(ns, prinfo, slba, reftag);
2659 if (status) {
2660 return status;
2661 }
2662
2663 if (prinfo & NVME_PRINFO_PRACT) {
2664 return NVME_INVALID_PROT_INFO | NVME_DNR;
2665 }
2666 }
2667
2668 if (len > n->page_size << n->params.vsl) {
2669 return NVME_INVALID_FIELD | NVME_DNR;
2670 }
2671
2672 status = nvme_check_bounds(ns, slba, nlb);
2673 if (status) {
2674 return status;
2675 }
2676
2677 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
2678 status = nvme_check_dulbe(ns, slba, nlb);
2679 if (status) {
2680 return status;
2681 }
2682 }
2683
2684 ctx = g_new0(NvmeBounceContext, 1);
2685 ctx->req = req;
2686
2687 ctx->data.bounce = g_malloc(len);
2688
2689 qemu_iovec_init(&ctx->data.iov, 1);
2690 qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len);
2691
2692 block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size,
2693 BLOCK_ACCT_READ);
2694
2695 req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0,
2696 nvme_verify_mdata_in_cb, ctx);
2697 return NVME_NO_COMPLETE;
2698 }
2699
2700 typedef struct NvmeCopyAIOCB {
2701 BlockAIOCB common;
2702 BlockAIOCB *aiocb;
2703 NvmeRequest *req;
2704 NvmeCtrl *n;
2705 int ret;
2706
2707 void *ranges;
2708 unsigned int format;
2709 int nr;
2710 int idx;
2711
2712 uint8_t *bounce;
2713 QEMUIOVector iov;
2714 struct {
2715 BlockAcctCookie read;
2716 BlockAcctCookie write;
2717 } acct;
2718
2719 uint64_t reftag;
2720 uint64_t slba;
2721
2722 NvmeZone *zone;
2723 NvmeNamespace *sns;
2724 uint32_t tcl;
2725 } NvmeCopyAIOCB;
2726
nvme_copy_cancel(BlockAIOCB * aiocb)2727 static void nvme_copy_cancel(BlockAIOCB *aiocb)
2728 {
2729 NvmeCopyAIOCB *iocb = container_of(aiocb, NvmeCopyAIOCB, common);
2730
2731 iocb->ret = -ECANCELED;
2732
2733 if (iocb->aiocb) {
2734 blk_aio_cancel_async(iocb->aiocb);
2735 iocb->aiocb = NULL;
2736 }
2737 }
2738
2739 static const AIOCBInfo nvme_copy_aiocb_info = {
2740 .aiocb_size = sizeof(NvmeCopyAIOCB),
2741 .cancel_async = nvme_copy_cancel,
2742 };
2743
nvme_copy_done(NvmeCopyAIOCB * iocb)2744 static void nvme_copy_done(NvmeCopyAIOCB *iocb)
2745 {
2746 NvmeRequest *req = iocb->req;
2747 NvmeNamespace *ns = req->ns;
2748 BlockAcctStats *stats = blk_get_stats(ns->blkconf.blk);
2749
2750 if (iocb->idx != iocb->nr) {
2751 req->cqe.result = cpu_to_le32(iocb->idx);
2752 }
2753
2754 qemu_iovec_destroy(&iocb->iov);
2755 g_free(iocb->bounce);
2756
2757 if (iocb->ret < 0) {
2758 block_acct_failed(stats, &iocb->acct.read);
2759 block_acct_failed(stats, &iocb->acct.write);
2760 } else {
2761 block_acct_done(stats, &iocb->acct.read);
2762 block_acct_done(stats, &iocb->acct.write);
2763 }
2764
2765 iocb->common.cb(iocb->common.opaque, iocb->ret);
2766 qemu_aio_unref(iocb);
2767 }
2768
2769 static void nvme_do_copy(NvmeCopyAIOCB *iocb);
2770
nvme_copy_source_range_parse_format0_2(void * ranges,int idx,uint64_t * slba,uint32_t * nlb,uint32_t * snsid,uint16_t * apptag,uint16_t * appmask,uint64_t * reftag)2771 static void nvme_copy_source_range_parse_format0_2(void *ranges,
2772 int idx, uint64_t *slba,
2773 uint32_t *nlb,
2774 uint32_t *snsid,
2775 uint16_t *apptag,
2776 uint16_t *appmask,
2777 uint64_t *reftag)
2778 {
2779 NvmeCopySourceRangeFormat0_2 *_ranges = ranges;
2780
2781 if (snsid) {
2782 *snsid = le32_to_cpu(_ranges[idx].sparams);
2783 }
2784
2785 if (slba) {
2786 *slba = le64_to_cpu(_ranges[idx].slba);
2787 }
2788
2789 if (nlb) {
2790 *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2791 }
2792
2793 if (apptag) {
2794 *apptag = le16_to_cpu(_ranges[idx].apptag);
2795 }
2796
2797 if (appmask) {
2798 *appmask = le16_to_cpu(_ranges[idx].appmask);
2799 }
2800
2801 if (reftag) {
2802 *reftag = le32_to_cpu(_ranges[idx].reftag);
2803 }
2804 }
2805
nvme_copy_source_range_parse_format1_3(void * ranges,int idx,uint64_t * slba,uint32_t * nlb,uint32_t * snsid,uint16_t * apptag,uint16_t * appmask,uint64_t * reftag)2806 static void nvme_copy_source_range_parse_format1_3(void *ranges, int idx,
2807 uint64_t *slba,
2808 uint32_t *nlb,
2809 uint32_t *snsid,
2810 uint16_t *apptag,
2811 uint16_t *appmask,
2812 uint64_t *reftag)
2813 {
2814 NvmeCopySourceRangeFormat1_3 *_ranges = ranges;
2815
2816 if (snsid) {
2817 *snsid = le32_to_cpu(_ranges[idx].sparams);
2818 }
2819
2820 if (slba) {
2821 *slba = le64_to_cpu(_ranges[idx].slba);
2822 }
2823
2824 if (nlb) {
2825 *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2826 }
2827
2828 if (apptag) {
2829 *apptag = le16_to_cpu(_ranges[idx].apptag);
2830 }
2831
2832 if (appmask) {
2833 *appmask = le16_to_cpu(_ranges[idx].appmask);
2834 }
2835
2836 if (reftag) {
2837 *reftag = 0;
2838
2839 *reftag |= (uint64_t)_ranges[idx].sr[4] << 40;
2840 *reftag |= (uint64_t)_ranges[idx].sr[5] << 32;
2841 *reftag |= (uint64_t)_ranges[idx].sr[6] << 24;
2842 *reftag |= (uint64_t)_ranges[idx].sr[7] << 16;
2843 *reftag |= (uint64_t)_ranges[idx].sr[8] << 8;
2844 *reftag |= (uint64_t)_ranges[idx].sr[9];
2845 }
2846 }
2847
nvme_copy_source_range_parse(void * ranges,int idx,uint8_t format,uint64_t * slba,uint32_t * nlb,uint32_t * snsid,uint16_t * apptag,uint16_t * appmask,uint64_t * reftag)2848 static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format,
2849 uint64_t *slba, uint32_t *nlb,
2850 uint32_t *snsid, uint16_t *apptag,
2851 uint16_t *appmask, uint64_t *reftag)
2852 {
2853 switch (format) {
2854 case NVME_COPY_FORMAT_0:
2855 case NVME_COPY_FORMAT_2:
2856 nvme_copy_source_range_parse_format0_2(ranges, idx, slba, nlb, snsid,
2857 apptag, appmask, reftag);
2858 break;
2859
2860 case NVME_COPY_FORMAT_1:
2861 case NVME_COPY_FORMAT_3:
2862 nvme_copy_source_range_parse_format1_3(ranges, idx, slba, nlb, snsid,
2863 apptag, appmask, reftag);
2864 break;
2865
2866 default:
2867 abort();
2868 }
2869 }
2870
nvme_check_copy_mcl(NvmeNamespace * ns,NvmeCopyAIOCB * iocb,uint16_t nr)2871 static inline uint16_t nvme_check_copy_mcl(NvmeNamespace *ns,
2872 NvmeCopyAIOCB *iocb, uint16_t nr)
2873 {
2874 uint32_t copy_len = 0;
2875
2876 for (int idx = 0; idx < nr; idx++) {
2877 uint32_t nlb;
2878 nvme_copy_source_range_parse(iocb->ranges, idx, iocb->format, NULL,
2879 &nlb, NULL, NULL, NULL, NULL);
2880 copy_len += nlb;
2881 }
2882 iocb->tcl = copy_len;
2883 if (copy_len > ns->id_ns.mcl) {
2884 return NVME_CMD_SIZE_LIMIT | NVME_DNR;
2885 }
2886
2887 return NVME_SUCCESS;
2888 }
2889
nvme_copy_out_completed_cb(void * opaque,int ret)2890 static void nvme_copy_out_completed_cb(void *opaque, int ret)
2891 {
2892 NvmeCopyAIOCB *iocb = opaque;
2893 NvmeRequest *req = iocb->req;
2894 NvmeNamespace *dns = req->ns;
2895 uint32_t nlb;
2896
2897 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2898 &nlb, NULL, NULL, NULL, NULL);
2899
2900 if (ret < 0) {
2901 iocb->ret = ret;
2902 goto out;
2903 } else if (iocb->ret < 0) {
2904 goto out;
2905 }
2906
2907 if (dns->params.zoned) {
2908 nvme_advance_zone_wp(dns, iocb->zone, nlb);
2909 }
2910
2911 iocb->idx++;
2912 iocb->slba += nlb;
2913 out:
2914 nvme_do_copy(iocb);
2915 }
2916
nvme_copy_out_cb(void * opaque,int ret)2917 static void nvme_copy_out_cb(void *opaque, int ret)
2918 {
2919 NvmeCopyAIOCB *iocb = opaque;
2920 NvmeRequest *req = iocb->req;
2921 NvmeNamespace *dns = req->ns;
2922 uint32_t nlb;
2923 size_t mlen;
2924 uint8_t *mbounce;
2925
2926 if (ret < 0 || iocb->ret < 0 || !dns->lbaf.ms) {
2927 goto out;
2928 }
2929
2930 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2931 &nlb, NULL, NULL, NULL, NULL);
2932
2933 mlen = nvme_m2b(dns, nlb);
2934 mbounce = iocb->bounce + nvme_l2b(dns, nlb);
2935
2936 qemu_iovec_reset(&iocb->iov);
2937 qemu_iovec_add(&iocb->iov, mbounce, mlen);
2938
2939 iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_moff(dns, iocb->slba),
2940 &iocb->iov, 0, nvme_copy_out_completed_cb,
2941 iocb);
2942
2943 return;
2944
2945 out:
2946 nvme_copy_out_completed_cb(iocb, ret);
2947 }
2948
nvme_copy_in_completed_cb(void * opaque,int ret)2949 static void nvme_copy_in_completed_cb(void *opaque, int ret)
2950 {
2951 NvmeCopyAIOCB *iocb = opaque;
2952 NvmeRequest *req = iocb->req;
2953 NvmeNamespace *sns = iocb->sns;
2954 NvmeNamespace *dns = req->ns;
2955 NvmeCopyCmd *copy = NULL;
2956 uint8_t *mbounce = NULL;
2957 uint32_t nlb;
2958 uint64_t slba;
2959 uint16_t apptag, appmask;
2960 uint64_t reftag;
2961 size_t len, mlen;
2962 uint16_t status;
2963
2964 if (ret < 0) {
2965 iocb->ret = ret;
2966 goto out;
2967 } else if (iocb->ret < 0) {
2968 goto out;
2969 }
2970
2971 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
2972 &nlb, NULL, &apptag, &appmask, &reftag);
2973
2974 trace_pci_nvme_copy_out(iocb->slba, nlb);
2975
2976 len = nvme_l2b(sns, nlb);
2977
2978 if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps)) {
2979 copy = (NvmeCopyCmd *)&req->cmd;
2980
2981 uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
2982
2983 mlen = nvme_m2b(sns, nlb);
2984 mbounce = iocb->bounce + nvme_l2b(sns, nlb);
2985
2986 status = nvme_dif_mangle_mdata(sns, mbounce, mlen, slba);
2987 if (status) {
2988 goto invalid;
2989 }
2990 status = nvme_dif_check(sns, iocb->bounce, len, mbounce, mlen, prinfor,
2991 slba, apptag, appmask, &reftag);
2992 if (status) {
2993 goto invalid;
2994 }
2995 }
2996
2997 if (NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
2998 copy = (NvmeCopyCmd *)&req->cmd;
2999 uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
3000
3001 mlen = nvme_m2b(dns, nlb);
3002 mbounce = iocb->bounce + nvme_l2b(dns, nlb);
3003
3004 apptag = le16_to_cpu(copy->apptag);
3005 appmask = le16_to_cpu(copy->appmask);
3006
3007 if (prinfow & NVME_PRINFO_PRACT) {
3008 status = nvme_check_prinfo(dns, prinfow, iocb->slba, iocb->reftag);
3009 if (status) {
3010 goto invalid;
3011 }
3012
3013 nvme_dif_pract_generate_dif(dns, iocb->bounce, len, mbounce, mlen,
3014 apptag, &iocb->reftag);
3015 } else {
3016 status = nvme_dif_check(dns, iocb->bounce, len, mbounce, mlen,
3017 prinfow, iocb->slba, apptag, appmask,
3018 &iocb->reftag);
3019 if (status) {
3020 goto invalid;
3021 }
3022 }
3023 }
3024
3025 status = nvme_check_bounds(dns, iocb->slba, nlb);
3026 if (status) {
3027 goto invalid;
3028 }
3029
3030 if (dns->params.zoned) {
3031 status = nvme_check_zone_write(dns, iocb->zone, iocb->slba, nlb);
3032 if (status) {
3033 goto invalid;
3034 }
3035
3036 if (!(iocb->zone->d.za & NVME_ZA_ZRWA_VALID)) {
3037 iocb->zone->w_ptr += nlb;
3038 }
3039 }
3040
3041 qemu_iovec_reset(&iocb->iov);
3042 qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3043
3044 block_acct_start(blk_get_stats(dns->blkconf.blk), &iocb->acct.write, 0,
3045 BLOCK_ACCT_WRITE);
3046
3047 iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_l2b(dns, iocb->slba),
3048 &iocb->iov, 0, nvme_copy_out_cb, iocb);
3049
3050 return;
3051
3052 invalid:
3053 req->status = status;
3054 iocb->ret = -1;
3055 out:
3056 nvme_do_copy(iocb);
3057 }
3058
nvme_copy_in_cb(void * opaque,int ret)3059 static void nvme_copy_in_cb(void *opaque, int ret)
3060 {
3061 NvmeCopyAIOCB *iocb = opaque;
3062 NvmeNamespace *sns = iocb->sns;
3063 uint64_t slba;
3064 uint32_t nlb;
3065
3066 if (ret < 0 || iocb->ret < 0 || !sns->lbaf.ms) {
3067 goto out;
3068 }
3069
3070 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3071 &nlb, NULL, NULL, NULL, NULL);
3072
3073 qemu_iovec_reset(&iocb->iov);
3074 qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(sns, nlb),
3075 nvme_m2b(sns, nlb));
3076
3077 iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_moff(sns, slba),
3078 &iocb->iov, 0, nvme_copy_in_completed_cb,
3079 iocb);
3080 return;
3081
3082 out:
3083 nvme_copy_in_completed_cb(iocb, ret);
3084 }
3085
nvme_csi_supports_copy(uint8_t csi)3086 static inline bool nvme_csi_supports_copy(uint8_t csi)
3087 {
3088 return csi == NVME_CSI_NVM || csi == NVME_CSI_ZONED;
3089 }
3090
nvme_copy_ns_format_match(NvmeNamespace * sns,NvmeNamespace * dns)3091 static inline bool nvme_copy_ns_format_match(NvmeNamespace *sns,
3092 NvmeNamespace *dns)
3093 {
3094 return sns->lbaf.ds == dns->lbaf.ds && sns->lbaf.ms == dns->lbaf.ms;
3095 }
3096
nvme_copy_matching_ns_format(NvmeNamespace * sns,NvmeNamespace * dns,bool pi_enable)3097 static bool nvme_copy_matching_ns_format(NvmeNamespace *sns, NvmeNamespace *dns,
3098 bool pi_enable)
3099 {
3100 if (!nvme_csi_supports_copy(sns->csi) ||
3101 !nvme_csi_supports_copy(dns->csi)) {
3102 return false;
3103 }
3104
3105 if (!pi_enable && !nvme_copy_ns_format_match(sns, dns)) {
3106 return false;
3107 }
3108
3109 if (pi_enable && (!nvme_copy_ns_format_match(sns, dns) ||
3110 sns->id_ns.dps != dns->id_ns.dps)) {
3111 return false;
3112 }
3113
3114 return true;
3115 }
3116
nvme_copy_corresp_pi_match(NvmeNamespace * sns,NvmeNamespace * dns)3117 static inline bool nvme_copy_corresp_pi_match(NvmeNamespace *sns,
3118 NvmeNamespace *dns)
3119 {
3120 return sns->lbaf.ms == 0 &&
3121 ((dns->lbaf.ms == 8 && dns->pif == 0) ||
3122 (dns->lbaf.ms == 16 && dns->pif == 1));
3123 }
3124
nvme_copy_corresp_pi_format(NvmeNamespace * sns,NvmeNamespace * dns,bool sns_pi_en)3125 static bool nvme_copy_corresp_pi_format(NvmeNamespace *sns, NvmeNamespace *dns,
3126 bool sns_pi_en)
3127 {
3128 if (!nvme_csi_supports_copy(sns->csi) ||
3129 !nvme_csi_supports_copy(dns->csi)) {
3130 return false;
3131 }
3132
3133 if (!sns_pi_en && !nvme_copy_corresp_pi_match(sns, dns)) {
3134 return false;
3135 }
3136
3137 if (sns_pi_en && !nvme_copy_corresp_pi_match(dns, sns)) {
3138 return false;
3139 }
3140
3141 return true;
3142 }
3143
nvme_do_copy(NvmeCopyAIOCB * iocb)3144 static void nvme_do_copy(NvmeCopyAIOCB *iocb)
3145 {
3146 NvmeRequest *req = iocb->req;
3147 NvmeNamespace *sns;
3148 NvmeNamespace *dns = req->ns;
3149 NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3150 uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
3151 uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
3152 uint64_t slba;
3153 uint32_t nlb;
3154 size_t len;
3155 uint16_t status;
3156 uint32_t dnsid = le32_to_cpu(req->cmd.nsid);
3157 uint32_t snsid = dnsid;
3158
3159 if (iocb->ret < 0) {
3160 goto done;
3161 }
3162
3163 if (iocb->idx == iocb->nr) {
3164 goto done;
3165 }
3166
3167 if (iocb->format == 2 || iocb->format == 3) {
3168 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format,
3169 &slba, &nlb, &snsid, NULL, NULL, NULL);
3170 if (snsid != dnsid) {
3171 if (snsid == NVME_NSID_BROADCAST ||
3172 !nvme_nsid_valid(iocb->n, snsid)) {
3173 status = NVME_INVALID_NSID | NVME_DNR;
3174 goto invalid;
3175 }
3176 iocb->sns = nvme_ns(iocb->n, snsid);
3177 if (unlikely(!iocb->sns)) {
3178 status = NVME_INVALID_FIELD | NVME_DNR;
3179 goto invalid;
3180 }
3181 } else {
3182 if (((slba + nlb) > iocb->slba) &&
3183 ((slba + nlb) < (iocb->slba + iocb->tcl))) {
3184 status = NVME_CMD_OVERLAP_IO_RANGE | NVME_DNR;
3185 goto invalid;
3186 }
3187 }
3188 } else {
3189 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format,
3190 &slba, &nlb, NULL, NULL, NULL, NULL);
3191 }
3192
3193 sns = iocb->sns;
3194 if ((snsid == dnsid) && NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3195 ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) {
3196 status = NVME_INVALID_FIELD | NVME_DNR;
3197 goto invalid;
3198 } else if (snsid != dnsid) {
3199 if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3200 !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3201 if (!nvme_copy_matching_ns_format(sns, dns, false)) {
3202 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3203 goto invalid;
3204 }
3205 }
3206 if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3207 NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3208 if ((prinfor & NVME_PRINFO_PRACT) !=
3209 (prinfow & NVME_PRINFO_PRACT)) {
3210 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3211 goto invalid;
3212 } else {
3213 if (!nvme_copy_matching_ns_format(sns, dns, true)) {
3214 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3215 goto invalid;
3216 }
3217 }
3218 }
3219
3220 if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3221 NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3222 if (!(prinfow & NVME_PRINFO_PRACT)) {
3223 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3224 goto invalid;
3225 } else {
3226 if (!nvme_copy_corresp_pi_format(sns, dns, false)) {
3227 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3228 goto invalid;
3229 }
3230 }
3231 }
3232
3233 if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3234 !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3235 if (!(prinfor & NVME_PRINFO_PRACT)) {
3236 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3237 goto invalid;
3238 } else {
3239 if (!nvme_copy_corresp_pi_format(sns, dns, true)) {
3240 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3241 goto invalid;
3242 }
3243 }
3244 }
3245 }
3246 len = nvme_l2b(sns, nlb);
3247
3248 trace_pci_nvme_copy_source_range(slba, nlb);
3249
3250 if (nlb > le16_to_cpu(sns->id_ns.mssrl)) {
3251 status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3252 goto invalid;
3253 }
3254
3255 status = nvme_check_bounds(sns, slba, nlb);
3256 if (status) {
3257 goto invalid;
3258 }
3259
3260 if (NVME_ERR_REC_DULBE(sns->features.err_rec)) {
3261 status = nvme_check_dulbe(sns, slba, nlb);
3262 if (status) {
3263 goto invalid;
3264 }
3265 }
3266
3267 if (sns->params.zoned) {
3268 status = nvme_check_zone_read(sns, slba, nlb);
3269 if (status) {
3270 goto invalid;
3271 }
3272 }
3273
3274 g_free(iocb->bounce);
3275 iocb->bounce = g_malloc_n(le16_to_cpu(sns->id_ns.mssrl),
3276 sns->lbasz + sns->lbaf.ms);
3277
3278 qemu_iovec_reset(&iocb->iov);
3279 qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3280
3281 block_acct_start(blk_get_stats(sns->blkconf.blk), &iocb->acct.read, 0,
3282 BLOCK_ACCT_READ);
3283
3284 iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_l2b(sns, slba),
3285 &iocb->iov, 0, nvme_copy_in_cb, iocb);
3286 return;
3287
3288 invalid:
3289 req->status = status;
3290 iocb->ret = -1;
3291 done:
3292 nvme_copy_done(iocb);
3293 }
3294
nvme_copy(NvmeCtrl * n,NvmeRequest * req)3295 static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req)
3296 {
3297 NvmeNamespace *ns = req->ns;
3298 NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3299 NvmeCopyAIOCB *iocb = blk_aio_get(&nvme_copy_aiocb_info, ns->blkconf.blk,
3300 nvme_misc_cb, req);
3301 uint16_t nr = copy->nr + 1;
3302 uint8_t format = copy->control[0] & 0xf;
3303 size_t len = sizeof(NvmeCopySourceRangeFormat0_2);
3304
3305 uint16_t status;
3306
3307 trace_pci_nvme_copy(nvme_cid(req), nvme_nsid(ns), nr, format);
3308
3309 iocb->ranges = NULL;
3310 iocb->zone = NULL;
3311
3312 if (!(n->id_ctrl.ocfs & (1 << format)) ||
3313 ((format == 2 || format == 3) &&
3314 !(n->features.hbs.cdfe & (1 << format)))) {
3315 trace_pci_nvme_err_copy_invalid_format(format);
3316 status = NVME_INVALID_FIELD | NVME_DNR;
3317 goto invalid;
3318 }
3319
3320 if (nr > ns->id_ns.msrc + 1) {
3321 status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3322 goto invalid;
3323 }
3324
3325 if ((ns->pif == 0x0 && (format != 0x0 && format != 0x2)) ||
3326 (ns->pif != 0x0 && (format != 0x1 && format != 0x3))) {
3327 status = NVME_INVALID_FORMAT | NVME_DNR;
3328 goto invalid;
3329 }
3330
3331 if (ns->pif) {
3332 len = sizeof(NvmeCopySourceRangeFormat1_3);
3333 }
3334
3335 iocb->format = format;
3336 iocb->ranges = g_malloc_n(nr, len);
3337 status = nvme_h2c(n, (uint8_t *)iocb->ranges, len * nr, req);
3338 if (status) {
3339 goto invalid;
3340 }
3341
3342 iocb->slba = le64_to_cpu(copy->sdlba);
3343
3344 if (ns->params.zoned) {
3345 iocb->zone = nvme_get_zone_by_slba(ns, iocb->slba);
3346 if (!iocb->zone) {
3347 status = NVME_LBA_RANGE | NVME_DNR;
3348 goto invalid;
3349 }
3350
3351 status = nvme_zrm_auto(n, ns, iocb->zone);
3352 if (status) {
3353 goto invalid;
3354 }
3355 }
3356
3357 status = nvme_check_copy_mcl(ns, iocb, nr);
3358 if (status) {
3359 goto invalid;
3360 }
3361
3362 iocb->req = req;
3363 iocb->ret = 0;
3364 iocb->nr = nr;
3365 iocb->idx = 0;
3366 iocb->reftag = le32_to_cpu(copy->reftag);
3367 iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32;
3368
3369 qemu_iovec_init(&iocb->iov, 1);
3370
3371 req->aiocb = &iocb->common;
3372 iocb->sns = req->ns;
3373 iocb->n = n;
3374 iocb->bounce = NULL;
3375 nvme_do_copy(iocb);
3376
3377 return NVME_NO_COMPLETE;
3378
3379 invalid:
3380 g_free(iocb->ranges);
3381 qemu_aio_unref(iocb);
3382 return status;
3383 }
3384
nvme_compare(NvmeCtrl * n,NvmeRequest * req)3385 static uint16_t nvme_compare(NvmeCtrl *n, NvmeRequest *req)
3386 {
3387 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3388 NvmeNamespace *ns = req->ns;
3389 BlockBackend *blk = ns->blkconf.blk;
3390 uint64_t slba = le64_to_cpu(rw->slba);
3391 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
3392 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3393 size_t data_len = nvme_l2b(ns, nlb);
3394 size_t len = data_len;
3395 int64_t offset = nvme_l2b(ns, slba);
3396 struct nvme_compare_ctx *ctx = NULL;
3397 uint16_t status;
3398
3399 trace_pci_nvme_compare(nvme_cid(req), nvme_nsid(ns), slba, nlb);
3400
3401 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) && (prinfo & NVME_PRINFO_PRACT)) {
3402 return NVME_INVALID_PROT_INFO | NVME_DNR;
3403 }
3404
3405 if (nvme_ns_ext(ns)) {
3406 len += nvme_m2b(ns, nlb);
3407 }
3408
3409 status = nvme_check_mdts(n, len);
3410 if (status) {
3411 return status;
3412 }
3413
3414 status = nvme_check_bounds(ns, slba, nlb);
3415 if (status) {
3416 return status;
3417 }
3418
3419 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3420 status = nvme_check_dulbe(ns, slba, nlb);
3421 if (status) {
3422 return status;
3423 }
3424 }
3425
3426 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
3427 if (status) {
3428 return status;
3429 }
3430
3431 ctx = g_new(struct nvme_compare_ctx, 1);
3432 ctx->data.bounce = g_malloc(data_len);
3433
3434 req->opaque = ctx;
3435
3436 qemu_iovec_init(&ctx->data.iov, 1);
3437 qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, data_len);
3438
3439 block_acct_start(blk_get_stats(blk), &req->acct, data_len,
3440 BLOCK_ACCT_READ);
3441 req->aiocb = blk_aio_preadv(blk, offset, &ctx->data.iov, 0,
3442 nvme_compare_data_cb, req);
3443
3444 return NVME_NO_COMPLETE;
3445 }
3446
3447 typedef struct NvmeFlushAIOCB {
3448 BlockAIOCB common;
3449 BlockAIOCB *aiocb;
3450 NvmeRequest *req;
3451 int ret;
3452
3453 NvmeNamespace *ns;
3454 uint32_t nsid;
3455 bool broadcast;
3456 } NvmeFlushAIOCB;
3457
nvme_flush_cancel(BlockAIOCB * acb)3458 static void nvme_flush_cancel(BlockAIOCB *acb)
3459 {
3460 NvmeFlushAIOCB *iocb = container_of(acb, NvmeFlushAIOCB, common);
3461
3462 iocb->ret = -ECANCELED;
3463
3464 if (iocb->aiocb) {
3465 blk_aio_cancel_async(iocb->aiocb);
3466 iocb->aiocb = NULL;
3467 }
3468 }
3469
3470 static const AIOCBInfo nvme_flush_aiocb_info = {
3471 .aiocb_size = sizeof(NvmeFlushAIOCB),
3472 .cancel_async = nvme_flush_cancel,
3473 };
3474
3475 static void nvme_do_flush(NvmeFlushAIOCB *iocb);
3476
nvme_flush_ns_cb(void * opaque,int ret)3477 static void nvme_flush_ns_cb(void *opaque, int ret)
3478 {
3479 NvmeFlushAIOCB *iocb = opaque;
3480 NvmeNamespace *ns = iocb->ns;
3481
3482 if (ret < 0) {
3483 iocb->ret = ret;
3484 goto out;
3485 } else if (iocb->ret < 0) {
3486 goto out;
3487 }
3488
3489 if (ns) {
3490 trace_pci_nvme_flush_ns(iocb->nsid);
3491
3492 iocb->ns = NULL;
3493 iocb->aiocb = blk_aio_flush(ns->blkconf.blk, nvme_flush_ns_cb, iocb);
3494 return;
3495 }
3496
3497 out:
3498 nvme_do_flush(iocb);
3499 }
3500
nvme_do_flush(NvmeFlushAIOCB * iocb)3501 static void nvme_do_flush(NvmeFlushAIOCB *iocb)
3502 {
3503 NvmeRequest *req = iocb->req;
3504 NvmeCtrl *n = nvme_ctrl(req);
3505 int i;
3506
3507 if (iocb->ret < 0) {
3508 goto done;
3509 }
3510
3511 if (iocb->broadcast) {
3512 for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
3513 iocb->ns = nvme_ns(n, i);
3514 if (iocb->ns) {
3515 iocb->nsid = i;
3516 break;
3517 }
3518 }
3519 }
3520
3521 if (!iocb->ns) {
3522 goto done;
3523 }
3524
3525 nvme_flush_ns_cb(iocb, 0);
3526 return;
3527
3528 done:
3529 iocb->common.cb(iocb->common.opaque, iocb->ret);
3530 qemu_aio_unref(iocb);
3531 }
3532
nvme_flush(NvmeCtrl * n,NvmeRequest * req)3533 static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req)
3534 {
3535 NvmeFlushAIOCB *iocb;
3536 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
3537 uint16_t status;
3538
3539 iocb = qemu_aio_get(&nvme_flush_aiocb_info, NULL, nvme_misc_cb, req);
3540
3541 iocb->req = req;
3542 iocb->ret = 0;
3543 iocb->ns = NULL;
3544 iocb->nsid = 0;
3545 iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
3546
3547 if (!iocb->broadcast) {
3548 if (!nvme_nsid_valid(n, nsid)) {
3549 status = NVME_INVALID_NSID | NVME_DNR;
3550 goto out;
3551 }
3552
3553 iocb->ns = nvme_ns(n, nsid);
3554 if (!iocb->ns) {
3555 status = NVME_INVALID_FIELD | NVME_DNR;
3556 goto out;
3557 }
3558
3559 iocb->nsid = nsid;
3560 }
3561
3562 req->aiocb = &iocb->common;
3563 nvme_do_flush(iocb);
3564
3565 return NVME_NO_COMPLETE;
3566
3567 out:
3568 qemu_aio_unref(iocb);
3569
3570 return status;
3571 }
3572
nvme_read(NvmeCtrl * n,NvmeRequest * req)3573 static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req)
3574 {
3575 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3576 NvmeNamespace *ns = req->ns;
3577 uint64_t slba = le64_to_cpu(rw->slba);
3578 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3579 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3580 uint64_t data_size = nvme_l2b(ns, nlb);
3581 uint64_t mapped_size = data_size;
3582 uint64_t data_offset;
3583 BlockBackend *blk = ns->blkconf.blk;
3584 uint16_t status;
3585
3586 if (nvme_ns_ext(ns)) {
3587 mapped_size += nvme_m2b(ns, nlb);
3588
3589 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3590 bool pract = prinfo & NVME_PRINFO_PRACT;
3591
3592 if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3593 mapped_size = data_size;
3594 }
3595 }
3596 }
3597
3598 trace_pci_nvme_read(nvme_cid(req), nvme_nsid(ns), nlb, mapped_size, slba);
3599
3600 status = nvme_check_mdts(n, mapped_size);
3601 if (status) {
3602 goto invalid;
3603 }
3604
3605 status = nvme_check_bounds(ns, slba, nlb);
3606 if (status) {
3607 goto invalid;
3608 }
3609
3610 if (ns->params.zoned) {
3611 status = nvme_check_zone_read(ns, slba, nlb);
3612 if (status) {
3613 trace_pci_nvme_err_zone_read_not_ok(slba, nlb, status);
3614 goto invalid;
3615 }
3616 }
3617
3618 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3619 status = nvme_check_dulbe(ns, slba, nlb);
3620 if (status) {
3621 goto invalid;
3622 }
3623 }
3624
3625 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3626 return nvme_dif_rw(n, req);
3627 }
3628
3629 status = nvme_map_data(n, nlb, req);
3630 if (status) {
3631 goto invalid;
3632 }
3633
3634 data_offset = nvme_l2b(ns, slba);
3635
3636 block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3637 BLOCK_ACCT_READ);
3638 nvme_blk_read(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3639 return NVME_NO_COMPLETE;
3640
3641 invalid:
3642 block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
3643 return status | NVME_DNR;
3644 }
3645
nvme_do_write_fdp(NvmeCtrl * n,NvmeRequest * req,uint64_t slba,uint32_t nlb)3646 static void nvme_do_write_fdp(NvmeCtrl *n, NvmeRequest *req, uint64_t slba,
3647 uint32_t nlb)
3648 {
3649 NvmeNamespace *ns = req->ns;
3650 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3651 uint64_t data_size = nvme_l2b(ns, nlb);
3652 uint32_t dw12 = le32_to_cpu(req->cmd.cdw12);
3653 uint8_t dtype = (dw12 >> 20) & 0xf;
3654 uint16_t pid = le16_to_cpu(rw->dspec);
3655 uint16_t ph, rg, ruhid;
3656 NvmeReclaimUnit *ru;
3657
3658 if (dtype != NVME_DIRECTIVE_DATA_PLACEMENT ||
3659 !nvme_parse_pid(ns, pid, &ph, &rg)) {
3660 ph = 0;
3661 rg = 0;
3662 }
3663
3664 ruhid = ns->fdp.phs[ph];
3665 ru = &ns->endgrp->fdp.ruhs[ruhid].rus[rg];
3666
3667 nvme_fdp_stat_inc(&ns->endgrp->fdp.hbmw, data_size);
3668 nvme_fdp_stat_inc(&ns->endgrp->fdp.mbmw, data_size);
3669
3670 while (nlb) {
3671 if (nlb < ru->ruamw) {
3672 ru->ruamw -= nlb;
3673 break;
3674 }
3675
3676 nlb -= ru->ruamw;
3677 nvme_update_ruh(n, ns, pid);
3678 }
3679 }
3680
nvme_do_write(NvmeCtrl * n,NvmeRequest * req,bool append,bool wrz)3681 static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append,
3682 bool wrz)
3683 {
3684 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3685 NvmeNamespace *ns = req->ns;
3686 uint64_t slba = le64_to_cpu(rw->slba);
3687 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3688 uint16_t ctrl = le16_to_cpu(rw->control);
3689 uint8_t prinfo = NVME_RW_PRINFO(ctrl);
3690 uint64_t data_size = nvme_l2b(ns, nlb);
3691 uint64_t mapped_size = data_size;
3692 uint64_t data_offset;
3693 NvmeZone *zone;
3694 NvmeZonedResult *res = (NvmeZonedResult *)&req->cqe;
3695 BlockBackend *blk = ns->blkconf.blk;
3696 uint16_t status;
3697
3698 if (nvme_ns_ext(ns)) {
3699 mapped_size += nvme_m2b(ns, nlb);
3700
3701 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3702 bool pract = prinfo & NVME_PRINFO_PRACT;
3703
3704 if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3705 mapped_size -= nvme_m2b(ns, nlb);
3706 }
3707 }
3708 }
3709
3710 trace_pci_nvme_write(nvme_cid(req), nvme_io_opc_str(rw->opcode),
3711 nvme_nsid(ns), nlb, mapped_size, slba);
3712
3713 if (!wrz) {
3714 status = nvme_check_mdts(n, mapped_size);
3715 if (status) {
3716 goto invalid;
3717 }
3718 }
3719
3720 status = nvme_check_bounds(ns, slba, nlb);
3721 if (status) {
3722 goto invalid;
3723 }
3724
3725 if (ns->params.zoned) {
3726 zone = nvme_get_zone_by_slba(ns, slba);
3727 assert(zone);
3728
3729 if (append) {
3730 bool piremap = !!(ctrl & NVME_RW_PIREMAP);
3731
3732 if (unlikely(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3733 return NVME_INVALID_ZONE_OP | NVME_DNR;
3734 }
3735
3736 if (unlikely(slba != zone->d.zslba)) {
3737 trace_pci_nvme_err_append_not_at_start(slba, zone->d.zslba);
3738 status = NVME_INVALID_FIELD;
3739 goto invalid;
3740 }
3741
3742 if (n->params.zasl &&
3743 data_size > (uint64_t)n->page_size << n->params.zasl) {
3744 trace_pci_nvme_err_zasl(data_size);
3745 return NVME_INVALID_FIELD | NVME_DNR;
3746 }
3747
3748 slba = zone->w_ptr;
3749 rw->slba = cpu_to_le64(slba);
3750 res->slba = cpu_to_le64(slba);
3751
3752 switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3753 case NVME_ID_NS_DPS_TYPE_1:
3754 if (!piremap) {
3755 return NVME_INVALID_PROT_INFO | NVME_DNR;
3756 }
3757
3758 /* fallthrough */
3759
3760 case NVME_ID_NS_DPS_TYPE_2:
3761 if (piremap) {
3762 uint32_t reftag = le32_to_cpu(rw->reftag);
3763 rw->reftag = cpu_to_le32(reftag + (slba - zone->d.zslba));
3764 }
3765
3766 break;
3767
3768 case NVME_ID_NS_DPS_TYPE_3:
3769 if (piremap) {
3770 return NVME_INVALID_PROT_INFO | NVME_DNR;
3771 }
3772
3773 break;
3774 }
3775 }
3776
3777 status = nvme_check_zone_write(ns, zone, slba, nlb);
3778 if (status) {
3779 goto invalid;
3780 }
3781
3782 status = nvme_zrm_auto(n, ns, zone);
3783 if (status) {
3784 goto invalid;
3785 }
3786
3787 if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3788 zone->w_ptr += nlb;
3789 }
3790 } else if (ns->endgrp && ns->endgrp->fdp.enabled) {
3791 nvme_do_write_fdp(n, req, slba, nlb);
3792 }
3793
3794 data_offset = nvme_l2b(ns, slba);
3795
3796 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3797 return nvme_dif_rw(n, req);
3798 }
3799
3800 if (!wrz) {
3801 status = nvme_map_data(n, nlb, req);
3802 if (status) {
3803 goto invalid;
3804 }
3805
3806 block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3807 BLOCK_ACCT_WRITE);
3808 nvme_blk_write(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3809 } else {
3810 req->aiocb = blk_aio_pwrite_zeroes(blk, data_offset, data_size,
3811 BDRV_REQ_MAY_UNMAP, nvme_rw_cb,
3812 req);
3813 }
3814
3815 return NVME_NO_COMPLETE;
3816
3817 invalid:
3818 block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
3819 return status | NVME_DNR;
3820 }
3821
nvme_write(NvmeCtrl * n,NvmeRequest * req)3822 static inline uint16_t nvme_write(NvmeCtrl *n, NvmeRequest *req)
3823 {
3824 return nvme_do_write(n, req, false, false);
3825 }
3826
nvme_write_zeroes(NvmeCtrl * n,NvmeRequest * req)3827 static inline uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req)
3828 {
3829 return nvme_do_write(n, req, false, true);
3830 }
3831
nvme_zone_append(NvmeCtrl * n,NvmeRequest * req)3832 static inline uint16_t nvme_zone_append(NvmeCtrl *n, NvmeRequest *req)
3833 {
3834 return nvme_do_write(n, req, true, false);
3835 }
3836
nvme_get_mgmt_zone_slba_idx(NvmeNamespace * ns,NvmeCmd * c,uint64_t * slba,uint32_t * zone_idx)3837 static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace *ns, NvmeCmd *c,
3838 uint64_t *slba, uint32_t *zone_idx)
3839 {
3840 uint32_t dw10 = le32_to_cpu(c->cdw10);
3841 uint32_t dw11 = le32_to_cpu(c->cdw11);
3842
3843 if (!ns->params.zoned) {
3844 trace_pci_nvme_err_invalid_opc(c->opcode);
3845 return NVME_INVALID_OPCODE | NVME_DNR;
3846 }
3847
3848 *slba = ((uint64_t)dw11) << 32 | dw10;
3849 if (unlikely(*slba >= ns->id_ns.nsze)) {
3850 trace_pci_nvme_err_invalid_lba_range(*slba, 0, ns->id_ns.nsze);
3851 *slba = 0;
3852 return NVME_LBA_RANGE | NVME_DNR;
3853 }
3854
3855 *zone_idx = nvme_zone_idx(ns, *slba);
3856 assert(*zone_idx < ns->num_zones);
3857
3858 return NVME_SUCCESS;
3859 }
3860
3861 typedef uint16_t (*op_handler_t)(NvmeNamespace *, NvmeZone *, NvmeZoneState,
3862 NvmeRequest *);
3863
3864 enum NvmeZoneProcessingMask {
3865 NVME_PROC_CURRENT_ZONE = 0,
3866 NVME_PROC_OPENED_ZONES = 1 << 0,
3867 NVME_PROC_CLOSED_ZONES = 1 << 1,
3868 NVME_PROC_READ_ONLY_ZONES = 1 << 2,
3869 NVME_PROC_FULL_ZONES = 1 << 3,
3870 };
3871
nvme_open_zone(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state,NvmeRequest * req)3872 static uint16_t nvme_open_zone(NvmeNamespace *ns, NvmeZone *zone,
3873 NvmeZoneState state, NvmeRequest *req)
3874 {
3875 NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
3876 int flags = 0;
3877
3878 if (cmd->zsflags & NVME_ZSFLAG_ZRWA_ALLOC) {
3879 uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3880
3881 if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3882 return NVME_INVALID_ZONE_OP | NVME_DNR;
3883 }
3884
3885 if (zone->w_ptr % ns->zns.zrwafg) {
3886 return NVME_NOZRWA | NVME_DNR;
3887 }
3888
3889 flags = NVME_ZRM_ZRWA;
3890 }
3891
3892 return nvme_zrm_open_flags(nvme_ctrl(req), ns, zone, flags);
3893 }
3894
nvme_close_zone(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state,NvmeRequest * req)3895 static uint16_t nvme_close_zone(NvmeNamespace *ns, NvmeZone *zone,
3896 NvmeZoneState state, NvmeRequest *req)
3897 {
3898 return nvme_zrm_close(ns, zone);
3899 }
3900
nvme_finish_zone(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state,NvmeRequest * req)3901 static uint16_t nvme_finish_zone(NvmeNamespace *ns, NvmeZone *zone,
3902 NvmeZoneState state, NvmeRequest *req)
3903 {
3904 return nvme_zrm_finish(ns, zone);
3905 }
3906
nvme_offline_zone(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state,NvmeRequest * req)3907 static uint16_t nvme_offline_zone(NvmeNamespace *ns, NvmeZone *zone,
3908 NvmeZoneState state, NvmeRequest *req)
3909 {
3910 switch (state) {
3911 case NVME_ZONE_STATE_READ_ONLY:
3912 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_OFFLINE);
3913 /* fall through */
3914 case NVME_ZONE_STATE_OFFLINE:
3915 return NVME_SUCCESS;
3916 default:
3917 return NVME_ZONE_INVAL_TRANSITION;
3918 }
3919 }
3920
nvme_set_zd_ext(NvmeNamespace * ns,NvmeZone * zone)3921 static uint16_t nvme_set_zd_ext(NvmeNamespace *ns, NvmeZone *zone)
3922 {
3923 uint16_t status;
3924 uint8_t state = nvme_get_zone_state(zone);
3925
3926 if (state == NVME_ZONE_STATE_EMPTY) {
3927 status = nvme_aor_check(ns, 1, 0);
3928 if (status) {
3929 return status;
3930 }
3931 nvme_aor_inc_active(ns);
3932 zone->d.za |= NVME_ZA_ZD_EXT_VALID;
3933 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
3934 return NVME_SUCCESS;
3935 }
3936
3937 return NVME_ZONE_INVAL_TRANSITION;
3938 }
3939
nvme_bulk_proc_zone(NvmeNamespace * ns,NvmeZone * zone,enum NvmeZoneProcessingMask proc_mask,op_handler_t op_hndlr,NvmeRequest * req)3940 static uint16_t nvme_bulk_proc_zone(NvmeNamespace *ns, NvmeZone *zone,
3941 enum NvmeZoneProcessingMask proc_mask,
3942 op_handler_t op_hndlr, NvmeRequest *req)
3943 {
3944 uint16_t status = NVME_SUCCESS;
3945 NvmeZoneState zs = nvme_get_zone_state(zone);
3946 bool proc_zone;
3947
3948 switch (zs) {
3949 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
3950 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
3951 proc_zone = proc_mask & NVME_PROC_OPENED_ZONES;
3952 break;
3953 case NVME_ZONE_STATE_CLOSED:
3954 proc_zone = proc_mask & NVME_PROC_CLOSED_ZONES;
3955 break;
3956 case NVME_ZONE_STATE_READ_ONLY:
3957 proc_zone = proc_mask & NVME_PROC_READ_ONLY_ZONES;
3958 break;
3959 case NVME_ZONE_STATE_FULL:
3960 proc_zone = proc_mask & NVME_PROC_FULL_ZONES;
3961 break;
3962 default:
3963 proc_zone = false;
3964 }
3965
3966 if (proc_zone) {
3967 status = op_hndlr(ns, zone, zs, req);
3968 }
3969
3970 return status;
3971 }
3972
nvme_do_zone_op(NvmeNamespace * ns,NvmeZone * zone,enum NvmeZoneProcessingMask proc_mask,op_handler_t op_hndlr,NvmeRequest * req)3973 static uint16_t nvme_do_zone_op(NvmeNamespace *ns, NvmeZone *zone,
3974 enum NvmeZoneProcessingMask proc_mask,
3975 op_handler_t op_hndlr, NvmeRequest *req)
3976 {
3977 NvmeZone *next;
3978 uint16_t status = NVME_SUCCESS;
3979 int i;
3980
3981 if (!proc_mask) {
3982 status = op_hndlr(ns, zone, nvme_get_zone_state(zone), req);
3983 } else {
3984 if (proc_mask & NVME_PROC_CLOSED_ZONES) {
3985 QTAILQ_FOREACH_SAFE(zone, &ns->closed_zones, entry, next) {
3986 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3987 req);
3988 if (status && status != NVME_NO_COMPLETE) {
3989 goto out;
3990 }
3991 }
3992 }
3993 if (proc_mask & NVME_PROC_OPENED_ZONES) {
3994 QTAILQ_FOREACH_SAFE(zone, &ns->imp_open_zones, entry, next) {
3995 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
3996 req);
3997 if (status && status != NVME_NO_COMPLETE) {
3998 goto out;
3999 }
4000 }
4001
4002 QTAILQ_FOREACH_SAFE(zone, &ns->exp_open_zones, entry, next) {
4003 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4004 req);
4005 if (status && status != NVME_NO_COMPLETE) {
4006 goto out;
4007 }
4008 }
4009 }
4010 if (proc_mask & NVME_PROC_FULL_ZONES) {
4011 QTAILQ_FOREACH_SAFE(zone, &ns->full_zones, entry, next) {
4012 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4013 req);
4014 if (status && status != NVME_NO_COMPLETE) {
4015 goto out;
4016 }
4017 }
4018 }
4019
4020 if (proc_mask & NVME_PROC_READ_ONLY_ZONES) {
4021 for (i = 0; i < ns->num_zones; i++, zone++) {
4022 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4023 req);
4024 if (status && status != NVME_NO_COMPLETE) {
4025 goto out;
4026 }
4027 }
4028 }
4029 }
4030
4031 out:
4032 return status;
4033 }
4034
4035 typedef struct NvmeZoneResetAIOCB {
4036 BlockAIOCB common;
4037 BlockAIOCB *aiocb;
4038 NvmeRequest *req;
4039 int ret;
4040
4041 bool all;
4042 int idx;
4043 NvmeZone *zone;
4044 } NvmeZoneResetAIOCB;
4045
nvme_zone_reset_cancel(BlockAIOCB * aiocb)4046 static void nvme_zone_reset_cancel(BlockAIOCB *aiocb)
4047 {
4048 NvmeZoneResetAIOCB *iocb = container_of(aiocb, NvmeZoneResetAIOCB, common);
4049 NvmeRequest *req = iocb->req;
4050 NvmeNamespace *ns = req->ns;
4051
4052 iocb->idx = ns->num_zones;
4053
4054 iocb->ret = -ECANCELED;
4055
4056 if (iocb->aiocb) {
4057 blk_aio_cancel_async(iocb->aiocb);
4058 iocb->aiocb = NULL;
4059 }
4060 }
4061
4062 static const AIOCBInfo nvme_zone_reset_aiocb_info = {
4063 .aiocb_size = sizeof(NvmeZoneResetAIOCB),
4064 .cancel_async = nvme_zone_reset_cancel,
4065 };
4066
4067 static void nvme_zone_reset_cb(void *opaque, int ret);
4068
nvme_zone_reset_epilogue_cb(void * opaque,int ret)4069 static void nvme_zone_reset_epilogue_cb(void *opaque, int ret)
4070 {
4071 NvmeZoneResetAIOCB *iocb = opaque;
4072 NvmeRequest *req = iocb->req;
4073 NvmeNamespace *ns = req->ns;
4074 int64_t moff;
4075 int count;
4076
4077 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
4078 goto out;
4079 }
4080
4081 moff = nvme_moff(ns, iocb->zone->d.zslba);
4082 count = nvme_m2b(ns, ns->zone_size);
4083
4084 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, moff, count,
4085 BDRV_REQ_MAY_UNMAP,
4086 nvme_zone_reset_cb, iocb);
4087 return;
4088
4089 out:
4090 nvme_zone_reset_cb(iocb, ret);
4091 }
4092
nvme_zone_reset_cb(void * opaque,int ret)4093 static void nvme_zone_reset_cb(void *opaque, int ret)
4094 {
4095 NvmeZoneResetAIOCB *iocb = opaque;
4096 NvmeRequest *req = iocb->req;
4097 NvmeNamespace *ns = req->ns;
4098
4099 if (iocb->ret < 0) {
4100 goto done;
4101 } else if (ret < 0) {
4102 iocb->ret = ret;
4103 goto done;
4104 }
4105
4106 if (iocb->zone) {
4107 nvme_zrm_reset(ns, iocb->zone);
4108
4109 if (!iocb->all) {
4110 goto done;
4111 }
4112 }
4113
4114 while (iocb->idx < ns->num_zones) {
4115 NvmeZone *zone = &ns->zone_array[iocb->idx++];
4116
4117 switch (nvme_get_zone_state(zone)) {
4118 case NVME_ZONE_STATE_EMPTY:
4119 if (!iocb->all) {
4120 goto done;
4121 }
4122
4123 continue;
4124
4125 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
4126 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
4127 case NVME_ZONE_STATE_CLOSED:
4128 case NVME_ZONE_STATE_FULL:
4129 iocb->zone = zone;
4130 break;
4131
4132 default:
4133 continue;
4134 }
4135
4136 trace_pci_nvme_zns_zone_reset(zone->d.zslba);
4137
4138 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk,
4139 nvme_l2b(ns, zone->d.zslba),
4140 nvme_l2b(ns, ns->zone_size),
4141 BDRV_REQ_MAY_UNMAP,
4142 nvme_zone_reset_epilogue_cb,
4143 iocb);
4144 return;
4145 }
4146
4147 done:
4148 iocb->aiocb = NULL;
4149
4150 iocb->common.cb(iocb->common.opaque, iocb->ret);
4151 qemu_aio_unref(iocb);
4152 }
4153
nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl * n,NvmeZone * zone,uint64_t elba,NvmeRequest * req)4154 static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl *n, NvmeZone *zone,
4155 uint64_t elba, NvmeRequest *req)
4156 {
4157 NvmeNamespace *ns = req->ns;
4158 uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
4159 uint64_t wp = zone->d.wp;
4160 uint32_t nlb = elba - wp + 1;
4161 uint16_t status;
4162
4163
4164 if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
4165 return NVME_INVALID_ZONE_OP | NVME_DNR;
4166 }
4167
4168 if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
4169 return NVME_INVALID_FIELD | NVME_DNR;
4170 }
4171
4172 if (elba < wp || elba > wp + ns->zns.zrwas) {
4173 return NVME_ZONE_BOUNDARY_ERROR | NVME_DNR;
4174 }
4175
4176 if (nlb % ns->zns.zrwafg) {
4177 return NVME_INVALID_FIELD | NVME_DNR;
4178 }
4179
4180 status = nvme_zrm_auto(n, ns, zone);
4181 if (status) {
4182 return status;
4183 }
4184
4185 zone->w_ptr += nlb;
4186
4187 nvme_advance_zone_wp(ns, zone, nlb);
4188
4189 return NVME_SUCCESS;
4190 }
4191
nvme_zone_mgmt_send(NvmeCtrl * n,NvmeRequest * req)4192 static uint16_t nvme_zone_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4193 {
4194 NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
4195 NvmeNamespace *ns = req->ns;
4196 NvmeZone *zone;
4197 NvmeZoneResetAIOCB *iocb;
4198 uint8_t *zd_ext;
4199 uint64_t slba = 0;
4200 uint32_t zone_idx = 0;
4201 uint16_t status;
4202 uint8_t action = cmd->zsa;
4203 bool all;
4204 enum NvmeZoneProcessingMask proc_mask = NVME_PROC_CURRENT_ZONE;
4205
4206 all = cmd->zsflags & NVME_ZSFLAG_SELECT_ALL;
4207
4208 req->status = NVME_SUCCESS;
4209
4210 if (!all) {
4211 status = nvme_get_mgmt_zone_slba_idx(ns, &req->cmd, &slba, &zone_idx);
4212 if (status) {
4213 return status;
4214 }
4215 }
4216
4217 zone = &ns->zone_array[zone_idx];
4218 if (slba != zone->d.zslba && action != NVME_ZONE_ACTION_ZRWA_FLUSH) {
4219 trace_pci_nvme_err_unaligned_zone_cmd(action, slba, zone->d.zslba);
4220 return NVME_INVALID_FIELD | NVME_DNR;
4221 }
4222
4223 switch (action) {
4224
4225 case NVME_ZONE_ACTION_OPEN:
4226 if (all) {
4227 proc_mask = NVME_PROC_CLOSED_ZONES;
4228 }
4229 trace_pci_nvme_open_zone(slba, zone_idx, all);
4230 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_open_zone, req);
4231 break;
4232
4233 case NVME_ZONE_ACTION_CLOSE:
4234 if (all) {
4235 proc_mask = NVME_PROC_OPENED_ZONES;
4236 }
4237 trace_pci_nvme_close_zone(slba, zone_idx, all);
4238 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_close_zone, req);
4239 break;
4240
4241 case NVME_ZONE_ACTION_FINISH:
4242 if (all) {
4243 proc_mask = NVME_PROC_OPENED_ZONES | NVME_PROC_CLOSED_ZONES;
4244 }
4245 trace_pci_nvme_finish_zone(slba, zone_idx, all);
4246 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_finish_zone, req);
4247 break;
4248
4249 case NVME_ZONE_ACTION_RESET:
4250 trace_pci_nvme_reset_zone(slba, zone_idx, all);
4251
4252 iocb = blk_aio_get(&nvme_zone_reset_aiocb_info, ns->blkconf.blk,
4253 nvme_misc_cb, req);
4254
4255 iocb->req = req;
4256 iocb->ret = 0;
4257 iocb->all = all;
4258 iocb->idx = zone_idx;
4259 iocb->zone = NULL;
4260
4261 req->aiocb = &iocb->common;
4262 nvme_zone_reset_cb(iocb, 0);
4263
4264 return NVME_NO_COMPLETE;
4265
4266 case NVME_ZONE_ACTION_OFFLINE:
4267 if (all) {
4268 proc_mask = NVME_PROC_READ_ONLY_ZONES;
4269 }
4270 trace_pci_nvme_offline_zone(slba, zone_idx, all);
4271 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_offline_zone, req);
4272 break;
4273
4274 case NVME_ZONE_ACTION_SET_ZD_EXT:
4275 trace_pci_nvme_set_descriptor_extension(slba, zone_idx);
4276 if (all || !ns->params.zd_extension_size) {
4277 return NVME_INVALID_FIELD | NVME_DNR;
4278 }
4279 zd_ext = nvme_get_zd_extension(ns, zone_idx);
4280 status = nvme_h2c(n, zd_ext, ns->params.zd_extension_size, req);
4281 if (status) {
4282 trace_pci_nvme_err_zd_extension_map_error(zone_idx);
4283 return status;
4284 }
4285
4286 status = nvme_set_zd_ext(ns, zone);
4287 if (status == NVME_SUCCESS) {
4288 trace_pci_nvme_zd_extension_set(zone_idx);
4289 return status;
4290 }
4291 break;
4292
4293 case NVME_ZONE_ACTION_ZRWA_FLUSH:
4294 if (all) {
4295 return NVME_INVALID_FIELD | NVME_DNR;
4296 }
4297
4298 return nvme_zone_mgmt_send_zrwa_flush(n, zone, slba, req);
4299
4300 default:
4301 trace_pci_nvme_err_invalid_mgmt_action(action);
4302 status = NVME_INVALID_FIELD;
4303 }
4304
4305 if (status == NVME_ZONE_INVAL_TRANSITION) {
4306 trace_pci_nvme_err_invalid_zone_state_transition(action, slba,
4307 zone->d.za);
4308 }
4309 if (status) {
4310 status |= NVME_DNR;
4311 }
4312
4313 return status;
4314 }
4315
nvme_zone_matches_filter(uint32_t zafs,NvmeZone * zl)4316 static bool nvme_zone_matches_filter(uint32_t zafs, NvmeZone *zl)
4317 {
4318 NvmeZoneState zs = nvme_get_zone_state(zl);
4319
4320 switch (zafs) {
4321 case NVME_ZONE_REPORT_ALL:
4322 return true;
4323 case NVME_ZONE_REPORT_EMPTY:
4324 return zs == NVME_ZONE_STATE_EMPTY;
4325 case NVME_ZONE_REPORT_IMPLICITLY_OPEN:
4326 return zs == NVME_ZONE_STATE_IMPLICITLY_OPEN;
4327 case NVME_ZONE_REPORT_EXPLICITLY_OPEN:
4328 return zs == NVME_ZONE_STATE_EXPLICITLY_OPEN;
4329 case NVME_ZONE_REPORT_CLOSED:
4330 return zs == NVME_ZONE_STATE_CLOSED;
4331 case NVME_ZONE_REPORT_FULL:
4332 return zs == NVME_ZONE_STATE_FULL;
4333 case NVME_ZONE_REPORT_READ_ONLY:
4334 return zs == NVME_ZONE_STATE_READ_ONLY;
4335 case NVME_ZONE_REPORT_OFFLINE:
4336 return zs == NVME_ZONE_STATE_OFFLINE;
4337 default:
4338 return false;
4339 }
4340 }
4341
nvme_zone_mgmt_recv(NvmeCtrl * n,NvmeRequest * req)4342 static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4343 {
4344 NvmeCmd *cmd = &req->cmd;
4345 NvmeNamespace *ns = req->ns;
4346 /* cdw12 is zero-based number of dwords to return. Convert to bytes */
4347 uint32_t data_size = (le32_to_cpu(cmd->cdw12) + 1) << 2;
4348 uint32_t dw13 = le32_to_cpu(cmd->cdw13);
4349 uint32_t zone_idx, zra, zrasf, partial;
4350 uint64_t max_zones, nr_zones = 0;
4351 uint16_t status;
4352 uint64_t slba;
4353 NvmeZoneDescr *z;
4354 NvmeZone *zone;
4355 NvmeZoneReportHeader *header;
4356 void *buf, *buf_p;
4357 size_t zone_entry_sz;
4358 int i;
4359
4360 req->status = NVME_SUCCESS;
4361
4362 status = nvme_get_mgmt_zone_slba_idx(ns, cmd, &slba, &zone_idx);
4363 if (status) {
4364 return status;
4365 }
4366
4367 zra = dw13 & 0xff;
4368 if (zra != NVME_ZONE_REPORT && zra != NVME_ZONE_REPORT_EXTENDED) {
4369 return NVME_INVALID_FIELD | NVME_DNR;
4370 }
4371 if (zra == NVME_ZONE_REPORT_EXTENDED && !ns->params.zd_extension_size) {
4372 return NVME_INVALID_FIELD | NVME_DNR;
4373 }
4374
4375 zrasf = (dw13 >> 8) & 0xff;
4376 if (zrasf > NVME_ZONE_REPORT_OFFLINE) {
4377 return NVME_INVALID_FIELD | NVME_DNR;
4378 }
4379
4380 if (data_size < sizeof(NvmeZoneReportHeader)) {
4381 return NVME_INVALID_FIELD | NVME_DNR;
4382 }
4383
4384 status = nvme_check_mdts(n, data_size);
4385 if (status) {
4386 return status;
4387 }
4388
4389 partial = (dw13 >> 16) & 0x01;
4390
4391 zone_entry_sz = sizeof(NvmeZoneDescr);
4392 if (zra == NVME_ZONE_REPORT_EXTENDED) {
4393 zone_entry_sz += ns->params.zd_extension_size;
4394 }
4395
4396 max_zones = (data_size - sizeof(NvmeZoneReportHeader)) / zone_entry_sz;
4397 buf = g_malloc0(data_size);
4398
4399 zone = &ns->zone_array[zone_idx];
4400 for (i = zone_idx; i < ns->num_zones; i++) {
4401 if (partial && nr_zones >= max_zones) {
4402 break;
4403 }
4404 if (nvme_zone_matches_filter(zrasf, zone++)) {
4405 nr_zones++;
4406 }
4407 }
4408 header = buf;
4409 header->nr_zones = cpu_to_le64(nr_zones);
4410
4411 buf_p = buf + sizeof(NvmeZoneReportHeader);
4412 for (; zone_idx < ns->num_zones && max_zones > 0; zone_idx++) {
4413 zone = &ns->zone_array[zone_idx];
4414 if (nvme_zone_matches_filter(zrasf, zone)) {
4415 z = buf_p;
4416 buf_p += sizeof(NvmeZoneDescr);
4417
4418 z->zt = zone->d.zt;
4419 z->zs = zone->d.zs;
4420 z->zcap = cpu_to_le64(zone->d.zcap);
4421 z->zslba = cpu_to_le64(zone->d.zslba);
4422 z->za = zone->d.za;
4423
4424 if (nvme_wp_is_valid(zone)) {
4425 z->wp = cpu_to_le64(zone->d.wp);
4426 } else {
4427 z->wp = cpu_to_le64(~0ULL);
4428 }
4429
4430 if (zra == NVME_ZONE_REPORT_EXTENDED) {
4431 if (zone->d.za & NVME_ZA_ZD_EXT_VALID) {
4432 memcpy(buf_p, nvme_get_zd_extension(ns, zone_idx),
4433 ns->params.zd_extension_size);
4434 }
4435 buf_p += ns->params.zd_extension_size;
4436 }
4437
4438 max_zones--;
4439 }
4440 }
4441
4442 status = nvme_c2h(n, (uint8_t *)buf, data_size, req);
4443
4444 g_free(buf);
4445
4446 return status;
4447 }
4448
nvme_io_mgmt_recv_ruhs(NvmeCtrl * n,NvmeRequest * req,size_t len)4449 static uint16_t nvme_io_mgmt_recv_ruhs(NvmeCtrl *n, NvmeRequest *req,
4450 size_t len)
4451 {
4452 NvmeNamespace *ns = req->ns;
4453 NvmeEnduranceGroup *endgrp;
4454 NvmeRuhStatus *hdr;
4455 NvmeRuhStatusDescr *ruhsd;
4456 unsigned int nruhsd;
4457 uint16_t rg, ph, *ruhid;
4458 size_t trans_len;
4459 g_autofree uint8_t *buf = NULL;
4460
4461 if (!n->subsys) {
4462 return NVME_INVALID_FIELD | NVME_DNR;
4463 }
4464
4465 if (ns->params.nsid == 0 || ns->params.nsid == 0xffffffff) {
4466 return NVME_INVALID_NSID | NVME_DNR;
4467 }
4468
4469 if (!n->subsys->endgrp.fdp.enabled) {
4470 return NVME_FDP_DISABLED | NVME_DNR;
4471 }
4472
4473 endgrp = ns->endgrp;
4474
4475 nruhsd = ns->fdp.nphs * endgrp->fdp.nrg;
4476 trans_len = sizeof(NvmeRuhStatus) + nruhsd * sizeof(NvmeRuhStatusDescr);
4477 buf = g_malloc0(trans_len);
4478
4479 trans_len = MIN(trans_len, len);
4480
4481 hdr = (NvmeRuhStatus *)buf;
4482 ruhsd = (NvmeRuhStatusDescr *)(buf + sizeof(NvmeRuhStatus));
4483
4484 hdr->nruhsd = cpu_to_le16(nruhsd);
4485
4486 ruhid = ns->fdp.phs;
4487
4488 for (ph = 0; ph < ns->fdp.nphs; ph++, ruhid++) {
4489 NvmeRuHandle *ruh = &endgrp->fdp.ruhs[*ruhid];
4490
4491 for (rg = 0; rg < endgrp->fdp.nrg; rg++, ruhsd++) {
4492 uint16_t pid = nvme_make_pid(ns, rg, ph);
4493
4494 ruhsd->pid = cpu_to_le16(pid);
4495 ruhsd->ruhid = *ruhid;
4496 ruhsd->earutr = 0;
4497 ruhsd->ruamw = cpu_to_le64(ruh->rus[rg].ruamw);
4498 }
4499 }
4500
4501 return nvme_c2h(n, buf, trans_len, req);
4502 }
4503
nvme_io_mgmt_recv(NvmeCtrl * n,NvmeRequest * req)4504 static uint16_t nvme_io_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4505 {
4506 NvmeCmd *cmd = &req->cmd;
4507 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4508 uint32_t numd = le32_to_cpu(cmd->cdw11);
4509 uint8_t mo = (cdw10 & 0xff);
4510 size_t len = (numd + 1) << 2;
4511
4512 switch (mo) {
4513 case NVME_IOMR_MO_NOP:
4514 return 0;
4515 case NVME_IOMR_MO_RUH_STATUS:
4516 return nvme_io_mgmt_recv_ruhs(n, req, len);
4517 default:
4518 return NVME_INVALID_FIELD | NVME_DNR;
4519 };
4520 }
4521
nvme_io_mgmt_send_ruh_update(NvmeCtrl * n,NvmeRequest * req)4522 static uint16_t nvme_io_mgmt_send_ruh_update(NvmeCtrl *n, NvmeRequest *req)
4523 {
4524 NvmeCmd *cmd = &req->cmd;
4525 NvmeNamespace *ns = req->ns;
4526 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4527 uint16_t ret = NVME_SUCCESS;
4528 uint32_t npid = (cdw10 >> 16) + 1;
4529 unsigned int i = 0;
4530 g_autofree uint16_t *pids = NULL;
4531 uint32_t maxnpid;
4532
4533 if (!ns->endgrp || !ns->endgrp->fdp.enabled) {
4534 return NVME_FDP_DISABLED | NVME_DNR;
4535 }
4536
4537 maxnpid = n->subsys->endgrp.fdp.nrg * n->subsys->endgrp.fdp.nruh;
4538
4539 if (unlikely(npid >= MIN(NVME_FDP_MAXPIDS, maxnpid))) {
4540 return NVME_INVALID_FIELD | NVME_DNR;
4541 }
4542
4543 pids = g_new(uint16_t, npid);
4544
4545 ret = nvme_h2c(n, pids, npid * sizeof(uint16_t), req);
4546 if (ret) {
4547 return ret;
4548 }
4549
4550 for (; i < npid; i++) {
4551 if (!nvme_update_ruh(n, ns, pids[i])) {
4552 return NVME_INVALID_FIELD | NVME_DNR;
4553 }
4554 }
4555
4556 return ret;
4557 }
4558
nvme_io_mgmt_send(NvmeCtrl * n,NvmeRequest * req)4559 static uint16_t nvme_io_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4560 {
4561 NvmeCmd *cmd = &req->cmd;
4562 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4563 uint8_t mo = (cdw10 & 0xff);
4564
4565 switch (mo) {
4566 case NVME_IOMS_MO_NOP:
4567 return 0;
4568 case NVME_IOMS_MO_RUH_UPDATE:
4569 return nvme_io_mgmt_send_ruh_update(n, req);
4570 default:
4571 return NVME_INVALID_FIELD | NVME_DNR;
4572 };
4573 }
4574
nvme_io_cmd(NvmeCtrl * n,NvmeRequest * req)4575 static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req)
4576 {
4577 NvmeNamespace *ns;
4578 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4579
4580 trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req),
4581 req->cmd.opcode, nvme_io_opc_str(req->cmd.opcode));
4582
4583 /*
4584 * In the base NVM command set, Flush may apply to all namespaces
4585 * (indicated by NSID being set to FFFFFFFFh). But if that feature is used
4586 * along with TP 4056 (Namespace Types), it may be pretty screwed up.
4587 *
4588 * If NSID is indeed set to FFFFFFFFh, we simply cannot associate the
4589 * opcode with a specific command since we cannot determine a unique I/O
4590 * command set. Opcode 0h could have any other meaning than something
4591 * equivalent to flushing and say it DOES have completely different
4592 * semantics in some other command set - does an NSID of FFFFFFFFh then
4593 * mean "for all namespaces, apply whatever command set specific command
4594 * that uses the 0h opcode?" Or does it mean "for all namespaces, apply
4595 * whatever command that uses the 0h opcode if, and only if, it allows NSID
4596 * to be FFFFFFFFh"?
4597 *
4598 * Anyway (and luckily), for now, we do not care about this since the
4599 * device only supports namespace types that includes the NVM Flush command
4600 * (NVM and Zoned), so always do an NVM Flush.
4601 */
4602
4603 if (req->cmd.opcode == NVME_CMD_FLUSH) {
4604 return nvme_flush(n, req);
4605 }
4606
4607 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
4608 return NVME_INVALID_NSID | NVME_DNR;
4609 }
4610
4611 ns = nvme_ns(n, nsid);
4612 if (unlikely(!ns)) {
4613 return NVME_INVALID_FIELD | NVME_DNR;
4614 }
4615
4616 if (!(ns->iocs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
4617 trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
4618 return NVME_INVALID_OPCODE | NVME_DNR;
4619 }
4620
4621 if (ns->status) {
4622 return ns->status;
4623 }
4624
4625 if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
4626 return NVME_INVALID_FIELD;
4627 }
4628
4629 req->ns = ns;
4630
4631 switch (req->cmd.opcode) {
4632 case NVME_CMD_WRITE_ZEROES:
4633 return nvme_write_zeroes(n, req);
4634 case NVME_CMD_ZONE_APPEND:
4635 return nvme_zone_append(n, req);
4636 case NVME_CMD_WRITE:
4637 return nvme_write(n, req);
4638 case NVME_CMD_READ:
4639 return nvme_read(n, req);
4640 case NVME_CMD_COMPARE:
4641 return nvme_compare(n, req);
4642 case NVME_CMD_DSM:
4643 return nvme_dsm(n, req);
4644 case NVME_CMD_VERIFY:
4645 return nvme_verify(n, req);
4646 case NVME_CMD_COPY:
4647 return nvme_copy(n, req);
4648 case NVME_CMD_ZONE_MGMT_SEND:
4649 return nvme_zone_mgmt_send(n, req);
4650 case NVME_CMD_ZONE_MGMT_RECV:
4651 return nvme_zone_mgmt_recv(n, req);
4652 case NVME_CMD_IO_MGMT_RECV:
4653 return nvme_io_mgmt_recv(n, req);
4654 case NVME_CMD_IO_MGMT_SEND:
4655 return nvme_io_mgmt_send(n, req);
4656 default:
4657 assert(false);
4658 }
4659
4660 return NVME_INVALID_OPCODE | NVME_DNR;
4661 }
4662
nvme_cq_notifier(EventNotifier * e)4663 static void nvme_cq_notifier(EventNotifier *e)
4664 {
4665 NvmeCQueue *cq = container_of(e, NvmeCQueue, notifier);
4666 NvmeCtrl *n = cq->ctrl;
4667
4668 if (!event_notifier_test_and_clear(e)) {
4669 return;
4670 }
4671
4672 nvme_update_cq_head(cq);
4673
4674 if (cq->tail == cq->head) {
4675 if (cq->irq_enabled) {
4676 n->cq_pending--;
4677 }
4678
4679 nvme_irq_deassert(n, cq);
4680 }
4681
4682 qemu_bh_schedule(cq->bh);
4683 }
4684
nvme_init_cq_ioeventfd(NvmeCQueue * cq)4685 static int nvme_init_cq_ioeventfd(NvmeCQueue *cq)
4686 {
4687 NvmeCtrl *n = cq->ctrl;
4688 uint16_t offset = (cq->cqid << 3) + (1 << 2);
4689 int ret;
4690
4691 ret = event_notifier_init(&cq->notifier, 0);
4692 if (ret < 0) {
4693 return ret;
4694 }
4695
4696 event_notifier_set_handler(&cq->notifier, nvme_cq_notifier);
4697 memory_region_add_eventfd(&n->iomem,
4698 0x1000 + offset, 4, false, 0, &cq->notifier);
4699
4700 return 0;
4701 }
4702
nvme_sq_notifier(EventNotifier * e)4703 static void nvme_sq_notifier(EventNotifier *e)
4704 {
4705 NvmeSQueue *sq = container_of(e, NvmeSQueue, notifier);
4706
4707 if (!event_notifier_test_and_clear(e)) {
4708 return;
4709 }
4710
4711 nvme_process_sq(sq);
4712 }
4713
nvme_init_sq_ioeventfd(NvmeSQueue * sq)4714 static int nvme_init_sq_ioeventfd(NvmeSQueue *sq)
4715 {
4716 NvmeCtrl *n = sq->ctrl;
4717 uint16_t offset = sq->sqid << 3;
4718 int ret;
4719
4720 ret = event_notifier_init(&sq->notifier, 0);
4721 if (ret < 0) {
4722 return ret;
4723 }
4724
4725 event_notifier_set_handler(&sq->notifier, nvme_sq_notifier);
4726 memory_region_add_eventfd(&n->iomem,
4727 0x1000 + offset, 4, false, 0, &sq->notifier);
4728
4729 return 0;
4730 }
4731
nvme_free_sq(NvmeSQueue * sq,NvmeCtrl * n)4732 static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
4733 {
4734 uint16_t offset = sq->sqid << 3;
4735
4736 n->sq[sq->sqid] = NULL;
4737 qemu_bh_delete(sq->bh);
4738 if (sq->ioeventfd_enabled) {
4739 memory_region_del_eventfd(&n->iomem,
4740 0x1000 + offset, 4, false, 0, &sq->notifier);
4741 event_notifier_set_handler(&sq->notifier, NULL);
4742 event_notifier_cleanup(&sq->notifier);
4743 }
4744 g_free(sq->io_req);
4745 if (sq->sqid) {
4746 g_free(sq);
4747 }
4748 }
4749
nvme_del_sq(NvmeCtrl * n,NvmeRequest * req)4750 static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req)
4751 {
4752 NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
4753 NvmeRequest *r, *next;
4754 NvmeSQueue *sq;
4755 NvmeCQueue *cq;
4756 uint16_t qid = le16_to_cpu(c->qid);
4757
4758 if (unlikely(!qid || nvme_check_sqid(n, qid))) {
4759 trace_pci_nvme_err_invalid_del_sq(qid);
4760 return NVME_INVALID_QID | NVME_DNR;
4761 }
4762
4763 trace_pci_nvme_del_sq(qid);
4764
4765 sq = n->sq[qid];
4766 while (!QTAILQ_EMPTY(&sq->out_req_list)) {
4767 r = QTAILQ_FIRST(&sq->out_req_list);
4768 assert(r->aiocb);
4769 blk_aio_cancel(r->aiocb);
4770 }
4771
4772 assert(QTAILQ_EMPTY(&sq->out_req_list));
4773
4774 if (!nvme_check_cqid(n, sq->cqid)) {
4775 cq = n->cq[sq->cqid];
4776 QTAILQ_REMOVE(&cq->sq_list, sq, entry);
4777
4778 nvme_post_cqes(cq);
4779 QTAILQ_FOREACH_SAFE(r, &cq->req_list, entry, next) {
4780 if (r->sq == sq) {
4781 QTAILQ_REMOVE(&cq->req_list, r, entry);
4782 QTAILQ_INSERT_TAIL(&sq->req_list, r, entry);
4783 }
4784 }
4785 }
4786
4787 nvme_free_sq(sq, n);
4788 return NVME_SUCCESS;
4789 }
4790
nvme_init_sq(NvmeSQueue * sq,NvmeCtrl * n,uint64_t dma_addr,uint16_t sqid,uint16_t cqid,uint16_t size)4791 static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
4792 uint16_t sqid, uint16_t cqid, uint16_t size)
4793 {
4794 int i;
4795 NvmeCQueue *cq;
4796
4797 sq->ctrl = n;
4798 sq->dma_addr = dma_addr;
4799 sq->sqid = sqid;
4800 sq->size = size;
4801 sq->cqid = cqid;
4802 sq->head = sq->tail = 0;
4803 sq->io_req = g_new0(NvmeRequest, sq->size);
4804
4805 QTAILQ_INIT(&sq->req_list);
4806 QTAILQ_INIT(&sq->out_req_list);
4807 for (i = 0; i < sq->size; i++) {
4808 sq->io_req[i].sq = sq;
4809 QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
4810 }
4811
4812 sq->bh = qemu_bh_new_guarded(nvme_process_sq, sq,
4813 &DEVICE(sq->ctrl)->mem_reentrancy_guard);
4814
4815 if (n->dbbuf_enabled) {
4816 sq->db_addr = n->dbbuf_dbs + (sqid << 3);
4817 sq->ei_addr = n->dbbuf_eis + (sqid << 3);
4818
4819 if (n->params.ioeventfd && sq->sqid != 0) {
4820 if (!nvme_init_sq_ioeventfd(sq)) {
4821 sq->ioeventfd_enabled = true;
4822 }
4823 }
4824 }
4825
4826 assert(n->cq[cqid]);
4827 cq = n->cq[cqid];
4828 QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
4829 n->sq[sqid] = sq;
4830 }
4831
nvme_create_sq(NvmeCtrl * n,NvmeRequest * req)4832 static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req)
4833 {
4834 NvmeSQueue *sq;
4835 NvmeCreateSq *c = (NvmeCreateSq *)&req->cmd;
4836
4837 uint16_t cqid = le16_to_cpu(c->cqid);
4838 uint16_t sqid = le16_to_cpu(c->sqid);
4839 uint16_t qsize = le16_to_cpu(c->qsize);
4840 uint16_t qflags = le16_to_cpu(c->sq_flags);
4841 uint64_t prp1 = le64_to_cpu(c->prp1);
4842
4843 trace_pci_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
4844
4845 if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
4846 trace_pci_nvme_err_invalid_create_sq_cqid(cqid);
4847 return NVME_INVALID_CQID | NVME_DNR;
4848 }
4849 if (unlikely(!sqid || sqid > n->conf_ioqpairs || n->sq[sqid] != NULL)) {
4850 trace_pci_nvme_err_invalid_create_sq_sqid(sqid);
4851 return NVME_INVALID_QID | NVME_DNR;
4852 }
4853 if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
4854 trace_pci_nvme_err_invalid_create_sq_size(qsize);
4855 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
4856 }
4857 if (unlikely(prp1 & (n->page_size - 1))) {
4858 trace_pci_nvme_err_invalid_create_sq_addr(prp1);
4859 return NVME_INVALID_PRP_OFFSET | NVME_DNR;
4860 }
4861 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
4862 trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
4863 return NVME_INVALID_FIELD | NVME_DNR;
4864 }
4865 sq = g_malloc0(sizeof(*sq));
4866 nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
4867 return NVME_SUCCESS;
4868 }
4869
4870 struct nvme_stats {
4871 uint64_t units_read;
4872 uint64_t units_written;
4873 uint64_t read_commands;
4874 uint64_t write_commands;
4875 };
4876
nvme_set_blk_stats(NvmeNamespace * ns,struct nvme_stats * stats)4877 static void nvme_set_blk_stats(NvmeNamespace *ns, struct nvme_stats *stats)
4878 {
4879 BlockAcctStats *s = blk_get_stats(ns->blkconf.blk);
4880
4881 stats->units_read += s->nr_bytes[BLOCK_ACCT_READ];
4882 stats->units_written += s->nr_bytes[BLOCK_ACCT_WRITE];
4883 stats->read_commands += s->nr_ops[BLOCK_ACCT_READ];
4884 stats->write_commands += s->nr_ops[BLOCK_ACCT_WRITE];
4885 }
4886
nvme_smart_info(NvmeCtrl * n,uint8_t rae,uint32_t buf_len,uint64_t off,NvmeRequest * req)4887 static uint16_t nvme_smart_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4888 uint64_t off, NvmeRequest *req)
4889 {
4890 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4891 struct nvme_stats stats = { 0 };
4892 NvmeSmartLog smart = { 0 };
4893 uint32_t trans_len;
4894 NvmeNamespace *ns;
4895 time_t current_ms;
4896 uint64_t u_read, u_written;
4897
4898 if (off >= sizeof(smart)) {
4899 return NVME_INVALID_FIELD | NVME_DNR;
4900 }
4901
4902 if (nsid != 0xffffffff) {
4903 ns = nvme_ns(n, nsid);
4904 if (!ns) {
4905 return NVME_INVALID_NSID | NVME_DNR;
4906 }
4907 nvme_set_blk_stats(ns, &stats);
4908 } else {
4909 int i;
4910
4911 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
4912 ns = nvme_ns(n, i);
4913 if (!ns) {
4914 continue;
4915 }
4916 nvme_set_blk_stats(ns, &stats);
4917 }
4918 }
4919
4920 trans_len = MIN(sizeof(smart) - off, buf_len);
4921 smart.critical_warning = n->smart_critical_warning;
4922
4923 u_read = DIV_ROUND_UP(stats.units_read >> BDRV_SECTOR_BITS, 1000);
4924 u_written = DIV_ROUND_UP(stats.units_written >> BDRV_SECTOR_BITS, 1000);
4925
4926 smart.data_units_read[0] = cpu_to_le64(u_read);
4927 smart.data_units_written[0] = cpu_to_le64(u_written);
4928 smart.host_read_commands[0] = cpu_to_le64(stats.read_commands);
4929 smart.host_write_commands[0] = cpu_to_le64(stats.write_commands);
4930
4931 smart.temperature = cpu_to_le16(n->temperature);
4932
4933 if ((n->temperature >= n->features.temp_thresh_hi) ||
4934 (n->temperature <= n->features.temp_thresh_low)) {
4935 smart.critical_warning |= NVME_SMART_TEMPERATURE;
4936 }
4937
4938 current_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
4939 smart.power_on_hours[0] =
4940 cpu_to_le64((((current_ms - n->starttime_ms) / 1000) / 60) / 60);
4941
4942 if (!rae) {
4943 nvme_clear_events(n, NVME_AER_TYPE_SMART);
4944 }
4945
4946 return nvme_c2h(n, (uint8_t *) &smart + off, trans_len, req);
4947 }
4948
nvme_endgrp_info(NvmeCtrl * n,uint8_t rae,uint32_t buf_len,uint64_t off,NvmeRequest * req)4949 static uint16_t nvme_endgrp_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4950 uint64_t off, NvmeRequest *req)
4951 {
4952 uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
4953 uint16_t endgrpid = (dw11 >> 16) & 0xffff;
4954 struct nvme_stats stats = {};
4955 NvmeEndGrpLog info = {};
4956 int i;
4957
4958 if (!n->subsys || endgrpid != 0x1) {
4959 return NVME_INVALID_FIELD | NVME_DNR;
4960 }
4961
4962 if (off >= sizeof(info)) {
4963 return NVME_INVALID_FIELD | NVME_DNR;
4964 }
4965
4966 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
4967 NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
4968 if (!ns) {
4969 continue;
4970 }
4971
4972 nvme_set_blk_stats(ns, &stats);
4973 }
4974
4975 info.data_units_read[0] =
4976 cpu_to_le64(DIV_ROUND_UP(stats.units_read / 1000000000, 1000000000));
4977 info.data_units_written[0] =
4978 cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4979 info.media_units_written[0] =
4980 cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4981
4982 info.host_read_commands[0] = cpu_to_le64(stats.read_commands);
4983 info.host_write_commands[0] = cpu_to_le64(stats.write_commands);
4984
4985 buf_len = MIN(sizeof(info) - off, buf_len);
4986
4987 return nvme_c2h(n, (uint8_t *)&info + off, buf_len, req);
4988 }
4989
4990
nvme_fw_log_info(NvmeCtrl * n,uint32_t buf_len,uint64_t off,NvmeRequest * req)4991 static uint16_t nvme_fw_log_info(NvmeCtrl *n, uint32_t buf_len, uint64_t off,
4992 NvmeRequest *req)
4993 {
4994 uint32_t trans_len;
4995 NvmeFwSlotInfoLog fw_log = {
4996 .afi = 0x1,
4997 };
4998
4999 if (off >= sizeof(fw_log)) {
5000 return NVME_INVALID_FIELD | NVME_DNR;
5001 }
5002
5003 strpadcpy((char *)&fw_log.frs1, sizeof(fw_log.frs1), "1.0", ' ');
5004 trans_len = MIN(sizeof(fw_log) - off, buf_len);
5005
5006 return nvme_c2h(n, (uint8_t *) &fw_log + off, trans_len, req);
5007 }
5008
nvme_error_info(NvmeCtrl * n,uint8_t rae,uint32_t buf_len,uint64_t off,NvmeRequest * req)5009 static uint16_t nvme_error_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
5010 uint64_t off, NvmeRequest *req)
5011 {
5012 uint32_t trans_len;
5013 NvmeErrorLog errlog;
5014
5015 if (off >= sizeof(errlog)) {
5016 return NVME_INVALID_FIELD | NVME_DNR;
5017 }
5018
5019 if (!rae) {
5020 nvme_clear_events(n, NVME_AER_TYPE_ERROR);
5021 }
5022
5023 memset(&errlog, 0x0, sizeof(errlog));
5024 trans_len = MIN(sizeof(errlog) - off, buf_len);
5025
5026 return nvme_c2h(n, (uint8_t *)&errlog, trans_len, req);
5027 }
5028
nvme_changed_nslist(NvmeCtrl * n,uint8_t rae,uint32_t buf_len,uint64_t off,NvmeRequest * req)5029 static uint16_t nvme_changed_nslist(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
5030 uint64_t off, NvmeRequest *req)
5031 {
5032 uint32_t nslist[1024];
5033 uint32_t trans_len;
5034 int i = 0;
5035 uint32_t nsid;
5036
5037 if (off >= sizeof(nslist)) {
5038 trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(nslist));
5039 return NVME_INVALID_FIELD | NVME_DNR;
5040 }
5041
5042 memset(nslist, 0x0, sizeof(nslist));
5043 trans_len = MIN(sizeof(nslist) - off, buf_len);
5044
5045 while ((nsid = find_first_bit(n->changed_nsids, NVME_CHANGED_NSID_SIZE)) !=
5046 NVME_CHANGED_NSID_SIZE) {
5047 /*
5048 * If more than 1024 namespaces, the first entry in the log page should
5049 * be set to FFFFFFFFh and the others to 0 as spec.
5050 */
5051 if (i == ARRAY_SIZE(nslist)) {
5052 memset(nslist, 0x0, sizeof(nslist));
5053 nslist[0] = 0xffffffff;
5054 break;
5055 }
5056
5057 nslist[i++] = nsid;
5058 clear_bit(nsid, n->changed_nsids);
5059 }
5060
5061 /*
5062 * Remove all the remaining list entries in case returns directly due to
5063 * more than 1024 namespaces.
5064 */
5065 if (nslist[0] == 0xffffffff) {
5066 bitmap_zero(n->changed_nsids, NVME_CHANGED_NSID_SIZE);
5067 }
5068
5069 if (!rae) {
5070 nvme_clear_events(n, NVME_AER_TYPE_NOTICE);
5071 }
5072
5073 return nvme_c2h(n, ((uint8_t *)nslist) + off, trans_len, req);
5074 }
5075
nvme_cmd_effects(NvmeCtrl * n,uint8_t csi,uint32_t buf_len,uint64_t off,NvmeRequest * req)5076 static uint16_t nvme_cmd_effects(NvmeCtrl *n, uint8_t csi, uint32_t buf_len,
5077 uint64_t off, NvmeRequest *req)
5078 {
5079 NvmeEffectsLog log = {};
5080 const uint32_t *src_iocs = NULL;
5081 uint32_t trans_len;
5082
5083 if (off >= sizeof(log)) {
5084 trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(log));
5085 return NVME_INVALID_FIELD | NVME_DNR;
5086 }
5087
5088 switch (NVME_CC_CSS(ldl_le_p(&n->bar.cc))) {
5089 case NVME_CC_CSS_NVM:
5090 src_iocs = nvme_cse_iocs_nvm;
5091 /* fall through */
5092 case NVME_CC_CSS_ADMIN_ONLY:
5093 break;
5094 case NVME_CC_CSS_CSI:
5095 switch (csi) {
5096 case NVME_CSI_NVM:
5097 src_iocs = nvme_cse_iocs_nvm;
5098 break;
5099 case NVME_CSI_ZONED:
5100 src_iocs = nvme_cse_iocs_zoned;
5101 break;
5102 }
5103 }
5104
5105 memcpy(log.acs, nvme_cse_acs, sizeof(nvme_cse_acs));
5106
5107 if (src_iocs) {
5108 memcpy(log.iocs, src_iocs, sizeof(log.iocs));
5109 }
5110
5111 trans_len = MIN(sizeof(log) - off, buf_len);
5112
5113 return nvme_c2h(n, ((uint8_t *)&log) + off, trans_len, req);
5114 }
5115
sizeof_fdp_conf_descr(size_t nruh,size_t vss)5116 static size_t sizeof_fdp_conf_descr(size_t nruh, size_t vss)
5117 {
5118 size_t entry_siz = sizeof(NvmeFdpDescrHdr) + nruh * sizeof(NvmeRuhDescr)
5119 + vss;
5120 return ROUND_UP(entry_siz, 8);
5121 }
5122
nvme_fdp_confs(NvmeCtrl * n,uint32_t endgrpid,uint32_t buf_len,uint64_t off,NvmeRequest * req)5123 static uint16_t nvme_fdp_confs(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
5124 uint64_t off, NvmeRequest *req)
5125 {
5126 uint32_t log_size, trans_len;
5127 g_autofree uint8_t *buf = NULL;
5128 NvmeFdpDescrHdr *hdr;
5129 NvmeRuhDescr *ruhd;
5130 NvmeEnduranceGroup *endgrp;
5131 NvmeFdpConfsHdr *log;
5132 size_t nruh, fdp_descr_size;
5133 int i;
5134
5135 if (endgrpid != 1 || !n->subsys) {
5136 return NVME_INVALID_FIELD | NVME_DNR;
5137 }
5138
5139 endgrp = &n->subsys->endgrp;
5140
5141 if (endgrp->fdp.enabled) {
5142 nruh = endgrp->fdp.nruh;
5143 } else {
5144 nruh = 1;
5145 }
5146
5147 fdp_descr_size = sizeof_fdp_conf_descr(nruh, FDPVSS);
5148 log_size = sizeof(NvmeFdpConfsHdr) + fdp_descr_size;
5149
5150 if (off >= log_size) {
5151 return NVME_INVALID_FIELD | NVME_DNR;
5152 }
5153
5154 trans_len = MIN(log_size - off, buf_len);
5155
5156 buf = g_malloc0(log_size);
5157 log = (NvmeFdpConfsHdr *)buf;
5158 hdr = (NvmeFdpDescrHdr *)(log + 1);
5159 ruhd = (NvmeRuhDescr *)(buf + sizeof(*log) + sizeof(*hdr));
5160
5161 log->num_confs = cpu_to_le16(0);
5162 log->size = cpu_to_le32(log_size);
5163
5164 hdr->descr_size = cpu_to_le16(fdp_descr_size);
5165 if (endgrp->fdp.enabled) {
5166 hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, VALID, 1);
5167 hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, RGIF, endgrp->fdp.rgif);
5168 hdr->nrg = cpu_to_le16(endgrp->fdp.nrg);
5169 hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
5170 hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
5171 hdr->nnss = cpu_to_le32(NVME_MAX_NAMESPACES);
5172 hdr->runs = cpu_to_le64(endgrp->fdp.runs);
5173
5174 for (i = 0; i < nruh; i++) {
5175 ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
5176 ruhd++;
5177 }
5178 } else {
5179 /* 1 bit for RUH in PIF -> 2 RUHs max. */
5180 hdr->nrg = cpu_to_le16(1);
5181 hdr->nruh = cpu_to_le16(1);
5182 hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
5183 hdr->nnss = cpu_to_le32(1);
5184 hdr->runs = cpu_to_le64(96 * MiB);
5185
5186 ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
5187 }
5188
5189 return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
5190 }
5191
nvme_fdp_ruh_usage(NvmeCtrl * n,uint32_t endgrpid,uint32_t dw10,uint32_t dw12,uint32_t buf_len,uint64_t off,NvmeRequest * req)5192 static uint16_t nvme_fdp_ruh_usage(NvmeCtrl *n, uint32_t endgrpid,
5193 uint32_t dw10, uint32_t dw12,
5194 uint32_t buf_len, uint64_t off,
5195 NvmeRequest *req)
5196 {
5197 NvmeRuHandle *ruh;
5198 NvmeRuhuLog *hdr;
5199 NvmeRuhuDescr *ruhud;
5200 NvmeEnduranceGroup *endgrp;
5201 g_autofree uint8_t *buf = NULL;
5202 uint32_t log_size, trans_len;
5203 uint16_t i;
5204
5205 if (endgrpid != 1 || !n->subsys) {
5206 return NVME_INVALID_FIELD | NVME_DNR;
5207 }
5208
5209 endgrp = &n->subsys->endgrp;
5210
5211 if (!endgrp->fdp.enabled) {
5212 return NVME_FDP_DISABLED | NVME_DNR;
5213 }
5214
5215 log_size = sizeof(NvmeRuhuLog) + endgrp->fdp.nruh * sizeof(NvmeRuhuDescr);
5216
5217 if (off >= log_size) {
5218 return NVME_INVALID_FIELD | NVME_DNR;
5219 }
5220
5221 trans_len = MIN(log_size - off, buf_len);
5222
5223 buf = g_malloc0(log_size);
5224 hdr = (NvmeRuhuLog *)buf;
5225 ruhud = (NvmeRuhuDescr *)(hdr + 1);
5226
5227 ruh = endgrp->fdp.ruhs;
5228 hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
5229
5230 for (i = 0; i < endgrp->fdp.nruh; i++, ruhud++, ruh++) {
5231 ruhud->ruha = ruh->ruha;
5232 }
5233
5234 return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
5235 }
5236
nvme_fdp_stats(NvmeCtrl * n,uint32_t endgrpid,uint32_t buf_len,uint64_t off,NvmeRequest * req)5237 static uint16_t nvme_fdp_stats(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
5238 uint64_t off, NvmeRequest *req)
5239 {
5240 NvmeEnduranceGroup *endgrp;
5241 NvmeFdpStatsLog log = {};
5242 uint32_t trans_len;
5243
5244 if (off >= sizeof(NvmeFdpStatsLog)) {
5245 return NVME_INVALID_FIELD | NVME_DNR;
5246 }
5247
5248 if (endgrpid != 1 || !n->subsys) {
5249 return NVME_INVALID_FIELD | NVME_DNR;
5250 }
5251
5252 if (!n->subsys->endgrp.fdp.enabled) {
5253 return NVME_FDP_DISABLED | NVME_DNR;
5254 }
5255
5256 endgrp = &n->subsys->endgrp;
5257
5258 trans_len = MIN(sizeof(log) - off, buf_len);
5259
5260 /* spec value is 128 bit, we only use 64 bit */
5261 log.hbmw[0] = cpu_to_le64(endgrp->fdp.hbmw);
5262 log.mbmw[0] = cpu_to_le64(endgrp->fdp.mbmw);
5263 log.mbe[0] = cpu_to_le64(endgrp->fdp.mbe);
5264
5265 return nvme_c2h(n, (uint8_t *)&log + off, trans_len, req);
5266 }
5267
nvme_fdp_events(NvmeCtrl * n,uint32_t endgrpid,uint32_t buf_len,uint64_t off,NvmeRequest * req)5268 static uint16_t nvme_fdp_events(NvmeCtrl *n, uint32_t endgrpid,
5269 uint32_t buf_len, uint64_t off,
5270 NvmeRequest *req)
5271 {
5272 NvmeEnduranceGroup *endgrp;
5273 NvmeCmd *cmd = &req->cmd;
5274 bool host_events = (cmd->cdw10 >> 8) & 0x1;
5275 uint32_t log_size, trans_len;
5276 NvmeFdpEventBuffer *ebuf;
5277 g_autofree NvmeFdpEventsLog *elog = NULL;
5278 NvmeFdpEvent *event;
5279
5280 if (endgrpid != 1 || !n->subsys) {
5281 return NVME_INVALID_FIELD | NVME_DNR;
5282 }
5283
5284 endgrp = &n->subsys->endgrp;
5285
5286 if (!endgrp->fdp.enabled) {
5287 return NVME_FDP_DISABLED | NVME_DNR;
5288 }
5289
5290 if (host_events) {
5291 ebuf = &endgrp->fdp.host_events;
5292 } else {
5293 ebuf = &endgrp->fdp.ctrl_events;
5294 }
5295
5296 log_size = sizeof(NvmeFdpEventsLog) + ebuf->nelems * sizeof(NvmeFdpEvent);
5297
5298 if (off >= log_size) {
5299 return NVME_INVALID_FIELD | NVME_DNR;
5300 }
5301
5302 trans_len = MIN(log_size - off, buf_len);
5303 elog = g_malloc0(log_size);
5304 elog->num_events = cpu_to_le32(ebuf->nelems);
5305 event = (NvmeFdpEvent *)(elog + 1);
5306
5307 if (ebuf->nelems && ebuf->start == ebuf->next) {
5308 unsigned int nelems = (NVME_FDP_MAX_EVENTS - ebuf->start);
5309 /* wrap over, copy [start;NVME_FDP_MAX_EVENTS[ and [0; next[ */
5310 memcpy(event, &ebuf->events[ebuf->start],
5311 sizeof(NvmeFdpEvent) * nelems);
5312 memcpy(event + nelems, ebuf->events,
5313 sizeof(NvmeFdpEvent) * ebuf->next);
5314 } else if (ebuf->start < ebuf->next) {
5315 memcpy(event, &ebuf->events[ebuf->start],
5316 sizeof(NvmeFdpEvent) * (ebuf->next - ebuf->start));
5317 }
5318
5319 return nvme_c2h(n, (uint8_t *)elog + off, trans_len, req);
5320 }
5321
nvme_get_log(NvmeCtrl * n,NvmeRequest * req)5322 static uint16_t nvme_get_log(NvmeCtrl *n, NvmeRequest *req)
5323 {
5324 NvmeCmd *cmd = &req->cmd;
5325
5326 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
5327 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
5328 uint32_t dw12 = le32_to_cpu(cmd->cdw12);
5329 uint32_t dw13 = le32_to_cpu(cmd->cdw13);
5330 uint8_t lid = dw10 & 0xff;
5331 uint8_t lsp = (dw10 >> 8) & 0xf;
5332 uint8_t rae = (dw10 >> 15) & 0x1;
5333 uint8_t csi = le32_to_cpu(cmd->cdw14) >> 24;
5334 uint32_t numdl, numdu, lspi;
5335 uint64_t off, lpol, lpou;
5336 size_t len;
5337 uint16_t status;
5338
5339 numdl = (dw10 >> 16);
5340 numdu = (dw11 & 0xffff);
5341 lspi = (dw11 >> 16);
5342 lpol = dw12;
5343 lpou = dw13;
5344
5345 len = (((numdu << 16) | numdl) + 1) << 2;
5346 off = (lpou << 32ULL) | lpol;
5347
5348 if (off & 0x3) {
5349 return NVME_INVALID_FIELD | NVME_DNR;
5350 }
5351
5352 trace_pci_nvme_get_log(nvme_cid(req), lid, lsp, rae, len, off);
5353
5354 status = nvme_check_mdts(n, len);
5355 if (status) {
5356 return status;
5357 }
5358
5359 switch (lid) {
5360 case NVME_LOG_ERROR_INFO:
5361 return nvme_error_info(n, rae, len, off, req);
5362 case NVME_LOG_SMART_INFO:
5363 return nvme_smart_info(n, rae, len, off, req);
5364 case NVME_LOG_FW_SLOT_INFO:
5365 return nvme_fw_log_info(n, len, off, req);
5366 case NVME_LOG_CHANGED_NSLIST:
5367 return nvme_changed_nslist(n, rae, len, off, req);
5368 case NVME_LOG_CMD_EFFECTS:
5369 return nvme_cmd_effects(n, csi, len, off, req);
5370 case NVME_LOG_ENDGRP:
5371 return nvme_endgrp_info(n, rae, len, off, req);
5372 case NVME_LOG_FDP_CONFS:
5373 return nvme_fdp_confs(n, lspi, len, off, req);
5374 case NVME_LOG_FDP_RUH_USAGE:
5375 return nvme_fdp_ruh_usage(n, lspi, dw10, dw12, len, off, req);
5376 case NVME_LOG_FDP_STATS:
5377 return nvme_fdp_stats(n, lspi, len, off, req);
5378 case NVME_LOG_FDP_EVENTS:
5379 return nvme_fdp_events(n, lspi, len, off, req);
5380 default:
5381 trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid);
5382 return NVME_INVALID_FIELD | NVME_DNR;
5383 }
5384 }
5385
nvme_free_cq(NvmeCQueue * cq,NvmeCtrl * n)5386 static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
5387 {
5388 PCIDevice *pci = PCI_DEVICE(n);
5389 uint16_t offset = (cq->cqid << 3) + (1 << 2);
5390
5391 n->cq[cq->cqid] = NULL;
5392 qemu_bh_delete(cq->bh);
5393 if (cq->ioeventfd_enabled) {
5394 memory_region_del_eventfd(&n->iomem,
5395 0x1000 + offset, 4, false, 0, &cq->notifier);
5396 event_notifier_set_handler(&cq->notifier, NULL);
5397 event_notifier_cleanup(&cq->notifier);
5398 }
5399 if (msix_enabled(pci)) {
5400 msix_vector_unuse(pci, cq->vector);
5401 }
5402 if (cq->cqid) {
5403 g_free(cq);
5404 }
5405 }
5406
nvme_del_cq(NvmeCtrl * n,NvmeRequest * req)5407 static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
5408 {
5409 NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
5410 NvmeCQueue *cq;
5411 uint16_t qid = le16_to_cpu(c->qid);
5412
5413 if (unlikely(!qid || nvme_check_cqid(n, qid))) {
5414 trace_pci_nvme_err_invalid_del_cq_cqid(qid);
5415 return NVME_INVALID_CQID | NVME_DNR;
5416 }
5417
5418 cq = n->cq[qid];
5419 if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
5420 trace_pci_nvme_err_invalid_del_cq_notempty(qid);
5421 return NVME_INVALID_QUEUE_DEL;
5422 }
5423
5424 if (cq->irq_enabled && cq->tail != cq->head) {
5425 n->cq_pending--;
5426 }
5427
5428 nvme_irq_deassert(n, cq);
5429 trace_pci_nvme_del_cq(qid);
5430 nvme_free_cq(cq, n);
5431 return NVME_SUCCESS;
5432 }
5433
nvme_init_cq(NvmeCQueue * cq,NvmeCtrl * n,uint64_t dma_addr,uint16_t cqid,uint16_t vector,uint16_t size,uint16_t irq_enabled)5434 static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
5435 uint16_t cqid, uint16_t vector, uint16_t size,
5436 uint16_t irq_enabled)
5437 {
5438 PCIDevice *pci = PCI_DEVICE(n);
5439
5440 if (msix_enabled(pci)) {
5441 msix_vector_use(pci, vector);
5442 }
5443 cq->ctrl = n;
5444 cq->cqid = cqid;
5445 cq->size = size;
5446 cq->dma_addr = dma_addr;
5447 cq->phase = 1;
5448 cq->irq_enabled = irq_enabled;
5449 cq->vector = vector;
5450 cq->head = cq->tail = 0;
5451 QTAILQ_INIT(&cq->req_list);
5452 QTAILQ_INIT(&cq->sq_list);
5453 if (n->dbbuf_enabled) {
5454 cq->db_addr = n->dbbuf_dbs + (cqid << 3) + (1 << 2);
5455 cq->ei_addr = n->dbbuf_eis + (cqid << 3) + (1 << 2);
5456
5457 if (n->params.ioeventfd && cqid != 0) {
5458 if (!nvme_init_cq_ioeventfd(cq)) {
5459 cq->ioeventfd_enabled = true;
5460 }
5461 }
5462 }
5463 n->cq[cqid] = cq;
5464 cq->bh = qemu_bh_new_guarded(nvme_post_cqes, cq,
5465 &DEVICE(cq->ctrl)->mem_reentrancy_guard);
5466 }
5467
nvme_create_cq(NvmeCtrl * n,NvmeRequest * req)5468 static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeRequest *req)
5469 {
5470 NvmeCQueue *cq;
5471 NvmeCreateCq *c = (NvmeCreateCq *)&req->cmd;
5472 uint16_t cqid = le16_to_cpu(c->cqid);
5473 uint16_t vector = le16_to_cpu(c->irq_vector);
5474 uint16_t qsize = le16_to_cpu(c->qsize);
5475 uint16_t qflags = le16_to_cpu(c->cq_flags);
5476 uint64_t prp1 = le64_to_cpu(c->prp1);
5477 uint32_t cc = ldq_le_p(&n->bar.cc);
5478 uint8_t iocqes = NVME_CC_IOCQES(cc);
5479 uint8_t iosqes = NVME_CC_IOSQES(cc);
5480
5481 trace_pci_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
5482 NVME_CQ_FLAGS_IEN(qflags) != 0);
5483
5484 if (iosqes != NVME_SQES || iocqes != NVME_CQES) {
5485 trace_pci_nvme_err_invalid_create_cq_entry_size(iosqes, iocqes);
5486 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
5487 }
5488
5489 if (unlikely(!cqid || cqid > n->conf_ioqpairs || n->cq[cqid] != NULL)) {
5490 trace_pci_nvme_err_invalid_create_cq_cqid(cqid);
5491 return NVME_INVALID_QID | NVME_DNR;
5492 }
5493 if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
5494 trace_pci_nvme_err_invalid_create_cq_size(qsize);
5495 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
5496 }
5497 if (unlikely(prp1 & (n->page_size - 1))) {
5498 trace_pci_nvme_err_invalid_create_cq_addr(prp1);
5499 return NVME_INVALID_PRP_OFFSET | NVME_DNR;
5500 }
5501 if (unlikely(!msix_enabled(PCI_DEVICE(n)) && vector)) {
5502 trace_pci_nvme_err_invalid_create_cq_vector(vector);
5503 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5504 }
5505 if (unlikely(vector >= n->conf_msix_qsize)) {
5506 trace_pci_nvme_err_invalid_create_cq_vector(vector);
5507 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
5508 }
5509 if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
5510 trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
5511 return NVME_INVALID_FIELD | NVME_DNR;
5512 }
5513
5514 cq = g_malloc0(sizeof(*cq));
5515 nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
5516 NVME_CQ_FLAGS_IEN(qflags));
5517
5518 /*
5519 * It is only required to set qs_created when creating a completion queue;
5520 * creating a submission queue without a matching completion queue will
5521 * fail.
5522 */
5523 n->qs_created = true;
5524 return NVME_SUCCESS;
5525 }
5526
nvme_rpt_empty_id_struct(NvmeCtrl * n,NvmeRequest * req)5527 static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl *n, NvmeRequest *req)
5528 {
5529 uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5530
5531 return nvme_c2h(n, id, sizeof(id), req);
5532 }
5533
nvme_identify_ctrl(NvmeCtrl * n,NvmeRequest * req)5534 static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeRequest *req)
5535 {
5536 trace_pci_nvme_identify_ctrl();
5537
5538 return nvme_c2h(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl), req);
5539 }
5540
nvme_identify_ctrl_csi(NvmeCtrl * n,NvmeRequest * req)5541 static uint16_t nvme_identify_ctrl_csi(NvmeCtrl *n, NvmeRequest *req)
5542 {
5543 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5544 uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
5545 NvmeIdCtrlNvm *id_nvm = (NvmeIdCtrlNvm *)&id;
5546
5547 trace_pci_nvme_identify_ctrl_csi(c->csi);
5548
5549 switch (c->csi) {
5550 case NVME_CSI_NVM:
5551 id_nvm->vsl = n->params.vsl;
5552 id_nvm->dmrsl = cpu_to_le32(n->dmrsl);
5553 break;
5554
5555 case NVME_CSI_ZONED:
5556 ((NvmeIdCtrlZoned *)&id)->zasl = n->params.zasl;
5557 break;
5558
5559 default:
5560 return NVME_INVALID_FIELD | NVME_DNR;
5561 }
5562
5563 return nvme_c2h(n, id, sizeof(id), req);
5564 }
5565
nvme_identify_ns(NvmeCtrl * n,NvmeRequest * req,bool active)5566 static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeRequest *req, bool active)
5567 {
5568 NvmeNamespace *ns;
5569 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5570 uint32_t nsid = le32_to_cpu(c->nsid);
5571
5572 trace_pci_nvme_identify_ns(nsid);
5573
5574 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5575 return NVME_INVALID_NSID | NVME_DNR;
5576 }
5577
5578 ns = nvme_ns(n, nsid);
5579 if (unlikely(!ns)) {
5580 if (!active) {
5581 ns = nvme_subsys_ns(n->subsys, nsid);
5582 if (!ns) {
5583 return nvme_rpt_empty_id_struct(n, req);
5584 }
5585 } else {
5586 return nvme_rpt_empty_id_struct(n, req);
5587 }
5588 }
5589
5590 if (active || ns->csi == NVME_CSI_NVM) {
5591 return nvme_c2h(n, (uint8_t *)&ns->id_ns, sizeof(NvmeIdNs), req);
5592 }
5593
5594 return NVME_INVALID_CMD_SET | NVME_DNR;
5595 }
5596
nvme_identify_ctrl_list(NvmeCtrl * n,NvmeRequest * req,bool attached)5597 static uint16_t nvme_identify_ctrl_list(NvmeCtrl *n, NvmeRequest *req,
5598 bool attached)
5599 {
5600 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5601 uint32_t nsid = le32_to_cpu(c->nsid);
5602 uint16_t min_id = le16_to_cpu(c->ctrlid);
5603 uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
5604 uint16_t *ids = &list[1];
5605 NvmeNamespace *ns;
5606 NvmeCtrl *ctrl;
5607 int cntlid, nr_ids = 0;
5608
5609 trace_pci_nvme_identify_ctrl_list(c->cns, min_id);
5610
5611 if (!n->subsys) {
5612 return NVME_INVALID_FIELD | NVME_DNR;
5613 }
5614
5615 if (attached) {
5616 if (nsid == NVME_NSID_BROADCAST) {
5617 return NVME_INVALID_FIELD | NVME_DNR;
5618 }
5619
5620 ns = nvme_subsys_ns(n->subsys, nsid);
5621 if (!ns) {
5622 return NVME_INVALID_FIELD | NVME_DNR;
5623 }
5624 }
5625
5626 for (cntlid = min_id; cntlid < ARRAY_SIZE(n->subsys->ctrls); cntlid++) {
5627 ctrl = nvme_subsys_ctrl(n->subsys, cntlid);
5628 if (!ctrl) {
5629 continue;
5630 }
5631
5632 if (attached && !nvme_ns(ctrl, nsid)) {
5633 continue;
5634 }
5635
5636 ids[nr_ids++] = cntlid;
5637 }
5638
5639 list[0] = nr_ids;
5640
5641 return nvme_c2h(n, (uint8_t *)list, sizeof(list), req);
5642 }
5643
nvme_identify_pri_ctrl_cap(NvmeCtrl * n,NvmeRequest * req)5644 static uint16_t nvme_identify_pri_ctrl_cap(NvmeCtrl *n, NvmeRequest *req)
5645 {
5646 trace_pci_nvme_identify_pri_ctrl_cap(le16_to_cpu(n->pri_ctrl_cap.cntlid));
5647
5648 return nvme_c2h(n, (uint8_t *)&n->pri_ctrl_cap,
5649 sizeof(NvmePriCtrlCap), req);
5650 }
5651
nvme_identify_sec_ctrl_list(NvmeCtrl * n,NvmeRequest * req)5652 static uint16_t nvme_identify_sec_ctrl_list(NvmeCtrl *n, NvmeRequest *req)
5653 {
5654 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5655 uint16_t pri_ctrl_id = le16_to_cpu(n->pri_ctrl_cap.cntlid);
5656 uint16_t min_id = le16_to_cpu(c->ctrlid);
5657 uint8_t num_sec_ctrl = n->nr_sec_ctrls;
5658 NvmeSecCtrlList list = {0};
5659 uint8_t i;
5660
5661 for (i = 0; i < num_sec_ctrl; i++) {
5662 if (n->sec_ctrl_list[i].scid >= min_id) {
5663 list.numcntl = MIN(num_sec_ctrl - i, 127);
5664 memcpy(&list.sec, n->sec_ctrl_list + i,
5665 list.numcntl * sizeof(NvmeSecCtrlEntry));
5666 break;
5667 }
5668 }
5669
5670 trace_pci_nvme_identify_sec_ctrl_list(pri_ctrl_id, list.numcntl);
5671
5672 return nvme_c2h(n, (uint8_t *)&list, sizeof(list), req);
5673 }
5674
nvme_identify_ns_csi(NvmeCtrl * n,NvmeRequest * req,bool active)5675 static uint16_t nvme_identify_ns_csi(NvmeCtrl *n, NvmeRequest *req,
5676 bool active)
5677 {
5678 NvmeNamespace *ns;
5679 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5680 uint32_t nsid = le32_to_cpu(c->nsid);
5681
5682 trace_pci_nvme_identify_ns_csi(nsid, c->csi);
5683
5684 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5685 return NVME_INVALID_NSID | NVME_DNR;
5686 }
5687
5688 ns = nvme_ns(n, nsid);
5689 if (unlikely(!ns)) {
5690 if (!active) {
5691 ns = nvme_subsys_ns(n->subsys, nsid);
5692 if (!ns) {
5693 return nvme_rpt_empty_id_struct(n, req);
5694 }
5695 } else {
5696 return nvme_rpt_empty_id_struct(n, req);
5697 }
5698 }
5699
5700 if (c->csi == NVME_CSI_NVM) {
5701 return nvme_c2h(n, (uint8_t *)&ns->id_ns_nvm, sizeof(NvmeIdNsNvm),
5702 req);
5703 } else if (c->csi == NVME_CSI_ZONED && ns->csi == NVME_CSI_ZONED) {
5704 return nvme_c2h(n, (uint8_t *)ns->id_ns_zoned, sizeof(NvmeIdNsZoned),
5705 req);
5706 }
5707
5708 return NVME_INVALID_FIELD | NVME_DNR;
5709 }
5710
nvme_identify_nslist(NvmeCtrl * n,NvmeRequest * req,bool active)5711 static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req,
5712 bool active)
5713 {
5714 NvmeNamespace *ns;
5715 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5716 uint32_t min_nsid = le32_to_cpu(c->nsid);
5717 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5718 static const int data_len = sizeof(list);
5719 uint32_t *list_ptr = (uint32_t *)list;
5720 int i, j = 0;
5721
5722 trace_pci_nvme_identify_nslist(min_nsid);
5723
5724 /*
5725 * Both FFFFFFFFh (NVME_NSID_BROADCAST) and FFFFFFFFEh are invalid values
5726 * since the Active Namespace ID List should return namespaces with ids
5727 * *higher* than the NSID specified in the command. This is also specified
5728 * in the spec (NVM Express v1.3d, Section 5.15.4).
5729 */
5730 if (min_nsid >= NVME_NSID_BROADCAST - 1) {
5731 return NVME_INVALID_NSID | NVME_DNR;
5732 }
5733
5734 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5735 ns = nvme_ns(n, i);
5736 if (!ns) {
5737 if (!active) {
5738 ns = nvme_subsys_ns(n->subsys, i);
5739 if (!ns) {
5740 continue;
5741 }
5742 } else {
5743 continue;
5744 }
5745 }
5746 if (ns->params.nsid <= min_nsid) {
5747 continue;
5748 }
5749 list_ptr[j++] = cpu_to_le32(ns->params.nsid);
5750 if (j == data_len / sizeof(uint32_t)) {
5751 break;
5752 }
5753 }
5754
5755 return nvme_c2h(n, list, data_len, req);
5756 }
5757
nvme_identify_nslist_csi(NvmeCtrl * n,NvmeRequest * req,bool active)5758 static uint16_t nvme_identify_nslist_csi(NvmeCtrl *n, NvmeRequest *req,
5759 bool active)
5760 {
5761 NvmeNamespace *ns;
5762 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5763 uint32_t min_nsid = le32_to_cpu(c->nsid);
5764 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5765 static const int data_len = sizeof(list);
5766 uint32_t *list_ptr = (uint32_t *)list;
5767 int i, j = 0;
5768
5769 trace_pci_nvme_identify_nslist_csi(min_nsid, c->csi);
5770
5771 /*
5772 * Same as in nvme_identify_nslist(), FFFFFFFFh/FFFFFFFFEh are invalid.
5773 */
5774 if (min_nsid >= NVME_NSID_BROADCAST - 1) {
5775 return NVME_INVALID_NSID | NVME_DNR;
5776 }
5777
5778 if (c->csi != NVME_CSI_NVM && c->csi != NVME_CSI_ZONED) {
5779 return NVME_INVALID_FIELD | NVME_DNR;
5780 }
5781
5782 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
5783 ns = nvme_ns(n, i);
5784 if (!ns) {
5785 if (!active) {
5786 ns = nvme_subsys_ns(n->subsys, i);
5787 if (!ns) {
5788 continue;
5789 }
5790 } else {
5791 continue;
5792 }
5793 }
5794 if (ns->params.nsid <= min_nsid || c->csi != ns->csi) {
5795 continue;
5796 }
5797 list_ptr[j++] = cpu_to_le32(ns->params.nsid);
5798 if (j == data_len / sizeof(uint32_t)) {
5799 break;
5800 }
5801 }
5802
5803 return nvme_c2h(n, list, data_len, req);
5804 }
5805
nvme_endurance_group_list(NvmeCtrl * n,NvmeRequest * req)5806 static uint16_t nvme_endurance_group_list(NvmeCtrl *n, NvmeRequest *req)
5807 {
5808 uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
5809 uint16_t *nr_ids = &list[0];
5810 uint16_t *ids = &list[1];
5811 uint16_t endgid = le32_to_cpu(req->cmd.cdw11) & 0xffff;
5812
5813 /*
5814 * The current nvme-subsys only supports Endurance Group #1.
5815 */
5816 if (!endgid) {
5817 *nr_ids = 1;
5818 ids[0] = 1;
5819 } else {
5820 *nr_ids = 0;
5821 }
5822
5823 return nvme_c2h(n, list, sizeof(list), req);
5824 }
5825
nvme_identify_ns_descr_list(NvmeCtrl * n,NvmeRequest * req)5826 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeRequest *req)
5827 {
5828 NvmeNamespace *ns;
5829 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5830 uint32_t nsid = le32_to_cpu(c->nsid);
5831 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5832 uint8_t *pos = list;
5833 struct {
5834 NvmeIdNsDescr hdr;
5835 uint8_t v[NVME_NIDL_UUID];
5836 } QEMU_PACKED uuid = {};
5837 struct {
5838 NvmeIdNsDescr hdr;
5839 uint8_t v[NVME_NIDL_NGUID];
5840 } QEMU_PACKED nguid = {};
5841 struct {
5842 NvmeIdNsDescr hdr;
5843 uint64_t v;
5844 } QEMU_PACKED eui64 = {};
5845 struct {
5846 NvmeIdNsDescr hdr;
5847 uint8_t v;
5848 } QEMU_PACKED csi = {};
5849
5850 trace_pci_nvme_identify_ns_descr_list(nsid);
5851
5852 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
5853 return NVME_INVALID_NSID | NVME_DNR;
5854 }
5855
5856 ns = nvme_ns(n, nsid);
5857 if (unlikely(!ns)) {
5858 return NVME_INVALID_FIELD | NVME_DNR;
5859 }
5860
5861 if (!qemu_uuid_is_null(&ns->params.uuid)) {
5862 uuid.hdr.nidt = NVME_NIDT_UUID;
5863 uuid.hdr.nidl = NVME_NIDL_UUID;
5864 memcpy(uuid.v, ns->params.uuid.data, NVME_NIDL_UUID);
5865 memcpy(pos, &uuid, sizeof(uuid));
5866 pos += sizeof(uuid);
5867 }
5868
5869 if (!nvme_nguid_is_null(&ns->params.nguid)) {
5870 nguid.hdr.nidt = NVME_NIDT_NGUID;
5871 nguid.hdr.nidl = NVME_NIDL_NGUID;
5872 memcpy(nguid.v, ns->params.nguid.data, NVME_NIDL_NGUID);
5873 memcpy(pos, &nguid, sizeof(nguid));
5874 pos += sizeof(nguid);
5875 }
5876
5877 if (ns->params.eui64) {
5878 eui64.hdr.nidt = NVME_NIDT_EUI64;
5879 eui64.hdr.nidl = NVME_NIDL_EUI64;
5880 eui64.v = cpu_to_be64(ns->params.eui64);
5881 memcpy(pos, &eui64, sizeof(eui64));
5882 pos += sizeof(eui64);
5883 }
5884
5885 csi.hdr.nidt = NVME_NIDT_CSI;
5886 csi.hdr.nidl = NVME_NIDL_CSI;
5887 csi.v = ns->csi;
5888 memcpy(pos, &csi, sizeof(csi));
5889 pos += sizeof(csi);
5890
5891 return nvme_c2h(n, list, sizeof(list), req);
5892 }
5893
nvme_identify_cmd_set(NvmeCtrl * n,NvmeRequest * req)5894 static uint16_t nvme_identify_cmd_set(NvmeCtrl *n, NvmeRequest *req)
5895 {
5896 uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
5897 static const int data_len = sizeof(list);
5898
5899 trace_pci_nvme_identify_cmd_set();
5900
5901 NVME_SET_CSI(*list, NVME_CSI_NVM);
5902 NVME_SET_CSI(*list, NVME_CSI_ZONED);
5903
5904 return nvme_c2h(n, list, data_len, req);
5905 }
5906
nvme_identify(NvmeCtrl * n,NvmeRequest * req)5907 static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req)
5908 {
5909 NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
5910
5911 trace_pci_nvme_identify(nvme_cid(req), c->cns, le16_to_cpu(c->ctrlid),
5912 c->csi);
5913
5914 switch (c->cns) {
5915 case NVME_ID_CNS_NS:
5916 return nvme_identify_ns(n, req, true);
5917 case NVME_ID_CNS_NS_PRESENT:
5918 return nvme_identify_ns(n, req, false);
5919 case NVME_ID_CNS_NS_ATTACHED_CTRL_LIST:
5920 return nvme_identify_ctrl_list(n, req, true);
5921 case NVME_ID_CNS_CTRL_LIST:
5922 return nvme_identify_ctrl_list(n, req, false);
5923 case NVME_ID_CNS_PRIMARY_CTRL_CAP:
5924 return nvme_identify_pri_ctrl_cap(n, req);
5925 case NVME_ID_CNS_SECONDARY_CTRL_LIST:
5926 return nvme_identify_sec_ctrl_list(n, req);
5927 case NVME_ID_CNS_CS_NS:
5928 return nvme_identify_ns_csi(n, req, true);
5929 case NVME_ID_CNS_CS_NS_PRESENT:
5930 return nvme_identify_ns_csi(n, req, false);
5931 case NVME_ID_CNS_CTRL:
5932 return nvme_identify_ctrl(n, req);
5933 case NVME_ID_CNS_CS_CTRL:
5934 return nvme_identify_ctrl_csi(n, req);
5935 case NVME_ID_CNS_NS_ACTIVE_LIST:
5936 return nvme_identify_nslist(n, req, true);
5937 case NVME_ID_CNS_NS_PRESENT_LIST:
5938 return nvme_identify_nslist(n, req, false);
5939 case NVME_ID_CNS_CS_NS_ACTIVE_LIST:
5940 return nvme_identify_nslist_csi(n, req, true);
5941 case NVME_ID_CNS_ENDURANCE_GROUP_LIST:
5942 return nvme_endurance_group_list(n, req);
5943 case NVME_ID_CNS_CS_NS_PRESENT_LIST:
5944 return nvme_identify_nslist_csi(n, req, false);
5945 case NVME_ID_CNS_NS_DESCR_LIST:
5946 return nvme_identify_ns_descr_list(n, req);
5947 case NVME_ID_CNS_IO_COMMAND_SET:
5948 return nvme_identify_cmd_set(n, req);
5949 default:
5950 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
5951 return NVME_INVALID_FIELD | NVME_DNR;
5952 }
5953 }
5954
nvme_abort(NvmeCtrl * n,NvmeRequest * req)5955 static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req)
5956 {
5957 uint16_t sqid = le32_to_cpu(req->cmd.cdw10) & 0xffff;
5958 uint16_t cid = (le32_to_cpu(req->cmd.cdw10) >> 16) & 0xffff;
5959 NvmeSQueue *sq = n->sq[sqid];
5960 NvmeRequest *r, *next;
5961 int i;
5962
5963 req->cqe.result = 1;
5964 if (nvme_check_sqid(n, sqid)) {
5965 return NVME_INVALID_FIELD | NVME_DNR;
5966 }
5967
5968 if (sqid == 0) {
5969 for (i = 0; i < n->outstanding_aers; i++) {
5970 NvmeRequest *re = n->aer_reqs[i];
5971 if (re->cqe.cid == cid) {
5972 memmove(n->aer_reqs + i, n->aer_reqs + i + 1,
5973 (n->outstanding_aers - i - 1) * sizeof(NvmeRequest *));
5974 n->outstanding_aers--;
5975 re->status = NVME_CMD_ABORT_REQ;
5976 req->cqe.result = 0;
5977 nvme_enqueue_req_completion(&n->admin_cq, re);
5978 return NVME_SUCCESS;
5979 }
5980 }
5981 }
5982
5983 QTAILQ_FOREACH_SAFE(r, &sq->out_req_list, entry, next) {
5984 if (r->cqe.cid == cid) {
5985 if (r->aiocb) {
5986 blk_aio_cancel_async(r->aiocb);
5987 }
5988 break;
5989 }
5990 }
5991
5992 return NVME_SUCCESS;
5993 }
5994
nvme_set_timestamp(NvmeCtrl * n,uint64_t ts)5995 static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts)
5996 {
5997 trace_pci_nvme_setfeat_timestamp(ts);
5998
5999 n->host_timestamp = le64_to_cpu(ts);
6000 n->timestamp_set_qemu_clock_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
6001 }
6002
nvme_get_timestamp(const NvmeCtrl * n)6003 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)
6004 {
6005 uint64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
6006 uint64_t elapsed_time = current_time - n->timestamp_set_qemu_clock_ms;
6007
6008 union nvme_timestamp {
6009 struct {
6010 uint64_t timestamp:48;
6011 uint64_t sync:1;
6012 uint64_t origin:3;
6013 uint64_t rsvd1:12;
6014 };
6015 uint64_t all;
6016 };
6017
6018 union nvme_timestamp ts;
6019 ts.all = 0;
6020 ts.timestamp = n->host_timestamp + elapsed_time;
6021
6022 /* If the host timestamp is non-zero, set the timestamp origin */
6023 ts.origin = n->host_timestamp ? 0x01 : 0x00;
6024
6025 trace_pci_nvme_getfeat_timestamp(ts.all);
6026
6027 return cpu_to_le64(ts.all);
6028 }
6029
nvme_get_feature_timestamp(NvmeCtrl * n,NvmeRequest * req)6030 static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
6031 {
6032 uint64_t timestamp = nvme_get_timestamp(n);
6033
6034 return nvme_c2h(n, (uint8_t *)×tamp, sizeof(timestamp), req);
6035 }
6036
nvme_get_feature_fdp(NvmeCtrl * n,uint32_t endgrpid,uint32_t * result)6037 static int nvme_get_feature_fdp(NvmeCtrl *n, uint32_t endgrpid,
6038 uint32_t *result)
6039 {
6040 *result = 0;
6041
6042 if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
6043 return NVME_INVALID_FIELD | NVME_DNR;
6044 }
6045
6046 *result = FIELD_DP16(0, FEAT_FDP, FDPE, 1);
6047 *result = FIELD_DP16(*result, FEAT_FDP, CONF_NDX, 0);
6048
6049 return NVME_SUCCESS;
6050 }
6051
nvme_get_feature_fdp_events(NvmeCtrl * n,NvmeNamespace * ns,NvmeRequest * req,uint32_t * result)6052 static uint16_t nvme_get_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
6053 NvmeRequest *req, uint32_t *result)
6054 {
6055 NvmeCmd *cmd = &req->cmd;
6056 uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
6057 uint16_t ph = cdw11 & 0xffff;
6058 uint8_t noet = (cdw11 >> 16) & 0xff;
6059 uint16_t ruhid, ret;
6060 uint32_t nentries = 0;
6061 uint8_t s_events_ndx = 0;
6062 size_t s_events_siz = sizeof(NvmeFdpEventDescr) * noet;
6063 g_autofree NvmeFdpEventDescr *s_events = g_malloc0(s_events_siz);
6064 NvmeRuHandle *ruh;
6065 NvmeFdpEventDescr *s_event;
6066
6067 if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
6068 return NVME_FDP_DISABLED | NVME_DNR;
6069 }
6070
6071 if (!nvme_ph_valid(ns, ph)) {
6072 return NVME_INVALID_FIELD | NVME_DNR;
6073 }
6074
6075 ruhid = ns->fdp.phs[ph];
6076 ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
6077
6078 assert(ruh);
6079
6080 if (unlikely(noet == 0)) {
6081 return NVME_INVALID_FIELD | NVME_DNR;
6082 }
6083
6084 for (uint8_t event_type = 0; event_type < FDP_EVT_MAX; event_type++) {
6085 uint8_t shift = nvme_fdp_evf_shifts[event_type];
6086 if (!shift && event_type) {
6087 /*
6088 * only first entry (event_type == 0) has a shift value of 0
6089 * other entries are simply unpopulated.
6090 */
6091 continue;
6092 }
6093
6094 nentries++;
6095
6096 s_event = &s_events[s_events_ndx];
6097 s_event->evt = event_type;
6098 s_event->evta = (ruh->event_filter >> shift) & 0x1;
6099
6100 /* break if all `noet` entries are filled */
6101 if ((++s_events_ndx) == noet) {
6102 break;
6103 }
6104 }
6105
6106 ret = nvme_c2h(n, s_events, s_events_siz, req);
6107 if (ret) {
6108 return ret;
6109 }
6110
6111 *result = nentries;
6112 return NVME_SUCCESS;
6113 }
6114
nvme_get_feature(NvmeCtrl * n,NvmeRequest * req)6115 static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req)
6116 {
6117 NvmeCmd *cmd = &req->cmd;
6118 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
6119 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
6120 uint32_t nsid = le32_to_cpu(cmd->nsid);
6121 uint32_t result = 0;
6122 uint8_t fid = NVME_GETSETFEAT_FID(dw10);
6123 NvmeGetFeatureSelect sel = NVME_GETFEAT_SELECT(dw10);
6124 uint16_t iv;
6125 NvmeNamespace *ns;
6126 int i;
6127 uint16_t endgrpid = 0, ret = NVME_SUCCESS;
6128
6129 static const uint32_t nvme_feature_default[NVME_FID_MAX] = {
6130 [NVME_ARBITRATION] = NVME_ARB_AB_NOLIMIT,
6131 };
6132
6133 trace_pci_nvme_getfeat(nvme_cid(req), nsid, fid, sel, dw11);
6134
6135 if (!nvme_feature_support[fid]) {
6136 return NVME_INVALID_FIELD | NVME_DNR;
6137 }
6138
6139 if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
6140 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
6141 /*
6142 * The Reservation Notification Mask and Reservation Persistence
6143 * features require a status code of Invalid Field in Command when
6144 * NSID is FFFFFFFFh. Since the device does not support those
6145 * features we can always return Invalid Namespace or Format as we
6146 * should do for all other features.
6147 */
6148 return NVME_INVALID_NSID | NVME_DNR;
6149 }
6150
6151 if (!nvme_ns(n, nsid)) {
6152 return NVME_INVALID_FIELD | NVME_DNR;
6153 }
6154 }
6155
6156 switch (sel) {
6157 case NVME_GETFEAT_SELECT_CURRENT:
6158 break;
6159 case NVME_GETFEAT_SELECT_SAVED:
6160 /* no features are saveable by the controller; fallthrough */
6161 case NVME_GETFEAT_SELECT_DEFAULT:
6162 goto defaults;
6163 case NVME_GETFEAT_SELECT_CAP:
6164 result = nvme_feature_cap[fid];
6165 goto out;
6166 }
6167
6168 switch (fid) {
6169 case NVME_TEMPERATURE_THRESHOLD:
6170 result = 0;
6171
6172 /*
6173 * The controller only implements the Composite Temperature sensor, so
6174 * return 0 for all other sensors.
6175 */
6176 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6177 goto out;
6178 }
6179
6180 switch (NVME_TEMP_THSEL(dw11)) {
6181 case NVME_TEMP_THSEL_OVER:
6182 result = n->features.temp_thresh_hi;
6183 goto out;
6184 case NVME_TEMP_THSEL_UNDER:
6185 result = n->features.temp_thresh_low;
6186 goto out;
6187 }
6188
6189 return NVME_INVALID_FIELD | NVME_DNR;
6190 case NVME_ERROR_RECOVERY:
6191 if (!nvme_nsid_valid(n, nsid)) {
6192 return NVME_INVALID_NSID | NVME_DNR;
6193 }
6194
6195 ns = nvme_ns(n, nsid);
6196 if (unlikely(!ns)) {
6197 return NVME_INVALID_FIELD | NVME_DNR;
6198 }
6199
6200 result = ns->features.err_rec;
6201 goto out;
6202 case NVME_VOLATILE_WRITE_CACHE:
6203 result = 0;
6204 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6205 ns = nvme_ns(n, i);
6206 if (!ns) {
6207 continue;
6208 }
6209
6210 result = blk_enable_write_cache(ns->blkconf.blk);
6211 if (result) {
6212 break;
6213 }
6214 }
6215 trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
6216 goto out;
6217 case NVME_ASYNCHRONOUS_EVENT_CONF:
6218 result = n->features.async_config;
6219 goto out;
6220 case NVME_TIMESTAMP:
6221 return nvme_get_feature_timestamp(n, req);
6222 case NVME_HOST_BEHAVIOR_SUPPORT:
6223 return nvme_c2h(n, (uint8_t *)&n->features.hbs,
6224 sizeof(n->features.hbs), req);
6225 case NVME_FDP_MODE:
6226 endgrpid = dw11 & 0xff;
6227
6228 if (endgrpid != 0x1) {
6229 return NVME_INVALID_FIELD | NVME_DNR;
6230 }
6231
6232 ret = nvme_get_feature_fdp(n, endgrpid, &result);
6233 if (ret) {
6234 return ret;
6235 }
6236 goto out;
6237 case NVME_FDP_EVENTS:
6238 if (!nvme_nsid_valid(n, nsid)) {
6239 return NVME_INVALID_NSID | NVME_DNR;
6240 }
6241
6242 ns = nvme_ns(n, nsid);
6243 if (unlikely(!ns)) {
6244 return NVME_INVALID_FIELD | NVME_DNR;
6245 }
6246
6247 ret = nvme_get_feature_fdp_events(n, ns, req, &result);
6248 if (ret) {
6249 return ret;
6250 }
6251 goto out;
6252 default:
6253 break;
6254 }
6255
6256 defaults:
6257 switch (fid) {
6258 case NVME_TEMPERATURE_THRESHOLD:
6259 result = 0;
6260
6261 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6262 break;
6263 }
6264
6265 if (NVME_TEMP_THSEL(dw11) == NVME_TEMP_THSEL_OVER) {
6266 result = NVME_TEMPERATURE_WARNING;
6267 }
6268
6269 break;
6270 case NVME_NUMBER_OF_QUEUES:
6271 result = (n->conf_ioqpairs - 1) | ((n->conf_ioqpairs - 1) << 16);
6272 trace_pci_nvme_getfeat_numq(result);
6273 break;
6274 case NVME_INTERRUPT_VECTOR_CONF:
6275 iv = dw11 & 0xffff;
6276 if (iv >= n->conf_ioqpairs + 1) {
6277 return NVME_INVALID_FIELD | NVME_DNR;
6278 }
6279
6280 result = iv;
6281 if (iv == n->admin_cq.vector) {
6282 result |= NVME_INTVC_NOCOALESCING;
6283 }
6284 break;
6285 case NVME_FDP_MODE:
6286 endgrpid = dw11 & 0xff;
6287
6288 if (endgrpid != 0x1) {
6289 return NVME_INVALID_FIELD | NVME_DNR;
6290 }
6291
6292 ret = nvme_get_feature_fdp(n, endgrpid, &result);
6293 if (ret) {
6294 return ret;
6295 }
6296 goto out;
6297
6298 break;
6299 default:
6300 result = nvme_feature_default[fid];
6301 break;
6302 }
6303
6304 out:
6305 req->cqe.result = cpu_to_le32(result);
6306 return ret;
6307 }
6308
nvme_set_feature_timestamp(NvmeCtrl * n,NvmeRequest * req)6309 static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
6310 {
6311 uint16_t ret;
6312 uint64_t timestamp;
6313
6314 ret = nvme_h2c(n, (uint8_t *)×tamp, sizeof(timestamp), req);
6315 if (ret) {
6316 return ret;
6317 }
6318
6319 nvme_set_timestamp(n, timestamp);
6320
6321 return NVME_SUCCESS;
6322 }
6323
nvme_set_feature_fdp_events(NvmeCtrl * n,NvmeNamespace * ns,NvmeRequest * req)6324 static uint16_t nvme_set_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
6325 NvmeRequest *req)
6326 {
6327 NvmeCmd *cmd = &req->cmd;
6328 uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
6329 uint16_t ph = cdw11 & 0xffff;
6330 uint8_t noet = (cdw11 >> 16) & 0xff;
6331 uint16_t ret, ruhid;
6332 uint8_t enable = le32_to_cpu(cmd->cdw12) & 0x1;
6333 uint8_t event_mask = 0;
6334 unsigned int i;
6335 g_autofree uint8_t *events = g_malloc0(noet);
6336 NvmeRuHandle *ruh = NULL;
6337
6338 assert(ns);
6339
6340 if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
6341 return NVME_FDP_DISABLED | NVME_DNR;
6342 }
6343
6344 if (!nvme_ph_valid(ns, ph)) {
6345 return NVME_INVALID_FIELD | NVME_DNR;
6346 }
6347
6348 ruhid = ns->fdp.phs[ph];
6349 ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
6350
6351 ret = nvme_h2c(n, events, noet, req);
6352 if (ret) {
6353 return ret;
6354 }
6355
6356 for (i = 0; i < noet; i++) {
6357 event_mask |= (1 << nvme_fdp_evf_shifts[events[i]]);
6358 }
6359
6360 if (enable) {
6361 ruh->event_filter |= event_mask;
6362 } else {
6363 ruh->event_filter = ruh->event_filter & ~event_mask;
6364 }
6365
6366 return NVME_SUCCESS;
6367 }
6368
nvme_set_feature(NvmeCtrl * n,NvmeRequest * req)6369 static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req)
6370 {
6371 NvmeNamespace *ns = NULL;
6372
6373 NvmeCmd *cmd = &req->cmd;
6374 uint32_t dw10 = le32_to_cpu(cmd->cdw10);
6375 uint32_t dw11 = le32_to_cpu(cmd->cdw11);
6376 uint32_t nsid = le32_to_cpu(cmd->nsid);
6377 uint8_t fid = NVME_GETSETFEAT_FID(dw10);
6378 uint8_t save = NVME_SETFEAT_SAVE(dw10);
6379 uint16_t status;
6380 int i;
6381
6382 trace_pci_nvme_setfeat(nvme_cid(req), nsid, fid, save, dw11);
6383
6384 if (save && !(nvme_feature_cap[fid] & NVME_FEAT_CAP_SAVE)) {
6385 return NVME_FID_NOT_SAVEABLE | NVME_DNR;
6386 }
6387
6388 if (!nvme_feature_support[fid]) {
6389 return NVME_INVALID_FIELD | NVME_DNR;
6390 }
6391
6392 if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
6393 if (nsid != NVME_NSID_BROADCAST) {
6394 if (!nvme_nsid_valid(n, nsid)) {
6395 return NVME_INVALID_NSID | NVME_DNR;
6396 }
6397
6398 ns = nvme_ns(n, nsid);
6399 if (unlikely(!ns)) {
6400 return NVME_INVALID_FIELD | NVME_DNR;
6401 }
6402 }
6403 } else if (nsid && nsid != NVME_NSID_BROADCAST) {
6404 if (!nvme_nsid_valid(n, nsid)) {
6405 return NVME_INVALID_NSID | NVME_DNR;
6406 }
6407
6408 return NVME_FEAT_NOT_NS_SPEC | NVME_DNR;
6409 }
6410
6411 if (!(nvme_feature_cap[fid] & NVME_FEAT_CAP_CHANGE)) {
6412 return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6413 }
6414
6415 switch (fid) {
6416 case NVME_TEMPERATURE_THRESHOLD:
6417 if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
6418 break;
6419 }
6420
6421 switch (NVME_TEMP_THSEL(dw11)) {
6422 case NVME_TEMP_THSEL_OVER:
6423 n->features.temp_thresh_hi = NVME_TEMP_TMPTH(dw11);
6424 break;
6425 case NVME_TEMP_THSEL_UNDER:
6426 n->features.temp_thresh_low = NVME_TEMP_TMPTH(dw11);
6427 break;
6428 default:
6429 return NVME_INVALID_FIELD | NVME_DNR;
6430 }
6431
6432 if ((n->temperature >= n->features.temp_thresh_hi) ||
6433 (n->temperature <= n->features.temp_thresh_low)) {
6434 nvme_smart_event(n, NVME_SMART_TEMPERATURE);
6435 }
6436
6437 break;
6438 case NVME_ERROR_RECOVERY:
6439 if (nsid == NVME_NSID_BROADCAST) {
6440 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6441 ns = nvme_ns(n, i);
6442
6443 if (!ns) {
6444 continue;
6445 }
6446
6447 if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
6448 ns->features.err_rec = dw11;
6449 }
6450 }
6451
6452 break;
6453 }
6454
6455 assert(ns);
6456 if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
6457 ns->features.err_rec = dw11;
6458 }
6459 break;
6460 case NVME_VOLATILE_WRITE_CACHE:
6461 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6462 ns = nvme_ns(n, i);
6463 if (!ns) {
6464 continue;
6465 }
6466
6467 if (!(dw11 & 0x1) && blk_enable_write_cache(ns->blkconf.blk)) {
6468 blk_flush(ns->blkconf.blk);
6469 }
6470
6471 blk_set_enable_write_cache(ns->blkconf.blk, dw11 & 1);
6472 }
6473
6474 break;
6475
6476 case NVME_NUMBER_OF_QUEUES:
6477 if (n->qs_created) {
6478 return NVME_CMD_SEQ_ERROR | NVME_DNR;
6479 }
6480
6481 /*
6482 * NVMe v1.3, Section 5.21.1.7: FFFFh is not an allowed value for NCQR
6483 * and NSQR.
6484 */
6485 if ((dw11 & 0xffff) == 0xffff || ((dw11 >> 16) & 0xffff) == 0xffff) {
6486 return NVME_INVALID_FIELD | NVME_DNR;
6487 }
6488
6489 trace_pci_nvme_setfeat_numq((dw11 & 0xffff) + 1,
6490 ((dw11 >> 16) & 0xffff) + 1,
6491 n->conf_ioqpairs,
6492 n->conf_ioqpairs);
6493 req->cqe.result = cpu_to_le32((n->conf_ioqpairs - 1) |
6494 ((n->conf_ioqpairs - 1) << 16));
6495 break;
6496 case NVME_ASYNCHRONOUS_EVENT_CONF:
6497 n->features.async_config = dw11;
6498 break;
6499 case NVME_TIMESTAMP:
6500 return nvme_set_feature_timestamp(n, req);
6501 case NVME_HOST_BEHAVIOR_SUPPORT:
6502 status = nvme_h2c(n, (uint8_t *)&n->features.hbs,
6503 sizeof(n->features.hbs), req);
6504 if (status) {
6505 return status;
6506 }
6507
6508 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6509 ns = nvme_ns(n, i);
6510
6511 if (!ns) {
6512 continue;
6513 }
6514
6515 ns->id_ns.nlbaf = ns->nlbaf - 1;
6516 if (!n->features.hbs.lbafee) {
6517 ns->id_ns.nlbaf = MIN(ns->id_ns.nlbaf, 15);
6518 }
6519 }
6520
6521 return status;
6522 case NVME_COMMAND_SET_PROFILE:
6523 if (dw11 & 0x1ff) {
6524 trace_pci_nvme_err_invalid_iocsci(dw11 & 0x1ff);
6525 return NVME_CMD_SET_CMB_REJECTED | NVME_DNR;
6526 }
6527 break;
6528 case NVME_FDP_MODE:
6529 /* spec: abort with cmd seq err if there's one or more NS' in endgrp */
6530 return NVME_CMD_SEQ_ERROR | NVME_DNR;
6531 case NVME_FDP_EVENTS:
6532 return nvme_set_feature_fdp_events(n, ns, req);
6533 default:
6534 return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
6535 }
6536 return NVME_SUCCESS;
6537 }
6538
nvme_aer(NvmeCtrl * n,NvmeRequest * req)6539 static uint16_t nvme_aer(NvmeCtrl *n, NvmeRequest *req)
6540 {
6541 trace_pci_nvme_aer(nvme_cid(req));
6542
6543 if (n->outstanding_aers > n->params.aerl) {
6544 trace_pci_nvme_aer_aerl_exceeded();
6545 return NVME_AER_LIMIT_EXCEEDED;
6546 }
6547
6548 n->aer_reqs[n->outstanding_aers] = req;
6549 n->outstanding_aers++;
6550
6551 if (!QTAILQ_EMPTY(&n->aer_queue)) {
6552 nvme_process_aers(n);
6553 }
6554
6555 return NVME_NO_COMPLETE;
6556 }
6557
nvme_update_dmrsl(NvmeCtrl * n)6558 static void nvme_update_dmrsl(NvmeCtrl *n)
6559 {
6560 int nsid;
6561
6562 for (nsid = 1; nsid <= NVME_MAX_NAMESPACES; nsid++) {
6563 NvmeNamespace *ns = nvme_ns(n, nsid);
6564 if (!ns) {
6565 continue;
6566 }
6567
6568 n->dmrsl = MIN_NON_ZERO(n->dmrsl,
6569 BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
6570 }
6571 }
6572
nvme_select_iocs_ns(NvmeCtrl * n,NvmeNamespace * ns)6573 static void nvme_select_iocs_ns(NvmeCtrl *n, NvmeNamespace *ns)
6574 {
6575 uint32_t cc = ldl_le_p(&n->bar.cc);
6576
6577 ns->iocs = nvme_cse_iocs_none;
6578 switch (ns->csi) {
6579 case NVME_CSI_NVM:
6580 if (NVME_CC_CSS(cc) != NVME_CC_CSS_ADMIN_ONLY) {
6581 ns->iocs = nvme_cse_iocs_nvm;
6582 }
6583 break;
6584 case NVME_CSI_ZONED:
6585 if (NVME_CC_CSS(cc) == NVME_CC_CSS_CSI) {
6586 ns->iocs = nvme_cse_iocs_zoned;
6587 } else if (NVME_CC_CSS(cc) == NVME_CC_CSS_NVM) {
6588 ns->iocs = nvme_cse_iocs_nvm;
6589 }
6590 break;
6591 }
6592 }
6593
nvme_ns_attachment(NvmeCtrl * n,NvmeRequest * req)6594 static uint16_t nvme_ns_attachment(NvmeCtrl *n, NvmeRequest *req)
6595 {
6596 NvmeNamespace *ns;
6597 NvmeCtrl *ctrl;
6598 uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
6599 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6600 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6601 uint8_t sel = dw10 & 0xf;
6602 uint16_t *nr_ids = &list[0];
6603 uint16_t *ids = &list[1];
6604 uint16_t ret;
6605 int i;
6606
6607 trace_pci_nvme_ns_attachment(nvme_cid(req), dw10 & 0xf);
6608
6609 if (!nvme_nsid_valid(n, nsid)) {
6610 return NVME_INVALID_NSID | NVME_DNR;
6611 }
6612
6613 ns = nvme_subsys_ns(n->subsys, nsid);
6614 if (!ns) {
6615 return NVME_INVALID_FIELD | NVME_DNR;
6616 }
6617
6618 ret = nvme_h2c(n, (uint8_t *)list, 4096, req);
6619 if (ret) {
6620 return ret;
6621 }
6622
6623 if (!*nr_ids) {
6624 return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
6625 }
6626
6627 *nr_ids = MIN(*nr_ids, NVME_CONTROLLER_LIST_SIZE - 1);
6628 for (i = 0; i < *nr_ids; i++) {
6629 ctrl = nvme_subsys_ctrl(n->subsys, ids[i]);
6630 if (!ctrl) {
6631 return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
6632 }
6633
6634 switch (sel) {
6635 case NVME_NS_ATTACHMENT_ATTACH:
6636 if (nvme_ns(ctrl, nsid)) {
6637 return NVME_NS_ALREADY_ATTACHED | NVME_DNR;
6638 }
6639
6640 if (ns->attached && !ns->params.shared) {
6641 return NVME_NS_PRIVATE | NVME_DNR;
6642 }
6643
6644 nvme_attach_ns(ctrl, ns);
6645 nvme_select_iocs_ns(ctrl, ns);
6646
6647 break;
6648
6649 case NVME_NS_ATTACHMENT_DETACH:
6650 if (!nvme_ns(ctrl, nsid)) {
6651 return NVME_NS_NOT_ATTACHED | NVME_DNR;
6652 }
6653
6654 ctrl->namespaces[nsid] = NULL;
6655 ns->attached--;
6656
6657 nvme_update_dmrsl(ctrl);
6658
6659 break;
6660
6661 default:
6662 return NVME_INVALID_FIELD | NVME_DNR;
6663 }
6664
6665 /*
6666 * Add namespace id to the changed namespace id list for event clearing
6667 * via Get Log Page command.
6668 */
6669 if (!test_and_set_bit(nsid, ctrl->changed_nsids)) {
6670 nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE,
6671 NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED,
6672 NVME_LOG_CHANGED_NSLIST);
6673 }
6674 }
6675
6676 return NVME_SUCCESS;
6677 }
6678
6679 typedef struct NvmeFormatAIOCB {
6680 BlockAIOCB common;
6681 BlockAIOCB *aiocb;
6682 NvmeRequest *req;
6683 int ret;
6684
6685 NvmeNamespace *ns;
6686 uint32_t nsid;
6687 bool broadcast;
6688 int64_t offset;
6689
6690 uint8_t lbaf;
6691 uint8_t mset;
6692 uint8_t pi;
6693 uint8_t pil;
6694 } NvmeFormatAIOCB;
6695
nvme_format_cancel(BlockAIOCB * aiocb)6696 static void nvme_format_cancel(BlockAIOCB *aiocb)
6697 {
6698 NvmeFormatAIOCB *iocb = container_of(aiocb, NvmeFormatAIOCB, common);
6699
6700 iocb->ret = -ECANCELED;
6701
6702 if (iocb->aiocb) {
6703 blk_aio_cancel_async(iocb->aiocb);
6704 iocb->aiocb = NULL;
6705 }
6706 }
6707
6708 static const AIOCBInfo nvme_format_aiocb_info = {
6709 .aiocb_size = sizeof(NvmeFormatAIOCB),
6710 .cancel_async = nvme_format_cancel,
6711 };
6712
nvme_format_set(NvmeNamespace * ns,uint8_t lbaf,uint8_t mset,uint8_t pi,uint8_t pil)6713 static void nvme_format_set(NvmeNamespace *ns, uint8_t lbaf, uint8_t mset,
6714 uint8_t pi, uint8_t pil)
6715 {
6716 uint8_t lbafl = lbaf & 0xf;
6717 uint8_t lbafu = lbaf >> 4;
6718
6719 trace_pci_nvme_format_set(ns->params.nsid, lbaf, mset, pi, pil);
6720
6721 ns->id_ns.dps = (pil << 3) | pi;
6722 ns->id_ns.flbas = (lbafu << 5) | (mset << 4) | lbafl;
6723
6724 nvme_ns_init_format(ns);
6725 }
6726
6727 static void nvme_do_format(NvmeFormatAIOCB *iocb);
6728
nvme_format_ns_cb(void * opaque,int ret)6729 static void nvme_format_ns_cb(void *opaque, int ret)
6730 {
6731 NvmeFormatAIOCB *iocb = opaque;
6732 NvmeNamespace *ns = iocb->ns;
6733 int bytes;
6734
6735 if (iocb->ret < 0) {
6736 goto done;
6737 } else if (ret < 0) {
6738 iocb->ret = ret;
6739 goto done;
6740 }
6741
6742 assert(ns);
6743
6744 if (iocb->offset < ns->size) {
6745 bytes = MIN(BDRV_REQUEST_MAX_BYTES, ns->size - iocb->offset);
6746
6747 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, iocb->offset,
6748 bytes, BDRV_REQ_MAY_UNMAP,
6749 nvme_format_ns_cb, iocb);
6750
6751 iocb->offset += bytes;
6752 return;
6753 }
6754
6755 nvme_format_set(ns, iocb->lbaf, iocb->mset, iocb->pi, iocb->pil);
6756 ns->status = 0x0;
6757 iocb->ns = NULL;
6758 iocb->offset = 0;
6759
6760 done:
6761 nvme_do_format(iocb);
6762 }
6763
nvme_format_check(NvmeNamespace * ns,uint8_t lbaf,uint8_t pi)6764 static uint16_t nvme_format_check(NvmeNamespace *ns, uint8_t lbaf, uint8_t pi)
6765 {
6766 if (ns->params.zoned) {
6767 return NVME_INVALID_FORMAT | NVME_DNR;
6768 }
6769
6770 if (lbaf > ns->id_ns.nlbaf) {
6771 return NVME_INVALID_FORMAT | NVME_DNR;
6772 }
6773
6774 if (pi && (ns->id_ns.lbaf[lbaf].ms < nvme_pi_tuple_size(ns))) {
6775 return NVME_INVALID_FORMAT | NVME_DNR;
6776 }
6777
6778 if (pi && pi > NVME_ID_NS_DPS_TYPE_3) {
6779 return NVME_INVALID_FIELD | NVME_DNR;
6780 }
6781
6782 return NVME_SUCCESS;
6783 }
6784
nvme_do_format(NvmeFormatAIOCB * iocb)6785 static void nvme_do_format(NvmeFormatAIOCB *iocb)
6786 {
6787 NvmeRequest *req = iocb->req;
6788 NvmeCtrl *n = nvme_ctrl(req);
6789 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6790 uint8_t lbaf = dw10 & 0xf;
6791 uint8_t pi = (dw10 >> 5) & 0x7;
6792 uint16_t status;
6793 int i;
6794
6795 if (iocb->ret < 0) {
6796 goto done;
6797 }
6798
6799 if (iocb->broadcast) {
6800 for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
6801 iocb->ns = nvme_ns(n, i);
6802 if (iocb->ns) {
6803 iocb->nsid = i;
6804 break;
6805 }
6806 }
6807 }
6808
6809 if (!iocb->ns) {
6810 goto done;
6811 }
6812
6813 status = nvme_format_check(iocb->ns, lbaf, pi);
6814 if (status) {
6815 req->status = status;
6816 goto done;
6817 }
6818
6819 iocb->ns->status = NVME_FORMAT_IN_PROGRESS;
6820 nvme_format_ns_cb(iocb, 0);
6821 return;
6822
6823 done:
6824 iocb->common.cb(iocb->common.opaque, iocb->ret);
6825 qemu_aio_unref(iocb);
6826 }
6827
nvme_format(NvmeCtrl * n,NvmeRequest * req)6828 static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req)
6829 {
6830 NvmeFormatAIOCB *iocb;
6831 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6832 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6833 uint8_t lbaf = dw10 & 0xf;
6834 uint8_t mset = (dw10 >> 4) & 0x1;
6835 uint8_t pi = (dw10 >> 5) & 0x7;
6836 uint8_t pil = (dw10 >> 8) & 0x1;
6837 uint8_t lbafu = (dw10 >> 12) & 0x3;
6838 uint16_t status;
6839
6840 iocb = qemu_aio_get(&nvme_format_aiocb_info, NULL, nvme_misc_cb, req);
6841
6842 iocb->req = req;
6843 iocb->ret = 0;
6844 iocb->ns = NULL;
6845 iocb->nsid = 0;
6846 iocb->lbaf = lbaf;
6847 iocb->mset = mset;
6848 iocb->pi = pi;
6849 iocb->pil = pil;
6850 iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
6851 iocb->offset = 0;
6852
6853 if (n->features.hbs.lbafee) {
6854 iocb->lbaf |= lbafu << 4;
6855 }
6856
6857 if (!iocb->broadcast) {
6858 if (!nvme_nsid_valid(n, nsid)) {
6859 status = NVME_INVALID_NSID | NVME_DNR;
6860 goto out;
6861 }
6862
6863 iocb->ns = nvme_ns(n, nsid);
6864 if (!iocb->ns) {
6865 status = NVME_INVALID_FIELD | NVME_DNR;
6866 goto out;
6867 }
6868 }
6869
6870 req->aiocb = &iocb->common;
6871 nvme_do_format(iocb);
6872
6873 return NVME_NO_COMPLETE;
6874
6875 out:
6876 qemu_aio_unref(iocb);
6877
6878 return status;
6879 }
6880
nvme_get_virt_res_num(NvmeCtrl * n,uint8_t rt,int * num_total,int * num_prim,int * num_sec)6881 static void nvme_get_virt_res_num(NvmeCtrl *n, uint8_t rt, int *num_total,
6882 int *num_prim, int *num_sec)
6883 {
6884 *num_total = le32_to_cpu(rt ?
6885 n->pri_ctrl_cap.vifrt : n->pri_ctrl_cap.vqfrt);
6886 *num_prim = le16_to_cpu(rt ?
6887 n->pri_ctrl_cap.virfap : n->pri_ctrl_cap.vqrfap);
6888 *num_sec = le16_to_cpu(rt ? n->pri_ctrl_cap.virfa : n->pri_ctrl_cap.vqrfa);
6889 }
6890
nvme_assign_virt_res_to_prim(NvmeCtrl * n,NvmeRequest * req,uint16_t cntlid,uint8_t rt,int nr)6891 static uint16_t nvme_assign_virt_res_to_prim(NvmeCtrl *n, NvmeRequest *req,
6892 uint16_t cntlid, uint8_t rt,
6893 int nr)
6894 {
6895 int num_total, num_prim, num_sec;
6896
6897 if (cntlid != n->cntlid) {
6898 return NVME_INVALID_CTRL_ID | NVME_DNR;
6899 }
6900
6901 nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
6902
6903 if (nr > num_total) {
6904 return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
6905 }
6906
6907 if (nr > num_total - num_sec) {
6908 return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6909 }
6910
6911 if (rt) {
6912 n->next_pri_ctrl_cap.virfap = cpu_to_le16(nr);
6913 } else {
6914 n->next_pri_ctrl_cap.vqrfap = cpu_to_le16(nr);
6915 }
6916
6917 req->cqe.result = cpu_to_le32(nr);
6918 return req->status;
6919 }
6920
nvme_update_virt_res(NvmeCtrl * n,NvmeSecCtrlEntry * sctrl,uint8_t rt,int nr)6921 static void nvme_update_virt_res(NvmeCtrl *n, NvmeSecCtrlEntry *sctrl,
6922 uint8_t rt, int nr)
6923 {
6924 int prev_nr, prev_total;
6925
6926 if (rt) {
6927 prev_nr = le16_to_cpu(sctrl->nvi);
6928 prev_total = le32_to_cpu(n->pri_ctrl_cap.virfa);
6929 sctrl->nvi = cpu_to_le16(nr);
6930 n->pri_ctrl_cap.virfa = cpu_to_le32(prev_total + nr - prev_nr);
6931 } else {
6932 prev_nr = le16_to_cpu(sctrl->nvq);
6933 prev_total = le32_to_cpu(n->pri_ctrl_cap.vqrfa);
6934 sctrl->nvq = cpu_to_le16(nr);
6935 n->pri_ctrl_cap.vqrfa = cpu_to_le32(prev_total + nr - prev_nr);
6936 }
6937 }
6938
nvme_assign_virt_res_to_sec(NvmeCtrl * n,NvmeRequest * req,uint16_t cntlid,uint8_t rt,int nr)6939 static uint16_t nvme_assign_virt_res_to_sec(NvmeCtrl *n, NvmeRequest *req,
6940 uint16_t cntlid, uint8_t rt, int nr)
6941 {
6942 int num_total, num_prim, num_sec, num_free, diff, limit;
6943 NvmeSecCtrlEntry *sctrl;
6944
6945 sctrl = nvme_sctrl_for_cntlid(n, cntlid);
6946 if (!sctrl) {
6947 return NVME_INVALID_CTRL_ID | NVME_DNR;
6948 }
6949
6950 if (sctrl->scs) {
6951 return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
6952 }
6953
6954 limit = le16_to_cpu(rt ? n->pri_ctrl_cap.vifrsm : n->pri_ctrl_cap.vqfrsm);
6955 if (nr > limit) {
6956 return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
6957 }
6958
6959 nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
6960 num_free = num_total - num_prim - num_sec;
6961 diff = nr - le16_to_cpu(rt ? sctrl->nvi : sctrl->nvq);
6962
6963 if (diff > num_free) {
6964 return NVME_INVALID_RESOURCE_ID | NVME_DNR;
6965 }
6966
6967 nvme_update_virt_res(n, sctrl, rt, nr);
6968 req->cqe.result = cpu_to_le32(nr);
6969
6970 return req->status;
6971 }
6972
nvme_virt_set_state(NvmeCtrl * n,uint16_t cntlid,bool online)6973 static uint16_t nvme_virt_set_state(NvmeCtrl *n, uint16_t cntlid, bool online)
6974 {
6975 PCIDevice *pci = PCI_DEVICE(n);
6976 NvmeCtrl *sn = NULL;
6977 NvmeSecCtrlEntry *sctrl;
6978 int vf_index;
6979
6980 sctrl = nvme_sctrl_for_cntlid(n, cntlid);
6981 if (!sctrl) {
6982 return NVME_INVALID_CTRL_ID | NVME_DNR;
6983 }
6984
6985 if (!pci_is_vf(pci)) {
6986 vf_index = le16_to_cpu(sctrl->vfn) - 1;
6987 sn = NVME(pcie_sriov_get_vf_at_index(pci, vf_index));
6988 }
6989
6990 if (online) {
6991 if (!sctrl->nvi || (le16_to_cpu(sctrl->nvq) < 2) || !sn) {
6992 return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
6993 }
6994
6995 if (!sctrl->scs) {
6996 sctrl->scs = 0x1;
6997 nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
6998 }
6999 } else {
7000 nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_INTERRUPT, 0);
7001 nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_QUEUE, 0);
7002
7003 if (sctrl->scs) {
7004 sctrl->scs = 0x0;
7005 if (sn) {
7006 nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
7007 }
7008 }
7009 }
7010
7011 return NVME_SUCCESS;
7012 }
7013
nvme_virt_mngmt(NvmeCtrl * n,NvmeRequest * req)7014 static uint16_t nvme_virt_mngmt(NvmeCtrl *n, NvmeRequest *req)
7015 {
7016 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7017 uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
7018 uint8_t act = dw10 & 0xf;
7019 uint8_t rt = (dw10 >> 8) & 0x7;
7020 uint16_t cntlid = (dw10 >> 16) & 0xffff;
7021 int nr = dw11 & 0xffff;
7022
7023 trace_pci_nvme_virt_mngmt(nvme_cid(req), act, cntlid, rt ? "VI" : "VQ", nr);
7024
7025 if (rt != NVME_VIRT_RES_QUEUE && rt != NVME_VIRT_RES_INTERRUPT) {
7026 return NVME_INVALID_RESOURCE_ID | NVME_DNR;
7027 }
7028
7029 switch (act) {
7030 case NVME_VIRT_MNGMT_ACTION_SEC_ASSIGN:
7031 return nvme_assign_virt_res_to_sec(n, req, cntlid, rt, nr);
7032 case NVME_VIRT_MNGMT_ACTION_PRM_ALLOC:
7033 return nvme_assign_virt_res_to_prim(n, req, cntlid, rt, nr);
7034 case NVME_VIRT_MNGMT_ACTION_SEC_ONLINE:
7035 return nvme_virt_set_state(n, cntlid, true);
7036 case NVME_VIRT_MNGMT_ACTION_SEC_OFFLINE:
7037 return nvme_virt_set_state(n, cntlid, false);
7038 default:
7039 return NVME_INVALID_FIELD | NVME_DNR;
7040 }
7041 }
7042
nvme_dbbuf_config(NvmeCtrl * n,const NvmeRequest * req)7043 static uint16_t nvme_dbbuf_config(NvmeCtrl *n, const NvmeRequest *req)
7044 {
7045 PCIDevice *pci = PCI_DEVICE(n);
7046 uint64_t dbs_addr = le64_to_cpu(req->cmd.dptr.prp1);
7047 uint64_t eis_addr = le64_to_cpu(req->cmd.dptr.prp2);
7048 int i;
7049
7050 /* Address should be page aligned */
7051 if (dbs_addr & (n->page_size - 1) || eis_addr & (n->page_size - 1)) {
7052 return NVME_INVALID_FIELD | NVME_DNR;
7053 }
7054
7055 /* Save shadow buffer base addr for use during queue creation */
7056 n->dbbuf_dbs = dbs_addr;
7057 n->dbbuf_eis = eis_addr;
7058 n->dbbuf_enabled = true;
7059
7060 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
7061 NvmeSQueue *sq = n->sq[i];
7062 NvmeCQueue *cq = n->cq[i];
7063
7064 if (sq) {
7065 /*
7066 * CAP.DSTRD is 0, so offset of ith sq db_addr is (i<<3)
7067 * nvme_process_db() uses this hard-coded way to calculate
7068 * doorbell offsets. Be consistent with that here.
7069 */
7070 sq->db_addr = dbs_addr + (i << 3);
7071 sq->ei_addr = eis_addr + (i << 3);
7072 stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED);
7073
7074 if (n->params.ioeventfd && sq->sqid != 0) {
7075 if (!nvme_init_sq_ioeventfd(sq)) {
7076 sq->ioeventfd_enabled = true;
7077 }
7078 }
7079 }
7080
7081 if (cq) {
7082 /* CAP.DSTRD is 0, so offset of ith cq db_addr is (i<<3)+(1<<2) */
7083 cq->db_addr = dbs_addr + (i << 3) + (1 << 2);
7084 cq->ei_addr = eis_addr + (i << 3) + (1 << 2);
7085 stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED);
7086
7087 if (n->params.ioeventfd && cq->cqid != 0) {
7088 if (!nvme_init_cq_ioeventfd(cq)) {
7089 cq->ioeventfd_enabled = true;
7090 }
7091 }
7092 }
7093 }
7094
7095 trace_pci_nvme_dbbuf_config(dbs_addr, eis_addr);
7096
7097 return NVME_SUCCESS;
7098 }
7099
nvme_directive_send(NvmeCtrl * n,NvmeRequest * req)7100 static uint16_t nvme_directive_send(NvmeCtrl *n, NvmeRequest *req)
7101 {
7102 return NVME_INVALID_FIELD | NVME_DNR;
7103 }
7104
nvme_directive_receive(NvmeCtrl * n,NvmeRequest * req)7105 static uint16_t nvme_directive_receive(NvmeCtrl *n, NvmeRequest *req)
7106 {
7107 NvmeNamespace *ns;
7108 uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
7109 uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
7110 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
7111 uint8_t doper, dtype;
7112 uint32_t numd, trans_len;
7113 NvmeDirectiveIdentify id = {
7114 .supported = 1 << NVME_DIRECTIVE_IDENTIFY,
7115 .enabled = 1 << NVME_DIRECTIVE_IDENTIFY,
7116 };
7117
7118 numd = dw10 + 1;
7119 doper = dw11 & 0xff;
7120 dtype = (dw11 >> 8) & 0xff;
7121
7122 trans_len = MIN(sizeof(NvmeDirectiveIdentify), numd << 2);
7123
7124 if (nsid == NVME_NSID_BROADCAST || dtype != NVME_DIRECTIVE_IDENTIFY ||
7125 doper != NVME_DIRECTIVE_RETURN_PARAMS) {
7126 return NVME_INVALID_FIELD | NVME_DNR;
7127 }
7128
7129 ns = nvme_ns(n, nsid);
7130 if (!ns) {
7131 return NVME_INVALID_FIELD | NVME_DNR;
7132 }
7133
7134 switch (dtype) {
7135 case NVME_DIRECTIVE_IDENTIFY:
7136 switch (doper) {
7137 case NVME_DIRECTIVE_RETURN_PARAMS:
7138 if (ns->endgrp && ns->endgrp->fdp.enabled) {
7139 id.supported |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
7140 id.enabled |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
7141 id.persistent |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
7142 }
7143
7144 return nvme_c2h(n, (uint8_t *)&id, trans_len, req);
7145
7146 default:
7147 return NVME_INVALID_FIELD | NVME_DNR;
7148 }
7149
7150 default:
7151 return NVME_INVALID_FIELD;
7152 }
7153 }
7154
nvme_admin_cmd(NvmeCtrl * n,NvmeRequest * req)7155 static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeRequest *req)
7156 {
7157 trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), req->cmd.opcode,
7158 nvme_adm_opc_str(req->cmd.opcode));
7159
7160 if (!(nvme_cse_acs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
7161 trace_pci_nvme_err_invalid_admin_opc(req->cmd.opcode);
7162 return NVME_INVALID_OPCODE | NVME_DNR;
7163 }
7164
7165 /* SGLs shall not be used for Admin commands in NVMe over PCIe */
7166 if (NVME_CMD_FLAGS_PSDT(req->cmd.flags) != NVME_PSDT_PRP) {
7167 return NVME_INVALID_FIELD | NVME_DNR;
7168 }
7169
7170 if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
7171 return NVME_INVALID_FIELD;
7172 }
7173
7174 switch (req->cmd.opcode) {
7175 case NVME_ADM_CMD_DELETE_SQ:
7176 return nvme_del_sq(n, req);
7177 case NVME_ADM_CMD_CREATE_SQ:
7178 return nvme_create_sq(n, req);
7179 case NVME_ADM_CMD_GET_LOG_PAGE:
7180 return nvme_get_log(n, req);
7181 case NVME_ADM_CMD_DELETE_CQ:
7182 return nvme_del_cq(n, req);
7183 case NVME_ADM_CMD_CREATE_CQ:
7184 return nvme_create_cq(n, req);
7185 case NVME_ADM_CMD_IDENTIFY:
7186 return nvme_identify(n, req);
7187 case NVME_ADM_CMD_ABORT:
7188 return nvme_abort(n, req);
7189 case NVME_ADM_CMD_SET_FEATURES:
7190 return nvme_set_feature(n, req);
7191 case NVME_ADM_CMD_GET_FEATURES:
7192 return nvme_get_feature(n, req);
7193 case NVME_ADM_CMD_ASYNC_EV_REQ:
7194 return nvme_aer(n, req);
7195 case NVME_ADM_CMD_NS_ATTACHMENT:
7196 return nvme_ns_attachment(n, req);
7197 case NVME_ADM_CMD_VIRT_MNGMT:
7198 return nvme_virt_mngmt(n, req);
7199 case NVME_ADM_CMD_DBBUF_CONFIG:
7200 return nvme_dbbuf_config(n, req);
7201 case NVME_ADM_CMD_FORMAT_NVM:
7202 return nvme_format(n, req);
7203 case NVME_ADM_CMD_DIRECTIVE_SEND:
7204 return nvme_directive_send(n, req);
7205 case NVME_ADM_CMD_DIRECTIVE_RECV:
7206 return nvme_directive_receive(n, req);
7207 default:
7208 assert(false);
7209 }
7210
7211 return NVME_INVALID_OPCODE | NVME_DNR;
7212 }
7213
nvme_update_sq_eventidx(const NvmeSQueue * sq)7214 static void nvme_update_sq_eventidx(const NvmeSQueue *sq)
7215 {
7216 trace_pci_nvme_update_sq_eventidx(sq->sqid, sq->tail);
7217
7218 stl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->ei_addr, sq->tail,
7219 MEMTXATTRS_UNSPECIFIED);
7220 }
7221
nvme_update_sq_tail(NvmeSQueue * sq)7222 static void nvme_update_sq_tail(NvmeSQueue *sq)
7223 {
7224 ldl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->db_addr, &sq->tail,
7225 MEMTXATTRS_UNSPECIFIED);
7226
7227 trace_pci_nvme_update_sq_tail(sq->sqid, sq->tail);
7228 }
7229
nvme_process_sq(void * opaque)7230 static void nvme_process_sq(void *opaque)
7231 {
7232 NvmeSQueue *sq = opaque;
7233 NvmeCtrl *n = sq->ctrl;
7234 NvmeCQueue *cq = n->cq[sq->cqid];
7235
7236 uint16_t status;
7237 hwaddr addr;
7238 NvmeCmd cmd;
7239 NvmeRequest *req;
7240
7241 if (n->dbbuf_enabled) {
7242 nvme_update_sq_tail(sq);
7243 }
7244
7245 while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
7246 addr = sq->dma_addr + (sq->head << NVME_SQES);
7247 if (nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd))) {
7248 trace_pci_nvme_err_addr_read(addr);
7249 trace_pci_nvme_err_cfs();
7250 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
7251 break;
7252 }
7253 nvme_inc_sq_head(sq);
7254
7255 req = QTAILQ_FIRST(&sq->req_list);
7256 QTAILQ_REMOVE(&sq->req_list, req, entry);
7257 QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
7258 nvme_req_clear(req);
7259 req->cqe.cid = cmd.cid;
7260 memcpy(&req->cmd, &cmd, sizeof(NvmeCmd));
7261
7262 status = sq->sqid ? nvme_io_cmd(n, req) :
7263 nvme_admin_cmd(n, req);
7264 if (status != NVME_NO_COMPLETE) {
7265 req->status = status;
7266 nvme_enqueue_req_completion(cq, req);
7267 }
7268
7269 if (n->dbbuf_enabled) {
7270 nvme_update_sq_eventidx(sq);
7271 nvme_update_sq_tail(sq);
7272 }
7273 }
7274 }
7275
nvme_update_msixcap_ts(PCIDevice * pci_dev,uint32_t table_size)7276 static void nvme_update_msixcap_ts(PCIDevice *pci_dev, uint32_t table_size)
7277 {
7278 uint8_t *config;
7279
7280 if (!msix_present(pci_dev)) {
7281 return;
7282 }
7283
7284 assert(table_size > 0 && table_size <= pci_dev->msix_entries_nr);
7285
7286 config = pci_dev->config + pci_dev->msix_cap;
7287 pci_set_word_by_mask(config + PCI_MSIX_FLAGS, PCI_MSIX_FLAGS_QSIZE,
7288 table_size - 1);
7289 }
7290
nvme_activate_virt_res(NvmeCtrl * n)7291 static void nvme_activate_virt_res(NvmeCtrl *n)
7292 {
7293 PCIDevice *pci_dev = PCI_DEVICE(n);
7294 NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
7295 NvmeSecCtrlEntry *sctrl;
7296
7297 /* -1 to account for the admin queue */
7298 if (pci_is_vf(pci_dev)) {
7299 sctrl = nvme_sctrl(n);
7300 cap->vqprt = sctrl->nvq;
7301 cap->viprt = sctrl->nvi;
7302 n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
7303 n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
7304 } else {
7305 cap->vqrfap = n->next_pri_ctrl_cap.vqrfap;
7306 cap->virfap = n->next_pri_ctrl_cap.virfap;
7307 n->conf_ioqpairs = le16_to_cpu(cap->vqprt) +
7308 le16_to_cpu(cap->vqrfap) - 1;
7309 n->conf_msix_qsize = le16_to_cpu(cap->viprt) +
7310 le16_to_cpu(cap->virfap);
7311 }
7312 }
7313
nvme_ctrl_reset(NvmeCtrl * n,NvmeResetType rst)7314 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst)
7315 {
7316 PCIDevice *pci_dev = PCI_DEVICE(n);
7317 NvmeSecCtrlEntry *sctrl;
7318 NvmeNamespace *ns;
7319 int i;
7320
7321 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7322 ns = nvme_ns(n, i);
7323 if (!ns) {
7324 continue;
7325 }
7326
7327 nvme_ns_drain(ns);
7328 }
7329
7330 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
7331 if (n->sq[i] != NULL) {
7332 nvme_free_sq(n->sq[i], n);
7333 }
7334 }
7335 for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
7336 if (n->cq[i] != NULL) {
7337 nvme_free_cq(n->cq[i], n);
7338 }
7339 }
7340
7341 while (!QTAILQ_EMPTY(&n->aer_queue)) {
7342 NvmeAsyncEvent *event = QTAILQ_FIRST(&n->aer_queue);
7343 QTAILQ_REMOVE(&n->aer_queue, event, entry);
7344 g_free(event);
7345 }
7346
7347 if (n->params.sriov_max_vfs) {
7348 if (!pci_is_vf(pci_dev)) {
7349 for (i = 0; i < n->nr_sec_ctrls; i++) {
7350 sctrl = &n->sec_ctrl_list[i];
7351 nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
7352 }
7353 }
7354
7355 if (rst != NVME_RESET_CONTROLLER) {
7356 nvme_activate_virt_res(n);
7357 }
7358 }
7359
7360 n->aer_queued = 0;
7361 n->aer_mask = 0;
7362 n->outstanding_aers = 0;
7363 n->qs_created = false;
7364
7365 nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
7366
7367 if (pci_is_vf(pci_dev)) {
7368 sctrl = nvme_sctrl(n);
7369
7370 stl_le_p(&n->bar.csts, sctrl->scs ? 0 : NVME_CSTS_FAILED);
7371 } else {
7372 stl_le_p(&n->bar.csts, 0);
7373 }
7374
7375 stl_le_p(&n->bar.intms, 0);
7376 stl_le_p(&n->bar.intmc, 0);
7377 stl_le_p(&n->bar.cc, 0);
7378
7379 n->dbbuf_dbs = 0;
7380 n->dbbuf_eis = 0;
7381 n->dbbuf_enabled = false;
7382 }
7383
nvme_ctrl_shutdown(NvmeCtrl * n)7384 static void nvme_ctrl_shutdown(NvmeCtrl *n)
7385 {
7386 NvmeNamespace *ns;
7387 int i;
7388
7389 if (n->pmr.dev) {
7390 memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
7391 }
7392
7393 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7394 ns = nvme_ns(n, i);
7395 if (!ns) {
7396 continue;
7397 }
7398
7399 nvme_ns_shutdown(ns);
7400 }
7401 }
7402
nvme_select_iocs(NvmeCtrl * n)7403 static void nvme_select_iocs(NvmeCtrl *n)
7404 {
7405 NvmeNamespace *ns;
7406 int i;
7407
7408 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
7409 ns = nvme_ns(n, i);
7410 if (!ns) {
7411 continue;
7412 }
7413
7414 nvme_select_iocs_ns(n, ns);
7415 }
7416 }
7417
nvme_start_ctrl(NvmeCtrl * n)7418 static int nvme_start_ctrl(NvmeCtrl *n)
7419 {
7420 uint64_t cap = ldq_le_p(&n->bar.cap);
7421 uint32_t cc = ldl_le_p(&n->bar.cc);
7422 uint32_t aqa = ldl_le_p(&n->bar.aqa);
7423 uint64_t asq = ldq_le_p(&n->bar.asq);
7424 uint64_t acq = ldq_le_p(&n->bar.acq);
7425 uint32_t page_bits = NVME_CC_MPS(cc) + 12;
7426 uint32_t page_size = 1 << page_bits;
7427 NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
7428
7429 if (pci_is_vf(PCI_DEVICE(n)) && !sctrl->scs) {
7430 trace_pci_nvme_err_startfail_virt_state(le16_to_cpu(sctrl->nvi),
7431 le16_to_cpu(sctrl->nvq));
7432 return -1;
7433 }
7434 if (unlikely(n->cq[0])) {
7435 trace_pci_nvme_err_startfail_cq();
7436 return -1;
7437 }
7438 if (unlikely(n->sq[0])) {
7439 trace_pci_nvme_err_startfail_sq();
7440 return -1;
7441 }
7442 if (unlikely(asq & (page_size - 1))) {
7443 trace_pci_nvme_err_startfail_asq_misaligned(asq);
7444 return -1;
7445 }
7446 if (unlikely(acq & (page_size - 1))) {
7447 trace_pci_nvme_err_startfail_acq_misaligned(acq);
7448 return -1;
7449 }
7450 if (unlikely(!(NVME_CAP_CSS(cap) & (1 << NVME_CC_CSS(cc))))) {
7451 trace_pci_nvme_err_startfail_css(NVME_CC_CSS(cc));
7452 return -1;
7453 }
7454 if (unlikely(NVME_CC_MPS(cc) < NVME_CAP_MPSMIN(cap))) {
7455 trace_pci_nvme_err_startfail_page_too_small(
7456 NVME_CC_MPS(cc),
7457 NVME_CAP_MPSMIN(cap));
7458 return -1;
7459 }
7460 if (unlikely(NVME_CC_MPS(cc) >
7461 NVME_CAP_MPSMAX(cap))) {
7462 trace_pci_nvme_err_startfail_page_too_large(
7463 NVME_CC_MPS(cc),
7464 NVME_CAP_MPSMAX(cap));
7465 return -1;
7466 }
7467 if (unlikely(!NVME_AQA_ASQS(aqa))) {
7468 trace_pci_nvme_err_startfail_asqent_sz_zero();
7469 return -1;
7470 }
7471 if (unlikely(!NVME_AQA_ACQS(aqa))) {
7472 trace_pci_nvme_err_startfail_acqent_sz_zero();
7473 return -1;
7474 }
7475
7476 n->page_bits = page_bits;
7477 n->page_size = page_size;
7478 n->max_prp_ents = n->page_size / sizeof(uint64_t);
7479 nvme_init_cq(&n->admin_cq, n, acq, 0, 0, NVME_AQA_ACQS(aqa) + 1, 1);
7480 nvme_init_sq(&n->admin_sq, n, asq, 0, 0, NVME_AQA_ASQS(aqa) + 1);
7481
7482 nvme_set_timestamp(n, 0ULL);
7483
7484 nvme_select_iocs(n);
7485
7486 return 0;
7487 }
7488
nvme_cmb_enable_regs(NvmeCtrl * n)7489 static void nvme_cmb_enable_regs(NvmeCtrl *n)
7490 {
7491 uint32_t cmbloc = ldl_le_p(&n->bar.cmbloc);
7492 uint32_t cmbsz = ldl_le_p(&n->bar.cmbsz);
7493
7494 NVME_CMBLOC_SET_CDPCILS(cmbloc, 1);
7495 NVME_CMBLOC_SET_CDPMLS(cmbloc, 1);
7496 NVME_CMBLOC_SET_BIR(cmbloc, NVME_CMB_BIR);
7497 stl_le_p(&n->bar.cmbloc, cmbloc);
7498
7499 NVME_CMBSZ_SET_SQS(cmbsz, 1);
7500 NVME_CMBSZ_SET_CQS(cmbsz, 0);
7501 NVME_CMBSZ_SET_LISTS(cmbsz, 1);
7502 NVME_CMBSZ_SET_RDS(cmbsz, 1);
7503 NVME_CMBSZ_SET_WDS(cmbsz, 1);
7504 NVME_CMBSZ_SET_SZU(cmbsz, 2); /* MBs */
7505 NVME_CMBSZ_SET_SZ(cmbsz, n->params.cmb_size_mb);
7506 stl_le_p(&n->bar.cmbsz, cmbsz);
7507 }
7508
nvme_write_bar(NvmeCtrl * n,hwaddr offset,uint64_t data,unsigned size)7509 static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
7510 unsigned size)
7511 {
7512 PCIDevice *pci = PCI_DEVICE(n);
7513 uint64_t cap = ldq_le_p(&n->bar.cap);
7514 uint32_t cc = ldl_le_p(&n->bar.cc);
7515 uint32_t intms = ldl_le_p(&n->bar.intms);
7516 uint32_t csts = ldl_le_p(&n->bar.csts);
7517 uint32_t pmrsts = ldl_le_p(&n->bar.pmrsts);
7518
7519 if (unlikely(offset & (sizeof(uint32_t) - 1))) {
7520 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32,
7521 "MMIO write not 32-bit aligned,"
7522 " offset=0x%"PRIx64"", offset);
7523 /* should be ignored, fall through for now */
7524 }
7525
7526 if (unlikely(size < sizeof(uint32_t))) {
7527 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall,
7528 "MMIO write smaller than 32-bits,"
7529 " offset=0x%"PRIx64", size=%u",
7530 offset, size);
7531 /* should be ignored, fall through for now */
7532 }
7533
7534 switch (offset) {
7535 case NVME_REG_INTMS:
7536 if (unlikely(msix_enabled(pci))) {
7537 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
7538 "undefined access to interrupt mask set"
7539 " when MSI-X is enabled");
7540 /* should be ignored, fall through for now */
7541 }
7542 intms |= data;
7543 stl_le_p(&n->bar.intms, intms);
7544 n->bar.intmc = n->bar.intms;
7545 trace_pci_nvme_mmio_intm_set(data & 0xffffffff, intms);
7546 nvme_irq_check(n);
7547 break;
7548 case NVME_REG_INTMC:
7549 if (unlikely(msix_enabled(pci))) {
7550 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
7551 "undefined access to interrupt mask clr"
7552 " when MSI-X is enabled");
7553 /* should be ignored, fall through for now */
7554 }
7555 intms &= ~data;
7556 stl_le_p(&n->bar.intms, intms);
7557 n->bar.intmc = n->bar.intms;
7558 trace_pci_nvme_mmio_intm_clr(data & 0xffffffff, intms);
7559 nvme_irq_check(n);
7560 break;
7561 case NVME_REG_CC:
7562 stl_le_p(&n->bar.cc, data);
7563
7564 trace_pci_nvme_mmio_cfg(data & 0xffffffff);
7565
7566 if (NVME_CC_SHN(data) && !(NVME_CC_SHN(cc))) {
7567 trace_pci_nvme_mmio_shutdown_set();
7568 nvme_ctrl_shutdown(n);
7569 csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
7570 csts |= NVME_CSTS_SHST_COMPLETE;
7571 } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(cc)) {
7572 trace_pci_nvme_mmio_shutdown_cleared();
7573 csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
7574 }
7575
7576 if (NVME_CC_EN(data) && !NVME_CC_EN(cc)) {
7577 if (unlikely(nvme_start_ctrl(n))) {
7578 trace_pci_nvme_err_startfail();
7579 csts = NVME_CSTS_FAILED;
7580 } else {
7581 trace_pci_nvme_mmio_start_success();
7582 csts = NVME_CSTS_READY;
7583 }
7584 } else if (!NVME_CC_EN(data) && NVME_CC_EN(cc)) {
7585 trace_pci_nvme_mmio_stopped();
7586 nvme_ctrl_reset(n, NVME_RESET_CONTROLLER);
7587
7588 break;
7589 }
7590
7591 stl_le_p(&n->bar.csts, csts);
7592
7593 break;
7594 case NVME_REG_CSTS:
7595 if (data & (1 << 4)) {
7596 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported,
7597 "attempted to W1C CSTS.NSSRO"
7598 " but CAP.NSSRS is zero (not supported)");
7599 } else if (data != 0) {
7600 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts,
7601 "attempted to set a read only bit"
7602 " of controller status");
7603 }
7604 break;
7605 case NVME_REG_NSSR:
7606 if (data == 0x4e564d65) {
7607 trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
7608 } else {
7609 /* The spec says that writes of other values have no effect */
7610 return;
7611 }
7612 break;
7613 case NVME_REG_AQA:
7614 stl_le_p(&n->bar.aqa, data);
7615 trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
7616 break;
7617 case NVME_REG_ASQ:
7618 stn_le_p(&n->bar.asq, size, data);
7619 trace_pci_nvme_mmio_asqaddr(data);
7620 break;
7621 case NVME_REG_ASQ + 4:
7622 stl_le_p((uint8_t *)&n->bar.asq + 4, data);
7623 trace_pci_nvme_mmio_asqaddr_hi(data, ldq_le_p(&n->bar.asq));
7624 break;
7625 case NVME_REG_ACQ:
7626 trace_pci_nvme_mmio_acqaddr(data);
7627 stn_le_p(&n->bar.acq, size, data);
7628 break;
7629 case NVME_REG_ACQ + 4:
7630 stl_le_p((uint8_t *)&n->bar.acq + 4, data);
7631 trace_pci_nvme_mmio_acqaddr_hi(data, ldq_le_p(&n->bar.acq));
7632 break;
7633 case NVME_REG_CMBLOC:
7634 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved,
7635 "invalid write to reserved CMBLOC"
7636 " when CMBSZ is zero, ignored");
7637 return;
7638 case NVME_REG_CMBSZ:
7639 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly,
7640 "invalid write to read only CMBSZ, ignored");
7641 return;
7642 case NVME_REG_CMBMSC:
7643 if (!NVME_CAP_CMBS(cap)) {
7644 return;
7645 }
7646
7647 stn_le_p(&n->bar.cmbmsc, size, data);
7648 n->cmb.cmse = false;
7649
7650 if (NVME_CMBMSC_CRE(data)) {
7651 nvme_cmb_enable_regs(n);
7652
7653 if (NVME_CMBMSC_CMSE(data)) {
7654 uint64_t cmbmsc = ldq_le_p(&n->bar.cmbmsc);
7655 hwaddr cba = NVME_CMBMSC_CBA(cmbmsc) << CMBMSC_CBA_SHIFT;
7656 if (cba + int128_get64(n->cmb.mem.size) < cba) {
7657 uint32_t cmbsts = ldl_le_p(&n->bar.cmbsts);
7658 NVME_CMBSTS_SET_CBAI(cmbsts, 1);
7659 stl_le_p(&n->bar.cmbsts, cmbsts);
7660 return;
7661 }
7662
7663 n->cmb.cba = cba;
7664 n->cmb.cmse = true;
7665 }
7666 } else {
7667 n->bar.cmbsz = 0;
7668 n->bar.cmbloc = 0;
7669 }
7670
7671 return;
7672 case NVME_REG_CMBMSC + 4:
7673 stl_le_p((uint8_t *)&n->bar.cmbmsc + 4, data);
7674 return;
7675
7676 case NVME_REG_PMRCAP:
7677 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly,
7678 "invalid write to PMRCAP register, ignored");
7679 return;
7680 case NVME_REG_PMRCTL:
7681 if (!NVME_CAP_PMRS(cap)) {
7682 return;
7683 }
7684
7685 stl_le_p(&n->bar.pmrctl, data);
7686 if (NVME_PMRCTL_EN(data)) {
7687 memory_region_set_enabled(&n->pmr.dev->mr, true);
7688 pmrsts = 0;
7689 } else {
7690 memory_region_set_enabled(&n->pmr.dev->mr, false);
7691 NVME_PMRSTS_SET_NRDY(pmrsts, 1);
7692 n->pmr.cmse = false;
7693 }
7694 stl_le_p(&n->bar.pmrsts, pmrsts);
7695 return;
7696 case NVME_REG_PMRSTS:
7697 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly,
7698 "invalid write to PMRSTS register, ignored");
7699 return;
7700 case NVME_REG_PMREBS:
7701 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly,
7702 "invalid write to PMREBS register, ignored");
7703 return;
7704 case NVME_REG_PMRSWTP:
7705 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly,
7706 "invalid write to PMRSWTP register, ignored");
7707 return;
7708 case NVME_REG_PMRMSCL:
7709 if (!NVME_CAP_PMRS(cap)) {
7710 return;
7711 }
7712
7713 stl_le_p(&n->bar.pmrmscl, data);
7714 n->pmr.cmse = false;
7715
7716 if (NVME_PMRMSCL_CMSE(data)) {
7717 uint64_t pmrmscu = ldl_le_p(&n->bar.pmrmscu);
7718 hwaddr cba = pmrmscu << 32 |
7719 (NVME_PMRMSCL_CBA(data) << PMRMSCL_CBA_SHIFT);
7720 if (cba + int128_get64(n->pmr.dev->mr.size) < cba) {
7721 NVME_PMRSTS_SET_CBAI(pmrsts, 1);
7722 stl_le_p(&n->bar.pmrsts, pmrsts);
7723 return;
7724 }
7725
7726 n->pmr.cmse = true;
7727 n->pmr.cba = cba;
7728 }
7729
7730 return;
7731 case NVME_REG_PMRMSCU:
7732 if (!NVME_CAP_PMRS(cap)) {
7733 return;
7734 }
7735
7736 stl_le_p(&n->bar.pmrmscu, data);
7737 return;
7738 default:
7739 NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid,
7740 "invalid MMIO write,"
7741 " offset=0x%"PRIx64", data=%"PRIx64"",
7742 offset, data);
7743 break;
7744 }
7745 }
7746
nvme_mmio_read(void * opaque,hwaddr addr,unsigned size)7747 static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
7748 {
7749 NvmeCtrl *n = (NvmeCtrl *)opaque;
7750 uint8_t *ptr = (uint8_t *)&n->bar;
7751
7752 trace_pci_nvme_mmio_read(addr, size);
7753
7754 if (unlikely(addr & (sizeof(uint32_t) - 1))) {
7755 NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32,
7756 "MMIO read not 32-bit aligned,"
7757 " offset=0x%"PRIx64"", addr);
7758 /* should RAZ, fall through for now */
7759 } else if (unlikely(size < sizeof(uint32_t))) {
7760 NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall,
7761 "MMIO read smaller than 32-bits,"
7762 " offset=0x%"PRIx64"", addr);
7763 /* should RAZ, fall through for now */
7764 }
7765
7766 if (addr > sizeof(n->bar) - size) {
7767 NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs,
7768 "MMIO read beyond last register,"
7769 " offset=0x%"PRIx64", returning 0", addr);
7770
7771 return 0;
7772 }
7773
7774 if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
7775 addr != NVME_REG_CSTS) {
7776 trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
7777 return 0;
7778 }
7779
7780 /*
7781 * When PMRWBM bit 1 is set then read from
7782 * from PMRSTS should ensure prior writes
7783 * made it to persistent media
7784 */
7785 if (addr == NVME_REG_PMRSTS &&
7786 (NVME_PMRCAP_PMRWBM(ldl_le_p(&n->bar.pmrcap)) & 0x02)) {
7787 memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
7788 }
7789
7790 return ldn_le_p(ptr + addr, size);
7791 }
7792
nvme_process_db(NvmeCtrl * n,hwaddr addr,int val)7793 static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
7794 {
7795 PCIDevice *pci = PCI_DEVICE(n);
7796 uint32_t qid;
7797
7798 if (unlikely(addr & ((1 << 2) - 1))) {
7799 NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned,
7800 "doorbell write not 32-bit aligned,"
7801 " offset=0x%"PRIx64", ignoring", addr);
7802 return;
7803 }
7804
7805 if (((addr - 0x1000) >> 2) & 1) {
7806 /* Completion queue doorbell write */
7807
7808 uint16_t new_head = val & 0xffff;
7809 int start_sqs;
7810 NvmeCQueue *cq;
7811
7812 qid = (addr - (0x1000 + (1 << 2))) >> 3;
7813 if (unlikely(nvme_check_cqid(n, qid))) {
7814 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq,
7815 "completion queue doorbell write"
7816 " for nonexistent queue,"
7817 " sqid=%"PRIu32", ignoring", qid);
7818
7819 /*
7820 * NVM Express v1.3d, Section 4.1 state: "If host software writes
7821 * an invalid value to the Submission Queue Tail Doorbell or
7822 * Completion Queue Head Doorbell register and an Asynchronous Event
7823 * Request command is outstanding, then an asynchronous event is
7824 * posted to the Admin Completion Queue with a status code of
7825 * Invalid Doorbell Write Value."
7826 *
7827 * Also note that the spec includes the "Invalid Doorbell Register"
7828 * status code, but nowhere does it specify when to use it.
7829 * However, it seems reasonable to use it here in a similar
7830 * fashion.
7831 */
7832 if (n->outstanding_aers) {
7833 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7834 NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
7835 NVME_LOG_ERROR_INFO);
7836 }
7837
7838 return;
7839 }
7840
7841 cq = n->cq[qid];
7842 if (unlikely(new_head >= cq->size)) {
7843 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead,
7844 "completion queue doorbell write value"
7845 " beyond queue size, sqid=%"PRIu32","
7846 " new_head=%"PRIu16", ignoring",
7847 qid, new_head);
7848
7849 if (n->outstanding_aers) {
7850 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7851 NVME_AER_INFO_ERR_INVALID_DB_VALUE,
7852 NVME_LOG_ERROR_INFO);
7853 }
7854
7855 return;
7856 }
7857
7858 trace_pci_nvme_mmio_doorbell_cq(cq->cqid, new_head);
7859
7860 start_sqs = nvme_cq_full(cq) ? 1 : 0;
7861 cq->head = new_head;
7862 if (!qid && n->dbbuf_enabled) {
7863 stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED);
7864 }
7865 if (start_sqs) {
7866 NvmeSQueue *sq;
7867 QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
7868 qemu_bh_schedule(sq->bh);
7869 }
7870 qemu_bh_schedule(cq->bh);
7871 }
7872
7873 if (cq->tail == cq->head) {
7874 if (cq->irq_enabled) {
7875 n->cq_pending--;
7876 }
7877
7878 nvme_irq_deassert(n, cq);
7879 }
7880 } else {
7881 /* Submission queue doorbell write */
7882
7883 uint16_t new_tail = val & 0xffff;
7884 NvmeSQueue *sq;
7885
7886 qid = (addr - 0x1000) >> 3;
7887 if (unlikely(nvme_check_sqid(n, qid))) {
7888 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq,
7889 "submission queue doorbell write"
7890 " for nonexistent queue,"
7891 " sqid=%"PRIu32", ignoring", qid);
7892
7893 if (n->outstanding_aers) {
7894 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7895 NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
7896 NVME_LOG_ERROR_INFO);
7897 }
7898
7899 return;
7900 }
7901
7902 sq = n->sq[qid];
7903 if (unlikely(new_tail >= sq->size)) {
7904 NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail,
7905 "submission queue doorbell write value"
7906 " beyond queue size, sqid=%"PRIu32","
7907 " new_tail=%"PRIu16", ignoring",
7908 qid, new_tail);
7909
7910 if (n->outstanding_aers) {
7911 nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
7912 NVME_AER_INFO_ERR_INVALID_DB_VALUE,
7913 NVME_LOG_ERROR_INFO);
7914 }
7915
7916 return;
7917 }
7918
7919 trace_pci_nvme_mmio_doorbell_sq(sq->sqid, new_tail);
7920
7921 sq->tail = new_tail;
7922 if (!qid && n->dbbuf_enabled) {
7923 /*
7924 * The spec states "the host shall also update the controller's
7925 * corresponding doorbell property to match the value of that entry
7926 * in the Shadow Doorbell buffer."
7927 *
7928 * Since this context is currently a VM trap, we can safely enforce
7929 * the requirement from the device side in case the host is
7930 * misbehaving.
7931 *
7932 * Note, we shouldn't have to do this, but various drivers
7933 * including ones that run on Linux, are not updating Admin Queues,
7934 * so we can't trust reading it for an appropriate sq tail.
7935 */
7936 stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED);
7937 }
7938
7939 qemu_bh_schedule(sq->bh);
7940 }
7941 }
7942
nvme_mmio_write(void * opaque,hwaddr addr,uint64_t data,unsigned size)7943 static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
7944 unsigned size)
7945 {
7946 NvmeCtrl *n = (NvmeCtrl *)opaque;
7947
7948 trace_pci_nvme_mmio_write(addr, data, size);
7949
7950 if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
7951 addr != NVME_REG_CSTS) {
7952 trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
7953 return;
7954 }
7955
7956 if (addr < sizeof(n->bar)) {
7957 nvme_write_bar(n, addr, data, size);
7958 } else {
7959 nvme_process_db(n, addr, data);
7960 }
7961 }
7962
7963 static const MemoryRegionOps nvme_mmio_ops = {
7964 .read = nvme_mmio_read,
7965 .write = nvme_mmio_write,
7966 .endianness = DEVICE_LITTLE_ENDIAN,
7967 .impl = {
7968 .min_access_size = 2,
7969 .max_access_size = 8,
7970 },
7971 };
7972
nvme_cmb_write(void * opaque,hwaddr addr,uint64_t data,unsigned size)7973 static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data,
7974 unsigned size)
7975 {
7976 NvmeCtrl *n = (NvmeCtrl *)opaque;
7977 stn_le_p(&n->cmb.buf[addr], size, data);
7978 }
7979
nvme_cmb_read(void * opaque,hwaddr addr,unsigned size)7980 static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size)
7981 {
7982 NvmeCtrl *n = (NvmeCtrl *)opaque;
7983 return ldn_le_p(&n->cmb.buf[addr], size);
7984 }
7985
7986 static const MemoryRegionOps nvme_cmb_ops = {
7987 .read = nvme_cmb_read,
7988 .write = nvme_cmb_write,
7989 .endianness = DEVICE_LITTLE_ENDIAN,
7990 .impl = {
7991 .min_access_size = 1,
7992 .max_access_size = 8,
7993 },
7994 };
7995
nvme_check_params(NvmeCtrl * n,Error ** errp)7996 static bool nvme_check_params(NvmeCtrl *n, Error **errp)
7997 {
7998 NvmeParams *params = &n->params;
7999
8000 if (params->num_queues) {
8001 warn_report("num_queues is deprecated; please use max_ioqpairs "
8002 "instead");
8003
8004 params->max_ioqpairs = params->num_queues - 1;
8005 }
8006
8007 if (n->namespace.blkconf.blk && n->subsys) {
8008 error_setg(errp, "subsystem support is unavailable with legacy "
8009 "namespace ('drive' property)");
8010 return false;
8011 }
8012
8013 if (params->max_ioqpairs < 1 ||
8014 params->max_ioqpairs > NVME_MAX_IOQPAIRS) {
8015 error_setg(errp, "max_ioqpairs must be between 1 and %d",
8016 NVME_MAX_IOQPAIRS);
8017 return false;
8018 }
8019
8020 if (params->msix_qsize < 1 ||
8021 params->msix_qsize > PCI_MSIX_FLAGS_QSIZE + 1) {
8022 error_setg(errp, "msix_qsize must be between 1 and %d",
8023 PCI_MSIX_FLAGS_QSIZE + 1);
8024 return false;
8025 }
8026
8027 if (!params->serial) {
8028 error_setg(errp, "serial property not set");
8029 return false;
8030 }
8031
8032 if (params->mqes < 1) {
8033 error_setg(errp, "mqes property cannot be less than 1");
8034 return false;
8035 }
8036
8037 if (n->pmr.dev) {
8038 if (params->msix_exclusive_bar) {
8039 error_setg(errp, "not enough BARs available to enable PMR");
8040 return false;
8041 }
8042
8043 if (host_memory_backend_is_mapped(n->pmr.dev)) {
8044 error_setg(errp, "can't use already busy memdev: %s",
8045 object_get_canonical_path_component(OBJECT(n->pmr.dev)));
8046 return false;
8047 }
8048
8049 if (!is_power_of_2(n->pmr.dev->size)) {
8050 error_setg(errp, "pmr backend size needs to be power of 2 in size");
8051 return false;
8052 }
8053
8054 host_memory_backend_set_mapped(n->pmr.dev, true);
8055 }
8056
8057 if (n->params.zasl > n->params.mdts) {
8058 error_setg(errp, "zoned.zasl (Zone Append Size Limit) must be less "
8059 "than or equal to mdts (Maximum Data Transfer Size)");
8060 return false;
8061 }
8062
8063 if (!n->params.vsl) {
8064 error_setg(errp, "vsl must be non-zero");
8065 return false;
8066 }
8067
8068 if (params->sriov_max_vfs) {
8069 if (!n->subsys) {
8070 error_setg(errp, "subsystem is required for the use of SR-IOV");
8071 return false;
8072 }
8073
8074 if (params->cmb_size_mb) {
8075 error_setg(errp, "CMB is not supported with SR-IOV");
8076 return false;
8077 }
8078
8079 if (n->pmr.dev) {
8080 error_setg(errp, "PMR is not supported with SR-IOV");
8081 return false;
8082 }
8083
8084 if (!params->sriov_vq_flexible || !params->sriov_vi_flexible) {
8085 error_setg(errp, "both sriov_vq_flexible and sriov_vi_flexible"
8086 " must be set for the use of SR-IOV");
8087 return false;
8088 }
8089
8090 if (params->sriov_vq_flexible < params->sriov_max_vfs * 2) {
8091 error_setg(errp, "sriov_vq_flexible must be greater than or equal"
8092 " to %d (sriov_max_vfs * 2)", params->sriov_max_vfs * 2);
8093 return false;
8094 }
8095
8096 if (params->max_ioqpairs < params->sriov_vq_flexible + 2) {
8097 error_setg(errp, "(max_ioqpairs - sriov_vq_flexible) must be"
8098 " greater than or equal to 2");
8099 return false;
8100 }
8101
8102 if (params->sriov_vi_flexible < params->sriov_max_vfs) {
8103 error_setg(errp, "sriov_vi_flexible must be greater than or equal"
8104 " to %d (sriov_max_vfs)", params->sriov_max_vfs);
8105 return false;
8106 }
8107
8108 if (params->msix_qsize < params->sriov_vi_flexible + 1) {
8109 error_setg(errp, "(msix_qsize - sriov_vi_flexible) must be"
8110 " greater than or equal to 1");
8111 return false;
8112 }
8113
8114 if (params->sriov_max_vi_per_vf &&
8115 (params->sriov_max_vi_per_vf - 1) % NVME_VF_RES_GRANULARITY) {
8116 error_setg(errp, "sriov_max_vi_per_vf must meet:"
8117 " (sriov_max_vi_per_vf - 1) %% %d == 0 and"
8118 " sriov_max_vi_per_vf >= 1", NVME_VF_RES_GRANULARITY);
8119 return false;
8120 }
8121
8122 if (params->sriov_max_vq_per_vf &&
8123 (params->sriov_max_vq_per_vf < 2 ||
8124 (params->sriov_max_vq_per_vf - 1) % NVME_VF_RES_GRANULARITY)) {
8125 error_setg(errp, "sriov_max_vq_per_vf must meet:"
8126 " (sriov_max_vq_per_vf - 1) %% %d == 0 and"
8127 " sriov_max_vq_per_vf >= 2", NVME_VF_RES_GRANULARITY);
8128 return false;
8129 }
8130 }
8131
8132 return true;
8133 }
8134
nvme_init_state(NvmeCtrl * n)8135 static void nvme_init_state(NvmeCtrl *n)
8136 {
8137 NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
8138 NvmeSecCtrlEntry *list = n->sec_ctrl_list;
8139 NvmeSecCtrlEntry *sctrl;
8140 PCIDevice *pci = PCI_DEVICE(n);
8141 uint8_t max_vfs;
8142 int i;
8143
8144 if (pci_is_vf(pci)) {
8145 sctrl = nvme_sctrl(n);
8146 max_vfs = 0;
8147 n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
8148 n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
8149 } else {
8150 max_vfs = n->params.sriov_max_vfs;
8151 n->conf_ioqpairs = n->params.max_ioqpairs;
8152 n->conf_msix_qsize = n->params.msix_qsize;
8153 }
8154
8155 n->sq = g_new0(NvmeSQueue *, n->params.max_ioqpairs + 1);
8156 n->cq = g_new0(NvmeCQueue *, n->params.max_ioqpairs + 1);
8157 n->temperature = NVME_TEMPERATURE;
8158 n->features.temp_thresh_hi = NVME_TEMPERATURE_WARNING;
8159 n->starttime_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
8160 n->aer_reqs = g_new0(NvmeRequest *, n->params.aerl + 1);
8161 QTAILQ_INIT(&n->aer_queue);
8162
8163 n->nr_sec_ctrls = max_vfs;
8164 for (i = 0; i < max_vfs; i++) {
8165 sctrl = &list[i];
8166 sctrl->pcid = cpu_to_le16(n->cntlid);
8167 sctrl->vfn = cpu_to_le16(i + 1);
8168 }
8169
8170 cap->cntlid = cpu_to_le16(n->cntlid);
8171 cap->crt = NVME_CRT_VQ | NVME_CRT_VI;
8172
8173 if (pci_is_vf(pci)) {
8174 cap->vqprt = cpu_to_le16(1 + n->conf_ioqpairs);
8175 } else {
8176 cap->vqprt = cpu_to_le16(1 + n->params.max_ioqpairs -
8177 n->params.sriov_vq_flexible);
8178 cap->vqfrt = cpu_to_le32(n->params.sriov_vq_flexible);
8179 cap->vqrfap = cap->vqfrt;
8180 cap->vqgran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
8181 cap->vqfrsm = n->params.sriov_max_vq_per_vf ?
8182 cpu_to_le16(n->params.sriov_max_vq_per_vf) :
8183 cap->vqfrt / MAX(max_vfs, 1);
8184 }
8185
8186 if (pci_is_vf(pci)) {
8187 cap->viprt = cpu_to_le16(n->conf_msix_qsize);
8188 } else {
8189 cap->viprt = cpu_to_le16(n->params.msix_qsize -
8190 n->params.sriov_vi_flexible);
8191 cap->vifrt = cpu_to_le32(n->params.sriov_vi_flexible);
8192 cap->virfap = cap->vifrt;
8193 cap->vigran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
8194 cap->vifrsm = n->params.sriov_max_vi_per_vf ?
8195 cpu_to_le16(n->params.sriov_max_vi_per_vf) :
8196 cap->vifrt / MAX(max_vfs, 1);
8197 }
8198 }
8199
nvme_init_cmb(NvmeCtrl * n,PCIDevice * pci_dev)8200 static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
8201 {
8202 uint64_t cmb_size = n->params.cmb_size_mb * MiB;
8203 uint64_t cap = ldq_le_p(&n->bar.cap);
8204
8205 n->cmb.buf = g_malloc0(cmb_size);
8206 memory_region_init_io(&n->cmb.mem, OBJECT(n), &nvme_cmb_ops, n,
8207 "nvme-cmb", cmb_size);
8208 pci_register_bar(pci_dev, NVME_CMB_BIR,
8209 PCI_BASE_ADDRESS_SPACE_MEMORY |
8210 PCI_BASE_ADDRESS_MEM_TYPE_64 |
8211 PCI_BASE_ADDRESS_MEM_PREFETCH, &n->cmb.mem);
8212
8213 NVME_CAP_SET_CMBS(cap, 1);
8214 stq_le_p(&n->bar.cap, cap);
8215
8216 if (n->params.legacy_cmb) {
8217 nvme_cmb_enable_regs(n);
8218 n->cmb.cmse = true;
8219 }
8220 }
8221
nvme_init_pmr(NvmeCtrl * n,PCIDevice * pci_dev)8222 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
8223 {
8224 uint32_t pmrcap = ldl_le_p(&n->bar.pmrcap);
8225
8226 NVME_PMRCAP_SET_RDS(pmrcap, 1);
8227 NVME_PMRCAP_SET_WDS(pmrcap, 1);
8228 NVME_PMRCAP_SET_BIR(pmrcap, NVME_PMR_BIR);
8229 /* Turn on bit 1 support */
8230 NVME_PMRCAP_SET_PMRWBM(pmrcap, 0x02);
8231 NVME_PMRCAP_SET_CMSS(pmrcap, 1);
8232 stl_le_p(&n->bar.pmrcap, pmrcap);
8233
8234 pci_register_bar(pci_dev, NVME_PMR_BIR,
8235 PCI_BASE_ADDRESS_SPACE_MEMORY |
8236 PCI_BASE_ADDRESS_MEM_TYPE_64 |
8237 PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmr.dev->mr);
8238
8239 memory_region_set_enabled(&n->pmr.dev->mr, false);
8240 }
8241
nvme_mbar_size(unsigned total_queues,unsigned total_irqs,unsigned * msix_table_offset,unsigned * msix_pba_offset)8242 static uint64_t nvme_mbar_size(unsigned total_queues, unsigned total_irqs,
8243 unsigned *msix_table_offset,
8244 unsigned *msix_pba_offset)
8245 {
8246 uint64_t bar_size, msix_table_size;
8247
8248 bar_size = sizeof(NvmeBar) + 2 * total_queues * NVME_DB_SIZE;
8249
8250 if (total_irqs == 0) {
8251 goto out;
8252 }
8253
8254 bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8255
8256 if (msix_table_offset) {
8257 *msix_table_offset = bar_size;
8258 }
8259
8260 msix_table_size = PCI_MSIX_ENTRY_SIZE * total_irqs;
8261 bar_size += msix_table_size;
8262 bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8263
8264 if (msix_pba_offset) {
8265 *msix_pba_offset = bar_size;
8266 }
8267
8268 bar_size += QEMU_ALIGN_UP(total_irqs, 64) / 8;
8269
8270 out:
8271 return pow2ceil(bar_size);
8272 }
8273
nvme_init_sriov(NvmeCtrl * n,PCIDevice * pci_dev,uint16_t offset)8274 static void nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset)
8275 {
8276 uint16_t vf_dev_id = n->params.use_intel_id ?
8277 PCI_DEVICE_ID_INTEL_NVME : PCI_DEVICE_ID_REDHAT_NVME;
8278 NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
8279 uint64_t bar_size = nvme_mbar_size(le16_to_cpu(cap->vqfrsm),
8280 le16_to_cpu(cap->vifrsm),
8281 NULL, NULL);
8282
8283 pcie_sriov_pf_init(pci_dev, offset, "nvme", vf_dev_id,
8284 n->params.sriov_max_vfs, n->params.sriov_max_vfs,
8285 NVME_VF_OFFSET, NVME_VF_STRIDE);
8286
8287 pcie_sriov_pf_init_vf_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
8288 PCI_BASE_ADDRESS_MEM_TYPE_64, bar_size);
8289 }
8290
nvme_add_pm_capability(PCIDevice * pci_dev,uint8_t offset)8291 static int nvme_add_pm_capability(PCIDevice *pci_dev, uint8_t offset)
8292 {
8293 Error *err = NULL;
8294 int ret;
8295
8296 ret = pci_add_capability(pci_dev, PCI_CAP_ID_PM, offset,
8297 PCI_PM_SIZEOF, &err);
8298 if (err) {
8299 error_report_err(err);
8300 return ret;
8301 }
8302
8303 pci_set_word(pci_dev->config + offset + PCI_PM_PMC,
8304 PCI_PM_CAP_VER_1_2);
8305 pci_set_word(pci_dev->config + offset + PCI_PM_CTRL,
8306 PCI_PM_CTRL_NO_SOFT_RESET);
8307 pci_set_word(pci_dev->wmask + offset + PCI_PM_CTRL,
8308 PCI_PM_CTRL_STATE_MASK);
8309
8310 return 0;
8311 }
8312
pcie_doe_spdm_rsp(DOECap * doe_cap)8313 static bool pcie_doe_spdm_rsp(DOECap *doe_cap)
8314 {
8315 void *req = pcie_doe_get_write_mbox_ptr(doe_cap);
8316 uint32_t req_len = pcie_doe_get_obj_len(req) * 4;
8317 void *rsp = doe_cap->read_mbox;
8318 uint32_t rsp_len = SPDM_SOCKET_MAX_MESSAGE_BUFFER_SIZE;
8319
8320 uint32_t recvd = spdm_socket_rsp(doe_cap->spdm_socket,
8321 SPDM_SOCKET_TRANSPORT_TYPE_PCI_DOE,
8322 req, req_len, rsp, rsp_len);
8323 doe_cap->read_mbox_len += DIV_ROUND_UP(recvd, 4);
8324
8325 return recvd != 0;
8326 }
8327
8328 static DOEProtocol doe_spdm_prot[] = {
8329 { PCI_VENDOR_ID_PCI_SIG, PCI_SIG_DOE_CMA, pcie_doe_spdm_rsp },
8330 { PCI_VENDOR_ID_PCI_SIG, PCI_SIG_DOE_SECURED_CMA, pcie_doe_spdm_rsp },
8331 { }
8332 };
8333
nvme_init_pci(NvmeCtrl * n,PCIDevice * pci_dev,Error ** errp)8334 static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
8335 {
8336 ERRP_GUARD();
8337 uint8_t *pci_conf = pci_dev->config;
8338 uint64_t bar_size;
8339 unsigned msix_table_offset = 0, msix_pba_offset = 0;
8340 unsigned nr_vectors;
8341 int ret;
8342
8343 pci_conf[PCI_INTERRUPT_PIN] = 1;
8344 pci_config_set_prog_interface(pci_conf, 0x2);
8345
8346 if (n->params.use_intel_id) {
8347 pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL);
8348 pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_INTEL_NVME);
8349 } else {
8350 pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_REDHAT);
8351 pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_REDHAT_NVME);
8352 }
8353
8354 pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
8355 nvme_add_pm_capability(pci_dev, 0x60);
8356 pcie_endpoint_cap_init(pci_dev, 0x80);
8357 pcie_cap_flr_init(pci_dev);
8358 if (n->params.sriov_max_vfs) {
8359 pcie_ari_init(pci_dev, 0x100);
8360 }
8361
8362 if (n->params.msix_exclusive_bar && !pci_is_vf(pci_dev)) {
8363 bar_size = nvme_mbar_size(n->params.max_ioqpairs + 1, 0, NULL, NULL);
8364 memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
8365 bar_size);
8366 pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
8367 PCI_BASE_ADDRESS_MEM_TYPE_64, &n->iomem);
8368 ret = msix_init_exclusive_bar(pci_dev, n->params.msix_qsize, 4, errp);
8369 } else {
8370 assert(n->params.msix_qsize >= 1);
8371
8372 /* add one to max_ioqpairs to account for the admin queue pair */
8373 if (!pci_is_vf(pci_dev)) {
8374 nr_vectors = n->params.msix_qsize;
8375 bar_size = nvme_mbar_size(n->params.max_ioqpairs + 1,
8376 nr_vectors, &msix_table_offset,
8377 &msix_pba_offset);
8378 } else {
8379 NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));
8380 NvmePriCtrlCap *cap = &pn->pri_ctrl_cap;
8381
8382 nr_vectors = le16_to_cpu(cap->vifrsm);
8383 bar_size = nvme_mbar_size(le16_to_cpu(cap->vqfrsm), nr_vectors,
8384 &msix_table_offset, &msix_pba_offset);
8385 }
8386
8387 memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
8388 memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
8389 msix_table_offset);
8390 memory_region_add_subregion(&n->bar0, 0, &n->iomem);
8391
8392 if (pci_is_vf(pci_dev)) {
8393 pcie_sriov_vf_register_bar(pci_dev, 0, &n->bar0);
8394 } else {
8395 pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
8396 PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
8397 }
8398
8399 ret = msix_init(pci_dev, nr_vectors,
8400 &n->bar0, 0, msix_table_offset,
8401 &n->bar0, 0, msix_pba_offset, 0, errp);
8402 }
8403
8404 if (ret == -ENOTSUP) {
8405 /* report that msix is not supported, but do not error out */
8406 warn_report_err(*errp);
8407 *errp = NULL;
8408 } else if (ret < 0) {
8409 /* propagate error to caller */
8410 return false;
8411 }
8412
8413 nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
8414
8415 pcie_cap_deverr_init(pci_dev);
8416
8417 /* DOE Initialisation */
8418 if (pci_dev->spdm_port) {
8419 uint16_t doe_offset = n->params.sriov_max_vfs ?
8420 PCI_CONFIG_SPACE_SIZE + PCI_ARI_SIZEOF
8421 : PCI_CONFIG_SPACE_SIZE;
8422
8423 pcie_doe_init(pci_dev, &pci_dev->doe_spdm, doe_offset,
8424 doe_spdm_prot, true, 0);
8425
8426 pci_dev->doe_spdm.spdm_socket = spdm_socket_connect(pci_dev->spdm_port,
8427 errp);
8428
8429 if (pci_dev->doe_spdm.spdm_socket < 0) {
8430 return false;
8431 }
8432 }
8433
8434 if (n->params.cmb_size_mb) {
8435 nvme_init_cmb(n, pci_dev);
8436 }
8437
8438 if (n->pmr.dev) {
8439 nvme_init_pmr(n, pci_dev);
8440 }
8441
8442 if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
8443 nvme_init_sriov(n, pci_dev, 0x120);
8444 }
8445
8446 return true;
8447 }
8448
nvme_init_subnqn(NvmeCtrl * n)8449 static void nvme_init_subnqn(NvmeCtrl *n)
8450 {
8451 NvmeSubsystem *subsys = n->subsys;
8452 NvmeIdCtrl *id = &n->id_ctrl;
8453
8454 if (!subsys) {
8455 snprintf((char *)id->subnqn, sizeof(id->subnqn),
8456 "nqn.2019-08.org.qemu:%s", n->params.serial);
8457 } else {
8458 pstrcpy((char *)id->subnqn, sizeof(id->subnqn), (char*)subsys->subnqn);
8459 }
8460 }
8461
nvme_init_ctrl(NvmeCtrl * n,PCIDevice * pci_dev)8462 static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
8463 {
8464 NvmeIdCtrl *id = &n->id_ctrl;
8465 uint8_t *pci_conf = pci_dev->config;
8466 uint64_t cap = ldq_le_p(&n->bar.cap);
8467 NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
8468 uint32_t ctratt;
8469
8470 id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
8471 id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
8472 strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
8473 strpadcpy((char *)id->fr, sizeof(id->fr), QEMU_VERSION, ' ');
8474 strpadcpy((char *)id->sn, sizeof(id->sn), n->params.serial, ' ');
8475
8476 id->cntlid = cpu_to_le16(n->cntlid);
8477
8478 id->oaes = cpu_to_le32(NVME_OAES_NS_ATTR);
8479 ctratt = NVME_CTRATT_ELBAS;
8480
8481 id->rab = 6;
8482
8483 if (n->params.use_intel_id) {
8484 id->ieee[0] = 0xb3;
8485 id->ieee[1] = 0x02;
8486 id->ieee[2] = 0x00;
8487 } else {
8488 id->ieee[0] = 0x00;
8489 id->ieee[1] = 0x54;
8490 id->ieee[2] = 0x52;
8491 }
8492
8493 id->mdts = n->params.mdts;
8494 id->ver = cpu_to_le32(NVME_SPEC_VER);
8495 id->oacs =
8496 cpu_to_le16(NVME_OACS_NS_MGMT | NVME_OACS_FORMAT | NVME_OACS_DBBUF |
8497 NVME_OACS_DIRECTIVES);
8498 id->cntrltype = 0x1;
8499
8500 /*
8501 * Because the controller always completes the Abort command immediately,
8502 * there can never be more than one concurrently executing Abort command,
8503 * so this value is never used for anything. Note that there can easily be
8504 * many Abort commands in the queues, but they are not considered
8505 * "executing" until processed by nvme_abort.
8506 *
8507 * The specification recommends a value of 3 for Abort Command Limit (four
8508 * concurrently outstanding Abort commands), so lets use that though it is
8509 * inconsequential.
8510 */
8511 id->acl = 3;
8512 id->aerl = n->params.aerl;
8513 id->frmw = (NVME_NUM_FW_SLOTS << 1) | NVME_FRMW_SLOT1_RO;
8514 id->lpa = NVME_LPA_NS_SMART | NVME_LPA_CSE | NVME_LPA_EXTENDED;
8515
8516 /* recommended default value (~70 C) */
8517 id->wctemp = cpu_to_le16(NVME_TEMPERATURE_WARNING);
8518 id->cctemp = cpu_to_le16(NVME_TEMPERATURE_CRITICAL);
8519
8520 id->sqes = (NVME_SQES << 4) | NVME_SQES;
8521 id->cqes = (NVME_CQES << 4) | NVME_CQES;
8522 id->nn = cpu_to_le32(NVME_MAX_NAMESPACES);
8523 id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROES | NVME_ONCS_TIMESTAMP |
8524 NVME_ONCS_FEATURES | NVME_ONCS_DSM |
8525 NVME_ONCS_COMPARE | NVME_ONCS_COPY |
8526 NVME_ONCS_NVMCSA | NVME_ONCS_NVMAFC);
8527
8528 /*
8529 * NOTE: If this device ever supports a command set that does NOT use 0x0
8530 * as a Flush-equivalent operation, support for the broadcast NSID in Flush
8531 * should probably be removed.
8532 *
8533 * See comment in nvme_io_cmd.
8534 */
8535 id->vwc = NVME_VWC_NSID_BROADCAST_SUPPORT | NVME_VWC_PRESENT;
8536
8537 id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1 |
8538 NVME_OCFS_COPY_FORMAT_2 | NVME_OCFS_COPY_FORMAT_3);
8539 id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN);
8540
8541 nvme_init_subnqn(n);
8542
8543 id->psd[0].mp = cpu_to_le16(0x9c4);
8544 id->psd[0].enlat = cpu_to_le32(0x10);
8545 id->psd[0].exlat = cpu_to_le32(0x4);
8546
8547 if (n->subsys) {
8548 id->cmic |= NVME_CMIC_MULTI_CTRL;
8549 ctratt |= NVME_CTRATT_ENDGRPS;
8550
8551 id->endgidmax = cpu_to_le16(0x1);
8552
8553 if (n->subsys->endgrp.fdp.enabled) {
8554 ctratt |= NVME_CTRATT_FDPS;
8555 }
8556 }
8557
8558 id->ctratt = cpu_to_le32(ctratt);
8559
8560 NVME_CAP_SET_MQES(cap, n->params.mqes);
8561 NVME_CAP_SET_CQR(cap, 1);
8562 NVME_CAP_SET_TO(cap, 0xf);
8563 NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_NVM);
8564 NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_CSI_SUPP);
8565 NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_ADMIN_ONLY);
8566 NVME_CAP_SET_MPSMAX(cap, 4);
8567 NVME_CAP_SET_CMBS(cap, n->params.cmb_size_mb ? 1 : 0);
8568 NVME_CAP_SET_PMRS(cap, n->pmr.dev ? 1 : 0);
8569 stq_le_p(&n->bar.cap, cap);
8570
8571 stl_le_p(&n->bar.vs, NVME_SPEC_VER);
8572 n->bar.intmc = n->bar.intms = 0;
8573
8574 if (pci_is_vf(pci_dev) && !sctrl->scs) {
8575 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
8576 }
8577 }
8578
nvme_init_subsys(NvmeCtrl * n,Error ** errp)8579 static int nvme_init_subsys(NvmeCtrl *n, Error **errp)
8580 {
8581 int cntlid;
8582
8583 if (!n->subsys) {
8584 return 0;
8585 }
8586
8587 cntlid = nvme_subsys_register_ctrl(n, errp);
8588 if (cntlid < 0) {
8589 return -1;
8590 }
8591
8592 n->cntlid = cntlid;
8593
8594 return 0;
8595 }
8596
nvme_attach_ns(NvmeCtrl * n,NvmeNamespace * ns)8597 void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns)
8598 {
8599 uint32_t nsid = ns->params.nsid;
8600 assert(nsid && nsid <= NVME_MAX_NAMESPACES);
8601
8602 n->namespaces[nsid] = ns;
8603 ns->attached++;
8604
8605 n->dmrsl = MIN_NON_ZERO(n->dmrsl,
8606 BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
8607 }
8608
nvme_realize(PCIDevice * pci_dev,Error ** errp)8609 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
8610 {
8611 NvmeCtrl *n = NVME(pci_dev);
8612 DeviceState *dev = DEVICE(pci_dev);
8613 NvmeNamespace *ns;
8614 NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));
8615
8616 if (pci_is_vf(pci_dev)) {
8617 /*
8618 * VFs derive settings from the parent. PF's lifespan exceeds
8619 * that of VF's.
8620 */
8621 memcpy(&n->params, &pn->params, sizeof(NvmeParams));
8622
8623 /*
8624 * Set PF's serial value to a new string memory to prevent 'serial'
8625 * property object release of PF when a VF is removed from the system.
8626 */
8627 n->params.serial = g_strdup(pn->params.serial);
8628 n->subsys = pn->subsys;
8629 }
8630
8631 if (!nvme_check_params(n, errp)) {
8632 return;
8633 }
8634
8635 qbus_init(&n->bus, sizeof(NvmeBus), TYPE_NVME_BUS, dev, dev->id);
8636
8637 if (nvme_init_subsys(n, errp)) {
8638 return;
8639 }
8640 nvme_init_state(n);
8641 if (!nvme_init_pci(n, pci_dev, errp)) {
8642 return;
8643 }
8644 nvme_init_ctrl(n, pci_dev);
8645
8646 /* setup a namespace if the controller drive property was given */
8647 if (n->namespace.blkconf.blk) {
8648 ns = &n->namespace;
8649 ns->params.nsid = 1;
8650
8651 if (nvme_ns_setup(ns, errp)) {
8652 return;
8653 }
8654
8655 nvme_attach_ns(n, ns);
8656 }
8657 }
8658
nvme_exit(PCIDevice * pci_dev)8659 static void nvme_exit(PCIDevice *pci_dev)
8660 {
8661 NvmeCtrl *n = NVME(pci_dev);
8662 NvmeNamespace *ns;
8663 int i;
8664
8665 nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
8666
8667 if (n->subsys) {
8668 for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
8669 ns = nvme_ns(n, i);
8670 if (ns) {
8671 ns->attached--;
8672 }
8673 }
8674
8675 nvme_subsys_unregister_ctrl(n->subsys, n);
8676 }
8677
8678 g_free(n->cq);
8679 g_free(n->sq);
8680 g_free(n->aer_reqs);
8681
8682 if (n->params.cmb_size_mb) {
8683 g_free(n->cmb.buf);
8684 }
8685
8686 if (pci_dev->doe_spdm.spdm_socket > 0) {
8687 spdm_socket_close(pci_dev->doe_spdm.spdm_socket,
8688 SPDM_SOCKET_TRANSPORT_TYPE_PCI_DOE);
8689 }
8690
8691 if (n->pmr.dev) {
8692 host_memory_backend_set_mapped(n->pmr.dev, false);
8693 }
8694
8695 if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
8696 pcie_sriov_pf_exit(pci_dev);
8697 }
8698
8699 msix_uninit(pci_dev, &n->bar0, &n->bar0);
8700 memory_region_del_subregion(&n->bar0, &n->iomem);
8701 }
8702
8703 static Property nvme_props[] = {
8704 DEFINE_BLOCK_PROPERTIES(NvmeCtrl, namespace.blkconf),
8705 DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmr.dev, TYPE_MEMORY_BACKEND,
8706 HostMemoryBackend *),
8707 DEFINE_PROP_LINK("subsys", NvmeCtrl, subsys, TYPE_NVME_SUBSYS,
8708 NvmeSubsystem *),
8709 DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial),
8710 DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0),
8711 DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0),
8712 DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl, params.max_ioqpairs, 64),
8713 DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl, params.msix_qsize, 65),
8714 DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
8715 DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64),
8716 DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7),
8717 DEFINE_PROP_UINT8("vsl", NvmeCtrl, params.vsl, 7),
8718 DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl, params.use_intel_id, false),
8719 DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl, params.legacy_cmb, false),
8720 DEFINE_PROP_BOOL("ioeventfd", NvmeCtrl, params.ioeventfd, false),
8721 DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl, params.zasl, 0),
8722 DEFINE_PROP_BOOL("zoned.auto_transition", NvmeCtrl,
8723 params.auto_transition_zones, true),
8724 DEFINE_PROP_UINT16("sriov_max_vfs", NvmeCtrl, params.sriov_max_vfs, 0),
8725 DEFINE_PROP_UINT16("sriov_vq_flexible", NvmeCtrl,
8726 params.sriov_vq_flexible, 0),
8727 DEFINE_PROP_UINT16("sriov_vi_flexible", NvmeCtrl,
8728 params.sriov_vi_flexible, 0),
8729 DEFINE_PROP_UINT32("sriov_max_vi_per_vf", NvmeCtrl,
8730 params.sriov_max_vi_per_vf, 0),
8731 DEFINE_PROP_UINT32("sriov_max_vq_per_vf", NvmeCtrl,
8732 params.sriov_max_vq_per_vf, 0),
8733 DEFINE_PROP_BOOL("msix-exclusive-bar", NvmeCtrl, params.msix_exclusive_bar,
8734 false),
8735 DEFINE_PROP_UINT16("mqes", NvmeCtrl, params.mqes, 0x7ff),
8736 DEFINE_PROP_UINT16("spdm_port", PCIDevice, spdm_port, 0),
8737 DEFINE_PROP_END_OF_LIST(),
8738 };
8739
nvme_get_smart_warning(Object * obj,Visitor * v,const char * name,void * opaque,Error ** errp)8740 static void nvme_get_smart_warning(Object *obj, Visitor *v, const char *name,
8741 void *opaque, Error **errp)
8742 {
8743 NvmeCtrl *n = NVME(obj);
8744 uint8_t value = n->smart_critical_warning;
8745
8746 visit_type_uint8(v, name, &value, errp);
8747 }
8748
nvme_set_smart_warning(Object * obj,Visitor * v,const char * name,void * opaque,Error ** errp)8749 static void nvme_set_smart_warning(Object *obj, Visitor *v, const char *name,
8750 void *opaque, Error **errp)
8751 {
8752 NvmeCtrl *n = NVME(obj);
8753 uint8_t value, old_value, cap = 0, index, event;
8754
8755 if (!visit_type_uint8(v, name, &value, errp)) {
8756 return;
8757 }
8758
8759 cap = NVME_SMART_SPARE | NVME_SMART_TEMPERATURE | NVME_SMART_RELIABILITY
8760 | NVME_SMART_MEDIA_READ_ONLY | NVME_SMART_FAILED_VOLATILE_MEDIA;
8761 if (NVME_CAP_PMRS(ldq_le_p(&n->bar.cap))) {
8762 cap |= NVME_SMART_PMR_UNRELIABLE;
8763 }
8764
8765 if ((value & cap) != value) {
8766 error_setg(errp, "unsupported smart critical warning bits: 0x%x",
8767 value & ~cap);
8768 return;
8769 }
8770
8771 old_value = n->smart_critical_warning;
8772 n->smart_critical_warning = value;
8773
8774 /* only inject new bits of smart critical warning */
8775 for (index = 0; index < NVME_SMART_WARN_MAX; index++) {
8776 event = 1 << index;
8777 if (value & ~old_value & event)
8778 nvme_smart_event(n, event);
8779 }
8780 }
8781
nvme_pci_reset(DeviceState * qdev)8782 static void nvme_pci_reset(DeviceState *qdev)
8783 {
8784 PCIDevice *pci_dev = PCI_DEVICE(qdev);
8785 NvmeCtrl *n = NVME(pci_dev);
8786
8787 trace_pci_nvme_pci_reset();
8788 nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
8789 }
8790
nvme_sriov_post_write_config(PCIDevice * dev,uint16_t old_num_vfs)8791 static void nvme_sriov_post_write_config(PCIDevice *dev, uint16_t old_num_vfs)
8792 {
8793 NvmeCtrl *n = NVME(dev);
8794 NvmeSecCtrlEntry *sctrl;
8795 int i;
8796
8797 for (i = pcie_sriov_num_vfs(dev); i < old_num_vfs; i++) {
8798 sctrl = &n->sec_ctrl_list[i];
8799 nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
8800 }
8801 }
8802
nvme_pci_write_config(PCIDevice * dev,uint32_t address,uint32_t val,int len)8803 static void nvme_pci_write_config(PCIDevice *dev, uint32_t address,
8804 uint32_t val, int len)
8805 {
8806 uint16_t old_num_vfs = pcie_sriov_num_vfs(dev);
8807
8808 if (pcie_find_capability(dev, PCI_EXT_CAP_ID_DOE)) {
8809 pcie_doe_write_config(&dev->doe_spdm, address, val, len);
8810 }
8811 pci_default_write_config(dev, address, val, len);
8812 pcie_cap_flr_write_config(dev, address, val, len);
8813 nvme_sriov_post_write_config(dev, old_num_vfs);
8814 }
8815
nvme_pci_read_config(PCIDevice * dev,uint32_t address,int len)8816 static uint32_t nvme_pci_read_config(PCIDevice *dev, uint32_t address, int len)
8817 {
8818 uint32_t val;
8819 if (dev->spdm_port && pcie_find_capability(dev, PCI_EXT_CAP_ID_DOE)) {
8820 if (pcie_doe_read_config(&dev->doe_spdm, address, len, &val)) {
8821 return val;
8822 }
8823 }
8824 return pci_default_read_config(dev, address, len);
8825 }
8826
8827 static const VMStateDescription nvme_vmstate = {
8828 .name = "nvme",
8829 .unmigratable = 1,
8830 };
8831
nvme_class_init(ObjectClass * oc,void * data)8832 static void nvme_class_init(ObjectClass *oc, void *data)
8833 {
8834 DeviceClass *dc = DEVICE_CLASS(oc);
8835 PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
8836
8837 pc->realize = nvme_realize;
8838 pc->config_write = nvme_pci_write_config;
8839 pc->config_read = nvme_pci_read_config;
8840 pc->exit = nvme_exit;
8841 pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
8842 pc->revision = 2;
8843
8844 set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
8845 dc->desc = "Non-Volatile Memory Express";
8846 device_class_set_props(dc, nvme_props);
8847 dc->vmsd = &nvme_vmstate;
8848 dc->reset = nvme_pci_reset;
8849 }
8850
nvme_instance_init(Object * obj)8851 static void nvme_instance_init(Object *obj)
8852 {
8853 NvmeCtrl *n = NVME(obj);
8854
8855 device_add_bootindex_property(obj, &n->namespace.blkconf.bootindex,
8856 "bootindex", "/namespace@1,0",
8857 DEVICE(obj));
8858
8859 object_property_add(obj, "smart_critical_warning", "uint8",
8860 nvme_get_smart_warning,
8861 nvme_set_smart_warning, NULL, NULL);
8862 }
8863
8864 static const TypeInfo nvme_info = {
8865 .name = TYPE_NVME,
8866 .parent = TYPE_PCI_DEVICE,
8867 .instance_size = sizeof(NvmeCtrl),
8868 .instance_init = nvme_instance_init,
8869 .class_init = nvme_class_init,
8870 .interfaces = (InterfaceInfo[]) {
8871 { INTERFACE_PCIE_DEVICE },
8872 { }
8873 },
8874 };
8875
8876 static const TypeInfo nvme_bus_info = {
8877 .name = TYPE_NVME_BUS,
8878 .parent = TYPE_BUS,
8879 .instance_size = sizeof(NvmeBus),
8880 };
8881
nvme_register_types(void)8882 static void nvme_register_types(void)
8883 {
8884 type_register_static(&nvme_info);
8885 type_register_static(&nvme_bus_info);
8886 }
8887
8888 type_init(nvme_register_types)
8889