188eea45cSKlaus Jensen /*
288eea45cSKlaus Jensen * QEMU NVM Express Controller
388eea45cSKlaus Jensen *
488eea45cSKlaus Jensen * Copyright (c) 2012, Intel Corporation
588eea45cSKlaus Jensen *
688eea45cSKlaus Jensen * Written by Keith Busch <keith.busch@intel.com>
788eea45cSKlaus Jensen *
888eea45cSKlaus Jensen * This code is licensed under the GNU GPL v2 or later.
988eea45cSKlaus Jensen */
1088eea45cSKlaus Jensen
1188eea45cSKlaus Jensen /**
1288eea45cSKlaus Jensen * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e
1388eea45cSKlaus Jensen *
1488eea45cSKlaus Jensen * https://nvmexpress.org/developers/nvme-specification/
1588eea45cSKlaus Jensen *
1688eea45cSKlaus Jensen *
1788eea45cSKlaus Jensen * Notes on coding style
1888eea45cSKlaus Jensen * ---------------------
1988eea45cSKlaus Jensen * While QEMU coding style prefers lowercase hexadecimals in constants, the
209b4b4e51SMichael Tokarev * NVMe subsystem use this format from the NVMe specifications in the comments
2188eea45cSKlaus Jensen * (i.e. 'h' suffix instead of '0x' prefix).
2288eea45cSKlaus Jensen *
2388eea45cSKlaus Jensen * Usage
2488eea45cSKlaus Jensen * -----
2588eea45cSKlaus Jensen * See docs/system/nvme.rst for extensive documentation.
2688eea45cSKlaus Jensen *
2788eea45cSKlaus Jensen * Add options:
2888eea45cSKlaus Jensen * -drive file=<file>,if=none,id=<drive_id>
2988eea45cSKlaus Jensen * -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id>
3088eea45cSKlaus Jensen * -device nvme,serial=<serial>,id=<bus_name>, \
3188eea45cSKlaus Jensen * cmb_size_mb=<cmb_size_mb[optional]>, \
3288eea45cSKlaus Jensen * [pmrdev=<mem_backend_file_id>,] \
3388eea45cSKlaus Jensen * max_ioqpairs=<N[optional]>, \
3488eea45cSKlaus Jensen * aerl=<N[optional]>,aer_max_queued=<N[optional]>, \
3588eea45cSKlaus Jensen * mdts=<N[optional]>,vsl=<N[optional]>, \
3688eea45cSKlaus Jensen * zoned.zasl=<N[optional]>, \
37cccc2651SNiklas Cassel * zoned.auto_transition=<on|off[optional]>, \
3844c2c094SLukasz Maniak * sriov_max_vfs=<N[optional]> \
39746d42b1SŁukasz Gieryk * sriov_vq_flexible=<N[optional]> \
40746d42b1SŁukasz Gieryk * sriov_vi_flexible=<N[optional]> \
41746d42b1SŁukasz Gieryk * sriov_max_vi_per_vf=<N[optional]> \
42746d42b1SŁukasz Gieryk * sriov_max_vq_per_vf=<N[optional]> \
4388eea45cSKlaus Jensen * subsys=<subsys_id>
4488eea45cSKlaus Jensen * -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
4588eea45cSKlaus Jensen * zoned=<true|false[optional]>, \
467491e0e4SMinwoo Im * subsys=<subsys_id>,shared=<true|false[optional]>, \
477491e0e4SMinwoo Im * detached=<true|false[optional]>, \
487491e0e4SMinwoo Im * zoned.zone_size=<N[optional]>, \
497491e0e4SMinwoo Im * zoned.zone_capacity=<N[optional]>, \
507491e0e4SMinwoo Im * zoned.descr_ext_size=<N[optional]>, \
517491e0e4SMinwoo Im * zoned.max_active=<N[optional]>, \
527491e0e4SMinwoo Im * zoned.max_open=<N[optional]>, \
537491e0e4SMinwoo Im * zoned.cross_read=<true|false[optional]>
5488eea45cSKlaus Jensen *
5588eea45cSKlaus Jensen * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
5688eea45cSKlaus Jensen * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
5788eea45cSKlaus Jensen * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to
5888eea45cSKlaus Jensen * always enable the CMBLOC and CMBSZ registers (v1.3 behavior).
5988eea45cSKlaus Jensen *
6088eea45cSKlaus Jensen * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
6188eea45cSKlaus Jensen * For example:
6288eea45cSKlaus Jensen * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
6388eea45cSKlaus Jensen * size=<size> .... -device nvme,...,pmrdev=<mem_id>
6488eea45cSKlaus Jensen *
6588eea45cSKlaus Jensen * The PMR will use BAR 4/5 exclusively.
6688eea45cSKlaus Jensen *
6788eea45cSKlaus Jensen * To place controller(s) and namespace(s) to a subsystem, then provide
6888eea45cSKlaus Jensen * nvme-subsys device as above.
6988eea45cSKlaus Jensen *
7088eea45cSKlaus Jensen * nvme subsystem device parameters
7188eea45cSKlaus Jensen * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
7288eea45cSKlaus Jensen * - `nqn`
7388eea45cSKlaus Jensen * This parameter provides the `<nqn_id>` part of the string
7488eea45cSKlaus Jensen * `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field
7588eea45cSKlaus Jensen * of subsystem controllers. Note that `<nqn_id>` should be unique per
7688eea45cSKlaus Jensen * subsystem, but this is not enforced by QEMU. If not specified, it will
7788eea45cSKlaus Jensen * default to the value of the `id` parameter (`<subsys_id>`).
7888eea45cSKlaus Jensen *
7988eea45cSKlaus Jensen * nvme device parameters
8088eea45cSKlaus Jensen * ~~~~~~~~~~~~~~~~~~~~~~
8188eea45cSKlaus Jensen * - `subsys`
8288eea45cSKlaus Jensen * Specifying this parameter attaches the controller to the subsystem and
8388eea45cSKlaus Jensen * the SUBNQN field in the controller will report the NQN of the subsystem
8488eea45cSKlaus Jensen * device. This also enables multi controller capability represented in
8588eea45cSKlaus Jensen * Identify Controller data structure in CMIC (Controller Multi-path I/O and
86a0984714SDr. David Alan Gilbert * Namespace Sharing Capabilities).
8788eea45cSKlaus Jensen *
8888eea45cSKlaus Jensen * - `aerl`
8988eea45cSKlaus Jensen * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
9088eea45cSKlaus Jensen * of concurrently outstanding Asynchronous Event Request commands support
9188eea45cSKlaus Jensen * by the controller. This is a 0's based value.
9288eea45cSKlaus Jensen *
9388eea45cSKlaus Jensen * - `aer_max_queued`
9488eea45cSKlaus Jensen * This is the maximum number of events that the device will enqueue for
9588eea45cSKlaus Jensen * completion when there are no outstanding AERs. When the maximum number of
9688eea45cSKlaus Jensen * enqueued events are reached, subsequent events will be dropped.
9788eea45cSKlaus Jensen *
9888eea45cSKlaus Jensen * - `mdts`
9988eea45cSKlaus Jensen * Indicates the maximum data transfer size for a command that transfers data
10088eea45cSKlaus Jensen * between host-accessible memory and the controller. The value is specified
10188eea45cSKlaus Jensen * as a power of two (2^n) and is in units of the minimum memory page size
10288eea45cSKlaus Jensen * (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB).
10388eea45cSKlaus Jensen *
10488eea45cSKlaus Jensen * - `vsl`
10588eea45cSKlaus Jensen * Indicates the maximum data size limit for the Verify command. Like `mdts`,
10688eea45cSKlaus Jensen * this value is specified as a power of two (2^n) and is in units of the
10788eea45cSKlaus Jensen * minimum memory page size (CAP.MPSMIN). The default value is 7 (i.e. 512
10888eea45cSKlaus Jensen * KiB).
10988eea45cSKlaus Jensen *
11088eea45cSKlaus Jensen * - `zoned.zasl`
11188eea45cSKlaus Jensen * Indicates the maximum data transfer size for the Zone Append command. Like
11288eea45cSKlaus Jensen * `mdts`, the value is specified as a power of two (2^n) and is in units of
11388eea45cSKlaus Jensen * the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e.
11488eea45cSKlaus Jensen * defaulting to the value of `mdts`).
11588eea45cSKlaus Jensen *
116cccc2651SNiklas Cassel * - `zoned.auto_transition`
117cccc2651SNiklas Cassel * Indicates if zones in zone state implicitly opened can be automatically
118cccc2651SNiklas Cassel * transitioned to zone state closed for resource management purposes.
119cccc2651SNiklas Cassel * Defaults to 'on'.
120cccc2651SNiklas Cassel *
12144c2c094SLukasz Maniak * - `sriov_max_vfs`
12244c2c094SLukasz Maniak * Indicates the maximum number of PCIe virtual functions supported
12344c2c094SLukasz Maniak * by the controller. The default value is 0. Specifying a non-zero value
12444c2c094SLukasz Maniak * enables reporting of both SR-IOV and ARI capabilities by the NVMe device.
12544c2c094SLukasz Maniak * Virtual function controllers will not report SR-IOV capability.
12644c2c094SLukasz Maniak *
127746d42b1SŁukasz Gieryk * NOTE: Single Root I/O Virtualization support is experimental.
128746d42b1SŁukasz Gieryk * All the related parameters may be subject to change.
129746d42b1SŁukasz Gieryk *
130746d42b1SŁukasz Gieryk * - `sriov_vq_flexible`
131746d42b1SŁukasz Gieryk * Indicates the total number of flexible queue resources assignable to all
132746d42b1SŁukasz Gieryk * the secondary controllers. Implicitly sets the number of primary
133746d42b1SŁukasz Gieryk * controller's private resources to `(max_ioqpairs - sriov_vq_flexible)`.
134746d42b1SŁukasz Gieryk *
135746d42b1SŁukasz Gieryk * - `sriov_vi_flexible`
136746d42b1SŁukasz Gieryk * Indicates the total number of flexible interrupt resources assignable to
137746d42b1SŁukasz Gieryk * all the secondary controllers. Implicitly sets the number of primary
138746d42b1SŁukasz Gieryk * controller's private resources to `(msix_qsize - sriov_vi_flexible)`.
139746d42b1SŁukasz Gieryk *
140746d42b1SŁukasz Gieryk * - `sriov_max_vi_per_vf`
141746d42b1SŁukasz Gieryk * Indicates the maximum number of virtual interrupt resources assignable
142746d42b1SŁukasz Gieryk * to a secondary controller. The default 0 resolves to
143746d42b1SŁukasz Gieryk * `(sriov_vi_flexible / sriov_max_vfs)`.
144746d42b1SŁukasz Gieryk *
145746d42b1SŁukasz Gieryk * - `sriov_max_vq_per_vf`
146746d42b1SŁukasz Gieryk * Indicates the maximum number of virtual queue resources assignable to
147746d42b1SŁukasz Gieryk * a secondary controller. The default 0 resolves to
148746d42b1SŁukasz Gieryk * `(sriov_vq_flexible / sriov_max_vfs)`.
149746d42b1SŁukasz Gieryk *
15088eea45cSKlaus Jensen * nvme namespace device parameters
15188eea45cSKlaus Jensen * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
15288eea45cSKlaus Jensen * - `shared`
15388eea45cSKlaus Jensen * When the parent nvme device (as defined explicitly by the 'bus' parameter
15488eea45cSKlaus Jensen * or implicitly by the most recently defined NvmeBus) is linked to an
15588eea45cSKlaus Jensen * nvme-subsys device, the namespace will be attached to all controllers in
15688eea45cSKlaus Jensen * the subsystem. If set to 'off' (the default), the namespace will remain a
15788eea45cSKlaus Jensen * private namespace and may only be attached to a single controller at a
15888eea45cSKlaus Jensen * time.
15988eea45cSKlaus Jensen *
16088eea45cSKlaus Jensen * - `detached`
16188eea45cSKlaus Jensen * This parameter is only valid together with the `subsys` parameter. If left
16288eea45cSKlaus Jensen * at the default value (`false/off`), the namespace will be attached to all
16388eea45cSKlaus Jensen * controllers in the NVMe subsystem at boot-up. If set to `true/on`, the
164eeef4329SGollu Appalanaidu * namespace will be available in the subsystem but not attached to any
16588eea45cSKlaus Jensen * controllers.
16688eea45cSKlaus Jensen *
16788eea45cSKlaus Jensen * Setting `zoned` to true selects Zoned Command Set at the namespace.
16888eea45cSKlaus Jensen * In this case, the following namespace properties are available to configure
16988eea45cSKlaus Jensen * zoned operation:
17088eea45cSKlaus Jensen * zoned.zone_size=<zone size in bytes, default: 128MiB>
17188eea45cSKlaus Jensen * The number may be followed by K, M, G as in kilo-, mega- or giga-.
17288eea45cSKlaus Jensen *
17388eea45cSKlaus Jensen * zoned.zone_capacity=<zone capacity in bytes, default: zone size>
17488eea45cSKlaus Jensen * The value 0 (default) forces zone capacity to be the same as zone
17588eea45cSKlaus Jensen * size. The value of this property may not exceed zone size.
17688eea45cSKlaus Jensen *
17788eea45cSKlaus Jensen * zoned.descr_ext_size=<zone descriptor extension size, default 0>
17888eea45cSKlaus Jensen * This value needs to be specified in 64B units. If it is zero,
17988eea45cSKlaus Jensen * namespace(s) will not support zone descriptor extensions.
18088eea45cSKlaus Jensen *
18188eea45cSKlaus Jensen * zoned.max_active=<Maximum Active Resources (zones), default: 0>
18288eea45cSKlaus Jensen * The default value means there is no limit to the number of
18388eea45cSKlaus Jensen * concurrently active zones.
18488eea45cSKlaus Jensen *
18588eea45cSKlaus Jensen * zoned.max_open=<Maximum Open Resources (zones), default: 0>
18688eea45cSKlaus Jensen * The default value means there is no limit to the number of
18788eea45cSKlaus Jensen * concurrently open zones.
18888eea45cSKlaus Jensen *
18988eea45cSKlaus Jensen * zoned.cross_read=<enable RAZB, default: false>
19088eea45cSKlaus Jensen * Setting this property to true enables Read Across Zone Boundaries.
19188eea45cSKlaus Jensen */
19288eea45cSKlaus Jensen
19388eea45cSKlaus Jensen #include "qemu/osdep.h"
19488eea45cSKlaus Jensen #include "qemu/cutils.h"
19588eea45cSKlaus Jensen #include "qemu/error-report.h"
19688eea45cSKlaus Jensen #include "qemu/log.h"
19788eea45cSKlaus Jensen #include "qemu/units.h"
19811871f53SŁukasz Gieryk #include "qemu/range.h"
19988eea45cSKlaus Jensen #include "qapi/error.h"
20088eea45cSKlaus Jensen #include "qapi/visitor.h"
20188eea45cSKlaus Jensen #include "sysemu/sysemu.h"
20288eea45cSKlaus Jensen #include "sysemu/block-backend.h"
20388eea45cSKlaus Jensen #include "sysemu/hostmem.h"
20488eea45cSKlaus Jensen #include "hw/pci/msix.h"
20544c2c094SLukasz Maniak #include "hw/pci/pcie_sriov.h"
20688eea45cSKlaus Jensen #include "migration/vmstate.h"
20788eea45cSKlaus Jensen
20888eea45cSKlaus Jensen #include "nvme.h"
20905f7ae45SKlaus Jensen #include "dif.h"
21088eea45cSKlaus Jensen #include "trace.h"
21188eea45cSKlaus Jensen
21288eea45cSKlaus Jensen #define NVME_MAX_IOQPAIRS 0xffff
21388eea45cSKlaus Jensen #define NVME_DB_SIZE 4
21488eea45cSKlaus Jensen #define NVME_SPEC_VER 0x00010400
21588eea45cSKlaus Jensen #define NVME_CMB_BIR 2
21688eea45cSKlaus Jensen #define NVME_PMR_BIR 4
21788eea45cSKlaus Jensen #define NVME_TEMPERATURE 0x143
21888eea45cSKlaus Jensen #define NVME_TEMPERATURE_WARNING 0x157
21988eea45cSKlaus Jensen #define NVME_TEMPERATURE_CRITICAL 0x175
22088eea45cSKlaus Jensen #define NVME_NUM_FW_SLOTS 1
22188eea45cSKlaus Jensen #define NVME_DEFAULT_MAX_ZA_SIZE (128 * KiB)
22244c2c094SLukasz Maniak #define NVME_MAX_VFS 127
223746d42b1SŁukasz Gieryk #define NVME_VF_RES_GRANULARITY 1
22444c2c094SLukasz Maniak #define NVME_VF_OFFSET 0x1
22544c2c094SLukasz Maniak #define NVME_VF_STRIDE 1
22688eea45cSKlaus Jensen
22788eea45cSKlaus Jensen #define NVME_GUEST_ERR(trace, fmt, ...) \
22888eea45cSKlaus Jensen do { \
22988eea45cSKlaus Jensen (trace_##trace)(__VA_ARGS__); \
23088eea45cSKlaus Jensen qemu_log_mask(LOG_GUEST_ERROR, #trace \
23188eea45cSKlaus Jensen " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
23288eea45cSKlaus Jensen } while (0)
23388eea45cSKlaus Jensen
23488eea45cSKlaus Jensen static const bool nvme_feature_support[NVME_FID_MAX] = {
23588eea45cSKlaus Jensen [NVME_ARBITRATION] = true,
23688eea45cSKlaus Jensen [NVME_POWER_MANAGEMENT] = true,
23788eea45cSKlaus Jensen [NVME_TEMPERATURE_THRESHOLD] = true,
23888eea45cSKlaus Jensen [NVME_ERROR_RECOVERY] = true,
23988eea45cSKlaus Jensen [NVME_VOLATILE_WRITE_CACHE] = true,
24088eea45cSKlaus Jensen [NVME_NUMBER_OF_QUEUES] = true,
24188eea45cSKlaus Jensen [NVME_INTERRUPT_COALESCING] = true,
24288eea45cSKlaus Jensen [NVME_INTERRUPT_VECTOR_CONF] = true,
24388eea45cSKlaus Jensen [NVME_WRITE_ATOMICITY] = true,
24488eea45cSKlaus Jensen [NVME_ASYNCHRONOUS_EVENT_CONF] = true,
24588eea45cSKlaus Jensen [NVME_TIMESTAMP] = true,
246d0c0697bSNaveen Nagar [NVME_HOST_BEHAVIOR_SUPPORT] = true,
24788eea45cSKlaus Jensen [NVME_COMMAND_SET_PROFILE] = true,
24873064edfSJesper Devantier [NVME_FDP_MODE] = true,
24973064edfSJesper Devantier [NVME_FDP_EVENTS] = true,
25088eea45cSKlaus Jensen };
25188eea45cSKlaus Jensen
25288eea45cSKlaus Jensen static const uint32_t nvme_feature_cap[NVME_FID_MAX] = {
25388eea45cSKlaus Jensen [NVME_TEMPERATURE_THRESHOLD] = NVME_FEAT_CAP_CHANGE,
25488eea45cSKlaus Jensen [NVME_ERROR_RECOVERY] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
25588eea45cSKlaus Jensen [NVME_VOLATILE_WRITE_CACHE] = NVME_FEAT_CAP_CHANGE,
25688eea45cSKlaus Jensen [NVME_NUMBER_OF_QUEUES] = NVME_FEAT_CAP_CHANGE,
25788eea45cSKlaus Jensen [NVME_ASYNCHRONOUS_EVENT_CONF] = NVME_FEAT_CAP_CHANGE,
25888eea45cSKlaus Jensen [NVME_TIMESTAMP] = NVME_FEAT_CAP_CHANGE,
259d0c0697bSNaveen Nagar [NVME_HOST_BEHAVIOR_SUPPORT] = NVME_FEAT_CAP_CHANGE,
26088eea45cSKlaus Jensen [NVME_COMMAND_SET_PROFILE] = NVME_FEAT_CAP_CHANGE,
26173064edfSJesper Devantier [NVME_FDP_MODE] = NVME_FEAT_CAP_CHANGE,
26273064edfSJesper Devantier [NVME_FDP_EVENTS] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
26388eea45cSKlaus Jensen };
26488eea45cSKlaus Jensen
26588eea45cSKlaus Jensen static const uint32_t nvme_cse_acs[256] = {
26688eea45cSKlaus Jensen [NVME_ADM_CMD_DELETE_SQ] = NVME_CMD_EFF_CSUPP,
26788eea45cSKlaus Jensen [NVME_ADM_CMD_CREATE_SQ] = NVME_CMD_EFF_CSUPP,
26888eea45cSKlaus Jensen [NVME_ADM_CMD_GET_LOG_PAGE] = NVME_CMD_EFF_CSUPP,
26988eea45cSKlaus Jensen [NVME_ADM_CMD_DELETE_CQ] = NVME_CMD_EFF_CSUPP,
27088eea45cSKlaus Jensen [NVME_ADM_CMD_CREATE_CQ] = NVME_CMD_EFF_CSUPP,
27188eea45cSKlaus Jensen [NVME_ADM_CMD_IDENTIFY] = NVME_CMD_EFF_CSUPP,
27288eea45cSKlaus Jensen [NVME_ADM_CMD_ABORT] = NVME_CMD_EFF_CSUPP,
27388eea45cSKlaus Jensen [NVME_ADM_CMD_SET_FEATURES] = NVME_CMD_EFF_CSUPP,
27488eea45cSKlaus Jensen [NVME_ADM_CMD_GET_FEATURES] = NVME_CMD_EFF_CSUPP,
27588eea45cSKlaus Jensen [NVME_ADM_CMD_ASYNC_EV_REQ] = NVME_CMD_EFF_CSUPP,
27688eea45cSKlaus Jensen [NVME_ADM_CMD_NS_ATTACHMENT] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_NIC,
27711871f53SŁukasz Gieryk [NVME_ADM_CMD_VIRT_MNGMT] = NVME_CMD_EFF_CSUPP,
2783f7fe8deSJinhao Fan [NVME_ADM_CMD_DBBUF_CONFIG] = NVME_CMD_EFF_CSUPP,
27988eea45cSKlaus Jensen [NVME_ADM_CMD_FORMAT_NVM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
280e181d3daSGollu Appalanaidu [NVME_ADM_CMD_DIRECTIVE_RECV] = NVME_CMD_EFF_CSUPP,
281e181d3daSGollu Appalanaidu [NVME_ADM_CMD_DIRECTIVE_SEND] = NVME_CMD_EFF_CSUPP,
28288eea45cSKlaus Jensen };
28388eea45cSKlaus Jensen
28488eea45cSKlaus Jensen static const uint32_t nvme_cse_iocs_none[256];
28588eea45cSKlaus Jensen
28688eea45cSKlaus Jensen static const uint32_t nvme_cse_iocs_nvm[256] = {
28788eea45cSKlaus Jensen [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
28888eea45cSKlaus Jensen [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
28988eea45cSKlaus Jensen [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
29088eea45cSKlaus Jensen [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
29188eea45cSKlaus Jensen [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
29288eea45cSKlaus Jensen [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
29388eea45cSKlaus Jensen [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
29488eea45cSKlaus Jensen [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
29573064edfSJesper Devantier [NVME_CMD_IO_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
29673064edfSJesper Devantier [NVME_CMD_IO_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
29788eea45cSKlaus Jensen };
29888eea45cSKlaus Jensen
29988eea45cSKlaus Jensen static const uint32_t nvme_cse_iocs_zoned[256] = {
30088eea45cSKlaus Jensen [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
30188eea45cSKlaus Jensen [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
30288eea45cSKlaus Jensen [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
30388eea45cSKlaus Jensen [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
30488eea45cSKlaus Jensen [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
30588eea45cSKlaus Jensen [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
30688eea45cSKlaus Jensen [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
30788eea45cSKlaus Jensen [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
30888eea45cSKlaus Jensen [NVME_CMD_ZONE_APPEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
30988eea45cSKlaus Jensen [NVME_CMD_ZONE_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
31088eea45cSKlaus Jensen [NVME_CMD_ZONE_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
31188eea45cSKlaus Jensen };
31288eea45cSKlaus Jensen
31388eea45cSKlaus Jensen static void nvme_process_sq(void *opaque);
31411871f53SŁukasz Gieryk static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst);
31573064edfSJesper Devantier static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n);
31688eea45cSKlaus Jensen
nvme_sqid(NvmeRequest * req)31788eea45cSKlaus Jensen static uint16_t nvme_sqid(NvmeRequest *req)
31888eea45cSKlaus Jensen {
31988eea45cSKlaus Jensen return le16_to_cpu(req->sq->sqid);
32088eea45cSKlaus Jensen }
32188eea45cSKlaus Jensen
nvme_make_pid(NvmeNamespace * ns,uint16_t rg,uint16_t ph)32273064edfSJesper Devantier static inline uint16_t nvme_make_pid(NvmeNamespace *ns, uint16_t rg,
32373064edfSJesper Devantier uint16_t ph)
32473064edfSJesper Devantier {
32573064edfSJesper Devantier uint16_t rgif = ns->endgrp->fdp.rgif;
32673064edfSJesper Devantier
32773064edfSJesper Devantier if (!rgif) {
32873064edfSJesper Devantier return ph;
32973064edfSJesper Devantier }
33073064edfSJesper Devantier
33173064edfSJesper Devantier return (rg << (16 - rgif)) | ph;
33273064edfSJesper Devantier }
33373064edfSJesper Devantier
nvme_ph_valid(NvmeNamespace * ns,uint16_t ph)33473064edfSJesper Devantier static inline bool nvme_ph_valid(NvmeNamespace *ns, uint16_t ph)
33573064edfSJesper Devantier {
33673064edfSJesper Devantier return ph < ns->fdp.nphs;
33773064edfSJesper Devantier }
33873064edfSJesper Devantier
nvme_rg_valid(NvmeEnduranceGroup * endgrp,uint16_t rg)33973064edfSJesper Devantier static inline bool nvme_rg_valid(NvmeEnduranceGroup *endgrp, uint16_t rg)
34073064edfSJesper Devantier {
34173064edfSJesper Devantier return rg < endgrp->fdp.nrg;
34273064edfSJesper Devantier }
34373064edfSJesper Devantier
nvme_pid2ph(NvmeNamespace * ns,uint16_t pid)34473064edfSJesper Devantier static inline uint16_t nvme_pid2ph(NvmeNamespace *ns, uint16_t pid)
34573064edfSJesper Devantier {
34673064edfSJesper Devantier uint16_t rgif = ns->endgrp->fdp.rgif;
34773064edfSJesper Devantier
34873064edfSJesper Devantier if (!rgif) {
34973064edfSJesper Devantier return pid;
35073064edfSJesper Devantier }
35173064edfSJesper Devantier
35273064edfSJesper Devantier return pid & ((1 << (15 - rgif)) - 1);
35373064edfSJesper Devantier }
35473064edfSJesper Devantier
nvme_pid2rg(NvmeNamespace * ns,uint16_t pid)35573064edfSJesper Devantier static inline uint16_t nvme_pid2rg(NvmeNamespace *ns, uint16_t pid)
35673064edfSJesper Devantier {
35773064edfSJesper Devantier uint16_t rgif = ns->endgrp->fdp.rgif;
35873064edfSJesper Devantier
35973064edfSJesper Devantier if (!rgif) {
36073064edfSJesper Devantier return 0;
36173064edfSJesper Devantier }
36273064edfSJesper Devantier
36373064edfSJesper Devantier return pid >> (16 - rgif);
36473064edfSJesper Devantier }
36573064edfSJesper Devantier
nvme_parse_pid(NvmeNamespace * ns,uint16_t pid,uint16_t * ph,uint16_t * rg)36673064edfSJesper Devantier static inline bool nvme_parse_pid(NvmeNamespace *ns, uint16_t pid,
36773064edfSJesper Devantier uint16_t *ph, uint16_t *rg)
36873064edfSJesper Devantier {
36973064edfSJesper Devantier *rg = nvme_pid2rg(ns, pid);
37073064edfSJesper Devantier *ph = nvme_pid2ph(ns, pid);
37173064edfSJesper Devantier
37273064edfSJesper Devantier return nvme_ph_valid(ns, *ph) && nvme_rg_valid(ns->endgrp, *rg);
37373064edfSJesper Devantier }
37473064edfSJesper Devantier
nvme_assign_zone_state(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state)37588eea45cSKlaus Jensen static void nvme_assign_zone_state(NvmeNamespace *ns, NvmeZone *zone,
37688eea45cSKlaus Jensen NvmeZoneState state)
37788eea45cSKlaus Jensen {
37888eea45cSKlaus Jensen if (QTAILQ_IN_USE(zone, entry)) {
37988eea45cSKlaus Jensen switch (nvme_get_zone_state(zone)) {
38088eea45cSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
38188eea45cSKlaus Jensen QTAILQ_REMOVE(&ns->exp_open_zones, zone, entry);
38288eea45cSKlaus Jensen break;
38388eea45cSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
38488eea45cSKlaus Jensen QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
38588eea45cSKlaus Jensen break;
38688eea45cSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
38788eea45cSKlaus Jensen QTAILQ_REMOVE(&ns->closed_zones, zone, entry);
38888eea45cSKlaus Jensen break;
38988eea45cSKlaus Jensen case NVME_ZONE_STATE_FULL:
39088eea45cSKlaus Jensen QTAILQ_REMOVE(&ns->full_zones, zone, entry);
39188eea45cSKlaus Jensen default:
39288eea45cSKlaus Jensen ;
39388eea45cSKlaus Jensen }
39488eea45cSKlaus Jensen }
39588eea45cSKlaus Jensen
39688eea45cSKlaus Jensen nvme_set_zone_state(zone, state);
39788eea45cSKlaus Jensen
39888eea45cSKlaus Jensen switch (state) {
39988eea45cSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
40088eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&ns->exp_open_zones, zone, entry);
40188eea45cSKlaus Jensen break;
40288eea45cSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
40388eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&ns->imp_open_zones, zone, entry);
40488eea45cSKlaus Jensen break;
40588eea45cSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
40688eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&ns->closed_zones, zone, entry);
40788eea45cSKlaus Jensen break;
40888eea45cSKlaus Jensen case NVME_ZONE_STATE_FULL:
40988eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&ns->full_zones, zone, entry);
41088eea45cSKlaus Jensen case NVME_ZONE_STATE_READ_ONLY:
41188eea45cSKlaus Jensen break;
41288eea45cSKlaus Jensen default:
41388eea45cSKlaus Jensen zone->d.za = 0;
41488eea45cSKlaus Jensen }
41588eea45cSKlaus Jensen }
41688eea45cSKlaus Jensen
nvme_zns_check_resources(NvmeNamespace * ns,uint32_t act,uint32_t opn,uint32_t zrwa)417e321b4cdSKlaus Jensen static uint16_t nvme_zns_check_resources(NvmeNamespace *ns, uint32_t act,
418e321b4cdSKlaus Jensen uint32_t opn, uint32_t zrwa)
41988eea45cSKlaus Jensen {
42088eea45cSKlaus Jensen if (ns->params.max_active_zones != 0 &&
42188eea45cSKlaus Jensen ns->nr_active_zones + act > ns->params.max_active_zones) {
42288eea45cSKlaus Jensen trace_pci_nvme_err_insuff_active_res(ns->params.max_active_zones);
42388eea45cSKlaus Jensen return NVME_ZONE_TOO_MANY_ACTIVE | NVME_DNR;
42488eea45cSKlaus Jensen }
425e321b4cdSKlaus Jensen
42688eea45cSKlaus Jensen if (ns->params.max_open_zones != 0 &&
42788eea45cSKlaus Jensen ns->nr_open_zones + opn > ns->params.max_open_zones) {
42888eea45cSKlaus Jensen trace_pci_nvme_err_insuff_open_res(ns->params.max_open_zones);
42988eea45cSKlaus Jensen return NVME_ZONE_TOO_MANY_OPEN | NVME_DNR;
43088eea45cSKlaus Jensen }
43188eea45cSKlaus Jensen
432e321b4cdSKlaus Jensen if (zrwa > ns->zns.numzrwa) {
433e321b4cdSKlaus Jensen return NVME_NOZRWA | NVME_DNR;
434e321b4cdSKlaus Jensen }
435e321b4cdSKlaus Jensen
43688eea45cSKlaus Jensen return NVME_SUCCESS;
43788eea45cSKlaus Jensen }
43888eea45cSKlaus Jensen
439e321b4cdSKlaus Jensen /*
440e321b4cdSKlaus Jensen * Check if we can open a zone without exceeding open/active limits.
441e321b4cdSKlaus Jensen * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5).
442e321b4cdSKlaus Jensen */
nvme_aor_check(NvmeNamespace * ns,uint32_t act,uint32_t opn)443e321b4cdSKlaus Jensen static uint16_t nvme_aor_check(NvmeNamespace *ns, uint32_t act, uint32_t opn)
444e321b4cdSKlaus Jensen {
445e321b4cdSKlaus Jensen return nvme_zns_check_resources(ns, act, opn, 0);
446e321b4cdSKlaus Jensen }
447e321b4cdSKlaus Jensen
nvme_fdp_alloc_event(NvmeCtrl * n,NvmeFdpEventBuffer * ebuf)44873064edfSJesper Devantier static NvmeFdpEvent *nvme_fdp_alloc_event(NvmeCtrl *n, NvmeFdpEventBuffer *ebuf)
44973064edfSJesper Devantier {
45073064edfSJesper Devantier NvmeFdpEvent *ret = NULL;
45173064edfSJesper Devantier bool is_full = ebuf->next == ebuf->start && ebuf->nelems;
45273064edfSJesper Devantier
45373064edfSJesper Devantier ret = &ebuf->events[ebuf->next++];
45473064edfSJesper Devantier if (unlikely(ebuf->next == NVME_FDP_MAX_EVENTS)) {
45573064edfSJesper Devantier ebuf->next = 0;
45673064edfSJesper Devantier }
45773064edfSJesper Devantier if (is_full) {
45873064edfSJesper Devantier ebuf->start = ebuf->next;
45973064edfSJesper Devantier } else {
46073064edfSJesper Devantier ebuf->nelems++;
46173064edfSJesper Devantier }
46273064edfSJesper Devantier
46373064edfSJesper Devantier memset(ret, 0, sizeof(NvmeFdpEvent));
46473064edfSJesper Devantier ret->timestamp = nvme_get_timestamp(n);
46573064edfSJesper Devantier
46673064edfSJesper Devantier return ret;
46773064edfSJesper Devantier }
46873064edfSJesper Devantier
log_event(NvmeRuHandle * ruh,uint8_t event_type)46973064edfSJesper Devantier static inline int log_event(NvmeRuHandle *ruh, uint8_t event_type)
47073064edfSJesper Devantier {
47173064edfSJesper Devantier return (ruh->event_filter >> nvme_fdp_evf_shifts[event_type]) & 0x1;
47273064edfSJesper Devantier }
47373064edfSJesper Devantier
nvme_update_ruh(NvmeCtrl * n,NvmeNamespace * ns,uint16_t pid)47473064edfSJesper Devantier static bool nvme_update_ruh(NvmeCtrl *n, NvmeNamespace *ns, uint16_t pid)
47573064edfSJesper Devantier {
47673064edfSJesper Devantier NvmeEnduranceGroup *endgrp = ns->endgrp;
47773064edfSJesper Devantier NvmeRuHandle *ruh;
47873064edfSJesper Devantier NvmeReclaimUnit *ru;
47973064edfSJesper Devantier NvmeFdpEvent *e = NULL;
48073064edfSJesper Devantier uint16_t ph, rg, ruhid;
48173064edfSJesper Devantier
48273064edfSJesper Devantier if (!nvme_parse_pid(ns, pid, &ph, &rg)) {
48373064edfSJesper Devantier return false;
48473064edfSJesper Devantier }
48573064edfSJesper Devantier
48673064edfSJesper Devantier ruhid = ns->fdp.phs[ph];
48773064edfSJesper Devantier
48873064edfSJesper Devantier ruh = &endgrp->fdp.ruhs[ruhid];
48973064edfSJesper Devantier ru = &ruh->rus[rg];
49073064edfSJesper Devantier
49173064edfSJesper Devantier if (ru->ruamw) {
49273064edfSJesper Devantier if (log_event(ruh, FDP_EVT_RU_NOT_FULLY_WRITTEN)) {
49373064edfSJesper Devantier e = nvme_fdp_alloc_event(n, &endgrp->fdp.host_events);
49473064edfSJesper Devantier e->type = FDP_EVT_RU_NOT_FULLY_WRITTEN;
49573064edfSJesper Devantier e->flags = FDPEF_PIV | FDPEF_NSIDV | FDPEF_LV;
49673064edfSJesper Devantier e->pid = cpu_to_le16(pid);
49773064edfSJesper Devantier e->nsid = cpu_to_le32(ns->params.nsid);
49873064edfSJesper Devantier e->rgid = cpu_to_le16(rg);
49973064edfSJesper Devantier e->ruhid = cpu_to_le16(ruhid);
50073064edfSJesper Devantier }
50173064edfSJesper Devantier
50273064edfSJesper Devantier /* log (eventual) GC overhead of prematurely swapping the RU */
50373064edfSJesper Devantier nvme_fdp_stat_inc(&endgrp->fdp.mbmw, nvme_l2b(ns, ru->ruamw));
50473064edfSJesper Devantier }
50573064edfSJesper Devantier
50673064edfSJesper Devantier ru->ruamw = ruh->ruamw;
50773064edfSJesper Devantier
50873064edfSJesper Devantier return true;
50973064edfSJesper Devantier }
51073064edfSJesper Devantier
nvme_addr_is_cmb(NvmeCtrl * n,hwaddr addr)51188eea45cSKlaus Jensen static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
51288eea45cSKlaus Jensen {
51388eea45cSKlaus Jensen hwaddr hi, lo;
51488eea45cSKlaus Jensen
51588eea45cSKlaus Jensen if (!n->cmb.cmse) {
51688eea45cSKlaus Jensen return false;
51788eea45cSKlaus Jensen }
51888eea45cSKlaus Jensen
51988eea45cSKlaus Jensen lo = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
52088eea45cSKlaus Jensen hi = lo + int128_get64(n->cmb.mem.size);
52188eea45cSKlaus Jensen
52288eea45cSKlaus Jensen return addr >= lo && addr < hi;
52388eea45cSKlaus Jensen }
52488eea45cSKlaus Jensen
nvme_addr_to_cmb(NvmeCtrl * n,hwaddr addr)52588eea45cSKlaus Jensen static inline void *nvme_addr_to_cmb(NvmeCtrl *n, hwaddr addr)
52688eea45cSKlaus Jensen {
52788eea45cSKlaus Jensen hwaddr base = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
52888eea45cSKlaus Jensen return &n->cmb.buf[addr - base];
52988eea45cSKlaus Jensen }
53088eea45cSKlaus Jensen
nvme_addr_is_pmr(NvmeCtrl * n,hwaddr addr)53188eea45cSKlaus Jensen static bool nvme_addr_is_pmr(NvmeCtrl *n, hwaddr addr)
53288eea45cSKlaus Jensen {
53388eea45cSKlaus Jensen hwaddr hi;
53488eea45cSKlaus Jensen
53588eea45cSKlaus Jensen if (!n->pmr.cmse) {
53688eea45cSKlaus Jensen return false;
53788eea45cSKlaus Jensen }
53888eea45cSKlaus Jensen
53988eea45cSKlaus Jensen hi = n->pmr.cba + int128_get64(n->pmr.dev->mr.size);
54088eea45cSKlaus Jensen
54188eea45cSKlaus Jensen return addr >= n->pmr.cba && addr < hi;
54288eea45cSKlaus Jensen }
54388eea45cSKlaus Jensen
nvme_addr_to_pmr(NvmeCtrl * n,hwaddr addr)54488eea45cSKlaus Jensen static inline void *nvme_addr_to_pmr(NvmeCtrl *n, hwaddr addr)
54588eea45cSKlaus Jensen {
54688eea45cSKlaus Jensen return memory_region_get_ram_ptr(&n->pmr.dev->mr) + (addr - n->pmr.cba);
54788eea45cSKlaus Jensen }
54888eea45cSKlaus Jensen
nvme_addr_is_iomem(NvmeCtrl * n,hwaddr addr)549736b0164SKlaus Jensen static inline bool nvme_addr_is_iomem(NvmeCtrl *n, hwaddr addr)
550736b0164SKlaus Jensen {
551736b0164SKlaus Jensen hwaddr hi, lo;
552736b0164SKlaus Jensen
553736b0164SKlaus Jensen /*
554736b0164SKlaus Jensen * The purpose of this check is to guard against invalid "local" access to
555736b0164SKlaus Jensen * the iomem (i.e. controller registers). Thus, we check against the range
556736b0164SKlaus Jensen * covered by the 'bar0' MemoryRegion since that is currently composed of
557736b0164SKlaus Jensen * two subregions (the NVMe "MBAR" and the MSI-X table/pba). Note, however,
558736b0164SKlaus Jensen * that if the device model is ever changed to allow the CMB to be located
559736b0164SKlaus Jensen * in BAR0 as well, then this must be changed.
560736b0164SKlaus Jensen */
561736b0164SKlaus Jensen lo = n->bar0.addr;
562736b0164SKlaus Jensen hi = lo + int128_get64(n->bar0.size);
563736b0164SKlaus Jensen
564736b0164SKlaus Jensen return addr >= lo && addr < hi;
565736b0164SKlaus Jensen }
566736b0164SKlaus Jensen
nvme_addr_read(NvmeCtrl * n,hwaddr addr,void * buf,int size)56788eea45cSKlaus Jensen static int nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
56888eea45cSKlaus Jensen {
56988eea45cSKlaus Jensen hwaddr hi = addr + size - 1;
57088eea45cSKlaus Jensen if (hi < addr) {
57188eea45cSKlaus Jensen return 1;
57288eea45cSKlaus Jensen }
57388eea45cSKlaus Jensen
57488eea45cSKlaus Jensen if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
57588eea45cSKlaus Jensen memcpy(buf, nvme_addr_to_cmb(n, addr), size);
57688eea45cSKlaus Jensen return 0;
57788eea45cSKlaus Jensen }
57888eea45cSKlaus Jensen
57988eea45cSKlaus Jensen if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
58088eea45cSKlaus Jensen memcpy(buf, nvme_addr_to_pmr(n, addr), size);
58188eea45cSKlaus Jensen return 0;
58288eea45cSKlaus Jensen }
58388eea45cSKlaus Jensen
58448b32c28SKlaus Jensen return pci_dma_read(PCI_DEVICE(n), addr, buf, size);
58588eea45cSKlaus Jensen }
58688eea45cSKlaus Jensen
nvme_addr_write(NvmeCtrl * n,hwaddr addr,const void * buf,int size)587e080ce86SPhilippe Mathieu-Daudé static int nvme_addr_write(NvmeCtrl *n, hwaddr addr, const void *buf, int size)
58888eea45cSKlaus Jensen {
58988eea45cSKlaus Jensen hwaddr hi = addr + size - 1;
59088eea45cSKlaus Jensen if (hi < addr) {
59188eea45cSKlaus Jensen return 1;
59288eea45cSKlaus Jensen }
59388eea45cSKlaus Jensen
59488eea45cSKlaus Jensen if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
59588eea45cSKlaus Jensen memcpy(nvme_addr_to_cmb(n, addr), buf, size);
59688eea45cSKlaus Jensen return 0;
59788eea45cSKlaus Jensen }
59888eea45cSKlaus Jensen
59988eea45cSKlaus Jensen if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
60088eea45cSKlaus Jensen memcpy(nvme_addr_to_pmr(n, addr), buf, size);
60188eea45cSKlaus Jensen return 0;
60288eea45cSKlaus Jensen }
60388eea45cSKlaus Jensen
60448b32c28SKlaus Jensen return pci_dma_write(PCI_DEVICE(n), addr, buf, size);
60588eea45cSKlaus Jensen }
60688eea45cSKlaus Jensen
nvme_nsid_valid(NvmeCtrl * n,uint32_t nsid)60788eea45cSKlaus Jensen static bool nvme_nsid_valid(NvmeCtrl *n, uint32_t nsid)
60888eea45cSKlaus Jensen {
60988eea45cSKlaus Jensen return nsid &&
61088eea45cSKlaus Jensen (nsid == NVME_NSID_BROADCAST || nsid <= NVME_MAX_NAMESPACES);
61188eea45cSKlaus Jensen }
61288eea45cSKlaus Jensen
nvme_check_sqid(NvmeCtrl * n,uint16_t sqid)61388eea45cSKlaus Jensen static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
61488eea45cSKlaus Jensen {
615decc0261SŁukasz Gieryk return sqid < n->conf_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1;
61688eea45cSKlaus Jensen }
61788eea45cSKlaus Jensen
nvme_check_cqid(NvmeCtrl * n,uint16_t cqid)61888eea45cSKlaus Jensen static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
61988eea45cSKlaus Jensen {
620decc0261SŁukasz Gieryk return cqid < n->conf_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1;
62188eea45cSKlaus Jensen }
62288eea45cSKlaus Jensen
nvme_inc_cq_tail(NvmeCQueue * cq)62388eea45cSKlaus Jensen static void nvme_inc_cq_tail(NvmeCQueue *cq)
62488eea45cSKlaus Jensen {
62588eea45cSKlaus Jensen cq->tail++;
62688eea45cSKlaus Jensen if (cq->tail >= cq->size) {
62788eea45cSKlaus Jensen cq->tail = 0;
62888eea45cSKlaus Jensen cq->phase = !cq->phase;
62988eea45cSKlaus Jensen }
63088eea45cSKlaus Jensen }
63188eea45cSKlaus Jensen
nvme_inc_sq_head(NvmeSQueue * sq)63288eea45cSKlaus Jensen static void nvme_inc_sq_head(NvmeSQueue *sq)
63388eea45cSKlaus Jensen {
63488eea45cSKlaus Jensen sq->head = (sq->head + 1) % sq->size;
63588eea45cSKlaus Jensen }
63688eea45cSKlaus Jensen
nvme_cq_full(NvmeCQueue * cq)63788eea45cSKlaus Jensen static uint8_t nvme_cq_full(NvmeCQueue *cq)
63888eea45cSKlaus Jensen {
63988eea45cSKlaus Jensen return (cq->tail + 1) % cq->size == cq->head;
64088eea45cSKlaus Jensen }
64188eea45cSKlaus Jensen
nvme_sq_empty(NvmeSQueue * sq)64288eea45cSKlaus Jensen static uint8_t nvme_sq_empty(NvmeSQueue *sq)
64388eea45cSKlaus Jensen {
64488eea45cSKlaus Jensen return sq->head == sq->tail;
64588eea45cSKlaus Jensen }
64688eea45cSKlaus Jensen
nvme_irq_check(NvmeCtrl * n)64788eea45cSKlaus Jensen static void nvme_irq_check(NvmeCtrl *n)
64888eea45cSKlaus Jensen {
64948b32c28SKlaus Jensen PCIDevice *pci = PCI_DEVICE(n);
65049e03457SKlaus Jensen uint32_t intms = ldl_le_p(&n->bar.intms);
65149e03457SKlaus Jensen
65248b32c28SKlaus Jensen if (msix_enabled(pci)) {
65388eea45cSKlaus Jensen return;
65488eea45cSKlaus Jensen }
65549e03457SKlaus Jensen if (~intms & n->irq_status) {
65648b32c28SKlaus Jensen pci_irq_assert(pci);
65788eea45cSKlaus Jensen } else {
65848b32c28SKlaus Jensen pci_irq_deassert(pci);
65988eea45cSKlaus Jensen }
66088eea45cSKlaus Jensen }
66188eea45cSKlaus Jensen
nvme_irq_assert(NvmeCtrl * n,NvmeCQueue * cq)66288eea45cSKlaus Jensen static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
66388eea45cSKlaus Jensen {
66448b32c28SKlaus Jensen PCIDevice *pci = PCI_DEVICE(n);
66548b32c28SKlaus Jensen
66688eea45cSKlaus Jensen if (cq->irq_enabled) {
66748b32c28SKlaus Jensen if (msix_enabled(pci)) {
66888eea45cSKlaus Jensen trace_pci_nvme_irq_msix(cq->vector);
66948b32c28SKlaus Jensen msix_notify(pci, cq->vector);
67088eea45cSKlaus Jensen } else {
67188eea45cSKlaus Jensen trace_pci_nvme_irq_pin();
67288eea45cSKlaus Jensen assert(cq->vector < 32);
67388eea45cSKlaus Jensen n->irq_status |= 1 << cq->vector;
67488eea45cSKlaus Jensen nvme_irq_check(n);
67588eea45cSKlaus Jensen }
67688eea45cSKlaus Jensen } else {
67788eea45cSKlaus Jensen trace_pci_nvme_irq_masked();
67888eea45cSKlaus Jensen }
67988eea45cSKlaus Jensen }
68088eea45cSKlaus Jensen
nvme_irq_deassert(NvmeCtrl * n,NvmeCQueue * cq)68188eea45cSKlaus Jensen static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
68288eea45cSKlaus Jensen {
68388eea45cSKlaus Jensen if (cq->irq_enabled) {
68448b32c28SKlaus Jensen if (msix_enabled(PCI_DEVICE(n))) {
68588eea45cSKlaus Jensen return;
68688eea45cSKlaus Jensen } else {
68788eea45cSKlaus Jensen assert(cq->vector < 32);
68883d7ed5cSKlaus Jensen if (!n->cq_pending) {
68988eea45cSKlaus Jensen n->irq_status &= ~(1 << cq->vector);
69083d7ed5cSKlaus Jensen }
69188eea45cSKlaus Jensen nvme_irq_check(n);
69288eea45cSKlaus Jensen }
69388eea45cSKlaus Jensen }
69488eea45cSKlaus Jensen }
69588eea45cSKlaus Jensen
nvme_req_clear(NvmeRequest * req)69688eea45cSKlaus Jensen static void nvme_req_clear(NvmeRequest *req)
69788eea45cSKlaus Jensen {
69888eea45cSKlaus Jensen req->ns = NULL;
69988eea45cSKlaus Jensen req->opaque = NULL;
70088eea45cSKlaus Jensen req->aiocb = NULL;
70188eea45cSKlaus Jensen memset(&req->cqe, 0x0, sizeof(req->cqe));
70288eea45cSKlaus Jensen req->status = NVME_SUCCESS;
70388eea45cSKlaus Jensen }
70488eea45cSKlaus Jensen
nvme_sg_init(NvmeCtrl * n,NvmeSg * sg,bool dma)70588eea45cSKlaus Jensen static inline void nvme_sg_init(NvmeCtrl *n, NvmeSg *sg, bool dma)
70688eea45cSKlaus Jensen {
70788eea45cSKlaus Jensen if (dma) {
70848b32c28SKlaus Jensen pci_dma_sglist_init(&sg->qsg, PCI_DEVICE(n), 0);
70988eea45cSKlaus Jensen sg->flags = NVME_SG_DMA;
71088eea45cSKlaus Jensen } else {
71188eea45cSKlaus Jensen qemu_iovec_init(&sg->iov, 0);
71288eea45cSKlaus Jensen }
71388eea45cSKlaus Jensen
71488eea45cSKlaus Jensen sg->flags |= NVME_SG_ALLOC;
71588eea45cSKlaus Jensen }
71688eea45cSKlaus Jensen
nvme_sg_unmap(NvmeSg * sg)71788eea45cSKlaus Jensen static inline void nvme_sg_unmap(NvmeSg *sg)
71888eea45cSKlaus Jensen {
71988eea45cSKlaus Jensen if (!(sg->flags & NVME_SG_ALLOC)) {
72088eea45cSKlaus Jensen return;
72188eea45cSKlaus Jensen }
72288eea45cSKlaus Jensen
72388eea45cSKlaus Jensen if (sg->flags & NVME_SG_DMA) {
72488eea45cSKlaus Jensen qemu_sglist_destroy(&sg->qsg);
72588eea45cSKlaus Jensen } else {
72688eea45cSKlaus Jensen qemu_iovec_destroy(&sg->iov);
72788eea45cSKlaus Jensen }
72888eea45cSKlaus Jensen
72988eea45cSKlaus Jensen memset(sg, 0x0, sizeof(*sg));
73088eea45cSKlaus Jensen }
73188eea45cSKlaus Jensen
73288eea45cSKlaus Jensen /*
7339b4b4e51SMichael Tokarev * When metadata is transferred as extended LBAs, the DPTR mapped into `sg`
73488eea45cSKlaus Jensen * holds both data and metadata. This function splits the data and metadata
73588eea45cSKlaus Jensen * into two separate QSG/IOVs.
73688eea45cSKlaus Jensen */
nvme_sg_split(NvmeSg * sg,NvmeNamespace * ns,NvmeSg * data,NvmeSg * mdata)73788eea45cSKlaus Jensen static void nvme_sg_split(NvmeSg *sg, NvmeNamespace *ns, NvmeSg *data,
73888eea45cSKlaus Jensen NvmeSg *mdata)
73988eea45cSKlaus Jensen {
74088eea45cSKlaus Jensen NvmeSg *dst = data;
74188eea45cSKlaus Jensen uint32_t trans_len, count = ns->lbasz;
74288eea45cSKlaus Jensen uint64_t offset = 0;
74388eea45cSKlaus Jensen bool dma = sg->flags & NVME_SG_DMA;
74488eea45cSKlaus Jensen size_t sge_len;
74588eea45cSKlaus Jensen size_t sg_len = dma ? sg->qsg.size : sg->iov.size;
74688eea45cSKlaus Jensen int sg_idx = 0;
74788eea45cSKlaus Jensen
74888eea45cSKlaus Jensen assert(sg->flags & NVME_SG_ALLOC);
74988eea45cSKlaus Jensen
75088eea45cSKlaus Jensen while (sg_len) {
75188eea45cSKlaus Jensen sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
75288eea45cSKlaus Jensen
75388eea45cSKlaus Jensen trans_len = MIN(sg_len, count);
75488eea45cSKlaus Jensen trans_len = MIN(trans_len, sge_len - offset);
75588eea45cSKlaus Jensen
75688eea45cSKlaus Jensen if (dst) {
75788eea45cSKlaus Jensen if (dma) {
75888eea45cSKlaus Jensen qemu_sglist_add(&dst->qsg, sg->qsg.sg[sg_idx].base + offset,
75988eea45cSKlaus Jensen trans_len);
76088eea45cSKlaus Jensen } else {
76188eea45cSKlaus Jensen qemu_iovec_add(&dst->iov,
76288eea45cSKlaus Jensen sg->iov.iov[sg_idx].iov_base + offset,
76388eea45cSKlaus Jensen trans_len);
76488eea45cSKlaus Jensen }
76588eea45cSKlaus Jensen }
76688eea45cSKlaus Jensen
76788eea45cSKlaus Jensen sg_len -= trans_len;
76888eea45cSKlaus Jensen count -= trans_len;
76988eea45cSKlaus Jensen offset += trans_len;
77088eea45cSKlaus Jensen
77188eea45cSKlaus Jensen if (count == 0) {
77288eea45cSKlaus Jensen dst = (dst == data) ? mdata : data;
77388eea45cSKlaus Jensen count = (dst == data) ? ns->lbasz : ns->lbaf.ms;
77488eea45cSKlaus Jensen }
77588eea45cSKlaus Jensen
77688eea45cSKlaus Jensen if (sge_len == offset) {
77788eea45cSKlaus Jensen offset = 0;
77888eea45cSKlaus Jensen sg_idx++;
77988eea45cSKlaus Jensen }
78088eea45cSKlaus Jensen }
78188eea45cSKlaus Jensen }
78288eea45cSKlaus Jensen
nvme_map_addr_cmb(NvmeCtrl * n,QEMUIOVector * iov,hwaddr addr,size_t len)78388eea45cSKlaus Jensen static uint16_t nvme_map_addr_cmb(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
78488eea45cSKlaus Jensen size_t len)
78588eea45cSKlaus Jensen {
78688eea45cSKlaus Jensen if (!len) {
78788eea45cSKlaus Jensen return NVME_SUCCESS;
78888eea45cSKlaus Jensen }
78988eea45cSKlaus Jensen
79088eea45cSKlaus Jensen trace_pci_nvme_map_addr_cmb(addr, len);
79188eea45cSKlaus Jensen
79288eea45cSKlaus Jensen if (!nvme_addr_is_cmb(n, addr) || !nvme_addr_is_cmb(n, addr + len - 1)) {
79388eea45cSKlaus Jensen return NVME_DATA_TRAS_ERROR;
79488eea45cSKlaus Jensen }
79588eea45cSKlaus Jensen
79688eea45cSKlaus Jensen qemu_iovec_add(iov, nvme_addr_to_cmb(n, addr), len);
79788eea45cSKlaus Jensen
79888eea45cSKlaus Jensen return NVME_SUCCESS;
79988eea45cSKlaus Jensen }
80088eea45cSKlaus Jensen
nvme_map_addr_pmr(NvmeCtrl * n,QEMUIOVector * iov,hwaddr addr,size_t len)80188eea45cSKlaus Jensen static uint16_t nvme_map_addr_pmr(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
80288eea45cSKlaus Jensen size_t len)
80388eea45cSKlaus Jensen {
80488eea45cSKlaus Jensen if (!len) {
80588eea45cSKlaus Jensen return NVME_SUCCESS;
80688eea45cSKlaus Jensen }
80788eea45cSKlaus Jensen
80888eea45cSKlaus Jensen if (!nvme_addr_is_pmr(n, addr) || !nvme_addr_is_pmr(n, addr + len - 1)) {
80988eea45cSKlaus Jensen return NVME_DATA_TRAS_ERROR;
81088eea45cSKlaus Jensen }
81188eea45cSKlaus Jensen
81288eea45cSKlaus Jensen qemu_iovec_add(iov, nvme_addr_to_pmr(n, addr), len);
81388eea45cSKlaus Jensen
81488eea45cSKlaus Jensen return NVME_SUCCESS;
81588eea45cSKlaus Jensen }
81688eea45cSKlaus Jensen
nvme_map_addr(NvmeCtrl * n,NvmeSg * sg,hwaddr addr,size_t len)81788eea45cSKlaus Jensen static uint16_t nvme_map_addr(NvmeCtrl *n, NvmeSg *sg, hwaddr addr, size_t len)
81888eea45cSKlaus Jensen {
81988eea45cSKlaus Jensen bool cmb = false, pmr = false;
82088eea45cSKlaus Jensen
82188eea45cSKlaus Jensen if (!len) {
82288eea45cSKlaus Jensen return NVME_SUCCESS;
82388eea45cSKlaus Jensen }
82488eea45cSKlaus Jensen
82588eea45cSKlaus Jensen trace_pci_nvme_map_addr(addr, len);
82688eea45cSKlaus Jensen
827736b0164SKlaus Jensen if (nvme_addr_is_iomem(n, addr)) {
828736b0164SKlaus Jensen return NVME_DATA_TRAS_ERROR;
829736b0164SKlaus Jensen }
830736b0164SKlaus Jensen
83188eea45cSKlaus Jensen if (nvme_addr_is_cmb(n, addr)) {
83288eea45cSKlaus Jensen cmb = true;
83388eea45cSKlaus Jensen } else if (nvme_addr_is_pmr(n, addr)) {
83488eea45cSKlaus Jensen pmr = true;
83588eea45cSKlaus Jensen }
83688eea45cSKlaus Jensen
83788eea45cSKlaus Jensen if (cmb || pmr) {
83888eea45cSKlaus Jensen if (sg->flags & NVME_SG_DMA) {
83988eea45cSKlaus Jensen return NVME_INVALID_USE_OF_CMB | NVME_DNR;
84088eea45cSKlaus Jensen }
84188eea45cSKlaus Jensen
84223421473SPadmakar Kalghatgi if (sg->iov.niov + 1 > IOV_MAX) {
84323421473SPadmakar Kalghatgi goto max_mappings_exceeded;
84423421473SPadmakar Kalghatgi }
84523421473SPadmakar Kalghatgi
84688eea45cSKlaus Jensen if (cmb) {
84788eea45cSKlaus Jensen return nvme_map_addr_cmb(n, &sg->iov, addr, len);
84888eea45cSKlaus Jensen } else {
84988eea45cSKlaus Jensen return nvme_map_addr_pmr(n, &sg->iov, addr, len);
85088eea45cSKlaus Jensen }
85188eea45cSKlaus Jensen }
85288eea45cSKlaus Jensen
85388eea45cSKlaus Jensen if (!(sg->flags & NVME_SG_DMA)) {
85488eea45cSKlaus Jensen return NVME_INVALID_USE_OF_CMB | NVME_DNR;
85588eea45cSKlaus Jensen }
85688eea45cSKlaus Jensen
85723421473SPadmakar Kalghatgi if (sg->qsg.nsg + 1 > IOV_MAX) {
85823421473SPadmakar Kalghatgi goto max_mappings_exceeded;
85923421473SPadmakar Kalghatgi }
86023421473SPadmakar Kalghatgi
86188eea45cSKlaus Jensen qemu_sglist_add(&sg->qsg, addr, len);
86288eea45cSKlaus Jensen
86388eea45cSKlaus Jensen return NVME_SUCCESS;
86423421473SPadmakar Kalghatgi
86523421473SPadmakar Kalghatgi max_mappings_exceeded:
86623421473SPadmakar Kalghatgi NVME_GUEST_ERR(pci_nvme_ub_too_many_mappings,
86723421473SPadmakar Kalghatgi "number of mappings exceed 1024");
86823421473SPadmakar Kalghatgi return NVME_INTERNAL_DEV_ERROR | NVME_DNR;
86988eea45cSKlaus Jensen }
87088eea45cSKlaus Jensen
nvme_addr_is_dma(NvmeCtrl * n,hwaddr addr)87188eea45cSKlaus Jensen static inline bool nvme_addr_is_dma(NvmeCtrl *n, hwaddr addr)
87288eea45cSKlaus Jensen {
87388eea45cSKlaus Jensen return !(nvme_addr_is_cmb(n, addr) || nvme_addr_is_pmr(n, addr));
87488eea45cSKlaus Jensen }
87588eea45cSKlaus Jensen
nvme_map_prp(NvmeCtrl * n,NvmeSg * sg,uint64_t prp1,uint64_t prp2,uint32_t len)87688eea45cSKlaus Jensen static uint16_t nvme_map_prp(NvmeCtrl *n, NvmeSg *sg, uint64_t prp1,
87788eea45cSKlaus Jensen uint64_t prp2, uint32_t len)
87888eea45cSKlaus Jensen {
87988eea45cSKlaus Jensen hwaddr trans_len = n->page_size - (prp1 % n->page_size);
88088eea45cSKlaus Jensen trans_len = MIN(len, trans_len);
88188eea45cSKlaus Jensen int num_prps = (len >> n->page_bits) + 1;
88288eea45cSKlaus Jensen uint16_t status;
88388eea45cSKlaus Jensen int ret;
88488eea45cSKlaus Jensen
88588eea45cSKlaus Jensen trace_pci_nvme_map_prp(trans_len, len, prp1, prp2, num_prps);
88688eea45cSKlaus Jensen
88788eea45cSKlaus Jensen nvme_sg_init(n, sg, nvme_addr_is_dma(n, prp1));
88888eea45cSKlaus Jensen
88988eea45cSKlaus Jensen status = nvme_map_addr(n, sg, prp1, trans_len);
89088eea45cSKlaus Jensen if (status) {
89188eea45cSKlaus Jensen goto unmap;
89288eea45cSKlaus Jensen }
89388eea45cSKlaus Jensen
89488eea45cSKlaus Jensen len -= trans_len;
89588eea45cSKlaus Jensen if (len) {
89688eea45cSKlaus Jensen if (len > n->page_size) {
897b3c82467SPeter Maydell g_autofree uint64_t *prp_list = g_new(uint64_t, n->max_prp_ents);
89888eea45cSKlaus Jensen uint32_t nents, prp_trans;
89988eea45cSKlaus Jensen int i = 0;
90088eea45cSKlaus Jensen
90188eea45cSKlaus Jensen /*
90288eea45cSKlaus Jensen * The first PRP list entry, pointed to by PRP2 may contain offset.
90388eea45cSKlaus Jensen * Hence, we need to calculate the number of entries in based on
90488eea45cSKlaus Jensen * that offset.
90588eea45cSKlaus Jensen */
90688eea45cSKlaus Jensen nents = (n->page_size - (prp2 & (n->page_size - 1))) >> 3;
90788eea45cSKlaus Jensen prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
90888eea45cSKlaus Jensen ret = nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
90988eea45cSKlaus Jensen if (ret) {
91088eea45cSKlaus Jensen trace_pci_nvme_err_addr_read(prp2);
91188eea45cSKlaus Jensen status = NVME_DATA_TRAS_ERROR;
91288eea45cSKlaus Jensen goto unmap;
91388eea45cSKlaus Jensen }
91488eea45cSKlaus Jensen while (len != 0) {
91588eea45cSKlaus Jensen uint64_t prp_ent = le64_to_cpu(prp_list[i]);
91688eea45cSKlaus Jensen
91788eea45cSKlaus Jensen if (i == nents - 1 && len > n->page_size) {
91888eea45cSKlaus Jensen if (unlikely(prp_ent & (n->page_size - 1))) {
91988eea45cSKlaus Jensen trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
92088eea45cSKlaus Jensen status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
92188eea45cSKlaus Jensen goto unmap;
92288eea45cSKlaus Jensen }
92388eea45cSKlaus Jensen
92488eea45cSKlaus Jensen i = 0;
92588eea45cSKlaus Jensen nents = (len + n->page_size - 1) >> n->page_bits;
92688eea45cSKlaus Jensen nents = MIN(nents, n->max_prp_ents);
92788eea45cSKlaus Jensen prp_trans = nents * sizeof(uint64_t);
92888eea45cSKlaus Jensen ret = nvme_addr_read(n, prp_ent, (void *)prp_list,
92988eea45cSKlaus Jensen prp_trans);
93088eea45cSKlaus Jensen if (ret) {
93188eea45cSKlaus Jensen trace_pci_nvme_err_addr_read(prp_ent);
93288eea45cSKlaus Jensen status = NVME_DATA_TRAS_ERROR;
93388eea45cSKlaus Jensen goto unmap;
93488eea45cSKlaus Jensen }
93588eea45cSKlaus Jensen prp_ent = le64_to_cpu(prp_list[i]);
93688eea45cSKlaus Jensen }
93788eea45cSKlaus Jensen
93888eea45cSKlaus Jensen if (unlikely(prp_ent & (n->page_size - 1))) {
93988eea45cSKlaus Jensen trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
94088eea45cSKlaus Jensen status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
94188eea45cSKlaus Jensen goto unmap;
94288eea45cSKlaus Jensen }
94388eea45cSKlaus Jensen
94488eea45cSKlaus Jensen trans_len = MIN(len, n->page_size);
94588eea45cSKlaus Jensen status = nvme_map_addr(n, sg, prp_ent, trans_len);
94688eea45cSKlaus Jensen if (status) {
94788eea45cSKlaus Jensen goto unmap;
94888eea45cSKlaus Jensen }
94988eea45cSKlaus Jensen
95088eea45cSKlaus Jensen len -= trans_len;
95188eea45cSKlaus Jensen i++;
95288eea45cSKlaus Jensen }
95388eea45cSKlaus Jensen } else {
95488eea45cSKlaus Jensen if (unlikely(prp2 & (n->page_size - 1))) {
95588eea45cSKlaus Jensen trace_pci_nvme_err_invalid_prp2_align(prp2);
95688eea45cSKlaus Jensen status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
95788eea45cSKlaus Jensen goto unmap;
95888eea45cSKlaus Jensen }
95988eea45cSKlaus Jensen status = nvme_map_addr(n, sg, prp2, len);
96088eea45cSKlaus Jensen if (status) {
96188eea45cSKlaus Jensen goto unmap;
96288eea45cSKlaus Jensen }
96388eea45cSKlaus Jensen }
96488eea45cSKlaus Jensen }
96588eea45cSKlaus Jensen
96688eea45cSKlaus Jensen return NVME_SUCCESS;
96788eea45cSKlaus Jensen
96888eea45cSKlaus Jensen unmap:
96988eea45cSKlaus Jensen nvme_sg_unmap(sg);
97088eea45cSKlaus Jensen return status;
97188eea45cSKlaus Jensen }
97288eea45cSKlaus Jensen
97388eea45cSKlaus Jensen /*
97488eea45cSKlaus Jensen * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
97588eea45cSKlaus Jensen * number of bytes mapped in len.
97688eea45cSKlaus Jensen */
nvme_map_sgl_data(NvmeCtrl * n,NvmeSg * sg,NvmeSglDescriptor * segment,uint64_t nsgld,size_t * len,NvmeCmd * cmd)97788eea45cSKlaus Jensen static uint16_t nvme_map_sgl_data(NvmeCtrl *n, NvmeSg *sg,
97888eea45cSKlaus Jensen NvmeSglDescriptor *segment, uint64_t nsgld,
97988eea45cSKlaus Jensen size_t *len, NvmeCmd *cmd)
98088eea45cSKlaus Jensen {
98188eea45cSKlaus Jensen dma_addr_t addr, trans_len;
98288eea45cSKlaus Jensen uint32_t dlen;
98388eea45cSKlaus Jensen uint16_t status;
98488eea45cSKlaus Jensen
98588eea45cSKlaus Jensen for (int i = 0; i < nsgld; i++) {
98688eea45cSKlaus Jensen uint8_t type = NVME_SGL_TYPE(segment[i].type);
98788eea45cSKlaus Jensen
98888eea45cSKlaus Jensen switch (type) {
98988eea45cSKlaus Jensen case NVME_SGL_DESCR_TYPE_DATA_BLOCK:
99088eea45cSKlaus Jensen break;
99188eea45cSKlaus Jensen case NVME_SGL_DESCR_TYPE_SEGMENT:
99288eea45cSKlaus Jensen case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
99388eea45cSKlaus Jensen return NVME_INVALID_NUM_SGL_DESCRS | NVME_DNR;
99488eea45cSKlaus Jensen default:
99588eea45cSKlaus Jensen return NVME_SGL_DESCR_TYPE_INVALID | NVME_DNR;
99688eea45cSKlaus Jensen }
99788eea45cSKlaus Jensen
99888eea45cSKlaus Jensen dlen = le32_to_cpu(segment[i].len);
99988eea45cSKlaus Jensen
100088eea45cSKlaus Jensen if (!dlen) {
100188eea45cSKlaus Jensen continue;
100288eea45cSKlaus Jensen }
100388eea45cSKlaus Jensen
100488eea45cSKlaus Jensen if (*len == 0) {
100588eea45cSKlaus Jensen /*
100688eea45cSKlaus Jensen * All data has been mapped, but the SGL contains additional
100788eea45cSKlaus Jensen * segments and/or descriptors. The controller might accept
100888eea45cSKlaus Jensen * ignoring the rest of the SGL.
100988eea45cSKlaus Jensen */
101088eea45cSKlaus Jensen uint32_t sgls = le32_to_cpu(n->id_ctrl.sgls);
101188eea45cSKlaus Jensen if (sgls & NVME_CTRL_SGLS_EXCESS_LENGTH) {
101288eea45cSKlaus Jensen break;
101388eea45cSKlaus Jensen }
101488eea45cSKlaus Jensen
101588eea45cSKlaus Jensen trace_pci_nvme_err_invalid_sgl_excess_length(dlen);
101688eea45cSKlaus Jensen return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
101788eea45cSKlaus Jensen }
101888eea45cSKlaus Jensen
101988eea45cSKlaus Jensen trans_len = MIN(*len, dlen);
102088eea45cSKlaus Jensen
102188eea45cSKlaus Jensen addr = le64_to_cpu(segment[i].addr);
102288eea45cSKlaus Jensen
102388eea45cSKlaus Jensen if (UINT64_MAX - addr < dlen) {
102488eea45cSKlaus Jensen return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
102588eea45cSKlaus Jensen }
102688eea45cSKlaus Jensen
102788eea45cSKlaus Jensen status = nvme_map_addr(n, sg, addr, trans_len);
102888eea45cSKlaus Jensen if (status) {
102988eea45cSKlaus Jensen return status;
103088eea45cSKlaus Jensen }
103188eea45cSKlaus Jensen
103288eea45cSKlaus Jensen *len -= trans_len;
103388eea45cSKlaus Jensen }
103488eea45cSKlaus Jensen
103588eea45cSKlaus Jensen return NVME_SUCCESS;
103688eea45cSKlaus Jensen }
103788eea45cSKlaus Jensen
nvme_map_sgl(NvmeCtrl * n,NvmeSg * sg,NvmeSglDescriptor sgl,size_t len,NvmeCmd * cmd)103888eea45cSKlaus Jensen static uint16_t nvme_map_sgl(NvmeCtrl *n, NvmeSg *sg, NvmeSglDescriptor sgl,
103988eea45cSKlaus Jensen size_t len, NvmeCmd *cmd)
104088eea45cSKlaus Jensen {
104188eea45cSKlaus Jensen /*
104288eea45cSKlaus Jensen * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
104388eea45cSKlaus Jensen * dynamically allocating a potentially huge SGL. The spec allows the SGL
104488eea45cSKlaus Jensen * to be larger (as in number of bytes required to describe the SGL
104588eea45cSKlaus Jensen * descriptors and segment chain) than the command transfer size, so it is
104688eea45cSKlaus Jensen * not bounded by MDTS.
104788eea45cSKlaus Jensen */
1048b02c2a85SPhilippe Mathieu-Daudé #define SEG_CHUNK_SIZE 256
104988eea45cSKlaus Jensen
105088eea45cSKlaus Jensen NvmeSglDescriptor segment[SEG_CHUNK_SIZE], *sgld, *last_sgld;
105188eea45cSKlaus Jensen uint64_t nsgld;
105288eea45cSKlaus Jensen uint32_t seg_len;
105388eea45cSKlaus Jensen uint16_t status;
105488eea45cSKlaus Jensen hwaddr addr;
105588eea45cSKlaus Jensen int ret;
105688eea45cSKlaus Jensen
105788eea45cSKlaus Jensen sgld = &sgl;
105888eea45cSKlaus Jensen addr = le64_to_cpu(sgl.addr);
105988eea45cSKlaus Jensen
106088eea45cSKlaus Jensen trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl.type), len);
106188eea45cSKlaus Jensen
106288eea45cSKlaus Jensen nvme_sg_init(n, sg, nvme_addr_is_dma(n, addr));
106388eea45cSKlaus Jensen
106488eea45cSKlaus Jensen /*
106588eea45cSKlaus Jensen * If the entire transfer can be described with a single data block it can
106688eea45cSKlaus Jensen * be mapped directly.
106788eea45cSKlaus Jensen */
106888eea45cSKlaus Jensen if (NVME_SGL_TYPE(sgl.type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
106988eea45cSKlaus Jensen status = nvme_map_sgl_data(n, sg, sgld, 1, &len, cmd);
107088eea45cSKlaus Jensen if (status) {
107188eea45cSKlaus Jensen goto unmap;
107288eea45cSKlaus Jensen }
107388eea45cSKlaus Jensen
107488eea45cSKlaus Jensen goto out;
107588eea45cSKlaus Jensen }
107688eea45cSKlaus Jensen
107788eea45cSKlaus Jensen for (;;) {
107888eea45cSKlaus Jensen switch (NVME_SGL_TYPE(sgld->type)) {
107988eea45cSKlaus Jensen case NVME_SGL_DESCR_TYPE_SEGMENT:
108088eea45cSKlaus Jensen case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
108188eea45cSKlaus Jensen break;
108288eea45cSKlaus Jensen default:
108388eea45cSKlaus Jensen return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
108488eea45cSKlaus Jensen }
108588eea45cSKlaus Jensen
108688eea45cSKlaus Jensen seg_len = le32_to_cpu(sgld->len);
108788eea45cSKlaus Jensen
108888eea45cSKlaus Jensen /* check the length of the (Last) Segment descriptor */
1089b9147a3aSKlaus Jensen if (!seg_len || seg_len & 0xf) {
109088eea45cSKlaus Jensen return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
109188eea45cSKlaus Jensen }
109288eea45cSKlaus Jensen
109388eea45cSKlaus Jensen if (UINT64_MAX - addr < seg_len) {
109488eea45cSKlaus Jensen return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
109588eea45cSKlaus Jensen }
109688eea45cSKlaus Jensen
109788eea45cSKlaus Jensen nsgld = seg_len / sizeof(NvmeSglDescriptor);
109888eea45cSKlaus Jensen
109988eea45cSKlaus Jensen while (nsgld > SEG_CHUNK_SIZE) {
110088eea45cSKlaus Jensen if (nvme_addr_read(n, addr, segment, sizeof(segment))) {
110188eea45cSKlaus Jensen trace_pci_nvme_err_addr_read(addr);
110288eea45cSKlaus Jensen status = NVME_DATA_TRAS_ERROR;
110388eea45cSKlaus Jensen goto unmap;
110488eea45cSKlaus Jensen }
110588eea45cSKlaus Jensen
110688eea45cSKlaus Jensen status = nvme_map_sgl_data(n, sg, segment, SEG_CHUNK_SIZE,
110788eea45cSKlaus Jensen &len, cmd);
110888eea45cSKlaus Jensen if (status) {
110988eea45cSKlaus Jensen goto unmap;
111088eea45cSKlaus Jensen }
111188eea45cSKlaus Jensen
111288eea45cSKlaus Jensen nsgld -= SEG_CHUNK_SIZE;
111388eea45cSKlaus Jensen addr += SEG_CHUNK_SIZE * sizeof(NvmeSglDescriptor);
111488eea45cSKlaus Jensen }
111588eea45cSKlaus Jensen
111688eea45cSKlaus Jensen ret = nvme_addr_read(n, addr, segment, nsgld *
111788eea45cSKlaus Jensen sizeof(NvmeSglDescriptor));
111888eea45cSKlaus Jensen if (ret) {
111988eea45cSKlaus Jensen trace_pci_nvme_err_addr_read(addr);
112088eea45cSKlaus Jensen status = NVME_DATA_TRAS_ERROR;
112188eea45cSKlaus Jensen goto unmap;
112288eea45cSKlaus Jensen }
112388eea45cSKlaus Jensen
112488eea45cSKlaus Jensen last_sgld = &segment[nsgld - 1];
112588eea45cSKlaus Jensen
112688eea45cSKlaus Jensen /*
1127b9147a3aSKlaus Jensen * If the segment ends with a Data Block, then we are done.
112888eea45cSKlaus Jensen */
1129b9147a3aSKlaus Jensen if (NVME_SGL_TYPE(last_sgld->type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
113088eea45cSKlaus Jensen status = nvme_map_sgl_data(n, sg, segment, nsgld, &len, cmd);
113188eea45cSKlaus Jensen if (status) {
113288eea45cSKlaus Jensen goto unmap;
113388eea45cSKlaus Jensen }
113488eea45cSKlaus Jensen
113588eea45cSKlaus Jensen goto out;
113688eea45cSKlaus Jensen }
113788eea45cSKlaus Jensen
113888eea45cSKlaus Jensen /*
1139b9147a3aSKlaus Jensen * If the last descriptor was not a Data Block, then the current
1140b9147a3aSKlaus Jensen * segment must not be a Last Segment.
114188eea45cSKlaus Jensen */
114288eea45cSKlaus Jensen if (NVME_SGL_TYPE(sgld->type) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT) {
114388eea45cSKlaus Jensen status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
114488eea45cSKlaus Jensen goto unmap;
114588eea45cSKlaus Jensen }
114688eea45cSKlaus Jensen
114788eea45cSKlaus Jensen sgld = last_sgld;
114888eea45cSKlaus Jensen addr = le64_to_cpu(sgld->addr);
114988eea45cSKlaus Jensen
115088eea45cSKlaus Jensen /*
115188eea45cSKlaus Jensen * Do not map the last descriptor; it will be a Segment or Last Segment
115288eea45cSKlaus Jensen * descriptor and is handled by the next iteration.
115388eea45cSKlaus Jensen */
115488eea45cSKlaus Jensen status = nvme_map_sgl_data(n, sg, segment, nsgld - 1, &len, cmd);
115588eea45cSKlaus Jensen if (status) {
115688eea45cSKlaus Jensen goto unmap;
115788eea45cSKlaus Jensen }
115888eea45cSKlaus Jensen }
115988eea45cSKlaus Jensen
116088eea45cSKlaus Jensen out:
116188eea45cSKlaus Jensen /* if there is any residual left in len, the SGL was too short */
116288eea45cSKlaus Jensen if (len) {
116388eea45cSKlaus Jensen status = NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
116488eea45cSKlaus Jensen goto unmap;
116588eea45cSKlaus Jensen }
116688eea45cSKlaus Jensen
116788eea45cSKlaus Jensen return NVME_SUCCESS;
116888eea45cSKlaus Jensen
116988eea45cSKlaus Jensen unmap:
117088eea45cSKlaus Jensen nvme_sg_unmap(sg);
117188eea45cSKlaus Jensen return status;
117288eea45cSKlaus Jensen }
117388eea45cSKlaus Jensen
nvme_map_dptr(NvmeCtrl * n,NvmeSg * sg,size_t len,NvmeCmd * cmd)117488eea45cSKlaus Jensen uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
117588eea45cSKlaus Jensen NvmeCmd *cmd)
117688eea45cSKlaus Jensen {
117788eea45cSKlaus Jensen uint64_t prp1, prp2;
117888eea45cSKlaus Jensen
117988eea45cSKlaus Jensen switch (NVME_CMD_FLAGS_PSDT(cmd->flags)) {
118088eea45cSKlaus Jensen case NVME_PSDT_PRP:
118188eea45cSKlaus Jensen prp1 = le64_to_cpu(cmd->dptr.prp1);
118288eea45cSKlaus Jensen prp2 = le64_to_cpu(cmd->dptr.prp2);
118388eea45cSKlaus Jensen
118488eea45cSKlaus Jensen return nvme_map_prp(n, sg, prp1, prp2, len);
118588eea45cSKlaus Jensen case NVME_PSDT_SGL_MPTR_CONTIGUOUS:
118688eea45cSKlaus Jensen case NVME_PSDT_SGL_MPTR_SGL:
118788eea45cSKlaus Jensen return nvme_map_sgl(n, sg, cmd->dptr.sgl, len, cmd);
118888eea45cSKlaus Jensen default:
118988eea45cSKlaus Jensen return NVME_INVALID_FIELD;
119088eea45cSKlaus Jensen }
119188eea45cSKlaus Jensen }
119288eea45cSKlaus Jensen
nvme_map_mptr(NvmeCtrl * n,NvmeSg * sg,size_t len,NvmeCmd * cmd)119388eea45cSKlaus Jensen static uint16_t nvme_map_mptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
119488eea45cSKlaus Jensen NvmeCmd *cmd)
119588eea45cSKlaus Jensen {
119688eea45cSKlaus Jensen int psdt = NVME_CMD_FLAGS_PSDT(cmd->flags);
119788eea45cSKlaus Jensen hwaddr mptr = le64_to_cpu(cmd->mptr);
119888eea45cSKlaus Jensen uint16_t status;
119988eea45cSKlaus Jensen
120088eea45cSKlaus Jensen if (psdt == NVME_PSDT_SGL_MPTR_SGL) {
120188eea45cSKlaus Jensen NvmeSglDescriptor sgl;
120288eea45cSKlaus Jensen
120388eea45cSKlaus Jensen if (nvme_addr_read(n, mptr, &sgl, sizeof(sgl))) {
120488eea45cSKlaus Jensen return NVME_DATA_TRAS_ERROR;
120588eea45cSKlaus Jensen }
120688eea45cSKlaus Jensen
120788eea45cSKlaus Jensen status = nvme_map_sgl(n, sg, sgl, len, cmd);
120888eea45cSKlaus Jensen if (status && (status & 0x7ff) == NVME_DATA_SGL_LEN_INVALID) {
120988eea45cSKlaus Jensen status = NVME_MD_SGL_LEN_INVALID | NVME_DNR;
121088eea45cSKlaus Jensen }
121188eea45cSKlaus Jensen
121288eea45cSKlaus Jensen return status;
121388eea45cSKlaus Jensen }
121488eea45cSKlaus Jensen
121588eea45cSKlaus Jensen nvme_sg_init(n, sg, nvme_addr_is_dma(n, mptr));
121688eea45cSKlaus Jensen status = nvme_map_addr(n, sg, mptr, len);
121788eea45cSKlaus Jensen if (status) {
121888eea45cSKlaus Jensen nvme_sg_unmap(sg);
121988eea45cSKlaus Jensen }
122088eea45cSKlaus Jensen
122188eea45cSKlaus Jensen return status;
122288eea45cSKlaus Jensen }
122388eea45cSKlaus Jensen
nvme_map_data(NvmeCtrl * n,uint32_t nlb,NvmeRequest * req)122488eea45cSKlaus Jensen static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
122588eea45cSKlaus Jensen {
122688eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
122788eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
12282a132309SKlaus Jensen bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
12292a132309SKlaus Jensen bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
123088eea45cSKlaus Jensen size_t len = nvme_l2b(ns, nlb);
123188eea45cSKlaus Jensen uint16_t status;
123288eea45cSKlaus Jensen
1233ac0b34c5SKlaus Jensen if (nvme_ns_ext(ns) &&
1234ac0b34c5SKlaus Jensen !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
123588eea45cSKlaus Jensen NvmeSg sg;
123688eea45cSKlaus Jensen
123788eea45cSKlaus Jensen len += nvme_m2b(ns, nlb);
123888eea45cSKlaus Jensen
123988eea45cSKlaus Jensen status = nvme_map_dptr(n, &sg, len, &req->cmd);
124088eea45cSKlaus Jensen if (status) {
124188eea45cSKlaus Jensen return status;
124288eea45cSKlaus Jensen }
124388eea45cSKlaus Jensen
124488eea45cSKlaus Jensen nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
124588eea45cSKlaus Jensen nvme_sg_split(&sg, ns, &req->sg, NULL);
124688eea45cSKlaus Jensen nvme_sg_unmap(&sg);
124788eea45cSKlaus Jensen
124888eea45cSKlaus Jensen return NVME_SUCCESS;
124988eea45cSKlaus Jensen }
125088eea45cSKlaus Jensen
125188eea45cSKlaus Jensen return nvme_map_dptr(n, &req->sg, len, &req->cmd);
125288eea45cSKlaus Jensen }
125388eea45cSKlaus Jensen
nvme_map_mdata(NvmeCtrl * n,uint32_t nlb,NvmeRequest * req)125488eea45cSKlaus Jensen static uint16_t nvme_map_mdata(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
125588eea45cSKlaus Jensen {
125688eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
125788eea45cSKlaus Jensen size_t len = nvme_m2b(ns, nlb);
125888eea45cSKlaus Jensen uint16_t status;
125988eea45cSKlaus Jensen
126088eea45cSKlaus Jensen if (nvme_ns_ext(ns)) {
126188eea45cSKlaus Jensen NvmeSg sg;
126288eea45cSKlaus Jensen
126388eea45cSKlaus Jensen len += nvme_l2b(ns, nlb);
126488eea45cSKlaus Jensen
126588eea45cSKlaus Jensen status = nvme_map_dptr(n, &sg, len, &req->cmd);
126688eea45cSKlaus Jensen if (status) {
126788eea45cSKlaus Jensen return status;
126888eea45cSKlaus Jensen }
126988eea45cSKlaus Jensen
127088eea45cSKlaus Jensen nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
127188eea45cSKlaus Jensen nvme_sg_split(&sg, ns, NULL, &req->sg);
127288eea45cSKlaus Jensen nvme_sg_unmap(&sg);
127388eea45cSKlaus Jensen
127488eea45cSKlaus Jensen return NVME_SUCCESS;
127588eea45cSKlaus Jensen }
127688eea45cSKlaus Jensen
127788eea45cSKlaus Jensen return nvme_map_mptr(n, &req->sg, len, &req->cmd);
127888eea45cSKlaus Jensen }
127988eea45cSKlaus Jensen
nvme_tx_interleaved(NvmeCtrl * n,NvmeSg * sg,uint8_t * ptr,uint32_t len,uint32_t bytes,int32_t skip_bytes,int64_t offset,NvmeTxDirection dir)128088eea45cSKlaus Jensen static uint16_t nvme_tx_interleaved(NvmeCtrl *n, NvmeSg *sg, uint8_t *ptr,
128188eea45cSKlaus Jensen uint32_t len, uint32_t bytes,
128288eea45cSKlaus Jensen int32_t skip_bytes, int64_t offset,
128388eea45cSKlaus Jensen NvmeTxDirection dir)
128488eea45cSKlaus Jensen {
128588eea45cSKlaus Jensen hwaddr addr;
128688eea45cSKlaus Jensen uint32_t trans_len, count = bytes;
128788eea45cSKlaus Jensen bool dma = sg->flags & NVME_SG_DMA;
128888eea45cSKlaus Jensen int64_t sge_len;
128988eea45cSKlaus Jensen int sg_idx = 0;
129088eea45cSKlaus Jensen int ret;
129188eea45cSKlaus Jensen
129288eea45cSKlaus Jensen assert(sg->flags & NVME_SG_ALLOC);
129388eea45cSKlaus Jensen
129488eea45cSKlaus Jensen while (len) {
129588eea45cSKlaus Jensen sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
129688eea45cSKlaus Jensen
129788eea45cSKlaus Jensen if (sge_len - offset < 0) {
129888eea45cSKlaus Jensen offset -= sge_len;
129988eea45cSKlaus Jensen sg_idx++;
130088eea45cSKlaus Jensen continue;
130188eea45cSKlaus Jensen }
130288eea45cSKlaus Jensen
130388eea45cSKlaus Jensen if (sge_len == offset) {
130488eea45cSKlaus Jensen offset = 0;
130588eea45cSKlaus Jensen sg_idx++;
130688eea45cSKlaus Jensen continue;
130788eea45cSKlaus Jensen }
130888eea45cSKlaus Jensen
130988eea45cSKlaus Jensen trans_len = MIN(len, count);
131088eea45cSKlaus Jensen trans_len = MIN(trans_len, sge_len - offset);
131188eea45cSKlaus Jensen
131288eea45cSKlaus Jensen if (dma) {
131388eea45cSKlaus Jensen addr = sg->qsg.sg[sg_idx].base + offset;
131488eea45cSKlaus Jensen } else {
131588eea45cSKlaus Jensen addr = (hwaddr)(uintptr_t)sg->iov.iov[sg_idx].iov_base + offset;
131688eea45cSKlaus Jensen }
131788eea45cSKlaus Jensen
131888eea45cSKlaus Jensen if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
131988eea45cSKlaus Jensen ret = nvme_addr_read(n, addr, ptr, trans_len);
132088eea45cSKlaus Jensen } else {
132188eea45cSKlaus Jensen ret = nvme_addr_write(n, addr, ptr, trans_len);
132288eea45cSKlaus Jensen }
132388eea45cSKlaus Jensen
132488eea45cSKlaus Jensen if (ret) {
132588eea45cSKlaus Jensen return NVME_DATA_TRAS_ERROR;
132688eea45cSKlaus Jensen }
132788eea45cSKlaus Jensen
132888eea45cSKlaus Jensen ptr += trans_len;
132988eea45cSKlaus Jensen len -= trans_len;
133088eea45cSKlaus Jensen count -= trans_len;
133188eea45cSKlaus Jensen offset += trans_len;
133288eea45cSKlaus Jensen
133388eea45cSKlaus Jensen if (count == 0) {
133488eea45cSKlaus Jensen count = bytes;
133588eea45cSKlaus Jensen offset += skip_bytes;
133688eea45cSKlaus Jensen }
133788eea45cSKlaus Jensen }
133888eea45cSKlaus Jensen
133988eea45cSKlaus Jensen return NVME_SUCCESS;
134088eea45cSKlaus Jensen }
134188eea45cSKlaus Jensen
nvme_tx(NvmeCtrl * n,NvmeSg * sg,void * ptr,uint32_t len,NvmeTxDirection dir)13428d3a17beSPhilippe Mathieu-Daudé static uint16_t nvme_tx(NvmeCtrl *n, NvmeSg *sg, void *ptr, uint32_t len,
134388eea45cSKlaus Jensen NvmeTxDirection dir)
134488eea45cSKlaus Jensen {
134588eea45cSKlaus Jensen assert(sg->flags & NVME_SG_ALLOC);
134688eea45cSKlaus Jensen
134788eea45cSKlaus Jensen if (sg->flags & NVME_SG_DMA) {
1348392e48afSPhilippe Mathieu-Daudé const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
1349bfa30f39SPhilippe Mathieu-Daudé dma_addr_t residual;
135088eea45cSKlaus Jensen
135188eea45cSKlaus Jensen if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1352f02b664aSPhilippe Mathieu-Daudé dma_buf_write(ptr, len, &residual, &sg->qsg, attrs);
135388eea45cSKlaus Jensen } else {
1354f02b664aSPhilippe Mathieu-Daudé dma_buf_read(ptr, len, &residual, &sg->qsg, attrs);
135588eea45cSKlaus Jensen }
135688eea45cSKlaus Jensen
135788eea45cSKlaus Jensen if (unlikely(residual)) {
135888eea45cSKlaus Jensen trace_pci_nvme_err_invalid_dma();
135988eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
136088eea45cSKlaus Jensen }
136188eea45cSKlaus Jensen } else {
136288eea45cSKlaus Jensen size_t bytes;
136388eea45cSKlaus Jensen
136488eea45cSKlaus Jensen if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
136588eea45cSKlaus Jensen bytes = qemu_iovec_to_buf(&sg->iov, 0, ptr, len);
136688eea45cSKlaus Jensen } else {
136788eea45cSKlaus Jensen bytes = qemu_iovec_from_buf(&sg->iov, 0, ptr, len);
136888eea45cSKlaus Jensen }
136988eea45cSKlaus Jensen
137088eea45cSKlaus Jensen if (unlikely(bytes != len)) {
137188eea45cSKlaus Jensen trace_pci_nvme_err_invalid_dma();
137288eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
137388eea45cSKlaus Jensen }
137488eea45cSKlaus Jensen }
137588eea45cSKlaus Jensen
137688eea45cSKlaus Jensen return NVME_SUCCESS;
137788eea45cSKlaus Jensen }
137888eea45cSKlaus Jensen
nvme_c2h(NvmeCtrl * n,void * ptr,uint32_t len,NvmeRequest * req)13798d3a17beSPhilippe Mathieu-Daudé static inline uint16_t nvme_c2h(NvmeCtrl *n, void *ptr, uint32_t len,
138088eea45cSKlaus Jensen NvmeRequest *req)
138188eea45cSKlaus Jensen {
138288eea45cSKlaus Jensen uint16_t status;
138388eea45cSKlaus Jensen
138488eea45cSKlaus Jensen status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
138588eea45cSKlaus Jensen if (status) {
138688eea45cSKlaus Jensen return status;
138788eea45cSKlaus Jensen }
138888eea45cSKlaus Jensen
138988eea45cSKlaus Jensen return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_FROM_DEVICE);
139088eea45cSKlaus Jensen }
139188eea45cSKlaus Jensen
nvme_h2c(NvmeCtrl * n,void * ptr,uint32_t len,NvmeRequest * req)13928d3a17beSPhilippe Mathieu-Daudé static inline uint16_t nvme_h2c(NvmeCtrl *n, void *ptr, uint32_t len,
139388eea45cSKlaus Jensen NvmeRequest *req)
139488eea45cSKlaus Jensen {
139588eea45cSKlaus Jensen uint16_t status;
139688eea45cSKlaus Jensen
139788eea45cSKlaus Jensen status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
139888eea45cSKlaus Jensen if (status) {
139988eea45cSKlaus Jensen return status;
140088eea45cSKlaus Jensen }
140188eea45cSKlaus Jensen
140288eea45cSKlaus Jensen return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_TO_DEVICE);
140388eea45cSKlaus Jensen }
140488eea45cSKlaus Jensen
nvme_bounce_data(NvmeCtrl * n,void * ptr,uint32_t len,NvmeTxDirection dir,NvmeRequest * req)14058d3a17beSPhilippe Mathieu-Daudé uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
140688eea45cSKlaus Jensen NvmeTxDirection dir, NvmeRequest *req)
140788eea45cSKlaus Jensen {
140888eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
140988eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
14102a132309SKlaus Jensen bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
14112a132309SKlaus Jensen bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
141288eea45cSKlaus Jensen
1413ac0b34c5SKlaus Jensen if (nvme_ns_ext(ns) &&
1414ac0b34c5SKlaus Jensen !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
141588eea45cSKlaus Jensen return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbasz,
141688eea45cSKlaus Jensen ns->lbaf.ms, 0, dir);
141788eea45cSKlaus Jensen }
141888eea45cSKlaus Jensen
141988eea45cSKlaus Jensen return nvme_tx(n, &req->sg, ptr, len, dir);
142088eea45cSKlaus Jensen }
142188eea45cSKlaus Jensen
nvme_bounce_mdata(NvmeCtrl * n,void * ptr,uint32_t len,NvmeTxDirection dir,NvmeRequest * req)14228d3a17beSPhilippe Mathieu-Daudé uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
142388eea45cSKlaus Jensen NvmeTxDirection dir, NvmeRequest *req)
142488eea45cSKlaus Jensen {
142588eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
142688eea45cSKlaus Jensen uint16_t status;
142788eea45cSKlaus Jensen
142888eea45cSKlaus Jensen if (nvme_ns_ext(ns)) {
142988eea45cSKlaus Jensen return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbaf.ms,
143088eea45cSKlaus Jensen ns->lbasz, ns->lbasz, dir);
143188eea45cSKlaus Jensen }
143288eea45cSKlaus Jensen
143388eea45cSKlaus Jensen nvme_sg_unmap(&req->sg);
143488eea45cSKlaus Jensen
143588eea45cSKlaus Jensen status = nvme_map_mptr(n, &req->sg, len, &req->cmd);
143688eea45cSKlaus Jensen if (status) {
143788eea45cSKlaus Jensen return status;
143888eea45cSKlaus Jensen }
143988eea45cSKlaus Jensen
144088eea45cSKlaus Jensen return nvme_tx(n, &req->sg, ptr, len, dir);
144188eea45cSKlaus Jensen }
144288eea45cSKlaus Jensen
nvme_blk_read(BlockBackend * blk,int64_t offset,uint32_t align,BlockCompletionFunc * cb,NvmeRequest * req)144388eea45cSKlaus Jensen static inline void nvme_blk_read(BlockBackend *blk, int64_t offset,
14449b4f0181SMateusz Kozlowski uint32_t align, BlockCompletionFunc *cb,
14459b4f0181SMateusz Kozlowski NvmeRequest *req)
144688eea45cSKlaus Jensen {
144788eea45cSKlaus Jensen assert(req->sg.flags & NVME_SG_ALLOC);
144888eea45cSKlaus Jensen
144988eea45cSKlaus Jensen if (req->sg.flags & NVME_SG_DMA) {
14509b4f0181SMateusz Kozlowski req->aiocb = dma_blk_read(blk, &req->sg.qsg, offset, align, cb, req);
145188eea45cSKlaus Jensen } else {
145288eea45cSKlaus Jensen req->aiocb = blk_aio_preadv(blk, offset, &req->sg.iov, 0, cb, req);
145388eea45cSKlaus Jensen }
145488eea45cSKlaus Jensen }
145588eea45cSKlaus Jensen
nvme_blk_write(BlockBackend * blk,int64_t offset,uint32_t align,BlockCompletionFunc * cb,NvmeRequest * req)145688eea45cSKlaus Jensen static inline void nvme_blk_write(BlockBackend *blk, int64_t offset,
14579b4f0181SMateusz Kozlowski uint32_t align, BlockCompletionFunc *cb,
14589b4f0181SMateusz Kozlowski NvmeRequest *req)
145988eea45cSKlaus Jensen {
146088eea45cSKlaus Jensen assert(req->sg.flags & NVME_SG_ALLOC);
146188eea45cSKlaus Jensen
146288eea45cSKlaus Jensen if (req->sg.flags & NVME_SG_DMA) {
14639b4f0181SMateusz Kozlowski req->aiocb = dma_blk_write(blk, &req->sg.qsg, offset, align, cb, req);
146488eea45cSKlaus Jensen } else {
146588eea45cSKlaus Jensen req->aiocb = blk_aio_pwritev(blk, offset, &req->sg.iov, 0, cb, req);
146688eea45cSKlaus Jensen }
146788eea45cSKlaus Jensen }
146888eea45cSKlaus Jensen
nvme_update_cq_eventidx(const NvmeCQueue * cq)1469fa5db2aaSKlaus Jensen static void nvme_update_cq_eventidx(const NvmeCQueue *cq)
1470fa5db2aaSKlaus Jensen {
1471fa5db2aaSKlaus Jensen trace_pci_nvme_update_cq_eventidx(cq->cqid, cq->head);
1472fa5db2aaSKlaus Jensen
1473c1e244b6SKlaus Jensen stl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->ei_addr, cq->head,
1474c1e244b6SKlaus Jensen MEMTXATTRS_UNSPECIFIED);
1475fa5db2aaSKlaus Jensen }
1476fa5db2aaSKlaus Jensen
nvme_update_cq_head(NvmeCQueue * cq)14773f7fe8deSJinhao Fan static void nvme_update_cq_head(NvmeCQueue *cq)
14783f7fe8deSJinhao Fan {
1479c1e244b6SKlaus Jensen ldl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->db_addr, &cq->head,
1480c1e244b6SKlaus Jensen MEMTXATTRS_UNSPECIFIED);
148147cd3539SKlaus Jensen
148247cd3539SKlaus Jensen trace_pci_nvme_update_cq_head(cq->cqid, cq->head);
14833f7fe8deSJinhao Fan }
14843f7fe8deSJinhao Fan
nvme_post_cqes(void * opaque)148588eea45cSKlaus Jensen static void nvme_post_cqes(void *opaque)
148688eea45cSKlaus Jensen {
148788eea45cSKlaus Jensen NvmeCQueue *cq = opaque;
148888eea45cSKlaus Jensen NvmeCtrl *n = cq->ctrl;
148988eea45cSKlaus Jensen NvmeRequest *req, *next;
149083d7ed5cSKlaus Jensen bool pending = cq->head != cq->tail;
149188eea45cSKlaus Jensen int ret;
149288eea45cSKlaus Jensen
149388eea45cSKlaus Jensen QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
149488eea45cSKlaus Jensen NvmeSQueue *sq;
149588eea45cSKlaus Jensen hwaddr addr;
149688eea45cSKlaus Jensen
14973f7fe8deSJinhao Fan if (n->dbbuf_enabled) {
1498fa5db2aaSKlaus Jensen nvme_update_cq_eventidx(cq);
14993f7fe8deSJinhao Fan nvme_update_cq_head(cq);
15003f7fe8deSJinhao Fan }
15013f7fe8deSJinhao Fan
150288eea45cSKlaus Jensen if (nvme_cq_full(cq)) {
150388eea45cSKlaus Jensen break;
150488eea45cSKlaus Jensen }
150588eea45cSKlaus Jensen
150688eea45cSKlaus Jensen sq = req->sq;
150788eea45cSKlaus Jensen req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
150888eea45cSKlaus Jensen req->cqe.sq_id = cpu_to_le16(sq->sqid);
150988eea45cSKlaus Jensen req->cqe.sq_head = cpu_to_le16(sq->head);
15106a33f2e9SKlaus Jensen addr = cq->dma_addr + (cq->tail << NVME_CQES);
151148b32c28SKlaus Jensen ret = pci_dma_write(PCI_DEVICE(n), addr, (void *)&req->cqe,
151288eea45cSKlaus Jensen sizeof(req->cqe));
151388eea45cSKlaus Jensen if (ret) {
151488eea45cSKlaus Jensen trace_pci_nvme_err_addr_write(addr);
151588eea45cSKlaus Jensen trace_pci_nvme_err_cfs();
151649e03457SKlaus Jensen stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
151788eea45cSKlaus Jensen break;
151888eea45cSKlaus Jensen }
151988eea45cSKlaus Jensen QTAILQ_REMOVE(&cq->req_list, req, entry);
152088eea45cSKlaus Jensen nvme_inc_cq_tail(cq);
152188eea45cSKlaus Jensen nvme_sg_unmap(&req->sg);
152288eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
152388eea45cSKlaus Jensen }
152488eea45cSKlaus Jensen if (cq->tail != cq->head) {
152583d7ed5cSKlaus Jensen if (cq->irq_enabled && !pending) {
152683d7ed5cSKlaus Jensen n->cq_pending++;
152783d7ed5cSKlaus Jensen }
152883d7ed5cSKlaus Jensen
152988eea45cSKlaus Jensen nvme_irq_assert(n, cq);
153088eea45cSKlaus Jensen }
153188eea45cSKlaus Jensen }
153288eea45cSKlaus Jensen
nvme_enqueue_req_completion(NvmeCQueue * cq,NvmeRequest * req)153388eea45cSKlaus Jensen static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
153488eea45cSKlaus Jensen {
153588eea45cSKlaus Jensen assert(cq->cqid == req->sq->cqid);
153688eea45cSKlaus Jensen trace_pci_nvme_enqueue_req_completion(nvme_cid(req), cq->cqid,
1537f1c97407SKlaus Jensen le32_to_cpu(req->cqe.result),
1538f1c97407SKlaus Jensen le32_to_cpu(req->cqe.dw1),
153988eea45cSKlaus Jensen req->status);
154088eea45cSKlaus Jensen
154188eea45cSKlaus Jensen if (req->status) {
154288eea45cSKlaus Jensen trace_pci_nvme_err_req_status(nvme_cid(req), nvme_nsid(req->ns),
154388eea45cSKlaus Jensen req->status, req->cmd.opcode);
154488eea45cSKlaus Jensen }
154588eea45cSKlaus Jensen
154688eea45cSKlaus Jensen QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
154788eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
15482e53b0b4SJinhao Fan
1549d38cc6fdSKlaus Jensen qemu_bh_schedule(cq->bh);
15502e53b0b4SJinhao Fan }
155188eea45cSKlaus Jensen
nvme_process_aers(void * opaque)155288eea45cSKlaus Jensen static void nvme_process_aers(void *opaque)
155388eea45cSKlaus Jensen {
155488eea45cSKlaus Jensen NvmeCtrl *n = opaque;
155588eea45cSKlaus Jensen NvmeAsyncEvent *event, *next;
155688eea45cSKlaus Jensen
155788eea45cSKlaus Jensen trace_pci_nvme_process_aers(n->aer_queued);
155888eea45cSKlaus Jensen
155988eea45cSKlaus Jensen QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
156088eea45cSKlaus Jensen NvmeRequest *req;
156188eea45cSKlaus Jensen NvmeAerResult *result;
156288eea45cSKlaus Jensen
156388eea45cSKlaus Jensen /* can't post cqe if there is nothing to complete */
156488eea45cSKlaus Jensen if (!n->outstanding_aers) {
156588eea45cSKlaus Jensen trace_pci_nvme_no_outstanding_aers();
156688eea45cSKlaus Jensen break;
156788eea45cSKlaus Jensen }
156888eea45cSKlaus Jensen
156988eea45cSKlaus Jensen /* ignore if masked (cqe posted, but event not cleared) */
157088eea45cSKlaus Jensen if (n->aer_mask & (1 << event->result.event_type)) {
157188eea45cSKlaus Jensen trace_pci_nvme_aer_masked(event->result.event_type, n->aer_mask);
157288eea45cSKlaus Jensen continue;
157388eea45cSKlaus Jensen }
157488eea45cSKlaus Jensen
157588eea45cSKlaus Jensen QTAILQ_REMOVE(&n->aer_queue, event, entry);
157688eea45cSKlaus Jensen n->aer_queued--;
157788eea45cSKlaus Jensen
157888eea45cSKlaus Jensen n->aer_mask |= 1 << event->result.event_type;
157988eea45cSKlaus Jensen n->outstanding_aers--;
158088eea45cSKlaus Jensen
158188eea45cSKlaus Jensen req = n->aer_reqs[n->outstanding_aers];
158288eea45cSKlaus Jensen
158388eea45cSKlaus Jensen result = (NvmeAerResult *) &req->cqe.result;
158488eea45cSKlaus Jensen result->event_type = event->result.event_type;
158588eea45cSKlaus Jensen result->event_info = event->result.event_info;
158688eea45cSKlaus Jensen result->log_page = event->result.log_page;
158788eea45cSKlaus Jensen g_free(event);
158888eea45cSKlaus Jensen
158988eea45cSKlaus Jensen trace_pci_nvme_aer_post_cqe(result->event_type, result->event_info,
159088eea45cSKlaus Jensen result->log_page);
159188eea45cSKlaus Jensen
159288eea45cSKlaus Jensen nvme_enqueue_req_completion(&n->admin_cq, req);
159388eea45cSKlaus Jensen }
159488eea45cSKlaus Jensen }
159588eea45cSKlaus Jensen
nvme_enqueue_event(NvmeCtrl * n,uint8_t event_type,uint8_t event_info,uint8_t log_page)159688eea45cSKlaus Jensen static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type,
159788eea45cSKlaus Jensen uint8_t event_info, uint8_t log_page)
159888eea45cSKlaus Jensen {
159988eea45cSKlaus Jensen NvmeAsyncEvent *event;
160088eea45cSKlaus Jensen
160188eea45cSKlaus Jensen trace_pci_nvme_enqueue_event(event_type, event_info, log_page);
160288eea45cSKlaus Jensen
160388eea45cSKlaus Jensen if (n->aer_queued == n->params.aer_max_queued) {
160488eea45cSKlaus Jensen trace_pci_nvme_enqueue_event_noqueue(n->aer_queued);
160588eea45cSKlaus Jensen return;
160688eea45cSKlaus Jensen }
160788eea45cSKlaus Jensen
160888eea45cSKlaus Jensen event = g_new(NvmeAsyncEvent, 1);
160988eea45cSKlaus Jensen event->result = (NvmeAerResult) {
161088eea45cSKlaus Jensen .event_type = event_type,
161188eea45cSKlaus Jensen .event_info = event_info,
161288eea45cSKlaus Jensen .log_page = log_page,
161388eea45cSKlaus Jensen };
161488eea45cSKlaus Jensen
161588eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&n->aer_queue, event, entry);
161688eea45cSKlaus Jensen n->aer_queued++;
161788eea45cSKlaus Jensen
161888eea45cSKlaus Jensen nvme_process_aers(n);
161988eea45cSKlaus Jensen }
162088eea45cSKlaus Jensen
nvme_smart_event(NvmeCtrl * n,uint8_t event)162188eea45cSKlaus Jensen static void nvme_smart_event(NvmeCtrl *n, uint8_t event)
162288eea45cSKlaus Jensen {
162388eea45cSKlaus Jensen uint8_t aer_info;
162488eea45cSKlaus Jensen
162588eea45cSKlaus Jensen /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */
162688eea45cSKlaus Jensen if (!(NVME_AEC_SMART(n->features.async_config) & event)) {
162788eea45cSKlaus Jensen return;
162888eea45cSKlaus Jensen }
162988eea45cSKlaus Jensen
163088eea45cSKlaus Jensen switch (event) {
163188eea45cSKlaus Jensen case NVME_SMART_SPARE:
163288eea45cSKlaus Jensen aer_info = NVME_AER_INFO_SMART_SPARE_THRESH;
163388eea45cSKlaus Jensen break;
163488eea45cSKlaus Jensen case NVME_SMART_TEMPERATURE:
163588eea45cSKlaus Jensen aer_info = NVME_AER_INFO_SMART_TEMP_THRESH;
163688eea45cSKlaus Jensen break;
163788eea45cSKlaus Jensen case NVME_SMART_RELIABILITY:
163888eea45cSKlaus Jensen case NVME_SMART_MEDIA_READ_ONLY:
163988eea45cSKlaus Jensen case NVME_SMART_FAILED_VOLATILE_MEDIA:
164088eea45cSKlaus Jensen case NVME_SMART_PMR_UNRELIABLE:
164188eea45cSKlaus Jensen aer_info = NVME_AER_INFO_SMART_RELIABILITY;
164288eea45cSKlaus Jensen break;
164388eea45cSKlaus Jensen default:
164488eea45cSKlaus Jensen return;
164588eea45cSKlaus Jensen }
164688eea45cSKlaus Jensen
164788eea45cSKlaus Jensen nvme_enqueue_event(n, NVME_AER_TYPE_SMART, aer_info, NVME_LOG_SMART_INFO);
164888eea45cSKlaus Jensen }
164988eea45cSKlaus Jensen
nvme_clear_events(NvmeCtrl * n,uint8_t event_type)165088eea45cSKlaus Jensen static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type)
165188eea45cSKlaus Jensen {
165288eea45cSKlaus Jensen n->aer_mask &= ~(1 << event_type);
165388eea45cSKlaus Jensen if (!QTAILQ_EMPTY(&n->aer_queue)) {
165488eea45cSKlaus Jensen nvme_process_aers(n);
165588eea45cSKlaus Jensen }
165688eea45cSKlaus Jensen }
165788eea45cSKlaus Jensen
nvme_check_mdts(NvmeCtrl * n,size_t len)165888eea45cSKlaus Jensen static inline uint16_t nvme_check_mdts(NvmeCtrl *n, size_t len)
165988eea45cSKlaus Jensen {
166088eea45cSKlaus Jensen uint8_t mdts = n->params.mdts;
166188eea45cSKlaus Jensen
166288eea45cSKlaus Jensen if (mdts && len > n->page_size << mdts) {
166388eea45cSKlaus Jensen trace_pci_nvme_err_mdts(len);
166488eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
166588eea45cSKlaus Jensen }
166688eea45cSKlaus Jensen
166788eea45cSKlaus Jensen return NVME_SUCCESS;
166888eea45cSKlaus Jensen }
166988eea45cSKlaus Jensen
nvme_check_bounds(NvmeNamespace * ns,uint64_t slba,uint32_t nlb)167088eea45cSKlaus Jensen static inline uint16_t nvme_check_bounds(NvmeNamespace *ns, uint64_t slba,
167188eea45cSKlaus Jensen uint32_t nlb)
167288eea45cSKlaus Jensen {
167388eea45cSKlaus Jensen uint64_t nsze = le64_to_cpu(ns->id_ns.nsze);
167488eea45cSKlaus Jensen
167588eea45cSKlaus Jensen if (unlikely(UINT64_MAX - slba < nlb || slba + nlb > nsze)) {
167688eea45cSKlaus Jensen trace_pci_nvme_err_invalid_lba_range(slba, nlb, nsze);
167788eea45cSKlaus Jensen return NVME_LBA_RANGE | NVME_DNR;
167888eea45cSKlaus Jensen }
167988eea45cSKlaus Jensen
168088eea45cSKlaus Jensen return NVME_SUCCESS;
168188eea45cSKlaus Jensen }
168288eea45cSKlaus Jensen
nvme_block_status_all(NvmeNamespace * ns,uint64_t slba,uint32_t nlb,int flags)1683ff0ac2c8SKlaus Jensen static int nvme_block_status_all(NvmeNamespace *ns, uint64_t slba,
1684ff0ac2c8SKlaus Jensen uint32_t nlb, int flags)
168588eea45cSKlaus Jensen {
168688eea45cSKlaus Jensen BlockDriverState *bs = blk_bs(ns->blkconf.blk);
168788eea45cSKlaus Jensen
168888eea45cSKlaus Jensen int64_t pnum = 0, bytes = nvme_l2b(ns, nlb);
168988eea45cSKlaus Jensen int64_t offset = nvme_l2b(ns, slba);
169088eea45cSKlaus Jensen int ret;
169188eea45cSKlaus Jensen
169288eea45cSKlaus Jensen /*
169388eea45cSKlaus Jensen * `pnum` holds the number of bytes after offset that shares the same
169488eea45cSKlaus Jensen * allocation status as the byte at offset. If `pnum` is different from
169588eea45cSKlaus Jensen * `bytes`, we should check the allocation status of the next range and
169688eea45cSKlaus Jensen * continue this until all bytes have been checked.
169788eea45cSKlaus Jensen */
169888eea45cSKlaus Jensen do {
169988eea45cSKlaus Jensen bytes -= pnum;
170088eea45cSKlaus Jensen
170188eea45cSKlaus Jensen ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL);
170288eea45cSKlaus Jensen if (ret < 0) {
1703ff0ac2c8SKlaus Jensen return ret;
170488eea45cSKlaus Jensen }
170588eea45cSKlaus Jensen
170688eea45cSKlaus Jensen
1707ff0ac2c8SKlaus Jensen trace_pci_nvme_block_status(offset, bytes, pnum, ret,
1708ff0ac2c8SKlaus Jensen !!(ret & BDRV_BLOCK_ZERO));
170988eea45cSKlaus Jensen
1710ff0ac2c8SKlaus Jensen if (!(ret & flags)) {
1711ff0ac2c8SKlaus Jensen return 1;
171288eea45cSKlaus Jensen }
171388eea45cSKlaus Jensen
171488eea45cSKlaus Jensen offset += pnum;
171588eea45cSKlaus Jensen } while (pnum != bytes);
171688eea45cSKlaus Jensen
1717ff0ac2c8SKlaus Jensen return 0;
1718ff0ac2c8SKlaus Jensen }
1719ff0ac2c8SKlaus Jensen
nvme_check_dulbe(NvmeNamespace * ns,uint64_t slba,uint32_t nlb)1720ff0ac2c8SKlaus Jensen static uint16_t nvme_check_dulbe(NvmeNamespace *ns, uint64_t slba,
1721ff0ac2c8SKlaus Jensen uint32_t nlb)
1722ff0ac2c8SKlaus Jensen {
1723ff0ac2c8SKlaus Jensen int ret;
1724ff0ac2c8SKlaus Jensen Error *err = NULL;
1725ff0ac2c8SKlaus Jensen
1726ff0ac2c8SKlaus Jensen ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_DATA);
1727ff0ac2c8SKlaus Jensen if (ret) {
1728ff0ac2c8SKlaus Jensen if (ret < 0) {
1729ff0ac2c8SKlaus Jensen error_setg_errno(&err, -ret, "unable to get block status");
1730ff0ac2c8SKlaus Jensen error_report_err(err);
1731ff0ac2c8SKlaus Jensen
1732ff0ac2c8SKlaus Jensen return NVME_INTERNAL_DEV_ERROR;
1733ff0ac2c8SKlaus Jensen }
1734ff0ac2c8SKlaus Jensen
1735ff0ac2c8SKlaus Jensen return NVME_DULB;
1736ff0ac2c8SKlaus Jensen }
1737ff0ac2c8SKlaus Jensen
173888eea45cSKlaus Jensen return NVME_SUCCESS;
173988eea45cSKlaus Jensen }
174088eea45cSKlaus Jensen
nvme_aio_err(NvmeRequest * req,int ret)174188eea45cSKlaus Jensen static void nvme_aio_err(NvmeRequest *req, int ret)
174288eea45cSKlaus Jensen {
174388eea45cSKlaus Jensen uint16_t status = NVME_SUCCESS;
174488eea45cSKlaus Jensen Error *local_err = NULL;
174588eea45cSKlaus Jensen
174688eea45cSKlaus Jensen switch (req->cmd.opcode) {
174788eea45cSKlaus Jensen case NVME_CMD_READ:
174888eea45cSKlaus Jensen status = NVME_UNRECOVERED_READ;
174988eea45cSKlaus Jensen break;
175088eea45cSKlaus Jensen case NVME_CMD_FLUSH:
175188eea45cSKlaus Jensen case NVME_CMD_WRITE:
175288eea45cSKlaus Jensen case NVME_CMD_WRITE_ZEROES:
175388eea45cSKlaus Jensen case NVME_CMD_ZONE_APPEND:
1754cab1da59SMinwoo Im case NVME_CMD_COPY:
175588eea45cSKlaus Jensen status = NVME_WRITE_FAULT;
175688eea45cSKlaus Jensen break;
175788eea45cSKlaus Jensen default:
175888eea45cSKlaus Jensen status = NVME_INTERNAL_DEV_ERROR;
175988eea45cSKlaus Jensen break;
176088eea45cSKlaus Jensen }
176188eea45cSKlaus Jensen
176288eea45cSKlaus Jensen trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), status);
176388eea45cSKlaus Jensen
176488eea45cSKlaus Jensen error_setg_errno(&local_err, -ret, "aio failed");
176588eea45cSKlaus Jensen error_report_err(local_err);
176688eea45cSKlaus Jensen
176788eea45cSKlaus Jensen /*
176888eea45cSKlaus Jensen * Set the command status code to the first encountered error but allow a
176988eea45cSKlaus Jensen * subsequent Internal Device Error to trump it.
177088eea45cSKlaus Jensen */
177188eea45cSKlaus Jensen if (req->status && status != NVME_INTERNAL_DEV_ERROR) {
177288eea45cSKlaus Jensen return;
177388eea45cSKlaus Jensen }
177488eea45cSKlaus Jensen
177588eea45cSKlaus Jensen req->status = status;
177688eea45cSKlaus Jensen }
177788eea45cSKlaus Jensen
nvme_zone_idx(NvmeNamespace * ns,uint64_t slba)177888eea45cSKlaus Jensen static inline uint32_t nvme_zone_idx(NvmeNamespace *ns, uint64_t slba)
177988eea45cSKlaus Jensen {
178088eea45cSKlaus Jensen return ns->zone_size_log2 > 0 ? slba >> ns->zone_size_log2 :
178188eea45cSKlaus Jensen slba / ns->zone_size;
178288eea45cSKlaus Jensen }
178388eea45cSKlaus Jensen
nvme_get_zone_by_slba(NvmeNamespace * ns,uint64_t slba)178488eea45cSKlaus Jensen static inline NvmeZone *nvme_get_zone_by_slba(NvmeNamespace *ns, uint64_t slba)
178588eea45cSKlaus Jensen {
178688eea45cSKlaus Jensen uint32_t zone_idx = nvme_zone_idx(ns, slba);
178788eea45cSKlaus Jensen
1788189a8bf7SKlaus Jensen if (zone_idx >= ns->num_zones) {
1789189a8bf7SKlaus Jensen return NULL;
1790189a8bf7SKlaus Jensen }
1791189a8bf7SKlaus Jensen
179288eea45cSKlaus Jensen return &ns->zone_array[zone_idx];
179388eea45cSKlaus Jensen }
179488eea45cSKlaus Jensen
nvme_check_zone_state_for_write(NvmeZone * zone)179588eea45cSKlaus Jensen static uint16_t nvme_check_zone_state_for_write(NvmeZone *zone)
179688eea45cSKlaus Jensen {
179788eea45cSKlaus Jensen uint64_t zslba = zone->d.zslba;
179888eea45cSKlaus Jensen
179988eea45cSKlaus Jensen switch (nvme_get_zone_state(zone)) {
180088eea45cSKlaus Jensen case NVME_ZONE_STATE_EMPTY:
180188eea45cSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
180288eea45cSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
180388eea45cSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
180488eea45cSKlaus Jensen return NVME_SUCCESS;
180588eea45cSKlaus Jensen case NVME_ZONE_STATE_FULL:
180688eea45cSKlaus Jensen trace_pci_nvme_err_zone_is_full(zslba);
180788eea45cSKlaus Jensen return NVME_ZONE_FULL;
180888eea45cSKlaus Jensen case NVME_ZONE_STATE_OFFLINE:
180988eea45cSKlaus Jensen trace_pci_nvme_err_zone_is_offline(zslba);
181088eea45cSKlaus Jensen return NVME_ZONE_OFFLINE;
181188eea45cSKlaus Jensen case NVME_ZONE_STATE_READ_ONLY:
181288eea45cSKlaus Jensen trace_pci_nvme_err_zone_is_read_only(zslba);
181388eea45cSKlaus Jensen return NVME_ZONE_READ_ONLY;
181488eea45cSKlaus Jensen default:
181588eea45cSKlaus Jensen assert(false);
181688eea45cSKlaus Jensen }
181788eea45cSKlaus Jensen
181888eea45cSKlaus Jensen return NVME_INTERNAL_DEV_ERROR;
181988eea45cSKlaus Jensen }
182088eea45cSKlaus Jensen
nvme_check_zone_write(NvmeNamespace * ns,NvmeZone * zone,uint64_t slba,uint32_t nlb)182188eea45cSKlaus Jensen static uint16_t nvme_check_zone_write(NvmeNamespace *ns, NvmeZone *zone,
182288eea45cSKlaus Jensen uint64_t slba, uint32_t nlb)
182388eea45cSKlaus Jensen {
182488eea45cSKlaus Jensen uint64_t zcap = nvme_zone_wr_boundary(zone);
182588eea45cSKlaus Jensen uint16_t status;
182688eea45cSKlaus Jensen
182788eea45cSKlaus Jensen status = nvme_check_zone_state_for_write(zone);
182888eea45cSKlaus Jensen if (status) {
182988eea45cSKlaus Jensen return status;
183088eea45cSKlaus Jensen }
183188eea45cSKlaus Jensen
1832e321b4cdSKlaus Jensen if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1833e321b4cdSKlaus Jensen uint64_t ezrwa = zone->w_ptr + 2 * ns->zns.zrwas;
1834e321b4cdSKlaus Jensen
1835e321b4cdSKlaus Jensen if (slba < zone->w_ptr || slba + nlb > ezrwa) {
1836e321b4cdSKlaus Jensen trace_pci_nvme_err_zone_invalid_write(slba, zone->w_ptr);
183788eea45cSKlaus Jensen return NVME_ZONE_INVALID_WRITE;
183888eea45cSKlaus Jensen }
1839e321b4cdSKlaus Jensen } else {
1840e321b4cdSKlaus Jensen if (unlikely(slba != zone->w_ptr)) {
1841e321b4cdSKlaus Jensen trace_pci_nvme_err_write_not_at_wp(slba, zone->d.zslba,
1842e321b4cdSKlaus Jensen zone->w_ptr);
1843e321b4cdSKlaus Jensen return NVME_ZONE_INVALID_WRITE;
1844e321b4cdSKlaus Jensen }
1845e321b4cdSKlaus Jensen }
184688eea45cSKlaus Jensen
184788eea45cSKlaus Jensen if (unlikely((slba + nlb) > zcap)) {
184888eea45cSKlaus Jensen trace_pci_nvme_err_zone_boundary(slba, nlb, zcap);
184988eea45cSKlaus Jensen return NVME_ZONE_BOUNDARY_ERROR;
185088eea45cSKlaus Jensen }
185188eea45cSKlaus Jensen
185288eea45cSKlaus Jensen return NVME_SUCCESS;
185388eea45cSKlaus Jensen }
185488eea45cSKlaus Jensen
nvme_check_zone_state_for_read(NvmeZone * zone)185588eea45cSKlaus Jensen static uint16_t nvme_check_zone_state_for_read(NvmeZone *zone)
185688eea45cSKlaus Jensen {
185788eea45cSKlaus Jensen switch (nvme_get_zone_state(zone)) {
185888eea45cSKlaus Jensen case NVME_ZONE_STATE_EMPTY:
185988eea45cSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
186088eea45cSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
186188eea45cSKlaus Jensen case NVME_ZONE_STATE_FULL:
186288eea45cSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
186388eea45cSKlaus Jensen case NVME_ZONE_STATE_READ_ONLY:
186488eea45cSKlaus Jensen return NVME_SUCCESS;
186588eea45cSKlaus Jensen case NVME_ZONE_STATE_OFFLINE:
186688eea45cSKlaus Jensen trace_pci_nvme_err_zone_is_offline(zone->d.zslba);
186788eea45cSKlaus Jensen return NVME_ZONE_OFFLINE;
186888eea45cSKlaus Jensen default:
186988eea45cSKlaus Jensen assert(false);
187088eea45cSKlaus Jensen }
187188eea45cSKlaus Jensen
187288eea45cSKlaus Jensen return NVME_INTERNAL_DEV_ERROR;
187388eea45cSKlaus Jensen }
187488eea45cSKlaus Jensen
nvme_check_zone_read(NvmeNamespace * ns,uint64_t slba,uint32_t nlb)187588eea45cSKlaus Jensen static uint16_t nvme_check_zone_read(NvmeNamespace *ns, uint64_t slba,
187688eea45cSKlaus Jensen uint32_t nlb)
187788eea45cSKlaus Jensen {
1878189a8bf7SKlaus Jensen NvmeZone *zone;
1879189a8bf7SKlaus Jensen uint64_t bndry, end;
188088eea45cSKlaus Jensen uint16_t status;
188188eea45cSKlaus Jensen
1882189a8bf7SKlaus Jensen zone = nvme_get_zone_by_slba(ns, slba);
1883189a8bf7SKlaus Jensen assert(zone);
1884189a8bf7SKlaus Jensen
1885189a8bf7SKlaus Jensen bndry = nvme_zone_rd_boundary(ns, zone);
1886189a8bf7SKlaus Jensen end = slba + nlb;
1887189a8bf7SKlaus Jensen
188888eea45cSKlaus Jensen status = nvme_check_zone_state_for_read(zone);
188988eea45cSKlaus Jensen if (status) {
189088eea45cSKlaus Jensen ;
189188eea45cSKlaus Jensen } else if (unlikely(end > bndry)) {
189288eea45cSKlaus Jensen if (!ns->params.cross_zone_read) {
189388eea45cSKlaus Jensen status = NVME_ZONE_BOUNDARY_ERROR;
189488eea45cSKlaus Jensen } else {
189588eea45cSKlaus Jensen /*
189688eea45cSKlaus Jensen * Read across zone boundary - check that all subsequent
189788eea45cSKlaus Jensen * zones that are being read have an appropriate state.
189888eea45cSKlaus Jensen */
189988eea45cSKlaus Jensen do {
190088eea45cSKlaus Jensen zone++;
190188eea45cSKlaus Jensen status = nvme_check_zone_state_for_read(zone);
190288eea45cSKlaus Jensen if (status) {
190388eea45cSKlaus Jensen break;
190488eea45cSKlaus Jensen }
190588eea45cSKlaus Jensen } while (end > nvme_zone_rd_boundary(ns, zone));
190688eea45cSKlaus Jensen }
190788eea45cSKlaus Jensen }
190888eea45cSKlaus Jensen
190988eea45cSKlaus Jensen return status;
191088eea45cSKlaus Jensen }
191188eea45cSKlaus Jensen
nvme_zrm_finish(NvmeNamespace * ns,NvmeZone * zone)191288eea45cSKlaus Jensen static uint16_t nvme_zrm_finish(NvmeNamespace *ns, NvmeZone *zone)
191388eea45cSKlaus Jensen {
191488eea45cSKlaus Jensen switch (nvme_get_zone_state(zone)) {
191588eea45cSKlaus Jensen case NVME_ZONE_STATE_FULL:
191688eea45cSKlaus Jensen return NVME_SUCCESS;
191788eea45cSKlaus Jensen
191888eea45cSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
191988eea45cSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
192088eea45cSKlaus Jensen nvme_aor_dec_open(ns);
192188eea45cSKlaus Jensen /* fallthrough */
192288eea45cSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
192388eea45cSKlaus Jensen nvme_aor_dec_active(ns);
1924e321b4cdSKlaus Jensen
1925e321b4cdSKlaus Jensen if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1926e321b4cdSKlaus Jensen zone->d.za &= ~NVME_ZA_ZRWA_VALID;
1927e321b4cdSKlaus Jensen if (ns->params.numzrwa) {
1928e321b4cdSKlaus Jensen ns->zns.numzrwa++;
1929e321b4cdSKlaus Jensen }
1930e321b4cdSKlaus Jensen }
1931e321b4cdSKlaus Jensen
193288eea45cSKlaus Jensen /* fallthrough */
193388eea45cSKlaus Jensen case NVME_ZONE_STATE_EMPTY:
193488eea45cSKlaus Jensen nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_FULL);
193588eea45cSKlaus Jensen return NVME_SUCCESS;
193688eea45cSKlaus Jensen
193788eea45cSKlaus Jensen default:
193888eea45cSKlaus Jensen return NVME_ZONE_INVAL_TRANSITION;
193988eea45cSKlaus Jensen }
194088eea45cSKlaus Jensen }
194188eea45cSKlaus Jensen
nvme_zrm_close(NvmeNamespace * ns,NvmeZone * zone)194288eea45cSKlaus Jensen static uint16_t nvme_zrm_close(NvmeNamespace *ns, NvmeZone *zone)
194388eea45cSKlaus Jensen {
194488eea45cSKlaus Jensen switch (nvme_get_zone_state(zone)) {
194588eea45cSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
194688eea45cSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
194788eea45cSKlaus Jensen nvme_aor_dec_open(ns);
194888eea45cSKlaus Jensen nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
194988eea45cSKlaus Jensen /* fall through */
195088eea45cSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
195188eea45cSKlaus Jensen return NVME_SUCCESS;
195288eea45cSKlaus Jensen
195388eea45cSKlaus Jensen default:
195488eea45cSKlaus Jensen return NVME_ZONE_INVAL_TRANSITION;
195588eea45cSKlaus Jensen }
195688eea45cSKlaus Jensen }
195788eea45cSKlaus Jensen
nvme_zrm_reset(NvmeNamespace * ns,NvmeZone * zone)195863d96e4fSKlaus Jensen static uint16_t nvme_zrm_reset(NvmeNamespace *ns, NvmeZone *zone)
195963d96e4fSKlaus Jensen {
196063d96e4fSKlaus Jensen switch (nvme_get_zone_state(zone)) {
196163d96e4fSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
196263d96e4fSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
196363d96e4fSKlaus Jensen nvme_aor_dec_open(ns);
196463d96e4fSKlaus Jensen /* fallthrough */
196563d96e4fSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
196663d96e4fSKlaus Jensen nvme_aor_dec_active(ns);
1967e321b4cdSKlaus Jensen
1968e321b4cdSKlaus Jensen if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1969e321b4cdSKlaus Jensen if (ns->params.numzrwa) {
1970e321b4cdSKlaus Jensen ns->zns.numzrwa++;
1971e321b4cdSKlaus Jensen }
1972e321b4cdSKlaus Jensen }
1973e321b4cdSKlaus Jensen
197463d96e4fSKlaus Jensen /* fallthrough */
197563d96e4fSKlaus Jensen case NVME_ZONE_STATE_FULL:
197663d96e4fSKlaus Jensen zone->w_ptr = zone->d.zslba;
197763d96e4fSKlaus Jensen zone->d.wp = zone->w_ptr;
197863d96e4fSKlaus Jensen nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EMPTY);
197963d96e4fSKlaus Jensen /* fallthrough */
198063d96e4fSKlaus Jensen case NVME_ZONE_STATE_EMPTY:
198163d96e4fSKlaus Jensen return NVME_SUCCESS;
198263d96e4fSKlaus Jensen
198363d96e4fSKlaus Jensen default:
198463d96e4fSKlaus Jensen return NVME_ZONE_INVAL_TRANSITION;
198563d96e4fSKlaus Jensen }
198663d96e4fSKlaus Jensen }
198763d96e4fSKlaus Jensen
nvme_zrm_auto_transition_zone(NvmeNamespace * ns)198888eea45cSKlaus Jensen static void nvme_zrm_auto_transition_zone(NvmeNamespace *ns)
198988eea45cSKlaus Jensen {
199088eea45cSKlaus Jensen NvmeZone *zone;
199188eea45cSKlaus Jensen
199288eea45cSKlaus Jensen if (ns->params.max_open_zones &&
199388eea45cSKlaus Jensen ns->nr_open_zones == ns->params.max_open_zones) {
199488eea45cSKlaus Jensen zone = QTAILQ_FIRST(&ns->imp_open_zones);
199588eea45cSKlaus Jensen if (zone) {
199688eea45cSKlaus Jensen /*
199788eea45cSKlaus Jensen * Automatically close this implicitly open zone.
199888eea45cSKlaus Jensen */
199988eea45cSKlaus Jensen QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
200088eea45cSKlaus Jensen nvme_zrm_close(ns, zone);
200188eea45cSKlaus Jensen }
200288eea45cSKlaus Jensen }
200388eea45cSKlaus Jensen }
200488eea45cSKlaus Jensen
200588eea45cSKlaus Jensen enum {
200688eea45cSKlaus Jensen NVME_ZRM_AUTO = 1 << 0,
2007e321b4cdSKlaus Jensen NVME_ZRM_ZRWA = 1 << 1,
200888eea45cSKlaus Jensen };
200988eea45cSKlaus Jensen
nvme_zrm_open_flags(NvmeCtrl * n,NvmeNamespace * ns,NvmeZone * zone,int flags)2010cccc2651SNiklas Cassel static uint16_t nvme_zrm_open_flags(NvmeCtrl *n, NvmeNamespace *ns,
2011cccc2651SNiklas Cassel NvmeZone *zone, int flags)
201288eea45cSKlaus Jensen {
201388eea45cSKlaus Jensen int act = 0;
201488eea45cSKlaus Jensen uint16_t status;
201588eea45cSKlaus Jensen
201688eea45cSKlaus Jensen switch (nvme_get_zone_state(zone)) {
201788eea45cSKlaus Jensen case NVME_ZONE_STATE_EMPTY:
201888eea45cSKlaus Jensen act = 1;
201988eea45cSKlaus Jensen
202088eea45cSKlaus Jensen /* fallthrough */
202188eea45cSKlaus Jensen
202288eea45cSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
2023cccc2651SNiklas Cassel if (n->params.auto_transition_zones) {
202488eea45cSKlaus Jensen nvme_zrm_auto_transition_zone(ns);
2025cccc2651SNiklas Cassel }
2026e321b4cdSKlaus Jensen status = nvme_zns_check_resources(ns, act, 1,
2027e321b4cdSKlaus Jensen (flags & NVME_ZRM_ZRWA) ? 1 : 0);
202888eea45cSKlaus Jensen if (status) {
202988eea45cSKlaus Jensen return status;
203088eea45cSKlaus Jensen }
203188eea45cSKlaus Jensen
203288eea45cSKlaus Jensen if (act) {
203388eea45cSKlaus Jensen nvme_aor_inc_active(ns);
203488eea45cSKlaus Jensen }
203588eea45cSKlaus Jensen
203688eea45cSKlaus Jensen nvme_aor_inc_open(ns);
203788eea45cSKlaus Jensen
203888eea45cSKlaus Jensen if (flags & NVME_ZRM_AUTO) {
203988eea45cSKlaus Jensen nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_IMPLICITLY_OPEN);
204088eea45cSKlaus Jensen return NVME_SUCCESS;
204188eea45cSKlaus Jensen }
204288eea45cSKlaus Jensen
204388eea45cSKlaus Jensen /* fallthrough */
204488eea45cSKlaus Jensen
204588eea45cSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
204688eea45cSKlaus Jensen if (flags & NVME_ZRM_AUTO) {
204788eea45cSKlaus Jensen return NVME_SUCCESS;
204888eea45cSKlaus Jensen }
204988eea45cSKlaus Jensen
205088eea45cSKlaus Jensen nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EXPLICITLY_OPEN);
205188eea45cSKlaus Jensen
205288eea45cSKlaus Jensen /* fallthrough */
205388eea45cSKlaus Jensen
205488eea45cSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
2055e321b4cdSKlaus Jensen if (flags & NVME_ZRM_ZRWA) {
2056e321b4cdSKlaus Jensen ns->zns.numzrwa--;
2057e321b4cdSKlaus Jensen
2058e321b4cdSKlaus Jensen zone->d.za |= NVME_ZA_ZRWA_VALID;
2059e321b4cdSKlaus Jensen }
2060e321b4cdSKlaus Jensen
206188eea45cSKlaus Jensen return NVME_SUCCESS;
206288eea45cSKlaus Jensen
206388eea45cSKlaus Jensen default:
206488eea45cSKlaus Jensen return NVME_ZONE_INVAL_TRANSITION;
206588eea45cSKlaus Jensen }
206688eea45cSKlaus Jensen }
206788eea45cSKlaus Jensen
nvme_zrm_auto(NvmeCtrl * n,NvmeNamespace * ns,NvmeZone * zone)2068cccc2651SNiklas Cassel static inline uint16_t nvme_zrm_auto(NvmeCtrl *n, NvmeNamespace *ns,
2069cccc2651SNiklas Cassel NvmeZone *zone)
207088eea45cSKlaus Jensen {
2071cccc2651SNiklas Cassel return nvme_zrm_open_flags(n, ns, zone, NVME_ZRM_AUTO);
207288eea45cSKlaus Jensen }
207388eea45cSKlaus Jensen
nvme_advance_zone_wp(NvmeNamespace * ns,NvmeZone * zone,uint32_t nlb)207488eea45cSKlaus Jensen static void nvme_advance_zone_wp(NvmeNamespace *ns, NvmeZone *zone,
207588eea45cSKlaus Jensen uint32_t nlb)
207688eea45cSKlaus Jensen {
207788eea45cSKlaus Jensen zone->d.wp += nlb;
207888eea45cSKlaus Jensen
207988eea45cSKlaus Jensen if (zone->d.wp == nvme_zone_wr_boundary(zone)) {
208088eea45cSKlaus Jensen nvme_zrm_finish(ns, zone);
208188eea45cSKlaus Jensen }
208288eea45cSKlaus Jensen }
208388eea45cSKlaus Jensen
nvme_zoned_zrwa_implicit_flush(NvmeNamespace * ns,NvmeZone * zone,uint32_t nlbc)2084e321b4cdSKlaus Jensen static void nvme_zoned_zrwa_implicit_flush(NvmeNamespace *ns, NvmeZone *zone,
2085e321b4cdSKlaus Jensen uint32_t nlbc)
2086e321b4cdSKlaus Jensen {
2087e321b4cdSKlaus Jensen uint16_t nzrwafgs = DIV_ROUND_UP(nlbc, ns->zns.zrwafg);
2088e321b4cdSKlaus Jensen
2089e321b4cdSKlaus Jensen nlbc = nzrwafgs * ns->zns.zrwafg;
2090e321b4cdSKlaus Jensen
2091e321b4cdSKlaus Jensen trace_pci_nvme_zoned_zrwa_implicit_flush(zone->d.zslba, nlbc);
2092e321b4cdSKlaus Jensen
2093e321b4cdSKlaus Jensen zone->w_ptr += nlbc;
2094e321b4cdSKlaus Jensen
2095e321b4cdSKlaus Jensen nvme_advance_zone_wp(ns, zone, nlbc);
2096e321b4cdSKlaus Jensen }
2097e321b4cdSKlaus Jensen
nvme_finalize_zoned_write(NvmeNamespace * ns,NvmeRequest * req)209888eea45cSKlaus Jensen static void nvme_finalize_zoned_write(NvmeNamespace *ns, NvmeRequest *req)
209988eea45cSKlaus Jensen {
210088eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
210188eea45cSKlaus Jensen NvmeZone *zone;
210288eea45cSKlaus Jensen uint64_t slba;
210388eea45cSKlaus Jensen uint32_t nlb;
210488eea45cSKlaus Jensen
210588eea45cSKlaus Jensen slba = le64_to_cpu(rw->slba);
210688eea45cSKlaus Jensen nlb = le16_to_cpu(rw->nlb) + 1;
210788eea45cSKlaus Jensen zone = nvme_get_zone_by_slba(ns, slba);
2108189a8bf7SKlaus Jensen assert(zone);
210988eea45cSKlaus Jensen
2110e321b4cdSKlaus Jensen if (zone->d.za & NVME_ZA_ZRWA_VALID) {
2111e321b4cdSKlaus Jensen uint64_t ezrwa = zone->w_ptr + ns->zns.zrwas - 1;
2112e321b4cdSKlaus Jensen uint64_t elba = slba + nlb - 1;
2113e321b4cdSKlaus Jensen
2114e321b4cdSKlaus Jensen if (elba > ezrwa) {
2115e321b4cdSKlaus Jensen nvme_zoned_zrwa_implicit_flush(ns, zone, elba - ezrwa);
2116e321b4cdSKlaus Jensen }
2117e321b4cdSKlaus Jensen
2118e321b4cdSKlaus Jensen return;
2119e321b4cdSKlaus Jensen }
2120e321b4cdSKlaus Jensen
212188eea45cSKlaus Jensen nvme_advance_zone_wp(ns, zone, nlb);
212288eea45cSKlaus Jensen }
212388eea45cSKlaus Jensen
nvme_is_write(NvmeRequest * req)212488eea45cSKlaus Jensen static inline bool nvme_is_write(NvmeRequest *req)
212588eea45cSKlaus Jensen {
212688eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
212788eea45cSKlaus Jensen
212888eea45cSKlaus Jensen return rw->opcode == NVME_CMD_WRITE ||
212988eea45cSKlaus Jensen rw->opcode == NVME_CMD_ZONE_APPEND ||
213088eea45cSKlaus Jensen rw->opcode == NVME_CMD_WRITE_ZEROES;
213188eea45cSKlaus Jensen }
213288eea45cSKlaus Jensen
nvme_misc_cb(void * opaque,int ret)213388eea45cSKlaus Jensen static void nvme_misc_cb(void *opaque, int ret)
213488eea45cSKlaus Jensen {
213588eea45cSKlaus Jensen NvmeRequest *req = opaque;
213688eea45cSKlaus Jensen
213738f4ac65SKlaus Jensen trace_pci_nvme_misc_cb(nvme_cid(req));
213888eea45cSKlaus Jensen
213988eea45cSKlaus Jensen if (ret) {
214088eea45cSKlaus Jensen nvme_aio_err(req, ret);
214188eea45cSKlaus Jensen }
214288eea45cSKlaus Jensen
214388eea45cSKlaus Jensen nvme_enqueue_req_completion(nvme_cq(req), req);
214488eea45cSKlaus Jensen }
214588eea45cSKlaus Jensen
nvme_rw_complete_cb(void * opaque,int ret)214688eea45cSKlaus Jensen void nvme_rw_complete_cb(void *opaque, int ret)
214788eea45cSKlaus Jensen {
214888eea45cSKlaus Jensen NvmeRequest *req = opaque;
214988eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
215088eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
215188eea45cSKlaus Jensen BlockAcctCookie *acct = &req->acct;
215288eea45cSKlaus Jensen BlockAcctStats *stats = blk_get_stats(blk);
215388eea45cSKlaus Jensen
215488eea45cSKlaus Jensen trace_pci_nvme_rw_complete_cb(nvme_cid(req), blk_name(blk));
215588eea45cSKlaus Jensen
215688eea45cSKlaus Jensen if (ret) {
215788eea45cSKlaus Jensen block_acct_failed(stats, acct);
215888eea45cSKlaus Jensen nvme_aio_err(req, ret);
215988eea45cSKlaus Jensen } else {
216088eea45cSKlaus Jensen block_acct_done(stats, acct);
216188eea45cSKlaus Jensen }
216288eea45cSKlaus Jensen
216388eea45cSKlaus Jensen if (ns->params.zoned && nvme_is_write(req)) {
216488eea45cSKlaus Jensen nvme_finalize_zoned_write(ns, req);
216588eea45cSKlaus Jensen }
216688eea45cSKlaus Jensen
216788eea45cSKlaus Jensen nvme_enqueue_req_completion(nvme_cq(req), req);
216888eea45cSKlaus Jensen }
216988eea45cSKlaus Jensen
nvme_rw_cb(void * opaque,int ret)217088eea45cSKlaus Jensen static void nvme_rw_cb(void *opaque, int ret)
217188eea45cSKlaus Jensen {
217288eea45cSKlaus Jensen NvmeRequest *req = opaque;
217388eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
217488eea45cSKlaus Jensen
217588eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
217688eea45cSKlaus Jensen
217788eea45cSKlaus Jensen trace_pci_nvme_rw_cb(nvme_cid(req), blk_name(blk));
217888eea45cSKlaus Jensen
217988eea45cSKlaus Jensen if (ret) {
218088eea45cSKlaus Jensen goto out;
218188eea45cSKlaus Jensen }
218288eea45cSKlaus Jensen
218388eea45cSKlaus Jensen if (ns->lbaf.ms) {
218488eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
218588eea45cSKlaus Jensen uint64_t slba = le64_to_cpu(rw->slba);
218688eea45cSKlaus Jensen uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
218788eea45cSKlaus Jensen uint64_t offset = nvme_moff(ns, slba);
218888eea45cSKlaus Jensen
218988eea45cSKlaus Jensen if (req->cmd.opcode == NVME_CMD_WRITE_ZEROES) {
219088eea45cSKlaus Jensen size_t mlen = nvme_m2b(ns, nlb);
219188eea45cSKlaus Jensen
219288eea45cSKlaus Jensen req->aiocb = blk_aio_pwrite_zeroes(blk, offset, mlen,
219388eea45cSKlaus Jensen BDRV_REQ_MAY_UNMAP,
219488eea45cSKlaus Jensen nvme_rw_complete_cb, req);
219588eea45cSKlaus Jensen return;
219688eea45cSKlaus Jensen }
219788eea45cSKlaus Jensen
219888eea45cSKlaus Jensen if (nvme_ns_ext(ns) || req->cmd.mptr) {
219988eea45cSKlaus Jensen uint16_t status;
220088eea45cSKlaus Jensen
220188eea45cSKlaus Jensen nvme_sg_unmap(&req->sg);
220288eea45cSKlaus Jensen status = nvme_map_mdata(nvme_ctrl(req), nlb, req);
220388eea45cSKlaus Jensen if (status) {
220488eea45cSKlaus Jensen ret = -EFAULT;
220588eea45cSKlaus Jensen goto out;
220688eea45cSKlaus Jensen }
220788eea45cSKlaus Jensen
220888eea45cSKlaus Jensen if (req->cmd.opcode == NVME_CMD_READ) {
22099b4f0181SMateusz Kozlowski return nvme_blk_read(blk, offset, 1, nvme_rw_complete_cb, req);
221088eea45cSKlaus Jensen }
221188eea45cSKlaus Jensen
22129b4f0181SMateusz Kozlowski return nvme_blk_write(blk, offset, 1, nvme_rw_complete_cb, req);
221388eea45cSKlaus Jensen }
221488eea45cSKlaus Jensen }
221588eea45cSKlaus Jensen
221688eea45cSKlaus Jensen out:
221788eea45cSKlaus Jensen nvme_rw_complete_cb(req, ret);
221888eea45cSKlaus Jensen }
221988eea45cSKlaus Jensen
nvme_verify_cb(void * opaque,int ret)222088eea45cSKlaus Jensen static void nvme_verify_cb(void *opaque, int ret)
222188eea45cSKlaus Jensen {
222288eea45cSKlaus Jensen NvmeBounceContext *ctx = opaque;
222388eea45cSKlaus Jensen NvmeRequest *req = ctx->req;
222488eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
222588eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
222688eea45cSKlaus Jensen BlockAcctCookie *acct = &req->acct;
222788eea45cSKlaus Jensen BlockAcctStats *stats = blk_get_stats(blk);
222888eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
222988eea45cSKlaus Jensen uint64_t slba = le64_to_cpu(rw->slba);
22302a132309SKlaus Jensen uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
223188eea45cSKlaus Jensen uint16_t apptag = le16_to_cpu(rw->apptag);
223288eea45cSKlaus Jensen uint16_t appmask = le16_to_cpu(rw->appmask);
223344219b60SNaveen Nagar uint64_t reftag = le32_to_cpu(rw->reftag);
223444219b60SNaveen Nagar uint64_t cdw3 = le32_to_cpu(rw->cdw3);
223588eea45cSKlaus Jensen uint16_t status;
223688eea45cSKlaus Jensen
223744219b60SNaveen Nagar reftag |= cdw3 << 32;
223844219b60SNaveen Nagar
22392a132309SKlaus Jensen trace_pci_nvme_verify_cb(nvme_cid(req), prinfo, apptag, appmask, reftag);
224088eea45cSKlaus Jensen
224188eea45cSKlaus Jensen if (ret) {
224288eea45cSKlaus Jensen block_acct_failed(stats, acct);
224388eea45cSKlaus Jensen nvme_aio_err(req, ret);
224488eea45cSKlaus Jensen goto out;
224588eea45cSKlaus Jensen }
224688eea45cSKlaus Jensen
224788eea45cSKlaus Jensen block_acct_done(stats, acct);
224888eea45cSKlaus Jensen
224988eea45cSKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
225088eea45cSKlaus Jensen status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce,
225188eea45cSKlaus Jensen ctx->mdata.iov.size, slba);
225288eea45cSKlaus Jensen if (status) {
225388eea45cSKlaus Jensen req->status = status;
225488eea45cSKlaus Jensen goto out;
225588eea45cSKlaus Jensen }
225688eea45cSKlaus Jensen
225788eea45cSKlaus Jensen req->status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
225888eea45cSKlaus Jensen ctx->mdata.bounce, ctx->mdata.iov.size,
22592a132309SKlaus Jensen prinfo, slba, apptag, appmask, &reftag);
226088eea45cSKlaus Jensen }
226188eea45cSKlaus Jensen
226288eea45cSKlaus Jensen out:
226388eea45cSKlaus Jensen qemu_iovec_destroy(&ctx->data.iov);
226488eea45cSKlaus Jensen g_free(ctx->data.bounce);
226588eea45cSKlaus Jensen
226688eea45cSKlaus Jensen qemu_iovec_destroy(&ctx->mdata.iov);
226788eea45cSKlaus Jensen g_free(ctx->mdata.bounce);
226888eea45cSKlaus Jensen
226988eea45cSKlaus Jensen g_free(ctx);
227088eea45cSKlaus Jensen
227188eea45cSKlaus Jensen nvme_enqueue_req_completion(nvme_cq(req), req);
227288eea45cSKlaus Jensen }
227388eea45cSKlaus Jensen
227488eea45cSKlaus Jensen
nvme_verify_mdata_in_cb(void * opaque,int ret)227588eea45cSKlaus Jensen static void nvme_verify_mdata_in_cb(void *opaque, int ret)
227688eea45cSKlaus Jensen {
227788eea45cSKlaus Jensen NvmeBounceContext *ctx = opaque;
227888eea45cSKlaus Jensen NvmeRequest *req = ctx->req;
227988eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
228088eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
228188eea45cSKlaus Jensen uint64_t slba = le64_to_cpu(rw->slba);
228288eea45cSKlaus Jensen uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
228388eea45cSKlaus Jensen size_t mlen = nvme_m2b(ns, nlb);
228488eea45cSKlaus Jensen uint64_t offset = nvme_moff(ns, slba);
228588eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
228688eea45cSKlaus Jensen
228788eea45cSKlaus Jensen trace_pci_nvme_verify_mdata_in_cb(nvme_cid(req), blk_name(blk));
228888eea45cSKlaus Jensen
228988eea45cSKlaus Jensen if (ret) {
229088eea45cSKlaus Jensen goto out;
229188eea45cSKlaus Jensen }
229288eea45cSKlaus Jensen
229388eea45cSKlaus Jensen ctx->mdata.bounce = g_malloc(mlen);
229488eea45cSKlaus Jensen
229588eea45cSKlaus Jensen qemu_iovec_reset(&ctx->mdata.iov);
229688eea45cSKlaus Jensen qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
229788eea45cSKlaus Jensen
229888eea45cSKlaus Jensen req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
229988eea45cSKlaus Jensen nvme_verify_cb, ctx);
230088eea45cSKlaus Jensen return;
230188eea45cSKlaus Jensen
230288eea45cSKlaus Jensen out:
230388eea45cSKlaus Jensen nvme_verify_cb(ctx, ret);
230488eea45cSKlaus Jensen }
230588eea45cSKlaus Jensen
230688eea45cSKlaus Jensen struct nvme_compare_ctx {
230788eea45cSKlaus Jensen struct {
230888eea45cSKlaus Jensen QEMUIOVector iov;
230988eea45cSKlaus Jensen uint8_t *bounce;
231088eea45cSKlaus Jensen } data;
231188eea45cSKlaus Jensen
231288eea45cSKlaus Jensen struct {
231388eea45cSKlaus Jensen QEMUIOVector iov;
231488eea45cSKlaus Jensen uint8_t *bounce;
231588eea45cSKlaus Jensen } mdata;
231688eea45cSKlaus Jensen };
231788eea45cSKlaus Jensen
nvme_compare_mdata_cb(void * opaque,int ret)231888eea45cSKlaus Jensen static void nvme_compare_mdata_cb(void *opaque, int ret)
231988eea45cSKlaus Jensen {
232088eea45cSKlaus Jensen NvmeRequest *req = opaque;
232188eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
232288eea45cSKlaus Jensen NvmeCtrl *n = nvme_ctrl(req);
232388eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
23242a132309SKlaus Jensen uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
232588eea45cSKlaus Jensen uint16_t apptag = le16_to_cpu(rw->apptag);
232688eea45cSKlaus Jensen uint16_t appmask = le16_to_cpu(rw->appmask);
232744219b60SNaveen Nagar uint64_t reftag = le32_to_cpu(rw->reftag);
232844219b60SNaveen Nagar uint64_t cdw3 = le32_to_cpu(rw->cdw3);
232988eea45cSKlaus Jensen struct nvme_compare_ctx *ctx = req->opaque;
233088eea45cSKlaus Jensen g_autofree uint8_t *buf = NULL;
233188eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
233288eea45cSKlaus Jensen BlockAcctCookie *acct = &req->acct;
233388eea45cSKlaus Jensen BlockAcctStats *stats = blk_get_stats(blk);
233488eea45cSKlaus Jensen uint16_t status = NVME_SUCCESS;
233588eea45cSKlaus Jensen
233644219b60SNaveen Nagar reftag |= cdw3 << 32;
233744219b60SNaveen Nagar
233888eea45cSKlaus Jensen trace_pci_nvme_compare_mdata_cb(nvme_cid(req));
233988eea45cSKlaus Jensen
234088eea45cSKlaus Jensen if (ret) {
234188eea45cSKlaus Jensen block_acct_failed(stats, acct);
234288eea45cSKlaus Jensen nvme_aio_err(req, ret);
234388eea45cSKlaus Jensen goto out;
234488eea45cSKlaus Jensen }
234588eea45cSKlaus Jensen
234688eea45cSKlaus Jensen buf = g_malloc(ctx->mdata.iov.size);
234788eea45cSKlaus Jensen
234888eea45cSKlaus Jensen status = nvme_bounce_mdata(n, buf, ctx->mdata.iov.size,
234988eea45cSKlaus Jensen NVME_TX_DIRECTION_TO_DEVICE, req);
235088eea45cSKlaus Jensen if (status) {
235188eea45cSKlaus Jensen req->status = status;
235288eea45cSKlaus Jensen goto out;
235388eea45cSKlaus Jensen }
235488eea45cSKlaus Jensen
235588eea45cSKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
235688eea45cSKlaus Jensen uint64_t slba = le64_to_cpu(rw->slba);
235788eea45cSKlaus Jensen uint8_t *bufp;
235888eea45cSKlaus Jensen uint8_t *mbufp = ctx->mdata.bounce;
235988eea45cSKlaus Jensen uint8_t *end = mbufp + ctx->mdata.iov.size;
236088eea45cSKlaus Jensen int16_t pil = 0;
236188eea45cSKlaus Jensen
236288eea45cSKlaus Jensen status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
23632a132309SKlaus Jensen ctx->mdata.bounce, ctx->mdata.iov.size, prinfo,
23640ca5c3ccSKlaus Jensen slba, apptag, appmask, &reftag);
236588eea45cSKlaus Jensen if (status) {
236688eea45cSKlaus Jensen req->status = status;
236788eea45cSKlaus Jensen goto out;
236888eea45cSKlaus Jensen }
236988eea45cSKlaus Jensen
237088eea45cSKlaus Jensen /*
237188eea45cSKlaus Jensen * When formatted with protection information, do not compare the DIF
237288eea45cSKlaus Jensen * tuple.
237388eea45cSKlaus Jensen */
237488eea45cSKlaus Jensen if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
2375ac0b34c5SKlaus Jensen pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
237688eea45cSKlaus Jensen }
237788eea45cSKlaus Jensen
237888eea45cSKlaus Jensen for (bufp = buf; mbufp < end; bufp += ns->lbaf.ms, mbufp += ns->lbaf.ms) {
237988eea45cSKlaus Jensen if (memcmp(bufp + pil, mbufp + pil, ns->lbaf.ms - pil)) {
2380ca2a0918SKlaus Jensen req->status = NVME_CMP_FAILURE | NVME_DNR;
238188eea45cSKlaus Jensen goto out;
238288eea45cSKlaus Jensen }
238388eea45cSKlaus Jensen }
238488eea45cSKlaus Jensen
238588eea45cSKlaus Jensen goto out;
238688eea45cSKlaus Jensen }
238788eea45cSKlaus Jensen
238888eea45cSKlaus Jensen if (memcmp(buf, ctx->mdata.bounce, ctx->mdata.iov.size)) {
2389ca2a0918SKlaus Jensen req->status = NVME_CMP_FAILURE | NVME_DNR;
239088eea45cSKlaus Jensen goto out;
239188eea45cSKlaus Jensen }
239288eea45cSKlaus Jensen
239388eea45cSKlaus Jensen block_acct_done(stats, acct);
239488eea45cSKlaus Jensen
239588eea45cSKlaus Jensen out:
239688eea45cSKlaus Jensen qemu_iovec_destroy(&ctx->data.iov);
239788eea45cSKlaus Jensen g_free(ctx->data.bounce);
239888eea45cSKlaus Jensen
239988eea45cSKlaus Jensen qemu_iovec_destroy(&ctx->mdata.iov);
240088eea45cSKlaus Jensen g_free(ctx->mdata.bounce);
240188eea45cSKlaus Jensen
240288eea45cSKlaus Jensen g_free(ctx);
240388eea45cSKlaus Jensen
240488eea45cSKlaus Jensen nvme_enqueue_req_completion(nvme_cq(req), req);
240588eea45cSKlaus Jensen }
240688eea45cSKlaus Jensen
nvme_compare_data_cb(void * opaque,int ret)240788eea45cSKlaus Jensen static void nvme_compare_data_cb(void *opaque, int ret)
240888eea45cSKlaus Jensen {
240988eea45cSKlaus Jensen NvmeRequest *req = opaque;
241088eea45cSKlaus Jensen NvmeCtrl *n = nvme_ctrl(req);
241188eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
241288eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
241388eea45cSKlaus Jensen BlockAcctCookie *acct = &req->acct;
241488eea45cSKlaus Jensen BlockAcctStats *stats = blk_get_stats(blk);
241588eea45cSKlaus Jensen
241688eea45cSKlaus Jensen struct nvme_compare_ctx *ctx = req->opaque;
241788eea45cSKlaus Jensen g_autofree uint8_t *buf = NULL;
241888eea45cSKlaus Jensen uint16_t status;
241988eea45cSKlaus Jensen
242088eea45cSKlaus Jensen trace_pci_nvme_compare_data_cb(nvme_cid(req));
242188eea45cSKlaus Jensen
242288eea45cSKlaus Jensen if (ret) {
242388eea45cSKlaus Jensen block_acct_failed(stats, acct);
242488eea45cSKlaus Jensen nvme_aio_err(req, ret);
242588eea45cSKlaus Jensen goto out;
242688eea45cSKlaus Jensen }
242788eea45cSKlaus Jensen
242888eea45cSKlaus Jensen buf = g_malloc(ctx->data.iov.size);
242988eea45cSKlaus Jensen
243088eea45cSKlaus Jensen status = nvme_bounce_data(n, buf, ctx->data.iov.size,
243188eea45cSKlaus Jensen NVME_TX_DIRECTION_TO_DEVICE, req);
243288eea45cSKlaus Jensen if (status) {
243388eea45cSKlaus Jensen req->status = status;
243488eea45cSKlaus Jensen goto out;
243588eea45cSKlaus Jensen }
243688eea45cSKlaus Jensen
243788eea45cSKlaus Jensen if (memcmp(buf, ctx->data.bounce, ctx->data.iov.size)) {
2438ca2a0918SKlaus Jensen req->status = NVME_CMP_FAILURE | NVME_DNR;
243988eea45cSKlaus Jensen goto out;
244088eea45cSKlaus Jensen }
244188eea45cSKlaus Jensen
244288eea45cSKlaus Jensen if (ns->lbaf.ms) {
244388eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
244488eea45cSKlaus Jensen uint64_t slba = le64_to_cpu(rw->slba);
244588eea45cSKlaus Jensen uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
244688eea45cSKlaus Jensen size_t mlen = nvme_m2b(ns, nlb);
244788eea45cSKlaus Jensen uint64_t offset = nvme_moff(ns, slba);
244888eea45cSKlaus Jensen
244988eea45cSKlaus Jensen ctx->mdata.bounce = g_malloc(mlen);
245088eea45cSKlaus Jensen
245188eea45cSKlaus Jensen qemu_iovec_init(&ctx->mdata.iov, 1);
245288eea45cSKlaus Jensen qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
245388eea45cSKlaus Jensen
245488eea45cSKlaus Jensen req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
245588eea45cSKlaus Jensen nvme_compare_mdata_cb, req);
245688eea45cSKlaus Jensen return;
245788eea45cSKlaus Jensen }
245888eea45cSKlaus Jensen
245988eea45cSKlaus Jensen block_acct_done(stats, acct);
246088eea45cSKlaus Jensen
246188eea45cSKlaus Jensen out:
246288eea45cSKlaus Jensen qemu_iovec_destroy(&ctx->data.iov);
246388eea45cSKlaus Jensen g_free(ctx->data.bounce);
246488eea45cSKlaus Jensen g_free(ctx);
246588eea45cSKlaus Jensen
246688eea45cSKlaus Jensen nvme_enqueue_req_completion(nvme_cq(req), req);
246788eea45cSKlaus Jensen }
246888eea45cSKlaus Jensen
2469d7d1474fSKlaus Jensen typedef struct NvmeDSMAIOCB {
2470d7d1474fSKlaus Jensen BlockAIOCB common;
2471d7d1474fSKlaus Jensen BlockAIOCB *aiocb;
2472d7d1474fSKlaus Jensen NvmeRequest *req;
2473d7d1474fSKlaus Jensen int ret;
2474d7d1474fSKlaus Jensen
2475d7d1474fSKlaus Jensen NvmeDsmRange *range;
2476d7d1474fSKlaus Jensen unsigned int nr;
2477d7d1474fSKlaus Jensen unsigned int idx;
2478d7d1474fSKlaus Jensen } NvmeDSMAIOCB;
2479d7d1474fSKlaus Jensen
nvme_dsm_cancel(BlockAIOCB * aiocb)2480d7d1474fSKlaus Jensen static void nvme_dsm_cancel(BlockAIOCB *aiocb)
2481d7d1474fSKlaus Jensen {
2482d7d1474fSKlaus Jensen NvmeDSMAIOCB *iocb = container_of(aiocb, NvmeDSMAIOCB, common);
2483d7d1474fSKlaus Jensen
2484d7d1474fSKlaus Jensen /* break nvme_dsm_cb loop */
2485d7d1474fSKlaus Jensen iocb->idx = iocb->nr;
2486d7d1474fSKlaus Jensen iocb->ret = -ECANCELED;
2487d7d1474fSKlaus Jensen
2488d7d1474fSKlaus Jensen if (iocb->aiocb) {
2489d7d1474fSKlaus Jensen blk_aio_cancel_async(iocb->aiocb);
2490d7d1474fSKlaus Jensen iocb->aiocb = NULL;
2491d7d1474fSKlaus Jensen } else {
2492d7d1474fSKlaus Jensen /*
2493d7d1474fSKlaus Jensen * We only reach this if nvme_dsm_cancel() has already been called or
2494818b9b8fSKlaus Jensen * the command ran to completion.
2495d7d1474fSKlaus Jensen */
2496d7d1474fSKlaus Jensen assert(iocb->idx == iocb->nr);
2497d7d1474fSKlaus Jensen }
2498d7d1474fSKlaus Jensen }
2499d7d1474fSKlaus Jensen
2500d7d1474fSKlaus Jensen static const AIOCBInfo nvme_dsm_aiocb_info = {
2501d7d1474fSKlaus Jensen .aiocb_size = sizeof(NvmeDSMAIOCB),
2502d7d1474fSKlaus Jensen .cancel_async = nvme_dsm_cancel,
2503d7d1474fSKlaus Jensen };
2504d7d1474fSKlaus Jensen
2505d7d1474fSKlaus Jensen static void nvme_dsm_cb(void *opaque, int ret);
2506d7d1474fSKlaus Jensen
nvme_dsm_md_cb(void * opaque,int ret)2507d7d1474fSKlaus Jensen static void nvme_dsm_md_cb(void *opaque, int ret)
2508d7d1474fSKlaus Jensen {
2509d7d1474fSKlaus Jensen NvmeDSMAIOCB *iocb = opaque;
2510d7d1474fSKlaus Jensen NvmeRequest *req = iocb->req;
2511d7d1474fSKlaus Jensen NvmeNamespace *ns = req->ns;
2512d7d1474fSKlaus Jensen NvmeDsmRange *range;
2513d7d1474fSKlaus Jensen uint64_t slba;
2514d7d1474fSKlaus Jensen uint32_t nlb;
2515d7d1474fSKlaus Jensen
2516818b9b8fSKlaus Jensen if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2517d7d1474fSKlaus Jensen goto done;
2518d7d1474fSKlaus Jensen }
2519d7d1474fSKlaus Jensen
2520d7d1474fSKlaus Jensen range = &iocb->range[iocb->idx - 1];
2521d7d1474fSKlaus Jensen slba = le64_to_cpu(range->slba);
2522d7d1474fSKlaus Jensen nlb = le32_to_cpu(range->nlb);
2523d7d1474fSKlaus Jensen
2524d7d1474fSKlaus Jensen /*
2525d7d1474fSKlaus Jensen * Check that all block were discarded (zeroed); otherwise we do not zero
2526d7d1474fSKlaus Jensen * the metadata.
2527d7d1474fSKlaus Jensen */
2528d7d1474fSKlaus Jensen
2529d7d1474fSKlaus Jensen ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_ZERO);
2530d7d1474fSKlaus Jensen if (ret) {
2531d7d1474fSKlaus Jensen if (ret < 0) {
2532d7d1474fSKlaus Jensen goto done;
2533d7d1474fSKlaus Jensen }
2534d7d1474fSKlaus Jensen
2535d7d1474fSKlaus Jensen nvme_dsm_cb(iocb, 0);
253651c45326SDmitry Tikhov return;
2537d7d1474fSKlaus Jensen }
2538d7d1474fSKlaus Jensen
2539d7d1474fSKlaus Jensen iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, nvme_moff(ns, slba),
2540d7d1474fSKlaus Jensen nvme_m2b(ns, nlb), BDRV_REQ_MAY_UNMAP,
2541d7d1474fSKlaus Jensen nvme_dsm_cb, iocb);
2542d7d1474fSKlaus Jensen return;
2543d7d1474fSKlaus Jensen
2544d7d1474fSKlaus Jensen done:
2545818b9b8fSKlaus Jensen nvme_dsm_cb(iocb, ret);
2546d7d1474fSKlaus Jensen }
2547d7d1474fSKlaus Jensen
nvme_dsm_cb(void * opaque,int ret)2548d7d1474fSKlaus Jensen static void nvme_dsm_cb(void *opaque, int ret)
2549d7d1474fSKlaus Jensen {
2550d7d1474fSKlaus Jensen NvmeDSMAIOCB *iocb = opaque;
2551d7d1474fSKlaus Jensen NvmeRequest *req = iocb->req;
2552d7d1474fSKlaus Jensen NvmeCtrl *n = nvme_ctrl(req);
2553d7d1474fSKlaus Jensen NvmeNamespace *ns = req->ns;
2554d7d1474fSKlaus Jensen NvmeDsmRange *range;
2555d7d1474fSKlaus Jensen uint64_t slba;
2556d7d1474fSKlaus Jensen uint32_t nlb;
2557d7d1474fSKlaus Jensen
2558818b9b8fSKlaus Jensen if (iocb->ret < 0) {
2559818b9b8fSKlaus Jensen goto done;
2560818b9b8fSKlaus Jensen } else if (ret < 0) {
2561d7d1474fSKlaus Jensen iocb->ret = ret;
2562d7d1474fSKlaus Jensen goto done;
2563d7d1474fSKlaus Jensen }
2564d7d1474fSKlaus Jensen
2565d7d1474fSKlaus Jensen next:
2566d7d1474fSKlaus Jensen if (iocb->idx == iocb->nr) {
2567d7d1474fSKlaus Jensen goto done;
2568d7d1474fSKlaus Jensen }
2569d7d1474fSKlaus Jensen
2570d7d1474fSKlaus Jensen range = &iocb->range[iocb->idx++];
2571d7d1474fSKlaus Jensen slba = le64_to_cpu(range->slba);
2572d7d1474fSKlaus Jensen nlb = le32_to_cpu(range->nlb);
2573d7d1474fSKlaus Jensen
2574d7d1474fSKlaus Jensen trace_pci_nvme_dsm_deallocate(slba, nlb);
2575d7d1474fSKlaus Jensen
2576d7d1474fSKlaus Jensen if (nlb > n->dmrsl) {
2577d7d1474fSKlaus Jensen trace_pci_nvme_dsm_single_range_limit_exceeded(nlb, n->dmrsl);
2578d7d1474fSKlaus Jensen goto next;
2579d7d1474fSKlaus Jensen }
2580d7d1474fSKlaus Jensen
2581d7d1474fSKlaus Jensen if (nvme_check_bounds(ns, slba, nlb)) {
2582d7d1474fSKlaus Jensen trace_pci_nvme_err_invalid_lba_range(slba, nlb,
2583d7d1474fSKlaus Jensen ns->id_ns.nsze);
2584d7d1474fSKlaus Jensen goto next;
2585d7d1474fSKlaus Jensen }
2586d7d1474fSKlaus Jensen
2587d7d1474fSKlaus Jensen iocb->aiocb = blk_aio_pdiscard(ns->blkconf.blk, nvme_l2b(ns, slba),
2588d7d1474fSKlaus Jensen nvme_l2b(ns, nlb),
2589d7d1474fSKlaus Jensen nvme_dsm_md_cb, iocb);
2590d7d1474fSKlaus Jensen return;
2591d7d1474fSKlaus Jensen
2592d7d1474fSKlaus Jensen done:
2593d7d1474fSKlaus Jensen iocb->aiocb = NULL;
2594818b9b8fSKlaus Jensen iocb->common.cb(iocb->common.opaque, iocb->ret);
2595818b9b8fSKlaus Jensen qemu_aio_unref(iocb);
2596d7d1474fSKlaus Jensen }
2597d7d1474fSKlaus Jensen
nvme_dsm(NvmeCtrl * n,NvmeRequest * req)259888eea45cSKlaus Jensen static uint16_t nvme_dsm(NvmeCtrl *n, NvmeRequest *req)
259988eea45cSKlaus Jensen {
260088eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
260188eea45cSKlaus Jensen NvmeDsmCmd *dsm = (NvmeDsmCmd *) &req->cmd;
260288eea45cSKlaus Jensen uint32_t attr = le32_to_cpu(dsm->attributes);
260388eea45cSKlaus Jensen uint32_t nr = (le32_to_cpu(dsm->nr) & 0xff) + 1;
260488eea45cSKlaus Jensen uint16_t status = NVME_SUCCESS;
260588eea45cSKlaus Jensen
2606d7d1474fSKlaus Jensen trace_pci_nvme_dsm(nr, attr);
260788eea45cSKlaus Jensen
260888eea45cSKlaus Jensen if (attr & NVME_DSMGMT_AD) {
2609d7d1474fSKlaus Jensen NvmeDSMAIOCB *iocb = blk_aio_get(&nvme_dsm_aiocb_info, ns->blkconf.blk,
2610d7d1474fSKlaus Jensen nvme_misc_cb, req);
261188eea45cSKlaus Jensen
2612d7d1474fSKlaus Jensen iocb->req = req;
2613d7d1474fSKlaus Jensen iocb->ret = 0;
2614d7d1474fSKlaus Jensen iocb->range = g_new(NvmeDsmRange, nr);
2615d7d1474fSKlaus Jensen iocb->nr = nr;
2616d7d1474fSKlaus Jensen iocb->idx = 0;
2617d7d1474fSKlaus Jensen
2618d7d1474fSKlaus Jensen status = nvme_h2c(n, (uint8_t *)iocb->range, sizeof(NvmeDsmRange) * nr,
2619d7d1474fSKlaus Jensen req);
262088eea45cSKlaus Jensen if (status) {
26214b32319cSKlaus Jensen g_free(iocb->range);
26224b32319cSKlaus Jensen qemu_aio_unref(iocb);
26234b32319cSKlaus Jensen
262488eea45cSKlaus Jensen return status;
262588eea45cSKlaus Jensen }
262688eea45cSKlaus Jensen
2627d7d1474fSKlaus Jensen req->aiocb = &iocb->common;
2628d7d1474fSKlaus Jensen nvme_dsm_cb(iocb, 0);
262988eea45cSKlaus Jensen
2630d7d1474fSKlaus Jensen return NVME_NO_COMPLETE;
263188eea45cSKlaus Jensen }
263288eea45cSKlaus Jensen
263388eea45cSKlaus Jensen return status;
263488eea45cSKlaus Jensen }
263588eea45cSKlaus Jensen
nvme_verify(NvmeCtrl * n,NvmeRequest * req)263688eea45cSKlaus Jensen static uint16_t nvme_verify(NvmeCtrl *n, NvmeRequest *req)
263788eea45cSKlaus Jensen {
263888eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
263988eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
264088eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
264188eea45cSKlaus Jensen uint64_t slba = le64_to_cpu(rw->slba);
264288eea45cSKlaus Jensen uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
264388eea45cSKlaus Jensen size_t len = nvme_l2b(ns, nlb);
264488eea45cSKlaus Jensen int64_t offset = nvme_l2b(ns, slba);
26452a132309SKlaus Jensen uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
264688eea45cSKlaus Jensen uint32_t reftag = le32_to_cpu(rw->reftag);
264788eea45cSKlaus Jensen NvmeBounceContext *ctx = NULL;
264888eea45cSKlaus Jensen uint16_t status;
264988eea45cSKlaus Jensen
265088eea45cSKlaus Jensen trace_pci_nvme_verify(nvme_cid(req), nvme_nsid(ns), slba, nlb);
265188eea45cSKlaus Jensen
265288eea45cSKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
26532a132309SKlaus Jensen status = nvme_check_prinfo(ns, prinfo, slba, reftag);
265488eea45cSKlaus Jensen if (status) {
265588eea45cSKlaus Jensen return status;
265688eea45cSKlaus Jensen }
265788eea45cSKlaus Jensen
26582a132309SKlaus Jensen if (prinfo & NVME_PRINFO_PRACT) {
265988eea45cSKlaus Jensen return NVME_INVALID_PROT_INFO | NVME_DNR;
266088eea45cSKlaus Jensen }
266188eea45cSKlaus Jensen }
266288eea45cSKlaus Jensen
266388eea45cSKlaus Jensen if (len > n->page_size << n->params.vsl) {
266488eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
266588eea45cSKlaus Jensen }
266688eea45cSKlaus Jensen
266788eea45cSKlaus Jensen status = nvme_check_bounds(ns, slba, nlb);
266888eea45cSKlaus Jensen if (status) {
266988eea45cSKlaus Jensen return status;
267088eea45cSKlaus Jensen }
267188eea45cSKlaus Jensen
267288eea45cSKlaus Jensen if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
267388eea45cSKlaus Jensen status = nvme_check_dulbe(ns, slba, nlb);
267488eea45cSKlaus Jensen if (status) {
267588eea45cSKlaus Jensen return status;
267688eea45cSKlaus Jensen }
267788eea45cSKlaus Jensen }
267888eea45cSKlaus Jensen
267988eea45cSKlaus Jensen ctx = g_new0(NvmeBounceContext, 1);
268088eea45cSKlaus Jensen ctx->req = req;
268188eea45cSKlaus Jensen
268288eea45cSKlaus Jensen ctx->data.bounce = g_malloc(len);
268388eea45cSKlaus Jensen
268488eea45cSKlaus Jensen qemu_iovec_init(&ctx->data.iov, 1);
268588eea45cSKlaus Jensen qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len);
268688eea45cSKlaus Jensen
268788eea45cSKlaus Jensen block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size,
268888eea45cSKlaus Jensen BLOCK_ACCT_READ);
268988eea45cSKlaus Jensen
269088eea45cSKlaus Jensen req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0,
269188eea45cSKlaus Jensen nvme_verify_mdata_in_cb, ctx);
269288eea45cSKlaus Jensen return NVME_NO_COMPLETE;
269388eea45cSKlaus Jensen }
269488eea45cSKlaus Jensen
2695796d2068SKlaus Jensen typedef struct NvmeCopyAIOCB {
2696796d2068SKlaus Jensen BlockAIOCB common;
2697796d2068SKlaus Jensen BlockAIOCB *aiocb;
2698796d2068SKlaus Jensen NvmeRequest *req;
2699796d2068SKlaus Jensen int ret;
2700796d2068SKlaus Jensen
270144219b60SNaveen Nagar void *ranges;
270244219b60SNaveen Nagar unsigned int format;
2703796d2068SKlaus Jensen int nr;
2704796d2068SKlaus Jensen int idx;
2705796d2068SKlaus Jensen
2706796d2068SKlaus Jensen uint8_t *bounce;
2707796d2068SKlaus Jensen QEMUIOVector iov;
2708796d2068SKlaus Jensen struct {
2709796d2068SKlaus Jensen BlockAcctCookie read;
2710796d2068SKlaus Jensen BlockAcctCookie write;
2711796d2068SKlaus Jensen } acct;
2712796d2068SKlaus Jensen
271344219b60SNaveen Nagar uint64_t reftag;
2714796d2068SKlaus Jensen uint64_t slba;
2715796d2068SKlaus Jensen
2716796d2068SKlaus Jensen NvmeZone *zone;
2717796d2068SKlaus Jensen } NvmeCopyAIOCB;
2718796d2068SKlaus Jensen
nvme_copy_cancel(BlockAIOCB * aiocb)2719796d2068SKlaus Jensen static void nvme_copy_cancel(BlockAIOCB *aiocb)
272088eea45cSKlaus Jensen {
2721796d2068SKlaus Jensen NvmeCopyAIOCB *iocb = container_of(aiocb, NvmeCopyAIOCB, common);
2722796d2068SKlaus Jensen
2723796d2068SKlaus Jensen iocb->ret = -ECANCELED;
2724796d2068SKlaus Jensen
2725796d2068SKlaus Jensen if (iocb->aiocb) {
2726796d2068SKlaus Jensen blk_aio_cancel_async(iocb->aiocb);
2727796d2068SKlaus Jensen iocb->aiocb = NULL;
2728796d2068SKlaus Jensen }
2729796d2068SKlaus Jensen }
2730796d2068SKlaus Jensen
2731796d2068SKlaus Jensen static const AIOCBInfo nvme_copy_aiocb_info = {
2732796d2068SKlaus Jensen .aiocb_size = sizeof(NvmeCopyAIOCB),
2733796d2068SKlaus Jensen .cancel_async = nvme_copy_cancel,
2734796d2068SKlaus Jensen };
2735796d2068SKlaus Jensen
nvme_copy_done(NvmeCopyAIOCB * iocb)273683f56ac3SKlaus Jensen static void nvme_copy_done(NvmeCopyAIOCB *iocb)
2737796d2068SKlaus Jensen {
2738796d2068SKlaus Jensen NvmeRequest *req = iocb->req;
273988eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
2740796d2068SKlaus Jensen BlockAcctStats *stats = blk_get_stats(ns->blkconf.blk);
2741796d2068SKlaus Jensen
2742796d2068SKlaus Jensen if (iocb->idx != iocb->nr) {
2743796d2068SKlaus Jensen req->cqe.result = cpu_to_le32(iocb->idx);
2744796d2068SKlaus Jensen }
2745796d2068SKlaus Jensen
2746796d2068SKlaus Jensen qemu_iovec_destroy(&iocb->iov);
2747796d2068SKlaus Jensen g_free(iocb->bounce);
2748796d2068SKlaus Jensen
2749796d2068SKlaus Jensen if (iocb->ret < 0) {
2750796d2068SKlaus Jensen block_acct_failed(stats, &iocb->acct.read);
2751796d2068SKlaus Jensen block_acct_failed(stats, &iocb->acct.write);
2752796d2068SKlaus Jensen } else {
2753796d2068SKlaus Jensen block_acct_done(stats, &iocb->acct.read);
2754796d2068SKlaus Jensen block_acct_done(stats, &iocb->acct.write);
2755796d2068SKlaus Jensen }
2756796d2068SKlaus Jensen
2757796d2068SKlaus Jensen iocb->common.cb(iocb->common.opaque, iocb->ret);
2758796d2068SKlaus Jensen qemu_aio_unref(iocb);
2759796d2068SKlaus Jensen }
2760796d2068SKlaus Jensen
276183f56ac3SKlaus Jensen static void nvme_do_copy(NvmeCopyAIOCB *iocb);
2762796d2068SKlaus Jensen
nvme_copy_source_range_parse_format0(void * ranges,int idx,uint64_t * slba,uint32_t * nlb,uint16_t * apptag,uint16_t * appmask,uint64_t * reftag)276344219b60SNaveen Nagar static void nvme_copy_source_range_parse_format0(void *ranges, int idx,
276444219b60SNaveen Nagar uint64_t *slba, uint32_t *nlb,
276544219b60SNaveen Nagar uint16_t *apptag,
276644219b60SNaveen Nagar uint16_t *appmask,
276744219b60SNaveen Nagar uint64_t *reftag)
276844219b60SNaveen Nagar {
276944219b60SNaveen Nagar NvmeCopySourceRangeFormat0 *_ranges = ranges;
277044219b60SNaveen Nagar
277144219b60SNaveen Nagar if (slba) {
277244219b60SNaveen Nagar *slba = le64_to_cpu(_ranges[idx].slba);
277344219b60SNaveen Nagar }
277444219b60SNaveen Nagar
277544219b60SNaveen Nagar if (nlb) {
277644219b60SNaveen Nagar *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
277744219b60SNaveen Nagar }
277844219b60SNaveen Nagar
277944219b60SNaveen Nagar if (apptag) {
278044219b60SNaveen Nagar *apptag = le16_to_cpu(_ranges[idx].apptag);
278144219b60SNaveen Nagar }
278244219b60SNaveen Nagar
278344219b60SNaveen Nagar if (appmask) {
278444219b60SNaveen Nagar *appmask = le16_to_cpu(_ranges[idx].appmask);
278544219b60SNaveen Nagar }
278644219b60SNaveen Nagar
278744219b60SNaveen Nagar if (reftag) {
278844219b60SNaveen Nagar *reftag = le32_to_cpu(_ranges[idx].reftag);
278944219b60SNaveen Nagar }
279044219b60SNaveen Nagar }
279144219b60SNaveen Nagar
nvme_copy_source_range_parse_format1(void * ranges,int idx,uint64_t * slba,uint32_t * nlb,uint16_t * apptag,uint16_t * appmask,uint64_t * reftag)279244219b60SNaveen Nagar static void nvme_copy_source_range_parse_format1(void *ranges, int idx,
279344219b60SNaveen Nagar uint64_t *slba, uint32_t *nlb,
279444219b60SNaveen Nagar uint16_t *apptag,
279544219b60SNaveen Nagar uint16_t *appmask,
279644219b60SNaveen Nagar uint64_t *reftag)
279744219b60SNaveen Nagar {
279844219b60SNaveen Nagar NvmeCopySourceRangeFormat1 *_ranges = ranges;
279944219b60SNaveen Nagar
280044219b60SNaveen Nagar if (slba) {
280144219b60SNaveen Nagar *slba = le64_to_cpu(_ranges[idx].slba);
280244219b60SNaveen Nagar }
280344219b60SNaveen Nagar
280444219b60SNaveen Nagar if (nlb) {
280544219b60SNaveen Nagar *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
280644219b60SNaveen Nagar }
280744219b60SNaveen Nagar
280844219b60SNaveen Nagar if (apptag) {
280944219b60SNaveen Nagar *apptag = le16_to_cpu(_ranges[idx].apptag);
281044219b60SNaveen Nagar }
281144219b60SNaveen Nagar
281244219b60SNaveen Nagar if (appmask) {
281344219b60SNaveen Nagar *appmask = le16_to_cpu(_ranges[idx].appmask);
281444219b60SNaveen Nagar }
281544219b60SNaveen Nagar
281644219b60SNaveen Nagar if (reftag) {
281744219b60SNaveen Nagar *reftag = 0;
281844219b60SNaveen Nagar
281944219b60SNaveen Nagar *reftag |= (uint64_t)_ranges[idx].sr[4] << 40;
282044219b60SNaveen Nagar *reftag |= (uint64_t)_ranges[idx].sr[5] << 32;
282144219b60SNaveen Nagar *reftag |= (uint64_t)_ranges[idx].sr[6] << 24;
282244219b60SNaveen Nagar *reftag |= (uint64_t)_ranges[idx].sr[7] << 16;
282344219b60SNaveen Nagar *reftag |= (uint64_t)_ranges[idx].sr[8] << 8;
282444219b60SNaveen Nagar *reftag |= (uint64_t)_ranges[idx].sr[9];
282544219b60SNaveen Nagar }
282644219b60SNaveen Nagar }
282744219b60SNaveen Nagar
nvme_copy_source_range_parse(void * ranges,int idx,uint8_t format,uint64_t * slba,uint32_t * nlb,uint16_t * apptag,uint16_t * appmask,uint64_t * reftag)282844219b60SNaveen Nagar static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format,
282944219b60SNaveen Nagar uint64_t *slba, uint32_t *nlb,
283044219b60SNaveen Nagar uint16_t *apptag, uint16_t *appmask,
283144219b60SNaveen Nagar uint64_t *reftag)
283244219b60SNaveen Nagar {
283344219b60SNaveen Nagar switch (format) {
283444219b60SNaveen Nagar case NVME_COPY_FORMAT_0:
283544219b60SNaveen Nagar nvme_copy_source_range_parse_format0(ranges, idx, slba, nlb, apptag,
283644219b60SNaveen Nagar appmask, reftag);
283744219b60SNaveen Nagar break;
283844219b60SNaveen Nagar
283944219b60SNaveen Nagar case NVME_COPY_FORMAT_1:
284044219b60SNaveen Nagar nvme_copy_source_range_parse_format1(ranges, idx, slba, nlb, apptag,
284144219b60SNaveen Nagar appmask, reftag);
284244219b60SNaveen Nagar break;
284344219b60SNaveen Nagar
284444219b60SNaveen Nagar default:
284544219b60SNaveen Nagar abort();
284644219b60SNaveen Nagar }
284744219b60SNaveen Nagar }
284844219b60SNaveen Nagar
nvme_check_copy_mcl(NvmeNamespace * ns,NvmeCopyAIOCB * iocb,uint16_t nr)2849381ab99dSMinwoo Im static inline uint16_t nvme_check_copy_mcl(NvmeNamespace *ns,
2850381ab99dSMinwoo Im NvmeCopyAIOCB *iocb, uint16_t nr)
2851381ab99dSMinwoo Im {
2852381ab99dSMinwoo Im uint32_t copy_len = 0;
2853381ab99dSMinwoo Im
2854381ab99dSMinwoo Im for (int idx = 0; idx < nr; idx++) {
2855381ab99dSMinwoo Im uint32_t nlb;
2856381ab99dSMinwoo Im nvme_copy_source_range_parse(iocb->ranges, idx, iocb->format, NULL,
2857381ab99dSMinwoo Im &nlb, NULL, NULL, NULL);
2858381ab99dSMinwoo Im copy_len += nlb + 1;
2859381ab99dSMinwoo Im }
2860381ab99dSMinwoo Im
2861381ab99dSMinwoo Im if (copy_len > ns->id_ns.mcl) {
2862381ab99dSMinwoo Im return NVME_CMD_SIZE_LIMIT | NVME_DNR;
2863381ab99dSMinwoo Im }
2864381ab99dSMinwoo Im
2865381ab99dSMinwoo Im return NVME_SUCCESS;
2866381ab99dSMinwoo Im }
2867381ab99dSMinwoo Im
nvme_copy_out_completed_cb(void * opaque,int ret)2868796d2068SKlaus Jensen static void nvme_copy_out_completed_cb(void *opaque, int ret)
2869796d2068SKlaus Jensen {
2870796d2068SKlaus Jensen NvmeCopyAIOCB *iocb = opaque;
2871796d2068SKlaus Jensen NvmeRequest *req = iocb->req;
2872796d2068SKlaus Jensen NvmeNamespace *ns = req->ns;
287344219b60SNaveen Nagar uint32_t nlb;
287444219b60SNaveen Nagar
287544219b60SNaveen Nagar nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
287644219b60SNaveen Nagar &nlb, NULL, NULL, NULL);
2877796d2068SKlaus Jensen
2878796d2068SKlaus Jensen if (ret < 0) {
2879796d2068SKlaus Jensen iocb->ret = ret;
2880796d2068SKlaus Jensen goto out;
2881796d2068SKlaus Jensen } else if (iocb->ret < 0) {
2882796d2068SKlaus Jensen goto out;
2883796d2068SKlaus Jensen }
2884796d2068SKlaus Jensen
2885796d2068SKlaus Jensen if (ns->params.zoned) {
2886796d2068SKlaus Jensen nvme_advance_zone_wp(ns, iocb->zone, nlb);
2887796d2068SKlaus Jensen }
2888796d2068SKlaus Jensen
2889796d2068SKlaus Jensen iocb->idx++;
2890796d2068SKlaus Jensen iocb->slba += nlb;
2891796d2068SKlaus Jensen out:
289283f56ac3SKlaus Jensen nvme_do_copy(iocb);
2893796d2068SKlaus Jensen }
2894796d2068SKlaus Jensen
nvme_copy_out_cb(void * opaque,int ret)2895796d2068SKlaus Jensen static void nvme_copy_out_cb(void *opaque, int ret)
2896796d2068SKlaus Jensen {
2897796d2068SKlaus Jensen NvmeCopyAIOCB *iocb = opaque;
2898796d2068SKlaus Jensen NvmeRequest *req = iocb->req;
2899796d2068SKlaus Jensen NvmeNamespace *ns = req->ns;
2900796d2068SKlaus Jensen uint32_t nlb;
2901796d2068SKlaus Jensen size_t mlen;
2902796d2068SKlaus Jensen uint8_t *mbounce;
2903796d2068SKlaus Jensen
290483f56ac3SKlaus Jensen if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2905796d2068SKlaus Jensen goto out;
2906796d2068SKlaus Jensen }
2907796d2068SKlaus Jensen
290844219b60SNaveen Nagar nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
290944219b60SNaveen Nagar &nlb, NULL, NULL, NULL);
2910796d2068SKlaus Jensen
2911796d2068SKlaus Jensen mlen = nvme_m2b(ns, nlb);
2912796d2068SKlaus Jensen mbounce = iocb->bounce + nvme_l2b(ns, nlb);
2913796d2068SKlaus Jensen
2914796d2068SKlaus Jensen qemu_iovec_reset(&iocb->iov);
2915796d2068SKlaus Jensen qemu_iovec_add(&iocb->iov, mbounce, mlen);
2916796d2068SKlaus Jensen
2917796d2068SKlaus Jensen iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_moff(ns, iocb->slba),
2918796d2068SKlaus Jensen &iocb->iov, 0, nvme_copy_out_completed_cb,
2919796d2068SKlaus Jensen iocb);
2920796d2068SKlaus Jensen
2921796d2068SKlaus Jensen return;
2922796d2068SKlaus Jensen
2923796d2068SKlaus Jensen out:
292483f56ac3SKlaus Jensen nvme_copy_out_completed_cb(iocb, ret);
2925796d2068SKlaus Jensen }
2926796d2068SKlaus Jensen
nvme_copy_in_completed_cb(void * opaque,int ret)2927796d2068SKlaus Jensen static void nvme_copy_in_completed_cb(void *opaque, int ret)
2928796d2068SKlaus Jensen {
2929796d2068SKlaus Jensen NvmeCopyAIOCB *iocb = opaque;
2930796d2068SKlaus Jensen NvmeRequest *req = iocb->req;
2931796d2068SKlaus Jensen NvmeNamespace *ns = req->ns;
2932796d2068SKlaus Jensen uint32_t nlb;
293344219b60SNaveen Nagar uint64_t slba;
293444219b60SNaveen Nagar uint16_t apptag, appmask;
293544219b60SNaveen Nagar uint64_t reftag;
2936796d2068SKlaus Jensen size_t len;
2937796d2068SKlaus Jensen uint16_t status;
2938796d2068SKlaus Jensen
2939796d2068SKlaus Jensen if (ret < 0) {
2940796d2068SKlaus Jensen iocb->ret = ret;
2941796d2068SKlaus Jensen goto out;
2942796d2068SKlaus Jensen } else if (iocb->ret < 0) {
2943796d2068SKlaus Jensen goto out;
2944796d2068SKlaus Jensen }
2945796d2068SKlaus Jensen
294644219b60SNaveen Nagar nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
294744219b60SNaveen Nagar &nlb, &apptag, &appmask, &reftag);
2948796d2068SKlaus Jensen len = nvme_l2b(ns, nlb);
2949796d2068SKlaus Jensen
2950796d2068SKlaus Jensen trace_pci_nvme_copy_out(iocb->slba, nlb);
2951796d2068SKlaus Jensen
2952796d2068SKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
295388eea45cSKlaus Jensen NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
295488eea45cSKlaus Jensen
2955796d2068SKlaus Jensen uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
2956796d2068SKlaus Jensen uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
295788eea45cSKlaus Jensen
2958796d2068SKlaus Jensen size_t mlen = nvme_m2b(ns, nlb);
2959796d2068SKlaus Jensen uint8_t *mbounce = iocb->bounce + nvme_l2b(ns, nlb);
2960796d2068SKlaus Jensen
29612e8f952aSDmitry Tikhov status = nvme_dif_mangle_mdata(ns, mbounce, mlen, slba);
29622e8f952aSDmitry Tikhov if (status) {
29632e8f952aSDmitry Tikhov goto invalid;
29642e8f952aSDmitry Tikhov }
2965796d2068SKlaus Jensen status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen, prinfor,
2966796d2068SKlaus Jensen slba, apptag, appmask, &reftag);
2967796d2068SKlaus Jensen if (status) {
2968796d2068SKlaus Jensen goto invalid;
2969796d2068SKlaus Jensen }
2970796d2068SKlaus Jensen
2971796d2068SKlaus Jensen apptag = le16_to_cpu(copy->apptag);
2972796d2068SKlaus Jensen appmask = le16_to_cpu(copy->appmask);
2973796d2068SKlaus Jensen
2974796d2068SKlaus Jensen if (prinfow & NVME_PRINFO_PRACT) {
2975796d2068SKlaus Jensen status = nvme_check_prinfo(ns, prinfow, iocb->slba, iocb->reftag);
2976796d2068SKlaus Jensen if (status) {
2977796d2068SKlaus Jensen goto invalid;
2978796d2068SKlaus Jensen }
2979796d2068SKlaus Jensen
2980796d2068SKlaus Jensen nvme_dif_pract_generate_dif(ns, iocb->bounce, len, mbounce, mlen,
2981796d2068SKlaus Jensen apptag, &iocb->reftag);
2982796d2068SKlaus Jensen } else {
2983796d2068SKlaus Jensen status = nvme_dif_check(ns, iocb->bounce, len, mbounce, mlen,
2984796d2068SKlaus Jensen prinfow, iocb->slba, apptag, appmask,
2985796d2068SKlaus Jensen &iocb->reftag);
2986796d2068SKlaus Jensen if (status) {
2987796d2068SKlaus Jensen goto invalid;
2988796d2068SKlaus Jensen }
2989796d2068SKlaus Jensen }
2990796d2068SKlaus Jensen }
2991796d2068SKlaus Jensen
2992796d2068SKlaus Jensen status = nvme_check_bounds(ns, iocb->slba, nlb);
2993796d2068SKlaus Jensen if (status) {
2994796d2068SKlaus Jensen goto invalid;
2995796d2068SKlaus Jensen }
2996796d2068SKlaus Jensen
2997796d2068SKlaus Jensen if (ns->params.zoned) {
2998796d2068SKlaus Jensen status = nvme_check_zone_write(ns, iocb->zone, iocb->slba, nlb);
2999796d2068SKlaus Jensen if (status) {
3000796d2068SKlaus Jensen goto invalid;
3001796d2068SKlaus Jensen }
3002796d2068SKlaus Jensen
3003e321b4cdSKlaus Jensen if (!(iocb->zone->d.za & NVME_ZA_ZRWA_VALID)) {
3004796d2068SKlaus Jensen iocb->zone->w_ptr += nlb;
3005796d2068SKlaus Jensen }
3006e321b4cdSKlaus Jensen }
3007796d2068SKlaus Jensen
3008796d2068SKlaus Jensen qemu_iovec_reset(&iocb->iov);
3009796d2068SKlaus Jensen qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3010796d2068SKlaus Jensen
3011796d2068SKlaus Jensen iocb->aiocb = blk_aio_pwritev(ns->blkconf.blk, nvme_l2b(ns, iocb->slba),
3012796d2068SKlaus Jensen &iocb->iov, 0, nvme_copy_out_cb, iocb);
3013796d2068SKlaus Jensen
3014796d2068SKlaus Jensen return;
3015796d2068SKlaus Jensen
3016796d2068SKlaus Jensen invalid:
3017796d2068SKlaus Jensen req->status = status;
301883f56ac3SKlaus Jensen iocb->ret = -1;
3019796d2068SKlaus Jensen out:
302083f56ac3SKlaus Jensen nvme_do_copy(iocb);
3021796d2068SKlaus Jensen }
3022796d2068SKlaus Jensen
nvme_copy_in_cb(void * opaque,int ret)3023796d2068SKlaus Jensen static void nvme_copy_in_cb(void *opaque, int ret)
3024796d2068SKlaus Jensen {
3025796d2068SKlaus Jensen NvmeCopyAIOCB *iocb = opaque;
3026796d2068SKlaus Jensen NvmeRequest *req = iocb->req;
3027796d2068SKlaus Jensen NvmeNamespace *ns = req->ns;
3028796d2068SKlaus Jensen uint64_t slba;
3029796d2068SKlaus Jensen uint32_t nlb;
3030796d2068SKlaus Jensen
303183f56ac3SKlaus Jensen if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
3032796d2068SKlaus Jensen goto out;
3033796d2068SKlaus Jensen }
3034796d2068SKlaus Jensen
303544219b60SNaveen Nagar nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
303644219b60SNaveen Nagar &nlb, NULL, NULL, NULL);
3037796d2068SKlaus Jensen
3038796d2068SKlaus Jensen qemu_iovec_reset(&iocb->iov);
3039796d2068SKlaus Jensen qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(ns, nlb),
3040796d2068SKlaus Jensen nvme_m2b(ns, nlb));
3041796d2068SKlaus Jensen
3042796d2068SKlaus Jensen iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_moff(ns, slba),
3043796d2068SKlaus Jensen &iocb->iov, 0, nvme_copy_in_completed_cb,
3044796d2068SKlaus Jensen iocb);
3045796d2068SKlaus Jensen return;
3046796d2068SKlaus Jensen
3047796d2068SKlaus Jensen out:
304883f56ac3SKlaus Jensen nvme_copy_in_completed_cb(iocb, ret);
3049796d2068SKlaus Jensen }
3050796d2068SKlaus Jensen
nvme_do_copy(NvmeCopyAIOCB * iocb)305183f56ac3SKlaus Jensen static void nvme_do_copy(NvmeCopyAIOCB *iocb)
3052796d2068SKlaus Jensen {
3053796d2068SKlaus Jensen NvmeRequest *req = iocb->req;
3054796d2068SKlaus Jensen NvmeNamespace *ns = req->ns;
3055796d2068SKlaus Jensen uint64_t slba;
3056796d2068SKlaus Jensen uint32_t nlb;
3057796d2068SKlaus Jensen size_t len;
305888eea45cSKlaus Jensen uint16_t status;
305988eea45cSKlaus Jensen
306083f56ac3SKlaus Jensen if (iocb->ret < 0) {
3061796d2068SKlaus Jensen goto done;
306288eea45cSKlaus Jensen }
306388eea45cSKlaus Jensen
3064796d2068SKlaus Jensen if (iocb->idx == iocb->nr) {
3065796d2068SKlaus Jensen goto done;
306688eea45cSKlaus Jensen }
306788eea45cSKlaus Jensen
306844219b60SNaveen Nagar nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
306944219b60SNaveen Nagar &nlb, NULL, NULL, NULL);
3070796d2068SKlaus Jensen len = nvme_l2b(ns, nlb);
307188eea45cSKlaus Jensen
3072796d2068SKlaus Jensen trace_pci_nvme_copy_source_range(slba, nlb);
307388eea45cSKlaus Jensen
3074796d2068SKlaus Jensen if (nlb > le16_to_cpu(ns->id_ns.mssrl)) {
307588eea45cSKlaus Jensen status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3076796d2068SKlaus Jensen goto invalid;
307788eea45cSKlaus Jensen }
307888eea45cSKlaus Jensen
3079796d2068SKlaus Jensen status = nvme_check_bounds(ns, slba, nlb);
308088eea45cSKlaus Jensen if (status) {
3081796d2068SKlaus Jensen goto invalid;
308288eea45cSKlaus Jensen }
308388eea45cSKlaus Jensen
308488eea45cSKlaus Jensen if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3085796d2068SKlaus Jensen status = nvme_check_dulbe(ns, slba, nlb);
308688eea45cSKlaus Jensen if (status) {
3087796d2068SKlaus Jensen goto invalid;
308888eea45cSKlaus Jensen }
308988eea45cSKlaus Jensen }
309088eea45cSKlaus Jensen
309188eea45cSKlaus Jensen if (ns->params.zoned) {
3092796d2068SKlaus Jensen status = nvme_check_zone_read(ns, slba, nlb);
309388eea45cSKlaus Jensen if (status) {
3094796d2068SKlaus Jensen goto invalid;
309588eea45cSKlaus Jensen }
309688eea45cSKlaus Jensen }
309788eea45cSKlaus Jensen
3098796d2068SKlaus Jensen qemu_iovec_reset(&iocb->iov);
3099796d2068SKlaus Jensen qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3100796d2068SKlaus Jensen
3101796d2068SKlaus Jensen iocb->aiocb = blk_aio_preadv(ns->blkconf.blk, nvme_l2b(ns, slba),
3102796d2068SKlaus Jensen &iocb->iov, 0, nvme_copy_in_cb, iocb);
3103796d2068SKlaus Jensen return;
3104796d2068SKlaus Jensen
3105796d2068SKlaus Jensen invalid:
3106796d2068SKlaus Jensen req->status = status;
310783f56ac3SKlaus Jensen iocb->ret = -1;
3108796d2068SKlaus Jensen done:
310983f56ac3SKlaus Jensen nvme_copy_done(iocb);
3110796d2068SKlaus Jensen }
3111796d2068SKlaus Jensen
nvme_copy(NvmeCtrl * n,NvmeRequest * req)3112796d2068SKlaus Jensen static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req)
3113796d2068SKlaus Jensen {
3114796d2068SKlaus Jensen NvmeNamespace *ns = req->ns;
3115796d2068SKlaus Jensen NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3116796d2068SKlaus Jensen NvmeCopyAIOCB *iocb = blk_aio_get(&nvme_copy_aiocb_info, ns->blkconf.blk,
3117796d2068SKlaus Jensen nvme_misc_cb, req);
3118796d2068SKlaus Jensen uint16_t nr = copy->nr + 1;
3119796d2068SKlaus Jensen uint8_t format = copy->control[0] & 0xf;
3120796d2068SKlaus Jensen uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
3121796d2068SKlaus Jensen uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
312244219b60SNaveen Nagar size_t len = sizeof(NvmeCopySourceRangeFormat0);
3123796d2068SKlaus Jensen
3124796d2068SKlaus Jensen uint16_t status;
3125796d2068SKlaus Jensen
3126796d2068SKlaus Jensen trace_pci_nvme_copy(nvme_cid(req), nvme_nsid(ns), nr, format);
3127796d2068SKlaus Jensen
3128796d2068SKlaus Jensen iocb->ranges = NULL;
3129796d2068SKlaus Jensen iocb->zone = NULL;
3130796d2068SKlaus Jensen
3131796d2068SKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) &&
3132796d2068SKlaus Jensen ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) {
3133796d2068SKlaus Jensen status = NVME_INVALID_FIELD | NVME_DNR;
3134796d2068SKlaus Jensen goto invalid;
3135796d2068SKlaus Jensen }
3136796d2068SKlaus Jensen
3137796d2068SKlaus Jensen if (!(n->id_ctrl.ocfs & (1 << format))) {
3138796d2068SKlaus Jensen trace_pci_nvme_err_copy_invalid_format(format);
3139796d2068SKlaus Jensen status = NVME_INVALID_FIELD | NVME_DNR;
3140796d2068SKlaus Jensen goto invalid;
3141796d2068SKlaus Jensen }
3142796d2068SKlaus Jensen
3143796d2068SKlaus Jensen if (nr > ns->id_ns.msrc + 1) {
314488eea45cSKlaus Jensen status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3145796d2068SKlaus Jensen goto invalid;
314688eea45cSKlaus Jensen }
314788eea45cSKlaus Jensen
3148632cb6cfSFrancis Pravin Antony Michael Raj if ((ns->pif == 0x0 && format != 0x0) ||
3149632cb6cfSFrancis Pravin Antony Michael Raj (ns->pif != 0x0 && format != 0x1)) {
315044219b60SNaveen Nagar status = NVME_INVALID_FORMAT | NVME_DNR;
315144219b60SNaveen Nagar goto invalid;
315244219b60SNaveen Nagar }
3153796d2068SKlaus Jensen
315444219b60SNaveen Nagar if (ns->pif) {
315544219b60SNaveen Nagar len = sizeof(NvmeCopySourceRangeFormat1);
315644219b60SNaveen Nagar }
315744219b60SNaveen Nagar
315844219b60SNaveen Nagar iocb->format = format;
315944219b60SNaveen Nagar iocb->ranges = g_malloc_n(nr, len);
316044219b60SNaveen Nagar status = nvme_h2c(n, (uint8_t *)iocb->ranges, len * nr, req);
3161796d2068SKlaus Jensen if (status) {
3162796d2068SKlaus Jensen goto invalid;
316388eea45cSKlaus Jensen }
316488eea45cSKlaus Jensen
3165796d2068SKlaus Jensen iocb->slba = le64_to_cpu(copy->sdlba);
3166796d2068SKlaus Jensen
3167796d2068SKlaus Jensen if (ns->params.zoned) {
3168796d2068SKlaus Jensen iocb->zone = nvme_get_zone_by_slba(ns, iocb->slba);
3169796d2068SKlaus Jensen if (!iocb->zone) {
3170796d2068SKlaus Jensen status = NVME_LBA_RANGE | NVME_DNR;
3171796d2068SKlaus Jensen goto invalid;
3172796d2068SKlaus Jensen }
3173796d2068SKlaus Jensen
3174796d2068SKlaus Jensen status = nvme_zrm_auto(n, ns, iocb->zone);
3175796d2068SKlaus Jensen if (status) {
3176796d2068SKlaus Jensen goto invalid;
3177796d2068SKlaus Jensen }
3178796d2068SKlaus Jensen }
3179796d2068SKlaus Jensen
3180381ab99dSMinwoo Im status = nvme_check_copy_mcl(ns, iocb, nr);
3181381ab99dSMinwoo Im if (status) {
3182381ab99dSMinwoo Im goto invalid;
3183381ab99dSMinwoo Im }
3184381ab99dSMinwoo Im
3185796d2068SKlaus Jensen iocb->req = req;
3186796d2068SKlaus Jensen iocb->ret = 0;
3187796d2068SKlaus Jensen iocb->nr = nr;
3188796d2068SKlaus Jensen iocb->idx = 0;
3189796d2068SKlaus Jensen iocb->reftag = le32_to_cpu(copy->reftag);
319044219b60SNaveen Nagar iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32;
3191796d2068SKlaus Jensen iocb->bounce = g_malloc_n(le16_to_cpu(ns->id_ns.mssrl),
3192796d2068SKlaus Jensen ns->lbasz + ns->lbaf.ms);
3193796d2068SKlaus Jensen
3194796d2068SKlaus Jensen qemu_iovec_init(&iocb->iov, 1);
3195796d2068SKlaus Jensen
3196796d2068SKlaus Jensen block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.read, 0,
319788eea45cSKlaus Jensen BLOCK_ACCT_READ);
3198796d2068SKlaus Jensen block_acct_start(blk_get_stats(ns->blkconf.blk), &iocb->acct.write, 0,
3199796d2068SKlaus Jensen BLOCK_ACCT_WRITE);
320088eea45cSKlaus Jensen
3201796d2068SKlaus Jensen req->aiocb = &iocb->common;
320283f56ac3SKlaus Jensen nvme_do_copy(iocb);
320388eea45cSKlaus Jensen
320488eea45cSKlaus Jensen return NVME_NO_COMPLETE;
320588eea45cSKlaus Jensen
3206796d2068SKlaus Jensen invalid:
3207796d2068SKlaus Jensen g_free(iocb->ranges);
3208796d2068SKlaus Jensen qemu_aio_unref(iocb);
320988eea45cSKlaus Jensen return status;
321088eea45cSKlaus Jensen }
321188eea45cSKlaus Jensen
nvme_compare(NvmeCtrl * n,NvmeRequest * req)321288eea45cSKlaus Jensen static uint16_t nvme_compare(NvmeCtrl *n, NvmeRequest *req)
321388eea45cSKlaus Jensen {
321488eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
321588eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
321688eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
321788eea45cSKlaus Jensen uint64_t slba = le64_to_cpu(rw->slba);
321888eea45cSKlaus Jensen uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
32192a132309SKlaus Jensen uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
322088eea45cSKlaus Jensen size_t data_len = nvme_l2b(ns, nlb);
322188eea45cSKlaus Jensen size_t len = data_len;
322288eea45cSKlaus Jensen int64_t offset = nvme_l2b(ns, slba);
322388eea45cSKlaus Jensen struct nvme_compare_ctx *ctx = NULL;
322488eea45cSKlaus Jensen uint16_t status;
322588eea45cSKlaus Jensen
322688eea45cSKlaus Jensen trace_pci_nvme_compare(nvme_cid(req), nvme_nsid(ns), slba, nlb);
322788eea45cSKlaus Jensen
32282a132309SKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) && (prinfo & NVME_PRINFO_PRACT)) {
322988eea45cSKlaus Jensen return NVME_INVALID_PROT_INFO | NVME_DNR;
323088eea45cSKlaus Jensen }
323188eea45cSKlaus Jensen
323288eea45cSKlaus Jensen if (nvme_ns_ext(ns)) {
323388eea45cSKlaus Jensen len += nvme_m2b(ns, nlb);
323488eea45cSKlaus Jensen }
323588eea45cSKlaus Jensen
323688eea45cSKlaus Jensen status = nvme_check_mdts(n, len);
323788eea45cSKlaus Jensen if (status) {
323888eea45cSKlaus Jensen return status;
323988eea45cSKlaus Jensen }
324088eea45cSKlaus Jensen
324188eea45cSKlaus Jensen status = nvme_check_bounds(ns, slba, nlb);
324288eea45cSKlaus Jensen if (status) {
324388eea45cSKlaus Jensen return status;
324488eea45cSKlaus Jensen }
324588eea45cSKlaus Jensen
324688eea45cSKlaus Jensen if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
324788eea45cSKlaus Jensen status = nvme_check_dulbe(ns, slba, nlb);
324888eea45cSKlaus Jensen if (status) {
324988eea45cSKlaus Jensen return status;
325088eea45cSKlaus Jensen }
325188eea45cSKlaus Jensen }
325288eea45cSKlaus Jensen
325388eea45cSKlaus Jensen status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
325488eea45cSKlaus Jensen if (status) {
325588eea45cSKlaus Jensen return status;
325688eea45cSKlaus Jensen }
325788eea45cSKlaus Jensen
325888eea45cSKlaus Jensen ctx = g_new(struct nvme_compare_ctx, 1);
325988eea45cSKlaus Jensen ctx->data.bounce = g_malloc(data_len);
326088eea45cSKlaus Jensen
326188eea45cSKlaus Jensen req->opaque = ctx;
326288eea45cSKlaus Jensen
326388eea45cSKlaus Jensen qemu_iovec_init(&ctx->data.iov, 1);
326488eea45cSKlaus Jensen qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, data_len);
326588eea45cSKlaus Jensen
326688eea45cSKlaus Jensen block_acct_start(blk_get_stats(blk), &req->acct, data_len,
326788eea45cSKlaus Jensen BLOCK_ACCT_READ);
326888eea45cSKlaus Jensen req->aiocb = blk_aio_preadv(blk, offset, &ctx->data.iov, 0,
326988eea45cSKlaus Jensen nvme_compare_data_cb, req);
327088eea45cSKlaus Jensen
327188eea45cSKlaus Jensen return NVME_NO_COMPLETE;
327288eea45cSKlaus Jensen }
327388eea45cSKlaus Jensen
327438f4ac65SKlaus Jensen typedef struct NvmeFlushAIOCB {
327538f4ac65SKlaus Jensen BlockAIOCB common;
327638f4ac65SKlaus Jensen BlockAIOCB *aiocb;
327738f4ac65SKlaus Jensen NvmeRequest *req;
327838f4ac65SKlaus Jensen int ret;
327938f4ac65SKlaus Jensen
328038f4ac65SKlaus Jensen NvmeNamespace *ns;
328138f4ac65SKlaus Jensen uint32_t nsid;
328238f4ac65SKlaus Jensen bool broadcast;
328338f4ac65SKlaus Jensen } NvmeFlushAIOCB;
328438f4ac65SKlaus Jensen
nvme_flush_cancel(BlockAIOCB * acb)328538f4ac65SKlaus Jensen static void nvme_flush_cancel(BlockAIOCB *acb)
328638f4ac65SKlaus Jensen {
328738f4ac65SKlaus Jensen NvmeFlushAIOCB *iocb = container_of(acb, NvmeFlushAIOCB, common);
328838f4ac65SKlaus Jensen
328938f4ac65SKlaus Jensen iocb->ret = -ECANCELED;
329038f4ac65SKlaus Jensen
329138f4ac65SKlaus Jensen if (iocb->aiocb) {
329238f4ac65SKlaus Jensen blk_aio_cancel_async(iocb->aiocb);
32933dbc1708SKlaus Jensen iocb->aiocb = NULL;
329438f4ac65SKlaus Jensen }
329538f4ac65SKlaus Jensen }
329638f4ac65SKlaus Jensen
329738f4ac65SKlaus Jensen static const AIOCBInfo nvme_flush_aiocb_info = {
329838f4ac65SKlaus Jensen .aiocb_size = sizeof(NvmeFlushAIOCB),
329938f4ac65SKlaus Jensen .cancel_async = nvme_flush_cancel,
330038f4ac65SKlaus Jensen };
330138f4ac65SKlaus Jensen
33023dbc1708SKlaus Jensen static void nvme_do_flush(NvmeFlushAIOCB *iocb);
33033dbc1708SKlaus Jensen
nvme_flush_ns_cb(void * opaque,int ret)330438f4ac65SKlaus Jensen static void nvme_flush_ns_cb(void *opaque, int ret)
330538f4ac65SKlaus Jensen {
330638f4ac65SKlaus Jensen NvmeFlushAIOCB *iocb = opaque;
330738f4ac65SKlaus Jensen NvmeNamespace *ns = iocb->ns;
330838f4ac65SKlaus Jensen
330938f4ac65SKlaus Jensen if (ret < 0) {
331038f4ac65SKlaus Jensen iocb->ret = ret;
331138f4ac65SKlaus Jensen goto out;
331238f4ac65SKlaus Jensen } else if (iocb->ret < 0) {
331338f4ac65SKlaus Jensen goto out;
331438f4ac65SKlaus Jensen }
331538f4ac65SKlaus Jensen
331638f4ac65SKlaus Jensen if (ns) {
331738f4ac65SKlaus Jensen trace_pci_nvme_flush_ns(iocb->nsid);
331838f4ac65SKlaus Jensen
331938f4ac65SKlaus Jensen iocb->ns = NULL;
332038f4ac65SKlaus Jensen iocb->aiocb = blk_aio_flush(ns->blkconf.blk, nvme_flush_ns_cb, iocb);
332138f4ac65SKlaus Jensen return;
332238f4ac65SKlaus Jensen }
332338f4ac65SKlaus Jensen
332438f4ac65SKlaus Jensen out:
33253dbc1708SKlaus Jensen nvme_do_flush(iocb);
332638f4ac65SKlaus Jensen }
332738f4ac65SKlaus Jensen
nvme_do_flush(NvmeFlushAIOCB * iocb)33283dbc1708SKlaus Jensen static void nvme_do_flush(NvmeFlushAIOCB *iocb)
332938f4ac65SKlaus Jensen {
333038f4ac65SKlaus Jensen NvmeRequest *req = iocb->req;
333138f4ac65SKlaus Jensen NvmeCtrl *n = nvme_ctrl(req);
333238f4ac65SKlaus Jensen int i;
333338f4ac65SKlaus Jensen
333438f4ac65SKlaus Jensen if (iocb->ret < 0) {
333538f4ac65SKlaus Jensen goto done;
333638f4ac65SKlaus Jensen }
333738f4ac65SKlaus Jensen
333838f4ac65SKlaus Jensen if (iocb->broadcast) {
333938f4ac65SKlaus Jensen for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
334038f4ac65SKlaus Jensen iocb->ns = nvme_ns(n, i);
334138f4ac65SKlaus Jensen if (iocb->ns) {
334238f4ac65SKlaus Jensen iocb->nsid = i;
334338f4ac65SKlaus Jensen break;
334438f4ac65SKlaus Jensen }
334538f4ac65SKlaus Jensen }
334638f4ac65SKlaus Jensen }
334738f4ac65SKlaus Jensen
334838f4ac65SKlaus Jensen if (!iocb->ns) {
334938f4ac65SKlaus Jensen goto done;
335038f4ac65SKlaus Jensen }
335138f4ac65SKlaus Jensen
335238f4ac65SKlaus Jensen nvme_flush_ns_cb(iocb, 0);
335338f4ac65SKlaus Jensen return;
335438f4ac65SKlaus Jensen
335538f4ac65SKlaus Jensen done:
335638f4ac65SKlaus Jensen iocb->common.cb(iocb->common.opaque, iocb->ret);
335738f4ac65SKlaus Jensen qemu_aio_unref(iocb);
335838f4ac65SKlaus Jensen }
335938f4ac65SKlaus Jensen
nvme_flush(NvmeCtrl * n,NvmeRequest * req)336088eea45cSKlaus Jensen static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req)
336188eea45cSKlaus Jensen {
336238f4ac65SKlaus Jensen NvmeFlushAIOCB *iocb;
336388eea45cSKlaus Jensen uint32_t nsid = le32_to_cpu(req->cmd.nsid);
336488eea45cSKlaus Jensen uint16_t status;
336588eea45cSKlaus Jensen
336638f4ac65SKlaus Jensen iocb = qemu_aio_get(&nvme_flush_aiocb_info, NULL, nvme_misc_cb, req);
336788eea45cSKlaus Jensen
336838f4ac65SKlaus Jensen iocb->req = req;
336938f4ac65SKlaus Jensen iocb->ret = 0;
337038f4ac65SKlaus Jensen iocb->ns = NULL;
337138f4ac65SKlaus Jensen iocb->nsid = 0;
337238f4ac65SKlaus Jensen iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
337338f4ac65SKlaus Jensen
337438f4ac65SKlaus Jensen if (!iocb->broadcast) {
337538f4ac65SKlaus Jensen if (!nvme_nsid_valid(n, nsid)) {
337638f4ac65SKlaus Jensen status = NVME_INVALID_NSID | NVME_DNR;
337738f4ac65SKlaus Jensen goto out;
337888eea45cSKlaus Jensen }
337988eea45cSKlaus Jensen
338038f4ac65SKlaus Jensen iocb->ns = nvme_ns(n, nsid);
338138f4ac65SKlaus Jensen if (!iocb->ns) {
338238f4ac65SKlaus Jensen status = NVME_INVALID_FIELD | NVME_DNR;
338338f4ac65SKlaus Jensen goto out;
338438f4ac65SKlaus Jensen }
338538f4ac65SKlaus Jensen
338638f4ac65SKlaus Jensen iocb->nsid = nsid;
338738f4ac65SKlaus Jensen }
338838f4ac65SKlaus Jensen
338938f4ac65SKlaus Jensen req->aiocb = &iocb->common;
33903dbc1708SKlaus Jensen nvme_do_flush(iocb);
339138f4ac65SKlaus Jensen
339288eea45cSKlaus Jensen return NVME_NO_COMPLETE;
339388eea45cSKlaus Jensen
339438f4ac65SKlaus Jensen out:
339538f4ac65SKlaus Jensen qemu_aio_unref(iocb);
339688eea45cSKlaus Jensen
339788eea45cSKlaus Jensen return status;
339888eea45cSKlaus Jensen }
339988eea45cSKlaus Jensen
nvme_read(NvmeCtrl * n,NvmeRequest * req)340088eea45cSKlaus Jensen static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req)
340188eea45cSKlaus Jensen {
340288eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
340388eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
340488eea45cSKlaus Jensen uint64_t slba = le64_to_cpu(rw->slba);
340588eea45cSKlaus Jensen uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
34062a132309SKlaus Jensen uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
340788eea45cSKlaus Jensen uint64_t data_size = nvme_l2b(ns, nlb);
340888eea45cSKlaus Jensen uint64_t mapped_size = data_size;
340988eea45cSKlaus Jensen uint64_t data_offset;
341088eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
341188eea45cSKlaus Jensen uint16_t status;
341288eea45cSKlaus Jensen
341388eea45cSKlaus Jensen if (nvme_ns_ext(ns)) {
341488eea45cSKlaus Jensen mapped_size += nvme_m2b(ns, nlb);
341588eea45cSKlaus Jensen
341688eea45cSKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
34172a132309SKlaus Jensen bool pract = prinfo & NVME_PRINFO_PRACT;
341888eea45cSKlaus Jensen
3419ac0b34c5SKlaus Jensen if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
342088eea45cSKlaus Jensen mapped_size = data_size;
342188eea45cSKlaus Jensen }
342288eea45cSKlaus Jensen }
342388eea45cSKlaus Jensen }
342488eea45cSKlaus Jensen
342588eea45cSKlaus Jensen trace_pci_nvme_read(nvme_cid(req), nvme_nsid(ns), nlb, mapped_size, slba);
342688eea45cSKlaus Jensen
342788eea45cSKlaus Jensen status = nvme_check_mdts(n, mapped_size);
342888eea45cSKlaus Jensen if (status) {
342988eea45cSKlaus Jensen goto invalid;
343088eea45cSKlaus Jensen }
343188eea45cSKlaus Jensen
343288eea45cSKlaus Jensen status = nvme_check_bounds(ns, slba, nlb);
343388eea45cSKlaus Jensen if (status) {
343488eea45cSKlaus Jensen goto invalid;
343588eea45cSKlaus Jensen }
343688eea45cSKlaus Jensen
343788eea45cSKlaus Jensen if (ns->params.zoned) {
343888eea45cSKlaus Jensen status = nvme_check_zone_read(ns, slba, nlb);
343988eea45cSKlaus Jensen if (status) {
344088eea45cSKlaus Jensen trace_pci_nvme_err_zone_read_not_ok(slba, nlb, status);
344188eea45cSKlaus Jensen goto invalid;
344288eea45cSKlaus Jensen }
344388eea45cSKlaus Jensen }
344488eea45cSKlaus Jensen
344588eea45cSKlaus Jensen if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
344688eea45cSKlaus Jensen status = nvme_check_dulbe(ns, slba, nlb);
344788eea45cSKlaus Jensen if (status) {
344888eea45cSKlaus Jensen goto invalid;
344988eea45cSKlaus Jensen }
345088eea45cSKlaus Jensen }
345188eea45cSKlaus Jensen
345288eea45cSKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
345388eea45cSKlaus Jensen return nvme_dif_rw(n, req);
345488eea45cSKlaus Jensen }
345588eea45cSKlaus Jensen
345688eea45cSKlaus Jensen status = nvme_map_data(n, nlb, req);
345788eea45cSKlaus Jensen if (status) {
345888eea45cSKlaus Jensen goto invalid;
345988eea45cSKlaus Jensen }
346088eea45cSKlaus Jensen
346188eea45cSKlaus Jensen data_offset = nvme_l2b(ns, slba);
346288eea45cSKlaus Jensen
346388eea45cSKlaus Jensen block_acct_start(blk_get_stats(blk), &req->acct, data_size,
346488eea45cSKlaus Jensen BLOCK_ACCT_READ);
34659b4f0181SMateusz Kozlowski nvme_blk_read(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
346688eea45cSKlaus Jensen return NVME_NO_COMPLETE;
346788eea45cSKlaus Jensen
346888eea45cSKlaus Jensen invalid:
346988eea45cSKlaus Jensen block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
347088eea45cSKlaus Jensen return status | NVME_DNR;
347188eea45cSKlaus Jensen }
347288eea45cSKlaus Jensen
nvme_do_write_fdp(NvmeCtrl * n,NvmeRequest * req,uint64_t slba,uint32_t nlb)347373064edfSJesper Devantier static void nvme_do_write_fdp(NvmeCtrl *n, NvmeRequest *req, uint64_t slba,
347473064edfSJesper Devantier uint32_t nlb)
347573064edfSJesper Devantier {
347673064edfSJesper Devantier NvmeNamespace *ns = req->ns;
347773064edfSJesper Devantier NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
347873064edfSJesper Devantier uint64_t data_size = nvme_l2b(ns, nlb);
347973064edfSJesper Devantier uint32_t dw12 = le32_to_cpu(req->cmd.cdw12);
348073064edfSJesper Devantier uint8_t dtype = (dw12 >> 20) & 0xf;
348173064edfSJesper Devantier uint16_t pid = le16_to_cpu(rw->dspec);
348273064edfSJesper Devantier uint16_t ph, rg, ruhid;
348373064edfSJesper Devantier NvmeReclaimUnit *ru;
348473064edfSJesper Devantier
348573064edfSJesper Devantier if (dtype != NVME_DIRECTIVE_DATA_PLACEMENT ||
348673064edfSJesper Devantier !nvme_parse_pid(ns, pid, &ph, &rg)) {
348773064edfSJesper Devantier ph = 0;
348873064edfSJesper Devantier rg = 0;
348973064edfSJesper Devantier }
349073064edfSJesper Devantier
349173064edfSJesper Devantier ruhid = ns->fdp.phs[ph];
349273064edfSJesper Devantier ru = &ns->endgrp->fdp.ruhs[ruhid].rus[rg];
349373064edfSJesper Devantier
349473064edfSJesper Devantier nvme_fdp_stat_inc(&ns->endgrp->fdp.hbmw, data_size);
349573064edfSJesper Devantier nvme_fdp_stat_inc(&ns->endgrp->fdp.mbmw, data_size);
349673064edfSJesper Devantier
349773064edfSJesper Devantier while (nlb) {
349873064edfSJesper Devantier if (nlb < ru->ruamw) {
349973064edfSJesper Devantier ru->ruamw -= nlb;
350073064edfSJesper Devantier break;
350173064edfSJesper Devantier }
350273064edfSJesper Devantier
350373064edfSJesper Devantier nlb -= ru->ruamw;
350473064edfSJesper Devantier nvme_update_ruh(n, ns, pid);
350573064edfSJesper Devantier }
350673064edfSJesper Devantier }
350773064edfSJesper Devantier
nvme_do_write(NvmeCtrl * n,NvmeRequest * req,bool append,bool wrz)350888eea45cSKlaus Jensen static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append,
350988eea45cSKlaus Jensen bool wrz)
351088eea45cSKlaus Jensen {
351188eea45cSKlaus Jensen NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
351288eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
351388eea45cSKlaus Jensen uint64_t slba = le64_to_cpu(rw->slba);
351488eea45cSKlaus Jensen uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
351588eea45cSKlaus Jensen uint16_t ctrl = le16_to_cpu(rw->control);
35162a132309SKlaus Jensen uint8_t prinfo = NVME_RW_PRINFO(ctrl);
351788eea45cSKlaus Jensen uint64_t data_size = nvme_l2b(ns, nlb);
351888eea45cSKlaus Jensen uint64_t mapped_size = data_size;
351988eea45cSKlaus Jensen uint64_t data_offset;
352088eea45cSKlaus Jensen NvmeZone *zone;
352188eea45cSKlaus Jensen NvmeZonedResult *res = (NvmeZonedResult *)&req->cqe;
352288eea45cSKlaus Jensen BlockBackend *blk = ns->blkconf.blk;
352388eea45cSKlaus Jensen uint16_t status;
352488eea45cSKlaus Jensen
352588eea45cSKlaus Jensen if (nvme_ns_ext(ns)) {
352688eea45cSKlaus Jensen mapped_size += nvme_m2b(ns, nlb);
352788eea45cSKlaus Jensen
352888eea45cSKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
35292a132309SKlaus Jensen bool pract = prinfo & NVME_PRINFO_PRACT;
353088eea45cSKlaus Jensen
3531ac0b34c5SKlaus Jensen if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
353288eea45cSKlaus Jensen mapped_size -= nvme_m2b(ns, nlb);
353388eea45cSKlaus Jensen }
353488eea45cSKlaus Jensen }
353588eea45cSKlaus Jensen }
353688eea45cSKlaus Jensen
353788eea45cSKlaus Jensen trace_pci_nvme_write(nvme_cid(req), nvme_io_opc_str(rw->opcode),
353888eea45cSKlaus Jensen nvme_nsid(ns), nlb, mapped_size, slba);
353988eea45cSKlaus Jensen
354088eea45cSKlaus Jensen if (!wrz) {
354188eea45cSKlaus Jensen status = nvme_check_mdts(n, mapped_size);
354288eea45cSKlaus Jensen if (status) {
354388eea45cSKlaus Jensen goto invalid;
354488eea45cSKlaus Jensen }
354588eea45cSKlaus Jensen }
354688eea45cSKlaus Jensen
354788eea45cSKlaus Jensen status = nvme_check_bounds(ns, slba, nlb);
354888eea45cSKlaus Jensen if (status) {
354988eea45cSKlaus Jensen goto invalid;
355088eea45cSKlaus Jensen }
355188eea45cSKlaus Jensen
355288eea45cSKlaus Jensen if (ns->params.zoned) {
355388eea45cSKlaus Jensen zone = nvme_get_zone_by_slba(ns, slba);
3554189a8bf7SKlaus Jensen assert(zone);
355588eea45cSKlaus Jensen
355688eea45cSKlaus Jensen if (append) {
355788eea45cSKlaus Jensen bool piremap = !!(ctrl & NVME_RW_PIREMAP);
355888eea45cSKlaus Jensen
3559e321b4cdSKlaus Jensen if (unlikely(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3560e321b4cdSKlaus Jensen return NVME_INVALID_ZONE_OP | NVME_DNR;
3561e321b4cdSKlaus Jensen }
3562e321b4cdSKlaus Jensen
356388eea45cSKlaus Jensen if (unlikely(slba != zone->d.zslba)) {
356488eea45cSKlaus Jensen trace_pci_nvme_err_append_not_at_start(slba, zone->d.zslba);
356588eea45cSKlaus Jensen status = NVME_INVALID_FIELD;
356688eea45cSKlaus Jensen goto invalid;
356788eea45cSKlaus Jensen }
356888eea45cSKlaus Jensen
356988eea45cSKlaus Jensen if (n->params.zasl &&
357088eea45cSKlaus Jensen data_size > (uint64_t)n->page_size << n->params.zasl) {
357188eea45cSKlaus Jensen trace_pci_nvme_err_zasl(data_size);
357288eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
357388eea45cSKlaus Jensen }
357488eea45cSKlaus Jensen
357588eea45cSKlaus Jensen slba = zone->w_ptr;
357688eea45cSKlaus Jensen rw->slba = cpu_to_le64(slba);
357788eea45cSKlaus Jensen res->slba = cpu_to_le64(slba);
357888eea45cSKlaus Jensen
357988eea45cSKlaus Jensen switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
358088eea45cSKlaus Jensen case NVME_ID_NS_DPS_TYPE_1:
358188eea45cSKlaus Jensen if (!piremap) {
358288eea45cSKlaus Jensen return NVME_INVALID_PROT_INFO | NVME_DNR;
358388eea45cSKlaus Jensen }
358488eea45cSKlaus Jensen
358588eea45cSKlaus Jensen /* fallthrough */
358688eea45cSKlaus Jensen
358788eea45cSKlaus Jensen case NVME_ID_NS_DPS_TYPE_2:
358888eea45cSKlaus Jensen if (piremap) {
358988eea45cSKlaus Jensen uint32_t reftag = le32_to_cpu(rw->reftag);
359088eea45cSKlaus Jensen rw->reftag = cpu_to_le32(reftag + (slba - zone->d.zslba));
359188eea45cSKlaus Jensen }
359288eea45cSKlaus Jensen
359388eea45cSKlaus Jensen break;
359488eea45cSKlaus Jensen
359588eea45cSKlaus Jensen case NVME_ID_NS_DPS_TYPE_3:
359688eea45cSKlaus Jensen if (piremap) {
359788eea45cSKlaus Jensen return NVME_INVALID_PROT_INFO | NVME_DNR;
359888eea45cSKlaus Jensen }
359988eea45cSKlaus Jensen
360088eea45cSKlaus Jensen break;
360188eea45cSKlaus Jensen }
360288eea45cSKlaus Jensen }
360388eea45cSKlaus Jensen
360488eea45cSKlaus Jensen status = nvme_check_zone_write(ns, zone, slba, nlb);
360588eea45cSKlaus Jensen if (status) {
360688eea45cSKlaus Jensen goto invalid;
360788eea45cSKlaus Jensen }
360888eea45cSKlaus Jensen
3609cccc2651SNiklas Cassel status = nvme_zrm_auto(n, ns, zone);
361088eea45cSKlaus Jensen if (status) {
361188eea45cSKlaus Jensen goto invalid;
361288eea45cSKlaus Jensen }
361388eea45cSKlaus Jensen
3614e321b4cdSKlaus Jensen if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
361588eea45cSKlaus Jensen zone->w_ptr += nlb;
361688eea45cSKlaus Jensen }
361773064edfSJesper Devantier } else if (ns->endgrp && ns->endgrp->fdp.enabled) {
361873064edfSJesper Devantier nvme_do_write_fdp(n, req, slba, nlb);
3619e321b4cdSKlaus Jensen }
362088eea45cSKlaus Jensen
362188eea45cSKlaus Jensen data_offset = nvme_l2b(ns, slba);
362288eea45cSKlaus Jensen
362388eea45cSKlaus Jensen if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
362488eea45cSKlaus Jensen return nvme_dif_rw(n, req);
362588eea45cSKlaus Jensen }
362688eea45cSKlaus Jensen
362788eea45cSKlaus Jensen if (!wrz) {
362888eea45cSKlaus Jensen status = nvme_map_data(n, nlb, req);
362988eea45cSKlaus Jensen if (status) {
363088eea45cSKlaus Jensen goto invalid;
363188eea45cSKlaus Jensen }
363288eea45cSKlaus Jensen
363388eea45cSKlaus Jensen block_acct_start(blk_get_stats(blk), &req->acct, data_size,
363488eea45cSKlaus Jensen BLOCK_ACCT_WRITE);
36359b4f0181SMateusz Kozlowski nvme_blk_write(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
363688eea45cSKlaus Jensen } else {
363788eea45cSKlaus Jensen req->aiocb = blk_aio_pwrite_zeroes(blk, data_offset, data_size,
363888eea45cSKlaus Jensen BDRV_REQ_MAY_UNMAP, nvme_rw_cb,
363988eea45cSKlaus Jensen req);
364088eea45cSKlaus Jensen }
364188eea45cSKlaus Jensen
364288eea45cSKlaus Jensen return NVME_NO_COMPLETE;
364388eea45cSKlaus Jensen
364488eea45cSKlaus Jensen invalid:
364588eea45cSKlaus Jensen block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
364688eea45cSKlaus Jensen return status | NVME_DNR;
364788eea45cSKlaus Jensen }
364888eea45cSKlaus Jensen
nvme_write(NvmeCtrl * n,NvmeRequest * req)364988eea45cSKlaus Jensen static inline uint16_t nvme_write(NvmeCtrl *n, NvmeRequest *req)
365088eea45cSKlaus Jensen {
365188eea45cSKlaus Jensen return nvme_do_write(n, req, false, false);
365288eea45cSKlaus Jensen }
365388eea45cSKlaus Jensen
nvme_write_zeroes(NvmeCtrl * n,NvmeRequest * req)365488eea45cSKlaus Jensen static inline uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req)
365588eea45cSKlaus Jensen {
365688eea45cSKlaus Jensen return nvme_do_write(n, req, false, true);
365788eea45cSKlaus Jensen }
365888eea45cSKlaus Jensen
nvme_zone_append(NvmeCtrl * n,NvmeRequest * req)365988eea45cSKlaus Jensen static inline uint16_t nvme_zone_append(NvmeCtrl *n, NvmeRequest *req)
366088eea45cSKlaus Jensen {
366188eea45cSKlaus Jensen return nvme_do_write(n, req, true, false);
366288eea45cSKlaus Jensen }
366388eea45cSKlaus Jensen
nvme_get_mgmt_zone_slba_idx(NvmeNamespace * ns,NvmeCmd * c,uint64_t * slba,uint32_t * zone_idx)366488eea45cSKlaus Jensen static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace *ns, NvmeCmd *c,
366588eea45cSKlaus Jensen uint64_t *slba, uint32_t *zone_idx)
366688eea45cSKlaus Jensen {
366788eea45cSKlaus Jensen uint32_t dw10 = le32_to_cpu(c->cdw10);
366888eea45cSKlaus Jensen uint32_t dw11 = le32_to_cpu(c->cdw11);
366988eea45cSKlaus Jensen
367088eea45cSKlaus Jensen if (!ns->params.zoned) {
367188eea45cSKlaus Jensen trace_pci_nvme_err_invalid_opc(c->opcode);
367288eea45cSKlaus Jensen return NVME_INVALID_OPCODE | NVME_DNR;
367388eea45cSKlaus Jensen }
367488eea45cSKlaus Jensen
367588eea45cSKlaus Jensen *slba = ((uint64_t)dw11) << 32 | dw10;
367688eea45cSKlaus Jensen if (unlikely(*slba >= ns->id_ns.nsze)) {
367788eea45cSKlaus Jensen trace_pci_nvme_err_invalid_lba_range(*slba, 0, ns->id_ns.nsze);
367888eea45cSKlaus Jensen *slba = 0;
367988eea45cSKlaus Jensen return NVME_LBA_RANGE | NVME_DNR;
368088eea45cSKlaus Jensen }
368188eea45cSKlaus Jensen
368288eea45cSKlaus Jensen *zone_idx = nvme_zone_idx(ns, *slba);
368388eea45cSKlaus Jensen assert(*zone_idx < ns->num_zones);
368488eea45cSKlaus Jensen
368588eea45cSKlaus Jensen return NVME_SUCCESS;
368688eea45cSKlaus Jensen }
368788eea45cSKlaus Jensen
368888eea45cSKlaus Jensen typedef uint16_t (*op_handler_t)(NvmeNamespace *, NvmeZone *, NvmeZoneState,
368988eea45cSKlaus Jensen NvmeRequest *);
369088eea45cSKlaus Jensen
369188eea45cSKlaus Jensen enum NvmeZoneProcessingMask {
369288eea45cSKlaus Jensen NVME_PROC_CURRENT_ZONE = 0,
369388eea45cSKlaus Jensen NVME_PROC_OPENED_ZONES = 1 << 0,
369488eea45cSKlaus Jensen NVME_PROC_CLOSED_ZONES = 1 << 1,
369588eea45cSKlaus Jensen NVME_PROC_READ_ONLY_ZONES = 1 << 2,
369688eea45cSKlaus Jensen NVME_PROC_FULL_ZONES = 1 << 3,
369788eea45cSKlaus Jensen };
369888eea45cSKlaus Jensen
nvme_open_zone(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state,NvmeRequest * req)369988eea45cSKlaus Jensen static uint16_t nvme_open_zone(NvmeNamespace *ns, NvmeZone *zone,
370088eea45cSKlaus Jensen NvmeZoneState state, NvmeRequest *req)
370188eea45cSKlaus Jensen {
3702e321b4cdSKlaus Jensen NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
3703e321b4cdSKlaus Jensen int flags = 0;
3704e321b4cdSKlaus Jensen
3705e321b4cdSKlaus Jensen if (cmd->zsflags & NVME_ZSFLAG_ZRWA_ALLOC) {
3706e321b4cdSKlaus Jensen uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3707e321b4cdSKlaus Jensen
3708e321b4cdSKlaus Jensen if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3709e321b4cdSKlaus Jensen return NVME_INVALID_ZONE_OP | NVME_DNR;
3710e321b4cdSKlaus Jensen }
3711e321b4cdSKlaus Jensen
3712e321b4cdSKlaus Jensen if (zone->w_ptr % ns->zns.zrwafg) {
3713e321b4cdSKlaus Jensen return NVME_NOZRWA | NVME_DNR;
3714e321b4cdSKlaus Jensen }
3715e321b4cdSKlaus Jensen
3716e321b4cdSKlaus Jensen flags = NVME_ZRM_ZRWA;
3717e321b4cdSKlaus Jensen }
3718e321b4cdSKlaus Jensen
3719e321b4cdSKlaus Jensen return nvme_zrm_open_flags(nvme_ctrl(req), ns, zone, flags);
372088eea45cSKlaus Jensen }
372188eea45cSKlaus Jensen
nvme_close_zone(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state,NvmeRequest * req)372288eea45cSKlaus Jensen static uint16_t nvme_close_zone(NvmeNamespace *ns, NvmeZone *zone,
372388eea45cSKlaus Jensen NvmeZoneState state, NvmeRequest *req)
372488eea45cSKlaus Jensen {
372588eea45cSKlaus Jensen return nvme_zrm_close(ns, zone);
372688eea45cSKlaus Jensen }
372788eea45cSKlaus Jensen
nvme_finish_zone(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state,NvmeRequest * req)372888eea45cSKlaus Jensen static uint16_t nvme_finish_zone(NvmeNamespace *ns, NvmeZone *zone,
372988eea45cSKlaus Jensen NvmeZoneState state, NvmeRequest *req)
373088eea45cSKlaus Jensen {
373188eea45cSKlaus Jensen return nvme_zrm_finish(ns, zone);
373288eea45cSKlaus Jensen }
373388eea45cSKlaus Jensen
nvme_offline_zone(NvmeNamespace * ns,NvmeZone * zone,NvmeZoneState state,NvmeRequest * req)373488eea45cSKlaus Jensen static uint16_t nvme_offline_zone(NvmeNamespace *ns, NvmeZone *zone,
373588eea45cSKlaus Jensen NvmeZoneState state, NvmeRequest *req)
373688eea45cSKlaus Jensen {
373788eea45cSKlaus Jensen switch (state) {
373888eea45cSKlaus Jensen case NVME_ZONE_STATE_READ_ONLY:
373988eea45cSKlaus Jensen nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_OFFLINE);
374088eea45cSKlaus Jensen /* fall through */
374188eea45cSKlaus Jensen case NVME_ZONE_STATE_OFFLINE:
374288eea45cSKlaus Jensen return NVME_SUCCESS;
374388eea45cSKlaus Jensen default:
374488eea45cSKlaus Jensen return NVME_ZONE_INVAL_TRANSITION;
374588eea45cSKlaus Jensen }
374688eea45cSKlaus Jensen }
374788eea45cSKlaus Jensen
nvme_set_zd_ext(NvmeNamespace * ns,NvmeZone * zone)374888eea45cSKlaus Jensen static uint16_t nvme_set_zd_ext(NvmeNamespace *ns, NvmeZone *zone)
374988eea45cSKlaus Jensen {
375088eea45cSKlaus Jensen uint16_t status;
375188eea45cSKlaus Jensen uint8_t state = nvme_get_zone_state(zone);
375288eea45cSKlaus Jensen
375388eea45cSKlaus Jensen if (state == NVME_ZONE_STATE_EMPTY) {
375488eea45cSKlaus Jensen status = nvme_aor_check(ns, 1, 0);
375588eea45cSKlaus Jensen if (status) {
375688eea45cSKlaus Jensen return status;
375788eea45cSKlaus Jensen }
375888eea45cSKlaus Jensen nvme_aor_inc_active(ns);
375988eea45cSKlaus Jensen zone->d.za |= NVME_ZA_ZD_EXT_VALID;
376088eea45cSKlaus Jensen nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
376188eea45cSKlaus Jensen return NVME_SUCCESS;
376288eea45cSKlaus Jensen }
376388eea45cSKlaus Jensen
376488eea45cSKlaus Jensen return NVME_ZONE_INVAL_TRANSITION;
376588eea45cSKlaus Jensen }
376688eea45cSKlaus Jensen
nvme_bulk_proc_zone(NvmeNamespace * ns,NvmeZone * zone,enum NvmeZoneProcessingMask proc_mask,op_handler_t op_hndlr,NvmeRequest * req)376788eea45cSKlaus Jensen static uint16_t nvme_bulk_proc_zone(NvmeNamespace *ns, NvmeZone *zone,
376888eea45cSKlaus Jensen enum NvmeZoneProcessingMask proc_mask,
376988eea45cSKlaus Jensen op_handler_t op_hndlr, NvmeRequest *req)
377088eea45cSKlaus Jensen {
377188eea45cSKlaus Jensen uint16_t status = NVME_SUCCESS;
377288eea45cSKlaus Jensen NvmeZoneState zs = nvme_get_zone_state(zone);
377388eea45cSKlaus Jensen bool proc_zone;
377488eea45cSKlaus Jensen
377588eea45cSKlaus Jensen switch (zs) {
377688eea45cSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
377788eea45cSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
377888eea45cSKlaus Jensen proc_zone = proc_mask & NVME_PROC_OPENED_ZONES;
377988eea45cSKlaus Jensen break;
378088eea45cSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
378188eea45cSKlaus Jensen proc_zone = proc_mask & NVME_PROC_CLOSED_ZONES;
378288eea45cSKlaus Jensen break;
378388eea45cSKlaus Jensen case NVME_ZONE_STATE_READ_ONLY:
378488eea45cSKlaus Jensen proc_zone = proc_mask & NVME_PROC_READ_ONLY_ZONES;
378588eea45cSKlaus Jensen break;
378688eea45cSKlaus Jensen case NVME_ZONE_STATE_FULL:
378788eea45cSKlaus Jensen proc_zone = proc_mask & NVME_PROC_FULL_ZONES;
378888eea45cSKlaus Jensen break;
378988eea45cSKlaus Jensen default:
379088eea45cSKlaus Jensen proc_zone = false;
379188eea45cSKlaus Jensen }
379288eea45cSKlaus Jensen
379388eea45cSKlaus Jensen if (proc_zone) {
379488eea45cSKlaus Jensen status = op_hndlr(ns, zone, zs, req);
379588eea45cSKlaus Jensen }
379688eea45cSKlaus Jensen
379788eea45cSKlaus Jensen return status;
379888eea45cSKlaus Jensen }
379988eea45cSKlaus Jensen
nvme_do_zone_op(NvmeNamespace * ns,NvmeZone * zone,enum NvmeZoneProcessingMask proc_mask,op_handler_t op_hndlr,NvmeRequest * req)380088eea45cSKlaus Jensen static uint16_t nvme_do_zone_op(NvmeNamespace *ns, NvmeZone *zone,
380188eea45cSKlaus Jensen enum NvmeZoneProcessingMask proc_mask,
380288eea45cSKlaus Jensen op_handler_t op_hndlr, NvmeRequest *req)
380388eea45cSKlaus Jensen {
380488eea45cSKlaus Jensen NvmeZone *next;
380588eea45cSKlaus Jensen uint16_t status = NVME_SUCCESS;
380688eea45cSKlaus Jensen int i;
380788eea45cSKlaus Jensen
380888eea45cSKlaus Jensen if (!proc_mask) {
380988eea45cSKlaus Jensen status = op_hndlr(ns, zone, nvme_get_zone_state(zone), req);
381088eea45cSKlaus Jensen } else {
381188eea45cSKlaus Jensen if (proc_mask & NVME_PROC_CLOSED_ZONES) {
381288eea45cSKlaus Jensen QTAILQ_FOREACH_SAFE(zone, &ns->closed_zones, entry, next) {
381388eea45cSKlaus Jensen status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
381488eea45cSKlaus Jensen req);
381588eea45cSKlaus Jensen if (status && status != NVME_NO_COMPLETE) {
381688eea45cSKlaus Jensen goto out;
381788eea45cSKlaus Jensen }
381888eea45cSKlaus Jensen }
381988eea45cSKlaus Jensen }
382088eea45cSKlaus Jensen if (proc_mask & NVME_PROC_OPENED_ZONES) {
382188eea45cSKlaus Jensen QTAILQ_FOREACH_SAFE(zone, &ns->imp_open_zones, entry, next) {
382288eea45cSKlaus Jensen status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
382388eea45cSKlaus Jensen req);
382488eea45cSKlaus Jensen if (status && status != NVME_NO_COMPLETE) {
382588eea45cSKlaus Jensen goto out;
382688eea45cSKlaus Jensen }
382788eea45cSKlaus Jensen }
382888eea45cSKlaus Jensen
382988eea45cSKlaus Jensen QTAILQ_FOREACH_SAFE(zone, &ns->exp_open_zones, entry, next) {
383088eea45cSKlaus Jensen status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
383188eea45cSKlaus Jensen req);
383288eea45cSKlaus Jensen if (status && status != NVME_NO_COMPLETE) {
383388eea45cSKlaus Jensen goto out;
383488eea45cSKlaus Jensen }
383588eea45cSKlaus Jensen }
383688eea45cSKlaus Jensen }
383788eea45cSKlaus Jensen if (proc_mask & NVME_PROC_FULL_ZONES) {
383888eea45cSKlaus Jensen QTAILQ_FOREACH_SAFE(zone, &ns->full_zones, entry, next) {
383988eea45cSKlaus Jensen status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
384088eea45cSKlaus Jensen req);
384188eea45cSKlaus Jensen if (status && status != NVME_NO_COMPLETE) {
384288eea45cSKlaus Jensen goto out;
384388eea45cSKlaus Jensen }
384488eea45cSKlaus Jensen }
384588eea45cSKlaus Jensen }
384688eea45cSKlaus Jensen
384788eea45cSKlaus Jensen if (proc_mask & NVME_PROC_READ_ONLY_ZONES) {
384888eea45cSKlaus Jensen for (i = 0; i < ns->num_zones; i++, zone++) {
384988eea45cSKlaus Jensen status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
385088eea45cSKlaus Jensen req);
385188eea45cSKlaus Jensen if (status && status != NVME_NO_COMPLETE) {
385288eea45cSKlaus Jensen goto out;
385388eea45cSKlaus Jensen }
385488eea45cSKlaus Jensen }
385588eea45cSKlaus Jensen }
385688eea45cSKlaus Jensen }
385788eea45cSKlaus Jensen
385888eea45cSKlaus Jensen out:
385988eea45cSKlaus Jensen return status;
386088eea45cSKlaus Jensen }
386188eea45cSKlaus Jensen
386263d96e4fSKlaus Jensen typedef struct NvmeZoneResetAIOCB {
386363d96e4fSKlaus Jensen BlockAIOCB common;
386463d96e4fSKlaus Jensen BlockAIOCB *aiocb;
386563d96e4fSKlaus Jensen NvmeRequest *req;
386663d96e4fSKlaus Jensen int ret;
386763d96e4fSKlaus Jensen
386863d96e4fSKlaus Jensen bool all;
386963d96e4fSKlaus Jensen int idx;
387063d96e4fSKlaus Jensen NvmeZone *zone;
387163d96e4fSKlaus Jensen } NvmeZoneResetAIOCB;
387263d96e4fSKlaus Jensen
nvme_zone_reset_cancel(BlockAIOCB * aiocb)387363d96e4fSKlaus Jensen static void nvme_zone_reset_cancel(BlockAIOCB *aiocb)
387463d96e4fSKlaus Jensen {
387563d96e4fSKlaus Jensen NvmeZoneResetAIOCB *iocb = container_of(aiocb, NvmeZoneResetAIOCB, common);
387663d96e4fSKlaus Jensen NvmeRequest *req = iocb->req;
387763d96e4fSKlaus Jensen NvmeNamespace *ns = req->ns;
387863d96e4fSKlaus Jensen
387963d96e4fSKlaus Jensen iocb->idx = ns->num_zones;
388063d96e4fSKlaus Jensen
388163d96e4fSKlaus Jensen iocb->ret = -ECANCELED;
388263d96e4fSKlaus Jensen
388363d96e4fSKlaus Jensen if (iocb->aiocb) {
388463d96e4fSKlaus Jensen blk_aio_cancel_async(iocb->aiocb);
388563d96e4fSKlaus Jensen iocb->aiocb = NULL;
388663d96e4fSKlaus Jensen }
388763d96e4fSKlaus Jensen }
388863d96e4fSKlaus Jensen
388963d96e4fSKlaus Jensen static const AIOCBInfo nvme_zone_reset_aiocb_info = {
389063d96e4fSKlaus Jensen .aiocb_size = sizeof(NvmeZoneResetAIOCB),
389163d96e4fSKlaus Jensen .cancel_async = nvme_zone_reset_cancel,
389263d96e4fSKlaus Jensen };
389363d96e4fSKlaus Jensen
389463d96e4fSKlaus Jensen static void nvme_zone_reset_cb(void *opaque, int ret);
389563d96e4fSKlaus Jensen
nvme_zone_reset_epilogue_cb(void * opaque,int ret)389663d96e4fSKlaus Jensen static void nvme_zone_reset_epilogue_cb(void *opaque, int ret)
389763d96e4fSKlaus Jensen {
389863d96e4fSKlaus Jensen NvmeZoneResetAIOCB *iocb = opaque;
389963d96e4fSKlaus Jensen NvmeRequest *req = iocb->req;
390063d96e4fSKlaus Jensen NvmeNamespace *ns = req->ns;
390163d96e4fSKlaus Jensen int64_t moff;
390263d96e4fSKlaus Jensen int count;
390363d96e4fSKlaus Jensen
390436a251c3SKlaus Jensen if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
390536a251c3SKlaus Jensen goto out;
390663d96e4fSKlaus Jensen }
390763d96e4fSKlaus Jensen
390863d96e4fSKlaus Jensen moff = nvme_moff(ns, iocb->zone->d.zslba);
390963d96e4fSKlaus Jensen count = nvme_m2b(ns, ns->zone_size);
391063d96e4fSKlaus Jensen
391163d96e4fSKlaus Jensen iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, moff, count,
391263d96e4fSKlaus Jensen BDRV_REQ_MAY_UNMAP,
391363d96e4fSKlaus Jensen nvme_zone_reset_cb, iocb);
391463d96e4fSKlaus Jensen return;
391536a251c3SKlaus Jensen
391636a251c3SKlaus Jensen out:
391736a251c3SKlaus Jensen nvme_zone_reset_cb(iocb, ret);
391863d96e4fSKlaus Jensen }
391963d96e4fSKlaus Jensen
nvme_zone_reset_cb(void * opaque,int ret)392063d96e4fSKlaus Jensen static void nvme_zone_reset_cb(void *opaque, int ret)
392163d96e4fSKlaus Jensen {
392263d96e4fSKlaus Jensen NvmeZoneResetAIOCB *iocb = opaque;
392363d96e4fSKlaus Jensen NvmeRequest *req = iocb->req;
392463d96e4fSKlaus Jensen NvmeNamespace *ns = req->ns;
392563d96e4fSKlaus Jensen
392636a251c3SKlaus Jensen if (iocb->ret < 0) {
392736a251c3SKlaus Jensen goto done;
392836a251c3SKlaus Jensen } else if (ret < 0) {
392963d96e4fSKlaus Jensen iocb->ret = ret;
393063d96e4fSKlaus Jensen goto done;
393163d96e4fSKlaus Jensen }
393263d96e4fSKlaus Jensen
393363d96e4fSKlaus Jensen if (iocb->zone) {
393463d96e4fSKlaus Jensen nvme_zrm_reset(ns, iocb->zone);
393563d96e4fSKlaus Jensen
393663d96e4fSKlaus Jensen if (!iocb->all) {
393763d96e4fSKlaus Jensen goto done;
393863d96e4fSKlaus Jensen }
393963d96e4fSKlaus Jensen }
394063d96e4fSKlaus Jensen
394163d96e4fSKlaus Jensen while (iocb->idx < ns->num_zones) {
394263d96e4fSKlaus Jensen NvmeZone *zone = &ns->zone_array[iocb->idx++];
394363d96e4fSKlaus Jensen
394463d96e4fSKlaus Jensen switch (nvme_get_zone_state(zone)) {
394563d96e4fSKlaus Jensen case NVME_ZONE_STATE_EMPTY:
394663d96e4fSKlaus Jensen if (!iocb->all) {
394763d96e4fSKlaus Jensen goto done;
394863d96e4fSKlaus Jensen }
394963d96e4fSKlaus Jensen
395063d96e4fSKlaus Jensen continue;
395163d96e4fSKlaus Jensen
395263d96e4fSKlaus Jensen case NVME_ZONE_STATE_EXPLICITLY_OPEN:
395363d96e4fSKlaus Jensen case NVME_ZONE_STATE_IMPLICITLY_OPEN:
395463d96e4fSKlaus Jensen case NVME_ZONE_STATE_CLOSED:
395563d96e4fSKlaus Jensen case NVME_ZONE_STATE_FULL:
395663d96e4fSKlaus Jensen iocb->zone = zone;
395763d96e4fSKlaus Jensen break;
395863d96e4fSKlaus Jensen
395963d96e4fSKlaus Jensen default:
396063d96e4fSKlaus Jensen continue;
396163d96e4fSKlaus Jensen }
396263d96e4fSKlaus Jensen
396363d96e4fSKlaus Jensen trace_pci_nvme_zns_zone_reset(zone->d.zslba);
396463d96e4fSKlaus Jensen
396563d96e4fSKlaus Jensen iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk,
396663d96e4fSKlaus Jensen nvme_l2b(ns, zone->d.zslba),
396763d96e4fSKlaus Jensen nvme_l2b(ns, ns->zone_size),
396863d96e4fSKlaus Jensen BDRV_REQ_MAY_UNMAP,
396963d96e4fSKlaus Jensen nvme_zone_reset_epilogue_cb,
397063d96e4fSKlaus Jensen iocb);
397163d96e4fSKlaus Jensen return;
397263d96e4fSKlaus Jensen }
397363d96e4fSKlaus Jensen
397463d96e4fSKlaus Jensen done:
397563d96e4fSKlaus Jensen iocb->aiocb = NULL;
397636a251c3SKlaus Jensen
397736a251c3SKlaus Jensen iocb->common.cb(iocb->common.opaque, iocb->ret);
397836a251c3SKlaus Jensen qemu_aio_unref(iocb);
397963d96e4fSKlaus Jensen }
398063d96e4fSKlaus Jensen
nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl * n,NvmeZone * zone,uint64_t elba,NvmeRequest * req)3981e321b4cdSKlaus Jensen static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl *n, NvmeZone *zone,
3982e321b4cdSKlaus Jensen uint64_t elba, NvmeRequest *req)
3983e321b4cdSKlaus Jensen {
3984e321b4cdSKlaus Jensen NvmeNamespace *ns = req->ns;
3985e321b4cdSKlaus Jensen uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3986e321b4cdSKlaus Jensen uint64_t wp = zone->d.wp;
3987e321b4cdSKlaus Jensen uint32_t nlb = elba - wp + 1;
3988e321b4cdSKlaus Jensen uint16_t status;
3989e321b4cdSKlaus Jensen
3990e321b4cdSKlaus Jensen
3991e321b4cdSKlaus Jensen if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3992e321b4cdSKlaus Jensen return NVME_INVALID_ZONE_OP | NVME_DNR;
3993e321b4cdSKlaus Jensen }
3994e321b4cdSKlaus Jensen
3995e321b4cdSKlaus Jensen if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3996e321b4cdSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
3997e321b4cdSKlaus Jensen }
3998e321b4cdSKlaus Jensen
3999e321b4cdSKlaus Jensen if (elba < wp || elba > wp + ns->zns.zrwas) {
4000e321b4cdSKlaus Jensen return NVME_ZONE_BOUNDARY_ERROR | NVME_DNR;
4001e321b4cdSKlaus Jensen }
4002e321b4cdSKlaus Jensen
4003e321b4cdSKlaus Jensen if (nlb % ns->zns.zrwafg) {
4004e321b4cdSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
4005e321b4cdSKlaus Jensen }
4006e321b4cdSKlaus Jensen
4007e321b4cdSKlaus Jensen status = nvme_zrm_auto(n, ns, zone);
4008e321b4cdSKlaus Jensen if (status) {
4009e321b4cdSKlaus Jensen return status;
4010e321b4cdSKlaus Jensen }
4011e321b4cdSKlaus Jensen
4012e321b4cdSKlaus Jensen zone->w_ptr += nlb;
4013e321b4cdSKlaus Jensen
4014e321b4cdSKlaus Jensen nvme_advance_zone_wp(ns, zone, nlb);
4015e321b4cdSKlaus Jensen
4016e321b4cdSKlaus Jensen return NVME_SUCCESS;
4017e321b4cdSKlaus Jensen }
4018e321b4cdSKlaus Jensen
nvme_zone_mgmt_send(NvmeCtrl * n,NvmeRequest * req)401988eea45cSKlaus Jensen static uint16_t nvme_zone_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
402088eea45cSKlaus Jensen {
40216190d92fSKlaus Jensen NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
402288eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
402388eea45cSKlaus Jensen NvmeZone *zone;
402463d96e4fSKlaus Jensen NvmeZoneResetAIOCB *iocb;
402588eea45cSKlaus Jensen uint8_t *zd_ext;
402688eea45cSKlaus Jensen uint64_t slba = 0;
402788eea45cSKlaus Jensen uint32_t zone_idx = 0;
402888eea45cSKlaus Jensen uint16_t status;
40296190d92fSKlaus Jensen uint8_t action = cmd->zsa;
403088eea45cSKlaus Jensen bool all;
403188eea45cSKlaus Jensen enum NvmeZoneProcessingMask proc_mask = NVME_PROC_CURRENT_ZONE;
403288eea45cSKlaus Jensen
40336190d92fSKlaus Jensen all = cmd->zsflags & NVME_ZSFLAG_SELECT_ALL;
403488eea45cSKlaus Jensen
403588eea45cSKlaus Jensen req->status = NVME_SUCCESS;
403688eea45cSKlaus Jensen
403788eea45cSKlaus Jensen if (!all) {
40386190d92fSKlaus Jensen status = nvme_get_mgmt_zone_slba_idx(ns, &req->cmd, &slba, &zone_idx);
403988eea45cSKlaus Jensen if (status) {
404088eea45cSKlaus Jensen return status;
404188eea45cSKlaus Jensen }
404288eea45cSKlaus Jensen }
404388eea45cSKlaus Jensen
404488eea45cSKlaus Jensen zone = &ns->zone_array[zone_idx];
4045e321b4cdSKlaus Jensen if (slba != zone->d.zslba && action != NVME_ZONE_ACTION_ZRWA_FLUSH) {
404688eea45cSKlaus Jensen trace_pci_nvme_err_unaligned_zone_cmd(action, slba, zone->d.zslba);
404788eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
404888eea45cSKlaus Jensen }
404988eea45cSKlaus Jensen
405088eea45cSKlaus Jensen switch (action) {
405188eea45cSKlaus Jensen
405288eea45cSKlaus Jensen case NVME_ZONE_ACTION_OPEN:
405388eea45cSKlaus Jensen if (all) {
405488eea45cSKlaus Jensen proc_mask = NVME_PROC_CLOSED_ZONES;
405588eea45cSKlaus Jensen }
405688eea45cSKlaus Jensen trace_pci_nvme_open_zone(slba, zone_idx, all);
405788eea45cSKlaus Jensen status = nvme_do_zone_op(ns, zone, proc_mask, nvme_open_zone, req);
405888eea45cSKlaus Jensen break;
405988eea45cSKlaus Jensen
406088eea45cSKlaus Jensen case NVME_ZONE_ACTION_CLOSE:
406188eea45cSKlaus Jensen if (all) {
406288eea45cSKlaus Jensen proc_mask = NVME_PROC_OPENED_ZONES;
406388eea45cSKlaus Jensen }
406488eea45cSKlaus Jensen trace_pci_nvme_close_zone(slba, zone_idx, all);
406588eea45cSKlaus Jensen status = nvme_do_zone_op(ns, zone, proc_mask, nvme_close_zone, req);
406688eea45cSKlaus Jensen break;
406788eea45cSKlaus Jensen
406888eea45cSKlaus Jensen case NVME_ZONE_ACTION_FINISH:
406988eea45cSKlaus Jensen if (all) {
407088eea45cSKlaus Jensen proc_mask = NVME_PROC_OPENED_ZONES | NVME_PROC_CLOSED_ZONES;
407188eea45cSKlaus Jensen }
407288eea45cSKlaus Jensen trace_pci_nvme_finish_zone(slba, zone_idx, all);
407388eea45cSKlaus Jensen status = nvme_do_zone_op(ns, zone, proc_mask, nvme_finish_zone, req);
407488eea45cSKlaus Jensen break;
407588eea45cSKlaus Jensen
407688eea45cSKlaus Jensen case NVME_ZONE_ACTION_RESET:
407788eea45cSKlaus Jensen trace_pci_nvme_reset_zone(slba, zone_idx, all);
407888eea45cSKlaus Jensen
407963d96e4fSKlaus Jensen iocb = blk_aio_get(&nvme_zone_reset_aiocb_info, ns->blkconf.blk,
408063d96e4fSKlaus Jensen nvme_misc_cb, req);
408188eea45cSKlaus Jensen
408263d96e4fSKlaus Jensen iocb->req = req;
408363d96e4fSKlaus Jensen iocb->ret = 0;
408463d96e4fSKlaus Jensen iocb->all = all;
408563d96e4fSKlaus Jensen iocb->idx = zone_idx;
408663d96e4fSKlaus Jensen iocb->zone = NULL;
408788eea45cSKlaus Jensen
408863d96e4fSKlaus Jensen req->aiocb = &iocb->common;
408963d96e4fSKlaus Jensen nvme_zone_reset_cb(iocb, 0);
409088eea45cSKlaus Jensen
409163d96e4fSKlaus Jensen return NVME_NO_COMPLETE;
409288eea45cSKlaus Jensen
409388eea45cSKlaus Jensen case NVME_ZONE_ACTION_OFFLINE:
409488eea45cSKlaus Jensen if (all) {
409588eea45cSKlaus Jensen proc_mask = NVME_PROC_READ_ONLY_ZONES;
409688eea45cSKlaus Jensen }
409788eea45cSKlaus Jensen trace_pci_nvme_offline_zone(slba, zone_idx, all);
409888eea45cSKlaus Jensen status = nvme_do_zone_op(ns, zone, proc_mask, nvme_offline_zone, req);
409988eea45cSKlaus Jensen break;
410088eea45cSKlaus Jensen
410188eea45cSKlaus Jensen case NVME_ZONE_ACTION_SET_ZD_EXT:
410288eea45cSKlaus Jensen trace_pci_nvme_set_descriptor_extension(slba, zone_idx);
410388eea45cSKlaus Jensen if (all || !ns->params.zd_extension_size) {
410488eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
410588eea45cSKlaus Jensen }
410688eea45cSKlaus Jensen zd_ext = nvme_get_zd_extension(ns, zone_idx);
410788eea45cSKlaus Jensen status = nvme_h2c(n, zd_ext, ns->params.zd_extension_size, req);
410888eea45cSKlaus Jensen if (status) {
410988eea45cSKlaus Jensen trace_pci_nvme_err_zd_extension_map_error(zone_idx);
411088eea45cSKlaus Jensen return status;
411188eea45cSKlaus Jensen }
411288eea45cSKlaus Jensen
411388eea45cSKlaus Jensen status = nvme_set_zd_ext(ns, zone);
411488eea45cSKlaus Jensen if (status == NVME_SUCCESS) {
411588eea45cSKlaus Jensen trace_pci_nvme_zd_extension_set(zone_idx);
411688eea45cSKlaus Jensen return status;
411788eea45cSKlaus Jensen }
411888eea45cSKlaus Jensen break;
411988eea45cSKlaus Jensen
4120e321b4cdSKlaus Jensen case NVME_ZONE_ACTION_ZRWA_FLUSH:
4121e321b4cdSKlaus Jensen if (all) {
4122e321b4cdSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
4123e321b4cdSKlaus Jensen }
4124e321b4cdSKlaus Jensen
4125e321b4cdSKlaus Jensen return nvme_zone_mgmt_send_zrwa_flush(n, zone, slba, req);
4126e321b4cdSKlaus Jensen
412788eea45cSKlaus Jensen default:
412888eea45cSKlaus Jensen trace_pci_nvme_err_invalid_mgmt_action(action);
412988eea45cSKlaus Jensen status = NVME_INVALID_FIELD;
413088eea45cSKlaus Jensen }
413188eea45cSKlaus Jensen
413288eea45cSKlaus Jensen if (status == NVME_ZONE_INVAL_TRANSITION) {
413388eea45cSKlaus Jensen trace_pci_nvme_err_invalid_zone_state_transition(action, slba,
413488eea45cSKlaus Jensen zone->d.za);
413588eea45cSKlaus Jensen }
413688eea45cSKlaus Jensen if (status) {
413788eea45cSKlaus Jensen status |= NVME_DNR;
413888eea45cSKlaus Jensen }
413988eea45cSKlaus Jensen
414088eea45cSKlaus Jensen return status;
414188eea45cSKlaus Jensen }
414288eea45cSKlaus Jensen
nvme_zone_matches_filter(uint32_t zafs,NvmeZone * zl)414388eea45cSKlaus Jensen static bool nvme_zone_matches_filter(uint32_t zafs, NvmeZone *zl)
414488eea45cSKlaus Jensen {
414588eea45cSKlaus Jensen NvmeZoneState zs = nvme_get_zone_state(zl);
414688eea45cSKlaus Jensen
414788eea45cSKlaus Jensen switch (zafs) {
414888eea45cSKlaus Jensen case NVME_ZONE_REPORT_ALL:
414988eea45cSKlaus Jensen return true;
415088eea45cSKlaus Jensen case NVME_ZONE_REPORT_EMPTY:
415188eea45cSKlaus Jensen return zs == NVME_ZONE_STATE_EMPTY;
415288eea45cSKlaus Jensen case NVME_ZONE_REPORT_IMPLICITLY_OPEN:
415388eea45cSKlaus Jensen return zs == NVME_ZONE_STATE_IMPLICITLY_OPEN;
415488eea45cSKlaus Jensen case NVME_ZONE_REPORT_EXPLICITLY_OPEN:
415588eea45cSKlaus Jensen return zs == NVME_ZONE_STATE_EXPLICITLY_OPEN;
415688eea45cSKlaus Jensen case NVME_ZONE_REPORT_CLOSED:
415788eea45cSKlaus Jensen return zs == NVME_ZONE_STATE_CLOSED;
415888eea45cSKlaus Jensen case NVME_ZONE_REPORT_FULL:
415988eea45cSKlaus Jensen return zs == NVME_ZONE_STATE_FULL;
416088eea45cSKlaus Jensen case NVME_ZONE_REPORT_READ_ONLY:
416188eea45cSKlaus Jensen return zs == NVME_ZONE_STATE_READ_ONLY;
416288eea45cSKlaus Jensen case NVME_ZONE_REPORT_OFFLINE:
416388eea45cSKlaus Jensen return zs == NVME_ZONE_STATE_OFFLINE;
416488eea45cSKlaus Jensen default:
416588eea45cSKlaus Jensen return false;
416688eea45cSKlaus Jensen }
416788eea45cSKlaus Jensen }
416888eea45cSKlaus Jensen
nvme_zone_mgmt_recv(NvmeCtrl * n,NvmeRequest * req)416988eea45cSKlaus Jensen static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
417088eea45cSKlaus Jensen {
417188eea45cSKlaus Jensen NvmeCmd *cmd = (NvmeCmd *)&req->cmd;
417288eea45cSKlaus Jensen NvmeNamespace *ns = req->ns;
417388eea45cSKlaus Jensen /* cdw12 is zero-based number of dwords to return. Convert to bytes */
417488eea45cSKlaus Jensen uint32_t data_size = (le32_to_cpu(cmd->cdw12) + 1) << 2;
417588eea45cSKlaus Jensen uint32_t dw13 = le32_to_cpu(cmd->cdw13);
417688eea45cSKlaus Jensen uint32_t zone_idx, zra, zrasf, partial;
417788eea45cSKlaus Jensen uint64_t max_zones, nr_zones = 0;
417888eea45cSKlaus Jensen uint16_t status;
417988eea45cSKlaus Jensen uint64_t slba;
418088eea45cSKlaus Jensen NvmeZoneDescr *z;
418188eea45cSKlaus Jensen NvmeZone *zone;
418288eea45cSKlaus Jensen NvmeZoneReportHeader *header;
418388eea45cSKlaus Jensen void *buf, *buf_p;
418488eea45cSKlaus Jensen size_t zone_entry_sz;
418588eea45cSKlaus Jensen int i;
418688eea45cSKlaus Jensen
418788eea45cSKlaus Jensen req->status = NVME_SUCCESS;
418888eea45cSKlaus Jensen
418988eea45cSKlaus Jensen status = nvme_get_mgmt_zone_slba_idx(ns, cmd, &slba, &zone_idx);
419088eea45cSKlaus Jensen if (status) {
419188eea45cSKlaus Jensen return status;
419288eea45cSKlaus Jensen }
419388eea45cSKlaus Jensen
419488eea45cSKlaus Jensen zra = dw13 & 0xff;
419588eea45cSKlaus Jensen if (zra != NVME_ZONE_REPORT && zra != NVME_ZONE_REPORT_EXTENDED) {
419688eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
419788eea45cSKlaus Jensen }
419888eea45cSKlaus Jensen if (zra == NVME_ZONE_REPORT_EXTENDED && !ns->params.zd_extension_size) {
419988eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
420088eea45cSKlaus Jensen }
420188eea45cSKlaus Jensen
420288eea45cSKlaus Jensen zrasf = (dw13 >> 8) & 0xff;
420388eea45cSKlaus Jensen if (zrasf > NVME_ZONE_REPORT_OFFLINE) {
420488eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
420588eea45cSKlaus Jensen }
420688eea45cSKlaus Jensen
420788eea45cSKlaus Jensen if (data_size < sizeof(NvmeZoneReportHeader)) {
420888eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
420988eea45cSKlaus Jensen }
421088eea45cSKlaus Jensen
421188eea45cSKlaus Jensen status = nvme_check_mdts(n, data_size);
421288eea45cSKlaus Jensen if (status) {
421388eea45cSKlaus Jensen return status;
421488eea45cSKlaus Jensen }
421588eea45cSKlaus Jensen
421688eea45cSKlaus Jensen partial = (dw13 >> 16) & 0x01;
421788eea45cSKlaus Jensen
421888eea45cSKlaus Jensen zone_entry_sz = sizeof(NvmeZoneDescr);
421988eea45cSKlaus Jensen if (zra == NVME_ZONE_REPORT_EXTENDED) {
422088eea45cSKlaus Jensen zone_entry_sz += ns->params.zd_extension_size;
422188eea45cSKlaus Jensen }
422288eea45cSKlaus Jensen
422388eea45cSKlaus Jensen max_zones = (data_size - sizeof(NvmeZoneReportHeader)) / zone_entry_sz;
422488eea45cSKlaus Jensen buf = g_malloc0(data_size);
422588eea45cSKlaus Jensen
422688eea45cSKlaus Jensen zone = &ns->zone_array[zone_idx];
422788eea45cSKlaus Jensen for (i = zone_idx; i < ns->num_zones; i++) {
422888eea45cSKlaus Jensen if (partial && nr_zones >= max_zones) {
422988eea45cSKlaus Jensen break;
423088eea45cSKlaus Jensen }
423188eea45cSKlaus Jensen if (nvme_zone_matches_filter(zrasf, zone++)) {
423288eea45cSKlaus Jensen nr_zones++;
423388eea45cSKlaus Jensen }
423488eea45cSKlaus Jensen }
42353d558330SMarkus Armbruster header = buf;
423688eea45cSKlaus Jensen header->nr_zones = cpu_to_le64(nr_zones);
423788eea45cSKlaus Jensen
423888eea45cSKlaus Jensen buf_p = buf + sizeof(NvmeZoneReportHeader);
423988eea45cSKlaus Jensen for (; zone_idx < ns->num_zones && max_zones > 0; zone_idx++) {
424088eea45cSKlaus Jensen zone = &ns->zone_array[zone_idx];
424188eea45cSKlaus Jensen if (nvme_zone_matches_filter(zrasf, zone)) {
42423d558330SMarkus Armbruster z = buf_p;
424388eea45cSKlaus Jensen buf_p += sizeof(NvmeZoneDescr);
424488eea45cSKlaus Jensen
424588eea45cSKlaus Jensen z->zt = zone->d.zt;
424688eea45cSKlaus Jensen z->zs = zone->d.zs;
424788eea45cSKlaus Jensen z->zcap = cpu_to_le64(zone->d.zcap);
424888eea45cSKlaus Jensen z->zslba = cpu_to_le64(zone->d.zslba);
424988eea45cSKlaus Jensen z->za = zone->d.za;
425088eea45cSKlaus Jensen
425188eea45cSKlaus Jensen if (nvme_wp_is_valid(zone)) {
425288eea45cSKlaus Jensen z->wp = cpu_to_le64(zone->d.wp);
425388eea45cSKlaus Jensen } else {
425488eea45cSKlaus Jensen z->wp = cpu_to_le64(~0ULL);
425588eea45cSKlaus Jensen }
425688eea45cSKlaus Jensen
425788eea45cSKlaus Jensen if (zra == NVME_ZONE_REPORT_EXTENDED) {
425888eea45cSKlaus Jensen if (zone->d.za & NVME_ZA_ZD_EXT_VALID) {
425988eea45cSKlaus Jensen memcpy(buf_p, nvme_get_zd_extension(ns, zone_idx),
426088eea45cSKlaus Jensen ns->params.zd_extension_size);
426188eea45cSKlaus Jensen }
426288eea45cSKlaus Jensen buf_p += ns->params.zd_extension_size;
426388eea45cSKlaus Jensen }
426488eea45cSKlaus Jensen
426588eea45cSKlaus Jensen max_zones--;
426688eea45cSKlaus Jensen }
426788eea45cSKlaus Jensen }
426888eea45cSKlaus Jensen
426988eea45cSKlaus Jensen status = nvme_c2h(n, (uint8_t *)buf, data_size, req);
427088eea45cSKlaus Jensen
427188eea45cSKlaus Jensen g_free(buf);
427288eea45cSKlaus Jensen
427388eea45cSKlaus Jensen return status;
427488eea45cSKlaus Jensen }
427588eea45cSKlaus Jensen
nvme_io_mgmt_recv_ruhs(NvmeCtrl * n,NvmeRequest * req,size_t len)427673064edfSJesper Devantier static uint16_t nvme_io_mgmt_recv_ruhs(NvmeCtrl *n, NvmeRequest *req,
427773064edfSJesper Devantier size_t len)
427873064edfSJesper Devantier {
427973064edfSJesper Devantier NvmeNamespace *ns = req->ns;
428073064edfSJesper Devantier NvmeEnduranceGroup *endgrp;
428173064edfSJesper Devantier NvmeRuhStatus *hdr;
428273064edfSJesper Devantier NvmeRuhStatusDescr *ruhsd;
428373064edfSJesper Devantier unsigned int nruhsd;
428473064edfSJesper Devantier uint16_t rg, ph, *ruhid;
428573064edfSJesper Devantier size_t trans_len;
428673064edfSJesper Devantier g_autofree uint8_t *buf = NULL;
428773064edfSJesper Devantier
428873064edfSJesper Devantier if (!n->subsys) {
428973064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
429073064edfSJesper Devantier }
429173064edfSJesper Devantier
429273064edfSJesper Devantier if (ns->params.nsid == 0 || ns->params.nsid == 0xffffffff) {
429373064edfSJesper Devantier return NVME_INVALID_NSID | NVME_DNR;
429473064edfSJesper Devantier }
429573064edfSJesper Devantier
429673064edfSJesper Devantier if (!n->subsys->endgrp.fdp.enabled) {
429773064edfSJesper Devantier return NVME_FDP_DISABLED | NVME_DNR;
429873064edfSJesper Devantier }
429973064edfSJesper Devantier
430073064edfSJesper Devantier endgrp = ns->endgrp;
430173064edfSJesper Devantier
430273064edfSJesper Devantier nruhsd = ns->fdp.nphs * endgrp->fdp.nrg;
430373064edfSJesper Devantier trans_len = sizeof(NvmeRuhStatus) + nruhsd * sizeof(NvmeRuhStatusDescr);
430473064edfSJesper Devantier buf = g_malloc(trans_len);
430573064edfSJesper Devantier
430673064edfSJesper Devantier trans_len = MIN(trans_len, len);
430773064edfSJesper Devantier
430873064edfSJesper Devantier hdr = (NvmeRuhStatus *)buf;
430973064edfSJesper Devantier ruhsd = (NvmeRuhStatusDescr *)(buf + sizeof(NvmeRuhStatus));
431073064edfSJesper Devantier
431173064edfSJesper Devantier hdr->nruhsd = cpu_to_le16(nruhsd);
431273064edfSJesper Devantier
431373064edfSJesper Devantier ruhid = ns->fdp.phs;
431473064edfSJesper Devantier
431573064edfSJesper Devantier for (ph = 0; ph < ns->fdp.nphs; ph++, ruhid++) {
431673064edfSJesper Devantier NvmeRuHandle *ruh = &endgrp->fdp.ruhs[*ruhid];
431773064edfSJesper Devantier
431873064edfSJesper Devantier for (rg = 0; rg < endgrp->fdp.nrg; rg++, ruhsd++) {
431973064edfSJesper Devantier uint16_t pid = nvme_make_pid(ns, rg, ph);
432073064edfSJesper Devantier
432173064edfSJesper Devantier ruhsd->pid = cpu_to_le16(pid);
432273064edfSJesper Devantier ruhsd->ruhid = *ruhid;
432373064edfSJesper Devantier ruhsd->earutr = 0;
432473064edfSJesper Devantier ruhsd->ruamw = cpu_to_le64(ruh->rus[rg].ruamw);
432573064edfSJesper Devantier }
432673064edfSJesper Devantier }
432773064edfSJesper Devantier
432873064edfSJesper Devantier return nvme_c2h(n, buf, trans_len, req);
432973064edfSJesper Devantier }
433073064edfSJesper Devantier
nvme_io_mgmt_recv(NvmeCtrl * n,NvmeRequest * req)433173064edfSJesper Devantier static uint16_t nvme_io_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
433273064edfSJesper Devantier {
433373064edfSJesper Devantier NvmeCmd *cmd = &req->cmd;
433473064edfSJesper Devantier uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
433573064edfSJesper Devantier uint32_t numd = le32_to_cpu(cmd->cdw11);
433673064edfSJesper Devantier uint8_t mo = (cdw10 & 0xff);
433773064edfSJesper Devantier size_t len = (numd + 1) << 2;
433873064edfSJesper Devantier
433973064edfSJesper Devantier switch (mo) {
434073064edfSJesper Devantier case NVME_IOMR_MO_NOP:
434173064edfSJesper Devantier return 0;
434273064edfSJesper Devantier case NVME_IOMR_MO_RUH_STATUS:
434373064edfSJesper Devantier return nvme_io_mgmt_recv_ruhs(n, req, len);
434473064edfSJesper Devantier default:
434573064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
434673064edfSJesper Devantier };
434773064edfSJesper Devantier }
434873064edfSJesper Devantier
nvme_io_mgmt_send_ruh_update(NvmeCtrl * n,NvmeRequest * req)434973064edfSJesper Devantier static uint16_t nvme_io_mgmt_send_ruh_update(NvmeCtrl *n, NvmeRequest *req)
435073064edfSJesper Devantier {
435173064edfSJesper Devantier NvmeCmd *cmd = &req->cmd;
435273064edfSJesper Devantier NvmeNamespace *ns = req->ns;
435373064edfSJesper Devantier uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
435473064edfSJesper Devantier uint16_t ret = NVME_SUCCESS;
435573064edfSJesper Devantier uint32_t npid = (cdw10 >> 1) + 1;
435673064edfSJesper Devantier unsigned int i = 0;
435773064edfSJesper Devantier g_autofree uint16_t *pids = NULL;
43583439ba9cSKlaus Jensen uint32_t maxnpid;
43593439ba9cSKlaus Jensen
43603439ba9cSKlaus Jensen if (!ns->endgrp || !ns->endgrp->fdp.enabled) {
43613439ba9cSKlaus Jensen return NVME_FDP_DISABLED | NVME_DNR;
43623439ba9cSKlaus Jensen }
43633439ba9cSKlaus Jensen
43643439ba9cSKlaus Jensen maxnpid = n->subsys->endgrp.fdp.nrg * n->subsys->endgrp.fdp.nruh;
436573064edfSJesper Devantier
436673064edfSJesper Devantier if (unlikely(npid >= MIN(NVME_FDP_MAXPIDS, maxnpid))) {
436773064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
436873064edfSJesper Devantier }
436973064edfSJesper Devantier
437073064edfSJesper Devantier pids = g_new(uint16_t, npid);
437173064edfSJesper Devantier
437273064edfSJesper Devantier ret = nvme_h2c(n, pids, npid * sizeof(uint16_t), req);
437373064edfSJesper Devantier if (ret) {
437473064edfSJesper Devantier return ret;
437573064edfSJesper Devantier }
437673064edfSJesper Devantier
437773064edfSJesper Devantier for (; i < npid; i++) {
437873064edfSJesper Devantier if (!nvme_update_ruh(n, ns, pids[i])) {
437973064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
438073064edfSJesper Devantier }
438173064edfSJesper Devantier }
438273064edfSJesper Devantier
438373064edfSJesper Devantier return ret;
438473064edfSJesper Devantier }
438573064edfSJesper Devantier
nvme_io_mgmt_send(NvmeCtrl * n,NvmeRequest * req)438673064edfSJesper Devantier static uint16_t nvme_io_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
438773064edfSJesper Devantier {
438873064edfSJesper Devantier NvmeCmd *cmd = &req->cmd;
438973064edfSJesper Devantier uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
439073064edfSJesper Devantier uint8_t mo = (cdw10 & 0xff);
439173064edfSJesper Devantier
439273064edfSJesper Devantier switch (mo) {
439373064edfSJesper Devantier case NVME_IOMS_MO_NOP:
439473064edfSJesper Devantier return 0;
439573064edfSJesper Devantier case NVME_IOMS_MO_RUH_UPDATE:
439673064edfSJesper Devantier return nvme_io_mgmt_send_ruh_update(n, req);
439773064edfSJesper Devantier default:
439873064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
439973064edfSJesper Devantier };
440073064edfSJesper Devantier }
440173064edfSJesper Devantier
nvme_io_cmd(NvmeCtrl * n,NvmeRequest * req)440288eea45cSKlaus Jensen static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req)
440388eea45cSKlaus Jensen {
440488eea45cSKlaus Jensen NvmeNamespace *ns;
440588eea45cSKlaus Jensen uint32_t nsid = le32_to_cpu(req->cmd.nsid);
440688eea45cSKlaus Jensen
440788eea45cSKlaus Jensen trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req),
440888eea45cSKlaus Jensen req->cmd.opcode, nvme_io_opc_str(req->cmd.opcode));
440988eea45cSKlaus Jensen
441088eea45cSKlaus Jensen if (!nvme_nsid_valid(n, nsid)) {
441188eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
441288eea45cSKlaus Jensen }
441388eea45cSKlaus Jensen
441488eea45cSKlaus Jensen /*
441588eea45cSKlaus Jensen * In the base NVM command set, Flush may apply to all namespaces
441688eea45cSKlaus Jensen * (indicated by NSID being set to FFFFFFFFh). But if that feature is used
441788eea45cSKlaus Jensen * along with TP 4056 (Namespace Types), it may be pretty screwed up.
441888eea45cSKlaus Jensen *
441988eea45cSKlaus Jensen * If NSID is indeed set to FFFFFFFFh, we simply cannot associate the
442088eea45cSKlaus Jensen * opcode with a specific command since we cannot determine a unique I/O
442188eea45cSKlaus Jensen * command set. Opcode 0h could have any other meaning than something
442288eea45cSKlaus Jensen * equivalent to flushing and say it DOES have completely different
442388eea45cSKlaus Jensen * semantics in some other command set - does an NSID of FFFFFFFFh then
442488eea45cSKlaus Jensen * mean "for all namespaces, apply whatever command set specific command
442588eea45cSKlaus Jensen * that uses the 0h opcode?" Or does it mean "for all namespaces, apply
442688eea45cSKlaus Jensen * whatever command that uses the 0h opcode if, and only if, it allows NSID
442788eea45cSKlaus Jensen * to be FFFFFFFFh"?
442888eea45cSKlaus Jensen *
442988eea45cSKlaus Jensen * Anyway (and luckily), for now, we do not care about this since the
443088eea45cSKlaus Jensen * device only supports namespace types that includes the NVM Flush command
443188eea45cSKlaus Jensen * (NVM and Zoned), so always do an NVM Flush.
443288eea45cSKlaus Jensen */
443388eea45cSKlaus Jensen if (req->cmd.opcode == NVME_CMD_FLUSH) {
443488eea45cSKlaus Jensen return nvme_flush(n, req);
443588eea45cSKlaus Jensen }
443688eea45cSKlaus Jensen
443788eea45cSKlaus Jensen ns = nvme_ns(n, nsid);
443888eea45cSKlaus Jensen if (unlikely(!ns)) {
443988eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
444088eea45cSKlaus Jensen }
444188eea45cSKlaus Jensen
444288eea45cSKlaus Jensen if (!(ns->iocs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
444388eea45cSKlaus Jensen trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
444488eea45cSKlaus Jensen return NVME_INVALID_OPCODE | NVME_DNR;
444588eea45cSKlaus Jensen }
444688eea45cSKlaus Jensen
444788eea45cSKlaus Jensen if (ns->status) {
444888eea45cSKlaus Jensen return ns->status;
444988eea45cSKlaus Jensen }
445088eea45cSKlaus Jensen
4451c53a9a91SPankaj Raghav if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
4452c53a9a91SPankaj Raghav return NVME_INVALID_FIELD;
4453c53a9a91SPankaj Raghav }
4454c53a9a91SPankaj Raghav
445588eea45cSKlaus Jensen req->ns = ns;
445688eea45cSKlaus Jensen
445788eea45cSKlaus Jensen switch (req->cmd.opcode) {
445888eea45cSKlaus Jensen case NVME_CMD_WRITE_ZEROES:
445988eea45cSKlaus Jensen return nvme_write_zeroes(n, req);
446088eea45cSKlaus Jensen case NVME_CMD_ZONE_APPEND:
446188eea45cSKlaus Jensen return nvme_zone_append(n, req);
446288eea45cSKlaus Jensen case NVME_CMD_WRITE:
446388eea45cSKlaus Jensen return nvme_write(n, req);
446488eea45cSKlaus Jensen case NVME_CMD_READ:
446588eea45cSKlaus Jensen return nvme_read(n, req);
446688eea45cSKlaus Jensen case NVME_CMD_COMPARE:
446788eea45cSKlaus Jensen return nvme_compare(n, req);
446888eea45cSKlaus Jensen case NVME_CMD_DSM:
446988eea45cSKlaus Jensen return nvme_dsm(n, req);
447088eea45cSKlaus Jensen case NVME_CMD_VERIFY:
447188eea45cSKlaus Jensen return nvme_verify(n, req);
447288eea45cSKlaus Jensen case NVME_CMD_COPY:
447388eea45cSKlaus Jensen return nvme_copy(n, req);
447488eea45cSKlaus Jensen case NVME_CMD_ZONE_MGMT_SEND:
447588eea45cSKlaus Jensen return nvme_zone_mgmt_send(n, req);
447688eea45cSKlaus Jensen case NVME_CMD_ZONE_MGMT_RECV:
447788eea45cSKlaus Jensen return nvme_zone_mgmt_recv(n, req);
447873064edfSJesper Devantier case NVME_CMD_IO_MGMT_RECV:
447973064edfSJesper Devantier return nvme_io_mgmt_recv(n, req);
448073064edfSJesper Devantier case NVME_CMD_IO_MGMT_SEND:
448173064edfSJesper Devantier return nvme_io_mgmt_send(n, req);
448288eea45cSKlaus Jensen default:
448388eea45cSKlaus Jensen assert(false);
448488eea45cSKlaus Jensen }
448588eea45cSKlaus Jensen
448688eea45cSKlaus Jensen return NVME_INVALID_OPCODE | NVME_DNR;
448788eea45cSKlaus Jensen }
448888eea45cSKlaus Jensen
nvme_cq_notifier(EventNotifier * e)44892e53b0b4SJinhao Fan static void nvme_cq_notifier(EventNotifier *e)
44902e53b0b4SJinhao Fan {
44912e53b0b4SJinhao Fan NvmeCQueue *cq = container_of(e, NvmeCQueue, notifier);
44922e53b0b4SJinhao Fan NvmeCtrl *n = cq->ctrl;
44932e53b0b4SJinhao Fan
4494a2da7377SKlaus Jensen if (!event_notifier_test_and_clear(e)) {
4495a2da7377SKlaus Jensen return;
4496a2da7377SKlaus Jensen }
44972e53b0b4SJinhao Fan
44982e53b0b4SJinhao Fan nvme_update_cq_head(cq);
44992e53b0b4SJinhao Fan
45002e53b0b4SJinhao Fan if (cq->tail == cq->head) {
45012e53b0b4SJinhao Fan if (cq->irq_enabled) {
45022e53b0b4SJinhao Fan n->cq_pending--;
45032e53b0b4SJinhao Fan }
45042e53b0b4SJinhao Fan
45052e53b0b4SJinhao Fan nvme_irq_deassert(n, cq);
45062e53b0b4SJinhao Fan }
45072e53b0b4SJinhao Fan
4508d38cc6fdSKlaus Jensen qemu_bh_schedule(cq->bh);
45092e53b0b4SJinhao Fan }
45102e53b0b4SJinhao Fan
nvme_init_cq_ioeventfd(NvmeCQueue * cq)45112e53b0b4SJinhao Fan static int nvme_init_cq_ioeventfd(NvmeCQueue *cq)
45122e53b0b4SJinhao Fan {
45132e53b0b4SJinhao Fan NvmeCtrl *n = cq->ctrl;
45142e53b0b4SJinhao Fan uint16_t offset = (cq->cqid << 3) + (1 << 2);
45152e53b0b4SJinhao Fan int ret;
45162e53b0b4SJinhao Fan
45172e53b0b4SJinhao Fan ret = event_notifier_init(&cq->notifier, 0);
45182e53b0b4SJinhao Fan if (ret < 0) {
45192e53b0b4SJinhao Fan return ret;
45202e53b0b4SJinhao Fan }
45212e53b0b4SJinhao Fan
45222e53b0b4SJinhao Fan event_notifier_set_handler(&cq->notifier, nvme_cq_notifier);
45232e53b0b4SJinhao Fan memory_region_add_eventfd(&n->iomem,
45242e53b0b4SJinhao Fan 0x1000 + offset, 4, false, 0, &cq->notifier);
45252e53b0b4SJinhao Fan
45262e53b0b4SJinhao Fan return 0;
45272e53b0b4SJinhao Fan }
45282e53b0b4SJinhao Fan
nvme_sq_notifier(EventNotifier * e)45292e53b0b4SJinhao Fan static void nvme_sq_notifier(EventNotifier *e)
45302e53b0b4SJinhao Fan {
45312e53b0b4SJinhao Fan NvmeSQueue *sq = container_of(e, NvmeSQueue, notifier);
45322e53b0b4SJinhao Fan
4533a2da7377SKlaus Jensen if (!event_notifier_test_and_clear(e)) {
4534a2da7377SKlaus Jensen return;
4535a2da7377SKlaus Jensen }
45362e53b0b4SJinhao Fan
45372e53b0b4SJinhao Fan nvme_process_sq(sq);
45382e53b0b4SJinhao Fan }
45392e53b0b4SJinhao Fan
nvme_init_sq_ioeventfd(NvmeSQueue * sq)45402e53b0b4SJinhao Fan static int nvme_init_sq_ioeventfd(NvmeSQueue *sq)
45412e53b0b4SJinhao Fan {
45422e53b0b4SJinhao Fan NvmeCtrl *n = sq->ctrl;
45432e53b0b4SJinhao Fan uint16_t offset = sq->sqid << 3;
45442e53b0b4SJinhao Fan int ret;
45452e53b0b4SJinhao Fan
45462e53b0b4SJinhao Fan ret = event_notifier_init(&sq->notifier, 0);
45472e53b0b4SJinhao Fan if (ret < 0) {
45482e53b0b4SJinhao Fan return ret;
45492e53b0b4SJinhao Fan }
45502e53b0b4SJinhao Fan
45512e53b0b4SJinhao Fan event_notifier_set_handler(&sq->notifier, nvme_sq_notifier);
45522e53b0b4SJinhao Fan memory_region_add_eventfd(&n->iomem,
45532e53b0b4SJinhao Fan 0x1000 + offset, 4, false, 0, &sq->notifier);
45542e53b0b4SJinhao Fan
45552e53b0b4SJinhao Fan return 0;
45562e53b0b4SJinhao Fan }
45572e53b0b4SJinhao Fan
nvme_free_sq(NvmeSQueue * sq,NvmeCtrl * n)455888eea45cSKlaus Jensen static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
455988eea45cSKlaus Jensen {
45602e53b0b4SJinhao Fan uint16_t offset = sq->sqid << 3;
45612e53b0b4SJinhao Fan
456288eea45cSKlaus Jensen n->sq[sq->sqid] = NULL;
4563d38cc6fdSKlaus Jensen qemu_bh_delete(sq->bh);
45642e53b0b4SJinhao Fan if (sq->ioeventfd_enabled) {
45652e53b0b4SJinhao Fan memory_region_del_eventfd(&n->iomem,
45662e53b0b4SJinhao Fan 0x1000 + offset, 4, false, 0, &sq->notifier);
456704e8da88SKlaus Jensen event_notifier_set_handler(&sq->notifier, NULL);
45682e53b0b4SJinhao Fan event_notifier_cleanup(&sq->notifier);
45692e53b0b4SJinhao Fan }
457088eea45cSKlaus Jensen g_free(sq->io_req);
457188eea45cSKlaus Jensen if (sq->sqid) {
457288eea45cSKlaus Jensen g_free(sq);
457388eea45cSKlaus Jensen }
457488eea45cSKlaus Jensen }
457588eea45cSKlaus Jensen
nvme_del_sq(NvmeCtrl * n,NvmeRequest * req)457688eea45cSKlaus Jensen static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req)
457788eea45cSKlaus Jensen {
457888eea45cSKlaus Jensen NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
457988eea45cSKlaus Jensen NvmeRequest *r, *next;
458088eea45cSKlaus Jensen NvmeSQueue *sq;
458188eea45cSKlaus Jensen NvmeCQueue *cq;
458288eea45cSKlaus Jensen uint16_t qid = le16_to_cpu(c->qid);
458388eea45cSKlaus Jensen
458488eea45cSKlaus Jensen if (unlikely(!qid || nvme_check_sqid(n, qid))) {
458588eea45cSKlaus Jensen trace_pci_nvme_err_invalid_del_sq(qid);
458688eea45cSKlaus Jensen return NVME_INVALID_QID | NVME_DNR;
458788eea45cSKlaus Jensen }
458888eea45cSKlaus Jensen
458988eea45cSKlaus Jensen trace_pci_nvme_del_sq(qid);
459088eea45cSKlaus Jensen
459188eea45cSKlaus Jensen sq = n->sq[qid];
459288eea45cSKlaus Jensen while (!QTAILQ_EMPTY(&sq->out_req_list)) {
459388eea45cSKlaus Jensen r = QTAILQ_FIRST(&sq->out_req_list);
4594e76fb260SKlaus Jensen assert(r->aiocb);
459588eea45cSKlaus Jensen blk_aio_cancel(r->aiocb);
459688eea45cSKlaus Jensen }
459788eea45cSKlaus Jensen
459888eea45cSKlaus Jensen assert(QTAILQ_EMPTY(&sq->out_req_list));
459988eea45cSKlaus Jensen
460088eea45cSKlaus Jensen if (!nvme_check_cqid(n, sq->cqid)) {
460188eea45cSKlaus Jensen cq = n->cq[sq->cqid];
460288eea45cSKlaus Jensen QTAILQ_REMOVE(&cq->sq_list, sq, entry);
460388eea45cSKlaus Jensen
460488eea45cSKlaus Jensen nvme_post_cqes(cq);
460588eea45cSKlaus Jensen QTAILQ_FOREACH_SAFE(r, &cq->req_list, entry, next) {
460688eea45cSKlaus Jensen if (r->sq == sq) {
460788eea45cSKlaus Jensen QTAILQ_REMOVE(&cq->req_list, r, entry);
460888eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&sq->req_list, r, entry);
460988eea45cSKlaus Jensen }
461088eea45cSKlaus Jensen }
461188eea45cSKlaus Jensen }
461288eea45cSKlaus Jensen
461388eea45cSKlaus Jensen nvme_free_sq(sq, n);
461488eea45cSKlaus Jensen return NVME_SUCCESS;
461588eea45cSKlaus Jensen }
461688eea45cSKlaus Jensen
nvme_init_sq(NvmeSQueue * sq,NvmeCtrl * n,uint64_t dma_addr,uint16_t sqid,uint16_t cqid,uint16_t size)461788eea45cSKlaus Jensen static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
461888eea45cSKlaus Jensen uint16_t sqid, uint16_t cqid, uint16_t size)
461988eea45cSKlaus Jensen {
462088eea45cSKlaus Jensen int i;
462188eea45cSKlaus Jensen NvmeCQueue *cq;
462288eea45cSKlaus Jensen
462388eea45cSKlaus Jensen sq->ctrl = n;
462488eea45cSKlaus Jensen sq->dma_addr = dma_addr;
462588eea45cSKlaus Jensen sq->sqid = sqid;
462688eea45cSKlaus Jensen sq->size = size;
462788eea45cSKlaus Jensen sq->cqid = cqid;
462888eea45cSKlaus Jensen sq->head = sq->tail = 0;
462988eea45cSKlaus Jensen sq->io_req = g_new0(NvmeRequest, sq->size);
463088eea45cSKlaus Jensen
463188eea45cSKlaus Jensen QTAILQ_INIT(&sq->req_list);
463288eea45cSKlaus Jensen QTAILQ_INIT(&sq->out_req_list);
463388eea45cSKlaus Jensen for (i = 0; i < sq->size; i++) {
463488eea45cSKlaus Jensen sq->io_req[i].sq = sq;
463588eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
463688eea45cSKlaus Jensen }
4637d38cc6fdSKlaus Jensen
4638f63192b0SAlexander Bulekov sq->bh = qemu_bh_new_guarded(nvme_process_sq, sq,
4639f63192b0SAlexander Bulekov &DEVICE(sq->ctrl)->mem_reentrancy_guard);
464088eea45cSKlaus Jensen
46413f7fe8deSJinhao Fan if (n->dbbuf_enabled) {
46423f7fe8deSJinhao Fan sq->db_addr = n->dbbuf_dbs + (sqid << 3);
46433f7fe8deSJinhao Fan sq->ei_addr = n->dbbuf_eis + (sqid << 3);
46442e53b0b4SJinhao Fan
46452e53b0b4SJinhao Fan if (n->params.ioeventfd && sq->sqid != 0) {
46462e53b0b4SJinhao Fan if (!nvme_init_sq_ioeventfd(sq)) {
46472e53b0b4SJinhao Fan sq->ioeventfd_enabled = true;
46482e53b0b4SJinhao Fan }
46492e53b0b4SJinhao Fan }
46503f7fe8deSJinhao Fan }
46513f7fe8deSJinhao Fan
465288eea45cSKlaus Jensen assert(n->cq[cqid]);
465388eea45cSKlaus Jensen cq = n->cq[cqid];
465488eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
465588eea45cSKlaus Jensen n->sq[sqid] = sq;
465688eea45cSKlaus Jensen }
465788eea45cSKlaus Jensen
nvme_create_sq(NvmeCtrl * n,NvmeRequest * req)465888eea45cSKlaus Jensen static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req)
465988eea45cSKlaus Jensen {
466088eea45cSKlaus Jensen NvmeSQueue *sq;
466188eea45cSKlaus Jensen NvmeCreateSq *c = (NvmeCreateSq *)&req->cmd;
466288eea45cSKlaus Jensen
466388eea45cSKlaus Jensen uint16_t cqid = le16_to_cpu(c->cqid);
466488eea45cSKlaus Jensen uint16_t sqid = le16_to_cpu(c->sqid);
466588eea45cSKlaus Jensen uint16_t qsize = le16_to_cpu(c->qsize);
466688eea45cSKlaus Jensen uint16_t qflags = le16_to_cpu(c->sq_flags);
466788eea45cSKlaus Jensen uint64_t prp1 = le64_to_cpu(c->prp1);
466888eea45cSKlaus Jensen
466988eea45cSKlaus Jensen trace_pci_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
467088eea45cSKlaus Jensen
467188eea45cSKlaus Jensen if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
467288eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_sq_cqid(cqid);
467388eea45cSKlaus Jensen return NVME_INVALID_CQID | NVME_DNR;
467488eea45cSKlaus Jensen }
4675decc0261SŁukasz Gieryk if (unlikely(!sqid || sqid > n->conf_ioqpairs || n->sq[sqid] != NULL)) {
467688eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_sq_sqid(sqid);
467788eea45cSKlaus Jensen return NVME_INVALID_QID | NVME_DNR;
467888eea45cSKlaus Jensen }
467949e03457SKlaus Jensen if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
468088eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_sq_size(qsize);
468188eea45cSKlaus Jensen return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
468288eea45cSKlaus Jensen }
468388eea45cSKlaus Jensen if (unlikely(prp1 & (n->page_size - 1))) {
468488eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_sq_addr(prp1);
468588eea45cSKlaus Jensen return NVME_INVALID_PRP_OFFSET | NVME_DNR;
468688eea45cSKlaus Jensen }
468788eea45cSKlaus Jensen if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
468888eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
468988eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
469088eea45cSKlaus Jensen }
469188eea45cSKlaus Jensen sq = g_malloc0(sizeof(*sq));
469288eea45cSKlaus Jensen nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
469388eea45cSKlaus Jensen return NVME_SUCCESS;
469488eea45cSKlaus Jensen }
469588eea45cSKlaus Jensen
469688eea45cSKlaus Jensen struct nvme_stats {
469788eea45cSKlaus Jensen uint64_t units_read;
469888eea45cSKlaus Jensen uint64_t units_written;
469988eea45cSKlaus Jensen uint64_t read_commands;
470088eea45cSKlaus Jensen uint64_t write_commands;
470188eea45cSKlaus Jensen };
470288eea45cSKlaus Jensen
nvme_set_blk_stats(NvmeNamespace * ns,struct nvme_stats * stats)470388eea45cSKlaus Jensen static void nvme_set_blk_stats(NvmeNamespace *ns, struct nvme_stats *stats)
470488eea45cSKlaus Jensen {
470588eea45cSKlaus Jensen BlockAcctStats *s = blk_get_stats(ns->blkconf.blk);
470688eea45cSKlaus Jensen
4707a555af17SJoel Granados stats->units_read += s->nr_bytes[BLOCK_ACCT_READ];
4708a555af17SJoel Granados stats->units_written += s->nr_bytes[BLOCK_ACCT_WRITE];
470988eea45cSKlaus Jensen stats->read_commands += s->nr_ops[BLOCK_ACCT_READ];
471088eea45cSKlaus Jensen stats->write_commands += s->nr_ops[BLOCK_ACCT_WRITE];
471188eea45cSKlaus Jensen }
471288eea45cSKlaus Jensen
nvme_smart_info(NvmeCtrl * n,uint8_t rae,uint32_t buf_len,uint64_t off,NvmeRequest * req)471388eea45cSKlaus Jensen static uint16_t nvme_smart_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
471488eea45cSKlaus Jensen uint64_t off, NvmeRequest *req)
471588eea45cSKlaus Jensen {
471688eea45cSKlaus Jensen uint32_t nsid = le32_to_cpu(req->cmd.nsid);
471788eea45cSKlaus Jensen struct nvme_stats stats = { 0 };
471888eea45cSKlaus Jensen NvmeSmartLog smart = { 0 };
471988eea45cSKlaus Jensen uint32_t trans_len;
472088eea45cSKlaus Jensen NvmeNamespace *ns;
472188eea45cSKlaus Jensen time_t current_ms;
4722a555af17SJoel Granados uint64_t u_read, u_written;
472388eea45cSKlaus Jensen
472488eea45cSKlaus Jensen if (off >= sizeof(smart)) {
472588eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
472688eea45cSKlaus Jensen }
472788eea45cSKlaus Jensen
472888eea45cSKlaus Jensen if (nsid != 0xffffffff) {
472988eea45cSKlaus Jensen ns = nvme_ns(n, nsid);
473088eea45cSKlaus Jensen if (!ns) {
473188eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
473288eea45cSKlaus Jensen }
473388eea45cSKlaus Jensen nvme_set_blk_stats(ns, &stats);
473488eea45cSKlaus Jensen } else {
473588eea45cSKlaus Jensen int i;
473688eea45cSKlaus Jensen
473788eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
473888eea45cSKlaus Jensen ns = nvme_ns(n, i);
473988eea45cSKlaus Jensen if (!ns) {
474088eea45cSKlaus Jensen continue;
474188eea45cSKlaus Jensen }
474288eea45cSKlaus Jensen nvme_set_blk_stats(ns, &stats);
474388eea45cSKlaus Jensen }
474488eea45cSKlaus Jensen }
474588eea45cSKlaus Jensen
474688eea45cSKlaus Jensen trans_len = MIN(sizeof(smart) - off, buf_len);
474788eea45cSKlaus Jensen smart.critical_warning = n->smart_critical_warning;
474888eea45cSKlaus Jensen
4749a555af17SJoel Granados u_read = DIV_ROUND_UP(stats.units_read >> BDRV_SECTOR_BITS, 1000);
4750a555af17SJoel Granados u_written = DIV_ROUND_UP(stats.units_written >> BDRV_SECTOR_BITS, 1000);
4751a555af17SJoel Granados
4752a555af17SJoel Granados smart.data_units_read[0] = cpu_to_le64(u_read);
4753a555af17SJoel Granados smart.data_units_written[0] = cpu_to_le64(u_written);
475488eea45cSKlaus Jensen smart.host_read_commands[0] = cpu_to_le64(stats.read_commands);
475588eea45cSKlaus Jensen smart.host_write_commands[0] = cpu_to_le64(stats.write_commands);
475688eea45cSKlaus Jensen
475788eea45cSKlaus Jensen smart.temperature = cpu_to_le16(n->temperature);
475888eea45cSKlaus Jensen
475988eea45cSKlaus Jensen if ((n->temperature >= n->features.temp_thresh_hi) ||
476088eea45cSKlaus Jensen (n->temperature <= n->features.temp_thresh_low)) {
476188eea45cSKlaus Jensen smart.critical_warning |= NVME_SMART_TEMPERATURE;
476288eea45cSKlaus Jensen }
476388eea45cSKlaus Jensen
476488eea45cSKlaus Jensen current_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
476588eea45cSKlaus Jensen smart.power_on_hours[0] =
476688eea45cSKlaus Jensen cpu_to_le64((((current_ms - n->starttime_ms) / 1000) / 60) / 60);
476788eea45cSKlaus Jensen
476888eea45cSKlaus Jensen if (!rae) {
476988eea45cSKlaus Jensen nvme_clear_events(n, NVME_AER_TYPE_SMART);
477088eea45cSKlaus Jensen }
477188eea45cSKlaus Jensen
477288eea45cSKlaus Jensen return nvme_c2h(n, (uint8_t *) &smart + off, trans_len, req);
477388eea45cSKlaus Jensen }
477488eea45cSKlaus Jensen
nvme_endgrp_info(NvmeCtrl * n,uint8_t rae,uint32_t buf_len,uint64_t off,NvmeRequest * req)4775771dbc3aSKlaus Jensen static uint16_t nvme_endgrp_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
4776771dbc3aSKlaus Jensen uint64_t off, NvmeRequest *req)
4777771dbc3aSKlaus Jensen {
4778771dbc3aSKlaus Jensen uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
4779771dbc3aSKlaus Jensen uint16_t endgrpid = (dw11 >> 16) & 0xffff;
4780771dbc3aSKlaus Jensen struct nvme_stats stats = {};
4781771dbc3aSKlaus Jensen NvmeEndGrpLog info = {};
4782771dbc3aSKlaus Jensen int i;
4783771dbc3aSKlaus Jensen
4784771dbc3aSKlaus Jensen if (!n->subsys || endgrpid != 0x1) {
4785771dbc3aSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
4786771dbc3aSKlaus Jensen }
4787771dbc3aSKlaus Jensen
4788771dbc3aSKlaus Jensen if (off >= sizeof(info)) {
4789771dbc3aSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
4790771dbc3aSKlaus Jensen }
4791771dbc3aSKlaus Jensen
4792771dbc3aSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
4793771dbc3aSKlaus Jensen NvmeNamespace *ns = nvme_subsys_ns(n->subsys, i);
4794771dbc3aSKlaus Jensen if (!ns) {
4795771dbc3aSKlaus Jensen continue;
4796771dbc3aSKlaus Jensen }
4797771dbc3aSKlaus Jensen
4798771dbc3aSKlaus Jensen nvme_set_blk_stats(ns, &stats);
4799771dbc3aSKlaus Jensen }
4800771dbc3aSKlaus Jensen
4801771dbc3aSKlaus Jensen info.data_units_read[0] =
4802771dbc3aSKlaus Jensen cpu_to_le64(DIV_ROUND_UP(stats.units_read / 1000000000, 1000000000));
4803771dbc3aSKlaus Jensen info.data_units_written[0] =
4804771dbc3aSKlaus Jensen cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4805771dbc3aSKlaus Jensen info.media_units_written[0] =
4806771dbc3aSKlaus Jensen cpu_to_le64(DIV_ROUND_UP(stats.units_written / 1000000000, 1000000000));
4807771dbc3aSKlaus Jensen
4808771dbc3aSKlaus Jensen info.host_read_commands[0] = cpu_to_le64(stats.read_commands);
4809771dbc3aSKlaus Jensen info.host_write_commands[0] = cpu_to_le64(stats.write_commands);
4810771dbc3aSKlaus Jensen
4811771dbc3aSKlaus Jensen buf_len = MIN(sizeof(info) - off, buf_len);
4812771dbc3aSKlaus Jensen
4813771dbc3aSKlaus Jensen return nvme_c2h(n, (uint8_t *)&info + off, buf_len, req);
4814771dbc3aSKlaus Jensen }
4815771dbc3aSKlaus Jensen
4816771dbc3aSKlaus Jensen
nvme_fw_log_info(NvmeCtrl * n,uint32_t buf_len,uint64_t off,NvmeRequest * req)481788eea45cSKlaus Jensen static uint16_t nvme_fw_log_info(NvmeCtrl *n, uint32_t buf_len, uint64_t off,
481888eea45cSKlaus Jensen NvmeRequest *req)
481988eea45cSKlaus Jensen {
482088eea45cSKlaus Jensen uint32_t trans_len;
482188eea45cSKlaus Jensen NvmeFwSlotInfoLog fw_log = {
482288eea45cSKlaus Jensen .afi = 0x1,
482388eea45cSKlaus Jensen };
482488eea45cSKlaus Jensen
482588eea45cSKlaus Jensen if (off >= sizeof(fw_log)) {
482688eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
482788eea45cSKlaus Jensen }
482888eea45cSKlaus Jensen
482988eea45cSKlaus Jensen strpadcpy((char *)&fw_log.frs1, sizeof(fw_log.frs1), "1.0", ' ');
483088eea45cSKlaus Jensen trans_len = MIN(sizeof(fw_log) - off, buf_len);
483188eea45cSKlaus Jensen
483288eea45cSKlaus Jensen return nvme_c2h(n, (uint8_t *) &fw_log + off, trans_len, req);
483388eea45cSKlaus Jensen }
483488eea45cSKlaus Jensen
nvme_error_info(NvmeCtrl * n,uint8_t rae,uint32_t buf_len,uint64_t off,NvmeRequest * req)483588eea45cSKlaus Jensen static uint16_t nvme_error_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
483688eea45cSKlaus Jensen uint64_t off, NvmeRequest *req)
483788eea45cSKlaus Jensen {
483888eea45cSKlaus Jensen uint32_t trans_len;
483988eea45cSKlaus Jensen NvmeErrorLog errlog;
484088eea45cSKlaus Jensen
484188eea45cSKlaus Jensen if (off >= sizeof(errlog)) {
484288eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
484388eea45cSKlaus Jensen }
484488eea45cSKlaus Jensen
484588eea45cSKlaus Jensen if (!rae) {
484688eea45cSKlaus Jensen nvme_clear_events(n, NVME_AER_TYPE_ERROR);
484788eea45cSKlaus Jensen }
484888eea45cSKlaus Jensen
484988eea45cSKlaus Jensen memset(&errlog, 0x0, sizeof(errlog));
485088eea45cSKlaus Jensen trans_len = MIN(sizeof(errlog) - off, buf_len);
485188eea45cSKlaus Jensen
485288eea45cSKlaus Jensen return nvme_c2h(n, (uint8_t *)&errlog, trans_len, req);
485388eea45cSKlaus Jensen }
485488eea45cSKlaus Jensen
nvme_changed_nslist(NvmeCtrl * n,uint8_t rae,uint32_t buf_len,uint64_t off,NvmeRequest * req)485588eea45cSKlaus Jensen static uint16_t nvme_changed_nslist(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,
485688eea45cSKlaus Jensen uint64_t off, NvmeRequest *req)
485788eea45cSKlaus Jensen {
485888eea45cSKlaus Jensen uint32_t nslist[1024];
485988eea45cSKlaus Jensen uint32_t trans_len;
486088eea45cSKlaus Jensen int i = 0;
486188eea45cSKlaus Jensen uint32_t nsid;
486288eea45cSKlaus Jensen
4863e2c57529SKlaus Jensen if (off >= sizeof(nslist)) {
4864e2c57529SKlaus Jensen trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(nslist));
4865e2c57529SKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
4866e2c57529SKlaus Jensen }
4867e2c57529SKlaus Jensen
486888eea45cSKlaus Jensen memset(nslist, 0x0, sizeof(nslist));
486988eea45cSKlaus Jensen trans_len = MIN(sizeof(nslist) - off, buf_len);
487088eea45cSKlaus Jensen
487188eea45cSKlaus Jensen while ((nsid = find_first_bit(n->changed_nsids, NVME_CHANGED_NSID_SIZE)) !=
487288eea45cSKlaus Jensen NVME_CHANGED_NSID_SIZE) {
487388eea45cSKlaus Jensen /*
487488eea45cSKlaus Jensen * If more than 1024 namespaces, the first entry in the log page should
487588eea45cSKlaus Jensen * be set to FFFFFFFFh and the others to 0 as spec.
487688eea45cSKlaus Jensen */
487788eea45cSKlaus Jensen if (i == ARRAY_SIZE(nslist)) {
487888eea45cSKlaus Jensen memset(nslist, 0x0, sizeof(nslist));
487988eea45cSKlaus Jensen nslist[0] = 0xffffffff;
488088eea45cSKlaus Jensen break;
488188eea45cSKlaus Jensen }
488288eea45cSKlaus Jensen
488388eea45cSKlaus Jensen nslist[i++] = nsid;
488488eea45cSKlaus Jensen clear_bit(nsid, n->changed_nsids);
488588eea45cSKlaus Jensen }
488688eea45cSKlaus Jensen
488788eea45cSKlaus Jensen /*
488888eea45cSKlaus Jensen * Remove all the remaining list entries in case returns directly due to
488988eea45cSKlaus Jensen * more than 1024 namespaces.
489088eea45cSKlaus Jensen */
489188eea45cSKlaus Jensen if (nslist[0] == 0xffffffff) {
489288eea45cSKlaus Jensen bitmap_zero(n->changed_nsids, NVME_CHANGED_NSID_SIZE);
489388eea45cSKlaus Jensen }
489488eea45cSKlaus Jensen
489588eea45cSKlaus Jensen if (!rae) {
489688eea45cSKlaus Jensen nvme_clear_events(n, NVME_AER_TYPE_NOTICE);
489788eea45cSKlaus Jensen }
489888eea45cSKlaus Jensen
489988eea45cSKlaus Jensen return nvme_c2h(n, ((uint8_t *)nslist) + off, trans_len, req);
490088eea45cSKlaus Jensen }
490188eea45cSKlaus Jensen
nvme_cmd_effects(NvmeCtrl * n,uint8_t csi,uint32_t buf_len,uint64_t off,NvmeRequest * req)490288eea45cSKlaus Jensen static uint16_t nvme_cmd_effects(NvmeCtrl *n, uint8_t csi, uint32_t buf_len,
490388eea45cSKlaus Jensen uint64_t off, NvmeRequest *req)
490488eea45cSKlaus Jensen {
490588eea45cSKlaus Jensen NvmeEffectsLog log = {};
490688eea45cSKlaus Jensen const uint32_t *src_iocs = NULL;
490788eea45cSKlaus Jensen uint32_t trans_len;
490888eea45cSKlaus Jensen
490988eea45cSKlaus Jensen if (off >= sizeof(log)) {
491088eea45cSKlaus Jensen trace_pci_nvme_err_invalid_log_page_offset(off, sizeof(log));
491188eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
491288eea45cSKlaus Jensen }
491388eea45cSKlaus Jensen
491449e03457SKlaus Jensen switch (NVME_CC_CSS(ldl_le_p(&n->bar.cc))) {
491588eea45cSKlaus Jensen case NVME_CC_CSS_NVM:
491688eea45cSKlaus Jensen src_iocs = nvme_cse_iocs_nvm;
491788eea45cSKlaus Jensen /* fall through */
491888eea45cSKlaus Jensen case NVME_CC_CSS_ADMIN_ONLY:
491988eea45cSKlaus Jensen break;
492088eea45cSKlaus Jensen case NVME_CC_CSS_CSI:
492188eea45cSKlaus Jensen switch (csi) {
492288eea45cSKlaus Jensen case NVME_CSI_NVM:
492388eea45cSKlaus Jensen src_iocs = nvme_cse_iocs_nvm;
492488eea45cSKlaus Jensen break;
492588eea45cSKlaus Jensen case NVME_CSI_ZONED:
492688eea45cSKlaus Jensen src_iocs = nvme_cse_iocs_zoned;
492788eea45cSKlaus Jensen break;
492888eea45cSKlaus Jensen }
492988eea45cSKlaus Jensen }
493088eea45cSKlaus Jensen
493188eea45cSKlaus Jensen memcpy(log.acs, nvme_cse_acs, sizeof(nvme_cse_acs));
493288eea45cSKlaus Jensen
493388eea45cSKlaus Jensen if (src_iocs) {
493488eea45cSKlaus Jensen memcpy(log.iocs, src_iocs, sizeof(log.iocs));
493588eea45cSKlaus Jensen }
493688eea45cSKlaus Jensen
493788eea45cSKlaus Jensen trans_len = MIN(sizeof(log) - off, buf_len);
493888eea45cSKlaus Jensen
493988eea45cSKlaus Jensen return nvme_c2h(n, ((uint8_t *)&log) + off, trans_len, req);
494088eea45cSKlaus Jensen }
494188eea45cSKlaus Jensen
sizeof_fdp_conf_descr(size_t nruh,size_t vss)494273064edfSJesper Devantier static size_t sizeof_fdp_conf_descr(size_t nruh, size_t vss)
494373064edfSJesper Devantier {
494473064edfSJesper Devantier size_t entry_siz = sizeof(NvmeFdpDescrHdr) + nruh * sizeof(NvmeRuhDescr)
494573064edfSJesper Devantier + vss;
494673064edfSJesper Devantier return ROUND_UP(entry_siz, 8);
494773064edfSJesper Devantier }
494873064edfSJesper Devantier
nvme_fdp_confs(NvmeCtrl * n,uint32_t endgrpid,uint32_t buf_len,uint64_t off,NvmeRequest * req)494973064edfSJesper Devantier static uint16_t nvme_fdp_confs(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
495073064edfSJesper Devantier uint64_t off, NvmeRequest *req)
495173064edfSJesper Devantier {
495273064edfSJesper Devantier uint32_t log_size, trans_len;
495373064edfSJesper Devantier g_autofree uint8_t *buf = NULL;
495473064edfSJesper Devantier NvmeFdpDescrHdr *hdr;
495573064edfSJesper Devantier NvmeRuhDescr *ruhd;
495673064edfSJesper Devantier NvmeEnduranceGroup *endgrp;
495773064edfSJesper Devantier NvmeFdpConfsHdr *log;
495873064edfSJesper Devantier size_t nruh, fdp_descr_size;
495973064edfSJesper Devantier int i;
496073064edfSJesper Devantier
496173064edfSJesper Devantier if (endgrpid != 1 || !n->subsys) {
496273064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
496373064edfSJesper Devantier }
496473064edfSJesper Devantier
496573064edfSJesper Devantier endgrp = &n->subsys->endgrp;
496673064edfSJesper Devantier
496773064edfSJesper Devantier if (endgrp->fdp.enabled) {
496873064edfSJesper Devantier nruh = endgrp->fdp.nruh;
496973064edfSJesper Devantier } else {
497073064edfSJesper Devantier nruh = 1;
497173064edfSJesper Devantier }
497273064edfSJesper Devantier
497373064edfSJesper Devantier fdp_descr_size = sizeof_fdp_conf_descr(nruh, FDPVSS);
497473064edfSJesper Devantier log_size = sizeof(NvmeFdpConfsHdr) + fdp_descr_size;
497573064edfSJesper Devantier
497673064edfSJesper Devantier if (off >= log_size) {
497773064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
497873064edfSJesper Devantier }
497973064edfSJesper Devantier
498073064edfSJesper Devantier trans_len = MIN(log_size - off, buf_len);
498173064edfSJesper Devantier
498273064edfSJesper Devantier buf = g_malloc0(log_size);
498373064edfSJesper Devantier log = (NvmeFdpConfsHdr *)buf;
498473064edfSJesper Devantier hdr = (NvmeFdpDescrHdr *)(log + 1);
498573064edfSJesper Devantier ruhd = (NvmeRuhDescr *)(buf + sizeof(*log) + sizeof(*hdr));
498673064edfSJesper Devantier
498773064edfSJesper Devantier log->num_confs = cpu_to_le16(0);
498873064edfSJesper Devantier log->size = cpu_to_le32(log_size);
498973064edfSJesper Devantier
499073064edfSJesper Devantier hdr->descr_size = cpu_to_le16(fdp_descr_size);
499173064edfSJesper Devantier if (endgrp->fdp.enabled) {
499273064edfSJesper Devantier hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, VALID, 1);
499373064edfSJesper Devantier hdr->fdpa = FIELD_DP8(hdr->fdpa, FDPA, RGIF, endgrp->fdp.rgif);
499473064edfSJesper Devantier hdr->nrg = cpu_to_le16(endgrp->fdp.nrg);
499573064edfSJesper Devantier hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
499673064edfSJesper Devantier hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
499773064edfSJesper Devantier hdr->nnss = cpu_to_le32(NVME_MAX_NAMESPACES);
499873064edfSJesper Devantier hdr->runs = cpu_to_le64(endgrp->fdp.runs);
499973064edfSJesper Devantier
500073064edfSJesper Devantier for (i = 0; i < nruh; i++) {
500173064edfSJesper Devantier ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
500273064edfSJesper Devantier ruhd++;
500373064edfSJesper Devantier }
500473064edfSJesper Devantier } else {
500573064edfSJesper Devantier /* 1 bit for RUH in PIF -> 2 RUHs max. */
500673064edfSJesper Devantier hdr->nrg = cpu_to_le16(1);
500773064edfSJesper Devantier hdr->nruh = cpu_to_le16(1);
500873064edfSJesper Devantier hdr->maxpids = cpu_to_le16(NVME_FDP_MAXPIDS - 1);
500973064edfSJesper Devantier hdr->nnss = cpu_to_le32(1);
501073064edfSJesper Devantier hdr->runs = cpu_to_le64(96 * MiB);
501173064edfSJesper Devantier
501273064edfSJesper Devantier ruhd->ruht = NVME_RUHT_INITIALLY_ISOLATED;
501373064edfSJesper Devantier }
501473064edfSJesper Devantier
501573064edfSJesper Devantier return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
501673064edfSJesper Devantier }
501773064edfSJesper Devantier
nvme_fdp_ruh_usage(NvmeCtrl * n,uint32_t endgrpid,uint32_t dw10,uint32_t dw12,uint32_t buf_len,uint64_t off,NvmeRequest * req)501873064edfSJesper Devantier static uint16_t nvme_fdp_ruh_usage(NvmeCtrl *n, uint32_t endgrpid,
501973064edfSJesper Devantier uint32_t dw10, uint32_t dw12,
502073064edfSJesper Devantier uint32_t buf_len, uint64_t off,
502173064edfSJesper Devantier NvmeRequest *req)
502273064edfSJesper Devantier {
502373064edfSJesper Devantier NvmeRuHandle *ruh;
502473064edfSJesper Devantier NvmeRuhuLog *hdr;
502573064edfSJesper Devantier NvmeRuhuDescr *ruhud;
502673064edfSJesper Devantier NvmeEnduranceGroup *endgrp;
502773064edfSJesper Devantier g_autofree uint8_t *buf = NULL;
502873064edfSJesper Devantier uint32_t log_size, trans_len;
502973064edfSJesper Devantier uint16_t i;
503073064edfSJesper Devantier
503173064edfSJesper Devantier if (endgrpid != 1 || !n->subsys) {
503273064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
503373064edfSJesper Devantier }
503473064edfSJesper Devantier
503573064edfSJesper Devantier endgrp = &n->subsys->endgrp;
503673064edfSJesper Devantier
503773064edfSJesper Devantier if (!endgrp->fdp.enabled) {
503873064edfSJesper Devantier return NVME_FDP_DISABLED | NVME_DNR;
503973064edfSJesper Devantier }
504073064edfSJesper Devantier
504173064edfSJesper Devantier log_size = sizeof(NvmeRuhuLog) + endgrp->fdp.nruh * sizeof(NvmeRuhuDescr);
504273064edfSJesper Devantier
504373064edfSJesper Devantier if (off >= log_size) {
504473064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
504573064edfSJesper Devantier }
504673064edfSJesper Devantier
504773064edfSJesper Devantier trans_len = MIN(log_size - off, buf_len);
504873064edfSJesper Devantier
504973064edfSJesper Devantier buf = g_malloc0(log_size);
505073064edfSJesper Devantier hdr = (NvmeRuhuLog *)buf;
505173064edfSJesper Devantier ruhud = (NvmeRuhuDescr *)(hdr + 1);
505273064edfSJesper Devantier
505373064edfSJesper Devantier ruh = endgrp->fdp.ruhs;
505473064edfSJesper Devantier hdr->nruh = cpu_to_le16(endgrp->fdp.nruh);
505573064edfSJesper Devantier
505673064edfSJesper Devantier for (i = 0; i < endgrp->fdp.nruh; i++, ruhud++, ruh++) {
505773064edfSJesper Devantier ruhud->ruha = ruh->ruha;
505873064edfSJesper Devantier }
505973064edfSJesper Devantier
506073064edfSJesper Devantier return nvme_c2h(n, (uint8_t *)buf + off, trans_len, req);
506173064edfSJesper Devantier }
506273064edfSJesper Devantier
nvme_fdp_stats(NvmeCtrl * n,uint32_t endgrpid,uint32_t buf_len,uint64_t off,NvmeRequest * req)506373064edfSJesper Devantier static uint16_t nvme_fdp_stats(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,
506473064edfSJesper Devantier uint64_t off, NvmeRequest *req)
506573064edfSJesper Devantier {
506673064edfSJesper Devantier NvmeEnduranceGroup *endgrp;
506773064edfSJesper Devantier NvmeFdpStatsLog log = {};
506873064edfSJesper Devantier uint32_t trans_len;
506973064edfSJesper Devantier
507073064edfSJesper Devantier if (off >= sizeof(NvmeFdpStatsLog)) {
507173064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
507273064edfSJesper Devantier }
507373064edfSJesper Devantier
507473064edfSJesper Devantier if (endgrpid != 1 || !n->subsys) {
507573064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
507673064edfSJesper Devantier }
507773064edfSJesper Devantier
507873064edfSJesper Devantier if (!n->subsys->endgrp.fdp.enabled) {
507973064edfSJesper Devantier return NVME_FDP_DISABLED | NVME_DNR;
508073064edfSJesper Devantier }
508173064edfSJesper Devantier
508273064edfSJesper Devantier endgrp = &n->subsys->endgrp;
508373064edfSJesper Devantier
508473064edfSJesper Devantier trans_len = MIN(sizeof(log) - off, buf_len);
508573064edfSJesper Devantier
508673064edfSJesper Devantier /* spec value is 128 bit, we only use 64 bit */
508773064edfSJesper Devantier log.hbmw[0] = cpu_to_le64(endgrp->fdp.hbmw);
508873064edfSJesper Devantier log.mbmw[0] = cpu_to_le64(endgrp->fdp.mbmw);
508973064edfSJesper Devantier log.mbe[0] = cpu_to_le64(endgrp->fdp.mbe);
509073064edfSJesper Devantier
509173064edfSJesper Devantier return nvme_c2h(n, (uint8_t *)&log + off, trans_len, req);
509273064edfSJesper Devantier }
509373064edfSJesper Devantier
nvme_fdp_events(NvmeCtrl * n,uint32_t endgrpid,uint32_t buf_len,uint64_t off,NvmeRequest * req)509473064edfSJesper Devantier static uint16_t nvme_fdp_events(NvmeCtrl *n, uint32_t endgrpid,
509573064edfSJesper Devantier uint32_t buf_len, uint64_t off,
509673064edfSJesper Devantier NvmeRequest *req)
509773064edfSJesper Devantier {
509873064edfSJesper Devantier NvmeEnduranceGroup *endgrp;
509973064edfSJesper Devantier NvmeCmd *cmd = &req->cmd;
510073064edfSJesper Devantier bool host_events = (cmd->cdw10 >> 8) & 0x1;
510173064edfSJesper Devantier uint32_t log_size, trans_len;
510273064edfSJesper Devantier NvmeFdpEventBuffer *ebuf;
510373064edfSJesper Devantier g_autofree NvmeFdpEventsLog *elog = NULL;
510473064edfSJesper Devantier NvmeFdpEvent *event;
510573064edfSJesper Devantier
510673064edfSJesper Devantier if (endgrpid != 1 || !n->subsys) {
510773064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
510873064edfSJesper Devantier }
510973064edfSJesper Devantier
511073064edfSJesper Devantier endgrp = &n->subsys->endgrp;
511173064edfSJesper Devantier
511273064edfSJesper Devantier if (!endgrp->fdp.enabled) {
511373064edfSJesper Devantier return NVME_FDP_DISABLED | NVME_DNR;
511473064edfSJesper Devantier }
511573064edfSJesper Devantier
511673064edfSJesper Devantier if (host_events) {
511773064edfSJesper Devantier ebuf = &endgrp->fdp.host_events;
511873064edfSJesper Devantier } else {
511973064edfSJesper Devantier ebuf = &endgrp->fdp.ctrl_events;
512073064edfSJesper Devantier }
512173064edfSJesper Devantier
512273064edfSJesper Devantier log_size = sizeof(NvmeFdpEventsLog) + ebuf->nelems * sizeof(NvmeFdpEvent);
5123ecb1b7b0SKlaus Jensen
5124ecb1b7b0SKlaus Jensen if (off >= log_size) {
5125ecb1b7b0SKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
5126ecb1b7b0SKlaus Jensen }
5127ecb1b7b0SKlaus Jensen
512873064edfSJesper Devantier trans_len = MIN(log_size - off, buf_len);
512973064edfSJesper Devantier elog = g_malloc0(log_size);
513073064edfSJesper Devantier elog->num_events = cpu_to_le32(ebuf->nelems);
513173064edfSJesper Devantier event = (NvmeFdpEvent *)(elog + 1);
513273064edfSJesper Devantier
513373064edfSJesper Devantier if (ebuf->nelems && ebuf->start == ebuf->next) {
513473064edfSJesper Devantier unsigned int nelems = (NVME_FDP_MAX_EVENTS - ebuf->start);
513573064edfSJesper Devantier /* wrap over, copy [start;NVME_FDP_MAX_EVENTS[ and [0; next[ */
513673064edfSJesper Devantier memcpy(event, &ebuf->events[ebuf->start],
513773064edfSJesper Devantier sizeof(NvmeFdpEvent) * nelems);
513873064edfSJesper Devantier memcpy(event + nelems, ebuf->events,
513973064edfSJesper Devantier sizeof(NvmeFdpEvent) * ebuf->next);
514073064edfSJesper Devantier } else if (ebuf->start < ebuf->next) {
514173064edfSJesper Devantier memcpy(event, &ebuf->events[ebuf->start],
514273064edfSJesper Devantier sizeof(NvmeFdpEvent) * (ebuf->next - ebuf->start));
514373064edfSJesper Devantier }
514473064edfSJesper Devantier
514573064edfSJesper Devantier return nvme_c2h(n, (uint8_t *)elog + off, trans_len, req);
514673064edfSJesper Devantier }
514773064edfSJesper Devantier
nvme_get_log(NvmeCtrl * n,NvmeRequest * req)514888eea45cSKlaus Jensen static uint16_t nvme_get_log(NvmeCtrl *n, NvmeRequest *req)
514988eea45cSKlaus Jensen {
515088eea45cSKlaus Jensen NvmeCmd *cmd = &req->cmd;
515188eea45cSKlaus Jensen
515288eea45cSKlaus Jensen uint32_t dw10 = le32_to_cpu(cmd->cdw10);
515388eea45cSKlaus Jensen uint32_t dw11 = le32_to_cpu(cmd->cdw11);
515488eea45cSKlaus Jensen uint32_t dw12 = le32_to_cpu(cmd->cdw12);
515588eea45cSKlaus Jensen uint32_t dw13 = le32_to_cpu(cmd->cdw13);
515688eea45cSKlaus Jensen uint8_t lid = dw10 & 0xff;
515788eea45cSKlaus Jensen uint8_t lsp = (dw10 >> 8) & 0xf;
515888eea45cSKlaus Jensen uint8_t rae = (dw10 >> 15) & 0x1;
515988eea45cSKlaus Jensen uint8_t csi = le32_to_cpu(cmd->cdw14) >> 24;
516073064edfSJesper Devantier uint32_t numdl, numdu, lspi;
516188eea45cSKlaus Jensen uint64_t off, lpol, lpou;
516288eea45cSKlaus Jensen size_t len;
516388eea45cSKlaus Jensen uint16_t status;
516488eea45cSKlaus Jensen
516588eea45cSKlaus Jensen numdl = (dw10 >> 16);
516688eea45cSKlaus Jensen numdu = (dw11 & 0xffff);
516773064edfSJesper Devantier lspi = (dw11 >> 16);
516888eea45cSKlaus Jensen lpol = dw12;
516988eea45cSKlaus Jensen lpou = dw13;
517088eea45cSKlaus Jensen
517188eea45cSKlaus Jensen len = (((numdu << 16) | numdl) + 1) << 2;
517288eea45cSKlaus Jensen off = (lpou << 32ULL) | lpol;
517388eea45cSKlaus Jensen
517488eea45cSKlaus Jensen if (off & 0x3) {
517588eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
517688eea45cSKlaus Jensen }
517788eea45cSKlaus Jensen
517888eea45cSKlaus Jensen trace_pci_nvme_get_log(nvme_cid(req), lid, lsp, rae, len, off);
517988eea45cSKlaus Jensen
518088eea45cSKlaus Jensen status = nvme_check_mdts(n, len);
518188eea45cSKlaus Jensen if (status) {
518288eea45cSKlaus Jensen return status;
518388eea45cSKlaus Jensen }
518488eea45cSKlaus Jensen
518588eea45cSKlaus Jensen switch (lid) {
518688eea45cSKlaus Jensen case NVME_LOG_ERROR_INFO:
518788eea45cSKlaus Jensen return nvme_error_info(n, rae, len, off, req);
518888eea45cSKlaus Jensen case NVME_LOG_SMART_INFO:
518988eea45cSKlaus Jensen return nvme_smart_info(n, rae, len, off, req);
519088eea45cSKlaus Jensen case NVME_LOG_FW_SLOT_INFO:
519188eea45cSKlaus Jensen return nvme_fw_log_info(n, len, off, req);
519288eea45cSKlaus Jensen case NVME_LOG_CHANGED_NSLIST:
519388eea45cSKlaus Jensen return nvme_changed_nslist(n, rae, len, off, req);
519488eea45cSKlaus Jensen case NVME_LOG_CMD_EFFECTS:
519588eea45cSKlaus Jensen return nvme_cmd_effects(n, csi, len, off, req);
5196771dbc3aSKlaus Jensen case NVME_LOG_ENDGRP:
5197771dbc3aSKlaus Jensen return nvme_endgrp_info(n, rae, len, off, req);
519873064edfSJesper Devantier case NVME_LOG_FDP_CONFS:
519973064edfSJesper Devantier return nvme_fdp_confs(n, lspi, len, off, req);
520073064edfSJesper Devantier case NVME_LOG_FDP_RUH_USAGE:
520173064edfSJesper Devantier return nvme_fdp_ruh_usage(n, lspi, dw10, dw12, len, off, req);
520273064edfSJesper Devantier case NVME_LOG_FDP_STATS:
520373064edfSJesper Devantier return nvme_fdp_stats(n, lspi, len, off, req);
520473064edfSJesper Devantier case NVME_LOG_FDP_EVENTS:
520573064edfSJesper Devantier return nvme_fdp_events(n, lspi, len, off, req);
520688eea45cSKlaus Jensen default:
520788eea45cSKlaus Jensen trace_pci_nvme_err_invalid_log_page(nvme_cid(req), lid);
520888eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
520988eea45cSKlaus Jensen }
521088eea45cSKlaus Jensen }
521188eea45cSKlaus Jensen
nvme_free_cq(NvmeCQueue * cq,NvmeCtrl * n)521288eea45cSKlaus Jensen static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
521388eea45cSKlaus Jensen {
521448b32c28SKlaus Jensen PCIDevice *pci = PCI_DEVICE(n);
52152e53b0b4SJinhao Fan uint16_t offset = (cq->cqid << 3) + (1 << 2);
52162e53b0b4SJinhao Fan
521788eea45cSKlaus Jensen n->cq[cq->cqid] = NULL;
5218d38cc6fdSKlaus Jensen qemu_bh_delete(cq->bh);
52192e53b0b4SJinhao Fan if (cq->ioeventfd_enabled) {
52202e53b0b4SJinhao Fan memory_region_del_eventfd(&n->iomem,
52212e53b0b4SJinhao Fan 0x1000 + offset, 4, false, 0, &cq->notifier);
522204e8da88SKlaus Jensen event_notifier_set_handler(&cq->notifier, NULL);
52232e53b0b4SJinhao Fan event_notifier_cleanup(&cq->notifier);
52242e53b0b4SJinhao Fan }
522548b32c28SKlaus Jensen if (msix_enabled(pci)) {
522648b32c28SKlaus Jensen msix_vector_unuse(pci, cq->vector);
522788eea45cSKlaus Jensen }
522888eea45cSKlaus Jensen if (cq->cqid) {
522988eea45cSKlaus Jensen g_free(cq);
523088eea45cSKlaus Jensen }
523188eea45cSKlaus Jensen }
523288eea45cSKlaus Jensen
nvme_del_cq(NvmeCtrl * n,NvmeRequest * req)523388eea45cSKlaus Jensen static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)
523488eea45cSKlaus Jensen {
523588eea45cSKlaus Jensen NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
523688eea45cSKlaus Jensen NvmeCQueue *cq;
523788eea45cSKlaus Jensen uint16_t qid = le16_to_cpu(c->qid);
523888eea45cSKlaus Jensen
523988eea45cSKlaus Jensen if (unlikely(!qid || nvme_check_cqid(n, qid))) {
524088eea45cSKlaus Jensen trace_pci_nvme_err_invalid_del_cq_cqid(qid);
524188eea45cSKlaus Jensen return NVME_INVALID_CQID | NVME_DNR;
524288eea45cSKlaus Jensen }
524388eea45cSKlaus Jensen
524488eea45cSKlaus Jensen cq = n->cq[qid];
524588eea45cSKlaus Jensen if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
524688eea45cSKlaus Jensen trace_pci_nvme_err_invalid_del_cq_notempty(qid);
524788eea45cSKlaus Jensen return NVME_INVALID_QUEUE_DEL;
524888eea45cSKlaus Jensen }
524983d7ed5cSKlaus Jensen
525083d7ed5cSKlaus Jensen if (cq->irq_enabled && cq->tail != cq->head) {
525183d7ed5cSKlaus Jensen n->cq_pending--;
525283d7ed5cSKlaus Jensen }
525383d7ed5cSKlaus Jensen
525488eea45cSKlaus Jensen nvme_irq_deassert(n, cq);
525588eea45cSKlaus Jensen trace_pci_nvme_del_cq(qid);
525688eea45cSKlaus Jensen nvme_free_cq(cq, n);
525788eea45cSKlaus Jensen return NVME_SUCCESS;
525888eea45cSKlaus Jensen }
525988eea45cSKlaus Jensen
nvme_init_cq(NvmeCQueue * cq,NvmeCtrl * n,uint64_t dma_addr,uint16_t cqid,uint16_t vector,uint16_t size,uint16_t irq_enabled)526088eea45cSKlaus Jensen static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
526188eea45cSKlaus Jensen uint16_t cqid, uint16_t vector, uint16_t size,
526288eea45cSKlaus Jensen uint16_t irq_enabled)
526388eea45cSKlaus Jensen {
526448b32c28SKlaus Jensen PCIDevice *pci = PCI_DEVICE(n);
526548b32c28SKlaus Jensen
526648b32c28SKlaus Jensen if (msix_enabled(pci)) {
526748b32c28SKlaus Jensen msix_vector_use(pci, vector);
526888eea45cSKlaus Jensen }
526988eea45cSKlaus Jensen cq->ctrl = n;
527088eea45cSKlaus Jensen cq->cqid = cqid;
527188eea45cSKlaus Jensen cq->size = size;
527288eea45cSKlaus Jensen cq->dma_addr = dma_addr;
527388eea45cSKlaus Jensen cq->phase = 1;
527488eea45cSKlaus Jensen cq->irq_enabled = irq_enabled;
527588eea45cSKlaus Jensen cq->vector = vector;
527688eea45cSKlaus Jensen cq->head = cq->tail = 0;
527788eea45cSKlaus Jensen QTAILQ_INIT(&cq->req_list);
527888eea45cSKlaus Jensen QTAILQ_INIT(&cq->sq_list);
52793f7fe8deSJinhao Fan if (n->dbbuf_enabled) {
52803f7fe8deSJinhao Fan cq->db_addr = n->dbbuf_dbs + (cqid << 3) + (1 << 2);
52813f7fe8deSJinhao Fan cq->ei_addr = n->dbbuf_eis + (cqid << 3) + (1 << 2);
52822e53b0b4SJinhao Fan
52832e53b0b4SJinhao Fan if (n->params.ioeventfd && cqid != 0) {
52842e53b0b4SJinhao Fan if (!nvme_init_cq_ioeventfd(cq)) {
52852e53b0b4SJinhao Fan cq->ioeventfd_enabled = true;
52862e53b0b4SJinhao Fan }
52872e53b0b4SJinhao Fan }
52883f7fe8deSJinhao Fan }
528988eea45cSKlaus Jensen n->cq[cqid] = cq;
5290f63192b0SAlexander Bulekov cq->bh = qemu_bh_new_guarded(nvme_post_cqes, cq,
5291f63192b0SAlexander Bulekov &DEVICE(cq->ctrl)->mem_reentrancy_guard);
529288eea45cSKlaus Jensen }
529388eea45cSKlaus Jensen
nvme_create_cq(NvmeCtrl * n,NvmeRequest * req)529488eea45cSKlaus Jensen static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeRequest *req)
529588eea45cSKlaus Jensen {
529688eea45cSKlaus Jensen NvmeCQueue *cq;
529788eea45cSKlaus Jensen NvmeCreateCq *c = (NvmeCreateCq *)&req->cmd;
529888eea45cSKlaus Jensen uint16_t cqid = le16_to_cpu(c->cqid);
529988eea45cSKlaus Jensen uint16_t vector = le16_to_cpu(c->irq_vector);
530088eea45cSKlaus Jensen uint16_t qsize = le16_to_cpu(c->qsize);
530188eea45cSKlaus Jensen uint16_t qflags = le16_to_cpu(c->cq_flags);
530288eea45cSKlaus Jensen uint64_t prp1 = le64_to_cpu(c->prp1);
53036a33f2e9SKlaus Jensen uint32_t cc = ldq_le_p(&n->bar.cc);
53046a33f2e9SKlaus Jensen uint8_t iocqes = NVME_CC_IOCQES(cc);
53056a33f2e9SKlaus Jensen uint8_t iosqes = NVME_CC_IOSQES(cc);
530688eea45cSKlaus Jensen
530788eea45cSKlaus Jensen trace_pci_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
530888eea45cSKlaus Jensen NVME_CQ_FLAGS_IEN(qflags) != 0);
530988eea45cSKlaus Jensen
53106a33f2e9SKlaus Jensen if (iosqes != NVME_SQES || iocqes != NVME_CQES) {
53116a33f2e9SKlaus Jensen trace_pci_nvme_err_invalid_create_cq_entry_size(iosqes, iocqes);
53126a33f2e9SKlaus Jensen return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
53136a33f2e9SKlaus Jensen }
53146a33f2e9SKlaus Jensen
5315decc0261SŁukasz Gieryk if (unlikely(!cqid || cqid > n->conf_ioqpairs || n->cq[cqid] != NULL)) {
531688eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_cq_cqid(cqid);
531788eea45cSKlaus Jensen return NVME_INVALID_QID | NVME_DNR;
531888eea45cSKlaus Jensen }
531949e03457SKlaus Jensen if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
532088eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_cq_size(qsize);
532188eea45cSKlaus Jensen return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
532288eea45cSKlaus Jensen }
532388eea45cSKlaus Jensen if (unlikely(prp1 & (n->page_size - 1))) {
532488eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_cq_addr(prp1);
532588eea45cSKlaus Jensen return NVME_INVALID_PRP_OFFSET | NVME_DNR;
532688eea45cSKlaus Jensen }
532748b32c28SKlaus Jensen if (unlikely(!msix_enabled(PCI_DEVICE(n)) && vector)) {
532888eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_cq_vector(vector);
532988eea45cSKlaus Jensen return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
533088eea45cSKlaus Jensen }
5331decc0261SŁukasz Gieryk if (unlikely(vector >= n->conf_msix_qsize)) {
533288eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_cq_vector(vector);
533388eea45cSKlaus Jensen return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
533488eea45cSKlaus Jensen }
533588eea45cSKlaus Jensen if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
533688eea45cSKlaus Jensen trace_pci_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
533788eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
533888eea45cSKlaus Jensen }
533988eea45cSKlaus Jensen
534088eea45cSKlaus Jensen cq = g_malloc0(sizeof(*cq));
534188eea45cSKlaus Jensen nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
534288eea45cSKlaus Jensen NVME_CQ_FLAGS_IEN(qflags));
534388eea45cSKlaus Jensen
534488eea45cSKlaus Jensen /*
534588eea45cSKlaus Jensen * It is only required to set qs_created when creating a completion queue;
534688eea45cSKlaus Jensen * creating a submission queue without a matching completion queue will
534788eea45cSKlaus Jensen * fail.
534888eea45cSKlaus Jensen */
534988eea45cSKlaus Jensen n->qs_created = true;
535088eea45cSKlaus Jensen return NVME_SUCCESS;
535188eea45cSKlaus Jensen }
535288eea45cSKlaus Jensen
nvme_rpt_empty_id_struct(NvmeCtrl * n,NvmeRequest * req)535388eea45cSKlaus Jensen static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl *n, NvmeRequest *req)
535488eea45cSKlaus Jensen {
535588eea45cSKlaus Jensen uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
535688eea45cSKlaus Jensen
535788eea45cSKlaus Jensen return nvme_c2h(n, id, sizeof(id), req);
535888eea45cSKlaus Jensen }
535988eea45cSKlaus Jensen
nvme_identify_ctrl(NvmeCtrl * n,NvmeRequest * req)536088eea45cSKlaus Jensen static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeRequest *req)
536188eea45cSKlaus Jensen {
536288eea45cSKlaus Jensen trace_pci_nvme_identify_ctrl();
536388eea45cSKlaus Jensen
536488eea45cSKlaus Jensen return nvme_c2h(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl), req);
536588eea45cSKlaus Jensen }
536688eea45cSKlaus Jensen
nvme_identify_ctrl_csi(NvmeCtrl * n,NvmeRequest * req)536788eea45cSKlaus Jensen static uint16_t nvme_identify_ctrl_csi(NvmeCtrl *n, NvmeRequest *req)
536888eea45cSKlaus Jensen {
536988eea45cSKlaus Jensen NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
537088eea45cSKlaus Jensen uint8_t id[NVME_IDENTIFY_DATA_SIZE] = {};
537188eea45cSKlaus Jensen NvmeIdCtrlNvm *id_nvm = (NvmeIdCtrlNvm *)&id;
537288eea45cSKlaus Jensen
537388eea45cSKlaus Jensen trace_pci_nvme_identify_ctrl_csi(c->csi);
537488eea45cSKlaus Jensen
537588eea45cSKlaus Jensen switch (c->csi) {
537688eea45cSKlaus Jensen case NVME_CSI_NVM:
537788eea45cSKlaus Jensen id_nvm->vsl = n->params.vsl;
537888eea45cSKlaus Jensen id_nvm->dmrsl = cpu_to_le32(n->dmrsl);
537988eea45cSKlaus Jensen break;
538088eea45cSKlaus Jensen
538188eea45cSKlaus Jensen case NVME_CSI_ZONED:
538288eea45cSKlaus Jensen ((NvmeIdCtrlZoned *)&id)->zasl = n->params.zasl;
538388eea45cSKlaus Jensen break;
538488eea45cSKlaus Jensen
538588eea45cSKlaus Jensen default:
538688eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
538788eea45cSKlaus Jensen }
538888eea45cSKlaus Jensen
538988eea45cSKlaus Jensen return nvme_c2h(n, id, sizeof(id), req);
539088eea45cSKlaus Jensen }
539188eea45cSKlaus Jensen
nvme_identify_ns(NvmeCtrl * n,NvmeRequest * req,bool active)539288eea45cSKlaus Jensen static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeRequest *req, bool active)
539388eea45cSKlaus Jensen {
539488eea45cSKlaus Jensen NvmeNamespace *ns;
539588eea45cSKlaus Jensen NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
539688eea45cSKlaus Jensen uint32_t nsid = le32_to_cpu(c->nsid);
539788eea45cSKlaus Jensen
539888eea45cSKlaus Jensen trace_pci_nvme_identify_ns(nsid);
539988eea45cSKlaus Jensen
540088eea45cSKlaus Jensen if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
540188eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
540288eea45cSKlaus Jensen }
540388eea45cSKlaus Jensen
540488eea45cSKlaus Jensen ns = nvme_ns(n, nsid);
540588eea45cSKlaus Jensen if (unlikely(!ns)) {
540688eea45cSKlaus Jensen if (!active) {
540788eea45cSKlaus Jensen ns = nvme_subsys_ns(n->subsys, nsid);
540888eea45cSKlaus Jensen if (!ns) {
540988eea45cSKlaus Jensen return nvme_rpt_empty_id_struct(n, req);
541088eea45cSKlaus Jensen }
541188eea45cSKlaus Jensen } else {
541288eea45cSKlaus Jensen return nvme_rpt_empty_id_struct(n, req);
541388eea45cSKlaus Jensen }
541488eea45cSKlaus Jensen }
541588eea45cSKlaus Jensen
54163553c48fSGollu Appalanaidu if (active || ns->csi == NVME_CSI_NVM) {
541788eea45cSKlaus Jensen return nvme_c2h(n, (uint8_t *)&ns->id_ns, sizeof(NvmeIdNs), req);
541888eea45cSKlaus Jensen }
541988eea45cSKlaus Jensen
542088eea45cSKlaus Jensen return NVME_INVALID_CMD_SET | NVME_DNR;
542188eea45cSKlaus Jensen }
542288eea45cSKlaus Jensen
nvme_identify_ctrl_list(NvmeCtrl * n,NvmeRequest * req,bool attached)54235f4eb94dSGollu Appalanaidu static uint16_t nvme_identify_ctrl_list(NvmeCtrl *n, NvmeRequest *req,
54245f4eb94dSGollu Appalanaidu bool attached)
542588eea45cSKlaus Jensen {
542688eea45cSKlaus Jensen NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
54275f4eb94dSGollu Appalanaidu uint32_t nsid = le32_to_cpu(c->nsid);
542888eea45cSKlaus Jensen uint16_t min_id = le16_to_cpu(c->ctrlid);
542988eea45cSKlaus Jensen uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
543088eea45cSKlaus Jensen uint16_t *ids = &list[1];
543188eea45cSKlaus Jensen NvmeNamespace *ns;
543288eea45cSKlaus Jensen NvmeCtrl *ctrl;
543388eea45cSKlaus Jensen int cntlid, nr_ids = 0;
543488eea45cSKlaus Jensen
54355f4eb94dSGollu Appalanaidu trace_pci_nvme_identify_ctrl_list(c->cns, min_id);
543688eea45cSKlaus Jensen
54375f4eb94dSGollu Appalanaidu if (!n->subsys) {
543888eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
543988eea45cSKlaus Jensen }
544088eea45cSKlaus Jensen
54415f4eb94dSGollu Appalanaidu if (attached) {
54425f4eb94dSGollu Appalanaidu if (nsid == NVME_NSID_BROADCAST) {
54435f4eb94dSGollu Appalanaidu return NVME_INVALID_FIELD | NVME_DNR;
54445f4eb94dSGollu Appalanaidu }
54455f4eb94dSGollu Appalanaidu
54465f4eb94dSGollu Appalanaidu ns = nvme_subsys_ns(n->subsys, nsid);
544788eea45cSKlaus Jensen if (!ns) {
544888eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
544988eea45cSKlaus Jensen }
54505f4eb94dSGollu Appalanaidu }
545188eea45cSKlaus Jensen
545288eea45cSKlaus Jensen for (cntlid = min_id; cntlid < ARRAY_SIZE(n->subsys->ctrls); cntlid++) {
545388eea45cSKlaus Jensen ctrl = nvme_subsys_ctrl(n->subsys, cntlid);
545488eea45cSKlaus Jensen if (!ctrl) {
545588eea45cSKlaus Jensen continue;
545688eea45cSKlaus Jensen }
545788eea45cSKlaus Jensen
54585f4eb94dSGollu Appalanaidu if (attached && !nvme_ns(ctrl, nsid)) {
545988eea45cSKlaus Jensen continue;
546088eea45cSKlaus Jensen }
546188eea45cSKlaus Jensen
546288eea45cSKlaus Jensen ids[nr_ids++] = cntlid;
546388eea45cSKlaus Jensen }
546488eea45cSKlaus Jensen
546588eea45cSKlaus Jensen list[0] = nr_ids;
546688eea45cSKlaus Jensen
546788eea45cSKlaus Jensen return nvme_c2h(n, (uint8_t *)list, sizeof(list), req);
546888eea45cSKlaus Jensen }
546988eea45cSKlaus Jensen
nvme_identify_pri_ctrl_cap(NvmeCtrl * n,NvmeRequest * req)54705e6f963fSLukasz Maniak static uint16_t nvme_identify_pri_ctrl_cap(NvmeCtrl *n, NvmeRequest *req)
54715e6f963fSLukasz Maniak {
54725e6f963fSLukasz Maniak trace_pci_nvme_identify_pri_ctrl_cap(le16_to_cpu(n->pri_ctrl_cap.cntlid));
54735e6f963fSLukasz Maniak
54745e6f963fSLukasz Maniak return nvme_c2h(n, (uint8_t *)&n->pri_ctrl_cap,
54755e6f963fSLukasz Maniak sizeof(NvmePriCtrlCap), req);
54765e6f963fSLukasz Maniak }
54775e6f963fSLukasz Maniak
nvme_identify_sec_ctrl_list(NvmeCtrl * n,NvmeRequest * req)547899f48ae7SLukasz Maniak static uint16_t nvme_identify_sec_ctrl_list(NvmeCtrl *n, NvmeRequest *req)
547999f48ae7SLukasz Maniak {
548099f48ae7SLukasz Maniak NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
548199f48ae7SLukasz Maniak uint16_t pri_ctrl_id = le16_to_cpu(n->pri_ctrl_cap.cntlid);
548299f48ae7SLukasz Maniak uint16_t min_id = le16_to_cpu(c->ctrlid);
548399f48ae7SLukasz Maniak uint8_t num_sec_ctrl = n->sec_ctrl_list.numcntl;
548499f48ae7SLukasz Maniak NvmeSecCtrlList list = {0};
548599f48ae7SLukasz Maniak uint8_t i;
548699f48ae7SLukasz Maniak
548799f48ae7SLukasz Maniak for (i = 0; i < num_sec_ctrl; i++) {
548899f48ae7SLukasz Maniak if (n->sec_ctrl_list.sec[i].scid >= min_id) {
548999f48ae7SLukasz Maniak list.numcntl = num_sec_ctrl - i;
549099f48ae7SLukasz Maniak memcpy(&list.sec, n->sec_ctrl_list.sec + i,
549199f48ae7SLukasz Maniak list.numcntl * sizeof(NvmeSecCtrlEntry));
549299f48ae7SLukasz Maniak break;
549399f48ae7SLukasz Maniak }
549499f48ae7SLukasz Maniak }
549599f48ae7SLukasz Maniak
549699f48ae7SLukasz Maniak trace_pci_nvme_identify_sec_ctrl_list(pri_ctrl_id, list.numcntl);
549799f48ae7SLukasz Maniak
549899f48ae7SLukasz Maniak return nvme_c2h(n, (uint8_t *)&list, sizeof(list), req);
549999f48ae7SLukasz Maniak }
550099f48ae7SLukasz Maniak
nvme_identify_ns_csi(NvmeCtrl * n,NvmeRequest * req,bool active)550188eea45cSKlaus Jensen static uint16_t nvme_identify_ns_csi(NvmeCtrl *n, NvmeRequest *req,
550288eea45cSKlaus Jensen bool active)
550388eea45cSKlaus Jensen {
550488eea45cSKlaus Jensen NvmeNamespace *ns;
550588eea45cSKlaus Jensen NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
550688eea45cSKlaus Jensen uint32_t nsid = le32_to_cpu(c->nsid);
550788eea45cSKlaus Jensen
550888eea45cSKlaus Jensen trace_pci_nvme_identify_ns_csi(nsid, c->csi);
550988eea45cSKlaus Jensen
551088eea45cSKlaus Jensen if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
551188eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
551288eea45cSKlaus Jensen }
551388eea45cSKlaus Jensen
551488eea45cSKlaus Jensen ns = nvme_ns(n, nsid);
551588eea45cSKlaus Jensen if (unlikely(!ns)) {
551688eea45cSKlaus Jensen if (!active) {
551788eea45cSKlaus Jensen ns = nvme_subsys_ns(n->subsys, nsid);
551888eea45cSKlaus Jensen if (!ns) {
551988eea45cSKlaus Jensen return nvme_rpt_empty_id_struct(n, req);
552088eea45cSKlaus Jensen }
552188eea45cSKlaus Jensen } else {
552288eea45cSKlaus Jensen return nvme_rpt_empty_id_struct(n, req);
552388eea45cSKlaus Jensen }
552488eea45cSKlaus Jensen }
552588eea45cSKlaus Jensen
55263553c48fSGollu Appalanaidu if (c->csi == NVME_CSI_NVM) {
552744219b60SNaveen Nagar return nvme_c2h(n, (uint8_t *)&ns->id_ns_nvm, sizeof(NvmeIdNsNvm),
552844219b60SNaveen Nagar req);
552988eea45cSKlaus Jensen } else if (c->csi == NVME_CSI_ZONED && ns->csi == NVME_CSI_ZONED) {
553088eea45cSKlaus Jensen return nvme_c2h(n, (uint8_t *)ns->id_ns_zoned, sizeof(NvmeIdNsZoned),
553188eea45cSKlaus Jensen req);
553288eea45cSKlaus Jensen }
553388eea45cSKlaus Jensen
553488eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
553588eea45cSKlaus Jensen }
553688eea45cSKlaus Jensen
nvme_identify_nslist(NvmeCtrl * n,NvmeRequest * req,bool active)553788eea45cSKlaus Jensen static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req,
553888eea45cSKlaus Jensen bool active)
553988eea45cSKlaus Jensen {
554088eea45cSKlaus Jensen NvmeNamespace *ns;
554188eea45cSKlaus Jensen NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
554288eea45cSKlaus Jensen uint32_t min_nsid = le32_to_cpu(c->nsid);
554388eea45cSKlaus Jensen uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
554488eea45cSKlaus Jensen static const int data_len = sizeof(list);
554588eea45cSKlaus Jensen uint32_t *list_ptr = (uint32_t *)list;
554688eea45cSKlaus Jensen int i, j = 0;
554788eea45cSKlaus Jensen
554888eea45cSKlaus Jensen trace_pci_nvme_identify_nslist(min_nsid);
554988eea45cSKlaus Jensen
555088eea45cSKlaus Jensen /*
555188eea45cSKlaus Jensen * Both FFFFFFFFh (NVME_NSID_BROADCAST) and FFFFFFFFEh are invalid values
555288eea45cSKlaus Jensen * since the Active Namespace ID List should return namespaces with ids
555388eea45cSKlaus Jensen * *higher* than the NSID specified in the command. This is also specified
555488eea45cSKlaus Jensen * in the spec (NVM Express v1.3d, Section 5.15.4).
555588eea45cSKlaus Jensen */
555688eea45cSKlaus Jensen if (min_nsid >= NVME_NSID_BROADCAST - 1) {
555788eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
555888eea45cSKlaus Jensen }
555988eea45cSKlaus Jensen
556088eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
556188eea45cSKlaus Jensen ns = nvme_ns(n, i);
556288eea45cSKlaus Jensen if (!ns) {
556388eea45cSKlaus Jensen if (!active) {
556488eea45cSKlaus Jensen ns = nvme_subsys_ns(n->subsys, i);
556588eea45cSKlaus Jensen if (!ns) {
556688eea45cSKlaus Jensen continue;
556788eea45cSKlaus Jensen }
556888eea45cSKlaus Jensen } else {
556988eea45cSKlaus Jensen continue;
557088eea45cSKlaus Jensen }
557188eea45cSKlaus Jensen }
557288eea45cSKlaus Jensen if (ns->params.nsid <= min_nsid) {
557388eea45cSKlaus Jensen continue;
557488eea45cSKlaus Jensen }
557588eea45cSKlaus Jensen list_ptr[j++] = cpu_to_le32(ns->params.nsid);
557688eea45cSKlaus Jensen if (j == data_len / sizeof(uint32_t)) {
557788eea45cSKlaus Jensen break;
557888eea45cSKlaus Jensen }
557988eea45cSKlaus Jensen }
558088eea45cSKlaus Jensen
558188eea45cSKlaus Jensen return nvme_c2h(n, list, data_len, req);
558288eea45cSKlaus Jensen }
558388eea45cSKlaus Jensen
nvme_identify_nslist_csi(NvmeCtrl * n,NvmeRequest * req,bool active)558488eea45cSKlaus Jensen static uint16_t nvme_identify_nslist_csi(NvmeCtrl *n, NvmeRequest *req,
558588eea45cSKlaus Jensen bool active)
558688eea45cSKlaus Jensen {
558788eea45cSKlaus Jensen NvmeNamespace *ns;
558888eea45cSKlaus Jensen NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
558988eea45cSKlaus Jensen uint32_t min_nsid = le32_to_cpu(c->nsid);
559088eea45cSKlaus Jensen uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
559188eea45cSKlaus Jensen static const int data_len = sizeof(list);
559288eea45cSKlaus Jensen uint32_t *list_ptr = (uint32_t *)list;
559388eea45cSKlaus Jensen int i, j = 0;
559488eea45cSKlaus Jensen
559588eea45cSKlaus Jensen trace_pci_nvme_identify_nslist_csi(min_nsid, c->csi);
559688eea45cSKlaus Jensen
559788eea45cSKlaus Jensen /*
559888eea45cSKlaus Jensen * Same as in nvme_identify_nslist(), FFFFFFFFh/FFFFFFFFEh are invalid.
559988eea45cSKlaus Jensen */
560088eea45cSKlaus Jensen if (min_nsid >= NVME_NSID_BROADCAST - 1) {
560188eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
560288eea45cSKlaus Jensen }
560388eea45cSKlaus Jensen
560488eea45cSKlaus Jensen if (c->csi != NVME_CSI_NVM && c->csi != NVME_CSI_ZONED) {
560588eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
560688eea45cSKlaus Jensen }
560788eea45cSKlaus Jensen
560888eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
560988eea45cSKlaus Jensen ns = nvme_ns(n, i);
561088eea45cSKlaus Jensen if (!ns) {
561188eea45cSKlaus Jensen if (!active) {
561288eea45cSKlaus Jensen ns = nvme_subsys_ns(n->subsys, i);
561388eea45cSKlaus Jensen if (!ns) {
561488eea45cSKlaus Jensen continue;
561588eea45cSKlaus Jensen }
561688eea45cSKlaus Jensen } else {
561788eea45cSKlaus Jensen continue;
561888eea45cSKlaus Jensen }
561988eea45cSKlaus Jensen }
562088eea45cSKlaus Jensen if (ns->params.nsid <= min_nsid || c->csi != ns->csi) {
562188eea45cSKlaus Jensen continue;
562288eea45cSKlaus Jensen }
562388eea45cSKlaus Jensen list_ptr[j++] = cpu_to_le32(ns->params.nsid);
562488eea45cSKlaus Jensen if (j == data_len / sizeof(uint32_t)) {
562588eea45cSKlaus Jensen break;
562688eea45cSKlaus Jensen }
562788eea45cSKlaus Jensen }
562888eea45cSKlaus Jensen
562988eea45cSKlaus Jensen return nvme_c2h(n, list, data_len, req);
563088eea45cSKlaus Jensen }
563188eea45cSKlaus Jensen
nvme_identify_ns_descr_list(NvmeCtrl * n,NvmeRequest * req)563288eea45cSKlaus Jensen static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeRequest *req)
563388eea45cSKlaus Jensen {
563488eea45cSKlaus Jensen NvmeNamespace *ns;
563588eea45cSKlaus Jensen NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
563688eea45cSKlaus Jensen uint32_t nsid = le32_to_cpu(c->nsid);
563788eea45cSKlaus Jensen uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
56386870cfb8SHeinrich Schuchardt uint8_t *pos = list;
563988eea45cSKlaus Jensen struct {
564088eea45cSKlaus Jensen NvmeIdNsDescr hdr;
564188eea45cSKlaus Jensen uint8_t v[NVME_NIDL_UUID];
56425f4884c4SKlaus Jensen } QEMU_PACKED uuid = {};
56436870cfb8SHeinrich Schuchardt struct {
56446870cfb8SHeinrich Schuchardt NvmeIdNsDescr hdr;
56456870cfb8SHeinrich Schuchardt uint64_t v;
56465f4884c4SKlaus Jensen } QEMU_PACKED eui64 = {};
564788eea45cSKlaus Jensen struct {
564888eea45cSKlaus Jensen NvmeIdNsDescr hdr;
564988eea45cSKlaus Jensen uint8_t v;
56505f4884c4SKlaus Jensen } QEMU_PACKED csi = {};
565188eea45cSKlaus Jensen
565288eea45cSKlaus Jensen trace_pci_nvme_identify_ns_descr_list(nsid);
565388eea45cSKlaus Jensen
565488eea45cSKlaus Jensen if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
565588eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
565688eea45cSKlaus Jensen }
565788eea45cSKlaus Jensen
565888eea45cSKlaus Jensen ns = nvme_ns(n, nsid);
565988eea45cSKlaus Jensen if (unlikely(!ns)) {
566088eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
566188eea45cSKlaus Jensen }
566288eea45cSKlaus Jensen
56639f2e1acfSKlaus Jensen if (!qemu_uuid_is_null(&ns->params.uuid)) {
56646870cfb8SHeinrich Schuchardt uuid.hdr.nidt = NVME_NIDT_UUID;
56656870cfb8SHeinrich Schuchardt uuid.hdr.nidl = NVME_NIDL_UUID;
56666870cfb8SHeinrich Schuchardt memcpy(uuid.v, ns->params.uuid.data, NVME_NIDL_UUID);
56676870cfb8SHeinrich Schuchardt memcpy(pos, &uuid, sizeof(uuid));
56686870cfb8SHeinrich Schuchardt pos += sizeof(uuid);
56699f2e1acfSKlaus Jensen }
567088eea45cSKlaus Jensen
56716870cfb8SHeinrich Schuchardt if (ns->params.eui64) {
56726870cfb8SHeinrich Schuchardt eui64.hdr.nidt = NVME_NIDT_EUI64;
56736870cfb8SHeinrich Schuchardt eui64.hdr.nidl = NVME_NIDL_EUI64;
56746870cfb8SHeinrich Schuchardt eui64.v = cpu_to_be64(ns->params.eui64);
56756870cfb8SHeinrich Schuchardt memcpy(pos, &eui64, sizeof(eui64));
56766870cfb8SHeinrich Schuchardt pos += sizeof(eui64);
56776870cfb8SHeinrich Schuchardt }
56786870cfb8SHeinrich Schuchardt
56796870cfb8SHeinrich Schuchardt csi.hdr.nidt = NVME_NIDT_CSI;
56806870cfb8SHeinrich Schuchardt csi.hdr.nidl = NVME_NIDL_CSI;
56816870cfb8SHeinrich Schuchardt csi.v = ns->csi;
56826870cfb8SHeinrich Schuchardt memcpy(pos, &csi, sizeof(csi));
56836870cfb8SHeinrich Schuchardt pos += sizeof(csi);
568488eea45cSKlaus Jensen
568588eea45cSKlaus Jensen return nvme_c2h(n, list, sizeof(list), req);
568688eea45cSKlaus Jensen }
568788eea45cSKlaus Jensen
nvme_identify_cmd_set(NvmeCtrl * n,NvmeRequest * req)568888eea45cSKlaus Jensen static uint16_t nvme_identify_cmd_set(NvmeCtrl *n, NvmeRequest *req)
568988eea45cSKlaus Jensen {
569088eea45cSKlaus Jensen uint8_t list[NVME_IDENTIFY_DATA_SIZE] = {};
569188eea45cSKlaus Jensen static const int data_len = sizeof(list);
569288eea45cSKlaus Jensen
569388eea45cSKlaus Jensen trace_pci_nvme_identify_cmd_set();
569488eea45cSKlaus Jensen
569588eea45cSKlaus Jensen NVME_SET_CSI(*list, NVME_CSI_NVM);
569688eea45cSKlaus Jensen NVME_SET_CSI(*list, NVME_CSI_ZONED);
569788eea45cSKlaus Jensen
569888eea45cSKlaus Jensen return nvme_c2h(n, list, data_len, req);
569988eea45cSKlaus Jensen }
570088eea45cSKlaus Jensen
nvme_identify(NvmeCtrl * n,NvmeRequest * req)570188eea45cSKlaus Jensen static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req)
570288eea45cSKlaus Jensen {
570388eea45cSKlaus Jensen NvmeIdentify *c = (NvmeIdentify *)&req->cmd;
570488eea45cSKlaus Jensen
570588eea45cSKlaus Jensen trace_pci_nvme_identify(nvme_cid(req), c->cns, le16_to_cpu(c->ctrlid),
570688eea45cSKlaus Jensen c->csi);
570788eea45cSKlaus Jensen
570888eea45cSKlaus Jensen switch (c->cns) {
570988eea45cSKlaus Jensen case NVME_ID_CNS_NS:
571088eea45cSKlaus Jensen return nvme_identify_ns(n, req, true);
571188eea45cSKlaus Jensen case NVME_ID_CNS_NS_PRESENT:
571288eea45cSKlaus Jensen return nvme_identify_ns(n, req, false);
571388eea45cSKlaus Jensen case NVME_ID_CNS_NS_ATTACHED_CTRL_LIST:
57145f4eb94dSGollu Appalanaidu return nvme_identify_ctrl_list(n, req, true);
57155f4eb94dSGollu Appalanaidu case NVME_ID_CNS_CTRL_LIST:
57165f4eb94dSGollu Appalanaidu return nvme_identify_ctrl_list(n, req, false);
57175e6f963fSLukasz Maniak case NVME_ID_CNS_PRIMARY_CTRL_CAP:
57185e6f963fSLukasz Maniak return nvme_identify_pri_ctrl_cap(n, req);
571999f48ae7SLukasz Maniak case NVME_ID_CNS_SECONDARY_CTRL_LIST:
572099f48ae7SLukasz Maniak return nvme_identify_sec_ctrl_list(n, req);
572188eea45cSKlaus Jensen case NVME_ID_CNS_CS_NS:
572288eea45cSKlaus Jensen return nvme_identify_ns_csi(n, req, true);
572388eea45cSKlaus Jensen case NVME_ID_CNS_CS_NS_PRESENT:
572488eea45cSKlaus Jensen return nvme_identify_ns_csi(n, req, false);
572588eea45cSKlaus Jensen case NVME_ID_CNS_CTRL:
572688eea45cSKlaus Jensen return nvme_identify_ctrl(n, req);
572788eea45cSKlaus Jensen case NVME_ID_CNS_CS_CTRL:
572888eea45cSKlaus Jensen return nvme_identify_ctrl_csi(n, req);
572988eea45cSKlaus Jensen case NVME_ID_CNS_NS_ACTIVE_LIST:
573088eea45cSKlaus Jensen return nvme_identify_nslist(n, req, true);
573188eea45cSKlaus Jensen case NVME_ID_CNS_NS_PRESENT_LIST:
573288eea45cSKlaus Jensen return nvme_identify_nslist(n, req, false);
573388eea45cSKlaus Jensen case NVME_ID_CNS_CS_NS_ACTIVE_LIST:
573488eea45cSKlaus Jensen return nvme_identify_nslist_csi(n, req, true);
573588eea45cSKlaus Jensen case NVME_ID_CNS_CS_NS_PRESENT_LIST:
573688eea45cSKlaus Jensen return nvme_identify_nslist_csi(n, req, false);
573788eea45cSKlaus Jensen case NVME_ID_CNS_NS_DESCR_LIST:
573888eea45cSKlaus Jensen return nvme_identify_ns_descr_list(n, req);
573988eea45cSKlaus Jensen case NVME_ID_CNS_IO_COMMAND_SET:
574088eea45cSKlaus Jensen return nvme_identify_cmd_set(n, req);
574188eea45cSKlaus Jensen default:
574288eea45cSKlaus Jensen trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
574388eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
574488eea45cSKlaus Jensen }
574588eea45cSKlaus Jensen }
574688eea45cSKlaus Jensen
nvme_abort(NvmeCtrl * n,NvmeRequest * req)574788eea45cSKlaus Jensen static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req)
574888eea45cSKlaus Jensen {
574988eea45cSKlaus Jensen uint16_t sqid = le32_to_cpu(req->cmd.cdw10) & 0xffff;
575088eea45cSKlaus Jensen
575188eea45cSKlaus Jensen req->cqe.result = 1;
575288eea45cSKlaus Jensen if (nvme_check_sqid(n, sqid)) {
575388eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
575488eea45cSKlaus Jensen }
575588eea45cSKlaus Jensen
575688eea45cSKlaus Jensen return NVME_SUCCESS;
575788eea45cSKlaus Jensen }
575888eea45cSKlaus Jensen
nvme_set_timestamp(NvmeCtrl * n,uint64_t ts)575988eea45cSKlaus Jensen static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts)
576088eea45cSKlaus Jensen {
576188eea45cSKlaus Jensen trace_pci_nvme_setfeat_timestamp(ts);
576288eea45cSKlaus Jensen
576388eea45cSKlaus Jensen n->host_timestamp = le64_to_cpu(ts);
576488eea45cSKlaus Jensen n->timestamp_set_qemu_clock_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
576588eea45cSKlaus Jensen }
576688eea45cSKlaus Jensen
nvme_get_timestamp(const NvmeCtrl * n)576788eea45cSKlaus Jensen static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)
576888eea45cSKlaus Jensen {
576988eea45cSKlaus Jensen uint64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
577088eea45cSKlaus Jensen uint64_t elapsed_time = current_time - n->timestamp_set_qemu_clock_ms;
577188eea45cSKlaus Jensen
577288eea45cSKlaus Jensen union nvme_timestamp {
577388eea45cSKlaus Jensen struct {
577488eea45cSKlaus Jensen uint64_t timestamp:48;
577588eea45cSKlaus Jensen uint64_t sync:1;
577688eea45cSKlaus Jensen uint64_t origin:3;
577788eea45cSKlaus Jensen uint64_t rsvd1:12;
577888eea45cSKlaus Jensen };
577988eea45cSKlaus Jensen uint64_t all;
578088eea45cSKlaus Jensen };
578188eea45cSKlaus Jensen
578288eea45cSKlaus Jensen union nvme_timestamp ts;
578388eea45cSKlaus Jensen ts.all = 0;
578488eea45cSKlaus Jensen ts.timestamp = n->host_timestamp + elapsed_time;
578588eea45cSKlaus Jensen
578688eea45cSKlaus Jensen /* If the host timestamp is non-zero, set the timestamp origin */
578788eea45cSKlaus Jensen ts.origin = n->host_timestamp ? 0x01 : 0x00;
578888eea45cSKlaus Jensen
578988eea45cSKlaus Jensen trace_pci_nvme_getfeat_timestamp(ts.all);
579088eea45cSKlaus Jensen
579188eea45cSKlaus Jensen return cpu_to_le64(ts.all);
579288eea45cSKlaus Jensen }
579388eea45cSKlaus Jensen
nvme_get_feature_timestamp(NvmeCtrl * n,NvmeRequest * req)579488eea45cSKlaus Jensen static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
579588eea45cSKlaus Jensen {
579688eea45cSKlaus Jensen uint64_t timestamp = nvme_get_timestamp(n);
579788eea45cSKlaus Jensen
579888eea45cSKlaus Jensen return nvme_c2h(n, (uint8_t *)×tamp, sizeof(timestamp), req);
579988eea45cSKlaus Jensen }
580088eea45cSKlaus Jensen
nvme_get_feature_fdp(NvmeCtrl * n,uint32_t endgrpid,uint32_t * result)580173064edfSJesper Devantier static int nvme_get_feature_fdp(NvmeCtrl *n, uint32_t endgrpid,
580273064edfSJesper Devantier uint32_t *result)
580373064edfSJesper Devantier {
580473064edfSJesper Devantier *result = 0;
580573064edfSJesper Devantier
580673064edfSJesper Devantier if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
580773064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
580873064edfSJesper Devantier }
580973064edfSJesper Devantier
581073064edfSJesper Devantier *result = FIELD_DP16(0, FEAT_FDP, FDPE, 1);
581173064edfSJesper Devantier *result = FIELD_DP16(*result, FEAT_FDP, CONF_NDX, 0);
581273064edfSJesper Devantier
581373064edfSJesper Devantier return NVME_SUCCESS;
581473064edfSJesper Devantier }
581573064edfSJesper Devantier
nvme_get_feature_fdp_events(NvmeCtrl * n,NvmeNamespace * ns,NvmeRequest * req,uint32_t * result)581673064edfSJesper Devantier static uint16_t nvme_get_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
581773064edfSJesper Devantier NvmeRequest *req, uint32_t *result)
581873064edfSJesper Devantier {
581973064edfSJesper Devantier NvmeCmd *cmd = &req->cmd;
582073064edfSJesper Devantier uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
582173064edfSJesper Devantier uint16_t ph = cdw11 & 0xffff;
582273064edfSJesper Devantier uint8_t noet = (cdw11 >> 16) & 0xff;
582373064edfSJesper Devantier uint16_t ruhid, ret;
582473064edfSJesper Devantier uint32_t nentries = 0;
582573064edfSJesper Devantier uint8_t s_events_ndx = 0;
582673064edfSJesper Devantier size_t s_events_siz = sizeof(NvmeFdpEventDescr) * noet;
582773064edfSJesper Devantier g_autofree NvmeFdpEventDescr *s_events = g_malloc0(s_events_siz);
582873064edfSJesper Devantier NvmeRuHandle *ruh;
582973064edfSJesper Devantier NvmeFdpEventDescr *s_event;
583073064edfSJesper Devantier
583173064edfSJesper Devantier if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
583273064edfSJesper Devantier return NVME_FDP_DISABLED | NVME_DNR;
583373064edfSJesper Devantier }
583473064edfSJesper Devantier
583573064edfSJesper Devantier if (!nvme_ph_valid(ns, ph)) {
583673064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
583773064edfSJesper Devantier }
583873064edfSJesper Devantier
583973064edfSJesper Devantier ruhid = ns->fdp.phs[ph];
584073064edfSJesper Devantier ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
584173064edfSJesper Devantier
584273064edfSJesper Devantier assert(ruh);
584373064edfSJesper Devantier
584473064edfSJesper Devantier if (unlikely(noet == 0)) {
584573064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
584673064edfSJesper Devantier }
584773064edfSJesper Devantier
584873064edfSJesper Devantier for (uint8_t event_type = 0; event_type < FDP_EVT_MAX; event_type++) {
584973064edfSJesper Devantier uint8_t shift = nvme_fdp_evf_shifts[event_type];
585073064edfSJesper Devantier if (!shift && event_type) {
585173064edfSJesper Devantier /*
585273064edfSJesper Devantier * only first entry (event_type == 0) has a shift value of 0
585373064edfSJesper Devantier * other entries are simply unpopulated.
585473064edfSJesper Devantier */
585573064edfSJesper Devantier continue;
585673064edfSJesper Devantier }
585773064edfSJesper Devantier
585873064edfSJesper Devantier nentries++;
585973064edfSJesper Devantier
586073064edfSJesper Devantier s_event = &s_events[s_events_ndx];
586173064edfSJesper Devantier s_event->evt = event_type;
586273064edfSJesper Devantier s_event->evta = (ruh->event_filter >> shift) & 0x1;
586373064edfSJesper Devantier
586473064edfSJesper Devantier /* break if all `noet` entries are filled */
586573064edfSJesper Devantier if ((++s_events_ndx) == noet) {
586673064edfSJesper Devantier break;
586773064edfSJesper Devantier }
586873064edfSJesper Devantier }
586973064edfSJesper Devantier
587073064edfSJesper Devantier ret = nvme_c2h(n, s_events, s_events_siz, req);
587173064edfSJesper Devantier if (ret) {
587273064edfSJesper Devantier return ret;
587373064edfSJesper Devantier }
587473064edfSJesper Devantier
587573064edfSJesper Devantier *result = nentries;
587673064edfSJesper Devantier return NVME_SUCCESS;
587773064edfSJesper Devantier }
587873064edfSJesper Devantier
nvme_get_feature(NvmeCtrl * n,NvmeRequest * req)587988eea45cSKlaus Jensen static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req)
588088eea45cSKlaus Jensen {
588188eea45cSKlaus Jensen NvmeCmd *cmd = &req->cmd;
588288eea45cSKlaus Jensen uint32_t dw10 = le32_to_cpu(cmd->cdw10);
588388eea45cSKlaus Jensen uint32_t dw11 = le32_to_cpu(cmd->cdw11);
588488eea45cSKlaus Jensen uint32_t nsid = le32_to_cpu(cmd->nsid);
588588eea45cSKlaus Jensen uint32_t result;
588688eea45cSKlaus Jensen uint8_t fid = NVME_GETSETFEAT_FID(dw10);
588788eea45cSKlaus Jensen NvmeGetFeatureSelect sel = NVME_GETFEAT_SELECT(dw10);
588888eea45cSKlaus Jensen uint16_t iv;
588988eea45cSKlaus Jensen NvmeNamespace *ns;
589088eea45cSKlaus Jensen int i;
589173064edfSJesper Devantier uint16_t endgrpid = 0, ret = NVME_SUCCESS;
589288eea45cSKlaus Jensen
589388eea45cSKlaus Jensen static const uint32_t nvme_feature_default[NVME_FID_MAX] = {
589488eea45cSKlaus Jensen [NVME_ARBITRATION] = NVME_ARB_AB_NOLIMIT,
589588eea45cSKlaus Jensen };
589688eea45cSKlaus Jensen
589788eea45cSKlaus Jensen trace_pci_nvme_getfeat(nvme_cid(req), nsid, fid, sel, dw11);
589888eea45cSKlaus Jensen
589988eea45cSKlaus Jensen if (!nvme_feature_support[fid]) {
590088eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
590188eea45cSKlaus Jensen }
590288eea45cSKlaus Jensen
590388eea45cSKlaus Jensen if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
590488eea45cSKlaus Jensen if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
590588eea45cSKlaus Jensen /*
590688eea45cSKlaus Jensen * The Reservation Notification Mask and Reservation Persistence
590788eea45cSKlaus Jensen * features require a status code of Invalid Field in Command when
590888eea45cSKlaus Jensen * NSID is FFFFFFFFh. Since the device does not support those
590988eea45cSKlaus Jensen * features we can always return Invalid Namespace or Format as we
591088eea45cSKlaus Jensen * should do for all other features.
591188eea45cSKlaus Jensen */
591288eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
591388eea45cSKlaus Jensen }
591488eea45cSKlaus Jensen
591588eea45cSKlaus Jensen if (!nvme_ns(n, nsid)) {
591688eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
591788eea45cSKlaus Jensen }
591888eea45cSKlaus Jensen }
591988eea45cSKlaus Jensen
592088eea45cSKlaus Jensen switch (sel) {
592188eea45cSKlaus Jensen case NVME_GETFEAT_SELECT_CURRENT:
592288eea45cSKlaus Jensen break;
592388eea45cSKlaus Jensen case NVME_GETFEAT_SELECT_SAVED:
592488eea45cSKlaus Jensen /* no features are saveable by the controller; fallthrough */
592588eea45cSKlaus Jensen case NVME_GETFEAT_SELECT_DEFAULT:
592688eea45cSKlaus Jensen goto defaults;
592788eea45cSKlaus Jensen case NVME_GETFEAT_SELECT_CAP:
592888eea45cSKlaus Jensen result = nvme_feature_cap[fid];
592988eea45cSKlaus Jensen goto out;
593088eea45cSKlaus Jensen }
593188eea45cSKlaus Jensen
593288eea45cSKlaus Jensen switch (fid) {
593388eea45cSKlaus Jensen case NVME_TEMPERATURE_THRESHOLD:
593488eea45cSKlaus Jensen result = 0;
593588eea45cSKlaus Jensen
593688eea45cSKlaus Jensen /*
593788eea45cSKlaus Jensen * The controller only implements the Composite Temperature sensor, so
593888eea45cSKlaus Jensen * return 0 for all other sensors.
593988eea45cSKlaus Jensen */
594088eea45cSKlaus Jensen if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
594188eea45cSKlaus Jensen goto out;
594288eea45cSKlaus Jensen }
594388eea45cSKlaus Jensen
594488eea45cSKlaus Jensen switch (NVME_TEMP_THSEL(dw11)) {
594588eea45cSKlaus Jensen case NVME_TEMP_THSEL_OVER:
594688eea45cSKlaus Jensen result = n->features.temp_thresh_hi;
594788eea45cSKlaus Jensen goto out;
594888eea45cSKlaus Jensen case NVME_TEMP_THSEL_UNDER:
594988eea45cSKlaus Jensen result = n->features.temp_thresh_low;
595088eea45cSKlaus Jensen goto out;
595188eea45cSKlaus Jensen }
595288eea45cSKlaus Jensen
595388eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
595488eea45cSKlaus Jensen case NVME_ERROR_RECOVERY:
595588eea45cSKlaus Jensen if (!nvme_nsid_valid(n, nsid)) {
595688eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
595788eea45cSKlaus Jensen }
595888eea45cSKlaus Jensen
595988eea45cSKlaus Jensen ns = nvme_ns(n, nsid);
596088eea45cSKlaus Jensen if (unlikely(!ns)) {
596188eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
596288eea45cSKlaus Jensen }
596388eea45cSKlaus Jensen
596488eea45cSKlaus Jensen result = ns->features.err_rec;
596588eea45cSKlaus Jensen goto out;
596688eea45cSKlaus Jensen case NVME_VOLATILE_WRITE_CACHE:
596788eea45cSKlaus Jensen result = 0;
596888eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
596988eea45cSKlaus Jensen ns = nvme_ns(n, i);
597088eea45cSKlaus Jensen if (!ns) {
597188eea45cSKlaus Jensen continue;
597288eea45cSKlaus Jensen }
597388eea45cSKlaus Jensen
597488eea45cSKlaus Jensen result = blk_enable_write_cache(ns->blkconf.blk);
597588eea45cSKlaus Jensen if (result) {
597688eea45cSKlaus Jensen break;
597788eea45cSKlaus Jensen }
597888eea45cSKlaus Jensen }
597988eea45cSKlaus Jensen trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
598088eea45cSKlaus Jensen goto out;
598188eea45cSKlaus Jensen case NVME_ASYNCHRONOUS_EVENT_CONF:
598288eea45cSKlaus Jensen result = n->features.async_config;
598388eea45cSKlaus Jensen goto out;
598488eea45cSKlaus Jensen case NVME_TIMESTAMP:
598588eea45cSKlaus Jensen return nvme_get_feature_timestamp(n, req);
5986d0c0697bSNaveen Nagar case NVME_HOST_BEHAVIOR_SUPPORT:
5987d0c0697bSNaveen Nagar return nvme_c2h(n, (uint8_t *)&n->features.hbs,
5988d0c0697bSNaveen Nagar sizeof(n->features.hbs), req);
598973064edfSJesper Devantier case NVME_FDP_MODE:
599073064edfSJesper Devantier endgrpid = dw11 & 0xff;
599173064edfSJesper Devantier
599273064edfSJesper Devantier if (endgrpid != 0x1) {
599373064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
599473064edfSJesper Devantier }
599573064edfSJesper Devantier
599673064edfSJesper Devantier ret = nvme_get_feature_fdp(n, endgrpid, &result);
599773064edfSJesper Devantier if (ret) {
599873064edfSJesper Devantier return ret;
599973064edfSJesper Devantier }
600073064edfSJesper Devantier goto out;
600173064edfSJesper Devantier case NVME_FDP_EVENTS:
600273064edfSJesper Devantier if (!nvme_nsid_valid(n, nsid)) {
600373064edfSJesper Devantier return NVME_INVALID_NSID | NVME_DNR;
600473064edfSJesper Devantier }
600573064edfSJesper Devantier
600673064edfSJesper Devantier ns = nvme_ns(n, nsid);
600773064edfSJesper Devantier if (unlikely(!ns)) {
600873064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
600973064edfSJesper Devantier }
601073064edfSJesper Devantier
601173064edfSJesper Devantier ret = nvme_get_feature_fdp_events(n, ns, req, &result);
601273064edfSJesper Devantier if (ret) {
601373064edfSJesper Devantier return ret;
601473064edfSJesper Devantier }
601573064edfSJesper Devantier goto out;
601688eea45cSKlaus Jensen default:
601788eea45cSKlaus Jensen break;
601888eea45cSKlaus Jensen }
601988eea45cSKlaus Jensen
602088eea45cSKlaus Jensen defaults:
602188eea45cSKlaus Jensen switch (fid) {
602288eea45cSKlaus Jensen case NVME_TEMPERATURE_THRESHOLD:
602388eea45cSKlaus Jensen result = 0;
602488eea45cSKlaus Jensen
602588eea45cSKlaus Jensen if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
602688eea45cSKlaus Jensen break;
602788eea45cSKlaus Jensen }
602888eea45cSKlaus Jensen
602988eea45cSKlaus Jensen if (NVME_TEMP_THSEL(dw11) == NVME_TEMP_THSEL_OVER) {
603088eea45cSKlaus Jensen result = NVME_TEMPERATURE_WARNING;
603188eea45cSKlaus Jensen }
603288eea45cSKlaus Jensen
603388eea45cSKlaus Jensen break;
603488eea45cSKlaus Jensen case NVME_NUMBER_OF_QUEUES:
6035decc0261SŁukasz Gieryk result = (n->conf_ioqpairs - 1) | ((n->conf_ioqpairs - 1) << 16);
603688eea45cSKlaus Jensen trace_pci_nvme_getfeat_numq(result);
603788eea45cSKlaus Jensen break;
603888eea45cSKlaus Jensen case NVME_INTERRUPT_VECTOR_CONF:
603988eea45cSKlaus Jensen iv = dw11 & 0xffff;
6040decc0261SŁukasz Gieryk if (iv >= n->conf_ioqpairs + 1) {
604188eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
604288eea45cSKlaus Jensen }
604388eea45cSKlaus Jensen
604488eea45cSKlaus Jensen result = iv;
604588eea45cSKlaus Jensen if (iv == n->admin_cq.vector) {
604688eea45cSKlaus Jensen result |= NVME_INTVC_NOCOALESCING;
604788eea45cSKlaus Jensen }
604888eea45cSKlaus Jensen break;
604973064edfSJesper Devantier case NVME_FDP_MODE:
605073064edfSJesper Devantier endgrpid = dw11 & 0xff;
605173064edfSJesper Devantier
605273064edfSJesper Devantier if (endgrpid != 0x1) {
605373064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
605473064edfSJesper Devantier }
605573064edfSJesper Devantier
605673064edfSJesper Devantier ret = nvme_get_feature_fdp(n, endgrpid, &result);
605773064edfSJesper Devantier if (ret) {
605873064edfSJesper Devantier return ret;
605973064edfSJesper Devantier }
606073064edfSJesper Devantier goto out;
606173064edfSJesper Devantier
606273064edfSJesper Devantier break;
606388eea45cSKlaus Jensen default:
606488eea45cSKlaus Jensen result = nvme_feature_default[fid];
606588eea45cSKlaus Jensen break;
606688eea45cSKlaus Jensen }
606788eea45cSKlaus Jensen
606888eea45cSKlaus Jensen out:
606988eea45cSKlaus Jensen req->cqe.result = cpu_to_le32(result);
607073064edfSJesper Devantier return ret;
607188eea45cSKlaus Jensen }
607288eea45cSKlaus Jensen
nvme_set_feature_timestamp(NvmeCtrl * n,NvmeRequest * req)607388eea45cSKlaus Jensen static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)
607488eea45cSKlaus Jensen {
607588eea45cSKlaus Jensen uint16_t ret;
607688eea45cSKlaus Jensen uint64_t timestamp;
607788eea45cSKlaus Jensen
607888eea45cSKlaus Jensen ret = nvme_h2c(n, (uint8_t *)×tamp, sizeof(timestamp), req);
607988eea45cSKlaus Jensen if (ret) {
608088eea45cSKlaus Jensen return ret;
608188eea45cSKlaus Jensen }
608288eea45cSKlaus Jensen
608388eea45cSKlaus Jensen nvme_set_timestamp(n, timestamp);
608488eea45cSKlaus Jensen
608588eea45cSKlaus Jensen return NVME_SUCCESS;
608688eea45cSKlaus Jensen }
608788eea45cSKlaus Jensen
nvme_set_feature_fdp_events(NvmeCtrl * n,NvmeNamespace * ns,NvmeRequest * req)608873064edfSJesper Devantier static uint16_t nvme_set_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,
608973064edfSJesper Devantier NvmeRequest *req)
609073064edfSJesper Devantier {
609173064edfSJesper Devantier NvmeCmd *cmd = &req->cmd;
609273064edfSJesper Devantier uint32_t cdw11 = le32_to_cpu(cmd->cdw11);
609373064edfSJesper Devantier uint16_t ph = cdw11 & 0xffff;
609473064edfSJesper Devantier uint8_t noet = (cdw11 >> 16) & 0xff;
609573064edfSJesper Devantier uint16_t ret, ruhid;
609673064edfSJesper Devantier uint8_t enable = le32_to_cpu(cmd->cdw12) & 0x1;
609773064edfSJesper Devantier uint8_t event_mask = 0;
609873064edfSJesper Devantier unsigned int i;
609973064edfSJesper Devantier g_autofree uint8_t *events = g_malloc0(noet);
610073064edfSJesper Devantier NvmeRuHandle *ruh = NULL;
610173064edfSJesper Devantier
610273064edfSJesper Devantier assert(ns);
610373064edfSJesper Devantier
610473064edfSJesper Devantier if (!n->subsys || !n->subsys->endgrp.fdp.enabled) {
610573064edfSJesper Devantier return NVME_FDP_DISABLED | NVME_DNR;
610673064edfSJesper Devantier }
610773064edfSJesper Devantier
610873064edfSJesper Devantier if (!nvme_ph_valid(ns, ph)) {
610973064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
611073064edfSJesper Devantier }
611173064edfSJesper Devantier
611273064edfSJesper Devantier ruhid = ns->fdp.phs[ph];
611373064edfSJesper Devantier ruh = &n->subsys->endgrp.fdp.ruhs[ruhid];
611473064edfSJesper Devantier
611573064edfSJesper Devantier ret = nvme_h2c(n, events, noet, req);
611673064edfSJesper Devantier if (ret) {
611773064edfSJesper Devantier return ret;
611873064edfSJesper Devantier }
611973064edfSJesper Devantier
612073064edfSJesper Devantier for (i = 0; i < noet; i++) {
612173064edfSJesper Devantier event_mask |= (1 << nvme_fdp_evf_shifts[events[i]]);
612273064edfSJesper Devantier }
612373064edfSJesper Devantier
612473064edfSJesper Devantier if (enable) {
612573064edfSJesper Devantier ruh->event_filter |= event_mask;
612673064edfSJesper Devantier } else {
612773064edfSJesper Devantier ruh->event_filter = ruh->event_filter & ~event_mask;
612873064edfSJesper Devantier }
612973064edfSJesper Devantier
613073064edfSJesper Devantier return NVME_SUCCESS;
613173064edfSJesper Devantier }
613273064edfSJesper Devantier
nvme_set_feature(NvmeCtrl * n,NvmeRequest * req)613388eea45cSKlaus Jensen static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req)
613488eea45cSKlaus Jensen {
613588eea45cSKlaus Jensen NvmeNamespace *ns = NULL;
613688eea45cSKlaus Jensen
613788eea45cSKlaus Jensen NvmeCmd *cmd = &req->cmd;
613888eea45cSKlaus Jensen uint32_t dw10 = le32_to_cpu(cmd->cdw10);
613988eea45cSKlaus Jensen uint32_t dw11 = le32_to_cpu(cmd->cdw11);
614088eea45cSKlaus Jensen uint32_t nsid = le32_to_cpu(cmd->nsid);
614188eea45cSKlaus Jensen uint8_t fid = NVME_GETSETFEAT_FID(dw10);
614288eea45cSKlaus Jensen uint8_t save = NVME_SETFEAT_SAVE(dw10);
6143763c05dfSNaveen Nagar uint16_t status;
614488eea45cSKlaus Jensen int i;
614588eea45cSKlaus Jensen
614688eea45cSKlaus Jensen trace_pci_nvme_setfeat(nvme_cid(req), nsid, fid, save, dw11);
614788eea45cSKlaus Jensen
614888eea45cSKlaus Jensen if (save && !(nvme_feature_cap[fid] & NVME_FEAT_CAP_SAVE)) {
614988eea45cSKlaus Jensen return NVME_FID_NOT_SAVEABLE | NVME_DNR;
615088eea45cSKlaus Jensen }
615188eea45cSKlaus Jensen
615288eea45cSKlaus Jensen if (!nvme_feature_support[fid]) {
615388eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
615488eea45cSKlaus Jensen }
615588eea45cSKlaus Jensen
615688eea45cSKlaus Jensen if (nvme_feature_cap[fid] & NVME_FEAT_CAP_NS) {
615788eea45cSKlaus Jensen if (nsid != NVME_NSID_BROADCAST) {
615888eea45cSKlaus Jensen if (!nvme_nsid_valid(n, nsid)) {
615988eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
616088eea45cSKlaus Jensen }
616188eea45cSKlaus Jensen
616288eea45cSKlaus Jensen ns = nvme_ns(n, nsid);
616388eea45cSKlaus Jensen if (unlikely(!ns)) {
616488eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
616588eea45cSKlaus Jensen }
616688eea45cSKlaus Jensen }
616788eea45cSKlaus Jensen } else if (nsid && nsid != NVME_NSID_BROADCAST) {
616888eea45cSKlaus Jensen if (!nvme_nsid_valid(n, nsid)) {
616988eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
617088eea45cSKlaus Jensen }
617188eea45cSKlaus Jensen
617288eea45cSKlaus Jensen return NVME_FEAT_NOT_NS_SPEC | NVME_DNR;
617388eea45cSKlaus Jensen }
617488eea45cSKlaus Jensen
617588eea45cSKlaus Jensen if (!(nvme_feature_cap[fid] & NVME_FEAT_CAP_CHANGE)) {
617688eea45cSKlaus Jensen return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
617788eea45cSKlaus Jensen }
617888eea45cSKlaus Jensen
617988eea45cSKlaus Jensen switch (fid) {
618088eea45cSKlaus Jensen case NVME_TEMPERATURE_THRESHOLD:
618188eea45cSKlaus Jensen if (NVME_TEMP_TMPSEL(dw11) != NVME_TEMP_TMPSEL_COMPOSITE) {
618288eea45cSKlaus Jensen break;
618388eea45cSKlaus Jensen }
618488eea45cSKlaus Jensen
618588eea45cSKlaus Jensen switch (NVME_TEMP_THSEL(dw11)) {
618688eea45cSKlaus Jensen case NVME_TEMP_THSEL_OVER:
618788eea45cSKlaus Jensen n->features.temp_thresh_hi = NVME_TEMP_TMPTH(dw11);
618888eea45cSKlaus Jensen break;
618988eea45cSKlaus Jensen case NVME_TEMP_THSEL_UNDER:
619088eea45cSKlaus Jensen n->features.temp_thresh_low = NVME_TEMP_TMPTH(dw11);
619188eea45cSKlaus Jensen break;
619288eea45cSKlaus Jensen default:
619388eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
619488eea45cSKlaus Jensen }
619588eea45cSKlaus Jensen
619688eea45cSKlaus Jensen if ((n->temperature >= n->features.temp_thresh_hi) ||
619788eea45cSKlaus Jensen (n->temperature <= n->features.temp_thresh_low)) {
61989235a72aSKlaus Jensen nvme_smart_event(n, NVME_SMART_TEMPERATURE);
619988eea45cSKlaus Jensen }
620088eea45cSKlaus Jensen
620188eea45cSKlaus Jensen break;
620288eea45cSKlaus Jensen case NVME_ERROR_RECOVERY:
620388eea45cSKlaus Jensen if (nsid == NVME_NSID_BROADCAST) {
620488eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
620588eea45cSKlaus Jensen ns = nvme_ns(n, i);
620688eea45cSKlaus Jensen
620788eea45cSKlaus Jensen if (!ns) {
620888eea45cSKlaus Jensen continue;
620988eea45cSKlaus Jensen }
621088eea45cSKlaus Jensen
621188eea45cSKlaus Jensen if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
621288eea45cSKlaus Jensen ns->features.err_rec = dw11;
621388eea45cSKlaus Jensen }
621488eea45cSKlaus Jensen }
621588eea45cSKlaus Jensen
621688eea45cSKlaus Jensen break;
621788eea45cSKlaus Jensen }
621888eea45cSKlaus Jensen
621988eea45cSKlaus Jensen assert(ns);
622088eea45cSKlaus Jensen if (NVME_ID_NS_NSFEAT_DULBE(ns->id_ns.nsfeat)) {
622188eea45cSKlaus Jensen ns->features.err_rec = dw11;
622288eea45cSKlaus Jensen }
622388eea45cSKlaus Jensen break;
622488eea45cSKlaus Jensen case NVME_VOLATILE_WRITE_CACHE:
622588eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
622688eea45cSKlaus Jensen ns = nvme_ns(n, i);
622788eea45cSKlaus Jensen if (!ns) {
622888eea45cSKlaus Jensen continue;
622988eea45cSKlaus Jensen }
623088eea45cSKlaus Jensen
623188eea45cSKlaus Jensen if (!(dw11 & 0x1) && blk_enable_write_cache(ns->blkconf.blk)) {
623288eea45cSKlaus Jensen blk_flush(ns->blkconf.blk);
623388eea45cSKlaus Jensen }
623488eea45cSKlaus Jensen
623588eea45cSKlaus Jensen blk_set_enable_write_cache(ns->blkconf.blk, dw11 & 1);
623688eea45cSKlaus Jensen }
623788eea45cSKlaus Jensen
623888eea45cSKlaus Jensen break;
623988eea45cSKlaus Jensen
624088eea45cSKlaus Jensen case NVME_NUMBER_OF_QUEUES:
624188eea45cSKlaus Jensen if (n->qs_created) {
624288eea45cSKlaus Jensen return NVME_CMD_SEQ_ERROR | NVME_DNR;
624388eea45cSKlaus Jensen }
624488eea45cSKlaus Jensen
624588eea45cSKlaus Jensen /*
624688eea45cSKlaus Jensen * NVMe v1.3, Section 5.21.1.7: FFFFh is not an allowed value for NCQR
624788eea45cSKlaus Jensen * and NSQR.
624888eea45cSKlaus Jensen */
624988eea45cSKlaus Jensen if ((dw11 & 0xffff) == 0xffff || ((dw11 >> 16) & 0xffff) == 0xffff) {
625088eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
625188eea45cSKlaus Jensen }
625288eea45cSKlaus Jensen
625388eea45cSKlaus Jensen trace_pci_nvme_setfeat_numq((dw11 & 0xffff) + 1,
625488eea45cSKlaus Jensen ((dw11 >> 16) & 0xffff) + 1,
6255decc0261SŁukasz Gieryk n->conf_ioqpairs,
6256decc0261SŁukasz Gieryk n->conf_ioqpairs);
6257decc0261SŁukasz Gieryk req->cqe.result = cpu_to_le32((n->conf_ioqpairs - 1) |
6258decc0261SŁukasz Gieryk ((n->conf_ioqpairs - 1) << 16));
625988eea45cSKlaus Jensen break;
626088eea45cSKlaus Jensen case NVME_ASYNCHRONOUS_EVENT_CONF:
626188eea45cSKlaus Jensen n->features.async_config = dw11;
626288eea45cSKlaus Jensen break;
626388eea45cSKlaus Jensen case NVME_TIMESTAMP:
626488eea45cSKlaus Jensen return nvme_set_feature_timestamp(n, req);
6265d0c0697bSNaveen Nagar case NVME_HOST_BEHAVIOR_SUPPORT:
6266763c05dfSNaveen Nagar status = nvme_h2c(n, (uint8_t *)&n->features.hbs,
6267d0c0697bSNaveen Nagar sizeof(n->features.hbs), req);
6268763c05dfSNaveen Nagar if (status) {
6269763c05dfSNaveen Nagar return status;
6270763c05dfSNaveen Nagar }
6271763c05dfSNaveen Nagar
6272763c05dfSNaveen Nagar for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
6273763c05dfSNaveen Nagar ns = nvme_ns(n, i);
6274763c05dfSNaveen Nagar
6275763c05dfSNaveen Nagar if (!ns) {
6276763c05dfSNaveen Nagar continue;
6277763c05dfSNaveen Nagar }
6278763c05dfSNaveen Nagar
6279763c05dfSNaveen Nagar ns->id_ns.nlbaf = ns->nlbaf - 1;
6280763c05dfSNaveen Nagar if (!n->features.hbs.lbafee) {
6281763c05dfSNaveen Nagar ns->id_ns.nlbaf = MIN(ns->id_ns.nlbaf, 15);
6282763c05dfSNaveen Nagar }
6283763c05dfSNaveen Nagar }
6284763c05dfSNaveen Nagar
6285763c05dfSNaveen Nagar return status;
628688eea45cSKlaus Jensen case NVME_COMMAND_SET_PROFILE:
628788eea45cSKlaus Jensen if (dw11 & 0x1ff) {
628888eea45cSKlaus Jensen trace_pci_nvme_err_invalid_iocsci(dw11 & 0x1ff);
628988eea45cSKlaus Jensen return NVME_CMD_SET_CMB_REJECTED | NVME_DNR;
629088eea45cSKlaus Jensen }
629188eea45cSKlaus Jensen break;
629273064edfSJesper Devantier case NVME_FDP_MODE:
629373064edfSJesper Devantier /* spec: abort with cmd seq err if there's one or more NS' in endgrp */
629473064edfSJesper Devantier return NVME_CMD_SEQ_ERROR | NVME_DNR;
629573064edfSJesper Devantier case NVME_FDP_EVENTS:
629673064edfSJesper Devantier return nvme_set_feature_fdp_events(n, ns, req);
629788eea45cSKlaus Jensen default:
629888eea45cSKlaus Jensen return NVME_FEAT_NOT_CHANGEABLE | NVME_DNR;
629988eea45cSKlaus Jensen }
630088eea45cSKlaus Jensen return NVME_SUCCESS;
630188eea45cSKlaus Jensen }
630288eea45cSKlaus Jensen
nvme_aer(NvmeCtrl * n,NvmeRequest * req)630388eea45cSKlaus Jensen static uint16_t nvme_aer(NvmeCtrl *n, NvmeRequest *req)
630488eea45cSKlaus Jensen {
630588eea45cSKlaus Jensen trace_pci_nvme_aer(nvme_cid(req));
630688eea45cSKlaus Jensen
630788eea45cSKlaus Jensen if (n->outstanding_aers > n->params.aerl) {
630888eea45cSKlaus Jensen trace_pci_nvme_aer_aerl_exceeded();
630988eea45cSKlaus Jensen return NVME_AER_LIMIT_EXCEEDED;
631088eea45cSKlaus Jensen }
631188eea45cSKlaus Jensen
631288eea45cSKlaus Jensen n->aer_reqs[n->outstanding_aers] = req;
631388eea45cSKlaus Jensen n->outstanding_aers++;
631488eea45cSKlaus Jensen
631588eea45cSKlaus Jensen if (!QTAILQ_EMPTY(&n->aer_queue)) {
631688eea45cSKlaus Jensen nvme_process_aers(n);
631788eea45cSKlaus Jensen }
631888eea45cSKlaus Jensen
631988eea45cSKlaus Jensen return NVME_NO_COMPLETE;
632088eea45cSKlaus Jensen }
632188eea45cSKlaus Jensen
nvme_update_dmrsl(NvmeCtrl * n)632288eea45cSKlaus Jensen static void nvme_update_dmrsl(NvmeCtrl *n)
632388eea45cSKlaus Jensen {
632488eea45cSKlaus Jensen int nsid;
632588eea45cSKlaus Jensen
632688eea45cSKlaus Jensen for (nsid = 1; nsid <= NVME_MAX_NAMESPACES; nsid++) {
632788eea45cSKlaus Jensen NvmeNamespace *ns = nvme_ns(n, nsid);
632888eea45cSKlaus Jensen if (!ns) {
632988eea45cSKlaus Jensen continue;
633088eea45cSKlaus Jensen }
633188eea45cSKlaus Jensen
633288eea45cSKlaus Jensen n->dmrsl = MIN_NON_ZERO(n->dmrsl,
633388eea45cSKlaus Jensen BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
633488eea45cSKlaus Jensen }
633588eea45cSKlaus Jensen }
633688eea45cSKlaus Jensen
nvme_select_iocs_ns(NvmeCtrl * n,NvmeNamespace * ns)633788eea45cSKlaus Jensen static void nvme_select_iocs_ns(NvmeCtrl *n, NvmeNamespace *ns)
633888eea45cSKlaus Jensen {
633949e03457SKlaus Jensen uint32_t cc = ldl_le_p(&n->bar.cc);
634049e03457SKlaus Jensen
634188eea45cSKlaus Jensen ns->iocs = nvme_cse_iocs_none;
634288eea45cSKlaus Jensen switch (ns->csi) {
634388eea45cSKlaus Jensen case NVME_CSI_NVM:
634449e03457SKlaus Jensen if (NVME_CC_CSS(cc) != NVME_CC_CSS_ADMIN_ONLY) {
634588eea45cSKlaus Jensen ns->iocs = nvme_cse_iocs_nvm;
634688eea45cSKlaus Jensen }
634788eea45cSKlaus Jensen break;
634888eea45cSKlaus Jensen case NVME_CSI_ZONED:
634949e03457SKlaus Jensen if (NVME_CC_CSS(cc) == NVME_CC_CSS_CSI) {
635088eea45cSKlaus Jensen ns->iocs = nvme_cse_iocs_zoned;
635149e03457SKlaus Jensen } else if (NVME_CC_CSS(cc) == NVME_CC_CSS_NVM) {
635288eea45cSKlaus Jensen ns->iocs = nvme_cse_iocs_nvm;
635388eea45cSKlaus Jensen }
635488eea45cSKlaus Jensen break;
635588eea45cSKlaus Jensen }
635688eea45cSKlaus Jensen }
635788eea45cSKlaus Jensen
nvme_ns_attachment(NvmeCtrl * n,NvmeRequest * req)635888eea45cSKlaus Jensen static uint16_t nvme_ns_attachment(NvmeCtrl *n, NvmeRequest *req)
635988eea45cSKlaus Jensen {
636088eea45cSKlaus Jensen NvmeNamespace *ns;
636188eea45cSKlaus Jensen NvmeCtrl *ctrl;
636288eea45cSKlaus Jensen uint16_t list[NVME_CONTROLLER_LIST_SIZE] = {};
636388eea45cSKlaus Jensen uint32_t nsid = le32_to_cpu(req->cmd.nsid);
636488eea45cSKlaus Jensen uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
636507a3dfa7SNaveen Nagar uint8_t sel = dw10 & 0xf;
636688eea45cSKlaus Jensen uint16_t *nr_ids = &list[0];
636788eea45cSKlaus Jensen uint16_t *ids = &list[1];
636888eea45cSKlaus Jensen uint16_t ret;
636988eea45cSKlaus Jensen int i;
637088eea45cSKlaus Jensen
637188eea45cSKlaus Jensen trace_pci_nvme_ns_attachment(nvme_cid(req), dw10 & 0xf);
637288eea45cSKlaus Jensen
637388eea45cSKlaus Jensen if (!nvme_nsid_valid(n, nsid)) {
637488eea45cSKlaus Jensen return NVME_INVALID_NSID | NVME_DNR;
637588eea45cSKlaus Jensen }
637688eea45cSKlaus Jensen
637788eea45cSKlaus Jensen ns = nvme_subsys_ns(n->subsys, nsid);
637888eea45cSKlaus Jensen if (!ns) {
637988eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
638088eea45cSKlaus Jensen }
638188eea45cSKlaus Jensen
638288eea45cSKlaus Jensen ret = nvme_h2c(n, (uint8_t *)list, 4096, req);
638388eea45cSKlaus Jensen if (ret) {
638488eea45cSKlaus Jensen return ret;
638588eea45cSKlaus Jensen }
638688eea45cSKlaus Jensen
638788eea45cSKlaus Jensen if (!*nr_ids) {
638888eea45cSKlaus Jensen return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
638988eea45cSKlaus Jensen }
639088eea45cSKlaus Jensen
639188eea45cSKlaus Jensen *nr_ids = MIN(*nr_ids, NVME_CONTROLLER_LIST_SIZE - 1);
639288eea45cSKlaus Jensen for (i = 0; i < *nr_ids; i++) {
639388eea45cSKlaus Jensen ctrl = nvme_subsys_ctrl(n->subsys, ids[i]);
639488eea45cSKlaus Jensen if (!ctrl) {
639588eea45cSKlaus Jensen return NVME_NS_CTRL_LIST_INVALID | NVME_DNR;
639688eea45cSKlaus Jensen }
639788eea45cSKlaus Jensen
639807a3dfa7SNaveen Nagar switch (sel) {
639907a3dfa7SNaveen Nagar case NVME_NS_ATTACHMENT_ATTACH:
640088eea45cSKlaus Jensen if (nvme_ns(ctrl, nsid)) {
640188eea45cSKlaus Jensen return NVME_NS_ALREADY_ATTACHED | NVME_DNR;
640288eea45cSKlaus Jensen }
640388eea45cSKlaus Jensen
640488eea45cSKlaus Jensen if (ns->attached && !ns->params.shared) {
640588eea45cSKlaus Jensen return NVME_NS_PRIVATE | NVME_DNR;
640688eea45cSKlaus Jensen }
640788eea45cSKlaus Jensen
640888eea45cSKlaus Jensen nvme_attach_ns(ctrl, ns);
640988eea45cSKlaus Jensen nvme_select_iocs_ns(ctrl, ns);
641007a3dfa7SNaveen Nagar
641107a3dfa7SNaveen Nagar break;
641207a3dfa7SNaveen Nagar
641307a3dfa7SNaveen Nagar case NVME_NS_ATTACHMENT_DETACH:
641488eea45cSKlaus Jensen if (!nvme_ns(ctrl, nsid)) {
641588eea45cSKlaus Jensen return NVME_NS_NOT_ATTACHED | NVME_DNR;
641688eea45cSKlaus Jensen }
641788eea45cSKlaus Jensen
641888eea45cSKlaus Jensen ctrl->namespaces[nsid] = NULL;
641988eea45cSKlaus Jensen ns->attached--;
642088eea45cSKlaus Jensen
642188eea45cSKlaus Jensen nvme_update_dmrsl(ctrl);
642207a3dfa7SNaveen Nagar
642307a3dfa7SNaveen Nagar break;
642407a3dfa7SNaveen Nagar
642507a3dfa7SNaveen Nagar default:
642607a3dfa7SNaveen Nagar return NVME_INVALID_FIELD | NVME_DNR;
642788eea45cSKlaus Jensen }
642888eea45cSKlaus Jensen
642988eea45cSKlaus Jensen /*
643088eea45cSKlaus Jensen * Add namespace id to the changed namespace id list for event clearing
643188eea45cSKlaus Jensen * via Get Log Page command.
643288eea45cSKlaus Jensen */
643388eea45cSKlaus Jensen if (!test_and_set_bit(nsid, ctrl->changed_nsids)) {
643488eea45cSKlaus Jensen nvme_enqueue_event(ctrl, NVME_AER_TYPE_NOTICE,
643588eea45cSKlaus Jensen NVME_AER_INFO_NOTICE_NS_ATTR_CHANGED,
643688eea45cSKlaus Jensen NVME_LOG_CHANGED_NSLIST);
643788eea45cSKlaus Jensen }
643888eea45cSKlaus Jensen }
643988eea45cSKlaus Jensen
644088eea45cSKlaus Jensen return NVME_SUCCESS;
644188eea45cSKlaus Jensen }
644288eea45cSKlaus Jensen
64433bcf26d3SKlaus Jensen typedef struct NvmeFormatAIOCB {
64443bcf26d3SKlaus Jensen BlockAIOCB common;
64453bcf26d3SKlaus Jensen BlockAIOCB *aiocb;
64463bcf26d3SKlaus Jensen NvmeRequest *req;
64473bcf26d3SKlaus Jensen int ret;
644888eea45cSKlaus Jensen
64493bcf26d3SKlaus Jensen NvmeNamespace *ns;
64503bcf26d3SKlaus Jensen uint32_t nsid;
64513bcf26d3SKlaus Jensen bool broadcast;
64523bcf26d3SKlaus Jensen int64_t offset;
6453a6de6ed5SKlaus Jensen
6454a6de6ed5SKlaus Jensen uint8_t lbaf;
6455a6de6ed5SKlaus Jensen uint8_t mset;
6456a6de6ed5SKlaus Jensen uint8_t pi;
6457a6de6ed5SKlaus Jensen uint8_t pil;
64583bcf26d3SKlaus Jensen } NvmeFormatAIOCB;
64593bcf26d3SKlaus Jensen
nvme_format_cancel(BlockAIOCB * aiocb)64603bcf26d3SKlaus Jensen static void nvme_format_cancel(BlockAIOCB *aiocb)
64613bcf26d3SKlaus Jensen {
64623bcf26d3SKlaus Jensen NvmeFormatAIOCB *iocb = container_of(aiocb, NvmeFormatAIOCB, common);
64633bcf26d3SKlaus Jensen
6464433c71e4SKlaus Jensen iocb->ret = -ECANCELED;
6465433c71e4SKlaus Jensen
64663bcf26d3SKlaus Jensen if (iocb->aiocb) {
64673bcf26d3SKlaus Jensen blk_aio_cancel_async(iocb->aiocb);
6468433c71e4SKlaus Jensen iocb->aiocb = NULL;
64693bcf26d3SKlaus Jensen }
64703bcf26d3SKlaus Jensen }
64713bcf26d3SKlaus Jensen
64723bcf26d3SKlaus Jensen static const AIOCBInfo nvme_format_aiocb_info = {
64733bcf26d3SKlaus Jensen .aiocb_size = sizeof(NvmeFormatAIOCB),
64743bcf26d3SKlaus Jensen .cancel_async = nvme_format_cancel,
64753bcf26d3SKlaus Jensen };
64763bcf26d3SKlaus Jensen
nvme_format_set(NvmeNamespace * ns,uint8_t lbaf,uint8_t mset,uint8_t pi,uint8_t pil)6477a6de6ed5SKlaus Jensen static void nvme_format_set(NvmeNamespace *ns, uint8_t lbaf, uint8_t mset,
6478a6de6ed5SKlaus Jensen uint8_t pi, uint8_t pil)
64793bcf26d3SKlaus Jensen {
6480763c05dfSNaveen Nagar uint8_t lbafl = lbaf & 0xf;
6481763c05dfSNaveen Nagar uint8_t lbafu = lbaf >> 4;
6482763c05dfSNaveen Nagar
64833bcf26d3SKlaus Jensen trace_pci_nvme_format_set(ns->params.nsid, lbaf, mset, pi, pil);
64843bcf26d3SKlaus Jensen
64853bcf26d3SKlaus Jensen ns->id_ns.dps = (pil << 3) | pi;
6486763c05dfSNaveen Nagar ns->id_ns.flbas = (lbafu << 5) | (mset << 4) | lbafl;
64873bcf26d3SKlaus Jensen
64883bcf26d3SKlaus Jensen nvme_ns_init_format(ns);
64893bcf26d3SKlaus Jensen }
64903bcf26d3SKlaus Jensen
6491433c71e4SKlaus Jensen static void nvme_do_format(NvmeFormatAIOCB *iocb);
6492433c71e4SKlaus Jensen
nvme_format_ns_cb(void * opaque,int ret)64933bcf26d3SKlaus Jensen static void nvme_format_ns_cb(void *opaque, int ret)
64943bcf26d3SKlaus Jensen {
64953bcf26d3SKlaus Jensen NvmeFormatAIOCB *iocb = opaque;
64963bcf26d3SKlaus Jensen NvmeNamespace *ns = iocb->ns;
64973bcf26d3SKlaus Jensen int bytes;
64983bcf26d3SKlaus Jensen
6499433c71e4SKlaus Jensen if (iocb->ret < 0) {
6500433c71e4SKlaus Jensen goto done;
6501433c71e4SKlaus Jensen } else if (ret < 0) {
65023bcf26d3SKlaus Jensen iocb->ret = ret;
65033bcf26d3SKlaus Jensen goto done;
65043bcf26d3SKlaus Jensen }
65053bcf26d3SKlaus Jensen
65063bcf26d3SKlaus Jensen assert(ns);
65073bcf26d3SKlaus Jensen
65083bcf26d3SKlaus Jensen if (iocb->offset < ns->size) {
65093bcf26d3SKlaus Jensen bytes = MIN(BDRV_REQUEST_MAX_BYTES, ns->size - iocb->offset);
65103bcf26d3SKlaus Jensen
65113bcf26d3SKlaus Jensen iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, iocb->offset,
65123bcf26d3SKlaus Jensen bytes, BDRV_REQ_MAY_UNMAP,
65133bcf26d3SKlaus Jensen nvme_format_ns_cb, iocb);
65143bcf26d3SKlaus Jensen
65153bcf26d3SKlaus Jensen iocb->offset += bytes;
65163bcf26d3SKlaus Jensen return;
65173bcf26d3SKlaus Jensen }
65183bcf26d3SKlaus Jensen
6519a6de6ed5SKlaus Jensen nvme_format_set(ns, iocb->lbaf, iocb->mset, iocb->pi, iocb->pil);
65203bcf26d3SKlaus Jensen ns->status = 0x0;
65213bcf26d3SKlaus Jensen iocb->ns = NULL;
65223bcf26d3SKlaus Jensen iocb->offset = 0;
65233bcf26d3SKlaus Jensen
65243bcf26d3SKlaus Jensen done:
6525433c71e4SKlaus Jensen nvme_do_format(iocb);
65263bcf26d3SKlaus Jensen }
65273bcf26d3SKlaus Jensen
nvme_format_check(NvmeNamespace * ns,uint8_t lbaf,uint8_t pi)65283bcf26d3SKlaus Jensen static uint16_t nvme_format_check(NvmeNamespace *ns, uint8_t lbaf, uint8_t pi)
65293bcf26d3SKlaus Jensen {
653088eea45cSKlaus Jensen if (ns->params.zoned) {
653188eea45cSKlaus Jensen return NVME_INVALID_FORMAT | NVME_DNR;
653288eea45cSKlaus Jensen }
653388eea45cSKlaus Jensen
653488eea45cSKlaus Jensen if (lbaf > ns->id_ns.nlbaf) {
653588eea45cSKlaus Jensen return NVME_INVALID_FORMAT | NVME_DNR;
653688eea45cSKlaus Jensen }
653788eea45cSKlaus Jensen
6538ac0b34c5SKlaus Jensen if (pi && (ns->id_ns.lbaf[lbaf].ms < nvme_pi_tuple_size(ns))) {
653988eea45cSKlaus Jensen return NVME_INVALID_FORMAT | NVME_DNR;
654088eea45cSKlaus Jensen }
654188eea45cSKlaus Jensen
654288eea45cSKlaus Jensen if (pi && pi > NVME_ID_NS_DPS_TYPE_3) {
654388eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
654488eea45cSKlaus Jensen }
654588eea45cSKlaus Jensen
654688eea45cSKlaus Jensen return NVME_SUCCESS;
654788eea45cSKlaus Jensen }
654888eea45cSKlaus Jensen
nvme_do_format(NvmeFormatAIOCB * iocb)6549433c71e4SKlaus Jensen static void nvme_do_format(NvmeFormatAIOCB *iocb)
655088eea45cSKlaus Jensen {
65513bcf26d3SKlaus Jensen NvmeRequest *req = iocb->req;
65523bcf26d3SKlaus Jensen NvmeCtrl *n = nvme_ctrl(req);
655344219b60SNaveen Nagar uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
655444219b60SNaveen Nagar uint8_t lbaf = dw10 & 0xf;
655544219b60SNaveen Nagar uint8_t pi = (dw10 >> 5) & 0x7;
655688eea45cSKlaus Jensen uint16_t status;
655788eea45cSKlaus Jensen int i;
655888eea45cSKlaus Jensen
65593bcf26d3SKlaus Jensen if (iocb->ret < 0) {
65603bcf26d3SKlaus Jensen goto done;
656188eea45cSKlaus Jensen }
656288eea45cSKlaus Jensen
65633bcf26d3SKlaus Jensen if (iocb->broadcast) {
65643bcf26d3SKlaus Jensen for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
65653bcf26d3SKlaus Jensen iocb->ns = nvme_ns(n, i);
65663bcf26d3SKlaus Jensen if (iocb->ns) {
65673bcf26d3SKlaus Jensen iocb->nsid = i;
656888eea45cSKlaus Jensen break;
656988eea45cSKlaus Jensen }
657088eea45cSKlaus Jensen }
657188eea45cSKlaus Jensen }
657288eea45cSKlaus Jensen
65733bcf26d3SKlaus Jensen if (!iocb->ns) {
65743bcf26d3SKlaus Jensen goto done;
657588eea45cSKlaus Jensen }
657688eea45cSKlaus Jensen
657744219b60SNaveen Nagar status = nvme_format_check(iocb->ns, lbaf, pi);
65783bcf26d3SKlaus Jensen if (status) {
65793bcf26d3SKlaus Jensen req->status = status;
65803bcf26d3SKlaus Jensen goto done;
65813bcf26d3SKlaus Jensen }
65823bcf26d3SKlaus Jensen
65833bcf26d3SKlaus Jensen iocb->ns->status = NVME_FORMAT_IN_PROGRESS;
65843bcf26d3SKlaus Jensen nvme_format_ns_cb(iocb, 0);
65853bcf26d3SKlaus Jensen return;
65863bcf26d3SKlaus Jensen
65873bcf26d3SKlaus Jensen done:
65883bcf26d3SKlaus Jensen iocb->common.cb(iocb->common.opaque, iocb->ret);
65893bcf26d3SKlaus Jensen qemu_aio_unref(iocb);
65903bcf26d3SKlaus Jensen }
65913bcf26d3SKlaus Jensen
nvme_format(NvmeCtrl * n,NvmeRequest * req)65923bcf26d3SKlaus Jensen static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req)
65933bcf26d3SKlaus Jensen {
65943bcf26d3SKlaus Jensen NvmeFormatAIOCB *iocb;
65953bcf26d3SKlaus Jensen uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6596a6de6ed5SKlaus Jensen uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6597a6de6ed5SKlaus Jensen uint8_t lbaf = dw10 & 0xf;
6598a6de6ed5SKlaus Jensen uint8_t mset = (dw10 >> 4) & 0x1;
6599a6de6ed5SKlaus Jensen uint8_t pi = (dw10 >> 5) & 0x7;
6600a6de6ed5SKlaus Jensen uint8_t pil = (dw10 >> 8) & 0x1;
6601763c05dfSNaveen Nagar uint8_t lbafu = (dw10 >> 12) & 0x3;
66023bcf26d3SKlaus Jensen uint16_t status;
66033bcf26d3SKlaus Jensen
66043bcf26d3SKlaus Jensen iocb = qemu_aio_get(&nvme_format_aiocb_info, NULL, nvme_misc_cb, req);
66053bcf26d3SKlaus Jensen
66063bcf26d3SKlaus Jensen iocb->req = req;
66073bcf26d3SKlaus Jensen iocb->ret = 0;
66083bcf26d3SKlaus Jensen iocb->ns = NULL;
66093bcf26d3SKlaus Jensen iocb->nsid = 0;
6610a6de6ed5SKlaus Jensen iocb->lbaf = lbaf;
6611a6de6ed5SKlaus Jensen iocb->mset = mset;
6612a6de6ed5SKlaus Jensen iocb->pi = pi;
6613a6de6ed5SKlaus Jensen iocb->pil = pil;
66143bcf26d3SKlaus Jensen iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
66153bcf26d3SKlaus Jensen iocb->offset = 0;
66163bcf26d3SKlaus Jensen
6617763c05dfSNaveen Nagar if (n->features.hbs.lbafee) {
6618763c05dfSNaveen Nagar iocb->lbaf |= lbafu << 4;
6619763c05dfSNaveen Nagar }
6620763c05dfSNaveen Nagar
66213bcf26d3SKlaus Jensen if (!iocb->broadcast) {
66223bcf26d3SKlaus Jensen if (!nvme_nsid_valid(n, nsid)) {
66233bcf26d3SKlaus Jensen status = NVME_INVALID_NSID | NVME_DNR;
66243bcf26d3SKlaus Jensen goto out;
66253bcf26d3SKlaus Jensen }
66263bcf26d3SKlaus Jensen
66273bcf26d3SKlaus Jensen iocb->ns = nvme_ns(n, nsid);
66283bcf26d3SKlaus Jensen if (!iocb->ns) {
66293bcf26d3SKlaus Jensen status = NVME_INVALID_FIELD | NVME_DNR;
66303bcf26d3SKlaus Jensen goto out;
66313bcf26d3SKlaus Jensen }
66323bcf26d3SKlaus Jensen }
66333bcf26d3SKlaus Jensen
66343bcf26d3SKlaus Jensen req->aiocb = &iocb->common;
6635433c71e4SKlaus Jensen nvme_do_format(iocb);
66363bcf26d3SKlaus Jensen
66373bcf26d3SKlaus Jensen return NVME_NO_COMPLETE;
66383bcf26d3SKlaus Jensen
66393bcf26d3SKlaus Jensen out:
66403bcf26d3SKlaus Jensen qemu_aio_unref(iocb);
6641433c71e4SKlaus Jensen
66423bcf26d3SKlaus Jensen return status;
664388eea45cSKlaus Jensen }
664488eea45cSKlaus Jensen
nvme_get_virt_res_num(NvmeCtrl * n,uint8_t rt,int * num_total,int * num_prim,int * num_sec)664511871f53SŁukasz Gieryk static void nvme_get_virt_res_num(NvmeCtrl *n, uint8_t rt, int *num_total,
664611871f53SŁukasz Gieryk int *num_prim, int *num_sec)
664711871f53SŁukasz Gieryk {
664811871f53SŁukasz Gieryk *num_total = le32_to_cpu(rt ?
664911871f53SŁukasz Gieryk n->pri_ctrl_cap.vifrt : n->pri_ctrl_cap.vqfrt);
665011871f53SŁukasz Gieryk *num_prim = le16_to_cpu(rt ?
665111871f53SŁukasz Gieryk n->pri_ctrl_cap.virfap : n->pri_ctrl_cap.vqrfap);
665211871f53SŁukasz Gieryk *num_sec = le16_to_cpu(rt ? n->pri_ctrl_cap.virfa : n->pri_ctrl_cap.vqrfa);
665311871f53SŁukasz Gieryk }
665411871f53SŁukasz Gieryk
nvme_assign_virt_res_to_prim(NvmeCtrl * n,NvmeRequest * req,uint16_t cntlid,uint8_t rt,int nr)665511871f53SŁukasz Gieryk static uint16_t nvme_assign_virt_res_to_prim(NvmeCtrl *n, NvmeRequest *req,
665611871f53SŁukasz Gieryk uint16_t cntlid, uint8_t rt,
665711871f53SŁukasz Gieryk int nr)
665811871f53SŁukasz Gieryk {
665911871f53SŁukasz Gieryk int num_total, num_prim, num_sec;
666011871f53SŁukasz Gieryk
666111871f53SŁukasz Gieryk if (cntlid != n->cntlid) {
666211871f53SŁukasz Gieryk return NVME_INVALID_CTRL_ID | NVME_DNR;
666311871f53SŁukasz Gieryk }
666411871f53SŁukasz Gieryk
666511871f53SŁukasz Gieryk nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
666611871f53SŁukasz Gieryk
666711871f53SŁukasz Gieryk if (nr > num_total) {
666811871f53SŁukasz Gieryk return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
666911871f53SŁukasz Gieryk }
667011871f53SŁukasz Gieryk
667111871f53SŁukasz Gieryk if (nr > num_total - num_sec) {
667211871f53SŁukasz Gieryk return NVME_INVALID_RESOURCE_ID | NVME_DNR;
667311871f53SŁukasz Gieryk }
667411871f53SŁukasz Gieryk
667511871f53SŁukasz Gieryk if (rt) {
667611871f53SŁukasz Gieryk n->next_pri_ctrl_cap.virfap = cpu_to_le16(nr);
667711871f53SŁukasz Gieryk } else {
667811871f53SŁukasz Gieryk n->next_pri_ctrl_cap.vqrfap = cpu_to_le16(nr);
667911871f53SŁukasz Gieryk }
668011871f53SŁukasz Gieryk
668111871f53SŁukasz Gieryk req->cqe.result = cpu_to_le32(nr);
668211871f53SŁukasz Gieryk return req->status;
668311871f53SŁukasz Gieryk }
668411871f53SŁukasz Gieryk
nvme_update_virt_res(NvmeCtrl * n,NvmeSecCtrlEntry * sctrl,uint8_t rt,int nr)668511871f53SŁukasz Gieryk static void nvme_update_virt_res(NvmeCtrl *n, NvmeSecCtrlEntry *sctrl,
668611871f53SŁukasz Gieryk uint8_t rt, int nr)
668711871f53SŁukasz Gieryk {
668811871f53SŁukasz Gieryk int prev_nr, prev_total;
668911871f53SŁukasz Gieryk
669011871f53SŁukasz Gieryk if (rt) {
669111871f53SŁukasz Gieryk prev_nr = le16_to_cpu(sctrl->nvi);
669211871f53SŁukasz Gieryk prev_total = le32_to_cpu(n->pri_ctrl_cap.virfa);
669311871f53SŁukasz Gieryk sctrl->nvi = cpu_to_le16(nr);
669411871f53SŁukasz Gieryk n->pri_ctrl_cap.virfa = cpu_to_le32(prev_total + nr - prev_nr);
669511871f53SŁukasz Gieryk } else {
669611871f53SŁukasz Gieryk prev_nr = le16_to_cpu(sctrl->nvq);
669711871f53SŁukasz Gieryk prev_total = le32_to_cpu(n->pri_ctrl_cap.vqrfa);
669811871f53SŁukasz Gieryk sctrl->nvq = cpu_to_le16(nr);
669911871f53SŁukasz Gieryk n->pri_ctrl_cap.vqrfa = cpu_to_le32(prev_total + nr - prev_nr);
670011871f53SŁukasz Gieryk }
670111871f53SŁukasz Gieryk }
670211871f53SŁukasz Gieryk
nvme_assign_virt_res_to_sec(NvmeCtrl * n,NvmeRequest * req,uint16_t cntlid,uint8_t rt,int nr)670311871f53SŁukasz Gieryk static uint16_t nvme_assign_virt_res_to_sec(NvmeCtrl *n, NvmeRequest *req,
670411871f53SŁukasz Gieryk uint16_t cntlid, uint8_t rt, int nr)
670511871f53SŁukasz Gieryk {
670611871f53SŁukasz Gieryk int num_total, num_prim, num_sec, num_free, diff, limit;
670711871f53SŁukasz Gieryk NvmeSecCtrlEntry *sctrl;
670811871f53SŁukasz Gieryk
670911871f53SŁukasz Gieryk sctrl = nvme_sctrl_for_cntlid(n, cntlid);
671011871f53SŁukasz Gieryk if (!sctrl) {
671111871f53SŁukasz Gieryk return NVME_INVALID_CTRL_ID | NVME_DNR;
671211871f53SŁukasz Gieryk }
671311871f53SŁukasz Gieryk
671411871f53SŁukasz Gieryk if (sctrl->scs) {
671511871f53SŁukasz Gieryk return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
671611871f53SŁukasz Gieryk }
671711871f53SŁukasz Gieryk
671811871f53SŁukasz Gieryk limit = le16_to_cpu(rt ? n->pri_ctrl_cap.vifrsm : n->pri_ctrl_cap.vqfrsm);
671911871f53SŁukasz Gieryk if (nr > limit) {
672011871f53SŁukasz Gieryk return NVME_INVALID_NUM_RESOURCES | NVME_DNR;
672111871f53SŁukasz Gieryk }
672211871f53SŁukasz Gieryk
672311871f53SŁukasz Gieryk nvme_get_virt_res_num(n, rt, &num_total, &num_prim, &num_sec);
672411871f53SŁukasz Gieryk num_free = num_total - num_prim - num_sec;
672511871f53SŁukasz Gieryk diff = nr - le16_to_cpu(rt ? sctrl->nvi : sctrl->nvq);
672611871f53SŁukasz Gieryk
672711871f53SŁukasz Gieryk if (diff > num_free) {
672811871f53SŁukasz Gieryk return NVME_INVALID_RESOURCE_ID | NVME_DNR;
672911871f53SŁukasz Gieryk }
673011871f53SŁukasz Gieryk
673111871f53SŁukasz Gieryk nvme_update_virt_res(n, sctrl, rt, nr);
673211871f53SŁukasz Gieryk req->cqe.result = cpu_to_le32(nr);
673311871f53SŁukasz Gieryk
673411871f53SŁukasz Gieryk return req->status;
673511871f53SŁukasz Gieryk }
673611871f53SŁukasz Gieryk
nvme_virt_set_state(NvmeCtrl * n,uint16_t cntlid,bool online)673711871f53SŁukasz Gieryk static uint16_t nvme_virt_set_state(NvmeCtrl *n, uint16_t cntlid, bool online)
673811871f53SŁukasz Gieryk {
673948b32c28SKlaus Jensen PCIDevice *pci = PCI_DEVICE(n);
674011871f53SŁukasz Gieryk NvmeCtrl *sn = NULL;
674111871f53SŁukasz Gieryk NvmeSecCtrlEntry *sctrl;
674211871f53SŁukasz Gieryk int vf_index;
674311871f53SŁukasz Gieryk
674411871f53SŁukasz Gieryk sctrl = nvme_sctrl_for_cntlid(n, cntlid);
674511871f53SŁukasz Gieryk if (!sctrl) {
674611871f53SŁukasz Gieryk return NVME_INVALID_CTRL_ID | NVME_DNR;
674711871f53SŁukasz Gieryk }
674811871f53SŁukasz Gieryk
674948b32c28SKlaus Jensen if (!pci_is_vf(pci)) {
675011871f53SŁukasz Gieryk vf_index = le16_to_cpu(sctrl->vfn) - 1;
675148b32c28SKlaus Jensen sn = NVME(pcie_sriov_get_vf_at_index(pci, vf_index));
675211871f53SŁukasz Gieryk }
675311871f53SŁukasz Gieryk
675411871f53SŁukasz Gieryk if (online) {
675511871f53SŁukasz Gieryk if (!sctrl->nvi || (le16_to_cpu(sctrl->nvq) < 2) || !sn) {
675611871f53SŁukasz Gieryk return NVME_INVALID_SEC_CTRL_STATE | NVME_DNR;
675711871f53SŁukasz Gieryk }
675811871f53SŁukasz Gieryk
675911871f53SŁukasz Gieryk if (!sctrl->scs) {
676011871f53SŁukasz Gieryk sctrl->scs = 0x1;
676111871f53SŁukasz Gieryk nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
676211871f53SŁukasz Gieryk }
676311871f53SŁukasz Gieryk } else {
676411871f53SŁukasz Gieryk nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_INTERRUPT, 0);
676511871f53SŁukasz Gieryk nvme_update_virt_res(n, sctrl, NVME_VIRT_RES_QUEUE, 0);
676611871f53SŁukasz Gieryk
676711871f53SŁukasz Gieryk if (sctrl->scs) {
676811871f53SŁukasz Gieryk sctrl->scs = 0x0;
676911871f53SŁukasz Gieryk if (sn) {
677011871f53SŁukasz Gieryk nvme_ctrl_reset(sn, NVME_RESET_FUNCTION);
677111871f53SŁukasz Gieryk }
677211871f53SŁukasz Gieryk }
677311871f53SŁukasz Gieryk }
677411871f53SŁukasz Gieryk
677511871f53SŁukasz Gieryk return NVME_SUCCESS;
677611871f53SŁukasz Gieryk }
677711871f53SŁukasz Gieryk
nvme_virt_mngmt(NvmeCtrl * n,NvmeRequest * req)677811871f53SŁukasz Gieryk static uint16_t nvme_virt_mngmt(NvmeCtrl *n, NvmeRequest *req)
677911871f53SŁukasz Gieryk {
678011871f53SŁukasz Gieryk uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
678111871f53SŁukasz Gieryk uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
678211871f53SŁukasz Gieryk uint8_t act = dw10 & 0xf;
678311871f53SŁukasz Gieryk uint8_t rt = (dw10 >> 8) & 0x7;
678411871f53SŁukasz Gieryk uint16_t cntlid = (dw10 >> 16) & 0xffff;
678511871f53SŁukasz Gieryk int nr = dw11 & 0xffff;
678611871f53SŁukasz Gieryk
678711871f53SŁukasz Gieryk trace_pci_nvme_virt_mngmt(nvme_cid(req), act, cntlid, rt ? "VI" : "VQ", nr);
678811871f53SŁukasz Gieryk
678911871f53SŁukasz Gieryk if (rt != NVME_VIRT_RES_QUEUE && rt != NVME_VIRT_RES_INTERRUPT) {
679011871f53SŁukasz Gieryk return NVME_INVALID_RESOURCE_ID | NVME_DNR;
679111871f53SŁukasz Gieryk }
679211871f53SŁukasz Gieryk
679311871f53SŁukasz Gieryk switch (act) {
679411871f53SŁukasz Gieryk case NVME_VIRT_MNGMT_ACTION_SEC_ASSIGN:
679511871f53SŁukasz Gieryk return nvme_assign_virt_res_to_sec(n, req, cntlid, rt, nr);
679611871f53SŁukasz Gieryk case NVME_VIRT_MNGMT_ACTION_PRM_ALLOC:
679711871f53SŁukasz Gieryk return nvme_assign_virt_res_to_prim(n, req, cntlid, rt, nr);
679811871f53SŁukasz Gieryk case NVME_VIRT_MNGMT_ACTION_SEC_ONLINE:
679911871f53SŁukasz Gieryk return nvme_virt_set_state(n, cntlid, true);
680011871f53SŁukasz Gieryk case NVME_VIRT_MNGMT_ACTION_SEC_OFFLINE:
680111871f53SŁukasz Gieryk return nvme_virt_set_state(n, cntlid, false);
680211871f53SŁukasz Gieryk default:
680311871f53SŁukasz Gieryk return NVME_INVALID_FIELD | NVME_DNR;
680411871f53SŁukasz Gieryk }
680511871f53SŁukasz Gieryk }
680611871f53SŁukasz Gieryk
nvme_dbbuf_config(NvmeCtrl * n,const NvmeRequest * req)68073f7fe8deSJinhao Fan static uint16_t nvme_dbbuf_config(NvmeCtrl *n, const NvmeRequest *req)
68083f7fe8deSJinhao Fan {
680948b32c28SKlaus Jensen PCIDevice *pci = PCI_DEVICE(n);
68103f7fe8deSJinhao Fan uint64_t dbs_addr = le64_to_cpu(req->cmd.dptr.prp1);
68113f7fe8deSJinhao Fan uint64_t eis_addr = le64_to_cpu(req->cmd.dptr.prp2);
68123f7fe8deSJinhao Fan int i;
68133f7fe8deSJinhao Fan
68143f7fe8deSJinhao Fan /* Address should be page aligned */
68153f7fe8deSJinhao Fan if (dbs_addr & (n->page_size - 1) || eis_addr & (n->page_size - 1)) {
68163f7fe8deSJinhao Fan return NVME_INVALID_FIELD | NVME_DNR;
68173f7fe8deSJinhao Fan }
68183f7fe8deSJinhao Fan
68193f7fe8deSJinhao Fan /* Save shadow buffer base addr for use during queue creation */
68203f7fe8deSJinhao Fan n->dbbuf_dbs = dbs_addr;
68213f7fe8deSJinhao Fan n->dbbuf_eis = eis_addr;
68223f7fe8deSJinhao Fan n->dbbuf_enabled = true;
68233f7fe8deSJinhao Fan
68243f7fe8deSJinhao Fan for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
68253f7fe8deSJinhao Fan NvmeSQueue *sq = n->sq[i];
68263f7fe8deSJinhao Fan NvmeCQueue *cq = n->cq[i];
68273f7fe8deSJinhao Fan
68283f7fe8deSJinhao Fan if (sq) {
68293f7fe8deSJinhao Fan /*
68303f7fe8deSJinhao Fan * CAP.DSTRD is 0, so offset of ith sq db_addr is (i<<3)
68313f7fe8deSJinhao Fan * nvme_process_db() uses this hard-coded way to calculate
68323f7fe8deSJinhao Fan * doorbell offsets. Be consistent with that here.
68333f7fe8deSJinhao Fan */
68343f7fe8deSJinhao Fan sq->db_addr = dbs_addr + (i << 3);
68353f7fe8deSJinhao Fan sq->ei_addr = eis_addr + (i << 3);
6836c1e244b6SKlaus Jensen stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED);
68372e53b0b4SJinhao Fan
68382e53b0b4SJinhao Fan if (n->params.ioeventfd && sq->sqid != 0) {
68392e53b0b4SJinhao Fan if (!nvme_init_sq_ioeventfd(sq)) {
68402e53b0b4SJinhao Fan sq->ioeventfd_enabled = true;
68412e53b0b4SJinhao Fan }
68422e53b0b4SJinhao Fan }
68433f7fe8deSJinhao Fan }
68443f7fe8deSJinhao Fan
68453f7fe8deSJinhao Fan if (cq) {
68463f7fe8deSJinhao Fan /* CAP.DSTRD is 0, so offset of ith cq db_addr is (i<<3)+(1<<2) */
68473f7fe8deSJinhao Fan cq->db_addr = dbs_addr + (i << 3) + (1 << 2);
68483f7fe8deSJinhao Fan cq->ei_addr = eis_addr + (i << 3) + (1 << 2);
6849c1e244b6SKlaus Jensen stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED);
68502e53b0b4SJinhao Fan
68512e53b0b4SJinhao Fan if (n->params.ioeventfd && cq->cqid != 0) {
68522e53b0b4SJinhao Fan if (!nvme_init_cq_ioeventfd(cq)) {
68532e53b0b4SJinhao Fan cq->ioeventfd_enabled = true;
68542e53b0b4SJinhao Fan }
68552e53b0b4SJinhao Fan }
68563f7fe8deSJinhao Fan }
68573f7fe8deSJinhao Fan }
68583f7fe8deSJinhao Fan
6859387350d5SJinhao Fan trace_pci_nvme_dbbuf_config(dbs_addr, eis_addr);
6860387350d5SJinhao Fan
68613f7fe8deSJinhao Fan return NVME_SUCCESS;
68623f7fe8deSJinhao Fan }
68633f7fe8deSJinhao Fan
nvme_directive_send(NvmeCtrl * n,NvmeRequest * req)6864e181d3daSGollu Appalanaidu static uint16_t nvme_directive_send(NvmeCtrl *n, NvmeRequest *req)
6865e181d3daSGollu Appalanaidu {
6866e181d3daSGollu Appalanaidu return NVME_INVALID_FIELD | NVME_DNR;
6867e181d3daSGollu Appalanaidu }
6868e181d3daSGollu Appalanaidu
nvme_directive_receive(NvmeCtrl * n,NvmeRequest * req)6869e181d3daSGollu Appalanaidu static uint16_t nvme_directive_receive(NvmeCtrl *n, NvmeRequest *req)
6870e181d3daSGollu Appalanaidu {
687173064edfSJesper Devantier NvmeNamespace *ns;
6872e181d3daSGollu Appalanaidu uint32_t dw10 = le32_to_cpu(req->cmd.cdw10);
6873e181d3daSGollu Appalanaidu uint32_t dw11 = le32_to_cpu(req->cmd.cdw11);
6874e181d3daSGollu Appalanaidu uint32_t nsid = le32_to_cpu(req->cmd.nsid);
6875e181d3daSGollu Appalanaidu uint8_t doper, dtype;
6876e181d3daSGollu Appalanaidu uint32_t numd, trans_len;
6877e181d3daSGollu Appalanaidu NvmeDirectiveIdentify id = {
6878e181d3daSGollu Appalanaidu .supported = 1 << NVME_DIRECTIVE_IDENTIFY,
6879e181d3daSGollu Appalanaidu .enabled = 1 << NVME_DIRECTIVE_IDENTIFY,
6880e181d3daSGollu Appalanaidu };
6881e181d3daSGollu Appalanaidu
6882e181d3daSGollu Appalanaidu numd = dw10 + 1;
6883e181d3daSGollu Appalanaidu doper = dw11 & 0xff;
6884e181d3daSGollu Appalanaidu dtype = (dw11 >> 8) & 0xff;
6885e181d3daSGollu Appalanaidu
6886e181d3daSGollu Appalanaidu trans_len = MIN(sizeof(NvmeDirectiveIdentify), numd << 2);
6887e181d3daSGollu Appalanaidu
6888e181d3daSGollu Appalanaidu if (nsid == NVME_NSID_BROADCAST || dtype != NVME_DIRECTIVE_IDENTIFY ||
6889e181d3daSGollu Appalanaidu doper != NVME_DIRECTIVE_RETURN_PARAMS) {
6890e181d3daSGollu Appalanaidu return NVME_INVALID_FIELD | NVME_DNR;
6891e181d3daSGollu Appalanaidu }
6892e181d3daSGollu Appalanaidu
689373064edfSJesper Devantier ns = nvme_ns(n, nsid);
689473064edfSJesper Devantier if (!ns) {
689573064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
689673064edfSJesper Devantier }
689773064edfSJesper Devantier
689873064edfSJesper Devantier switch (dtype) {
689973064edfSJesper Devantier case NVME_DIRECTIVE_IDENTIFY:
690073064edfSJesper Devantier switch (doper) {
690173064edfSJesper Devantier case NVME_DIRECTIVE_RETURN_PARAMS:
69026c8f8456SKlaus Jensen if (ns->endgrp && ns->endgrp->fdp.enabled) {
690373064edfSJesper Devantier id.supported |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
690473064edfSJesper Devantier id.enabled |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
690573064edfSJesper Devantier id.persistent |= 1 << NVME_DIRECTIVE_DATA_PLACEMENT;
690673064edfSJesper Devantier }
690773064edfSJesper Devantier
6908e181d3daSGollu Appalanaidu return nvme_c2h(n, (uint8_t *)&id, trans_len, req);
690973064edfSJesper Devantier
691073064edfSJesper Devantier default:
691173064edfSJesper Devantier return NVME_INVALID_FIELD | NVME_DNR;
691273064edfSJesper Devantier }
691373064edfSJesper Devantier
691473064edfSJesper Devantier default:
691573064edfSJesper Devantier return NVME_INVALID_FIELD;
691673064edfSJesper Devantier }
6917e181d3daSGollu Appalanaidu }
6918e181d3daSGollu Appalanaidu
nvme_admin_cmd(NvmeCtrl * n,NvmeRequest * req)691988eea45cSKlaus Jensen static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeRequest *req)
692088eea45cSKlaus Jensen {
692188eea45cSKlaus Jensen trace_pci_nvme_admin_cmd(nvme_cid(req), nvme_sqid(req), req->cmd.opcode,
692288eea45cSKlaus Jensen nvme_adm_opc_str(req->cmd.opcode));
692388eea45cSKlaus Jensen
692488eea45cSKlaus Jensen if (!(nvme_cse_acs[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
692588eea45cSKlaus Jensen trace_pci_nvme_err_invalid_admin_opc(req->cmd.opcode);
692688eea45cSKlaus Jensen return NVME_INVALID_OPCODE | NVME_DNR;
692788eea45cSKlaus Jensen }
692888eea45cSKlaus Jensen
692988eea45cSKlaus Jensen /* SGLs shall not be used for Admin commands in NVMe over PCIe */
693088eea45cSKlaus Jensen if (NVME_CMD_FLAGS_PSDT(req->cmd.flags) != NVME_PSDT_PRP) {
693188eea45cSKlaus Jensen return NVME_INVALID_FIELD | NVME_DNR;
693288eea45cSKlaus Jensen }
693388eea45cSKlaus Jensen
6934c53a9a91SPankaj Raghav if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
6935c53a9a91SPankaj Raghav return NVME_INVALID_FIELD;
6936c53a9a91SPankaj Raghav }
6937c53a9a91SPankaj Raghav
693888eea45cSKlaus Jensen switch (req->cmd.opcode) {
693988eea45cSKlaus Jensen case NVME_ADM_CMD_DELETE_SQ:
694088eea45cSKlaus Jensen return nvme_del_sq(n, req);
694188eea45cSKlaus Jensen case NVME_ADM_CMD_CREATE_SQ:
694288eea45cSKlaus Jensen return nvme_create_sq(n, req);
694388eea45cSKlaus Jensen case NVME_ADM_CMD_GET_LOG_PAGE:
694488eea45cSKlaus Jensen return nvme_get_log(n, req);
694588eea45cSKlaus Jensen case NVME_ADM_CMD_DELETE_CQ:
694688eea45cSKlaus Jensen return nvme_del_cq(n, req);
694788eea45cSKlaus Jensen case NVME_ADM_CMD_CREATE_CQ:
694888eea45cSKlaus Jensen return nvme_create_cq(n, req);
694988eea45cSKlaus Jensen case NVME_ADM_CMD_IDENTIFY:
695088eea45cSKlaus Jensen return nvme_identify(n, req);
695188eea45cSKlaus Jensen case NVME_ADM_CMD_ABORT:
695288eea45cSKlaus Jensen return nvme_abort(n, req);
695388eea45cSKlaus Jensen case NVME_ADM_CMD_SET_FEATURES:
695488eea45cSKlaus Jensen return nvme_set_feature(n, req);
695588eea45cSKlaus Jensen case NVME_ADM_CMD_GET_FEATURES:
695688eea45cSKlaus Jensen return nvme_get_feature(n, req);
695788eea45cSKlaus Jensen case NVME_ADM_CMD_ASYNC_EV_REQ:
695888eea45cSKlaus Jensen return nvme_aer(n, req);
695988eea45cSKlaus Jensen case NVME_ADM_CMD_NS_ATTACHMENT:
696088eea45cSKlaus Jensen return nvme_ns_attachment(n, req);
696111871f53SŁukasz Gieryk case NVME_ADM_CMD_VIRT_MNGMT:
696211871f53SŁukasz Gieryk return nvme_virt_mngmt(n, req);
69633f7fe8deSJinhao Fan case NVME_ADM_CMD_DBBUF_CONFIG:
69643f7fe8deSJinhao Fan return nvme_dbbuf_config(n, req);
696588eea45cSKlaus Jensen case NVME_ADM_CMD_FORMAT_NVM:
696688eea45cSKlaus Jensen return nvme_format(n, req);
6967e181d3daSGollu Appalanaidu case NVME_ADM_CMD_DIRECTIVE_SEND:
6968e181d3daSGollu Appalanaidu return nvme_directive_send(n, req);
6969e181d3daSGollu Appalanaidu case NVME_ADM_CMD_DIRECTIVE_RECV:
6970e181d3daSGollu Appalanaidu return nvme_directive_receive(n, req);
697188eea45cSKlaus Jensen default:
697288eea45cSKlaus Jensen assert(false);
697388eea45cSKlaus Jensen }
697488eea45cSKlaus Jensen
697588eea45cSKlaus Jensen return NVME_INVALID_OPCODE | NVME_DNR;
697688eea45cSKlaus Jensen }
697788eea45cSKlaus Jensen
nvme_update_sq_eventidx(const NvmeSQueue * sq)69783f7fe8deSJinhao Fan static void nvme_update_sq_eventidx(const NvmeSQueue *sq)
69793f7fe8deSJinhao Fan {
698047cd3539SKlaus Jensen trace_pci_nvme_update_sq_eventidx(sq->sqid, sq->tail);
698147cd3539SKlaus Jensen
6982c1e244b6SKlaus Jensen stl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->ei_addr, sq->tail,
6983c1e244b6SKlaus Jensen MEMTXATTRS_UNSPECIFIED);
69843f7fe8deSJinhao Fan }
69853f7fe8deSJinhao Fan
nvme_update_sq_tail(NvmeSQueue * sq)69863f7fe8deSJinhao Fan static void nvme_update_sq_tail(NvmeSQueue *sq)
69873f7fe8deSJinhao Fan {
6988c1e244b6SKlaus Jensen ldl_le_pci_dma(PCI_DEVICE(sq->ctrl), sq->db_addr, &sq->tail,
6989c1e244b6SKlaus Jensen MEMTXATTRS_UNSPECIFIED);
699047cd3539SKlaus Jensen
699147cd3539SKlaus Jensen trace_pci_nvme_update_sq_tail(sq->sqid, sq->tail);
69923f7fe8deSJinhao Fan }
69933f7fe8deSJinhao Fan
nvme_process_sq(void * opaque)699488eea45cSKlaus Jensen static void nvme_process_sq(void *opaque)
699588eea45cSKlaus Jensen {
699688eea45cSKlaus Jensen NvmeSQueue *sq = opaque;
699788eea45cSKlaus Jensen NvmeCtrl *n = sq->ctrl;
699888eea45cSKlaus Jensen NvmeCQueue *cq = n->cq[sq->cqid];
699988eea45cSKlaus Jensen
700088eea45cSKlaus Jensen uint16_t status;
700188eea45cSKlaus Jensen hwaddr addr;
700288eea45cSKlaus Jensen NvmeCmd cmd;
700388eea45cSKlaus Jensen NvmeRequest *req;
700488eea45cSKlaus Jensen
70053f7fe8deSJinhao Fan if (n->dbbuf_enabled) {
70063f7fe8deSJinhao Fan nvme_update_sq_tail(sq);
70073f7fe8deSJinhao Fan }
70083f7fe8deSJinhao Fan
700988eea45cSKlaus Jensen while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
70106a33f2e9SKlaus Jensen addr = sq->dma_addr + (sq->head << NVME_SQES);
701188eea45cSKlaus Jensen if (nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd))) {
701288eea45cSKlaus Jensen trace_pci_nvme_err_addr_read(addr);
701388eea45cSKlaus Jensen trace_pci_nvme_err_cfs();
701449e03457SKlaus Jensen stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
701588eea45cSKlaus Jensen break;
701688eea45cSKlaus Jensen }
701788eea45cSKlaus Jensen nvme_inc_sq_head(sq);
701888eea45cSKlaus Jensen
701988eea45cSKlaus Jensen req = QTAILQ_FIRST(&sq->req_list);
702088eea45cSKlaus Jensen QTAILQ_REMOVE(&sq->req_list, req, entry);
702188eea45cSKlaus Jensen QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
702288eea45cSKlaus Jensen nvme_req_clear(req);
702388eea45cSKlaus Jensen req->cqe.cid = cmd.cid;
702488eea45cSKlaus Jensen memcpy(&req->cmd, &cmd, sizeof(NvmeCmd));
702588eea45cSKlaus Jensen
702688eea45cSKlaus Jensen status = sq->sqid ? nvme_io_cmd(n, req) :
702788eea45cSKlaus Jensen nvme_admin_cmd(n, req);
702888eea45cSKlaus Jensen if (status != NVME_NO_COMPLETE) {
702988eea45cSKlaus Jensen req->status = status;
703088eea45cSKlaus Jensen nvme_enqueue_req_completion(cq, req);
703188eea45cSKlaus Jensen }
70323f7fe8deSJinhao Fan
70333f7fe8deSJinhao Fan if (n->dbbuf_enabled) {
70343f7fe8deSJinhao Fan nvme_update_sq_eventidx(sq);
70353f7fe8deSJinhao Fan nvme_update_sq_tail(sq);
70363f7fe8deSJinhao Fan }
703788eea45cSKlaus Jensen }
703888eea45cSKlaus Jensen }
703988eea45cSKlaus Jensen
nvme_update_msixcap_ts(PCIDevice * pci_dev,uint32_t table_size)7040decc0261SŁukasz Gieryk static void nvme_update_msixcap_ts(PCIDevice *pci_dev, uint32_t table_size)
7041decc0261SŁukasz Gieryk {
7042decc0261SŁukasz Gieryk uint8_t *config;
7043decc0261SŁukasz Gieryk
7044decc0261SŁukasz Gieryk if (!msix_present(pci_dev)) {
7045decc0261SŁukasz Gieryk return;
7046decc0261SŁukasz Gieryk }
7047decc0261SŁukasz Gieryk
7048decc0261SŁukasz Gieryk assert(table_size > 0 && table_size <= pci_dev->msix_entries_nr);
7049decc0261SŁukasz Gieryk
7050decc0261SŁukasz Gieryk config = pci_dev->config + pci_dev->msix_cap;
7051decc0261SŁukasz Gieryk pci_set_word_by_mask(config + PCI_MSIX_FLAGS, PCI_MSIX_FLAGS_QSIZE,
7052decc0261SŁukasz Gieryk table_size - 1);
7053decc0261SŁukasz Gieryk }
7054decc0261SŁukasz Gieryk
nvme_activate_virt_res(NvmeCtrl * n)705511871f53SŁukasz Gieryk static void nvme_activate_virt_res(NvmeCtrl *n)
705611871f53SŁukasz Gieryk {
705748b32c28SKlaus Jensen PCIDevice *pci_dev = PCI_DEVICE(n);
705811871f53SŁukasz Gieryk NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
705911871f53SŁukasz Gieryk NvmeSecCtrlEntry *sctrl;
706011871f53SŁukasz Gieryk
706111871f53SŁukasz Gieryk /* -1 to account for the admin queue */
706211871f53SŁukasz Gieryk if (pci_is_vf(pci_dev)) {
706311871f53SŁukasz Gieryk sctrl = nvme_sctrl(n);
706411871f53SŁukasz Gieryk cap->vqprt = sctrl->nvq;
706511871f53SŁukasz Gieryk cap->viprt = sctrl->nvi;
706611871f53SŁukasz Gieryk n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
706711871f53SŁukasz Gieryk n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
706811871f53SŁukasz Gieryk } else {
706911871f53SŁukasz Gieryk cap->vqrfap = n->next_pri_ctrl_cap.vqrfap;
707011871f53SŁukasz Gieryk cap->virfap = n->next_pri_ctrl_cap.virfap;
707111871f53SŁukasz Gieryk n->conf_ioqpairs = le16_to_cpu(cap->vqprt) +
707211871f53SŁukasz Gieryk le16_to_cpu(cap->vqrfap) - 1;
707311871f53SŁukasz Gieryk n->conf_msix_qsize = le16_to_cpu(cap->viprt) +
707411871f53SŁukasz Gieryk le16_to_cpu(cap->virfap);
707511871f53SŁukasz Gieryk }
707611871f53SŁukasz Gieryk }
707711871f53SŁukasz Gieryk
nvme_ctrl_reset(NvmeCtrl * n,NvmeResetType rst)70781e9c685eSŁukasz Gieryk static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst)
707988eea45cSKlaus Jensen {
708048b32c28SKlaus Jensen PCIDevice *pci_dev = PCI_DEVICE(n);
708111871f53SŁukasz Gieryk NvmeSecCtrlEntry *sctrl;
708288eea45cSKlaus Jensen NvmeNamespace *ns;
708388eea45cSKlaus Jensen int i;
708488eea45cSKlaus Jensen
708588eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
708688eea45cSKlaus Jensen ns = nvme_ns(n, i);
708788eea45cSKlaus Jensen if (!ns) {
708888eea45cSKlaus Jensen continue;
708988eea45cSKlaus Jensen }
709088eea45cSKlaus Jensen
709188eea45cSKlaus Jensen nvme_ns_drain(ns);
709288eea45cSKlaus Jensen }
709388eea45cSKlaus Jensen
709488eea45cSKlaus Jensen for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
709588eea45cSKlaus Jensen if (n->sq[i] != NULL) {
709688eea45cSKlaus Jensen nvme_free_sq(n->sq[i], n);
709788eea45cSKlaus Jensen }
709888eea45cSKlaus Jensen }
709988eea45cSKlaus Jensen for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
710088eea45cSKlaus Jensen if (n->cq[i] != NULL) {
710188eea45cSKlaus Jensen nvme_free_cq(n->cq[i], n);
710288eea45cSKlaus Jensen }
710388eea45cSKlaus Jensen }
710488eea45cSKlaus Jensen
710588eea45cSKlaus Jensen while (!QTAILQ_EMPTY(&n->aer_queue)) {
710688eea45cSKlaus Jensen NvmeAsyncEvent *event = QTAILQ_FIRST(&n->aer_queue);
710788eea45cSKlaus Jensen QTAILQ_REMOVE(&n->aer_queue, event, entry);
710888eea45cSKlaus Jensen g_free(event);
710988eea45cSKlaus Jensen }
711088eea45cSKlaus Jensen
711111871f53SŁukasz Gieryk if (n->params.sriov_max_vfs) {
711211871f53SŁukasz Gieryk if (!pci_is_vf(pci_dev)) {
711311871f53SŁukasz Gieryk for (i = 0; i < n->sec_ctrl_list.numcntl; i++) {
711411871f53SŁukasz Gieryk sctrl = &n->sec_ctrl_list.sec[i];
711511871f53SŁukasz Gieryk nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
711611871f53SŁukasz Gieryk }
711711871f53SŁukasz Gieryk
71181e9c685eSŁukasz Gieryk if (rst != NVME_RESET_CONTROLLER) {
7119decc0261SŁukasz Gieryk pcie_sriov_pf_disable_vfs(pci_dev);
712044c2c094SLukasz Maniak }
71211e9c685eSŁukasz Gieryk }
712244c2c094SLukasz Maniak
712311871f53SŁukasz Gieryk if (rst != NVME_RESET_CONTROLLER) {
712411871f53SŁukasz Gieryk nvme_activate_virt_res(n);
712511871f53SŁukasz Gieryk }
712611871f53SŁukasz Gieryk }
712711871f53SŁukasz Gieryk
712888eea45cSKlaus Jensen n->aer_queued = 0;
712998836e8eSKlaus Jensen n->aer_mask = 0;
713088eea45cSKlaus Jensen n->outstanding_aers = 0;
713188eea45cSKlaus Jensen n->qs_created = false;
7132decc0261SŁukasz Gieryk
7133decc0261SŁukasz Gieryk nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
713411871f53SŁukasz Gieryk
713511871f53SŁukasz Gieryk if (pci_is_vf(pci_dev)) {
713611871f53SŁukasz Gieryk sctrl = nvme_sctrl(n);
7137cc9bcee2SKlaus Jensen
713811871f53SŁukasz Gieryk stl_le_p(&n->bar.csts, sctrl->scs ? 0 : NVME_CSTS_FAILED);
713911871f53SŁukasz Gieryk } else {
714011871f53SŁukasz Gieryk stl_le_p(&n->bar.csts, 0);
714111871f53SŁukasz Gieryk }
7142cc9bcee2SKlaus Jensen
7143cc9bcee2SKlaus Jensen stl_le_p(&n->bar.intms, 0);
7144cc9bcee2SKlaus Jensen stl_le_p(&n->bar.intmc, 0);
7145cc9bcee2SKlaus Jensen stl_le_p(&n->bar.cc, 0);
71463f7fe8deSJinhao Fan
71473f7fe8deSJinhao Fan n->dbbuf_dbs = 0;
71483f7fe8deSJinhao Fan n->dbbuf_eis = 0;
71493f7fe8deSJinhao Fan n->dbbuf_enabled = false;
715088eea45cSKlaus Jensen }
715188eea45cSKlaus Jensen
nvme_ctrl_shutdown(NvmeCtrl * n)715288eea45cSKlaus Jensen static void nvme_ctrl_shutdown(NvmeCtrl *n)
715388eea45cSKlaus Jensen {
715488eea45cSKlaus Jensen NvmeNamespace *ns;
715588eea45cSKlaus Jensen int i;
715688eea45cSKlaus Jensen
715788eea45cSKlaus Jensen if (n->pmr.dev) {
715888eea45cSKlaus Jensen memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
715988eea45cSKlaus Jensen }
716088eea45cSKlaus Jensen
716188eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
716288eea45cSKlaus Jensen ns = nvme_ns(n, i);
716388eea45cSKlaus Jensen if (!ns) {
716488eea45cSKlaus Jensen continue;
716588eea45cSKlaus Jensen }
716688eea45cSKlaus Jensen
716788eea45cSKlaus Jensen nvme_ns_shutdown(ns);
716888eea45cSKlaus Jensen }
716988eea45cSKlaus Jensen }
717088eea45cSKlaus Jensen
nvme_select_iocs(NvmeCtrl * n)717188eea45cSKlaus Jensen static void nvme_select_iocs(NvmeCtrl *n)
717288eea45cSKlaus Jensen {
717388eea45cSKlaus Jensen NvmeNamespace *ns;
717488eea45cSKlaus Jensen int i;
717588eea45cSKlaus Jensen
717688eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
717788eea45cSKlaus Jensen ns = nvme_ns(n, i);
717888eea45cSKlaus Jensen if (!ns) {
717988eea45cSKlaus Jensen continue;
718088eea45cSKlaus Jensen }
718188eea45cSKlaus Jensen
718288eea45cSKlaus Jensen nvme_select_iocs_ns(n, ns);
718388eea45cSKlaus Jensen }
718488eea45cSKlaus Jensen }
718588eea45cSKlaus Jensen
nvme_start_ctrl(NvmeCtrl * n)718688eea45cSKlaus Jensen static int nvme_start_ctrl(NvmeCtrl *n)
718788eea45cSKlaus Jensen {
718849e03457SKlaus Jensen uint64_t cap = ldq_le_p(&n->bar.cap);
718949e03457SKlaus Jensen uint32_t cc = ldl_le_p(&n->bar.cc);
719049e03457SKlaus Jensen uint32_t aqa = ldl_le_p(&n->bar.aqa);
719149e03457SKlaus Jensen uint64_t asq = ldq_le_p(&n->bar.asq);
719249e03457SKlaus Jensen uint64_t acq = ldq_le_p(&n->bar.acq);
719349e03457SKlaus Jensen uint32_t page_bits = NVME_CC_MPS(cc) + 12;
719488eea45cSKlaus Jensen uint32_t page_size = 1 << page_bits;
719511871f53SŁukasz Gieryk NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
719688eea45cSKlaus Jensen
719748b32c28SKlaus Jensen if (pci_is_vf(PCI_DEVICE(n)) && !sctrl->scs) {
719811871f53SŁukasz Gieryk trace_pci_nvme_err_startfail_virt_state(le16_to_cpu(sctrl->nvi),
71993488fc32SPaolo Bonzini le16_to_cpu(sctrl->nvq));
720011871f53SŁukasz Gieryk return -1;
720111871f53SŁukasz Gieryk }
720288eea45cSKlaus Jensen if (unlikely(n->cq[0])) {
720388eea45cSKlaus Jensen trace_pci_nvme_err_startfail_cq();
720488eea45cSKlaus Jensen return -1;
720588eea45cSKlaus Jensen }
720688eea45cSKlaus Jensen if (unlikely(n->sq[0])) {
720788eea45cSKlaus Jensen trace_pci_nvme_err_startfail_sq();
720888eea45cSKlaus Jensen return -1;
720988eea45cSKlaus Jensen }
721049e03457SKlaus Jensen if (unlikely(asq & (page_size - 1))) {
721149e03457SKlaus Jensen trace_pci_nvme_err_startfail_asq_misaligned(asq);
721288eea45cSKlaus Jensen return -1;
721388eea45cSKlaus Jensen }
721449e03457SKlaus Jensen if (unlikely(acq & (page_size - 1))) {
721549e03457SKlaus Jensen trace_pci_nvme_err_startfail_acq_misaligned(acq);
721688eea45cSKlaus Jensen return -1;
721788eea45cSKlaus Jensen }
721849e03457SKlaus Jensen if (unlikely(!(NVME_CAP_CSS(cap) & (1 << NVME_CC_CSS(cc))))) {
721949e03457SKlaus Jensen trace_pci_nvme_err_startfail_css(NVME_CC_CSS(cc));
722088eea45cSKlaus Jensen return -1;
722188eea45cSKlaus Jensen }
722249e03457SKlaus Jensen if (unlikely(NVME_CC_MPS(cc) < NVME_CAP_MPSMIN(cap))) {
722388eea45cSKlaus Jensen trace_pci_nvme_err_startfail_page_too_small(
722449e03457SKlaus Jensen NVME_CC_MPS(cc),
722549e03457SKlaus Jensen NVME_CAP_MPSMIN(cap));
722688eea45cSKlaus Jensen return -1;
722788eea45cSKlaus Jensen }
722849e03457SKlaus Jensen if (unlikely(NVME_CC_MPS(cc) >
722949e03457SKlaus Jensen NVME_CAP_MPSMAX(cap))) {
723088eea45cSKlaus Jensen trace_pci_nvme_err_startfail_page_too_large(
723149e03457SKlaus Jensen NVME_CC_MPS(cc),
723249e03457SKlaus Jensen NVME_CAP_MPSMAX(cap));
723388eea45cSKlaus Jensen return -1;
723488eea45cSKlaus Jensen }
723549e03457SKlaus Jensen if (unlikely(!NVME_AQA_ASQS(aqa))) {
723688eea45cSKlaus Jensen trace_pci_nvme_err_startfail_asqent_sz_zero();
723788eea45cSKlaus Jensen return -1;
723888eea45cSKlaus Jensen }
723949e03457SKlaus Jensen if (unlikely(!NVME_AQA_ACQS(aqa))) {
724088eea45cSKlaus Jensen trace_pci_nvme_err_startfail_acqent_sz_zero();
724188eea45cSKlaus Jensen return -1;
724288eea45cSKlaus Jensen }
724388eea45cSKlaus Jensen
724488eea45cSKlaus Jensen n->page_bits = page_bits;
724588eea45cSKlaus Jensen n->page_size = page_size;
724688eea45cSKlaus Jensen n->max_prp_ents = n->page_size / sizeof(uint64_t);
724749e03457SKlaus Jensen nvme_init_cq(&n->admin_cq, n, acq, 0, 0, NVME_AQA_ACQS(aqa) + 1, 1);
724849e03457SKlaus Jensen nvme_init_sq(&n->admin_sq, n, asq, 0, 0, NVME_AQA_ASQS(aqa) + 1);
724988eea45cSKlaus Jensen
725088eea45cSKlaus Jensen nvme_set_timestamp(n, 0ULL);
725188eea45cSKlaus Jensen
725288eea45cSKlaus Jensen nvme_select_iocs(n);
725388eea45cSKlaus Jensen
725488eea45cSKlaus Jensen return 0;
725588eea45cSKlaus Jensen }
725688eea45cSKlaus Jensen
nvme_cmb_enable_regs(NvmeCtrl * n)725788eea45cSKlaus Jensen static void nvme_cmb_enable_regs(NvmeCtrl *n)
725888eea45cSKlaus Jensen {
725949e03457SKlaus Jensen uint32_t cmbloc = ldl_le_p(&n->bar.cmbloc);
726049e03457SKlaus Jensen uint32_t cmbsz = ldl_le_p(&n->bar.cmbsz);
726188eea45cSKlaus Jensen
726249e03457SKlaus Jensen NVME_CMBLOC_SET_CDPCILS(cmbloc, 1);
726349e03457SKlaus Jensen NVME_CMBLOC_SET_CDPMLS(cmbloc, 1);
726449e03457SKlaus Jensen NVME_CMBLOC_SET_BIR(cmbloc, NVME_CMB_BIR);
726549e03457SKlaus Jensen stl_le_p(&n->bar.cmbloc, cmbloc);
726649e03457SKlaus Jensen
726749e03457SKlaus Jensen NVME_CMBSZ_SET_SQS(cmbsz, 1);
726849e03457SKlaus Jensen NVME_CMBSZ_SET_CQS(cmbsz, 0);
726949e03457SKlaus Jensen NVME_CMBSZ_SET_LISTS(cmbsz, 1);
727049e03457SKlaus Jensen NVME_CMBSZ_SET_RDS(cmbsz, 1);
727149e03457SKlaus Jensen NVME_CMBSZ_SET_WDS(cmbsz, 1);
727249e03457SKlaus Jensen NVME_CMBSZ_SET_SZU(cmbsz, 2); /* MBs */
727349e03457SKlaus Jensen NVME_CMBSZ_SET_SZ(cmbsz, n->params.cmb_size_mb);
727449e03457SKlaus Jensen stl_le_p(&n->bar.cmbsz, cmbsz);
727588eea45cSKlaus Jensen }
727688eea45cSKlaus Jensen
nvme_write_bar(NvmeCtrl * n,hwaddr offset,uint64_t data,unsigned size)727788eea45cSKlaus Jensen static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
727888eea45cSKlaus Jensen unsigned size)
727988eea45cSKlaus Jensen {
728048b32c28SKlaus Jensen PCIDevice *pci = PCI_DEVICE(n);
728149e03457SKlaus Jensen uint64_t cap = ldq_le_p(&n->bar.cap);
728249e03457SKlaus Jensen uint32_t cc = ldl_le_p(&n->bar.cc);
728349e03457SKlaus Jensen uint32_t intms = ldl_le_p(&n->bar.intms);
728449e03457SKlaus Jensen uint32_t csts = ldl_le_p(&n->bar.csts);
728549e03457SKlaus Jensen uint32_t pmrsts = ldl_le_p(&n->bar.pmrsts);
728649e03457SKlaus Jensen
728788eea45cSKlaus Jensen if (unlikely(offset & (sizeof(uint32_t) - 1))) {
728888eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_misaligned32,
728988eea45cSKlaus Jensen "MMIO write not 32-bit aligned,"
729088eea45cSKlaus Jensen " offset=0x%"PRIx64"", offset);
729188eea45cSKlaus Jensen /* should be ignored, fall through for now */
729288eea45cSKlaus Jensen }
729388eea45cSKlaus Jensen
729488eea45cSKlaus Jensen if (unlikely(size < sizeof(uint32_t))) {
729588eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_toosmall,
729688eea45cSKlaus Jensen "MMIO write smaller than 32-bits,"
729788eea45cSKlaus Jensen " offset=0x%"PRIx64", size=%u",
729888eea45cSKlaus Jensen offset, size);
729988eea45cSKlaus Jensen /* should be ignored, fall through for now */
730088eea45cSKlaus Jensen }
730188eea45cSKlaus Jensen
730288eea45cSKlaus Jensen switch (offset) {
7303a316aa50SKlaus Jensen case NVME_REG_INTMS:
730448b32c28SKlaus Jensen if (unlikely(msix_enabled(pci))) {
730588eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
730688eea45cSKlaus Jensen "undefined access to interrupt mask set"
730788eea45cSKlaus Jensen " when MSI-X is enabled");
730888eea45cSKlaus Jensen /* should be ignored, fall through for now */
730988eea45cSKlaus Jensen }
731049e03457SKlaus Jensen intms |= data;
731149e03457SKlaus Jensen stl_le_p(&n->bar.intms, intms);
731288eea45cSKlaus Jensen n->bar.intmc = n->bar.intms;
731349e03457SKlaus Jensen trace_pci_nvme_mmio_intm_set(data & 0xffffffff, intms);
731488eea45cSKlaus Jensen nvme_irq_check(n);
731588eea45cSKlaus Jensen break;
7316a316aa50SKlaus Jensen case NVME_REG_INTMC:
731748b32c28SKlaus Jensen if (unlikely(msix_enabled(pci))) {
731888eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_intmask_with_msix,
731988eea45cSKlaus Jensen "undefined access to interrupt mask clr"
732088eea45cSKlaus Jensen " when MSI-X is enabled");
732188eea45cSKlaus Jensen /* should be ignored, fall through for now */
732288eea45cSKlaus Jensen }
732349e03457SKlaus Jensen intms &= ~data;
732449e03457SKlaus Jensen stl_le_p(&n->bar.intms, intms);
732588eea45cSKlaus Jensen n->bar.intmc = n->bar.intms;
732649e03457SKlaus Jensen trace_pci_nvme_mmio_intm_clr(data & 0xffffffff, intms);
732788eea45cSKlaus Jensen nvme_irq_check(n);
732888eea45cSKlaus Jensen break;
7329a316aa50SKlaus Jensen case NVME_REG_CC:
7330cc9bcee2SKlaus Jensen stl_le_p(&n->bar.cc, data);
7331cc9bcee2SKlaus Jensen
733288eea45cSKlaus Jensen trace_pci_nvme_mmio_cfg(data & 0xffffffff);
733349e03457SKlaus Jensen
7334cc9bcee2SKlaus Jensen if (NVME_CC_SHN(data) && !(NVME_CC_SHN(cc))) {
7335cc9bcee2SKlaus Jensen trace_pci_nvme_mmio_shutdown_set();
7336cc9bcee2SKlaus Jensen nvme_ctrl_shutdown(n);
7337cc9bcee2SKlaus Jensen csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
7338cc9bcee2SKlaus Jensen csts |= NVME_CSTS_SHST_COMPLETE;
7339cc9bcee2SKlaus Jensen } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(cc)) {
7340cc9bcee2SKlaus Jensen trace_pci_nvme_mmio_shutdown_cleared();
7341cc9bcee2SKlaus Jensen csts &= ~(CSTS_SHST_MASK << CSTS_SHST_SHIFT);
734288eea45cSKlaus Jensen }
734388eea45cSKlaus Jensen
734449e03457SKlaus Jensen if (NVME_CC_EN(data) && !NVME_CC_EN(cc)) {
734588eea45cSKlaus Jensen if (unlikely(nvme_start_ctrl(n))) {
734688eea45cSKlaus Jensen trace_pci_nvme_err_startfail();
734749e03457SKlaus Jensen csts = NVME_CSTS_FAILED;
734888eea45cSKlaus Jensen } else {
734988eea45cSKlaus Jensen trace_pci_nvme_mmio_start_success();
735049e03457SKlaus Jensen csts = NVME_CSTS_READY;
735188eea45cSKlaus Jensen }
735249e03457SKlaus Jensen } else if (!NVME_CC_EN(data) && NVME_CC_EN(cc)) {
735388eea45cSKlaus Jensen trace_pci_nvme_mmio_stopped();
73541e9c685eSŁukasz Gieryk nvme_ctrl_reset(n, NVME_RESET_CONTROLLER);
7355cc9bcee2SKlaus Jensen
7356cc9bcee2SKlaus Jensen break;
735788eea45cSKlaus Jensen }
735849e03457SKlaus Jensen
735949e03457SKlaus Jensen stl_le_p(&n->bar.csts, csts);
736049e03457SKlaus Jensen
736188eea45cSKlaus Jensen break;
7362a316aa50SKlaus Jensen case NVME_REG_CSTS:
736388eea45cSKlaus Jensen if (data & (1 << 4)) {
736488eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ssreset_w1c_unsupported,
736588eea45cSKlaus Jensen "attempted to W1C CSTS.NSSRO"
736688eea45cSKlaus Jensen " but CAP.NSSRS is zero (not supported)");
736788eea45cSKlaus Jensen } else if (data != 0) {
736888eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_ro_csts,
736988eea45cSKlaus Jensen "attempted to set a read only bit"
737088eea45cSKlaus Jensen " of controller status");
737188eea45cSKlaus Jensen }
737288eea45cSKlaus Jensen break;
7373a316aa50SKlaus Jensen case NVME_REG_NSSR:
737488eea45cSKlaus Jensen if (data == 0x4e564d65) {
737588eea45cSKlaus Jensen trace_pci_nvme_ub_mmiowr_ssreset_unsupported();
737688eea45cSKlaus Jensen } else {
737788eea45cSKlaus Jensen /* The spec says that writes of other values have no effect */
737888eea45cSKlaus Jensen return;
737988eea45cSKlaus Jensen }
738088eea45cSKlaus Jensen break;
7381a316aa50SKlaus Jensen case NVME_REG_AQA:
738249e03457SKlaus Jensen stl_le_p(&n->bar.aqa, data);
738388eea45cSKlaus Jensen trace_pci_nvme_mmio_aqattr(data & 0xffffffff);
738488eea45cSKlaus Jensen break;
7385a316aa50SKlaus Jensen case NVME_REG_ASQ:
738649e03457SKlaus Jensen stn_le_p(&n->bar.asq, size, data);
738788eea45cSKlaus Jensen trace_pci_nvme_mmio_asqaddr(data);
738888eea45cSKlaus Jensen break;
7389a316aa50SKlaus Jensen case NVME_REG_ASQ + 4:
739049e03457SKlaus Jensen stl_le_p((uint8_t *)&n->bar.asq + 4, data);
739149e03457SKlaus Jensen trace_pci_nvme_mmio_asqaddr_hi(data, ldq_le_p(&n->bar.asq));
739288eea45cSKlaus Jensen break;
7393a316aa50SKlaus Jensen case NVME_REG_ACQ:
739488eea45cSKlaus Jensen trace_pci_nvme_mmio_acqaddr(data);
739549e03457SKlaus Jensen stn_le_p(&n->bar.acq, size, data);
739688eea45cSKlaus Jensen break;
7397a316aa50SKlaus Jensen case NVME_REG_ACQ + 4:
739849e03457SKlaus Jensen stl_le_p((uint8_t *)&n->bar.acq + 4, data);
739949e03457SKlaus Jensen trace_pci_nvme_mmio_acqaddr_hi(data, ldq_le_p(&n->bar.acq));
740088eea45cSKlaus Jensen break;
7401a316aa50SKlaus Jensen case NVME_REG_CMBLOC:
740288eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbloc_reserved,
740388eea45cSKlaus Jensen "invalid write to reserved CMBLOC"
740488eea45cSKlaus Jensen " when CMBSZ is zero, ignored");
740588eea45cSKlaus Jensen return;
7406a316aa50SKlaus Jensen case NVME_REG_CMBSZ:
740788eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_cmbsz_readonly,
740888eea45cSKlaus Jensen "invalid write to read only CMBSZ, ignored");
740988eea45cSKlaus Jensen return;
7410a316aa50SKlaus Jensen case NVME_REG_CMBMSC:
741149e03457SKlaus Jensen if (!NVME_CAP_CMBS(cap)) {
741288eea45cSKlaus Jensen return;
741388eea45cSKlaus Jensen }
741488eea45cSKlaus Jensen
741549e03457SKlaus Jensen stn_le_p(&n->bar.cmbmsc, size, data);
741688eea45cSKlaus Jensen n->cmb.cmse = false;
741788eea45cSKlaus Jensen
741888eea45cSKlaus Jensen if (NVME_CMBMSC_CRE(data)) {
741988eea45cSKlaus Jensen nvme_cmb_enable_regs(n);
742088eea45cSKlaus Jensen
742188eea45cSKlaus Jensen if (NVME_CMBMSC_CMSE(data)) {
742249e03457SKlaus Jensen uint64_t cmbmsc = ldq_le_p(&n->bar.cmbmsc);
742349e03457SKlaus Jensen hwaddr cba = NVME_CMBMSC_CBA(cmbmsc) << CMBMSC_CBA_SHIFT;
742488eea45cSKlaus Jensen if (cba + int128_get64(n->cmb.mem.size) < cba) {
742549e03457SKlaus Jensen uint32_t cmbsts = ldl_le_p(&n->bar.cmbsts);
742649e03457SKlaus Jensen NVME_CMBSTS_SET_CBAI(cmbsts, 1);
742749e03457SKlaus Jensen stl_le_p(&n->bar.cmbsts, cmbsts);
742888eea45cSKlaus Jensen return;
742988eea45cSKlaus Jensen }
743088eea45cSKlaus Jensen
743188eea45cSKlaus Jensen n->cmb.cba = cba;
743288eea45cSKlaus Jensen n->cmb.cmse = true;
743388eea45cSKlaus Jensen }
743488eea45cSKlaus Jensen } else {
743588eea45cSKlaus Jensen n->bar.cmbsz = 0;
743688eea45cSKlaus Jensen n->bar.cmbloc = 0;
743788eea45cSKlaus Jensen }
743888eea45cSKlaus Jensen
743988eea45cSKlaus Jensen return;
7440a316aa50SKlaus Jensen case NVME_REG_CMBMSC + 4:
744149e03457SKlaus Jensen stl_le_p((uint8_t *)&n->bar.cmbmsc + 4, data);
744288eea45cSKlaus Jensen return;
744388eea45cSKlaus Jensen
7444a316aa50SKlaus Jensen case NVME_REG_PMRCAP:
744588eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrcap_readonly,
744688eea45cSKlaus Jensen "invalid write to PMRCAP register, ignored");
744788eea45cSKlaus Jensen return;
7448a316aa50SKlaus Jensen case NVME_REG_PMRCTL:
744949e03457SKlaus Jensen if (!NVME_CAP_PMRS(cap)) {
74502b02aabcSKlaus Jensen return;
74512b02aabcSKlaus Jensen }
74522b02aabcSKlaus Jensen
745349e03457SKlaus Jensen stl_le_p(&n->bar.pmrctl, data);
745488eea45cSKlaus Jensen if (NVME_PMRCTL_EN(data)) {
745588eea45cSKlaus Jensen memory_region_set_enabled(&n->pmr.dev->mr, true);
745649e03457SKlaus Jensen pmrsts = 0;
745788eea45cSKlaus Jensen } else {
745888eea45cSKlaus Jensen memory_region_set_enabled(&n->pmr.dev->mr, false);
745949e03457SKlaus Jensen NVME_PMRSTS_SET_NRDY(pmrsts, 1);
746088eea45cSKlaus Jensen n->pmr.cmse = false;
746188eea45cSKlaus Jensen }
746249e03457SKlaus Jensen stl_le_p(&n->bar.pmrsts, pmrsts);
746388eea45cSKlaus Jensen return;
7464a316aa50SKlaus Jensen case NVME_REG_PMRSTS:
746588eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrsts_readonly,
746688eea45cSKlaus Jensen "invalid write to PMRSTS register, ignored");
746788eea45cSKlaus Jensen return;
7468a316aa50SKlaus Jensen case NVME_REG_PMREBS:
746988eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrebs_readonly,
747088eea45cSKlaus Jensen "invalid write to PMREBS register, ignored");
747188eea45cSKlaus Jensen return;
7472a316aa50SKlaus Jensen case NVME_REG_PMRSWTP:
747388eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_pmrswtp_readonly,
747488eea45cSKlaus Jensen "invalid write to PMRSWTP register, ignored");
747588eea45cSKlaus Jensen return;
7476a316aa50SKlaus Jensen case NVME_REG_PMRMSCL:
747749e03457SKlaus Jensen if (!NVME_CAP_PMRS(cap)) {
747888eea45cSKlaus Jensen return;
747988eea45cSKlaus Jensen }
748088eea45cSKlaus Jensen
748149e03457SKlaus Jensen stl_le_p(&n->bar.pmrmscl, data);
748288eea45cSKlaus Jensen n->pmr.cmse = false;
748388eea45cSKlaus Jensen
748449e03457SKlaus Jensen if (NVME_PMRMSCL_CMSE(data)) {
748549e03457SKlaus Jensen uint64_t pmrmscu = ldl_le_p(&n->bar.pmrmscu);
748649e03457SKlaus Jensen hwaddr cba = pmrmscu << 32 |
748749e03457SKlaus Jensen (NVME_PMRMSCL_CBA(data) << PMRMSCL_CBA_SHIFT);
748888eea45cSKlaus Jensen if (cba + int128_get64(n->pmr.dev->mr.size) < cba) {
748949e03457SKlaus Jensen NVME_PMRSTS_SET_CBAI(pmrsts, 1);
749049e03457SKlaus Jensen stl_le_p(&n->bar.pmrsts, pmrsts);
749188eea45cSKlaus Jensen return;
749288eea45cSKlaus Jensen }
749388eea45cSKlaus Jensen
749488eea45cSKlaus Jensen n->pmr.cmse = true;
749588eea45cSKlaus Jensen n->pmr.cba = cba;
749688eea45cSKlaus Jensen }
749788eea45cSKlaus Jensen
749888eea45cSKlaus Jensen return;
7499a316aa50SKlaus Jensen case NVME_REG_PMRMSCU:
750049e03457SKlaus Jensen if (!NVME_CAP_PMRS(cap)) {
750188eea45cSKlaus Jensen return;
750288eea45cSKlaus Jensen }
750388eea45cSKlaus Jensen
750449e03457SKlaus Jensen stl_le_p(&n->bar.pmrmscu, data);
750588eea45cSKlaus Jensen return;
750688eea45cSKlaus Jensen default:
750788eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiowr_invalid,
750888eea45cSKlaus Jensen "invalid MMIO write,"
750988eea45cSKlaus Jensen " offset=0x%"PRIx64", data=%"PRIx64"",
751088eea45cSKlaus Jensen offset, data);
751188eea45cSKlaus Jensen break;
751288eea45cSKlaus Jensen }
751388eea45cSKlaus Jensen }
751488eea45cSKlaus Jensen
nvme_mmio_read(void * opaque,hwaddr addr,unsigned size)751588eea45cSKlaus Jensen static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
751688eea45cSKlaus Jensen {
751788eea45cSKlaus Jensen NvmeCtrl *n = (NvmeCtrl *)opaque;
751888eea45cSKlaus Jensen uint8_t *ptr = (uint8_t *)&n->bar;
751988eea45cSKlaus Jensen
752088eea45cSKlaus Jensen trace_pci_nvme_mmio_read(addr, size);
752188eea45cSKlaus Jensen
752288eea45cSKlaus Jensen if (unlikely(addr & (sizeof(uint32_t) - 1))) {
752388eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiord_misaligned32,
752488eea45cSKlaus Jensen "MMIO read not 32-bit aligned,"
752588eea45cSKlaus Jensen " offset=0x%"PRIx64"", addr);
752688eea45cSKlaus Jensen /* should RAZ, fall through for now */
752788eea45cSKlaus Jensen } else if (unlikely(size < sizeof(uint32_t))) {
752888eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiord_toosmall,
752988eea45cSKlaus Jensen "MMIO read smaller than 32-bits,"
753088eea45cSKlaus Jensen " offset=0x%"PRIx64"", addr);
753188eea45cSKlaus Jensen /* should RAZ, fall through for now */
753288eea45cSKlaus Jensen }
753388eea45cSKlaus Jensen
75345029de44SKlaus Jensen if (addr > sizeof(n->bar) - size) {
75355029de44SKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_mmiord_invalid_ofs,
75365029de44SKlaus Jensen "MMIO read beyond last register,"
75375029de44SKlaus Jensen " offset=0x%"PRIx64", returning 0", addr);
75385029de44SKlaus Jensen
75395029de44SKlaus Jensen return 0;
75405029de44SKlaus Jensen }
75415029de44SKlaus Jensen
754248b32c28SKlaus Jensen if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
754311871f53SŁukasz Gieryk addr != NVME_REG_CSTS) {
754411871f53SŁukasz Gieryk trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
754511871f53SŁukasz Gieryk return 0;
754611871f53SŁukasz Gieryk }
754711871f53SŁukasz Gieryk
754888eea45cSKlaus Jensen /*
754988eea45cSKlaus Jensen * When PMRWBM bit 1 is set then read from
755088eea45cSKlaus Jensen * from PMRSTS should ensure prior writes
755188eea45cSKlaus Jensen * made it to persistent media
755288eea45cSKlaus Jensen */
7553a316aa50SKlaus Jensen if (addr == NVME_REG_PMRSTS &&
755449e03457SKlaus Jensen (NVME_PMRCAP_PMRWBM(ldl_le_p(&n->bar.pmrcap)) & 0x02)) {
755588eea45cSKlaus Jensen memory_region_msync(&n->pmr.dev->mr, 0, n->pmr.dev->size);
755688eea45cSKlaus Jensen }
75575029de44SKlaus Jensen
755849e03457SKlaus Jensen return ldn_le_p(ptr + addr, size);
755988eea45cSKlaus Jensen }
756088eea45cSKlaus Jensen
nvme_process_db(NvmeCtrl * n,hwaddr addr,int val)756188eea45cSKlaus Jensen static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
756288eea45cSKlaus Jensen {
756348b32c28SKlaus Jensen PCIDevice *pci = PCI_DEVICE(n);
7564c1e244b6SKlaus Jensen uint32_t qid;
756588eea45cSKlaus Jensen
756688eea45cSKlaus Jensen if (unlikely(addr & ((1 << 2) - 1))) {
756788eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_db_wr_misaligned,
756888eea45cSKlaus Jensen "doorbell write not 32-bit aligned,"
756988eea45cSKlaus Jensen " offset=0x%"PRIx64", ignoring", addr);
757088eea45cSKlaus Jensen return;
757188eea45cSKlaus Jensen }
757288eea45cSKlaus Jensen
757388eea45cSKlaus Jensen if (((addr - 0x1000) >> 2) & 1) {
757488eea45cSKlaus Jensen /* Completion queue doorbell write */
757588eea45cSKlaus Jensen
757688eea45cSKlaus Jensen uint16_t new_head = val & 0xffff;
757788eea45cSKlaus Jensen int start_sqs;
757888eea45cSKlaus Jensen NvmeCQueue *cq;
757988eea45cSKlaus Jensen
758088eea45cSKlaus Jensen qid = (addr - (0x1000 + (1 << 2))) >> 3;
758188eea45cSKlaus Jensen if (unlikely(nvme_check_cqid(n, qid))) {
758288eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cq,
758388eea45cSKlaus Jensen "completion queue doorbell write"
758488eea45cSKlaus Jensen " for nonexistent queue,"
758588eea45cSKlaus Jensen " sqid=%"PRIu32", ignoring", qid);
758688eea45cSKlaus Jensen
758788eea45cSKlaus Jensen /*
758888eea45cSKlaus Jensen * NVM Express v1.3d, Section 4.1 state: "If host software writes
758988eea45cSKlaus Jensen * an invalid value to the Submission Queue Tail Doorbell or
75909b4b4e51SMichael Tokarev * Completion Queue Head Doorbell register and an Asynchronous Event
759188eea45cSKlaus Jensen * Request command is outstanding, then an asynchronous event is
759288eea45cSKlaus Jensen * posted to the Admin Completion Queue with a status code of
759388eea45cSKlaus Jensen * Invalid Doorbell Write Value."
759488eea45cSKlaus Jensen *
759588eea45cSKlaus Jensen * Also note that the spec includes the "Invalid Doorbell Register"
759688eea45cSKlaus Jensen * status code, but nowhere does it specify when to use it.
759788eea45cSKlaus Jensen * However, it seems reasonable to use it here in a similar
759888eea45cSKlaus Jensen * fashion.
759988eea45cSKlaus Jensen */
760088eea45cSKlaus Jensen if (n->outstanding_aers) {
760188eea45cSKlaus Jensen nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
760288eea45cSKlaus Jensen NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
760388eea45cSKlaus Jensen NVME_LOG_ERROR_INFO);
760488eea45cSKlaus Jensen }
760588eea45cSKlaus Jensen
760688eea45cSKlaus Jensen return;
760788eea45cSKlaus Jensen }
760888eea45cSKlaus Jensen
760988eea45cSKlaus Jensen cq = n->cq[qid];
761088eea45cSKlaus Jensen if (unlikely(new_head >= cq->size)) {
761188eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_cqhead,
761288eea45cSKlaus Jensen "completion queue doorbell write value"
761388eea45cSKlaus Jensen " beyond queue size, sqid=%"PRIu32","
761488eea45cSKlaus Jensen " new_head=%"PRIu16", ignoring",
761588eea45cSKlaus Jensen qid, new_head);
761688eea45cSKlaus Jensen
761788eea45cSKlaus Jensen if (n->outstanding_aers) {
761888eea45cSKlaus Jensen nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
761988eea45cSKlaus Jensen NVME_AER_INFO_ERR_INVALID_DB_VALUE,
762088eea45cSKlaus Jensen NVME_LOG_ERROR_INFO);
762188eea45cSKlaus Jensen }
762288eea45cSKlaus Jensen
762388eea45cSKlaus Jensen return;
762488eea45cSKlaus Jensen }
762588eea45cSKlaus Jensen
762688eea45cSKlaus Jensen trace_pci_nvme_mmio_doorbell_cq(cq->cqid, new_head);
762788eea45cSKlaus Jensen
762888eea45cSKlaus Jensen start_sqs = nvme_cq_full(cq) ? 1 : 0;
762988eea45cSKlaus Jensen cq->head = new_head;
76303f7fe8deSJinhao Fan if (!qid && n->dbbuf_enabled) {
7631c1e244b6SKlaus Jensen stl_le_pci_dma(pci, cq->db_addr, cq->head, MEMTXATTRS_UNSPECIFIED);
76323f7fe8deSJinhao Fan }
763388eea45cSKlaus Jensen if (start_sqs) {
763488eea45cSKlaus Jensen NvmeSQueue *sq;
763588eea45cSKlaus Jensen QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
7636d38cc6fdSKlaus Jensen qemu_bh_schedule(sq->bh);
763788eea45cSKlaus Jensen }
7638d38cc6fdSKlaus Jensen qemu_bh_schedule(cq->bh);
763988eea45cSKlaus Jensen }
764088eea45cSKlaus Jensen
764188eea45cSKlaus Jensen if (cq->tail == cq->head) {
764283d7ed5cSKlaus Jensen if (cq->irq_enabled) {
764383d7ed5cSKlaus Jensen n->cq_pending--;
764483d7ed5cSKlaus Jensen }
764583d7ed5cSKlaus Jensen
764688eea45cSKlaus Jensen nvme_irq_deassert(n, cq);
764788eea45cSKlaus Jensen }
764888eea45cSKlaus Jensen } else {
764988eea45cSKlaus Jensen /* Submission queue doorbell write */
765088eea45cSKlaus Jensen
765188eea45cSKlaus Jensen uint16_t new_tail = val & 0xffff;
765288eea45cSKlaus Jensen NvmeSQueue *sq;
765388eea45cSKlaus Jensen
765488eea45cSKlaus Jensen qid = (addr - 0x1000) >> 3;
765588eea45cSKlaus Jensen if (unlikely(nvme_check_sqid(n, qid))) {
765688eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sq,
765788eea45cSKlaus Jensen "submission queue doorbell write"
765888eea45cSKlaus Jensen " for nonexistent queue,"
765988eea45cSKlaus Jensen " sqid=%"PRIu32", ignoring", qid);
766088eea45cSKlaus Jensen
766188eea45cSKlaus Jensen if (n->outstanding_aers) {
766288eea45cSKlaus Jensen nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
766388eea45cSKlaus Jensen NVME_AER_INFO_ERR_INVALID_DB_REGISTER,
766488eea45cSKlaus Jensen NVME_LOG_ERROR_INFO);
766588eea45cSKlaus Jensen }
766688eea45cSKlaus Jensen
766788eea45cSKlaus Jensen return;
766888eea45cSKlaus Jensen }
766988eea45cSKlaus Jensen
767088eea45cSKlaus Jensen sq = n->sq[qid];
767188eea45cSKlaus Jensen if (unlikely(new_tail >= sq->size)) {
767288eea45cSKlaus Jensen NVME_GUEST_ERR(pci_nvme_ub_db_wr_invalid_sqtail,
767388eea45cSKlaus Jensen "submission queue doorbell write value"
767488eea45cSKlaus Jensen " beyond queue size, sqid=%"PRIu32","
767588eea45cSKlaus Jensen " new_tail=%"PRIu16", ignoring",
767688eea45cSKlaus Jensen qid, new_tail);
767788eea45cSKlaus Jensen
767888eea45cSKlaus Jensen if (n->outstanding_aers) {
767988eea45cSKlaus Jensen nvme_enqueue_event(n, NVME_AER_TYPE_ERROR,
768088eea45cSKlaus Jensen NVME_AER_INFO_ERR_INVALID_DB_VALUE,
768188eea45cSKlaus Jensen NVME_LOG_ERROR_INFO);
768288eea45cSKlaus Jensen }
768388eea45cSKlaus Jensen
768488eea45cSKlaus Jensen return;
768588eea45cSKlaus Jensen }
768688eea45cSKlaus Jensen
768788eea45cSKlaus Jensen trace_pci_nvme_mmio_doorbell_sq(sq->sqid, new_tail);
768888eea45cSKlaus Jensen
768988eea45cSKlaus Jensen sq->tail = new_tail;
76903f7fe8deSJinhao Fan if (!qid && n->dbbuf_enabled) {
76913f7fe8deSJinhao Fan /*
76923f7fe8deSJinhao Fan * The spec states "the host shall also update the controller's
76933f7fe8deSJinhao Fan * corresponding doorbell property to match the value of that entry
76943f7fe8deSJinhao Fan * in the Shadow Doorbell buffer."
76953f7fe8deSJinhao Fan *
76963f7fe8deSJinhao Fan * Since this context is currently a VM trap, we can safely enforce
76973f7fe8deSJinhao Fan * the requirement from the device side in case the host is
76983f7fe8deSJinhao Fan * misbehaving.
76993f7fe8deSJinhao Fan *
77003f7fe8deSJinhao Fan * Note, we shouldn't have to do this, but various drivers
77013f7fe8deSJinhao Fan * including ones that run on Linux, are not updating Admin Queues,
77023f7fe8deSJinhao Fan * so we can't trust reading it for an appropriate sq tail.
77033f7fe8deSJinhao Fan */
7704c1e244b6SKlaus Jensen stl_le_pci_dma(pci, sq->db_addr, sq->tail, MEMTXATTRS_UNSPECIFIED);
77053f7fe8deSJinhao Fan }
7706d38cc6fdSKlaus Jensen
7707d38cc6fdSKlaus Jensen qemu_bh_schedule(sq->bh);
770888eea45cSKlaus Jensen }
770988eea45cSKlaus Jensen }
771088eea45cSKlaus Jensen
nvme_mmio_write(void * opaque,hwaddr addr,uint64_t data,unsigned size)771188eea45cSKlaus Jensen static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
771288eea45cSKlaus Jensen unsigned size)
771388eea45cSKlaus Jensen {
771488eea45cSKlaus Jensen NvmeCtrl *n = (NvmeCtrl *)opaque;
771588eea45cSKlaus Jensen
771688eea45cSKlaus Jensen trace_pci_nvme_mmio_write(addr, data, size);
771788eea45cSKlaus Jensen
771848b32c28SKlaus Jensen if (pci_is_vf(PCI_DEVICE(n)) && !nvme_sctrl(n)->scs &&
771911871f53SŁukasz Gieryk addr != NVME_REG_CSTS) {
772011871f53SŁukasz Gieryk trace_pci_nvme_err_ignored_mmio_vf_offline(addr, size);
772111871f53SŁukasz Gieryk return;
772211871f53SŁukasz Gieryk }
772311871f53SŁukasz Gieryk
772488eea45cSKlaus Jensen if (addr < sizeof(n->bar)) {
772588eea45cSKlaus Jensen nvme_write_bar(n, addr, data, size);
772688eea45cSKlaus Jensen } else {
772788eea45cSKlaus Jensen nvme_process_db(n, addr, data);
772888eea45cSKlaus Jensen }
772988eea45cSKlaus Jensen }
773088eea45cSKlaus Jensen
773188eea45cSKlaus Jensen static const MemoryRegionOps nvme_mmio_ops = {
773288eea45cSKlaus Jensen .read = nvme_mmio_read,
773388eea45cSKlaus Jensen .write = nvme_mmio_write,
773488eea45cSKlaus Jensen .endianness = DEVICE_LITTLE_ENDIAN,
773588eea45cSKlaus Jensen .impl = {
773688eea45cSKlaus Jensen .min_access_size = 2,
773788eea45cSKlaus Jensen .max_access_size = 8,
773888eea45cSKlaus Jensen },
773988eea45cSKlaus Jensen };
774088eea45cSKlaus Jensen
nvme_cmb_write(void * opaque,hwaddr addr,uint64_t data,unsigned size)774188eea45cSKlaus Jensen static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data,
774288eea45cSKlaus Jensen unsigned size)
774388eea45cSKlaus Jensen {
774488eea45cSKlaus Jensen NvmeCtrl *n = (NvmeCtrl *)opaque;
774588eea45cSKlaus Jensen stn_le_p(&n->cmb.buf[addr], size, data);
774688eea45cSKlaus Jensen }
774788eea45cSKlaus Jensen
nvme_cmb_read(void * opaque,hwaddr addr,unsigned size)774888eea45cSKlaus Jensen static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size)
774988eea45cSKlaus Jensen {
775088eea45cSKlaus Jensen NvmeCtrl *n = (NvmeCtrl *)opaque;
775188eea45cSKlaus Jensen return ldn_le_p(&n->cmb.buf[addr], size);
775288eea45cSKlaus Jensen }
775388eea45cSKlaus Jensen
775488eea45cSKlaus Jensen static const MemoryRegionOps nvme_cmb_ops = {
775588eea45cSKlaus Jensen .read = nvme_cmb_read,
775688eea45cSKlaus Jensen .write = nvme_cmb_write,
775788eea45cSKlaus Jensen .endianness = DEVICE_LITTLE_ENDIAN,
775888eea45cSKlaus Jensen .impl = {
775988eea45cSKlaus Jensen .min_access_size = 1,
776088eea45cSKlaus Jensen .max_access_size = 8,
776188eea45cSKlaus Jensen },
776288eea45cSKlaus Jensen };
776388eea45cSKlaus Jensen
nvme_check_params(NvmeCtrl * n,Error ** errp)7764784fd353SKlaus Jensen static bool nvme_check_params(NvmeCtrl *n, Error **errp)
776588eea45cSKlaus Jensen {
776688eea45cSKlaus Jensen NvmeParams *params = &n->params;
776788eea45cSKlaus Jensen
776888eea45cSKlaus Jensen if (params->num_queues) {
776988eea45cSKlaus Jensen warn_report("num_queues is deprecated; please use max_ioqpairs "
777088eea45cSKlaus Jensen "instead");
777188eea45cSKlaus Jensen
777288eea45cSKlaus Jensen params->max_ioqpairs = params->num_queues - 1;
777388eea45cSKlaus Jensen }
777488eea45cSKlaus Jensen
777588eea45cSKlaus Jensen if (n->namespace.blkconf.blk && n->subsys) {
777688eea45cSKlaus Jensen error_setg(errp, "subsystem support is unavailable with legacy "
777788eea45cSKlaus Jensen "namespace ('drive' property)");
7778784fd353SKlaus Jensen return false;
777988eea45cSKlaus Jensen }
778088eea45cSKlaus Jensen
778188eea45cSKlaus Jensen if (params->max_ioqpairs < 1 ||
778288eea45cSKlaus Jensen params->max_ioqpairs > NVME_MAX_IOQPAIRS) {
778388eea45cSKlaus Jensen error_setg(errp, "max_ioqpairs must be between 1 and %d",
778488eea45cSKlaus Jensen NVME_MAX_IOQPAIRS);
7785784fd353SKlaus Jensen return false;
778688eea45cSKlaus Jensen }
778788eea45cSKlaus Jensen
778888eea45cSKlaus Jensen if (params->msix_qsize < 1 ||
778988eea45cSKlaus Jensen params->msix_qsize > PCI_MSIX_FLAGS_QSIZE + 1) {
779088eea45cSKlaus Jensen error_setg(errp, "msix_qsize must be between 1 and %d",
779188eea45cSKlaus Jensen PCI_MSIX_FLAGS_QSIZE + 1);
7792784fd353SKlaus Jensen return false;
779388eea45cSKlaus Jensen }
779488eea45cSKlaus Jensen
779588eea45cSKlaus Jensen if (!params->serial) {
779688eea45cSKlaus Jensen error_setg(errp, "serial property not set");
7797784fd353SKlaus Jensen return false;
779888eea45cSKlaus Jensen }
779988eea45cSKlaus Jensen
780088eea45cSKlaus Jensen if (n->pmr.dev) {
780188eea45cSKlaus Jensen if (host_memory_backend_is_mapped(n->pmr.dev)) {
780288eea45cSKlaus Jensen error_setg(errp, "can't use already busy memdev: %s",
780388eea45cSKlaus Jensen object_get_canonical_path_component(OBJECT(n->pmr.dev)));
7804784fd353SKlaus Jensen return false;
780588eea45cSKlaus Jensen }
780688eea45cSKlaus Jensen
780788eea45cSKlaus Jensen if (!is_power_of_2(n->pmr.dev->size)) {
780888eea45cSKlaus Jensen error_setg(errp, "pmr backend size needs to be power of 2 in size");
7809784fd353SKlaus Jensen return false;
781088eea45cSKlaus Jensen }
781188eea45cSKlaus Jensen
781288eea45cSKlaus Jensen host_memory_backend_set_mapped(n->pmr.dev, true);
781388eea45cSKlaus Jensen }
781488eea45cSKlaus Jensen
781588eea45cSKlaus Jensen if (n->params.zasl > n->params.mdts) {
781688eea45cSKlaus Jensen error_setg(errp, "zoned.zasl (Zone Append Size Limit) must be less "
781788eea45cSKlaus Jensen "than or equal to mdts (Maximum Data Transfer Size)");
7818784fd353SKlaus Jensen return false;
781988eea45cSKlaus Jensen }
782088eea45cSKlaus Jensen
782188eea45cSKlaus Jensen if (!n->params.vsl) {
782288eea45cSKlaus Jensen error_setg(errp, "vsl must be non-zero");
7823784fd353SKlaus Jensen return false;
782488eea45cSKlaus Jensen }
782544c2c094SLukasz Maniak
782644c2c094SLukasz Maniak if (params->sriov_max_vfs) {
782744c2c094SLukasz Maniak if (!n->subsys) {
782844c2c094SLukasz Maniak error_setg(errp, "subsystem is required for the use of SR-IOV");
7829784fd353SKlaus Jensen return false;
783044c2c094SLukasz Maniak }
783144c2c094SLukasz Maniak
783244c2c094SLukasz Maniak if (params->sriov_max_vfs > NVME_MAX_VFS) {
783344c2c094SLukasz Maniak error_setg(errp, "sriov_max_vfs must be between 0 and %d",
783444c2c094SLukasz Maniak NVME_MAX_VFS);
7835784fd353SKlaus Jensen return false;
783644c2c094SLukasz Maniak }
783744c2c094SLukasz Maniak
783844c2c094SLukasz Maniak if (params->cmb_size_mb) {
783944c2c094SLukasz Maniak error_setg(errp, "CMB is not supported with SR-IOV");
7840784fd353SKlaus Jensen return false;
784144c2c094SLukasz Maniak }
784244c2c094SLukasz Maniak
784344c2c094SLukasz Maniak if (n->pmr.dev) {
784444c2c094SLukasz Maniak error_setg(errp, "PMR is not supported with SR-IOV");
7845784fd353SKlaus Jensen return false;
784644c2c094SLukasz Maniak }
7847746d42b1SŁukasz Gieryk
7848746d42b1SŁukasz Gieryk if (!params->sriov_vq_flexible || !params->sriov_vi_flexible) {
7849746d42b1SŁukasz Gieryk error_setg(errp, "both sriov_vq_flexible and sriov_vi_flexible"
7850746d42b1SŁukasz Gieryk " must be set for the use of SR-IOV");
7851784fd353SKlaus Jensen return false;
7852746d42b1SŁukasz Gieryk }
7853746d42b1SŁukasz Gieryk
7854746d42b1SŁukasz Gieryk if (params->sriov_vq_flexible < params->sriov_max_vfs * 2) {
7855746d42b1SŁukasz Gieryk error_setg(errp, "sriov_vq_flexible must be greater than or equal"
7856746d42b1SŁukasz Gieryk " to %d (sriov_max_vfs * 2)", params->sriov_max_vfs * 2);
7857784fd353SKlaus Jensen return false;
7858746d42b1SŁukasz Gieryk }
7859746d42b1SŁukasz Gieryk
7860746d42b1SŁukasz Gieryk if (params->max_ioqpairs < params->sriov_vq_flexible + 2) {
7861746d42b1SŁukasz Gieryk error_setg(errp, "(max_ioqpairs - sriov_vq_flexible) must be"
7862746d42b1SŁukasz Gieryk " greater than or equal to 2");
7863784fd353SKlaus Jensen return false;
7864746d42b1SŁukasz Gieryk }
7865746d42b1SŁukasz Gieryk
7866746d42b1SŁukasz Gieryk if (params->sriov_vi_flexible < params->sriov_max_vfs) {
7867746d42b1SŁukasz Gieryk error_setg(errp, "sriov_vi_flexible must be greater than or equal"
7868746d42b1SŁukasz Gieryk " to %d (sriov_max_vfs)", params->sriov_max_vfs);
7869784fd353SKlaus Jensen return false;
7870746d42b1SŁukasz Gieryk }
7871746d42b1SŁukasz Gieryk
7872746d42b1SŁukasz Gieryk if (params->msix_qsize < params->sriov_vi_flexible + 1) {
7873746d42b1SŁukasz Gieryk error_setg(errp, "(msix_qsize - sriov_vi_flexible) must be"
7874746d42b1SŁukasz Gieryk " greater than or equal to 1");
7875784fd353SKlaus Jensen return false;
7876746d42b1SŁukasz Gieryk }
7877746d42b1SŁukasz Gieryk
7878746d42b1SŁukasz Gieryk if (params->sriov_max_vi_per_vf &&
7879746d42b1SŁukasz Gieryk (params->sriov_max_vi_per_vf - 1) % NVME_VF_RES_GRANULARITY) {
7880746d42b1SŁukasz Gieryk error_setg(errp, "sriov_max_vi_per_vf must meet:"
7881746d42b1SŁukasz Gieryk " (sriov_max_vi_per_vf - 1) %% %d == 0 and"
7882746d42b1SŁukasz Gieryk " sriov_max_vi_per_vf >= 1", NVME_VF_RES_GRANULARITY);
7883784fd353SKlaus Jensen return false;
7884746d42b1SŁukasz Gieryk }
7885746d42b1SŁukasz Gieryk
7886746d42b1SŁukasz Gieryk if (params->sriov_max_vq_per_vf &&
7887746d42b1SŁukasz Gieryk (params->sriov_max_vq_per_vf < 2 ||
7888746d42b1SŁukasz Gieryk (params->sriov_max_vq_per_vf - 1) % NVME_VF_RES_GRANULARITY)) {
7889746d42b1SŁukasz Gieryk error_setg(errp, "sriov_max_vq_per_vf must meet:"
7890746d42b1SŁukasz Gieryk " (sriov_max_vq_per_vf - 1) %% %d == 0 and"
7891746d42b1SŁukasz Gieryk " sriov_max_vq_per_vf >= 2", NVME_VF_RES_GRANULARITY);
7892784fd353SKlaus Jensen return false;
7893746d42b1SŁukasz Gieryk }
789444c2c094SLukasz Maniak }
7895784fd353SKlaus Jensen
7896784fd353SKlaus Jensen return true;
789788eea45cSKlaus Jensen }
789888eea45cSKlaus Jensen
nvme_init_state(NvmeCtrl * n)789988eea45cSKlaus Jensen static void nvme_init_state(NvmeCtrl *n)
790088eea45cSKlaus Jensen {
79015e6f963fSLukasz Maniak NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
790299f48ae7SLukasz Maniak NvmeSecCtrlList *list = &n->sec_ctrl_list;
790399f48ae7SLukasz Maniak NvmeSecCtrlEntry *sctrl;
790448b32c28SKlaus Jensen PCIDevice *pci = PCI_DEVICE(n);
7905746d42b1SŁukasz Gieryk uint8_t max_vfs;
790699f48ae7SLukasz Maniak int i;
79075e6f963fSLukasz Maniak
790848b32c28SKlaus Jensen if (pci_is_vf(pci)) {
7909746d42b1SŁukasz Gieryk sctrl = nvme_sctrl(n);
7910746d42b1SŁukasz Gieryk max_vfs = 0;
7911746d42b1SŁukasz Gieryk n->conf_ioqpairs = sctrl->nvq ? le16_to_cpu(sctrl->nvq) - 1 : 0;
7912746d42b1SŁukasz Gieryk n->conf_msix_qsize = sctrl->nvi ? le16_to_cpu(sctrl->nvi) : 1;
7913746d42b1SŁukasz Gieryk } else {
7914746d42b1SŁukasz Gieryk max_vfs = n->params.sriov_max_vfs;
7915decc0261SŁukasz Gieryk n->conf_ioqpairs = n->params.max_ioqpairs;
7916decc0261SŁukasz Gieryk n->conf_msix_qsize = n->params.msix_qsize;
7917746d42b1SŁukasz Gieryk }
7918decc0261SŁukasz Gieryk
791988eea45cSKlaus Jensen n->sq = g_new0(NvmeSQueue *, n->params.max_ioqpairs + 1);
792088eea45cSKlaus Jensen n->cq = g_new0(NvmeCQueue *, n->params.max_ioqpairs + 1);
792188eea45cSKlaus Jensen n->temperature = NVME_TEMPERATURE;
792288eea45cSKlaus Jensen n->features.temp_thresh_hi = NVME_TEMPERATURE_WARNING;
792388eea45cSKlaus Jensen n->starttime_ms = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
792488eea45cSKlaus Jensen n->aer_reqs = g_new0(NvmeRequest *, n->params.aerl + 1);
7925b7698b91SŁukasz Gieryk QTAILQ_INIT(&n->aer_queue);
79265e6f963fSLukasz Maniak
7927*e4e36e65SKlaus Jensen list->numcntl = max_vfs;
7928746d42b1SŁukasz Gieryk for (i = 0; i < max_vfs; i++) {
792999f48ae7SLukasz Maniak sctrl = &list->sec[i];
793099f48ae7SLukasz Maniak sctrl->pcid = cpu_to_le16(n->cntlid);
793199f48ae7SLukasz Maniak sctrl->vfn = cpu_to_le16(i + 1);
793299f48ae7SLukasz Maniak }
793399f48ae7SLukasz Maniak
79345e6f963fSLukasz Maniak cap->cntlid = cpu_to_le16(n->cntlid);
7935746d42b1SŁukasz Gieryk cap->crt = NVME_CRT_VQ | NVME_CRT_VI;
7936746d42b1SŁukasz Gieryk
793748b32c28SKlaus Jensen if (pci_is_vf(pci)) {
7938746d42b1SŁukasz Gieryk cap->vqprt = cpu_to_le16(1 + n->conf_ioqpairs);
7939746d42b1SŁukasz Gieryk } else {
7940746d42b1SŁukasz Gieryk cap->vqprt = cpu_to_le16(1 + n->params.max_ioqpairs -
7941746d42b1SŁukasz Gieryk n->params.sriov_vq_flexible);
7942746d42b1SŁukasz Gieryk cap->vqfrt = cpu_to_le32(n->params.sriov_vq_flexible);
7943746d42b1SŁukasz Gieryk cap->vqrfap = cap->vqfrt;
7944746d42b1SŁukasz Gieryk cap->vqgran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
7945746d42b1SŁukasz Gieryk cap->vqfrsm = n->params.sriov_max_vq_per_vf ?
7946746d42b1SŁukasz Gieryk cpu_to_le16(n->params.sriov_max_vq_per_vf) :
7947746d42b1SŁukasz Gieryk cap->vqfrt / MAX(max_vfs, 1);
7948746d42b1SŁukasz Gieryk }
7949746d42b1SŁukasz Gieryk
795048b32c28SKlaus Jensen if (pci_is_vf(pci)) {
7951746d42b1SŁukasz Gieryk cap->viprt = cpu_to_le16(n->conf_msix_qsize);
7952746d42b1SŁukasz Gieryk } else {
7953746d42b1SŁukasz Gieryk cap->viprt = cpu_to_le16(n->params.msix_qsize -
7954746d42b1SŁukasz Gieryk n->params.sriov_vi_flexible);
7955746d42b1SŁukasz Gieryk cap->vifrt = cpu_to_le32(n->params.sriov_vi_flexible);
7956746d42b1SŁukasz Gieryk cap->virfap = cap->vifrt;
7957746d42b1SŁukasz Gieryk cap->vigran = cpu_to_le16(NVME_VF_RES_GRANULARITY);
7958746d42b1SŁukasz Gieryk cap->vifrsm = n->params.sriov_max_vi_per_vf ?
7959746d42b1SŁukasz Gieryk cpu_to_le16(n->params.sriov_max_vi_per_vf) :
7960746d42b1SŁukasz Gieryk cap->vifrt / MAX(max_vfs, 1);
7961746d42b1SŁukasz Gieryk }
796288eea45cSKlaus Jensen }
796388eea45cSKlaus Jensen
nvme_init_cmb(NvmeCtrl * n,PCIDevice * pci_dev)796488eea45cSKlaus Jensen static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
796588eea45cSKlaus Jensen {
796688eea45cSKlaus Jensen uint64_t cmb_size = n->params.cmb_size_mb * MiB;
796749e03457SKlaus Jensen uint64_t cap = ldq_le_p(&n->bar.cap);
796888eea45cSKlaus Jensen
796988eea45cSKlaus Jensen n->cmb.buf = g_malloc0(cmb_size);
797088eea45cSKlaus Jensen memory_region_init_io(&n->cmb.mem, OBJECT(n), &nvme_cmb_ops, n,
797188eea45cSKlaus Jensen "nvme-cmb", cmb_size);
797288eea45cSKlaus Jensen pci_register_bar(pci_dev, NVME_CMB_BIR,
797388eea45cSKlaus Jensen PCI_BASE_ADDRESS_SPACE_MEMORY |
797488eea45cSKlaus Jensen PCI_BASE_ADDRESS_MEM_TYPE_64 |
797588eea45cSKlaus Jensen PCI_BASE_ADDRESS_MEM_PREFETCH, &n->cmb.mem);
797688eea45cSKlaus Jensen
797749e03457SKlaus Jensen NVME_CAP_SET_CMBS(cap, 1);
797849e03457SKlaus Jensen stq_le_p(&n->bar.cap, cap);
797988eea45cSKlaus Jensen
798088eea45cSKlaus Jensen if (n->params.legacy_cmb) {
798188eea45cSKlaus Jensen nvme_cmb_enable_regs(n);
798288eea45cSKlaus Jensen n->cmb.cmse = true;
798388eea45cSKlaus Jensen }
798488eea45cSKlaus Jensen }
798588eea45cSKlaus Jensen
nvme_init_pmr(NvmeCtrl * n,PCIDevice * pci_dev)798688eea45cSKlaus Jensen static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
798788eea45cSKlaus Jensen {
798849e03457SKlaus Jensen uint32_t pmrcap = ldl_le_p(&n->bar.pmrcap);
798988eea45cSKlaus Jensen
799049e03457SKlaus Jensen NVME_PMRCAP_SET_RDS(pmrcap, 1);
799149e03457SKlaus Jensen NVME_PMRCAP_SET_WDS(pmrcap, 1);
799249e03457SKlaus Jensen NVME_PMRCAP_SET_BIR(pmrcap, NVME_PMR_BIR);
799349e03457SKlaus Jensen /* Turn on bit 1 support */
799449e03457SKlaus Jensen NVME_PMRCAP_SET_PMRWBM(pmrcap, 0x02);
799549e03457SKlaus Jensen NVME_PMRCAP_SET_CMSS(pmrcap, 1);
799649e03457SKlaus Jensen stl_le_p(&n->bar.pmrcap, pmrcap);
799749e03457SKlaus Jensen
799849e03457SKlaus Jensen pci_register_bar(pci_dev, NVME_PMR_BIR,
799988eea45cSKlaus Jensen PCI_BASE_ADDRESS_SPACE_MEMORY |
800088eea45cSKlaus Jensen PCI_BASE_ADDRESS_MEM_TYPE_64 |
800188eea45cSKlaus Jensen PCI_BASE_ADDRESS_MEM_PREFETCH, &n->pmr.dev->mr);
800288eea45cSKlaus Jensen
800388eea45cSKlaus Jensen memory_region_set_enabled(&n->pmr.dev->mr, false);
800488eea45cSKlaus Jensen }
800588eea45cSKlaus Jensen
nvme_bar_size(unsigned total_queues,unsigned total_irqs,unsigned * msix_table_offset,unsigned * msix_pba_offset)8006aa817713SŁukasz Gieryk static uint64_t nvme_bar_size(unsigned total_queues, unsigned total_irqs,
8007aa817713SŁukasz Gieryk unsigned *msix_table_offset,
8008aa817713SŁukasz Gieryk unsigned *msix_pba_offset)
8009aa817713SŁukasz Gieryk {
8010aa817713SŁukasz Gieryk uint64_t bar_size, msix_table_size, msix_pba_size;
8011aa817713SŁukasz Gieryk
8012aa817713SŁukasz Gieryk bar_size = sizeof(NvmeBar) + 2 * total_queues * NVME_DB_SIZE;
8013aa817713SŁukasz Gieryk bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8014aa817713SŁukasz Gieryk
8015aa817713SŁukasz Gieryk if (msix_table_offset) {
8016aa817713SŁukasz Gieryk *msix_table_offset = bar_size;
8017aa817713SŁukasz Gieryk }
8018aa817713SŁukasz Gieryk
8019aa817713SŁukasz Gieryk msix_table_size = PCI_MSIX_ENTRY_SIZE * total_irqs;
8020aa817713SŁukasz Gieryk bar_size += msix_table_size;
8021aa817713SŁukasz Gieryk bar_size = QEMU_ALIGN_UP(bar_size, 4 * KiB);
8022aa817713SŁukasz Gieryk
8023aa817713SŁukasz Gieryk if (msix_pba_offset) {
8024aa817713SŁukasz Gieryk *msix_pba_offset = bar_size;
8025aa817713SŁukasz Gieryk }
8026aa817713SŁukasz Gieryk
8027aa817713SŁukasz Gieryk msix_pba_size = QEMU_ALIGN_UP(total_irqs, 64) / 8;
8028aa817713SŁukasz Gieryk bar_size += msix_pba_size;
8029aa817713SŁukasz Gieryk
8030aa817713SŁukasz Gieryk bar_size = pow2ceil(bar_size);
8031aa817713SŁukasz Gieryk return bar_size;
8032aa817713SŁukasz Gieryk }
8033aa817713SŁukasz Gieryk
nvme_init_sriov(NvmeCtrl * n,PCIDevice * pci_dev,uint16_t offset)8034746d42b1SŁukasz Gieryk static void nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset)
803544c2c094SLukasz Maniak {
803644c2c094SLukasz Maniak uint16_t vf_dev_id = n->params.use_intel_id ?
803744c2c094SLukasz Maniak PCI_DEVICE_ID_INTEL_NVME : PCI_DEVICE_ID_REDHAT_NVME;
8038746d42b1SŁukasz Gieryk NvmePriCtrlCap *cap = &n->pri_ctrl_cap;
8039746d42b1SŁukasz Gieryk uint64_t bar_size = nvme_bar_size(le16_to_cpu(cap->vqfrsm),
8040746d42b1SŁukasz Gieryk le16_to_cpu(cap->vifrsm),
8041746d42b1SŁukasz Gieryk NULL, NULL);
804244c2c094SLukasz Maniak
804344c2c094SLukasz Maniak pcie_sriov_pf_init(pci_dev, offset, "nvme", vf_dev_id,
804444c2c094SLukasz Maniak n->params.sriov_max_vfs, n->params.sriov_max_vfs,
804544c2c094SLukasz Maniak NVME_VF_OFFSET, NVME_VF_STRIDE);
804644c2c094SLukasz Maniak
804744c2c094SLukasz Maniak pcie_sriov_pf_init_vf_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
804844c2c094SLukasz Maniak PCI_BASE_ADDRESS_MEM_TYPE_64, bar_size);
804944c2c094SLukasz Maniak }
805044c2c094SLukasz Maniak
nvme_add_pm_capability(PCIDevice * pci_dev,uint8_t offset)80511e9c685eSŁukasz Gieryk static int nvme_add_pm_capability(PCIDevice *pci_dev, uint8_t offset)
80521e9c685eSŁukasz Gieryk {
80531e9c685eSŁukasz Gieryk Error *err = NULL;
80541e9c685eSŁukasz Gieryk int ret;
80551e9c685eSŁukasz Gieryk
80561e9c685eSŁukasz Gieryk ret = pci_add_capability(pci_dev, PCI_CAP_ID_PM, offset,
80571e9c685eSŁukasz Gieryk PCI_PM_SIZEOF, &err);
80581e9c685eSŁukasz Gieryk if (err) {
80591e9c685eSŁukasz Gieryk error_report_err(err);
80601e9c685eSŁukasz Gieryk return ret;
80611e9c685eSŁukasz Gieryk }
80621e9c685eSŁukasz Gieryk
80631e9c685eSŁukasz Gieryk pci_set_word(pci_dev->config + offset + PCI_PM_PMC,
80641e9c685eSŁukasz Gieryk PCI_PM_CAP_VER_1_2);
80651e9c685eSŁukasz Gieryk pci_set_word(pci_dev->config + offset + PCI_PM_CTRL,
80661e9c685eSŁukasz Gieryk PCI_PM_CTRL_NO_SOFT_RESET);
80671e9c685eSŁukasz Gieryk pci_set_word(pci_dev->wmask + offset + PCI_PM_CTRL,
80681e9c685eSŁukasz Gieryk PCI_PM_CTRL_STATE_MASK);
80691e9c685eSŁukasz Gieryk
80701e9c685eSŁukasz Gieryk return 0;
80711e9c685eSŁukasz Gieryk }
80721e9c685eSŁukasz Gieryk
nvme_init_pci(NvmeCtrl * n,PCIDevice * pci_dev,Error ** errp)8073973f76cfSKlaus Jensen static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)
807488eea45cSKlaus Jensen {
8075973f76cfSKlaus Jensen ERRP_GUARD();
807688eea45cSKlaus Jensen uint8_t *pci_conf = pci_dev->config;
8077aa817713SŁukasz Gieryk uint64_t bar_size;
807888eea45cSKlaus Jensen unsigned msix_table_offset, msix_pba_offset;
807988eea45cSKlaus Jensen int ret;
808088eea45cSKlaus Jensen
808188eea45cSKlaus Jensen pci_conf[PCI_INTERRUPT_PIN] = 1;
808288eea45cSKlaus Jensen pci_config_set_prog_interface(pci_conf, 0x2);
808388eea45cSKlaus Jensen
808488eea45cSKlaus Jensen if (n->params.use_intel_id) {
808588eea45cSKlaus Jensen pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_INTEL);
808644c2c094SLukasz Maniak pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_INTEL_NVME);
808788eea45cSKlaus Jensen } else {
808888eea45cSKlaus Jensen pci_config_set_vendor_id(pci_conf, PCI_VENDOR_ID_REDHAT);
808988eea45cSKlaus Jensen pci_config_set_device_id(pci_conf, PCI_DEVICE_ID_REDHAT_NVME);
809088eea45cSKlaus Jensen }
809188eea45cSKlaus Jensen
809288eea45cSKlaus Jensen pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
80931e9c685eSŁukasz Gieryk nvme_add_pm_capability(pci_dev, 0x60);
809488eea45cSKlaus Jensen pcie_endpoint_cap_init(pci_dev, 0x80);
80951e9c685eSŁukasz Gieryk pcie_cap_flr_init(pci_dev);
809644c2c094SLukasz Maniak if (n->params.sriov_max_vfs) {
8097445416e3SAkihiko Odaki pcie_ari_init(pci_dev, 0x100);
809844c2c094SLukasz Maniak }
809988eea45cSKlaus Jensen
81003bfcc517SŁukasz Gieryk /* add one to max_ioqpairs to account for the admin queue pair */
8101aa817713SŁukasz Gieryk bar_size = nvme_bar_size(n->params.max_ioqpairs + 1, n->params.msix_qsize,
8102aa817713SŁukasz Gieryk &msix_table_offset, &msix_pba_offset);
810388eea45cSKlaus Jensen
810488eea45cSKlaus Jensen memory_region_init(&n->bar0, OBJECT(n), "nvme-bar0", bar_size);
810588eea45cSKlaus Jensen memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n, "nvme",
81063bfcc517SŁukasz Gieryk msix_table_offset);
810788eea45cSKlaus Jensen memory_region_add_subregion(&n->bar0, 0, &n->iomem);
810888eea45cSKlaus Jensen
810944c2c094SLukasz Maniak if (pci_is_vf(pci_dev)) {
811044c2c094SLukasz Maniak pcie_sriov_vf_register_bar(pci_dev, 0, &n->bar0);
811144c2c094SLukasz Maniak } else {
811288eea45cSKlaus Jensen pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
811388eea45cSKlaus Jensen PCI_BASE_ADDRESS_MEM_TYPE_64, &n->bar0);
811444c2c094SLukasz Maniak }
811588eea45cSKlaus Jensen ret = msix_init(pci_dev, n->params.msix_qsize,
811688eea45cSKlaus Jensen &n->bar0, 0, msix_table_offset,
8117973f76cfSKlaus Jensen &n->bar0, 0, msix_pba_offset, 0, errp);
811888eea45cSKlaus Jensen if (ret == -ENOTSUP) {
8119973f76cfSKlaus Jensen /* report that msix is not supported, but do not error out */
8120973f76cfSKlaus Jensen warn_report_err(*errp);
8121973f76cfSKlaus Jensen *errp = NULL;
8122973f76cfSKlaus Jensen } else if (ret < 0) {
8123973f76cfSKlaus Jensen /* propagate error to caller */
8124973f76cfSKlaus Jensen return false;
812588eea45cSKlaus Jensen }
812688eea45cSKlaus Jensen
8127decc0261SŁukasz Gieryk nvme_update_msixcap_ts(pci_dev, n->conf_msix_qsize);
8128decc0261SŁukasz Gieryk
812988eea45cSKlaus Jensen if (n->params.cmb_size_mb) {
813088eea45cSKlaus Jensen nvme_init_cmb(n, pci_dev);
813188eea45cSKlaus Jensen }
813288eea45cSKlaus Jensen
813388eea45cSKlaus Jensen if (n->pmr.dev) {
813488eea45cSKlaus Jensen nvme_init_pmr(n, pci_dev);
813588eea45cSKlaus Jensen }
813688eea45cSKlaus Jensen
813744c2c094SLukasz Maniak if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
8138746d42b1SŁukasz Gieryk nvme_init_sriov(n, pci_dev, 0x120);
813944c2c094SLukasz Maniak }
814044c2c094SLukasz Maniak
8141973f76cfSKlaus Jensen return true;
814288eea45cSKlaus Jensen }
814388eea45cSKlaus Jensen
nvme_init_subnqn(NvmeCtrl * n)814488eea45cSKlaus Jensen static void nvme_init_subnqn(NvmeCtrl *n)
814588eea45cSKlaus Jensen {
814688eea45cSKlaus Jensen NvmeSubsystem *subsys = n->subsys;
814788eea45cSKlaus Jensen NvmeIdCtrl *id = &n->id_ctrl;
814888eea45cSKlaus Jensen
814988eea45cSKlaus Jensen if (!subsys) {
815088eea45cSKlaus Jensen snprintf((char *)id->subnqn, sizeof(id->subnqn),
815188eea45cSKlaus Jensen "nqn.2019-08.org.qemu:%s", n->params.serial);
815288eea45cSKlaus Jensen } else {
815388eea45cSKlaus Jensen pstrcpy((char *)id->subnqn, sizeof(id->subnqn), (char*)subsys->subnqn);
815488eea45cSKlaus Jensen }
815588eea45cSKlaus Jensen }
815688eea45cSKlaus Jensen
nvme_init_ctrl(NvmeCtrl * n,PCIDevice * pci_dev)815788eea45cSKlaus Jensen static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
815888eea45cSKlaus Jensen {
815988eea45cSKlaus Jensen NvmeIdCtrl *id = &n->id_ctrl;
816088eea45cSKlaus Jensen uint8_t *pci_conf = pci_dev->config;
816149e03457SKlaus Jensen uint64_t cap = ldq_le_p(&n->bar.cap);
8162746d42b1SŁukasz Gieryk NvmeSecCtrlEntry *sctrl = nvme_sctrl(n);
8163771dbc3aSKlaus Jensen uint32_t ctratt;
816488eea45cSKlaus Jensen
816588eea45cSKlaus Jensen id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
816688eea45cSKlaus Jensen id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
816788eea45cSKlaus Jensen strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
8168fbba243bSKlaus Jensen strpadcpy((char *)id->fr, sizeof(id->fr), QEMU_VERSION, ' ');
816988eea45cSKlaus Jensen strpadcpy((char *)id->sn, sizeof(id->sn), n->params.serial, ' ');
817088eea45cSKlaus Jensen
817188eea45cSKlaus Jensen id->cntlid = cpu_to_le16(n->cntlid);
817288eea45cSKlaus Jensen
817388eea45cSKlaus Jensen id->oaes = cpu_to_le32(NVME_OAES_NS_ATTR);
8174771dbc3aSKlaus Jensen ctratt = NVME_CTRATT_ELBAS;
817588eea45cSKlaus Jensen
817688eea45cSKlaus Jensen id->rab = 6;
817788eea45cSKlaus Jensen
817888eea45cSKlaus Jensen if (n->params.use_intel_id) {
817988eea45cSKlaus Jensen id->ieee[0] = 0xb3;
818088eea45cSKlaus Jensen id->ieee[1] = 0x02;
818188eea45cSKlaus Jensen id->ieee[2] = 0x00;
818288eea45cSKlaus Jensen } else {
818388eea45cSKlaus Jensen id->ieee[0] = 0x00;
818488eea45cSKlaus Jensen id->ieee[1] = 0x54;
818588eea45cSKlaus Jensen id->ieee[2] = 0x52;
818688eea45cSKlaus Jensen }
818788eea45cSKlaus Jensen
818888eea45cSKlaus Jensen id->mdts = n->params.mdts;
818988eea45cSKlaus Jensen id->ver = cpu_to_le32(NVME_SPEC_VER);
81903f7fe8deSJinhao Fan id->oacs =
8191e181d3daSGollu Appalanaidu cpu_to_le16(NVME_OACS_NS_MGMT | NVME_OACS_FORMAT | NVME_OACS_DBBUF |
8192e181d3daSGollu Appalanaidu NVME_OACS_DIRECTIVES);
819388eea45cSKlaus Jensen id->cntrltype = 0x1;
819488eea45cSKlaus Jensen
819588eea45cSKlaus Jensen /*
819688eea45cSKlaus Jensen * Because the controller always completes the Abort command immediately,
819788eea45cSKlaus Jensen * there can never be more than one concurrently executing Abort command,
819888eea45cSKlaus Jensen * so this value is never used for anything. Note that there can easily be
819988eea45cSKlaus Jensen * many Abort commands in the queues, but they are not considered
820088eea45cSKlaus Jensen * "executing" until processed by nvme_abort.
820188eea45cSKlaus Jensen *
820288eea45cSKlaus Jensen * The specification recommends a value of 3 for Abort Command Limit (four
820388eea45cSKlaus Jensen * concurrently outstanding Abort commands), so lets use that though it is
820488eea45cSKlaus Jensen * inconsequential.
820588eea45cSKlaus Jensen */
820688eea45cSKlaus Jensen id->acl = 3;
820788eea45cSKlaus Jensen id->aerl = n->params.aerl;
820888eea45cSKlaus Jensen id->frmw = (NVME_NUM_FW_SLOTS << 1) | NVME_FRMW_SLOT1_RO;
820988eea45cSKlaus Jensen id->lpa = NVME_LPA_NS_SMART | NVME_LPA_CSE | NVME_LPA_EXTENDED;
821088eea45cSKlaus Jensen
821188eea45cSKlaus Jensen /* recommended default value (~70 C) */
821288eea45cSKlaus Jensen id->wctemp = cpu_to_le16(NVME_TEMPERATURE_WARNING);
821388eea45cSKlaus Jensen id->cctemp = cpu_to_le16(NVME_TEMPERATURE_CRITICAL);
821488eea45cSKlaus Jensen
82156a33f2e9SKlaus Jensen id->sqes = (NVME_SQES << 4) | NVME_SQES;
82166a33f2e9SKlaus Jensen id->cqes = (NVME_CQES << 4) | NVME_CQES;
821788eea45cSKlaus Jensen id->nn = cpu_to_le32(NVME_MAX_NAMESPACES);
821888eea45cSKlaus Jensen id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROES | NVME_ONCS_TIMESTAMP |
821988eea45cSKlaus Jensen NVME_ONCS_FEATURES | NVME_ONCS_DSM |
822088eea45cSKlaus Jensen NVME_ONCS_COMPARE | NVME_ONCS_COPY);
822188eea45cSKlaus Jensen
822288eea45cSKlaus Jensen /*
822388eea45cSKlaus Jensen * NOTE: If this device ever supports a command set that does NOT use 0x0
822488eea45cSKlaus Jensen * as a Flush-equivalent operation, support for the broadcast NSID in Flush
822588eea45cSKlaus Jensen * should probably be removed.
822688eea45cSKlaus Jensen *
822788eea45cSKlaus Jensen * See comment in nvme_io_cmd.
822888eea45cSKlaus Jensen */
822988eea45cSKlaus Jensen id->vwc = NVME_VWC_NSID_BROADCAST_SUPPORT | NVME_VWC_PRESENT;
823088eea45cSKlaus Jensen
823144219b60SNaveen Nagar id->ocfs = cpu_to_le16(NVME_OCFS_COPY_FORMAT_0 | NVME_OCFS_COPY_FORMAT_1);
8232b9147a3aSKlaus Jensen id->sgls = cpu_to_le32(NVME_CTRL_SGLS_SUPPORT_NO_ALIGN);
823388eea45cSKlaus Jensen
823488eea45cSKlaus Jensen nvme_init_subnqn(n);
823588eea45cSKlaus Jensen
823688eea45cSKlaus Jensen id->psd[0].mp = cpu_to_le16(0x9c4);
823788eea45cSKlaus Jensen id->psd[0].enlat = cpu_to_le32(0x10);
823888eea45cSKlaus Jensen id->psd[0].exlat = cpu_to_le32(0x4);
823988eea45cSKlaus Jensen
824088eea45cSKlaus Jensen if (n->subsys) {
824188eea45cSKlaus Jensen id->cmic |= NVME_CMIC_MULTI_CTRL;
8242771dbc3aSKlaus Jensen ctratt |= NVME_CTRATT_ENDGRPS;
8243771dbc3aSKlaus Jensen
8244771dbc3aSKlaus Jensen id->endgidmax = cpu_to_le16(0x1);
824573064edfSJesper Devantier
824673064edfSJesper Devantier if (n->subsys->endgrp.fdp.enabled) {
824773064edfSJesper Devantier ctratt |= NVME_CTRATT_FDPS;
824873064edfSJesper Devantier }
824988eea45cSKlaus Jensen }
825088eea45cSKlaus Jensen
8251771dbc3aSKlaus Jensen id->ctratt = cpu_to_le32(ctratt);
8252771dbc3aSKlaus Jensen
825349e03457SKlaus Jensen NVME_CAP_SET_MQES(cap, 0x7ff);
825449e03457SKlaus Jensen NVME_CAP_SET_CQR(cap, 1);
825549e03457SKlaus Jensen NVME_CAP_SET_TO(cap, 0xf);
825649e03457SKlaus Jensen NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_NVM);
825749e03457SKlaus Jensen NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_CSI_SUPP);
825849e03457SKlaus Jensen NVME_CAP_SET_CSS(cap, NVME_CAP_CSS_ADMIN_ONLY);
825949e03457SKlaus Jensen NVME_CAP_SET_MPSMAX(cap, 4);
826049e03457SKlaus Jensen NVME_CAP_SET_CMBS(cap, n->params.cmb_size_mb ? 1 : 0);
826149e03457SKlaus Jensen NVME_CAP_SET_PMRS(cap, n->pmr.dev ? 1 : 0);
826249e03457SKlaus Jensen stq_le_p(&n->bar.cap, cap);
826388eea45cSKlaus Jensen
826449e03457SKlaus Jensen stl_le_p(&n->bar.vs, NVME_SPEC_VER);
826588eea45cSKlaus Jensen n->bar.intmc = n->bar.intms = 0;
8266746d42b1SŁukasz Gieryk
826748b32c28SKlaus Jensen if (pci_is_vf(pci_dev) && !sctrl->scs) {
8268746d42b1SŁukasz Gieryk stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
8269746d42b1SŁukasz Gieryk }
827088eea45cSKlaus Jensen }
827188eea45cSKlaus Jensen
nvme_init_subsys(NvmeCtrl * n,Error ** errp)827288eea45cSKlaus Jensen static int nvme_init_subsys(NvmeCtrl *n, Error **errp)
827388eea45cSKlaus Jensen {
827488eea45cSKlaus Jensen int cntlid;
827588eea45cSKlaus Jensen
827688eea45cSKlaus Jensen if (!n->subsys) {
827788eea45cSKlaus Jensen return 0;
827888eea45cSKlaus Jensen }
827988eea45cSKlaus Jensen
828088eea45cSKlaus Jensen cntlid = nvme_subsys_register_ctrl(n, errp);
828188eea45cSKlaus Jensen if (cntlid < 0) {
828288eea45cSKlaus Jensen return -1;
828388eea45cSKlaus Jensen }
828488eea45cSKlaus Jensen
828588eea45cSKlaus Jensen n->cntlid = cntlid;
828688eea45cSKlaus Jensen
828788eea45cSKlaus Jensen return 0;
828888eea45cSKlaus Jensen }
828988eea45cSKlaus Jensen
nvme_attach_ns(NvmeCtrl * n,NvmeNamespace * ns)829088eea45cSKlaus Jensen void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns)
829188eea45cSKlaus Jensen {
829288eea45cSKlaus Jensen uint32_t nsid = ns->params.nsid;
829388eea45cSKlaus Jensen assert(nsid && nsid <= NVME_MAX_NAMESPACES);
829488eea45cSKlaus Jensen
829588eea45cSKlaus Jensen n->namespaces[nsid] = ns;
829688eea45cSKlaus Jensen ns->attached++;
829788eea45cSKlaus Jensen
829888eea45cSKlaus Jensen n->dmrsl = MIN_NON_ZERO(n->dmrsl,
829988eea45cSKlaus Jensen BDRV_REQUEST_MAX_BYTES / nvme_l2b(ns, 1));
830088eea45cSKlaus Jensen }
830188eea45cSKlaus Jensen
nvme_realize(PCIDevice * pci_dev,Error ** errp)830288eea45cSKlaus Jensen static void nvme_realize(PCIDevice *pci_dev, Error **errp)
830388eea45cSKlaus Jensen {
830488eea45cSKlaus Jensen NvmeCtrl *n = NVME(pci_dev);
830548b32c28SKlaus Jensen DeviceState *dev = DEVICE(pci_dev);
830688eea45cSKlaus Jensen NvmeNamespace *ns;
830744c2c094SLukasz Maniak NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));
830844c2c094SLukasz Maniak
830944c2c094SLukasz Maniak if (pci_is_vf(pci_dev)) {
831044c2c094SLukasz Maniak /*
831144c2c094SLukasz Maniak * VFs derive settings from the parent. PF's lifespan exceeds
831244c2c094SLukasz Maniak * that of VF's, so it's safe to share params.serial.
831344c2c094SLukasz Maniak */
831444c2c094SLukasz Maniak memcpy(&n->params, &pn->params, sizeof(NvmeParams));
831544c2c094SLukasz Maniak n->subsys = pn->subsys;
831644c2c094SLukasz Maniak }
831788eea45cSKlaus Jensen
8318784fd353SKlaus Jensen if (!nvme_check_params(n, errp)) {
831988eea45cSKlaus Jensen return;
832088eea45cSKlaus Jensen }
832188eea45cSKlaus Jensen
832248b32c28SKlaus Jensen qbus_init(&n->bus, sizeof(NvmeBus), TYPE_NVME_BUS, dev, dev->id);
832388eea45cSKlaus Jensen
832488eea45cSKlaus Jensen if (nvme_init_subsys(n, errp)) {
832588eea45cSKlaus Jensen return;
832688eea45cSKlaus Jensen }
83275e6f963fSLukasz Maniak nvme_init_state(n);
8328973f76cfSKlaus Jensen if (!nvme_init_pci(n, pci_dev, errp)) {
83295e6f963fSLukasz Maniak return;
83305e6f963fSLukasz Maniak }
833188eea45cSKlaus Jensen nvme_init_ctrl(n, pci_dev);
833288eea45cSKlaus Jensen
833388eea45cSKlaus Jensen /* setup a namespace if the controller drive property was given */
833488eea45cSKlaus Jensen if (n->namespace.blkconf.blk) {
833588eea45cSKlaus Jensen ns = &n->namespace;
833688eea45cSKlaus Jensen ns->params.nsid = 1;
833788eea45cSKlaus Jensen
83385e4f6bccSKlaus Jensen if (nvme_ns_setup(ns, errp)) {
833988eea45cSKlaus Jensen return;
834088eea45cSKlaus Jensen }
834188eea45cSKlaus Jensen
834288eea45cSKlaus Jensen nvme_attach_ns(n, ns);
834388eea45cSKlaus Jensen }
834488eea45cSKlaus Jensen }
834588eea45cSKlaus Jensen
nvme_exit(PCIDevice * pci_dev)834688eea45cSKlaus Jensen static void nvme_exit(PCIDevice *pci_dev)
834788eea45cSKlaus Jensen {
834888eea45cSKlaus Jensen NvmeCtrl *n = NVME(pci_dev);
834988eea45cSKlaus Jensen NvmeNamespace *ns;
835088eea45cSKlaus Jensen int i;
835188eea45cSKlaus Jensen
83521e9c685eSŁukasz Gieryk nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
835388eea45cSKlaus Jensen
83545ffbaeedSKlaus Jensen if (n->subsys) {
835588eea45cSKlaus Jensen for (i = 1; i <= NVME_MAX_NAMESPACES; i++) {
835688eea45cSKlaus Jensen ns = nvme_ns(n, i);
83575ffbaeedSKlaus Jensen if (ns) {
83585ffbaeedSKlaus Jensen ns->attached--;
83595ffbaeedSKlaus Jensen }
836088eea45cSKlaus Jensen }
836188eea45cSKlaus Jensen
8362b0fde9e8SKlaus Jensen nvme_subsys_unregister_ctrl(n->subsys, n);
8363b0fde9e8SKlaus Jensen }
8364b0fde9e8SKlaus Jensen
836588eea45cSKlaus Jensen g_free(n->cq);
836688eea45cSKlaus Jensen g_free(n->sq);
836788eea45cSKlaus Jensen g_free(n->aer_reqs);
836888eea45cSKlaus Jensen
836988eea45cSKlaus Jensen if (n->params.cmb_size_mb) {
837088eea45cSKlaus Jensen g_free(n->cmb.buf);
837188eea45cSKlaus Jensen }
837288eea45cSKlaus Jensen
837388eea45cSKlaus Jensen if (n->pmr.dev) {
837488eea45cSKlaus Jensen host_memory_backend_set_mapped(n->pmr.dev, false);
837588eea45cSKlaus Jensen }
837644c2c094SLukasz Maniak
837744c2c094SLukasz Maniak if (!pci_is_vf(pci_dev) && n->params.sriov_max_vfs) {
837844c2c094SLukasz Maniak pcie_sriov_pf_exit(pci_dev);
837944c2c094SLukasz Maniak }
838044c2c094SLukasz Maniak
838188eea45cSKlaus Jensen msix_uninit(pci_dev, &n->bar0, &n->bar0);
838288eea45cSKlaus Jensen memory_region_del_subregion(&n->bar0, &n->iomem);
838388eea45cSKlaus Jensen }
838488eea45cSKlaus Jensen
838588eea45cSKlaus Jensen static Property nvme_props[] = {
838688eea45cSKlaus Jensen DEFINE_BLOCK_PROPERTIES(NvmeCtrl, namespace.blkconf),
838788eea45cSKlaus Jensen DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmr.dev, TYPE_MEMORY_BACKEND,
838888eea45cSKlaus Jensen HostMemoryBackend *),
838988eea45cSKlaus Jensen DEFINE_PROP_LINK("subsys", NvmeCtrl, subsys, TYPE_NVME_SUBSYS,
839088eea45cSKlaus Jensen NvmeSubsystem *),
839188eea45cSKlaus Jensen DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial),
839288eea45cSKlaus Jensen DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0),
839388eea45cSKlaus Jensen DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0),
839488eea45cSKlaus Jensen DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl, params.max_ioqpairs, 64),
839588eea45cSKlaus Jensen DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl, params.msix_qsize, 65),
839688eea45cSKlaus Jensen DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
839788eea45cSKlaus Jensen DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64),
839888eea45cSKlaus Jensen DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7),
839988eea45cSKlaus Jensen DEFINE_PROP_UINT8("vsl", NvmeCtrl, params.vsl, 7),
840088eea45cSKlaus Jensen DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl, params.use_intel_id, false),
840188eea45cSKlaus Jensen DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl, params.legacy_cmb, false),
8402e2e137f6SKlaus Jensen DEFINE_PROP_BOOL("ioeventfd", NvmeCtrl, params.ioeventfd, false),
840388eea45cSKlaus Jensen DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl, params.zasl, 0),
8404cccc2651SNiklas Cassel DEFINE_PROP_BOOL("zoned.auto_transition", NvmeCtrl,
8405cccc2651SNiklas Cassel params.auto_transition_zones, true),
840644c2c094SLukasz Maniak DEFINE_PROP_UINT8("sriov_max_vfs", NvmeCtrl, params.sriov_max_vfs, 0),
8407746d42b1SŁukasz Gieryk DEFINE_PROP_UINT16("sriov_vq_flexible", NvmeCtrl,
8408746d42b1SŁukasz Gieryk params.sriov_vq_flexible, 0),
8409746d42b1SŁukasz Gieryk DEFINE_PROP_UINT16("sriov_vi_flexible", NvmeCtrl,
8410746d42b1SŁukasz Gieryk params.sriov_vi_flexible, 0),
8411746d42b1SŁukasz Gieryk DEFINE_PROP_UINT8("sriov_max_vi_per_vf", NvmeCtrl,
8412746d42b1SŁukasz Gieryk params.sriov_max_vi_per_vf, 0),
8413746d42b1SŁukasz Gieryk DEFINE_PROP_UINT8("sriov_max_vq_per_vf", NvmeCtrl,
8414746d42b1SŁukasz Gieryk params.sriov_max_vq_per_vf, 0),
841588eea45cSKlaus Jensen DEFINE_PROP_END_OF_LIST(),
841688eea45cSKlaus Jensen };
841788eea45cSKlaus Jensen
nvme_get_smart_warning(Object * obj,Visitor * v,const char * name,void * opaque,Error ** errp)841888eea45cSKlaus Jensen static void nvme_get_smart_warning(Object *obj, Visitor *v, const char *name,
841988eea45cSKlaus Jensen void *opaque, Error **errp)
842088eea45cSKlaus Jensen {
842188eea45cSKlaus Jensen NvmeCtrl *n = NVME(obj);
842288eea45cSKlaus Jensen uint8_t value = n->smart_critical_warning;
842388eea45cSKlaus Jensen
842488eea45cSKlaus Jensen visit_type_uint8(v, name, &value, errp);
842588eea45cSKlaus Jensen }
842688eea45cSKlaus Jensen
nvme_set_smart_warning(Object * obj,Visitor * v,const char * name,void * opaque,Error ** errp)842788eea45cSKlaus Jensen static void nvme_set_smart_warning(Object *obj, Visitor *v, const char *name,
842888eea45cSKlaus Jensen void *opaque, Error **errp)
842988eea45cSKlaus Jensen {
843088eea45cSKlaus Jensen NvmeCtrl *n = NVME(obj);
843188eea45cSKlaus Jensen uint8_t value, old_value, cap = 0, index, event;
843288eea45cSKlaus Jensen
843388eea45cSKlaus Jensen if (!visit_type_uint8(v, name, &value, errp)) {
843488eea45cSKlaus Jensen return;
843588eea45cSKlaus Jensen }
843688eea45cSKlaus Jensen
843788eea45cSKlaus Jensen cap = NVME_SMART_SPARE | NVME_SMART_TEMPERATURE | NVME_SMART_RELIABILITY
843888eea45cSKlaus Jensen | NVME_SMART_MEDIA_READ_ONLY | NVME_SMART_FAILED_VOLATILE_MEDIA;
843949e03457SKlaus Jensen if (NVME_CAP_PMRS(ldq_le_p(&n->bar.cap))) {
844088eea45cSKlaus Jensen cap |= NVME_SMART_PMR_UNRELIABLE;
844188eea45cSKlaus Jensen }
844288eea45cSKlaus Jensen
844388eea45cSKlaus Jensen if ((value & cap) != value) {
844488eea45cSKlaus Jensen error_setg(errp, "unsupported smart critical warning bits: 0x%x",
844588eea45cSKlaus Jensen value & ~cap);
844688eea45cSKlaus Jensen return;
844788eea45cSKlaus Jensen }
844888eea45cSKlaus Jensen
844988eea45cSKlaus Jensen old_value = n->smart_critical_warning;
845088eea45cSKlaus Jensen n->smart_critical_warning = value;
845188eea45cSKlaus Jensen
845288eea45cSKlaus Jensen /* only inject new bits of smart critical warning */
845388eea45cSKlaus Jensen for (index = 0; index < NVME_SMART_WARN_MAX; index++) {
845488eea45cSKlaus Jensen event = 1 << index;
845588eea45cSKlaus Jensen if (value & ~old_value & event)
845688eea45cSKlaus Jensen nvme_smart_event(n, event);
845788eea45cSKlaus Jensen }
845888eea45cSKlaus Jensen }
845988eea45cSKlaus Jensen
nvme_pci_reset(DeviceState * qdev)84601e9c685eSŁukasz Gieryk static void nvme_pci_reset(DeviceState *qdev)
84611e9c685eSŁukasz Gieryk {
84621e9c685eSŁukasz Gieryk PCIDevice *pci_dev = PCI_DEVICE(qdev);
84631e9c685eSŁukasz Gieryk NvmeCtrl *n = NVME(pci_dev);
84641e9c685eSŁukasz Gieryk
84651e9c685eSŁukasz Gieryk trace_pci_nvme_pci_reset();
84661e9c685eSŁukasz Gieryk nvme_ctrl_reset(n, NVME_RESET_FUNCTION);
84671e9c685eSŁukasz Gieryk }
84681e9c685eSŁukasz Gieryk
nvme_sriov_pre_write_ctrl(PCIDevice * dev,uint32_t address,uint32_t val,int len)846911871f53SŁukasz Gieryk static void nvme_sriov_pre_write_ctrl(PCIDevice *dev, uint32_t address,
847011871f53SŁukasz Gieryk uint32_t val, int len)
847111871f53SŁukasz Gieryk {
847211871f53SŁukasz Gieryk NvmeCtrl *n = NVME(dev);
847311871f53SŁukasz Gieryk NvmeSecCtrlEntry *sctrl;
847411871f53SŁukasz Gieryk uint16_t sriov_cap = dev->exp.sriov_cap;
847511871f53SŁukasz Gieryk uint32_t off = address - sriov_cap;
847611871f53SŁukasz Gieryk int i, num_vfs;
847711871f53SŁukasz Gieryk
847811871f53SŁukasz Gieryk if (!sriov_cap) {
847911871f53SŁukasz Gieryk return;
848011871f53SŁukasz Gieryk }
848111871f53SŁukasz Gieryk
848211871f53SŁukasz Gieryk if (range_covers_byte(off, len, PCI_SRIOV_CTRL)) {
848311871f53SŁukasz Gieryk if (!(val & PCI_SRIOV_CTRL_VFE)) {
848411871f53SŁukasz Gieryk num_vfs = pci_get_word(dev->config + sriov_cap + PCI_SRIOV_NUM_VF);
848511871f53SŁukasz Gieryk for (i = 0; i < num_vfs; i++) {
848611871f53SŁukasz Gieryk sctrl = &n->sec_ctrl_list.sec[i];
848711871f53SŁukasz Gieryk nvme_virt_set_state(n, le16_to_cpu(sctrl->scid), false);
848811871f53SŁukasz Gieryk }
848911871f53SŁukasz Gieryk }
849011871f53SŁukasz Gieryk }
849111871f53SŁukasz Gieryk }
849211871f53SŁukasz Gieryk
nvme_pci_write_config(PCIDevice * dev,uint32_t address,uint32_t val,int len)84931e9c685eSŁukasz Gieryk static void nvme_pci_write_config(PCIDevice *dev, uint32_t address,
84941e9c685eSŁukasz Gieryk uint32_t val, int len)
84951e9c685eSŁukasz Gieryk {
849611871f53SŁukasz Gieryk nvme_sriov_pre_write_ctrl(dev, address, val, len);
84971e9c685eSŁukasz Gieryk pci_default_write_config(dev, address, val, len);
84981e9c685eSŁukasz Gieryk pcie_cap_flr_write_config(dev, address, val, len);
84991e9c685eSŁukasz Gieryk }
85001e9c685eSŁukasz Gieryk
850188eea45cSKlaus Jensen static const VMStateDescription nvme_vmstate = {
850288eea45cSKlaus Jensen .name = "nvme",
850388eea45cSKlaus Jensen .unmigratable = 1,
850488eea45cSKlaus Jensen };
850588eea45cSKlaus Jensen
nvme_class_init(ObjectClass * oc,void * data)850688eea45cSKlaus Jensen static void nvme_class_init(ObjectClass *oc, void *data)
850788eea45cSKlaus Jensen {
850888eea45cSKlaus Jensen DeviceClass *dc = DEVICE_CLASS(oc);
850988eea45cSKlaus Jensen PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
851088eea45cSKlaus Jensen
851188eea45cSKlaus Jensen pc->realize = nvme_realize;
85121e9c685eSŁukasz Gieryk pc->config_write = nvme_pci_write_config;
851388eea45cSKlaus Jensen pc->exit = nvme_exit;
851488eea45cSKlaus Jensen pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
851588eea45cSKlaus Jensen pc->revision = 2;
851688eea45cSKlaus Jensen
851788eea45cSKlaus Jensen set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
851888eea45cSKlaus Jensen dc->desc = "Non-Volatile Memory Express";
851988eea45cSKlaus Jensen device_class_set_props(dc, nvme_props);
852088eea45cSKlaus Jensen dc->vmsd = &nvme_vmstate;
85211e9c685eSŁukasz Gieryk dc->reset = nvme_pci_reset;
852288eea45cSKlaus Jensen }
852388eea45cSKlaus Jensen
nvme_instance_init(Object * obj)852488eea45cSKlaus Jensen static void nvme_instance_init(Object *obj)
852588eea45cSKlaus Jensen {
852688eea45cSKlaus Jensen NvmeCtrl *n = NVME(obj);
852788eea45cSKlaus Jensen
852888eea45cSKlaus Jensen device_add_bootindex_property(obj, &n->namespace.blkconf.bootindex,
852988eea45cSKlaus Jensen "bootindex", "/namespace@1,0",
853088eea45cSKlaus Jensen DEVICE(obj));
853188eea45cSKlaus Jensen
853288eea45cSKlaus Jensen object_property_add(obj, "smart_critical_warning", "uint8",
853388eea45cSKlaus Jensen nvme_get_smart_warning,
853488eea45cSKlaus Jensen nvme_set_smart_warning, NULL, NULL);
853588eea45cSKlaus Jensen }
853688eea45cSKlaus Jensen
853788eea45cSKlaus Jensen static const TypeInfo nvme_info = {
853888eea45cSKlaus Jensen .name = TYPE_NVME,
853988eea45cSKlaus Jensen .parent = TYPE_PCI_DEVICE,
854088eea45cSKlaus Jensen .instance_size = sizeof(NvmeCtrl),
854188eea45cSKlaus Jensen .instance_init = nvme_instance_init,
854288eea45cSKlaus Jensen .class_init = nvme_class_init,
854388eea45cSKlaus Jensen .interfaces = (InterfaceInfo[]) {
854488eea45cSKlaus Jensen { INTERFACE_PCIE_DEVICE },
854588eea45cSKlaus Jensen { }
854688eea45cSKlaus Jensen },
854788eea45cSKlaus Jensen };
854888eea45cSKlaus Jensen
854988eea45cSKlaus Jensen static const TypeInfo nvme_bus_info = {
855088eea45cSKlaus Jensen .name = TYPE_NVME_BUS,
855188eea45cSKlaus Jensen .parent = TYPE_BUS,
855288eea45cSKlaus Jensen .instance_size = sizeof(NvmeBus),
855388eea45cSKlaus Jensen };
855488eea45cSKlaus Jensen
nvme_register_types(void)855588eea45cSKlaus Jensen static void nvme_register_types(void)
855688eea45cSKlaus Jensen {
855788eea45cSKlaus Jensen type_register_static(&nvme_info);
855888eea45cSKlaus Jensen type_register_static(&nvme_bus_info);
855988eea45cSKlaus Jensen }
856088eea45cSKlaus Jensen
856188eea45cSKlaus Jensen type_init(nvme_register_types)
8562