318 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst);
319 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n);
452 static NvmeFdpEvent *nvme_fdp_alloc_event(NvmeCtrl *n, NvmeFdpEventBuffer *ebuf)  in nvme_fdp_alloc_event()
478 static bool nvme_update_ruh(NvmeCtrl *n, NvmeNamespace *ns, uint16_t pid)  in nvme_update_ruh()
515 static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)  in nvme_addr_is_cmb()
529 static inline void *nvme_addr_to_cmb(NvmeCtrl *n, hwaddr addr)  in nvme_addr_to_cmb()
535 static bool nvme_addr_is_pmr(NvmeCtrl *n, hwaddr addr)  in nvme_addr_is_pmr()
548 static inline void *nvme_addr_to_pmr(NvmeCtrl *n, hwaddr addr)  in nvme_addr_to_pmr()
553 static inline bool nvme_addr_is_iomem(NvmeCtrl *n, hwaddr addr)  in nvme_addr_is_iomem()
571 static int nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)  in nvme_addr_read()
591 static int nvme_addr_write(NvmeCtrl *n, hwaddr addr, const void *buf, int size)  in nvme_addr_write()
611 static bool nvme_nsid_valid(NvmeCtrl *n, uint32_t nsid)  in nvme_nsid_valid()
617 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)  in nvme_check_sqid()
622 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)  in nvme_check_cqid()
651 static void nvme_irq_check(NvmeCtrl *n)  in nvme_irq_check()
672 static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)  in nvme_irq_assert()
691 static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)  in nvme_irq_deassert()
715 static inline void nvme_sg_init(NvmeCtrl *n, NvmeSg *sg, bool dma)  in nvme_sg_init()
793 static uint16_t nvme_map_addr_cmb(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,  in nvme_map_addr_cmb()
811 static uint16_t nvme_map_addr_pmr(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,  in nvme_map_addr_pmr()
827 static uint16_t nvme_map_addr(NvmeCtrl *n, NvmeSg *sg, hwaddr addr, size_t len)  in nvme_map_addr()
881 static inline bool nvme_addr_is_dma(NvmeCtrl *n, hwaddr addr)  in nvme_addr_is_dma()
886 static uint16_t nvme_map_prp(NvmeCtrl *n, NvmeSg *sg, uint64_t prp1,  in nvme_map_prp()
987 static uint16_t nvme_map_sgl_data(NvmeCtrl *n, NvmeSg *sg,  in nvme_map_sgl_data()
1048 static uint16_t nvme_map_sgl(NvmeCtrl *n, NvmeSg *sg, NvmeSglDescriptor sgl,  in nvme_map_sgl()
1184 uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,  in nvme_map_dptr()
1203 static uint16_t nvme_map_mptr(NvmeCtrl *n, NvmeSg *sg, size_t len,  in nvme_map_mptr()
1234 static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)  in nvme_map_data()
1264 static uint16_t nvme_map_mdata(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)  in nvme_map_mdata()
1290 static uint16_t nvme_tx_interleaved(NvmeCtrl *n, NvmeSg *sg, uint8_t *ptr,  in nvme_tx_interleaved()
1352 static uint16_t nvme_tx(NvmeCtrl *n, NvmeSg *sg, void *ptr, uint32_t len,  in nvme_tx()
1389 static inline uint16_t nvme_c2h(NvmeCtrl *n, void *ptr, uint32_t len,  in nvme_c2h()
1402 static inline uint16_t nvme_h2c(NvmeCtrl *n, void *ptr, uint32_t len,  in nvme_h2c()
1415 uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,  in nvme_bounce_data()
1432 uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,  in nvme_bounce_mdata()
1498     NvmeCtrl *n = cq->ctrl;  in nvme_post_cqes()
1571     NvmeCtrl *n = opaque;  in nvme_process_aers()
1613 static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type,  in nvme_enqueue_event()
1638 static void nvme_smart_event(NvmeCtrl *n, uint8_t event)  in nvme_smart_event()
1667 static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type)  in nvme_clear_events()
1682 static inline uint16_t nvme_check_mdts(NvmeCtrl *n, size_t len)  in nvme_check_mdts()
2038 static uint16_t nvme_zrm_open_flags(NvmeCtrl *n, NvmeNamespace *ns,  in nvme_zrm_open_flags()
2096 static inline uint16_t nvme_zrm_auto(NvmeCtrl *n, NvmeNamespace *ns,  in nvme_zrm_auto()
2350     NvmeCtrl *n = nvme_ctrl(req);  in nvme_compare_mdata_cb()
2438     NvmeCtrl *n = nvme_ctrl(req);  in nvme_compare_data_cb()
2580     NvmeCtrl *n = nvme_ctrl(req);  in nvme_dsm_cb()
2627 static uint16_t nvme_dsm(NvmeCtrl *n, NvmeRequest *req)  in nvme_dsm()
2665 static uint16_t nvme_verify(NvmeCtrl *n, NvmeRequest *req)  in nvme_verify()
2733     NvmeCtrl *n;
3324 static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req)  in nvme_copy()
3414 static uint16_t nvme_compare(NvmeCtrl *n, NvmeRequest *req)  in nvme_compare()
3537     NvmeCtrl *n = nvme_ctrl(req);  in nvme_do_flush()
3566 static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req)  in nvme_flush()
3606 static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req)  in nvme_read()
3679 static void nvme_do_write_fdp(NvmeCtrl *n, NvmeRequest *req, uint64_t slba,  in nvme_do_write_fdp()
3714 static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append,  in nvme_do_write()
3855 static inline uint16_t nvme_write(NvmeCtrl *n, NvmeRequest *req)  in nvme_write()
3860 static inline uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req)  in nvme_write_zeroes()
3865 static inline uint16_t nvme_zone_append(NvmeCtrl *n, NvmeRequest *req)  in nvme_zone_append()
4187 static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl *n, NvmeZone *zone,  in nvme_zone_mgmt_send_zrwa_flush()
4225 static uint16_t nvme_zone_mgmt_send(NvmeCtrl *n, NvmeRequest *req)  in nvme_zone_mgmt_send()
4375 static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)  in nvme_zone_mgmt_recv()
4482 static uint16_t nvme_io_mgmt_recv_ruhs(NvmeCtrl *n, NvmeRequest *req,  in nvme_io_mgmt_recv_ruhs()
4537 static uint16_t nvme_io_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)  in nvme_io_mgmt_recv()
4555 static uint16_t nvme_io_mgmt_send_ruh_update(NvmeCtrl *n, NvmeRequest *req)  in nvme_io_mgmt_send_ruh_update()
4592 static uint16_t nvme_io_mgmt_send(NvmeCtrl *n, NvmeRequest *req)  in nvme_io_mgmt_send()
4608 static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req)  in nvme_io_cmd()
4699     NvmeCtrl *n = cq->ctrl;  in nvme_cq_notifier()
4720     NvmeCtrl *n = cq->ctrl;  in nvme_init_cq_ioeventfd()
4749     NvmeCtrl *n = sq->ctrl;  in nvme_init_sq_ioeventfd()
4765 static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)  in nvme_free_sq()
4783 static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req)  in nvme_del_sq()
4824 static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,  in nvme_init_sq()
4865 static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req)  in nvme_create_sq()
4920 static uint16_t nvme_smart_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,  in nvme_smart_info()
4982 static uint16_t nvme_endgrp_info(NvmeCtrl *n,  uint8_t rae, uint32_t buf_len,  in nvme_endgrp_info()
5024 static uint16_t nvme_fw_log_info(NvmeCtrl *n, uint32_t buf_len, uint64_t off,  in nvme_fw_log_info()
5042 static uint16_t nvme_error_info(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,  in nvme_error_info()
5062 static uint16_t nvme_changed_nslist(NvmeCtrl *n, uint8_t rae, uint32_t buf_len,  in nvme_changed_nslist()
5109 static uint16_t nvme_cmd_effects(NvmeCtrl *n, uint8_t csi, uint32_t buf_len,  in nvme_cmd_effects()
5156 static uint16_t nvme_fdp_confs(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,  in nvme_fdp_confs()
5225 static uint16_t nvme_fdp_ruh_usage(NvmeCtrl *n, uint32_t endgrpid,  in nvme_fdp_ruh_usage()
5270 static uint16_t nvme_fdp_stats(NvmeCtrl *n, uint32_t endgrpid, uint32_t buf_len,  in nvme_fdp_stats()
5301 static uint16_t nvme_fdp_events(NvmeCtrl *n, uint32_t endgrpid,  in nvme_fdp_events()
5355 static uint16_t nvme_get_log(NvmeCtrl *n, NvmeRequest *req)  in nvme_get_log()
5419 static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)  in nvme_free_cq()
5440 static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeRequest *req)  in nvme_del_cq()
5467 static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,  in nvme_init_cq()
5502 static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeRequest *req)  in nvme_create_cq()
5561 static uint16_t nvme_rpt_empty_id_struct(NvmeCtrl *n, NvmeRequest *req)  in nvme_rpt_empty_id_struct()
5568 static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeRequest *req)  in nvme_identify_ctrl()
5575 static uint16_t nvme_identify_ctrl_csi(NvmeCtrl *n, NvmeRequest *req)  in nvme_identify_ctrl_csi()
5600 static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeRequest *req, bool active)  in nvme_identify_ns()
5631 static uint16_t nvme_identify_ctrl_list(NvmeCtrl *n, NvmeRequest *req,  in nvme_identify_ctrl_list()
5640     NvmeCtrl *ctrl;  in nvme_identify_ctrl_list()
5678 static uint16_t nvme_identify_pri_ctrl_cap(NvmeCtrl *n, NvmeRequest *req)  in nvme_identify_pri_ctrl_cap()
5686 static uint16_t nvme_identify_sec_ctrl_list(NvmeCtrl *n, NvmeRequest *req)  in nvme_identify_sec_ctrl_list()
5709 static uint16_t nvme_identify_ns_ind(NvmeCtrl *n, NvmeRequest *req, bool alloc)  in nvme_identify_ns_ind()
5736 static uint16_t nvme_identify_ns_csi(NvmeCtrl *n, NvmeRequest *req,  in nvme_identify_ns_csi()
5772 static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeRequest *req,  in nvme_identify_nslist()
5819 static uint16_t nvme_identify_nslist_csi(NvmeCtrl *n, NvmeRequest *req,  in nvme_identify_nslist_csi()
5867 static uint16_t nvme_endurance_group_list(NvmeCtrl *n, NvmeRequest *req)  in nvme_endurance_group_list()
5887 static uint16_t nvme_identify_ns_descr_list(NvmeCtrl *n, NvmeRequest *req)  in nvme_identify_ns_descr_list()
5955 static uint16_t nvme_identify_cmd_set(NvmeCtrl *n, NvmeRequest *req)  in nvme_identify_cmd_set()
5968 static uint16_t nvme_identify(NvmeCtrl *n, NvmeRequest *req)  in nvme_identify()
6020 static uint16_t nvme_abort(NvmeCtrl *n, NvmeRequest *req)  in nvme_abort()
6060 static inline void nvme_set_timestamp(NvmeCtrl *n, uint64_t ts)  in nvme_set_timestamp()
6068 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n)  in nvme_get_timestamp()
6095 static uint16_t nvme_get_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)  in nvme_get_feature_timestamp()
6102 static int nvme_get_feature_fdp(NvmeCtrl *n, uint32_t endgrpid,  in nvme_get_feature_fdp()
6117 static uint16_t nvme_get_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,  in nvme_get_feature_fdp_events()
6180 static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeRequest *req)  in nvme_get_feature()
6376 static uint16_t nvme_set_feature_timestamp(NvmeCtrl *n, NvmeRequest *req)  in nvme_set_feature_timestamp()
6391 static uint16_t nvme_set_feature_fdp_events(NvmeCtrl *n, NvmeNamespace *ns,  in nvme_set_feature_fdp_events()
6436 static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeRequest *req)  in nvme_set_feature()
6624 static uint16_t nvme_aer(NvmeCtrl *n, NvmeRequest *req)  in nvme_aer()
6643 static void nvme_update_dmrsl(NvmeCtrl *n)  in nvme_update_dmrsl()
6658 static void nvme_select_iocs_ns(NvmeCtrl *n, NvmeNamespace *ns)  in nvme_select_iocs_ns()
6679 static uint16_t nvme_ns_attachment(NvmeCtrl *n, NvmeRequest *req)  in nvme_ns_attachment()
6682     NvmeCtrl *ctrl;  in nvme_ns_attachment()
6873     NvmeCtrl *n = nvme_ctrl(req);  in nvme_do_format()
6913 static uint16_t nvme_format(NvmeCtrl *n, NvmeRequest *req)  in nvme_format()
6966 static void nvme_get_virt_res_num(NvmeCtrl *n, uint8_t rt, int *num_total,  in nvme_get_virt_res_num()
6976 static uint16_t nvme_assign_virt_res_to_prim(NvmeCtrl *n, NvmeRequest *req,  in nvme_assign_virt_res_to_prim()
7006 static void nvme_update_virt_res(NvmeCtrl *n, NvmeSecCtrlEntry *sctrl,  in nvme_update_virt_res()
7024 static uint16_t nvme_assign_virt_res_to_sec(NvmeCtrl *n, NvmeRequest *req,  in nvme_assign_virt_res_to_sec()
7058 static uint16_t nvme_virt_set_state(NvmeCtrl *n, uint16_t cntlid, bool online)  in nvme_virt_set_state()
7061     NvmeCtrl *sn = NULL;  in nvme_virt_set_state()
7099 static uint16_t nvme_virt_mngmt(NvmeCtrl *n, NvmeRequest *req)  in nvme_virt_mngmt()
7128 static uint16_t nvme_dbbuf_config(NvmeCtrl *n, const NvmeRequest *req)  in nvme_dbbuf_config()
7185 static uint16_t nvme_directive_send(NvmeCtrl *n, NvmeRequest *req)  in nvme_directive_send()
7190 static uint16_t nvme_directive_receive(NvmeCtrl *n, NvmeRequest *req)  in nvme_directive_receive()
7240 static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeRequest *req)  in nvme_admin_cmd()
7319 static int nvme_atomic_write_check(NvmeCtrl *n, NvmeCmd *cmd,  in nvme_atomic_write_check()
7382 static NvmeAtomic *nvme_get_atomic(NvmeCtrl *n, NvmeCmd *cmd)  in nvme_get_atomic()
7393     NvmeCtrl *n = sq->ctrl;  in nvme_process_sq()
7478 static void nvme_activate_virt_res(NvmeCtrl *n)  in nvme_activate_virt_res()
7501 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst)  in nvme_ctrl_reset()
7573 static void nvme_ctrl_shutdown(NvmeCtrl *n)  in nvme_ctrl_shutdown()
7592 static void nvme_select_iocs(NvmeCtrl *n)  in nvme_select_iocs()
7607 static int nvme_start_ctrl(NvmeCtrl *n)  in nvme_start_ctrl()
7678 static void nvme_cmb_enable_regs(NvmeCtrl *n)  in nvme_cmb_enable_regs()
7698 static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,  in nvme_write_bar()
7938     NvmeCtrl *n = (NvmeCtrl *)opaque;  in nvme_mmio_read()
7982 static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)  in nvme_process_db()
8131     NvmeCtrl *n = (NvmeCtrl *)opaque;  in nvme_mmio_write()
8161     NvmeCtrl *n = (NvmeCtrl *)opaque;  in nvme_cmb_write()
8167     NvmeCtrl *n = (NvmeCtrl *)opaque;  in nvme_cmb_read()
8181 static bool nvme_check_params(NvmeCtrl *n, Error **errp)  in nvme_check_params()
8320 static void nvme_init_state(NvmeCtrl *n)  in nvme_init_state()
8410 static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)  in nvme_init_cmb()
8432 static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)  in nvme_init_pmr()
8484 static void nvme_init_sriov(NvmeCtrl *n, PCIDevice *pci_dev, uint16_t offset)  in nvme_init_sriov()
8544 static bool nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev, Error **errp)  in nvme_init_pci()
8589             NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));  in nvme_init_pci()
8659 static void nvme_init_subnqn(NvmeCtrl *n)  in nvme_init_subnqn()
8672 static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)  in nvme_init_ctrl()
8794 static int nvme_init_subsys(NvmeCtrl *n, Error **errp)  in nvme_init_subsys()
8812 void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns)  in nvme_attach_ns()
8826     NvmeCtrl *n = NVME(pci_dev);  in nvme_realize()
8829     NvmeCtrl *pn = NVME(pcie_sriov_get_pf(pci_dev));  in nvme_realize()
8883     NvmeCtrl *n = NVME(pci_dev);  in nvme_exit()
8931     DEFINE_BLOCK_PROPERTIES(NvmeCtrl, namespace.blkconf),
8932     DEFINE_PROP_LINK("pmrdev", NvmeCtrl, pmr.dev, TYPE_MEMORY_BACKEND,
8934     DEFINE_PROP_LINK("subsys", NvmeCtrl, subsys, TYPE_NVME_SUBSYS,
8936     DEFINE_PROP_STRING("serial", NvmeCtrl, params.serial),
8937     DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, params.cmb_size_mb, 0),
8938     DEFINE_PROP_UINT32("num_queues", NvmeCtrl, params.num_queues, 0),
8939     DEFINE_PROP_UINT32("max_ioqpairs", NvmeCtrl, params.max_ioqpairs, 64),
8940     DEFINE_PROP_UINT16("msix_qsize", NvmeCtrl, params.msix_qsize, 65),
8941     DEFINE_PROP_UINT8("aerl", NvmeCtrl, params.aerl, 3),
8942     DEFINE_PROP_UINT32("aer_max_queued", NvmeCtrl, params.aer_max_queued, 64),
8943     DEFINE_PROP_UINT8("mdts", NvmeCtrl, params.mdts, 7),
8944     DEFINE_PROP_UINT8("vsl", NvmeCtrl, params.vsl, 7),
8945     DEFINE_PROP_BOOL("use-intel-id", NvmeCtrl, params.use_intel_id, false),
8946     DEFINE_PROP_BOOL("legacy-cmb", NvmeCtrl, params.legacy_cmb, false),
8947     DEFINE_PROP_BOOL("ioeventfd", NvmeCtrl, params.ioeventfd, false),
8948     DEFINE_PROP_UINT8("zoned.zasl", NvmeCtrl, params.zasl, 0),
8949     DEFINE_PROP_BOOL("zoned.auto_transition", NvmeCtrl,
8951     DEFINE_PROP_UINT16("sriov_max_vfs", NvmeCtrl, params.sriov_max_vfs, 0),
8952     DEFINE_PROP_UINT16("sriov_vq_flexible", NvmeCtrl,
8954     DEFINE_PROP_UINT16("sriov_vi_flexible", NvmeCtrl,
8956     DEFINE_PROP_UINT32("sriov_max_vi_per_vf", NvmeCtrl,
8958     DEFINE_PROP_UINT32("sriov_max_vq_per_vf", NvmeCtrl,
8960     DEFINE_PROP_BOOL("msix-exclusive-bar", NvmeCtrl, params.msix_exclusive_bar,
8962     DEFINE_PROP_UINT16("mqes", NvmeCtrl, params.mqes, 0x7ff),
8964     DEFINE_PROP_BOOL("ctratt.mem", NvmeCtrl, params.ctratt.mem, false),
8965     DEFINE_PROP_BOOL("atomic.dn", NvmeCtrl, params.atomic_dn, 0),
8966     DEFINE_PROP_UINT16("atomic.awun", NvmeCtrl, params.atomic_awun, 0),
8967     DEFINE_PROP_UINT16("atomic.awupf", NvmeCtrl, params.atomic_awupf, 0),
8974     NvmeCtrl *n = NVME(obj);  in nvme_get_smart_warning()
8983     NvmeCtrl *n = NVME(obj);  in nvme_set_smart_warning()
9016     NvmeCtrl *n = NVME(pci_dev);  in nvme_pci_reset()
9024     NvmeCtrl *n = NVME(dev);  in nvme_sriov_post_write_config()
9084     NvmeCtrl *n = NVME(obj);  in nvme_instance_init()
9098     .instance_size = sizeof(NvmeCtrl),