1 /* 2 * QEMU NVM Express 3 * 4 * Copyright (c) 2012 Intel Corporation 5 * Copyright (c) 2021 Minwoo Im 6 * Copyright (c) 2021 Samsung Electronics Co., Ltd. 7 * 8 * Authors: 9 * Keith Busch <kbusch@kernel.org> 10 * Klaus Jensen <k.jensen@samsung.com> 11 * Gollu Appalanaidu <anaidu.gollu@samsung.com> 12 * Dmitry Fomichev <dmitry.fomichev@wdc.com> 13 * Minwoo Im <minwoo.im.dev@gmail.com> 14 * 15 * This code is licensed under the GNU GPL v2 or later. 16 */ 17 18 #ifndef HW_NVME_NVME_H 19 #define HW_NVME_NVME_H 20 21 #include "qemu/uuid.h" 22 #include "hw/pci/pci_device.h" 23 #include "hw/block/block.h" 24 25 #include "block/nvme.h" 26 27 #define NVME_MAX_CONTROLLERS 256 28 #define NVME_MAX_NAMESPACES 256 29 #define NVME_EUI64_DEFAULT ((uint64_t)0x5254000000000000) 30 31 QEMU_BUILD_BUG_ON(NVME_MAX_NAMESPACES > NVME_NSID_BROADCAST - 1); 32 33 typedef struct NvmeCtrl NvmeCtrl; 34 typedef struct NvmeNamespace NvmeNamespace; 35 36 #define TYPE_NVME_BUS "nvme-bus" 37 OBJECT_DECLARE_SIMPLE_TYPE(NvmeBus, NVME_BUS) 38 39 typedef struct NvmeBus { 40 BusState parent_bus; 41 } NvmeBus; 42 43 #define TYPE_NVME_SUBSYS "nvme-subsys" 44 #define NVME_SUBSYS(obj) \ 45 OBJECT_CHECK(NvmeSubsystem, (obj), TYPE_NVME_SUBSYS) 46 #define SUBSYS_SLOT_RSVD (void *)0xFFFF 47 48 typedef struct NvmeSubsystem { 49 DeviceState parent_obj; 50 NvmeBus bus; 51 uint8_t subnqn[256]; 52 char *serial; 53 54 NvmeCtrl *ctrls[NVME_MAX_CONTROLLERS]; 55 NvmeNamespace *namespaces[NVME_MAX_NAMESPACES + 1]; 56 57 struct { 58 char *nqn; 59 } params; 60 } NvmeSubsystem; 61 62 int nvme_subsys_register_ctrl(NvmeCtrl *n, Error **errp); 63 void nvme_subsys_unregister_ctrl(NvmeSubsystem *subsys, NvmeCtrl *n); 64 65 static inline NvmeCtrl *nvme_subsys_ctrl(NvmeSubsystem *subsys, 66 uint32_t cntlid) 67 { 68 if (!subsys || cntlid >= NVME_MAX_CONTROLLERS) { 69 return NULL; 70 } 71 72 if (subsys->ctrls[cntlid] == SUBSYS_SLOT_RSVD) { 73 return NULL; 74 } 75 76 return subsys->ctrls[cntlid]; 77 } 78 79 static inline NvmeNamespace *nvme_subsys_ns(NvmeSubsystem *subsys, 80 uint32_t nsid) 81 { 82 if (!subsys || !nsid || nsid > NVME_MAX_NAMESPACES) { 83 return NULL; 84 } 85 86 return subsys->namespaces[nsid]; 87 } 88 89 #define TYPE_NVME_NS "nvme-ns" 90 #define NVME_NS(obj) \ 91 OBJECT_CHECK(NvmeNamespace, (obj), TYPE_NVME_NS) 92 93 typedef struct NvmeZone { 94 NvmeZoneDescr d; 95 uint64_t w_ptr; 96 QTAILQ_ENTRY(NvmeZone) entry; 97 } NvmeZone; 98 99 typedef struct NvmeNamespaceParams { 100 bool detached; 101 bool shared; 102 uint32_t nsid; 103 QemuUUID uuid; 104 uint64_t eui64; 105 bool eui64_default; 106 107 uint16_t ms; 108 uint8_t mset; 109 uint8_t pi; 110 uint8_t pil; 111 uint8_t pif; 112 113 uint16_t mssrl; 114 uint32_t mcl; 115 uint8_t msrc; 116 117 bool zoned; 118 bool cross_zone_read; 119 uint64_t zone_size_bs; 120 uint64_t zone_cap_bs; 121 uint32_t max_active_zones; 122 uint32_t max_open_zones; 123 uint32_t zd_extension_size; 124 125 uint32_t numzrwa; 126 uint64_t zrwas; 127 uint64_t zrwafg; 128 } NvmeNamespaceParams; 129 130 typedef struct NvmeNamespace { 131 DeviceState parent_obj; 132 BlockConf blkconf; 133 int32_t bootindex; 134 int64_t size; 135 int64_t moff; 136 NvmeIdNs id_ns; 137 NvmeIdNsNvm id_ns_nvm; 138 NvmeLBAF lbaf; 139 unsigned int nlbaf; 140 size_t lbasz; 141 const uint32_t *iocs; 142 uint8_t csi; 143 uint16_t status; 144 int attached; 145 uint8_t pif; 146 147 struct { 148 uint16_t zrwas; 149 uint16_t zrwafg; 150 uint32_t numzrwa; 151 } zns; 152 153 QTAILQ_ENTRY(NvmeNamespace) entry; 154 155 NvmeIdNsZoned *id_ns_zoned; 156 NvmeZone *zone_array; 157 QTAILQ_HEAD(, NvmeZone) exp_open_zones; 158 QTAILQ_HEAD(, NvmeZone) imp_open_zones; 159 QTAILQ_HEAD(, NvmeZone) closed_zones; 160 QTAILQ_HEAD(, NvmeZone) full_zones; 161 uint32_t num_zones; 162 uint64_t zone_size; 163 uint64_t zone_capacity; 164 uint32_t zone_size_log2; 165 uint8_t *zd_extensions; 166 int32_t nr_open_zones; 167 int32_t nr_active_zones; 168 169 NvmeNamespaceParams params; 170 171 struct { 172 uint32_t err_rec; 173 } features; 174 } NvmeNamespace; 175 176 static inline uint32_t nvme_nsid(NvmeNamespace *ns) 177 { 178 if (ns) { 179 return ns->params.nsid; 180 } 181 182 return 0; 183 } 184 185 static inline size_t nvme_l2b(NvmeNamespace *ns, uint64_t lba) 186 { 187 return lba << ns->lbaf.ds; 188 } 189 190 static inline size_t nvme_m2b(NvmeNamespace *ns, uint64_t lba) 191 { 192 return ns->lbaf.ms * lba; 193 } 194 195 static inline int64_t nvme_moff(NvmeNamespace *ns, uint64_t lba) 196 { 197 return ns->moff + nvme_m2b(ns, lba); 198 } 199 200 static inline bool nvme_ns_ext(NvmeNamespace *ns) 201 { 202 return !!NVME_ID_NS_FLBAS_EXTENDED(ns->id_ns.flbas); 203 } 204 205 static inline NvmeZoneState nvme_get_zone_state(NvmeZone *zone) 206 { 207 return zone->d.zs >> 4; 208 } 209 210 static inline void nvme_set_zone_state(NvmeZone *zone, NvmeZoneState state) 211 { 212 zone->d.zs = state << 4; 213 } 214 215 static inline uint64_t nvme_zone_rd_boundary(NvmeNamespace *ns, NvmeZone *zone) 216 { 217 return zone->d.zslba + ns->zone_size; 218 } 219 220 static inline uint64_t nvme_zone_wr_boundary(NvmeZone *zone) 221 { 222 return zone->d.zslba + zone->d.zcap; 223 } 224 225 static inline bool nvme_wp_is_valid(NvmeZone *zone) 226 { 227 uint8_t st = nvme_get_zone_state(zone); 228 229 return st != NVME_ZONE_STATE_FULL && 230 st != NVME_ZONE_STATE_READ_ONLY && 231 st != NVME_ZONE_STATE_OFFLINE; 232 } 233 234 static inline uint8_t *nvme_get_zd_extension(NvmeNamespace *ns, 235 uint32_t zone_idx) 236 { 237 return &ns->zd_extensions[zone_idx * ns->params.zd_extension_size]; 238 } 239 240 static inline void nvme_aor_inc_open(NvmeNamespace *ns) 241 { 242 assert(ns->nr_open_zones >= 0); 243 if (ns->params.max_open_zones) { 244 ns->nr_open_zones++; 245 assert(ns->nr_open_zones <= ns->params.max_open_zones); 246 } 247 } 248 249 static inline void nvme_aor_dec_open(NvmeNamespace *ns) 250 { 251 if (ns->params.max_open_zones) { 252 assert(ns->nr_open_zones > 0); 253 ns->nr_open_zones--; 254 } 255 assert(ns->nr_open_zones >= 0); 256 } 257 258 static inline void nvme_aor_inc_active(NvmeNamespace *ns) 259 { 260 assert(ns->nr_active_zones >= 0); 261 if (ns->params.max_active_zones) { 262 ns->nr_active_zones++; 263 assert(ns->nr_active_zones <= ns->params.max_active_zones); 264 } 265 } 266 267 static inline void nvme_aor_dec_active(NvmeNamespace *ns) 268 { 269 if (ns->params.max_active_zones) { 270 assert(ns->nr_active_zones > 0); 271 ns->nr_active_zones--; 272 assert(ns->nr_active_zones >= ns->nr_open_zones); 273 } 274 assert(ns->nr_active_zones >= 0); 275 } 276 277 void nvme_ns_init_format(NvmeNamespace *ns); 278 int nvme_ns_setup(NvmeNamespace *ns, Error **errp); 279 void nvme_ns_drain(NvmeNamespace *ns); 280 void nvme_ns_shutdown(NvmeNamespace *ns); 281 void nvme_ns_cleanup(NvmeNamespace *ns); 282 283 typedef struct NvmeAsyncEvent { 284 QTAILQ_ENTRY(NvmeAsyncEvent) entry; 285 NvmeAerResult result; 286 } NvmeAsyncEvent; 287 288 enum { 289 NVME_SG_ALLOC = 1 << 0, 290 NVME_SG_DMA = 1 << 1, 291 }; 292 293 typedef struct NvmeSg { 294 int flags; 295 296 union { 297 QEMUSGList qsg; 298 QEMUIOVector iov; 299 }; 300 } NvmeSg; 301 302 typedef enum NvmeTxDirection { 303 NVME_TX_DIRECTION_TO_DEVICE = 0, 304 NVME_TX_DIRECTION_FROM_DEVICE = 1, 305 } NvmeTxDirection; 306 307 typedef struct NvmeRequest { 308 struct NvmeSQueue *sq; 309 struct NvmeNamespace *ns; 310 BlockAIOCB *aiocb; 311 uint16_t status; 312 void *opaque; 313 NvmeCqe cqe; 314 NvmeCmd cmd; 315 BlockAcctCookie acct; 316 NvmeSg sg; 317 QTAILQ_ENTRY(NvmeRequest)entry; 318 } NvmeRequest; 319 320 typedef struct NvmeBounceContext { 321 NvmeRequest *req; 322 323 struct { 324 QEMUIOVector iov; 325 uint8_t *bounce; 326 } data, mdata; 327 } NvmeBounceContext; 328 329 static inline const char *nvme_adm_opc_str(uint8_t opc) 330 { 331 switch (opc) { 332 case NVME_ADM_CMD_DELETE_SQ: return "NVME_ADM_CMD_DELETE_SQ"; 333 case NVME_ADM_CMD_CREATE_SQ: return "NVME_ADM_CMD_CREATE_SQ"; 334 case NVME_ADM_CMD_GET_LOG_PAGE: return "NVME_ADM_CMD_GET_LOG_PAGE"; 335 case NVME_ADM_CMD_DELETE_CQ: return "NVME_ADM_CMD_DELETE_CQ"; 336 case NVME_ADM_CMD_CREATE_CQ: return "NVME_ADM_CMD_CREATE_CQ"; 337 case NVME_ADM_CMD_IDENTIFY: return "NVME_ADM_CMD_IDENTIFY"; 338 case NVME_ADM_CMD_ABORT: return "NVME_ADM_CMD_ABORT"; 339 case NVME_ADM_CMD_SET_FEATURES: return "NVME_ADM_CMD_SET_FEATURES"; 340 case NVME_ADM_CMD_GET_FEATURES: return "NVME_ADM_CMD_GET_FEATURES"; 341 case NVME_ADM_CMD_ASYNC_EV_REQ: return "NVME_ADM_CMD_ASYNC_EV_REQ"; 342 case NVME_ADM_CMD_NS_ATTACHMENT: return "NVME_ADM_CMD_NS_ATTACHMENT"; 343 case NVME_ADM_CMD_VIRT_MNGMT: return "NVME_ADM_CMD_VIRT_MNGMT"; 344 case NVME_ADM_CMD_DBBUF_CONFIG: return "NVME_ADM_CMD_DBBUF_CONFIG"; 345 case NVME_ADM_CMD_FORMAT_NVM: return "NVME_ADM_CMD_FORMAT_NVM"; 346 default: return "NVME_ADM_CMD_UNKNOWN"; 347 } 348 } 349 350 static inline const char *nvme_io_opc_str(uint8_t opc) 351 { 352 switch (opc) { 353 case NVME_CMD_FLUSH: return "NVME_NVM_CMD_FLUSH"; 354 case NVME_CMD_WRITE: return "NVME_NVM_CMD_WRITE"; 355 case NVME_CMD_READ: return "NVME_NVM_CMD_READ"; 356 case NVME_CMD_COMPARE: return "NVME_NVM_CMD_COMPARE"; 357 case NVME_CMD_WRITE_ZEROES: return "NVME_NVM_CMD_WRITE_ZEROES"; 358 case NVME_CMD_DSM: return "NVME_NVM_CMD_DSM"; 359 case NVME_CMD_VERIFY: return "NVME_NVM_CMD_VERIFY"; 360 case NVME_CMD_COPY: return "NVME_NVM_CMD_COPY"; 361 case NVME_CMD_ZONE_MGMT_SEND: return "NVME_ZONED_CMD_MGMT_SEND"; 362 case NVME_CMD_ZONE_MGMT_RECV: return "NVME_ZONED_CMD_MGMT_RECV"; 363 case NVME_CMD_ZONE_APPEND: return "NVME_ZONED_CMD_ZONE_APPEND"; 364 default: return "NVME_NVM_CMD_UNKNOWN"; 365 } 366 } 367 368 typedef struct NvmeSQueue { 369 struct NvmeCtrl *ctrl; 370 uint16_t sqid; 371 uint16_t cqid; 372 uint32_t head; 373 uint32_t tail; 374 uint32_t size; 375 uint64_t dma_addr; 376 uint64_t db_addr; 377 uint64_t ei_addr; 378 QEMUBH *bh; 379 EventNotifier notifier; 380 bool ioeventfd_enabled; 381 NvmeRequest *io_req; 382 QTAILQ_HEAD(, NvmeRequest) req_list; 383 QTAILQ_HEAD(, NvmeRequest) out_req_list; 384 QTAILQ_ENTRY(NvmeSQueue) entry; 385 } NvmeSQueue; 386 387 typedef struct NvmeCQueue { 388 struct NvmeCtrl *ctrl; 389 uint8_t phase; 390 uint16_t cqid; 391 uint16_t irq_enabled; 392 uint32_t head; 393 uint32_t tail; 394 uint32_t vector; 395 uint32_t size; 396 uint64_t dma_addr; 397 uint64_t db_addr; 398 uint64_t ei_addr; 399 QEMUBH *bh; 400 EventNotifier notifier; 401 bool ioeventfd_enabled; 402 QTAILQ_HEAD(, NvmeSQueue) sq_list; 403 QTAILQ_HEAD(, NvmeRequest) req_list; 404 } NvmeCQueue; 405 406 #define TYPE_NVME "nvme" 407 #define NVME(obj) \ 408 OBJECT_CHECK(NvmeCtrl, (obj), TYPE_NVME) 409 410 typedef struct NvmeParams { 411 char *serial; 412 uint32_t num_queues; /* deprecated since 5.1 */ 413 uint32_t max_ioqpairs; 414 uint16_t msix_qsize; 415 uint32_t cmb_size_mb; 416 uint8_t aerl; 417 uint32_t aer_max_queued; 418 uint8_t mdts; 419 uint8_t vsl; 420 bool use_intel_id; 421 uint8_t zasl; 422 bool auto_transition_zones; 423 bool legacy_cmb; 424 bool ioeventfd; 425 uint8_t sriov_max_vfs; 426 uint16_t sriov_vq_flexible; 427 uint16_t sriov_vi_flexible; 428 uint8_t sriov_max_vq_per_vf; 429 uint8_t sriov_max_vi_per_vf; 430 } NvmeParams; 431 432 typedef struct NvmeCtrl { 433 PCIDevice parent_obj; 434 MemoryRegion bar0; 435 MemoryRegion iomem; 436 NvmeBar bar; 437 NvmeParams params; 438 NvmeBus bus; 439 440 uint16_t cntlid; 441 bool qs_created; 442 uint32_t page_size; 443 uint16_t page_bits; 444 uint16_t max_prp_ents; 445 uint16_t cqe_size; 446 uint16_t sqe_size; 447 uint32_t max_q_ents; 448 uint8_t outstanding_aers; 449 uint32_t irq_status; 450 int cq_pending; 451 uint64_t host_timestamp; /* Timestamp sent by the host */ 452 uint64_t timestamp_set_qemu_clock_ms; /* QEMU clock time */ 453 uint64_t starttime_ms; 454 uint16_t temperature; 455 uint8_t smart_critical_warning; 456 uint32_t conf_msix_qsize; 457 uint32_t conf_ioqpairs; 458 uint64_t dbbuf_dbs; 459 uint64_t dbbuf_eis; 460 bool dbbuf_enabled; 461 462 struct { 463 MemoryRegion mem; 464 uint8_t *buf; 465 bool cmse; 466 hwaddr cba; 467 } cmb; 468 469 struct { 470 HostMemoryBackend *dev; 471 bool cmse; 472 hwaddr cba; 473 } pmr; 474 475 uint8_t aer_mask; 476 NvmeRequest **aer_reqs; 477 QTAILQ_HEAD(, NvmeAsyncEvent) aer_queue; 478 int aer_queued; 479 480 uint32_t dmrsl; 481 482 /* Namespace ID is started with 1 so bitmap should be 1-based */ 483 #define NVME_CHANGED_NSID_SIZE (NVME_MAX_NAMESPACES + 1) 484 DECLARE_BITMAP(changed_nsids, NVME_CHANGED_NSID_SIZE); 485 486 NvmeSubsystem *subsys; 487 488 NvmeNamespace namespace; 489 NvmeNamespace *namespaces[NVME_MAX_NAMESPACES + 1]; 490 NvmeSQueue **sq; 491 NvmeCQueue **cq; 492 NvmeSQueue admin_sq; 493 NvmeCQueue admin_cq; 494 NvmeIdCtrl id_ctrl; 495 496 struct { 497 struct { 498 uint16_t temp_thresh_hi; 499 uint16_t temp_thresh_low; 500 }; 501 502 uint32_t async_config; 503 NvmeHostBehaviorSupport hbs; 504 } features; 505 506 NvmePriCtrlCap pri_ctrl_cap; 507 NvmeSecCtrlList sec_ctrl_list; 508 struct { 509 uint16_t vqrfap; 510 uint16_t virfap; 511 } next_pri_ctrl_cap; /* These override pri_ctrl_cap after reset */ 512 } NvmeCtrl; 513 514 typedef enum NvmeResetType { 515 NVME_RESET_FUNCTION = 0, 516 NVME_RESET_CONTROLLER = 1, 517 } NvmeResetType; 518 519 static inline NvmeNamespace *nvme_ns(NvmeCtrl *n, uint32_t nsid) 520 { 521 if (!nsid || nsid > NVME_MAX_NAMESPACES) { 522 return NULL; 523 } 524 525 return n->namespaces[nsid]; 526 } 527 528 static inline NvmeCQueue *nvme_cq(NvmeRequest *req) 529 { 530 NvmeSQueue *sq = req->sq; 531 NvmeCtrl *n = sq->ctrl; 532 533 return n->cq[sq->cqid]; 534 } 535 536 static inline NvmeCtrl *nvme_ctrl(NvmeRequest *req) 537 { 538 NvmeSQueue *sq = req->sq; 539 return sq->ctrl; 540 } 541 542 static inline uint16_t nvme_cid(NvmeRequest *req) 543 { 544 if (!req) { 545 return 0xffff; 546 } 547 548 return le16_to_cpu(req->cqe.cid); 549 } 550 551 static inline NvmeSecCtrlEntry *nvme_sctrl(NvmeCtrl *n) 552 { 553 PCIDevice *pci_dev = &n->parent_obj; 554 NvmeCtrl *pf = NVME(pcie_sriov_get_pf(pci_dev)); 555 556 if (pci_is_vf(pci_dev)) { 557 return &pf->sec_ctrl_list.sec[pcie_sriov_vf_number(pci_dev)]; 558 } 559 560 return NULL; 561 } 562 563 static inline NvmeSecCtrlEntry *nvme_sctrl_for_cntlid(NvmeCtrl *n, 564 uint16_t cntlid) 565 { 566 NvmeSecCtrlList *list = &n->sec_ctrl_list; 567 uint8_t i; 568 569 for (i = 0; i < list->numcntl; i++) { 570 if (le16_to_cpu(list->sec[i].scid) == cntlid) { 571 return &list->sec[i]; 572 } 573 } 574 575 return NULL; 576 } 577 578 void nvme_attach_ns(NvmeCtrl *n, NvmeNamespace *ns); 579 uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len, 580 NvmeTxDirection dir, NvmeRequest *req); 581 uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len, 582 NvmeTxDirection dir, NvmeRequest *req); 583 void nvme_rw_complete_cb(void *opaque, int ret); 584 uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len, 585 NvmeCmd *cmd); 586 587 #endif /* HW_NVME_NVME_H */ 588