1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVM Express device driver 4 * Copyright (c) 2011-2014, Intel Corporation. 5 */ 6 7 #include <linux/blkdev.h> 8 #include <linux/blk-mq.h> 9 #include <linux/compat.h> 10 #include <linux/delay.h> 11 #include <linux/errno.h> 12 #include <linux/hdreg.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/backing-dev.h> 16 #include <linux/list_sort.h> 17 #include <linux/slab.h> 18 #include <linux/types.h> 19 #include <linux/pr.h> 20 #include <linux/ptrace.h> 21 #include <linux/nvme_ioctl.h> 22 #include <linux/pm_qos.h> 23 #include <asm/unaligned.h> 24 25 #include "nvme.h" 26 #include "fabrics.h" 27 28 #define CREATE_TRACE_POINTS 29 #include "trace.h" 30 31 #define NVME_MINORS (1U << MINORBITS) 32 33 unsigned int admin_timeout = 60; 34 module_param(admin_timeout, uint, 0644); 35 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands"); 36 EXPORT_SYMBOL_GPL(admin_timeout); 37 38 unsigned int nvme_io_timeout = 30; 39 module_param_named(io_timeout, nvme_io_timeout, uint, 0644); 40 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O"); 41 EXPORT_SYMBOL_GPL(nvme_io_timeout); 42 43 static unsigned char shutdown_timeout = 5; 44 module_param(shutdown_timeout, byte, 0644); 45 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown"); 46 47 static u8 nvme_max_retries = 5; 48 module_param_named(max_retries, nvme_max_retries, byte, 0644); 49 MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); 50 51 static unsigned long default_ps_max_latency_us = 100000; 52 module_param(default_ps_max_latency_us, ulong, 0644); 53 MODULE_PARM_DESC(default_ps_max_latency_us, 54 "max power saving latency for new devices; use PM QOS to change per device"); 55 56 static bool force_apst; 57 module_param(force_apst, bool, 0644); 58 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off"); 59 60 static bool streams; 61 module_param(streams, bool, 0644); 62 MODULE_PARM_DESC(streams, "turn on support for Streams write directives"); 63 64 /* 65 * nvme_wq - hosts nvme related works that are not reset or delete 66 * nvme_reset_wq - hosts nvme reset works 67 * nvme_delete_wq - hosts nvme delete works 68 * 69 * nvme_wq will host works such as scan, aen handling, fw activation, 70 * keep-alive, periodic reconnects etc. nvme_reset_wq 71 * runs reset works which also flush works hosted on nvme_wq for 72 * serialization purposes. nvme_delete_wq host controller deletion 73 * works which flush reset works for serialization. 74 */ 75 struct workqueue_struct *nvme_wq; 76 EXPORT_SYMBOL_GPL(nvme_wq); 77 78 struct workqueue_struct *nvme_reset_wq; 79 EXPORT_SYMBOL_GPL(nvme_reset_wq); 80 81 struct workqueue_struct *nvme_delete_wq; 82 EXPORT_SYMBOL_GPL(nvme_delete_wq); 83 84 static LIST_HEAD(nvme_subsystems); 85 static DEFINE_MUTEX(nvme_subsystems_lock); 86 87 static DEFINE_IDA(nvme_instance_ida); 88 static dev_t nvme_ctrl_base_chr_devt; 89 static struct class *nvme_class; 90 static struct class *nvme_subsys_class; 91 92 static void nvme_put_subsystem(struct nvme_subsystem *subsys); 93 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, 94 unsigned nsid); 95 96 /* 97 * Prepare a queue for teardown. 98 * 99 * This must forcibly unquiesce queues to avoid blocking dispatch, and only set 100 * the capacity to 0 after that to avoid blocking dispatchers that may be 101 * holding bd_butex. This will end buffered writers dirtying pages that can't 102 * be synced. 103 */ 104 static void nvme_set_queue_dying(struct nvme_ns *ns) 105 { 106 if (test_and_set_bit(NVME_NS_DEAD, &ns->flags)) 107 return; 108 109 blk_set_queue_dying(ns->queue); 110 blk_mq_unquiesce_queue(ns->queue); 111 112 set_capacity_and_notify(ns->disk, 0); 113 } 114 115 static void nvme_queue_scan(struct nvme_ctrl *ctrl) 116 { 117 /* 118 * Only new queue scan work when admin and IO queues are both alive 119 */ 120 if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset) 121 queue_work(nvme_wq, &ctrl->scan_work); 122 } 123 124 /* 125 * Use this function to proceed with scheduling reset_work for a controller 126 * that had previously been set to the resetting state. This is intended for 127 * code paths that can't be interrupted by other reset attempts. A hot removal 128 * may prevent this from succeeding. 129 */ 130 int nvme_try_sched_reset(struct nvme_ctrl *ctrl) 131 { 132 if (ctrl->state != NVME_CTRL_RESETTING) 133 return -EBUSY; 134 if (!queue_work(nvme_reset_wq, &ctrl->reset_work)) 135 return -EBUSY; 136 return 0; 137 } 138 EXPORT_SYMBOL_GPL(nvme_try_sched_reset); 139 140 static void nvme_failfast_work(struct work_struct *work) 141 { 142 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work), 143 struct nvme_ctrl, failfast_work); 144 145 if (ctrl->state != NVME_CTRL_CONNECTING) 146 return; 147 148 set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags); 149 dev_info(ctrl->device, "failfast expired\n"); 150 nvme_kick_requeue_lists(ctrl); 151 } 152 153 static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl) 154 { 155 if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1) 156 return; 157 158 schedule_delayed_work(&ctrl->failfast_work, 159 ctrl->opts->fast_io_fail_tmo * HZ); 160 } 161 162 static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl) 163 { 164 if (!ctrl->opts) 165 return; 166 167 cancel_delayed_work_sync(&ctrl->failfast_work); 168 clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags); 169 } 170 171 172 int nvme_reset_ctrl(struct nvme_ctrl *ctrl) 173 { 174 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 175 return -EBUSY; 176 if (!queue_work(nvme_reset_wq, &ctrl->reset_work)) 177 return -EBUSY; 178 return 0; 179 } 180 EXPORT_SYMBOL_GPL(nvme_reset_ctrl); 181 182 static int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl) 183 { 184 int ret; 185 186 ret = nvme_reset_ctrl(ctrl); 187 if (!ret) { 188 flush_work(&ctrl->reset_work); 189 if (ctrl->state != NVME_CTRL_LIVE) 190 ret = -ENETRESET; 191 } 192 193 return ret; 194 } 195 196 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl) 197 { 198 dev_info(ctrl->device, 199 "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn); 200 201 flush_work(&ctrl->reset_work); 202 nvme_stop_ctrl(ctrl); 203 nvme_remove_namespaces(ctrl); 204 ctrl->ops->delete_ctrl(ctrl); 205 nvme_uninit_ctrl(ctrl); 206 } 207 208 static void nvme_delete_ctrl_work(struct work_struct *work) 209 { 210 struct nvme_ctrl *ctrl = 211 container_of(work, struct nvme_ctrl, delete_work); 212 213 nvme_do_delete_ctrl(ctrl); 214 } 215 216 int nvme_delete_ctrl(struct nvme_ctrl *ctrl) 217 { 218 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING)) 219 return -EBUSY; 220 if (!queue_work(nvme_delete_wq, &ctrl->delete_work)) 221 return -EBUSY; 222 return 0; 223 } 224 EXPORT_SYMBOL_GPL(nvme_delete_ctrl); 225 226 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl) 227 { 228 /* 229 * Keep a reference until nvme_do_delete_ctrl() complete, 230 * since ->delete_ctrl can free the controller. 231 */ 232 nvme_get_ctrl(ctrl); 233 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING)) 234 nvme_do_delete_ctrl(ctrl); 235 nvme_put_ctrl(ctrl); 236 } 237 238 static blk_status_t nvme_error_status(u16 status) 239 { 240 switch (status & 0x7ff) { 241 case NVME_SC_SUCCESS: 242 return BLK_STS_OK; 243 case NVME_SC_CAP_EXCEEDED: 244 return BLK_STS_NOSPC; 245 case NVME_SC_LBA_RANGE: 246 case NVME_SC_CMD_INTERRUPTED: 247 case NVME_SC_NS_NOT_READY: 248 return BLK_STS_TARGET; 249 case NVME_SC_BAD_ATTRIBUTES: 250 case NVME_SC_ONCS_NOT_SUPPORTED: 251 case NVME_SC_INVALID_OPCODE: 252 case NVME_SC_INVALID_FIELD: 253 case NVME_SC_INVALID_NS: 254 return BLK_STS_NOTSUPP; 255 case NVME_SC_WRITE_FAULT: 256 case NVME_SC_READ_ERROR: 257 case NVME_SC_UNWRITTEN_BLOCK: 258 case NVME_SC_ACCESS_DENIED: 259 case NVME_SC_READ_ONLY: 260 case NVME_SC_COMPARE_FAILED: 261 return BLK_STS_MEDIUM; 262 case NVME_SC_GUARD_CHECK: 263 case NVME_SC_APPTAG_CHECK: 264 case NVME_SC_REFTAG_CHECK: 265 case NVME_SC_INVALID_PI: 266 return BLK_STS_PROTECTION; 267 case NVME_SC_RESERVATION_CONFLICT: 268 return BLK_STS_NEXUS; 269 case NVME_SC_HOST_PATH_ERROR: 270 return BLK_STS_TRANSPORT; 271 case NVME_SC_ZONE_TOO_MANY_ACTIVE: 272 return BLK_STS_ZONE_ACTIVE_RESOURCE; 273 case NVME_SC_ZONE_TOO_MANY_OPEN: 274 return BLK_STS_ZONE_OPEN_RESOURCE; 275 default: 276 return BLK_STS_IOERR; 277 } 278 } 279 280 static void nvme_retry_req(struct request *req) 281 { 282 struct nvme_ns *ns = req->q->queuedata; 283 unsigned long delay = 0; 284 u16 crd; 285 286 /* The mask and shift result must be <= 3 */ 287 crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11; 288 if (ns && crd) 289 delay = ns->ctrl->crdt[crd - 1] * 100; 290 291 nvme_req(req)->retries++; 292 blk_mq_requeue_request(req, false); 293 blk_mq_delay_kick_requeue_list(req->q, delay); 294 } 295 296 enum nvme_disposition { 297 COMPLETE, 298 RETRY, 299 FAILOVER, 300 }; 301 302 static inline enum nvme_disposition nvme_decide_disposition(struct request *req) 303 { 304 if (likely(nvme_req(req)->status == 0)) 305 return COMPLETE; 306 307 if (blk_noretry_request(req) || 308 (nvme_req(req)->status & NVME_SC_DNR) || 309 nvme_req(req)->retries >= nvme_max_retries) 310 return COMPLETE; 311 312 if (req->cmd_flags & REQ_NVME_MPATH) { 313 if (nvme_is_path_error(nvme_req(req)->status) || 314 blk_queue_dying(req->q)) 315 return FAILOVER; 316 } else { 317 if (blk_queue_dying(req->q)) 318 return COMPLETE; 319 } 320 321 return RETRY; 322 } 323 324 static inline void nvme_end_req(struct request *req) 325 { 326 blk_status_t status = nvme_error_status(nvme_req(req)->status); 327 328 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && 329 req_op(req) == REQ_OP_ZONE_APPEND) 330 req->__sector = nvme_lba_to_sect(req->q->queuedata, 331 le64_to_cpu(nvme_req(req)->result.u64)); 332 333 nvme_trace_bio_complete(req); 334 blk_mq_end_request(req, status); 335 } 336 337 void nvme_complete_rq(struct request *req) 338 { 339 trace_nvme_complete_rq(req); 340 nvme_cleanup_cmd(req); 341 342 if (nvme_req(req)->ctrl->kas) 343 nvme_req(req)->ctrl->comp_seen = true; 344 345 switch (nvme_decide_disposition(req)) { 346 case COMPLETE: 347 nvme_end_req(req); 348 return; 349 case RETRY: 350 nvme_retry_req(req); 351 return; 352 case FAILOVER: 353 nvme_failover_req(req); 354 return; 355 } 356 } 357 EXPORT_SYMBOL_GPL(nvme_complete_rq); 358 359 bool nvme_cancel_request(struct request *req, void *data, bool reserved) 360 { 361 dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device, 362 "Cancelling I/O %d", req->tag); 363 364 /* don't abort one completed request */ 365 if (blk_mq_request_completed(req)) 366 return true; 367 368 nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD; 369 blk_mq_complete_request(req); 370 return true; 371 } 372 EXPORT_SYMBOL_GPL(nvme_cancel_request); 373 374 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, 375 enum nvme_ctrl_state new_state) 376 { 377 enum nvme_ctrl_state old_state; 378 unsigned long flags; 379 bool changed = false; 380 381 spin_lock_irqsave(&ctrl->lock, flags); 382 383 old_state = ctrl->state; 384 switch (new_state) { 385 case NVME_CTRL_LIVE: 386 switch (old_state) { 387 case NVME_CTRL_NEW: 388 case NVME_CTRL_RESETTING: 389 case NVME_CTRL_CONNECTING: 390 changed = true; 391 fallthrough; 392 default: 393 break; 394 } 395 break; 396 case NVME_CTRL_RESETTING: 397 switch (old_state) { 398 case NVME_CTRL_NEW: 399 case NVME_CTRL_LIVE: 400 changed = true; 401 fallthrough; 402 default: 403 break; 404 } 405 break; 406 case NVME_CTRL_CONNECTING: 407 switch (old_state) { 408 case NVME_CTRL_NEW: 409 case NVME_CTRL_RESETTING: 410 changed = true; 411 fallthrough; 412 default: 413 break; 414 } 415 break; 416 case NVME_CTRL_DELETING: 417 switch (old_state) { 418 case NVME_CTRL_LIVE: 419 case NVME_CTRL_RESETTING: 420 case NVME_CTRL_CONNECTING: 421 changed = true; 422 fallthrough; 423 default: 424 break; 425 } 426 break; 427 case NVME_CTRL_DELETING_NOIO: 428 switch (old_state) { 429 case NVME_CTRL_DELETING: 430 case NVME_CTRL_DEAD: 431 changed = true; 432 fallthrough; 433 default: 434 break; 435 } 436 break; 437 case NVME_CTRL_DEAD: 438 switch (old_state) { 439 case NVME_CTRL_DELETING: 440 changed = true; 441 fallthrough; 442 default: 443 break; 444 } 445 break; 446 default: 447 break; 448 } 449 450 if (changed) { 451 ctrl->state = new_state; 452 wake_up_all(&ctrl->state_wq); 453 } 454 455 spin_unlock_irqrestore(&ctrl->lock, flags); 456 if (!changed) 457 return false; 458 459 if (ctrl->state == NVME_CTRL_LIVE) { 460 if (old_state == NVME_CTRL_CONNECTING) 461 nvme_stop_failfast_work(ctrl); 462 nvme_kick_requeue_lists(ctrl); 463 } else if (ctrl->state == NVME_CTRL_CONNECTING && 464 old_state == NVME_CTRL_RESETTING) { 465 nvme_start_failfast_work(ctrl); 466 } 467 return changed; 468 } 469 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state); 470 471 /* 472 * Returns true for sink states that can't ever transition back to live. 473 */ 474 static bool nvme_state_terminal(struct nvme_ctrl *ctrl) 475 { 476 switch (ctrl->state) { 477 case NVME_CTRL_NEW: 478 case NVME_CTRL_LIVE: 479 case NVME_CTRL_RESETTING: 480 case NVME_CTRL_CONNECTING: 481 return false; 482 case NVME_CTRL_DELETING: 483 case NVME_CTRL_DELETING_NOIO: 484 case NVME_CTRL_DEAD: 485 return true; 486 default: 487 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state); 488 return true; 489 } 490 } 491 492 /* 493 * Waits for the controller state to be resetting, or returns false if it is 494 * not possible to ever transition to that state. 495 */ 496 bool nvme_wait_reset(struct nvme_ctrl *ctrl) 497 { 498 wait_event(ctrl->state_wq, 499 nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) || 500 nvme_state_terminal(ctrl)); 501 return ctrl->state == NVME_CTRL_RESETTING; 502 } 503 EXPORT_SYMBOL_GPL(nvme_wait_reset); 504 505 static void nvme_free_ns_head(struct kref *ref) 506 { 507 struct nvme_ns_head *head = 508 container_of(ref, struct nvme_ns_head, ref); 509 510 nvme_mpath_remove_disk(head); 511 ida_simple_remove(&head->subsys->ns_ida, head->instance); 512 cleanup_srcu_struct(&head->srcu); 513 nvme_put_subsystem(head->subsys); 514 kfree(head); 515 } 516 517 static void nvme_put_ns_head(struct nvme_ns_head *head) 518 { 519 kref_put(&head->ref, nvme_free_ns_head); 520 } 521 522 static void nvme_free_ns(struct kref *kref) 523 { 524 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref); 525 526 if (ns->ndev) 527 nvme_nvm_unregister(ns); 528 529 put_disk(ns->disk); 530 nvme_put_ns_head(ns->head); 531 nvme_put_ctrl(ns->ctrl); 532 kfree(ns); 533 } 534 535 void nvme_put_ns(struct nvme_ns *ns) 536 { 537 kref_put(&ns->kref, nvme_free_ns); 538 } 539 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU); 540 541 static inline void nvme_clear_nvme_request(struct request *req) 542 { 543 if (!(req->rq_flags & RQF_DONTPREP)) { 544 nvme_req(req)->retries = 0; 545 nvme_req(req)->flags = 0; 546 req->rq_flags |= RQF_DONTPREP; 547 } 548 } 549 550 static inline unsigned int nvme_req_op(struct nvme_command *cmd) 551 { 552 return nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN; 553 } 554 555 static inline void nvme_init_request(struct request *req, 556 struct nvme_command *cmd) 557 { 558 if (req->q->queuedata) 559 req->timeout = NVME_IO_TIMEOUT; 560 else /* no queuedata implies admin queue */ 561 req->timeout = NVME_ADMIN_TIMEOUT; 562 563 req->cmd_flags |= REQ_FAILFAST_DRIVER; 564 nvme_clear_nvme_request(req); 565 nvme_req(req)->cmd = cmd; 566 } 567 568 struct request *nvme_alloc_request(struct request_queue *q, 569 struct nvme_command *cmd, blk_mq_req_flags_t flags) 570 { 571 struct request *req; 572 573 req = blk_mq_alloc_request(q, nvme_req_op(cmd), flags); 574 if (!IS_ERR(req)) 575 nvme_init_request(req, cmd); 576 return req; 577 } 578 EXPORT_SYMBOL_GPL(nvme_alloc_request); 579 580 static struct request *nvme_alloc_request_qid(struct request_queue *q, 581 struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid) 582 { 583 struct request *req; 584 585 req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), flags, 586 qid ? qid - 1 : 0); 587 if (!IS_ERR(req)) 588 nvme_init_request(req, cmd); 589 return req; 590 } 591 592 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable) 593 { 594 struct nvme_command c; 595 596 memset(&c, 0, sizeof(c)); 597 598 c.directive.opcode = nvme_admin_directive_send; 599 c.directive.nsid = cpu_to_le32(NVME_NSID_ALL); 600 c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE; 601 c.directive.dtype = NVME_DIR_IDENTIFY; 602 c.directive.tdtype = NVME_DIR_STREAMS; 603 c.directive.endir = enable ? NVME_DIR_ENDIR : 0; 604 605 return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0); 606 } 607 608 static int nvme_disable_streams(struct nvme_ctrl *ctrl) 609 { 610 return nvme_toggle_streams(ctrl, false); 611 } 612 613 static int nvme_enable_streams(struct nvme_ctrl *ctrl) 614 { 615 return nvme_toggle_streams(ctrl, true); 616 } 617 618 static int nvme_get_stream_params(struct nvme_ctrl *ctrl, 619 struct streams_directive_params *s, u32 nsid) 620 { 621 struct nvme_command c; 622 623 memset(&c, 0, sizeof(c)); 624 memset(s, 0, sizeof(*s)); 625 626 c.directive.opcode = nvme_admin_directive_recv; 627 c.directive.nsid = cpu_to_le32(nsid); 628 c.directive.numd = cpu_to_le32(nvme_bytes_to_numd(sizeof(*s))); 629 c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM; 630 c.directive.dtype = NVME_DIR_STREAMS; 631 632 return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s)); 633 } 634 635 static int nvme_configure_directives(struct nvme_ctrl *ctrl) 636 { 637 struct streams_directive_params s; 638 int ret; 639 640 if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES)) 641 return 0; 642 if (!streams) 643 return 0; 644 645 ret = nvme_enable_streams(ctrl); 646 if (ret) 647 return ret; 648 649 ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL); 650 if (ret) 651 goto out_disable_stream; 652 653 ctrl->nssa = le16_to_cpu(s.nssa); 654 if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) { 655 dev_info(ctrl->device, "too few streams (%u) available\n", 656 ctrl->nssa); 657 goto out_disable_stream; 658 } 659 660 ctrl->nr_streams = min_t(u16, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1); 661 dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams); 662 return 0; 663 664 out_disable_stream: 665 nvme_disable_streams(ctrl); 666 return ret; 667 } 668 669 /* 670 * Check if 'req' has a write hint associated with it. If it does, assign 671 * a valid namespace stream to the write. 672 */ 673 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl, 674 struct request *req, u16 *control, 675 u32 *dsmgmt) 676 { 677 enum rw_hint streamid = req->write_hint; 678 679 if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE) 680 streamid = 0; 681 else { 682 streamid--; 683 if (WARN_ON_ONCE(streamid > ctrl->nr_streams)) 684 return; 685 686 *control |= NVME_RW_DTYPE_STREAMS; 687 *dsmgmt |= streamid << 16; 688 } 689 690 if (streamid < ARRAY_SIZE(req->q->write_hints)) 691 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9; 692 } 693 694 static void nvme_setup_passthrough(struct request *req, 695 struct nvme_command *cmd) 696 { 697 memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd)); 698 /* passthru commands should let the driver set the SGL flags */ 699 cmd->common.flags &= ~NVME_CMD_SGL_ALL; 700 } 701 702 static inline void nvme_setup_flush(struct nvme_ns *ns, 703 struct nvme_command *cmnd) 704 { 705 cmnd->common.opcode = nvme_cmd_flush; 706 cmnd->common.nsid = cpu_to_le32(ns->head->ns_id); 707 } 708 709 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req, 710 struct nvme_command *cmnd) 711 { 712 unsigned short segments = blk_rq_nr_discard_segments(req), n = 0; 713 struct nvme_dsm_range *range; 714 struct bio *bio; 715 716 /* 717 * Some devices do not consider the DSM 'Number of Ranges' field when 718 * determining how much data to DMA. Always allocate memory for maximum 719 * number of segments to prevent device reading beyond end of buffer. 720 */ 721 static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES; 722 723 range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN); 724 if (!range) { 725 /* 726 * If we fail allocation our range, fallback to the controller 727 * discard page. If that's also busy, it's safe to return 728 * busy, as we know we can make progress once that's freed. 729 */ 730 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy)) 731 return BLK_STS_RESOURCE; 732 733 range = page_address(ns->ctrl->discard_page); 734 } 735 736 __rq_for_each_bio(bio, req) { 737 u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector); 738 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift; 739 740 if (n < segments) { 741 range[n].cattr = cpu_to_le32(0); 742 range[n].nlb = cpu_to_le32(nlb); 743 range[n].slba = cpu_to_le64(slba); 744 } 745 n++; 746 } 747 748 if (WARN_ON_ONCE(n != segments)) { 749 if (virt_to_page(range) == ns->ctrl->discard_page) 750 clear_bit_unlock(0, &ns->ctrl->discard_page_busy); 751 else 752 kfree(range); 753 return BLK_STS_IOERR; 754 } 755 756 cmnd->dsm.opcode = nvme_cmd_dsm; 757 cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id); 758 cmnd->dsm.nr = cpu_to_le32(segments - 1); 759 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD); 760 761 req->special_vec.bv_page = virt_to_page(range); 762 req->special_vec.bv_offset = offset_in_page(range); 763 req->special_vec.bv_len = alloc_size; 764 req->rq_flags |= RQF_SPECIAL_PAYLOAD; 765 766 return BLK_STS_OK; 767 } 768 769 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns, 770 struct request *req, struct nvme_command *cmnd) 771 { 772 if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES) 773 return nvme_setup_discard(ns, req, cmnd); 774 775 cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes; 776 cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id); 777 cmnd->write_zeroes.slba = 778 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); 779 cmnd->write_zeroes.length = 780 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); 781 cmnd->write_zeroes.control = 0; 782 return BLK_STS_OK; 783 } 784 785 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns, 786 struct request *req, struct nvme_command *cmnd, 787 enum nvme_opcode op) 788 { 789 struct nvme_ctrl *ctrl = ns->ctrl; 790 u16 control = 0; 791 u32 dsmgmt = 0; 792 793 if (req->cmd_flags & REQ_FUA) 794 control |= NVME_RW_FUA; 795 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 796 control |= NVME_RW_LR; 797 798 if (req->cmd_flags & REQ_RAHEAD) 799 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 800 801 cmnd->rw.opcode = op; 802 cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id); 803 cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); 804 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); 805 806 if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams) 807 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt); 808 809 if (ns->ms) { 810 /* 811 * If formated with metadata, the block layer always provides a 812 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled. Else 813 * we enable the PRACT bit for protection information or set the 814 * namespace capacity to zero to prevent any I/O. 815 */ 816 if (!blk_integrity_rq(req)) { 817 if (WARN_ON_ONCE(!nvme_ns_has_pi(ns))) 818 return BLK_STS_NOTSUPP; 819 control |= NVME_RW_PRINFO_PRACT; 820 } 821 822 switch (ns->pi_type) { 823 case NVME_NS_DPS_PI_TYPE3: 824 control |= NVME_RW_PRINFO_PRCHK_GUARD; 825 break; 826 case NVME_NS_DPS_PI_TYPE1: 827 case NVME_NS_DPS_PI_TYPE2: 828 control |= NVME_RW_PRINFO_PRCHK_GUARD | 829 NVME_RW_PRINFO_PRCHK_REF; 830 if (op == nvme_cmd_zone_append) 831 control |= NVME_RW_APPEND_PIREMAP; 832 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req)); 833 break; 834 } 835 } 836 837 cmnd->rw.control = cpu_to_le16(control); 838 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 839 return 0; 840 } 841 842 void nvme_cleanup_cmd(struct request *req) 843 { 844 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) { 845 struct nvme_ns *ns = req->rq_disk->private_data; 846 struct page *page = req->special_vec.bv_page; 847 848 if (page == ns->ctrl->discard_page) 849 clear_bit_unlock(0, &ns->ctrl->discard_page_busy); 850 else 851 kfree(page_address(page) + req->special_vec.bv_offset); 852 } 853 } 854 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd); 855 856 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req, 857 struct nvme_command *cmd) 858 { 859 blk_status_t ret = BLK_STS_OK; 860 861 nvme_clear_nvme_request(req); 862 863 memset(cmd, 0, sizeof(*cmd)); 864 switch (req_op(req)) { 865 case REQ_OP_DRV_IN: 866 case REQ_OP_DRV_OUT: 867 nvme_setup_passthrough(req, cmd); 868 break; 869 case REQ_OP_FLUSH: 870 nvme_setup_flush(ns, cmd); 871 break; 872 case REQ_OP_ZONE_RESET_ALL: 873 case REQ_OP_ZONE_RESET: 874 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET); 875 break; 876 case REQ_OP_ZONE_OPEN: 877 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN); 878 break; 879 case REQ_OP_ZONE_CLOSE: 880 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE); 881 break; 882 case REQ_OP_ZONE_FINISH: 883 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH); 884 break; 885 case REQ_OP_WRITE_ZEROES: 886 ret = nvme_setup_write_zeroes(ns, req, cmd); 887 break; 888 case REQ_OP_DISCARD: 889 ret = nvme_setup_discard(ns, req, cmd); 890 break; 891 case REQ_OP_READ: 892 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read); 893 break; 894 case REQ_OP_WRITE: 895 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write); 896 break; 897 case REQ_OP_ZONE_APPEND: 898 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append); 899 break; 900 default: 901 WARN_ON_ONCE(1); 902 return BLK_STS_IOERR; 903 } 904 905 cmd->common.command_id = req->tag; 906 trace_nvme_setup_cmd(req, cmd); 907 return ret; 908 } 909 EXPORT_SYMBOL_GPL(nvme_setup_cmd); 910 911 static void nvme_end_sync_rq(struct request *rq, blk_status_t error) 912 { 913 struct completion *waiting = rq->end_io_data; 914 915 rq->end_io_data = NULL; 916 complete(waiting); 917 } 918 919 static void nvme_execute_rq_polled(struct request_queue *q, 920 struct gendisk *bd_disk, struct request *rq, int at_head) 921 { 922 DECLARE_COMPLETION_ONSTACK(wait); 923 924 WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)); 925 926 rq->cmd_flags |= REQ_HIPRI; 927 rq->end_io_data = &wait; 928 blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq); 929 930 while (!completion_done(&wait)) { 931 blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true); 932 cond_resched(); 933 } 934 } 935 936 /* 937 * Returns 0 on success. If the result is negative, it's a Linux error code; 938 * if the result is positive, it's an NVM Express status code 939 */ 940 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 941 union nvme_result *result, void *buffer, unsigned bufflen, 942 unsigned timeout, int qid, int at_head, 943 blk_mq_req_flags_t flags, bool poll) 944 { 945 struct request *req; 946 int ret; 947 948 if (qid == NVME_QID_ANY) 949 req = nvme_alloc_request(q, cmd, flags); 950 else 951 req = nvme_alloc_request_qid(q, cmd, flags, qid); 952 if (IS_ERR(req)) 953 return PTR_ERR(req); 954 955 if (timeout) 956 req->timeout = timeout; 957 958 if (buffer && bufflen) { 959 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL); 960 if (ret) 961 goto out; 962 } 963 964 if (poll) 965 nvme_execute_rq_polled(req->q, NULL, req, at_head); 966 else 967 blk_execute_rq(req->q, NULL, req, at_head); 968 if (result) 969 *result = nvme_req(req)->result; 970 if (nvme_req(req)->flags & NVME_REQ_CANCELLED) 971 ret = -EINTR; 972 else 973 ret = nvme_req(req)->status; 974 out: 975 blk_mq_free_request(req); 976 return ret; 977 } 978 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd); 979 980 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 981 void *buffer, unsigned bufflen) 982 { 983 return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0, 984 NVME_QID_ANY, 0, 0, false); 985 } 986 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd); 987 988 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf, 989 unsigned len, u32 seed, bool write) 990 { 991 struct bio_integrity_payload *bip; 992 int ret = -ENOMEM; 993 void *buf; 994 995 buf = kmalloc(len, GFP_KERNEL); 996 if (!buf) 997 goto out; 998 999 ret = -EFAULT; 1000 if (write && copy_from_user(buf, ubuf, len)) 1001 goto out_free_meta; 1002 1003 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1); 1004 if (IS_ERR(bip)) { 1005 ret = PTR_ERR(bip); 1006 goto out_free_meta; 1007 } 1008 1009 bip->bip_iter.bi_size = len; 1010 bip->bip_iter.bi_sector = seed; 1011 ret = bio_integrity_add_page(bio, virt_to_page(buf), len, 1012 offset_in_page(buf)); 1013 if (ret == len) 1014 return buf; 1015 ret = -ENOMEM; 1016 out_free_meta: 1017 kfree(buf); 1018 out: 1019 return ERR_PTR(ret); 1020 } 1021 1022 static u32 nvme_known_admin_effects(u8 opcode) 1023 { 1024 switch (opcode) { 1025 case nvme_admin_format_nvm: 1026 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_NCC | 1027 NVME_CMD_EFFECTS_CSE_MASK; 1028 case nvme_admin_sanitize_nvm: 1029 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK; 1030 default: 1031 break; 1032 } 1033 return 0; 1034 } 1035 1036 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode) 1037 { 1038 u32 effects = 0; 1039 1040 if (ns) { 1041 if (ns->head->effects) 1042 effects = le32_to_cpu(ns->head->effects->iocs[opcode]); 1043 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC)) 1044 dev_warn(ctrl->device, 1045 "IO command:%02x has unhandled effects:%08x\n", 1046 opcode, effects); 1047 return 0; 1048 } 1049 1050 if (ctrl->effects) 1051 effects = le32_to_cpu(ctrl->effects->acs[opcode]); 1052 effects |= nvme_known_admin_effects(opcode); 1053 1054 return effects; 1055 } 1056 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU); 1057 1058 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 1059 u8 opcode) 1060 { 1061 u32 effects = nvme_command_effects(ctrl, ns, opcode); 1062 1063 /* 1064 * For simplicity, IO to all namespaces is quiesced even if the command 1065 * effects say only one namespace is affected. 1066 */ 1067 if (effects & NVME_CMD_EFFECTS_CSE_MASK) { 1068 mutex_lock(&ctrl->scan_lock); 1069 mutex_lock(&ctrl->subsys->lock); 1070 nvme_mpath_start_freeze(ctrl->subsys); 1071 nvme_mpath_wait_freeze(ctrl->subsys); 1072 nvme_start_freeze(ctrl); 1073 nvme_wait_freeze(ctrl); 1074 } 1075 return effects; 1076 } 1077 1078 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects) 1079 { 1080 if (effects & NVME_CMD_EFFECTS_CSE_MASK) { 1081 nvme_unfreeze(ctrl); 1082 nvme_mpath_unfreeze(ctrl->subsys); 1083 mutex_unlock(&ctrl->subsys->lock); 1084 nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL); 1085 mutex_unlock(&ctrl->scan_lock); 1086 } 1087 if (effects & NVME_CMD_EFFECTS_CCC) 1088 nvme_init_identify(ctrl); 1089 if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) { 1090 nvme_queue_scan(ctrl); 1091 flush_work(&ctrl->scan_work); 1092 } 1093 } 1094 1095 void nvme_execute_passthru_rq(struct request *rq) 1096 { 1097 struct nvme_command *cmd = nvme_req(rq)->cmd; 1098 struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl; 1099 struct nvme_ns *ns = rq->q->queuedata; 1100 struct gendisk *disk = ns ? ns->disk : NULL; 1101 u32 effects; 1102 1103 effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode); 1104 blk_execute_rq(rq->q, disk, rq, 0); 1105 nvme_passthru_end(ctrl, effects); 1106 } 1107 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU); 1108 1109 static int nvme_submit_user_cmd(struct request_queue *q, 1110 struct nvme_command *cmd, void __user *ubuffer, 1111 unsigned bufflen, void __user *meta_buffer, unsigned meta_len, 1112 u32 meta_seed, u64 *result, unsigned timeout) 1113 { 1114 bool write = nvme_is_write(cmd); 1115 struct nvme_ns *ns = q->queuedata; 1116 struct gendisk *disk = ns ? ns->disk : NULL; 1117 struct request *req; 1118 struct bio *bio = NULL; 1119 void *meta = NULL; 1120 int ret; 1121 1122 req = nvme_alloc_request(q, cmd, 0); 1123 if (IS_ERR(req)) 1124 return PTR_ERR(req); 1125 1126 if (timeout) 1127 req->timeout = timeout; 1128 nvme_req(req)->flags |= NVME_REQ_USERCMD; 1129 1130 if (ubuffer && bufflen) { 1131 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen, 1132 GFP_KERNEL); 1133 if (ret) 1134 goto out; 1135 bio = req->bio; 1136 bio->bi_disk = disk; 1137 if (disk && meta_buffer && meta_len) { 1138 meta = nvme_add_user_metadata(bio, meta_buffer, meta_len, 1139 meta_seed, write); 1140 if (IS_ERR(meta)) { 1141 ret = PTR_ERR(meta); 1142 goto out_unmap; 1143 } 1144 req->cmd_flags |= REQ_INTEGRITY; 1145 } 1146 } 1147 1148 nvme_execute_passthru_rq(req); 1149 if (nvme_req(req)->flags & NVME_REQ_CANCELLED) 1150 ret = -EINTR; 1151 else 1152 ret = nvme_req(req)->status; 1153 if (result) 1154 *result = le64_to_cpu(nvme_req(req)->result.u64); 1155 if (meta && !ret && !write) { 1156 if (copy_to_user(meta_buffer, meta, meta_len)) 1157 ret = -EFAULT; 1158 } 1159 kfree(meta); 1160 out_unmap: 1161 if (bio) 1162 blk_rq_unmap_user(bio); 1163 out: 1164 blk_mq_free_request(req); 1165 return ret; 1166 } 1167 1168 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status) 1169 { 1170 struct nvme_ctrl *ctrl = rq->end_io_data; 1171 unsigned long flags; 1172 bool startka = false; 1173 1174 blk_mq_free_request(rq); 1175 1176 if (status) { 1177 dev_err(ctrl->device, 1178 "failed nvme_keep_alive_end_io error=%d\n", 1179 status); 1180 return; 1181 } 1182 1183 ctrl->comp_seen = false; 1184 spin_lock_irqsave(&ctrl->lock, flags); 1185 if (ctrl->state == NVME_CTRL_LIVE || 1186 ctrl->state == NVME_CTRL_CONNECTING) 1187 startka = true; 1188 spin_unlock_irqrestore(&ctrl->lock, flags); 1189 if (startka) 1190 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ); 1191 } 1192 1193 static int nvme_keep_alive(struct nvme_ctrl *ctrl) 1194 { 1195 struct request *rq; 1196 1197 rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, 1198 BLK_MQ_REQ_RESERVED); 1199 if (IS_ERR(rq)) 1200 return PTR_ERR(rq); 1201 1202 rq->timeout = ctrl->kato * HZ; 1203 rq->end_io_data = ctrl; 1204 1205 blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io); 1206 1207 return 0; 1208 } 1209 1210 static void nvme_keep_alive_work(struct work_struct *work) 1211 { 1212 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work), 1213 struct nvme_ctrl, ka_work); 1214 bool comp_seen = ctrl->comp_seen; 1215 1216 if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) { 1217 dev_dbg(ctrl->device, 1218 "reschedule traffic based keep-alive timer\n"); 1219 ctrl->comp_seen = false; 1220 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ); 1221 return; 1222 } 1223 1224 if (nvme_keep_alive(ctrl)) { 1225 /* allocation failure, reset the controller */ 1226 dev_err(ctrl->device, "keep-alive failed\n"); 1227 nvme_reset_ctrl(ctrl); 1228 return; 1229 } 1230 } 1231 1232 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl) 1233 { 1234 if (unlikely(ctrl->kato == 0)) 1235 return; 1236 1237 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ); 1238 } 1239 1240 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl) 1241 { 1242 if (unlikely(ctrl->kato == 0)) 1243 return; 1244 1245 cancel_delayed_work_sync(&ctrl->ka_work); 1246 } 1247 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive); 1248 1249 /* 1250 * In NVMe 1.0 the CNS field was just a binary controller or namespace 1251 * flag, thus sending any new CNS opcodes has a big chance of not working. 1252 * Qemu unfortunately had that bug after reporting a 1.1 version compliance 1253 * (but not for any later version). 1254 */ 1255 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl) 1256 { 1257 if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS) 1258 return ctrl->vs < NVME_VS(1, 2, 0); 1259 return ctrl->vs < NVME_VS(1, 1, 0); 1260 } 1261 1262 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id) 1263 { 1264 struct nvme_command c = { }; 1265 int error; 1266 1267 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */ 1268 c.identify.opcode = nvme_admin_identify; 1269 c.identify.cns = NVME_ID_CNS_CTRL; 1270 1271 *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL); 1272 if (!*id) 1273 return -ENOMEM; 1274 1275 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id, 1276 sizeof(struct nvme_id_ctrl)); 1277 if (error) 1278 kfree(*id); 1279 return error; 1280 } 1281 1282 static bool nvme_multi_css(struct nvme_ctrl *ctrl) 1283 { 1284 return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI; 1285 } 1286 1287 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids, 1288 struct nvme_ns_id_desc *cur, bool *csi_seen) 1289 { 1290 const char *warn_str = "ctrl returned bogus length:"; 1291 void *data = cur; 1292 1293 switch (cur->nidt) { 1294 case NVME_NIDT_EUI64: 1295 if (cur->nidl != NVME_NIDT_EUI64_LEN) { 1296 dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n", 1297 warn_str, cur->nidl); 1298 return -1; 1299 } 1300 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN); 1301 return NVME_NIDT_EUI64_LEN; 1302 case NVME_NIDT_NGUID: 1303 if (cur->nidl != NVME_NIDT_NGUID_LEN) { 1304 dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n", 1305 warn_str, cur->nidl); 1306 return -1; 1307 } 1308 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN); 1309 return NVME_NIDT_NGUID_LEN; 1310 case NVME_NIDT_UUID: 1311 if (cur->nidl != NVME_NIDT_UUID_LEN) { 1312 dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n", 1313 warn_str, cur->nidl); 1314 return -1; 1315 } 1316 uuid_copy(&ids->uuid, data + sizeof(*cur)); 1317 return NVME_NIDT_UUID_LEN; 1318 case NVME_NIDT_CSI: 1319 if (cur->nidl != NVME_NIDT_CSI_LEN) { 1320 dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n", 1321 warn_str, cur->nidl); 1322 return -1; 1323 } 1324 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN); 1325 *csi_seen = true; 1326 return NVME_NIDT_CSI_LEN; 1327 default: 1328 /* Skip unknown types */ 1329 return cur->nidl; 1330 } 1331 } 1332 1333 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid, 1334 struct nvme_ns_ids *ids) 1335 { 1336 struct nvme_command c = { }; 1337 bool csi_seen = false; 1338 int status, pos, len; 1339 void *data; 1340 1341 if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl)) 1342 return 0; 1343 if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST) 1344 return 0; 1345 1346 c.identify.opcode = nvme_admin_identify; 1347 c.identify.nsid = cpu_to_le32(nsid); 1348 c.identify.cns = NVME_ID_CNS_NS_DESC_LIST; 1349 1350 data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL); 1351 if (!data) 1352 return -ENOMEM; 1353 1354 status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data, 1355 NVME_IDENTIFY_DATA_SIZE); 1356 if (status) { 1357 dev_warn(ctrl->device, 1358 "Identify Descriptors failed (nsid=%u, status=0x%x)\n", 1359 nsid, status); 1360 goto free_data; 1361 } 1362 1363 for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) { 1364 struct nvme_ns_id_desc *cur = data + pos; 1365 1366 if (cur->nidl == 0) 1367 break; 1368 1369 len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen); 1370 if (len < 0) 1371 break; 1372 1373 len += sizeof(*cur); 1374 } 1375 1376 if (nvme_multi_css(ctrl) && !csi_seen) { 1377 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n", 1378 nsid); 1379 status = -EINVAL; 1380 } 1381 1382 free_data: 1383 kfree(data); 1384 return status; 1385 } 1386 1387 static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid, 1388 struct nvme_ns_ids *ids, struct nvme_id_ns **id) 1389 { 1390 struct nvme_command c = { }; 1391 int error; 1392 1393 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */ 1394 c.identify.opcode = nvme_admin_identify; 1395 c.identify.nsid = cpu_to_le32(nsid); 1396 c.identify.cns = NVME_ID_CNS_NS; 1397 1398 *id = kmalloc(sizeof(**id), GFP_KERNEL); 1399 if (!*id) 1400 return -ENOMEM; 1401 1402 error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id)); 1403 if (error) { 1404 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error); 1405 goto out_free_id; 1406 } 1407 1408 error = -ENODEV; 1409 if ((*id)->ncap == 0) /* namespace not allocated or attached */ 1410 goto out_free_id; 1411 1412 if (ctrl->vs >= NVME_VS(1, 1, 0) && 1413 !memchr_inv(ids->eui64, 0, sizeof(ids->eui64))) 1414 memcpy(ids->eui64, (*id)->eui64, sizeof(ids->eui64)); 1415 if (ctrl->vs >= NVME_VS(1, 2, 0) && 1416 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid))) 1417 memcpy(ids->nguid, (*id)->nguid, sizeof(ids->nguid)); 1418 1419 return 0; 1420 1421 out_free_id: 1422 kfree(*id); 1423 return error; 1424 } 1425 1426 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid, 1427 unsigned int dword11, void *buffer, size_t buflen, u32 *result) 1428 { 1429 union nvme_result res = { 0 }; 1430 struct nvme_command c; 1431 int ret; 1432 1433 memset(&c, 0, sizeof(c)); 1434 c.features.opcode = op; 1435 c.features.fid = cpu_to_le32(fid); 1436 c.features.dword11 = cpu_to_le32(dword11); 1437 1438 ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res, 1439 buffer, buflen, 0, NVME_QID_ANY, 0, 0, false); 1440 if (ret >= 0 && result) 1441 *result = le32_to_cpu(res.u32); 1442 return ret; 1443 } 1444 1445 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid, 1446 unsigned int dword11, void *buffer, size_t buflen, 1447 u32 *result) 1448 { 1449 return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer, 1450 buflen, result); 1451 } 1452 EXPORT_SYMBOL_GPL(nvme_set_features); 1453 1454 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid, 1455 unsigned int dword11, void *buffer, size_t buflen, 1456 u32 *result) 1457 { 1458 return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer, 1459 buflen, result); 1460 } 1461 EXPORT_SYMBOL_GPL(nvme_get_features); 1462 1463 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count) 1464 { 1465 u32 q_count = (*count - 1) | ((*count - 1) << 16); 1466 u32 result; 1467 int status, nr_io_queues; 1468 1469 status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0, 1470 &result); 1471 if (status < 0) 1472 return status; 1473 1474 /* 1475 * Degraded controllers might return an error when setting the queue 1476 * count. We still want to be able to bring them online and offer 1477 * access to the admin queue, as that might be only way to fix them up. 1478 */ 1479 if (status > 0) { 1480 dev_err(ctrl->device, "Could not set queue count (%d)\n", status); 1481 *count = 0; 1482 } else { 1483 nr_io_queues = min(result & 0xffff, result >> 16) + 1; 1484 *count = min(*count, nr_io_queues); 1485 } 1486 1487 return 0; 1488 } 1489 EXPORT_SYMBOL_GPL(nvme_set_queue_count); 1490 1491 #define NVME_AEN_SUPPORTED \ 1492 (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \ 1493 NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE) 1494 1495 static void nvme_enable_aen(struct nvme_ctrl *ctrl) 1496 { 1497 u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED; 1498 int status; 1499 1500 if (!supported_aens) 1501 return; 1502 1503 status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens, 1504 NULL, 0, &result); 1505 if (status) 1506 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n", 1507 supported_aens); 1508 1509 queue_work(nvme_wq, &ctrl->async_event_work); 1510 } 1511 1512 /* 1513 * Convert integer values from ioctl structures to user pointers, silently 1514 * ignoring the upper bits in the compat case to match behaviour of 32-bit 1515 * kernels. 1516 */ 1517 static void __user *nvme_to_user_ptr(uintptr_t ptrval) 1518 { 1519 if (in_compat_syscall()) 1520 ptrval = (compat_uptr_t)ptrval; 1521 return (void __user *)ptrval; 1522 } 1523 1524 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 1525 { 1526 struct nvme_user_io io; 1527 struct nvme_command c; 1528 unsigned length, meta_len; 1529 void __user *metadata; 1530 1531 if (copy_from_user(&io, uio, sizeof(io))) 1532 return -EFAULT; 1533 if (io.flags) 1534 return -EINVAL; 1535 1536 switch (io.opcode) { 1537 case nvme_cmd_write: 1538 case nvme_cmd_read: 1539 case nvme_cmd_compare: 1540 break; 1541 default: 1542 return -EINVAL; 1543 } 1544 1545 length = (io.nblocks + 1) << ns->lba_shift; 1546 1547 if ((io.control & NVME_RW_PRINFO_PRACT) && 1548 ns->ms == sizeof(struct t10_pi_tuple)) { 1549 /* 1550 * Protection information is stripped/inserted by the 1551 * controller. 1552 */ 1553 if (nvme_to_user_ptr(io.metadata)) 1554 return -EINVAL; 1555 meta_len = 0; 1556 metadata = NULL; 1557 } else { 1558 meta_len = (io.nblocks + 1) * ns->ms; 1559 metadata = nvme_to_user_ptr(io.metadata); 1560 } 1561 1562 if (ns->features & NVME_NS_EXT_LBAS) { 1563 length += meta_len; 1564 meta_len = 0; 1565 } else if (meta_len) { 1566 if ((io.metadata & 3) || !io.metadata) 1567 return -EINVAL; 1568 } 1569 1570 memset(&c, 0, sizeof(c)); 1571 c.rw.opcode = io.opcode; 1572 c.rw.flags = io.flags; 1573 c.rw.nsid = cpu_to_le32(ns->head->ns_id); 1574 c.rw.slba = cpu_to_le64(io.slba); 1575 c.rw.length = cpu_to_le16(io.nblocks); 1576 c.rw.control = cpu_to_le16(io.control); 1577 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 1578 c.rw.reftag = cpu_to_le32(io.reftag); 1579 c.rw.apptag = cpu_to_le16(io.apptag); 1580 c.rw.appmask = cpu_to_le16(io.appmask); 1581 1582 return nvme_submit_user_cmd(ns->queue, &c, 1583 nvme_to_user_ptr(io.addr), length, 1584 metadata, meta_len, lower_32_bits(io.slba), NULL, 0); 1585 } 1586 1587 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 1588 struct nvme_passthru_cmd __user *ucmd) 1589 { 1590 struct nvme_passthru_cmd cmd; 1591 struct nvme_command c; 1592 unsigned timeout = 0; 1593 u64 result; 1594 int status; 1595 1596 if (!capable(CAP_SYS_ADMIN)) 1597 return -EACCES; 1598 if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 1599 return -EFAULT; 1600 if (cmd.flags) 1601 return -EINVAL; 1602 1603 memset(&c, 0, sizeof(c)); 1604 c.common.opcode = cmd.opcode; 1605 c.common.flags = cmd.flags; 1606 c.common.nsid = cpu_to_le32(cmd.nsid); 1607 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 1608 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 1609 c.common.cdw10 = cpu_to_le32(cmd.cdw10); 1610 c.common.cdw11 = cpu_to_le32(cmd.cdw11); 1611 c.common.cdw12 = cpu_to_le32(cmd.cdw12); 1612 c.common.cdw13 = cpu_to_le32(cmd.cdw13); 1613 c.common.cdw14 = cpu_to_le32(cmd.cdw14); 1614 c.common.cdw15 = cpu_to_le32(cmd.cdw15); 1615 1616 if (cmd.timeout_ms) 1617 timeout = msecs_to_jiffies(cmd.timeout_ms); 1618 1619 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, 1620 nvme_to_user_ptr(cmd.addr), cmd.data_len, 1621 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len, 1622 0, &result, timeout); 1623 1624 if (status >= 0) { 1625 if (put_user(result, &ucmd->result)) 1626 return -EFAULT; 1627 } 1628 1629 return status; 1630 } 1631 1632 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 1633 struct nvme_passthru_cmd64 __user *ucmd) 1634 { 1635 struct nvme_passthru_cmd64 cmd; 1636 struct nvme_command c; 1637 unsigned timeout = 0; 1638 int status; 1639 1640 if (!capable(CAP_SYS_ADMIN)) 1641 return -EACCES; 1642 if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 1643 return -EFAULT; 1644 if (cmd.flags) 1645 return -EINVAL; 1646 1647 memset(&c, 0, sizeof(c)); 1648 c.common.opcode = cmd.opcode; 1649 c.common.flags = cmd.flags; 1650 c.common.nsid = cpu_to_le32(cmd.nsid); 1651 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 1652 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 1653 c.common.cdw10 = cpu_to_le32(cmd.cdw10); 1654 c.common.cdw11 = cpu_to_le32(cmd.cdw11); 1655 c.common.cdw12 = cpu_to_le32(cmd.cdw12); 1656 c.common.cdw13 = cpu_to_le32(cmd.cdw13); 1657 c.common.cdw14 = cpu_to_le32(cmd.cdw14); 1658 c.common.cdw15 = cpu_to_le32(cmd.cdw15); 1659 1660 if (cmd.timeout_ms) 1661 timeout = msecs_to_jiffies(cmd.timeout_ms); 1662 1663 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, 1664 nvme_to_user_ptr(cmd.addr), cmd.data_len, 1665 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len, 1666 0, &cmd.result, timeout); 1667 1668 if (status >= 0) { 1669 if (put_user(cmd.result, &ucmd->result)) 1670 return -EFAULT; 1671 } 1672 1673 return status; 1674 } 1675 1676 /* 1677 * Issue ioctl requests on the first available path. Note that unlike normal 1678 * block layer requests we will not retry failed request on another controller. 1679 */ 1680 struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk, 1681 struct nvme_ns_head **head, int *srcu_idx) 1682 { 1683 #ifdef CONFIG_NVME_MULTIPATH 1684 if (disk->fops == &nvme_ns_head_ops) { 1685 struct nvme_ns *ns; 1686 1687 *head = disk->private_data; 1688 *srcu_idx = srcu_read_lock(&(*head)->srcu); 1689 ns = nvme_find_path(*head); 1690 if (!ns) 1691 srcu_read_unlock(&(*head)->srcu, *srcu_idx); 1692 return ns; 1693 } 1694 #endif 1695 *head = NULL; 1696 *srcu_idx = -1; 1697 return disk->private_data; 1698 } 1699 1700 void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx) 1701 { 1702 if (head) 1703 srcu_read_unlock(&head->srcu, idx); 1704 } 1705 1706 static bool is_ctrl_ioctl(unsigned int cmd) 1707 { 1708 if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD) 1709 return true; 1710 if (is_sed_ioctl(cmd)) 1711 return true; 1712 return false; 1713 } 1714 1715 static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, 1716 void __user *argp, 1717 struct nvme_ns_head *head, 1718 int srcu_idx) 1719 { 1720 struct nvme_ctrl *ctrl = ns->ctrl; 1721 int ret; 1722 1723 nvme_get_ctrl(ns->ctrl); 1724 nvme_put_ns_from_disk(head, srcu_idx); 1725 1726 switch (cmd) { 1727 case NVME_IOCTL_ADMIN_CMD: 1728 ret = nvme_user_cmd(ctrl, NULL, argp); 1729 break; 1730 case NVME_IOCTL_ADMIN64_CMD: 1731 ret = nvme_user_cmd64(ctrl, NULL, argp); 1732 break; 1733 default: 1734 ret = sed_ioctl(ctrl->opal_dev, cmd, argp); 1735 break; 1736 } 1737 nvme_put_ctrl(ctrl); 1738 return ret; 1739 } 1740 1741 static int nvme_ioctl(struct block_device *bdev, fmode_t mode, 1742 unsigned int cmd, unsigned long arg) 1743 { 1744 struct nvme_ns_head *head = NULL; 1745 void __user *argp = (void __user *)arg; 1746 struct nvme_ns *ns; 1747 int srcu_idx, ret; 1748 1749 ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx); 1750 if (unlikely(!ns)) 1751 return -EWOULDBLOCK; 1752 1753 /* 1754 * Handle ioctls that apply to the controller instead of the namespace 1755 * seperately and drop the ns SRCU reference early. This avoids a 1756 * deadlock when deleting namespaces using the passthrough interface. 1757 */ 1758 if (is_ctrl_ioctl(cmd)) 1759 return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx); 1760 1761 switch (cmd) { 1762 case NVME_IOCTL_ID: 1763 force_successful_syscall_return(); 1764 ret = ns->head->ns_id; 1765 break; 1766 case NVME_IOCTL_IO_CMD: 1767 ret = nvme_user_cmd(ns->ctrl, ns, argp); 1768 break; 1769 case NVME_IOCTL_SUBMIT_IO: 1770 ret = nvme_submit_io(ns, argp); 1771 break; 1772 case NVME_IOCTL_IO64_CMD: 1773 ret = nvme_user_cmd64(ns->ctrl, ns, argp); 1774 break; 1775 default: 1776 if (ns->ndev) 1777 ret = nvme_nvm_ioctl(ns, cmd, arg); 1778 else 1779 ret = -ENOTTY; 1780 } 1781 1782 nvme_put_ns_from_disk(head, srcu_idx); 1783 return ret; 1784 } 1785 1786 #ifdef CONFIG_COMPAT 1787 struct nvme_user_io32 { 1788 __u8 opcode; 1789 __u8 flags; 1790 __u16 control; 1791 __u16 nblocks; 1792 __u16 rsvd; 1793 __u64 metadata; 1794 __u64 addr; 1795 __u64 slba; 1796 __u32 dsmgmt; 1797 __u32 reftag; 1798 __u16 apptag; 1799 __u16 appmask; 1800 } __attribute__((__packed__)); 1801 1802 #define NVME_IOCTL_SUBMIT_IO32 _IOW('N', 0x42, struct nvme_user_io32) 1803 1804 static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode, 1805 unsigned int cmd, unsigned long arg) 1806 { 1807 /* 1808 * Corresponds to the difference of NVME_IOCTL_SUBMIT_IO 1809 * between 32 bit programs and 64 bit kernel. 1810 * The cause is that the results of sizeof(struct nvme_user_io), 1811 * which is used to define NVME_IOCTL_SUBMIT_IO, 1812 * are not same between 32 bit compiler and 64 bit compiler. 1813 * NVME_IOCTL_SUBMIT_IO32 is for 64 bit kernel handling 1814 * NVME_IOCTL_SUBMIT_IO issued from 32 bit programs. 1815 * Other IOCTL numbers are same between 32 bit and 64 bit. 1816 * So there is nothing to do regarding to other IOCTL numbers. 1817 */ 1818 if (cmd == NVME_IOCTL_SUBMIT_IO32) 1819 return nvme_ioctl(bdev, mode, NVME_IOCTL_SUBMIT_IO, arg); 1820 1821 return nvme_ioctl(bdev, mode, cmd, arg); 1822 } 1823 #else 1824 #define nvme_compat_ioctl NULL 1825 #endif /* CONFIG_COMPAT */ 1826 1827 static int nvme_open(struct block_device *bdev, fmode_t mode) 1828 { 1829 struct nvme_ns *ns = bdev->bd_disk->private_data; 1830 1831 #ifdef CONFIG_NVME_MULTIPATH 1832 /* should never be called due to GENHD_FL_HIDDEN */ 1833 if (WARN_ON_ONCE(ns->head->disk)) 1834 goto fail; 1835 #endif 1836 if (!kref_get_unless_zero(&ns->kref)) 1837 goto fail; 1838 if (!try_module_get(ns->ctrl->ops->module)) 1839 goto fail_put_ns; 1840 1841 return 0; 1842 1843 fail_put_ns: 1844 nvme_put_ns(ns); 1845 fail: 1846 return -ENXIO; 1847 } 1848 1849 static void nvme_release(struct gendisk *disk, fmode_t mode) 1850 { 1851 struct nvme_ns *ns = disk->private_data; 1852 1853 module_put(ns->ctrl->ops->module); 1854 nvme_put_ns(ns); 1855 } 1856 1857 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo) 1858 { 1859 /* some standard values */ 1860 geo->heads = 1 << 6; 1861 geo->sectors = 1 << 5; 1862 geo->cylinders = get_capacity(bdev->bd_disk) >> 11; 1863 return 0; 1864 } 1865 1866 #ifdef CONFIG_BLK_DEV_INTEGRITY 1867 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type, 1868 u32 max_integrity_segments) 1869 { 1870 struct blk_integrity integrity; 1871 1872 memset(&integrity, 0, sizeof(integrity)); 1873 switch (pi_type) { 1874 case NVME_NS_DPS_PI_TYPE3: 1875 integrity.profile = &t10_pi_type3_crc; 1876 integrity.tag_size = sizeof(u16) + sizeof(u32); 1877 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE; 1878 break; 1879 case NVME_NS_DPS_PI_TYPE1: 1880 case NVME_NS_DPS_PI_TYPE2: 1881 integrity.profile = &t10_pi_type1_crc; 1882 integrity.tag_size = sizeof(u16); 1883 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE; 1884 break; 1885 default: 1886 integrity.profile = NULL; 1887 break; 1888 } 1889 integrity.tuple_size = ms; 1890 blk_integrity_register(disk, &integrity); 1891 blk_queue_max_integrity_segments(disk->queue, max_integrity_segments); 1892 } 1893 #else 1894 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type, 1895 u32 max_integrity_segments) 1896 { 1897 } 1898 #endif /* CONFIG_BLK_DEV_INTEGRITY */ 1899 1900 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns) 1901 { 1902 struct nvme_ctrl *ctrl = ns->ctrl; 1903 struct request_queue *queue = disk->queue; 1904 u32 size = queue_logical_block_size(queue); 1905 1906 if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) { 1907 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue); 1908 return; 1909 } 1910 1911 if (ctrl->nr_streams && ns->sws && ns->sgs) 1912 size *= ns->sws * ns->sgs; 1913 1914 BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) < 1915 NVME_DSM_MAX_RANGES); 1916 1917 queue->limits.discard_alignment = 0; 1918 queue->limits.discard_granularity = size; 1919 1920 /* If discard is already enabled, don't reset queue limits */ 1921 if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue)) 1922 return; 1923 1924 blk_queue_max_discard_sectors(queue, UINT_MAX); 1925 blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES); 1926 1927 if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES) 1928 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX); 1929 } 1930 1931 static void nvme_config_write_zeroes(struct gendisk *disk, struct nvme_ns *ns) 1932 { 1933 u64 max_blocks; 1934 1935 if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) || 1936 (ns->ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES)) 1937 return; 1938 /* 1939 * Even though NVMe spec explicitly states that MDTS is not 1940 * applicable to the write-zeroes:- "The restriction does not apply to 1941 * commands that do not transfer data between the host and the 1942 * controller (e.g., Write Uncorrectable ro Write Zeroes command).". 1943 * In order to be more cautious use controller's max_hw_sectors value 1944 * to configure the maximum sectors for the write-zeroes which is 1945 * configured based on the controller's MDTS field in the 1946 * nvme_init_identify() if available. 1947 */ 1948 if (ns->ctrl->max_hw_sectors == UINT_MAX) 1949 max_blocks = (u64)USHRT_MAX + 1; 1950 else 1951 max_blocks = ns->ctrl->max_hw_sectors + 1; 1952 1953 blk_queue_max_write_zeroes_sectors(disk->queue, 1954 nvme_lba_to_sect(ns, max_blocks)); 1955 } 1956 1957 static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids) 1958 { 1959 return !uuid_is_null(&ids->uuid) || 1960 memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) || 1961 memchr_inv(ids->eui64, 0, sizeof(ids->eui64)); 1962 } 1963 1964 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b) 1965 { 1966 return uuid_equal(&a->uuid, &b->uuid) && 1967 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 && 1968 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 && 1969 a->csi == b->csi; 1970 } 1971 1972 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 1973 u32 *phys_bs, u32 *io_opt) 1974 { 1975 struct streams_directive_params s; 1976 int ret; 1977 1978 if (!ctrl->nr_streams) 1979 return 0; 1980 1981 ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id); 1982 if (ret) 1983 return ret; 1984 1985 ns->sws = le32_to_cpu(s.sws); 1986 ns->sgs = le16_to_cpu(s.sgs); 1987 1988 if (ns->sws) { 1989 *phys_bs = ns->sws * (1 << ns->lba_shift); 1990 if (ns->sgs) 1991 *io_opt = *phys_bs * ns->sgs; 1992 } 1993 1994 return 0; 1995 } 1996 1997 static int nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id) 1998 { 1999 struct nvme_ctrl *ctrl = ns->ctrl; 2000 2001 /* 2002 * The PI implementation requires the metadata size to be equal to the 2003 * t10 pi tuple size. 2004 */ 2005 ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms); 2006 if (ns->ms == sizeof(struct t10_pi_tuple)) 2007 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK; 2008 else 2009 ns->pi_type = 0; 2010 2011 ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS); 2012 if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED)) 2013 return 0; 2014 if (ctrl->ops->flags & NVME_F_FABRICS) { 2015 /* 2016 * The NVMe over Fabrics specification only supports metadata as 2017 * part of the extended data LBA. We rely on HCA/HBA support to 2018 * remap the separate metadata buffer from the block layer. 2019 */ 2020 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT))) 2021 return -EINVAL; 2022 if (ctrl->max_integrity_segments) 2023 ns->features |= 2024 (NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS); 2025 } else { 2026 /* 2027 * For PCIe controllers, we can't easily remap the separate 2028 * metadata buffer from the block layer and thus require a 2029 * separate metadata buffer for block layer metadata/PI support. 2030 * We allow extended LBAs for the passthrough interface, though. 2031 */ 2032 if (id->flbas & NVME_NS_FLBAS_META_EXT) 2033 ns->features |= NVME_NS_EXT_LBAS; 2034 else 2035 ns->features |= NVME_NS_METADATA_SUPPORTED; 2036 } 2037 2038 return 0; 2039 } 2040 2041 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl, 2042 struct request_queue *q) 2043 { 2044 bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT; 2045 2046 if (ctrl->max_hw_sectors) { 2047 u32 max_segments = 2048 (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1; 2049 2050 max_segments = min_not_zero(max_segments, ctrl->max_segments); 2051 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors); 2052 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX)); 2053 } 2054 blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1); 2055 blk_queue_dma_alignment(q, 7); 2056 blk_queue_write_cache(q, vwc, vwc); 2057 } 2058 2059 static void nvme_update_disk_info(struct gendisk *disk, 2060 struct nvme_ns *ns, struct nvme_id_ns *id) 2061 { 2062 sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze)); 2063 unsigned short bs = 1 << ns->lba_shift; 2064 u32 atomic_bs, phys_bs, io_opt = 0; 2065 2066 /* 2067 * The block layer can't support LBA sizes larger than the page size 2068 * yet, so catch this early and don't allow block I/O. 2069 */ 2070 if (ns->lba_shift > PAGE_SHIFT) { 2071 capacity = 0; 2072 bs = (1 << 9); 2073 } 2074 2075 blk_integrity_unregister(disk); 2076 2077 atomic_bs = phys_bs = bs; 2078 nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt); 2079 if (id->nabo == 0) { 2080 /* 2081 * Bit 1 indicates whether NAWUPF is defined for this namespace 2082 * and whether it should be used instead of AWUPF. If NAWUPF == 2083 * 0 then AWUPF must be used instead. 2084 */ 2085 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf) 2086 atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs; 2087 else 2088 atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs; 2089 } 2090 2091 if (id->nsfeat & NVME_NS_FEAT_IO_OPT) { 2092 /* NPWG = Namespace Preferred Write Granularity */ 2093 phys_bs = bs * (1 + le16_to_cpu(id->npwg)); 2094 /* NOWS = Namespace Optimal Write Size */ 2095 io_opt = bs * (1 + le16_to_cpu(id->nows)); 2096 } 2097 2098 blk_queue_logical_block_size(disk->queue, bs); 2099 /* 2100 * Linux filesystems assume writing a single physical block is 2101 * an atomic operation. Hence limit the physical block size to the 2102 * value of the Atomic Write Unit Power Fail parameter. 2103 */ 2104 blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs)); 2105 blk_queue_io_min(disk->queue, phys_bs); 2106 blk_queue_io_opt(disk->queue, io_opt); 2107 2108 /* 2109 * Register a metadata profile for PI, or the plain non-integrity NVMe 2110 * metadata masquerading as Type 0 if supported, otherwise reject block 2111 * I/O to namespaces with metadata except when the namespace supports 2112 * PI, as it can strip/insert in that case. 2113 */ 2114 if (ns->ms) { 2115 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && 2116 (ns->features & NVME_NS_METADATA_SUPPORTED)) 2117 nvme_init_integrity(disk, ns->ms, ns->pi_type, 2118 ns->ctrl->max_integrity_segments); 2119 else if (!nvme_ns_has_pi(ns)) 2120 capacity = 0; 2121 } 2122 2123 set_capacity_and_notify(disk, capacity); 2124 2125 nvme_config_discard(disk, ns); 2126 nvme_config_write_zeroes(disk, ns); 2127 2128 if ((id->nsattr & NVME_NS_ATTR_RO) || 2129 test_bit(NVME_NS_FORCE_RO, &ns->flags)) 2130 set_disk_ro(disk, true); 2131 } 2132 2133 static inline bool nvme_first_scan(struct gendisk *disk) 2134 { 2135 /* nvme_alloc_ns() scans the disk prior to adding it */ 2136 return !(disk->flags & GENHD_FL_UP); 2137 } 2138 2139 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id) 2140 { 2141 struct nvme_ctrl *ctrl = ns->ctrl; 2142 u32 iob; 2143 2144 if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && 2145 is_power_of_2(ctrl->max_hw_sectors)) 2146 iob = ctrl->max_hw_sectors; 2147 else 2148 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob)); 2149 2150 if (!iob) 2151 return; 2152 2153 if (!is_power_of_2(iob)) { 2154 if (nvme_first_scan(ns->disk)) 2155 pr_warn("%s: ignoring unaligned IO boundary:%u\n", 2156 ns->disk->disk_name, iob); 2157 return; 2158 } 2159 2160 if (blk_queue_is_zoned(ns->disk->queue)) { 2161 if (nvme_first_scan(ns->disk)) 2162 pr_warn("%s: ignoring zoned namespace IO boundary\n", 2163 ns->disk->disk_name); 2164 return; 2165 } 2166 2167 blk_queue_chunk_sectors(ns->queue, iob); 2168 } 2169 2170 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id) 2171 { 2172 unsigned lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK; 2173 int ret; 2174 2175 blk_mq_freeze_queue(ns->disk->queue); 2176 ns->lba_shift = id->lbaf[lbaf].ds; 2177 nvme_set_queue_limits(ns->ctrl, ns->queue); 2178 2179 if (ns->head->ids.csi == NVME_CSI_ZNS) { 2180 ret = nvme_update_zone_info(ns, lbaf); 2181 if (ret) 2182 goto out_unfreeze; 2183 } 2184 2185 ret = nvme_configure_metadata(ns, id); 2186 if (ret) 2187 goto out_unfreeze; 2188 nvme_set_chunk_sectors(ns, id); 2189 nvme_update_disk_info(ns->disk, ns, id); 2190 blk_mq_unfreeze_queue(ns->disk->queue); 2191 2192 if (blk_queue_is_zoned(ns->queue)) { 2193 ret = nvme_revalidate_zones(ns); 2194 if (ret && !nvme_first_scan(ns->disk)) 2195 return ret; 2196 } 2197 2198 #ifdef CONFIG_NVME_MULTIPATH 2199 if (ns->head->disk) { 2200 blk_mq_freeze_queue(ns->head->disk->queue); 2201 nvme_update_disk_info(ns->head->disk, ns, id); 2202 blk_stack_limits(&ns->head->disk->queue->limits, 2203 &ns->queue->limits, 0); 2204 blk_queue_update_readahead(ns->head->disk->queue); 2205 blk_mq_unfreeze_queue(ns->head->disk->queue); 2206 } 2207 #endif 2208 return 0; 2209 2210 out_unfreeze: 2211 blk_mq_unfreeze_queue(ns->disk->queue); 2212 return ret; 2213 } 2214 2215 static char nvme_pr_type(enum pr_type type) 2216 { 2217 switch (type) { 2218 case PR_WRITE_EXCLUSIVE: 2219 return 1; 2220 case PR_EXCLUSIVE_ACCESS: 2221 return 2; 2222 case PR_WRITE_EXCLUSIVE_REG_ONLY: 2223 return 3; 2224 case PR_EXCLUSIVE_ACCESS_REG_ONLY: 2225 return 4; 2226 case PR_WRITE_EXCLUSIVE_ALL_REGS: 2227 return 5; 2228 case PR_EXCLUSIVE_ACCESS_ALL_REGS: 2229 return 6; 2230 default: 2231 return 0; 2232 } 2233 }; 2234 2235 static int nvme_pr_command(struct block_device *bdev, u32 cdw10, 2236 u64 key, u64 sa_key, u8 op) 2237 { 2238 struct nvme_ns_head *head = NULL; 2239 struct nvme_ns *ns; 2240 struct nvme_command c; 2241 int srcu_idx, ret; 2242 u8 data[16] = { 0, }; 2243 2244 ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx); 2245 if (unlikely(!ns)) 2246 return -EWOULDBLOCK; 2247 2248 put_unaligned_le64(key, &data[0]); 2249 put_unaligned_le64(sa_key, &data[8]); 2250 2251 memset(&c, 0, sizeof(c)); 2252 c.common.opcode = op; 2253 c.common.nsid = cpu_to_le32(ns->head->ns_id); 2254 c.common.cdw10 = cpu_to_le32(cdw10); 2255 2256 ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16); 2257 nvme_put_ns_from_disk(head, srcu_idx); 2258 return ret; 2259 } 2260 2261 static int nvme_pr_register(struct block_device *bdev, u64 old, 2262 u64 new, unsigned flags) 2263 { 2264 u32 cdw10; 2265 2266 if (flags & ~PR_FL_IGNORE_KEY) 2267 return -EOPNOTSUPP; 2268 2269 cdw10 = old ? 2 : 0; 2270 cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0; 2271 cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */ 2272 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register); 2273 } 2274 2275 static int nvme_pr_reserve(struct block_device *bdev, u64 key, 2276 enum pr_type type, unsigned flags) 2277 { 2278 u32 cdw10; 2279 2280 if (flags & ~PR_FL_IGNORE_KEY) 2281 return -EOPNOTSUPP; 2282 2283 cdw10 = nvme_pr_type(type) << 8; 2284 cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0); 2285 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire); 2286 } 2287 2288 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new, 2289 enum pr_type type, bool abort) 2290 { 2291 u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1); 2292 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire); 2293 } 2294 2295 static int nvme_pr_clear(struct block_device *bdev, u64 key) 2296 { 2297 u32 cdw10 = 1 | (key ? 1 << 3 : 0); 2298 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register); 2299 } 2300 2301 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 2302 { 2303 u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0); 2304 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release); 2305 } 2306 2307 static const struct pr_ops nvme_pr_ops = { 2308 .pr_register = nvme_pr_register, 2309 .pr_reserve = nvme_pr_reserve, 2310 .pr_release = nvme_pr_release, 2311 .pr_preempt = nvme_pr_preempt, 2312 .pr_clear = nvme_pr_clear, 2313 }; 2314 2315 #ifdef CONFIG_BLK_SED_OPAL 2316 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len, 2317 bool send) 2318 { 2319 struct nvme_ctrl *ctrl = data; 2320 struct nvme_command cmd; 2321 2322 memset(&cmd, 0, sizeof(cmd)); 2323 if (send) 2324 cmd.common.opcode = nvme_admin_security_send; 2325 else 2326 cmd.common.opcode = nvme_admin_security_recv; 2327 cmd.common.nsid = 0; 2328 cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8); 2329 cmd.common.cdw11 = cpu_to_le32(len); 2330 2331 return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len, 0, 2332 NVME_QID_ANY, 1, 0, false); 2333 } 2334 EXPORT_SYMBOL_GPL(nvme_sec_submit); 2335 #endif /* CONFIG_BLK_SED_OPAL */ 2336 2337 static const struct block_device_operations nvme_bdev_ops = { 2338 .owner = THIS_MODULE, 2339 .ioctl = nvme_ioctl, 2340 .compat_ioctl = nvme_compat_ioctl, 2341 .open = nvme_open, 2342 .release = nvme_release, 2343 .getgeo = nvme_getgeo, 2344 .report_zones = nvme_report_zones, 2345 .pr_ops = &nvme_pr_ops, 2346 }; 2347 2348 #ifdef CONFIG_NVME_MULTIPATH 2349 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode) 2350 { 2351 struct nvme_ns_head *head = bdev->bd_disk->private_data; 2352 2353 if (!kref_get_unless_zero(&head->ref)) 2354 return -ENXIO; 2355 return 0; 2356 } 2357 2358 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode) 2359 { 2360 nvme_put_ns_head(disk->private_data); 2361 } 2362 2363 const struct block_device_operations nvme_ns_head_ops = { 2364 .owner = THIS_MODULE, 2365 .submit_bio = nvme_ns_head_submit_bio, 2366 .open = nvme_ns_head_open, 2367 .release = nvme_ns_head_release, 2368 .ioctl = nvme_ioctl, 2369 .compat_ioctl = nvme_compat_ioctl, 2370 .getgeo = nvme_getgeo, 2371 .report_zones = nvme_report_zones, 2372 .pr_ops = &nvme_pr_ops, 2373 }; 2374 #endif /* CONFIG_NVME_MULTIPATH */ 2375 2376 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled) 2377 { 2378 unsigned long timeout = 2379 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; 2380 u32 csts, bit = enabled ? NVME_CSTS_RDY : 0; 2381 int ret; 2382 2383 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) { 2384 if (csts == ~0) 2385 return -ENODEV; 2386 if ((csts & NVME_CSTS_RDY) == bit) 2387 break; 2388 2389 usleep_range(1000, 2000); 2390 if (fatal_signal_pending(current)) 2391 return -EINTR; 2392 if (time_after(jiffies, timeout)) { 2393 dev_err(ctrl->device, 2394 "Device not ready; aborting %s, CSTS=0x%x\n", 2395 enabled ? "initialisation" : "reset", csts); 2396 return -ENODEV; 2397 } 2398 } 2399 2400 return ret; 2401 } 2402 2403 /* 2404 * If the device has been passed off to us in an enabled state, just clear 2405 * the enabled bit. The spec says we should set the 'shutdown notification 2406 * bits', but doing so may cause the device to complete commands to the 2407 * admin queue ... and we don't know what memory that might be pointing at! 2408 */ 2409 int nvme_disable_ctrl(struct nvme_ctrl *ctrl) 2410 { 2411 int ret; 2412 2413 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK; 2414 ctrl->ctrl_config &= ~NVME_CC_ENABLE; 2415 2416 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config); 2417 if (ret) 2418 return ret; 2419 2420 if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) 2421 msleep(NVME_QUIRK_DELAY_AMOUNT); 2422 2423 return nvme_wait_ready(ctrl, ctrl->cap, false); 2424 } 2425 EXPORT_SYMBOL_GPL(nvme_disable_ctrl); 2426 2427 int nvme_enable_ctrl(struct nvme_ctrl *ctrl) 2428 { 2429 unsigned dev_page_min; 2430 int ret; 2431 2432 ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap); 2433 if (ret) { 2434 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret); 2435 return ret; 2436 } 2437 dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12; 2438 2439 if (NVME_CTRL_PAGE_SHIFT < dev_page_min) { 2440 dev_err(ctrl->device, 2441 "Minimum device page size %u too large for host (%u)\n", 2442 1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT); 2443 return -ENODEV; 2444 } 2445 2446 if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI) 2447 ctrl->ctrl_config = NVME_CC_CSS_CSI; 2448 else 2449 ctrl->ctrl_config = NVME_CC_CSS_NVM; 2450 ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; 2451 ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE; 2452 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 2453 ctrl->ctrl_config |= NVME_CC_ENABLE; 2454 2455 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config); 2456 if (ret) 2457 return ret; 2458 return nvme_wait_ready(ctrl, ctrl->cap, true); 2459 } 2460 EXPORT_SYMBOL_GPL(nvme_enable_ctrl); 2461 2462 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl) 2463 { 2464 unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ); 2465 u32 csts; 2466 int ret; 2467 2468 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK; 2469 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL; 2470 2471 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config); 2472 if (ret) 2473 return ret; 2474 2475 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) { 2476 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT) 2477 break; 2478 2479 msleep(100); 2480 if (fatal_signal_pending(current)) 2481 return -EINTR; 2482 if (time_after(jiffies, timeout)) { 2483 dev_err(ctrl->device, 2484 "Device shutdown incomplete; abort shutdown\n"); 2485 return -ENODEV; 2486 } 2487 } 2488 2489 return ret; 2490 } 2491 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl); 2492 2493 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl) 2494 { 2495 __le64 ts; 2496 int ret; 2497 2498 if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP)) 2499 return 0; 2500 2501 ts = cpu_to_le64(ktime_to_ms(ktime_get_real())); 2502 ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts), 2503 NULL); 2504 if (ret) 2505 dev_warn_once(ctrl->device, 2506 "could not set timestamp (%d)\n", ret); 2507 return ret; 2508 } 2509 2510 static int nvme_configure_acre(struct nvme_ctrl *ctrl) 2511 { 2512 struct nvme_feat_host_behavior *host; 2513 int ret; 2514 2515 /* Don't bother enabling the feature if retry delay is not reported */ 2516 if (!ctrl->crdt[0]) 2517 return 0; 2518 2519 host = kzalloc(sizeof(*host), GFP_KERNEL); 2520 if (!host) 2521 return 0; 2522 2523 host->acre = NVME_ENABLE_ACRE; 2524 ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0, 2525 host, sizeof(*host), NULL); 2526 kfree(host); 2527 return ret; 2528 } 2529 2530 static int nvme_configure_apst(struct nvme_ctrl *ctrl) 2531 { 2532 /* 2533 * APST (Autonomous Power State Transition) lets us program a 2534 * table of power state transitions that the controller will 2535 * perform automatically. We configure it with a simple 2536 * heuristic: we are willing to spend at most 2% of the time 2537 * transitioning between power states. Therefore, when running 2538 * in any given state, we will enter the next lower-power 2539 * non-operational state after waiting 50 * (enlat + exlat) 2540 * microseconds, as long as that state's exit latency is under 2541 * the requested maximum latency. 2542 * 2543 * We will not autonomously enter any non-operational state for 2544 * which the total latency exceeds ps_max_latency_us. Users 2545 * can set ps_max_latency_us to zero to turn off APST. 2546 */ 2547 2548 unsigned apste; 2549 struct nvme_feat_auto_pst *table; 2550 u64 max_lat_us = 0; 2551 int max_ps = -1; 2552 int ret; 2553 2554 /* 2555 * If APST isn't supported or if we haven't been initialized yet, 2556 * then don't do anything. 2557 */ 2558 if (!ctrl->apsta) 2559 return 0; 2560 2561 if (ctrl->npss > 31) { 2562 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n"); 2563 return 0; 2564 } 2565 2566 table = kzalloc(sizeof(*table), GFP_KERNEL); 2567 if (!table) 2568 return 0; 2569 2570 if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) { 2571 /* Turn off APST. */ 2572 apste = 0; 2573 dev_dbg(ctrl->device, "APST disabled\n"); 2574 } else { 2575 __le64 target = cpu_to_le64(0); 2576 int state; 2577 2578 /* 2579 * Walk through all states from lowest- to highest-power. 2580 * According to the spec, lower-numbered states use more 2581 * power. NPSS, despite the name, is the index of the 2582 * lowest-power state, not the number of states. 2583 */ 2584 for (state = (int)ctrl->npss; state >= 0; state--) { 2585 u64 total_latency_us, exit_latency_us, transition_ms; 2586 2587 if (target) 2588 table->entries[state] = target; 2589 2590 /* 2591 * Don't allow transitions to the deepest state 2592 * if it's quirked off. 2593 */ 2594 if (state == ctrl->npss && 2595 (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) 2596 continue; 2597 2598 /* 2599 * Is this state a useful non-operational state for 2600 * higher-power states to autonomously transition to? 2601 */ 2602 if (!(ctrl->psd[state].flags & 2603 NVME_PS_FLAGS_NON_OP_STATE)) 2604 continue; 2605 2606 exit_latency_us = 2607 (u64)le32_to_cpu(ctrl->psd[state].exit_lat); 2608 if (exit_latency_us > ctrl->ps_max_latency_us) 2609 continue; 2610 2611 total_latency_us = 2612 exit_latency_us + 2613 le32_to_cpu(ctrl->psd[state].entry_lat); 2614 2615 /* 2616 * This state is good. Use it as the APST idle 2617 * target for higher power states. 2618 */ 2619 transition_ms = total_latency_us + 19; 2620 do_div(transition_ms, 20); 2621 if (transition_ms > (1 << 24) - 1) 2622 transition_ms = (1 << 24) - 1; 2623 2624 target = cpu_to_le64((state << 3) | 2625 (transition_ms << 8)); 2626 2627 if (max_ps == -1) 2628 max_ps = state; 2629 2630 if (total_latency_us > max_lat_us) 2631 max_lat_us = total_latency_us; 2632 } 2633 2634 apste = 1; 2635 2636 if (max_ps == -1) { 2637 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n"); 2638 } else { 2639 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n", 2640 max_ps, max_lat_us, (int)sizeof(*table), table); 2641 } 2642 } 2643 2644 ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste, 2645 table, sizeof(*table), NULL); 2646 if (ret) 2647 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret); 2648 2649 kfree(table); 2650 return ret; 2651 } 2652 2653 static void nvme_set_latency_tolerance(struct device *dev, s32 val) 2654 { 2655 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 2656 u64 latency; 2657 2658 switch (val) { 2659 case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT: 2660 case PM_QOS_LATENCY_ANY: 2661 latency = U64_MAX; 2662 break; 2663 2664 default: 2665 latency = val; 2666 } 2667 2668 if (ctrl->ps_max_latency_us != latency) { 2669 ctrl->ps_max_latency_us = latency; 2670 nvme_configure_apst(ctrl); 2671 } 2672 } 2673 2674 struct nvme_core_quirk_entry { 2675 /* 2676 * NVMe model and firmware strings are padded with spaces. For 2677 * simplicity, strings in the quirk table are padded with NULLs 2678 * instead. 2679 */ 2680 u16 vid; 2681 const char *mn; 2682 const char *fr; 2683 unsigned long quirks; 2684 }; 2685 2686 static const struct nvme_core_quirk_entry core_quirks[] = { 2687 { 2688 /* 2689 * This Toshiba device seems to die using any APST states. See: 2690 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11 2691 */ 2692 .vid = 0x1179, 2693 .mn = "THNSF5256GPUK TOSHIBA", 2694 .quirks = NVME_QUIRK_NO_APST, 2695 }, 2696 { 2697 /* 2698 * This LiteON CL1-3D*-Q11 firmware version has a race 2699 * condition associated with actions related to suspend to idle 2700 * LiteON has resolved the problem in future firmware 2701 */ 2702 .vid = 0x14a4, 2703 .fr = "22301111", 2704 .quirks = NVME_QUIRK_SIMPLE_SUSPEND, 2705 } 2706 }; 2707 2708 /* match is null-terminated but idstr is space-padded. */ 2709 static bool string_matches(const char *idstr, const char *match, size_t len) 2710 { 2711 size_t matchlen; 2712 2713 if (!match) 2714 return true; 2715 2716 matchlen = strlen(match); 2717 WARN_ON_ONCE(matchlen > len); 2718 2719 if (memcmp(idstr, match, matchlen)) 2720 return false; 2721 2722 for (; matchlen < len; matchlen++) 2723 if (idstr[matchlen] != ' ') 2724 return false; 2725 2726 return true; 2727 } 2728 2729 static bool quirk_matches(const struct nvme_id_ctrl *id, 2730 const struct nvme_core_quirk_entry *q) 2731 { 2732 return q->vid == le16_to_cpu(id->vid) && 2733 string_matches(id->mn, q->mn, sizeof(id->mn)) && 2734 string_matches(id->fr, q->fr, sizeof(id->fr)); 2735 } 2736 2737 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl, 2738 struct nvme_id_ctrl *id) 2739 { 2740 size_t nqnlen; 2741 int off; 2742 2743 if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) { 2744 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE); 2745 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) { 2746 strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE); 2747 return; 2748 } 2749 2750 if (ctrl->vs >= NVME_VS(1, 2, 1)) 2751 dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n"); 2752 } 2753 2754 /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */ 2755 off = snprintf(subsys->subnqn, NVMF_NQN_SIZE, 2756 "nqn.2014.08.org.nvmexpress:%04x%04x", 2757 le16_to_cpu(id->vid), le16_to_cpu(id->ssvid)); 2758 memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn)); 2759 off += sizeof(id->sn); 2760 memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn)); 2761 off += sizeof(id->mn); 2762 memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off); 2763 } 2764 2765 static void nvme_release_subsystem(struct device *dev) 2766 { 2767 struct nvme_subsystem *subsys = 2768 container_of(dev, struct nvme_subsystem, dev); 2769 2770 if (subsys->instance >= 0) 2771 ida_simple_remove(&nvme_instance_ida, subsys->instance); 2772 kfree(subsys); 2773 } 2774 2775 static void nvme_destroy_subsystem(struct kref *ref) 2776 { 2777 struct nvme_subsystem *subsys = 2778 container_of(ref, struct nvme_subsystem, ref); 2779 2780 mutex_lock(&nvme_subsystems_lock); 2781 list_del(&subsys->entry); 2782 mutex_unlock(&nvme_subsystems_lock); 2783 2784 ida_destroy(&subsys->ns_ida); 2785 device_del(&subsys->dev); 2786 put_device(&subsys->dev); 2787 } 2788 2789 static void nvme_put_subsystem(struct nvme_subsystem *subsys) 2790 { 2791 kref_put(&subsys->ref, nvme_destroy_subsystem); 2792 } 2793 2794 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn) 2795 { 2796 struct nvme_subsystem *subsys; 2797 2798 lockdep_assert_held(&nvme_subsystems_lock); 2799 2800 /* 2801 * Fail matches for discovery subsystems. This results 2802 * in each discovery controller bound to a unique subsystem. 2803 * This avoids issues with validating controller values 2804 * that can only be true when there is a single unique subsystem. 2805 * There may be multiple and completely independent entities 2806 * that provide discovery controllers. 2807 */ 2808 if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME)) 2809 return NULL; 2810 2811 list_for_each_entry(subsys, &nvme_subsystems, entry) { 2812 if (strcmp(subsys->subnqn, subsysnqn)) 2813 continue; 2814 if (!kref_get_unless_zero(&subsys->ref)) 2815 continue; 2816 return subsys; 2817 } 2818 2819 return NULL; 2820 } 2821 2822 #define SUBSYS_ATTR_RO(_name, _mode, _show) \ 2823 struct device_attribute subsys_attr_##_name = \ 2824 __ATTR(_name, _mode, _show, NULL) 2825 2826 static ssize_t nvme_subsys_show_nqn(struct device *dev, 2827 struct device_attribute *attr, 2828 char *buf) 2829 { 2830 struct nvme_subsystem *subsys = 2831 container_of(dev, struct nvme_subsystem, dev); 2832 2833 return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn); 2834 } 2835 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn); 2836 2837 #define nvme_subsys_show_str_function(field) \ 2838 static ssize_t subsys_##field##_show(struct device *dev, \ 2839 struct device_attribute *attr, char *buf) \ 2840 { \ 2841 struct nvme_subsystem *subsys = \ 2842 container_of(dev, struct nvme_subsystem, dev); \ 2843 return sprintf(buf, "%.*s\n", \ 2844 (int)sizeof(subsys->field), subsys->field); \ 2845 } \ 2846 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show); 2847 2848 nvme_subsys_show_str_function(model); 2849 nvme_subsys_show_str_function(serial); 2850 nvme_subsys_show_str_function(firmware_rev); 2851 2852 static struct attribute *nvme_subsys_attrs[] = { 2853 &subsys_attr_model.attr, 2854 &subsys_attr_serial.attr, 2855 &subsys_attr_firmware_rev.attr, 2856 &subsys_attr_subsysnqn.attr, 2857 #ifdef CONFIG_NVME_MULTIPATH 2858 &subsys_attr_iopolicy.attr, 2859 #endif 2860 NULL, 2861 }; 2862 2863 static struct attribute_group nvme_subsys_attrs_group = { 2864 .attrs = nvme_subsys_attrs, 2865 }; 2866 2867 static const struct attribute_group *nvme_subsys_attrs_groups[] = { 2868 &nvme_subsys_attrs_group, 2869 NULL, 2870 }; 2871 2872 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl) 2873 { 2874 return ctrl->opts && ctrl->opts->discovery_nqn; 2875 } 2876 2877 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys, 2878 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 2879 { 2880 struct nvme_ctrl *tmp; 2881 2882 lockdep_assert_held(&nvme_subsystems_lock); 2883 2884 list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) { 2885 if (nvme_state_terminal(tmp)) 2886 continue; 2887 2888 if (tmp->cntlid == ctrl->cntlid) { 2889 dev_err(ctrl->device, 2890 "Duplicate cntlid %u with %s, rejecting\n", 2891 ctrl->cntlid, dev_name(tmp->device)); 2892 return false; 2893 } 2894 2895 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || 2896 nvme_discovery_ctrl(ctrl)) 2897 continue; 2898 2899 dev_err(ctrl->device, 2900 "Subsystem does not support multiple controllers\n"); 2901 return false; 2902 } 2903 2904 return true; 2905 } 2906 2907 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 2908 { 2909 struct nvme_subsystem *subsys, *found; 2910 int ret; 2911 2912 subsys = kzalloc(sizeof(*subsys), GFP_KERNEL); 2913 if (!subsys) 2914 return -ENOMEM; 2915 2916 subsys->instance = -1; 2917 mutex_init(&subsys->lock); 2918 kref_init(&subsys->ref); 2919 INIT_LIST_HEAD(&subsys->ctrls); 2920 INIT_LIST_HEAD(&subsys->nsheads); 2921 nvme_init_subnqn(subsys, ctrl, id); 2922 memcpy(subsys->serial, id->sn, sizeof(subsys->serial)); 2923 memcpy(subsys->model, id->mn, sizeof(subsys->model)); 2924 memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev)); 2925 subsys->vendor_id = le16_to_cpu(id->vid); 2926 subsys->cmic = id->cmic; 2927 subsys->awupf = le16_to_cpu(id->awupf); 2928 #ifdef CONFIG_NVME_MULTIPATH 2929 subsys->iopolicy = NVME_IOPOLICY_NUMA; 2930 #endif 2931 2932 subsys->dev.class = nvme_subsys_class; 2933 subsys->dev.release = nvme_release_subsystem; 2934 subsys->dev.groups = nvme_subsys_attrs_groups; 2935 dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance); 2936 device_initialize(&subsys->dev); 2937 2938 mutex_lock(&nvme_subsystems_lock); 2939 found = __nvme_find_get_subsystem(subsys->subnqn); 2940 if (found) { 2941 put_device(&subsys->dev); 2942 subsys = found; 2943 2944 if (!nvme_validate_cntlid(subsys, ctrl, id)) { 2945 ret = -EINVAL; 2946 goto out_put_subsystem; 2947 } 2948 } else { 2949 ret = device_add(&subsys->dev); 2950 if (ret) { 2951 dev_err(ctrl->device, 2952 "failed to register subsystem device.\n"); 2953 put_device(&subsys->dev); 2954 goto out_unlock; 2955 } 2956 ida_init(&subsys->ns_ida); 2957 list_add_tail(&subsys->entry, &nvme_subsystems); 2958 } 2959 2960 ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj, 2961 dev_name(ctrl->device)); 2962 if (ret) { 2963 dev_err(ctrl->device, 2964 "failed to create sysfs link from subsystem.\n"); 2965 goto out_put_subsystem; 2966 } 2967 2968 if (!found) 2969 subsys->instance = ctrl->instance; 2970 ctrl->subsys = subsys; 2971 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls); 2972 mutex_unlock(&nvme_subsystems_lock); 2973 return 0; 2974 2975 out_put_subsystem: 2976 nvme_put_subsystem(subsys); 2977 out_unlock: 2978 mutex_unlock(&nvme_subsystems_lock); 2979 return ret; 2980 } 2981 2982 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, 2983 void *log, size_t size, u64 offset) 2984 { 2985 struct nvme_command c = { }; 2986 u32 dwlen = nvme_bytes_to_numd(size); 2987 2988 c.get_log_page.opcode = nvme_admin_get_log_page; 2989 c.get_log_page.nsid = cpu_to_le32(nsid); 2990 c.get_log_page.lid = log_page; 2991 c.get_log_page.lsp = lsp; 2992 c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1)); 2993 c.get_log_page.numdu = cpu_to_le16(dwlen >> 16); 2994 c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset)); 2995 c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset)); 2996 c.get_log_page.csi = csi; 2997 2998 return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size); 2999 } 3000 3001 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi, 3002 struct nvme_effects_log **log) 3003 { 3004 struct nvme_effects_log *cel = xa_load(&ctrl->cels, csi); 3005 int ret; 3006 3007 if (cel) 3008 goto out; 3009 3010 cel = kzalloc(sizeof(*cel), GFP_KERNEL); 3011 if (!cel) 3012 return -ENOMEM; 3013 3014 ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi, 3015 cel, sizeof(*cel), 0); 3016 if (ret) { 3017 kfree(cel); 3018 return ret; 3019 } 3020 3021 xa_store(&ctrl->cels, csi, cel, GFP_KERNEL); 3022 out: 3023 *log = cel; 3024 return 0; 3025 } 3026 3027 /* 3028 * Initialize the cached copies of the Identify data and various controller 3029 * register in our nvme_ctrl structure. This should be called as soon as 3030 * the admin queue is fully up and running. 3031 */ 3032 int nvme_init_identify(struct nvme_ctrl *ctrl) 3033 { 3034 struct nvme_id_ctrl *id; 3035 int ret, page_shift; 3036 u32 max_hw_sectors; 3037 bool prev_apst_enabled; 3038 3039 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs); 3040 if (ret) { 3041 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret); 3042 return ret; 3043 } 3044 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12; 3045 ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize); 3046 3047 if (ctrl->vs >= NVME_VS(1, 1, 0)) 3048 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap); 3049 3050 ret = nvme_identify_ctrl(ctrl, &id); 3051 if (ret) { 3052 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret); 3053 return -EIO; 3054 } 3055 3056 if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) { 3057 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects); 3058 if (ret < 0) 3059 goto out_free; 3060 } 3061 3062 if (!(ctrl->ops->flags & NVME_F_FABRICS)) 3063 ctrl->cntlid = le16_to_cpu(id->cntlid); 3064 3065 if (!ctrl->identified) { 3066 int i; 3067 3068 ret = nvme_init_subsystem(ctrl, id); 3069 if (ret) 3070 goto out_free; 3071 3072 /* 3073 * Check for quirks. Quirk can depend on firmware version, 3074 * so, in principle, the set of quirks present can change 3075 * across a reset. As a possible future enhancement, we 3076 * could re-scan for quirks every time we reinitialize 3077 * the device, but we'd have to make sure that the driver 3078 * behaves intelligently if the quirks change. 3079 */ 3080 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) { 3081 if (quirk_matches(id, &core_quirks[i])) 3082 ctrl->quirks |= core_quirks[i].quirks; 3083 } 3084 } 3085 3086 if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) { 3087 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n"); 3088 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS; 3089 } 3090 3091 ctrl->crdt[0] = le16_to_cpu(id->crdt1); 3092 ctrl->crdt[1] = le16_to_cpu(id->crdt2); 3093 ctrl->crdt[2] = le16_to_cpu(id->crdt3); 3094 3095 ctrl->oacs = le16_to_cpu(id->oacs); 3096 ctrl->oncs = le16_to_cpu(id->oncs); 3097 ctrl->mtfa = le16_to_cpu(id->mtfa); 3098 ctrl->oaes = le32_to_cpu(id->oaes); 3099 ctrl->wctemp = le16_to_cpu(id->wctemp); 3100 ctrl->cctemp = le16_to_cpu(id->cctemp); 3101 3102 atomic_set(&ctrl->abort_limit, id->acl + 1); 3103 ctrl->vwc = id->vwc; 3104 if (id->mdts) 3105 max_hw_sectors = 1 << (id->mdts + page_shift - 9); 3106 else 3107 max_hw_sectors = UINT_MAX; 3108 ctrl->max_hw_sectors = 3109 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors); 3110 3111 nvme_set_queue_limits(ctrl, ctrl->admin_q); 3112 ctrl->sgls = le32_to_cpu(id->sgls); 3113 ctrl->kas = le16_to_cpu(id->kas); 3114 ctrl->max_namespaces = le32_to_cpu(id->mnan); 3115 ctrl->ctratt = le32_to_cpu(id->ctratt); 3116 3117 if (id->rtd3e) { 3118 /* us -> s */ 3119 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC; 3120 3121 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time, 3122 shutdown_timeout, 60); 3123 3124 if (ctrl->shutdown_timeout != shutdown_timeout) 3125 dev_info(ctrl->device, 3126 "Shutdown timeout set to %u seconds\n", 3127 ctrl->shutdown_timeout); 3128 } else 3129 ctrl->shutdown_timeout = shutdown_timeout; 3130 3131 ctrl->npss = id->npss; 3132 ctrl->apsta = id->apsta; 3133 prev_apst_enabled = ctrl->apst_enabled; 3134 if (ctrl->quirks & NVME_QUIRK_NO_APST) { 3135 if (force_apst && id->apsta) { 3136 dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n"); 3137 ctrl->apst_enabled = true; 3138 } else { 3139 ctrl->apst_enabled = false; 3140 } 3141 } else { 3142 ctrl->apst_enabled = id->apsta; 3143 } 3144 memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd)); 3145 3146 if (ctrl->ops->flags & NVME_F_FABRICS) { 3147 ctrl->icdoff = le16_to_cpu(id->icdoff); 3148 ctrl->ioccsz = le32_to_cpu(id->ioccsz); 3149 ctrl->iorcsz = le32_to_cpu(id->iorcsz); 3150 ctrl->maxcmd = le16_to_cpu(id->maxcmd); 3151 3152 /* 3153 * In fabrics we need to verify the cntlid matches the 3154 * admin connect 3155 */ 3156 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) { 3157 dev_err(ctrl->device, 3158 "Mismatching cntlid: Connect %u vs Identify " 3159 "%u, rejecting\n", 3160 ctrl->cntlid, le16_to_cpu(id->cntlid)); 3161 ret = -EINVAL; 3162 goto out_free; 3163 } 3164 3165 if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) { 3166 dev_err(ctrl->device, 3167 "keep-alive support is mandatory for fabrics\n"); 3168 ret = -EINVAL; 3169 goto out_free; 3170 } 3171 } else { 3172 ctrl->hmpre = le32_to_cpu(id->hmpre); 3173 ctrl->hmmin = le32_to_cpu(id->hmmin); 3174 ctrl->hmminds = le32_to_cpu(id->hmminds); 3175 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd); 3176 } 3177 3178 ret = nvme_mpath_init(ctrl, id); 3179 kfree(id); 3180 3181 if (ret < 0) 3182 return ret; 3183 3184 if (ctrl->apst_enabled && !prev_apst_enabled) 3185 dev_pm_qos_expose_latency_tolerance(ctrl->device); 3186 else if (!ctrl->apst_enabled && prev_apst_enabled) 3187 dev_pm_qos_hide_latency_tolerance(ctrl->device); 3188 3189 ret = nvme_configure_apst(ctrl); 3190 if (ret < 0) 3191 return ret; 3192 3193 ret = nvme_configure_timestamp(ctrl); 3194 if (ret < 0) 3195 return ret; 3196 3197 ret = nvme_configure_directives(ctrl); 3198 if (ret < 0) 3199 return ret; 3200 3201 ret = nvme_configure_acre(ctrl); 3202 if (ret < 0) 3203 return ret; 3204 3205 if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) { 3206 ret = nvme_hwmon_init(ctrl); 3207 if (ret < 0) 3208 return ret; 3209 } 3210 3211 ctrl->identified = true; 3212 3213 return 0; 3214 3215 out_free: 3216 kfree(id); 3217 return ret; 3218 } 3219 EXPORT_SYMBOL_GPL(nvme_init_identify); 3220 3221 static int nvme_dev_open(struct inode *inode, struct file *file) 3222 { 3223 struct nvme_ctrl *ctrl = 3224 container_of(inode->i_cdev, struct nvme_ctrl, cdev); 3225 3226 switch (ctrl->state) { 3227 case NVME_CTRL_LIVE: 3228 break; 3229 default: 3230 return -EWOULDBLOCK; 3231 } 3232 3233 nvme_get_ctrl(ctrl); 3234 if (!try_module_get(ctrl->ops->module)) { 3235 nvme_put_ctrl(ctrl); 3236 return -EINVAL; 3237 } 3238 3239 file->private_data = ctrl; 3240 return 0; 3241 } 3242 3243 static int nvme_dev_release(struct inode *inode, struct file *file) 3244 { 3245 struct nvme_ctrl *ctrl = 3246 container_of(inode->i_cdev, struct nvme_ctrl, cdev); 3247 3248 module_put(ctrl->ops->module); 3249 nvme_put_ctrl(ctrl); 3250 return 0; 3251 } 3252 3253 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp) 3254 { 3255 struct nvme_ns *ns; 3256 int ret; 3257 3258 down_read(&ctrl->namespaces_rwsem); 3259 if (list_empty(&ctrl->namespaces)) { 3260 ret = -ENOTTY; 3261 goto out_unlock; 3262 } 3263 3264 ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list); 3265 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) { 3266 dev_warn(ctrl->device, 3267 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n"); 3268 ret = -EINVAL; 3269 goto out_unlock; 3270 } 3271 3272 dev_warn(ctrl->device, 3273 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n"); 3274 kref_get(&ns->kref); 3275 up_read(&ctrl->namespaces_rwsem); 3276 3277 ret = nvme_user_cmd(ctrl, ns, argp); 3278 nvme_put_ns(ns); 3279 return ret; 3280 3281 out_unlock: 3282 up_read(&ctrl->namespaces_rwsem); 3283 return ret; 3284 } 3285 3286 static long nvme_dev_ioctl(struct file *file, unsigned int cmd, 3287 unsigned long arg) 3288 { 3289 struct nvme_ctrl *ctrl = file->private_data; 3290 void __user *argp = (void __user *)arg; 3291 3292 switch (cmd) { 3293 case NVME_IOCTL_ADMIN_CMD: 3294 return nvme_user_cmd(ctrl, NULL, argp); 3295 case NVME_IOCTL_ADMIN64_CMD: 3296 return nvme_user_cmd64(ctrl, NULL, argp); 3297 case NVME_IOCTL_IO_CMD: 3298 return nvme_dev_user_cmd(ctrl, argp); 3299 case NVME_IOCTL_RESET: 3300 dev_warn(ctrl->device, "resetting controller\n"); 3301 return nvme_reset_ctrl_sync(ctrl); 3302 case NVME_IOCTL_SUBSYS_RESET: 3303 return nvme_reset_subsystem(ctrl); 3304 case NVME_IOCTL_RESCAN: 3305 nvme_queue_scan(ctrl); 3306 return 0; 3307 default: 3308 return -ENOTTY; 3309 } 3310 } 3311 3312 static const struct file_operations nvme_dev_fops = { 3313 .owner = THIS_MODULE, 3314 .open = nvme_dev_open, 3315 .release = nvme_dev_release, 3316 .unlocked_ioctl = nvme_dev_ioctl, 3317 .compat_ioctl = compat_ptr_ioctl, 3318 }; 3319 3320 static ssize_t nvme_sysfs_reset(struct device *dev, 3321 struct device_attribute *attr, const char *buf, 3322 size_t count) 3323 { 3324 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3325 int ret; 3326 3327 ret = nvme_reset_ctrl_sync(ctrl); 3328 if (ret < 0) 3329 return ret; 3330 return count; 3331 } 3332 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset); 3333 3334 static ssize_t nvme_sysfs_rescan(struct device *dev, 3335 struct device_attribute *attr, const char *buf, 3336 size_t count) 3337 { 3338 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3339 3340 nvme_queue_scan(ctrl); 3341 return count; 3342 } 3343 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan); 3344 3345 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev) 3346 { 3347 struct gendisk *disk = dev_to_disk(dev); 3348 3349 if (disk->fops == &nvme_bdev_ops) 3350 return nvme_get_ns_from_dev(dev)->head; 3351 else 3352 return disk->private_data; 3353 } 3354 3355 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr, 3356 char *buf) 3357 { 3358 struct nvme_ns_head *head = dev_to_ns_head(dev); 3359 struct nvme_ns_ids *ids = &head->ids; 3360 struct nvme_subsystem *subsys = head->subsys; 3361 int serial_len = sizeof(subsys->serial); 3362 int model_len = sizeof(subsys->model); 3363 3364 if (!uuid_is_null(&ids->uuid)) 3365 return sprintf(buf, "uuid.%pU\n", &ids->uuid); 3366 3367 if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid))) 3368 return sprintf(buf, "eui.%16phN\n", ids->nguid); 3369 3370 if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64))) 3371 return sprintf(buf, "eui.%8phN\n", ids->eui64); 3372 3373 while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' || 3374 subsys->serial[serial_len - 1] == '\0')) 3375 serial_len--; 3376 while (model_len > 0 && (subsys->model[model_len - 1] == ' ' || 3377 subsys->model[model_len - 1] == '\0')) 3378 model_len--; 3379 3380 return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id, 3381 serial_len, subsys->serial, model_len, subsys->model, 3382 head->ns_id); 3383 } 3384 static DEVICE_ATTR_RO(wwid); 3385 3386 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr, 3387 char *buf) 3388 { 3389 return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid); 3390 } 3391 static DEVICE_ATTR_RO(nguid); 3392 3393 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, 3394 char *buf) 3395 { 3396 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids; 3397 3398 /* For backward compatibility expose the NGUID to userspace if 3399 * we have no UUID set 3400 */ 3401 if (uuid_is_null(&ids->uuid)) { 3402 printk_ratelimited(KERN_WARNING 3403 "No UUID available providing old NGUID\n"); 3404 return sprintf(buf, "%pU\n", ids->nguid); 3405 } 3406 return sprintf(buf, "%pU\n", &ids->uuid); 3407 } 3408 static DEVICE_ATTR_RO(uuid); 3409 3410 static ssize_t eui_show(struct device *dev, struct device_attribute *attr, 3411 char *buf) 3412 { 3413 return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64); 3414 } 3415 static DEVICE_ATTR_RO(eui); 3416 3417 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr, 3418 char *buf) 3419 { 3420 return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id); 3421 } 3422 static DEVICE_ATTR_RO(nsid); 3423 3424 static struct attribute *nvme_ns_id_attrs[] = { 3425 &dev_attr_wwid.attr, 3426 &dev_attr_uuid.attr, 3427 &dev_attr_nguid.attr, 3428 &dev_attr_eui.attr, 3429 &dev_attr_nsid.attr, 3430 #ifdef CONFIG_NVME_MULTIPATH 3431 &dev_attr_ana_grpid.attr, 3432 &dev_attr_ana_state.attr, 3433 #endif 3434 NULL, 3435 }; 3436 3437 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj, 3438 struct attribute *a, int n) 3439 { 3440 struct device *dev = container_of(kobj, struct device, kobj); 3441 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids; 3442 3443 if (a == &dev_attr_uuid.attr) { 3444 if (uuid_is_null(&ids->uuid) && 3445 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid))) 3446 return 0; 3447 } 3448 if (a == &dev_attr_nguid.attr) { 3449 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid))) 3450 return 0; 3451 } 3452 if (a == &dev_attr_eui.attr) { 3453 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64))) 3454 return 0; 3455 } 3456 #ifdef CONFIG_NVME_MULTIPATH 3457 if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) { 3458 if (dev_to_disk(dev)->fops != &nvme_bdev_ops) /* per-path attr */ 3459 return 0; 3460 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl)) 3461 return 0; 3462 } 3463 #endif 3464 return a->mode; 3465 } 3466 3467 static const struct attribute_group nvme_ns_id_attr_group = { 3468 .attrs = nvme_ns_id_attrs, 3469 .is_visible = nvme_ns_id_attrs_are_visible, 3470 }; 3471 3472 const struct attribute_group *nvme_ns_id_attr_groups[] = { 3473 &nvme_ns_id_attr_group, 3474 #ifdef CONFIG_NVM 3475 &nvme_nvm_attr_group, 3476 #endif 3477 NULL, 3478 }; 3479 3480 #define nvme_show_str_function(field) \ 3481 static ssize_t field##_show(struct device *dev, \ 3482 struct device_attribute *attr, char *buf) \ 3483 { \ 3484 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \ 3485 return sprintf(buf, "%.*s\n", \ 3486 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field); \ 3487 } \ 3488 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL); 3489 3490 nvme_show_str_function(model); 3491 nvme_show_str_function(serial); 3492 nvme_show_str_function(firmware_rev); 3493 3494 #define nvme_show_int_function(field) \ 3495 static ssize_t field##_show(struct device *dev, \ 3496 struct device_attribute *attr, char *buf) \ 3497 { \ 3498 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \ 3499 return sprintf(buf, "%d\n", ctrl->field); \ 3500 } \ 3501 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL); 3502 3503 nvme_show_int_function(cntlid); 3504 nvme_show_int_function(numa_node); 3505 nvme_show_int_function(queue_count); 3506 nvme_show_int_function(sqsize); 3507 3508 static ssize_t nvme_sysfs_delete(struct device *dev, 3509 struct device_attribute *attr, const char *buf, 3510 size_t count) 3511 { 3512 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3513 3514 if (device_remove_file_self(dev, attr)) 3515 nvme_delete_ctrl_sync(ctrl); 3516 return count; 3517 } 3518 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete); 3519 3520 static ssize_t nvme_sysfs_show_transport(struct device *dev, 3521 struct device_attribute *attr, 3522 char *buf) 3523 { 3524 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3525 3526 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name); 3527 } 3528 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL); 3529 3530 static ssize_t nvme_sysfs_show_state(struct device *dev, 3531 struct device_attribute *attr, 3532 char *buf) 3533 { 3534 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3535 static const char *const state_name[] = { 3536 [NVME_CTRL_NEW] = "new", 3537 [NVME_CTRL_LIVE] = "live", 3538 [NVME_CTRL_RESETTING] = "resetting", 3539 [NVME_CTRL_CONNECTING] = "connecting", 3540 [NVME_CTRL_DELETING] = "deleting", 3541 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)", 3542 [NVME_CTRL_DEAD] = "dead", 3543 }; 3544 3545 if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) && 3546 state_name[ctrl->state]) 3547 return sprintf(buf, "%s\n", state_name[ctrl->state]); 3548 3549 return sprintf(buf, "unknown state\n"); 3550 } 3551 3552 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL); 3553 3554 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev, 3555 struct device_attribute *attr, 3556 char *buf) 3557 { 3558 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3559 3560 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn); 3561 } 3562 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL); 3563 3564 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev, 3565 struct device_attribute *attr, 3566 char *buf) 3567 { 3568 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3569 3570 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->opts->host->nqn); 3571 } 3572 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL); 3573 3574 static ssize_t nvme_sysfs_show_hostid(struct device *dev, 3575 struct device_attribute *attr, 3576 char *buf) 3577 { 3578 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3579 3580 return snprintf(buf, PAGE_SIZE, "%pU\n", &ctrl->opts->host->id); 3581 } 3582 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL); 3583 3584 static ssize_t nvme_sysfs_show_address(struct device *dev, 3585 struct device_attribute *attr, 3586 char *buf) 3587 { 3588 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3589 3590 return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE); 3591 } 3592 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL); 3593 3594 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev, 3595 struct device_attribute *attr, char *buf) 3596 { 3597 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3598 struct nvmf_ctrl_options *opts = ctrl->opts; 3599 3600 if (ctrl->opts->max_reconnects == -1) 3601 return sprintf(buf, "off\n"); 3602 return sprintf(buf, "%d\n", 3603 opts->max_reconnects * opts->reconnect_delay); 3604 } 3605 3606 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev, 3607 struct device_attribute *attr, const char *buf, size_t count) 3608 { 3609 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3610 struct nvmf_ctrl_options *opts = ctrl->opts; 3611 int ctrl_loss_tmo, err; 3612 3613 err = kstrtoint(buf, 10, &ctrl_loss_tmo); 3614 if (err) 3615 return -EINVAL; 3616 3617 else if (ctrl_loss_tmo < 0) 3618 opts->max_reconnects = -1; 3619 else 3620 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo, 3621 opts->reconnect_delay); 3622 return count; 3623 } 3624 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR, 3625 nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store); 3626 3627 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev, 3628 struct device_attribute *attr, char *buf) 3629 { 3630 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3631 3632 if (ctrl->opts->reconnect_delay == -1) 3633 return sprintf(buf, "off\n"); 3634 return sprintf(buf, "%d\n", ctrl->opts->reconnect_delay); 3635 } 3636 3637 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev, 3638 struct device_attribute *attr, const char *buf, size_t count) 3639 { 3640 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3641 unsigned int v; 3642 int err; 3643 3644 err = kstrtou32(buf, 10, &v); 3645 if (err) 3646 return err; 3647 3648 ctrl->opts->reconnect_delay = v; 3649 return count; 3650 } 3651 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR, 3652 nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store); 3653 3654 static struct attribute *nvme_dev_attrs[] = { 3655 &dev_attr_reset_controller.attr, 3656 &dev_attr_rescan_controller.attr, 3657 &dev_attr_model.attr, 3658 &dev_attr_serial.attr, 3659 &dev_attr_firmware_rev.attr, 3660 &dev_attr_cntlid.attr, 3661 &dev_attr_delete_controller.attr, 3662 &dev_attr_transport.attr, 3663 &dev_attr_subsysnqn.attr, 3664 &dev_attr_address.attr, 3665 &dev_attr_state.attr, 3666 &dev_attr_numa_node.attr, 3667 &dev_attr_queue_count.attr, 3668 &dev_attr_sqsize.attr, 3669 &dev_attr_hostnqn.attr, 3670 &dev_attr_hostid.attr, 3671 &dev_attr_ctrl_loss_tmo.attr, 3672 &dev_attr_reconnect_delay.attr, 3673 NULL 3674 }; 3675 3676 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj, 3677 struct attribute *a, int n) 3678 { 3679 struct device *dev = container_of(kobj, struct device, kobj); 3680 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 3681 3682 if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl) 3683 return 0; 3684 if (a == &dev_attr_address.attr && !ctrl->ops->get_address) 3685 return 0; 3686 if (a == &dev_attr_hostnqn.attr && !ctrl->opts) 3687 return 0; 3688 if (a == &dev_attr_hostid.attr && !ctrl->opts) 3689 return 0; 3690 if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts) 3691 return 0; 3692 if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts) 3693 return 0; 3694 3695 return a->mode; 3696 } 3697 3698 static struct attribute_group nvme_dev_attrs_group = { 3699 .attrs = nvme_dev_attrs, 3700 .is_visible = nvme_dev_attrs_are_visible, 3701 }; 3702 3703 static const struct attribute_group *nvme_dev_attr_groups[] = { 3704 &nvme_dev_attrs_group, 3705 NULL, 3706 }; 3707 3708 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys, 3709 unsigned nsid) 3710 { 3711 struct nvme_ns_head *h; 3712 3713 lockdep_assert_held(&subsys->lock); 3714 3715 list_for_each_entry(h, &subsys->nsheads, entry) { 3716 if (h->ns_id == nsid && kref_get_unless_zero(&h->ref)) 3717 return h; 3718 } 3719 3720 return NULL; 3721 } 3722 3723 static int __nvme_check_ids(struct nvme_subsystem *subsys, 3724 struct nvme_ns_head *new) 3725 { 3726 struct nvme_ns_head *h; 3727 3728 lockdep_assert_held(&subsys->lock); 3729 3730 list_for_each_entry(h, &subsys->nsheads, entry) { 3731 if (nvme_ns_ids_valid(&new->ids) && 3732 nvme_ns_ids_equal(&new->ids, &h->ids)) 3733 return -EINVAL; 3734 } 3735 3736 return 0; 3737 } 3738 3739 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, 3740 unsigned nsid, struct nvme_ns_ids *ids) 3741 { 3742 struct nvme_ns_head *head; 3743 size_t size = sizeof(*head); 3744 int ret = -ENOMEM; 3745 3746 #ifdef CONFIG_NVME_MULTIPATH 3747 size += num_possible_nodes() * sizeof(struct nvme_ns *); 3748 #endif 3749 3750 head = kzalloc(size, GFP_KERNEL); 3751 if (!head) 3752 goto out; 3753 ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL); 3754 if (ret < 0) 3755 goto out_free_head; 3756 head->instance = ret; 3757 INIT_LIST_HEAD(&head->list); 3758 ret = init_srcu_struct(&head->srcu); 3759 if (ret) 3760 goto out_ida_remove; 3761 head->subsys = ctrl->subsys; 3762 head->ns_id = nsid; 3763 head->ids = *ids; 3764 kref_init(&head->ref); 3765 3766 ret = __nvme_check_ids(ctrl->subsys, head); 3767 if (ret) { 3768 dev_err(ctrl->device, 3769 "duplicate IDs for nsid %d\n", nsid); 3770 goto out_cleanup_srcu; 3771 } 3772 3773 if (head->ids.csi) { 3774 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects); 3775 if (ret) 3776 goto out_cleanup_srcu; 3777 } else 3778 head->effects = ctrl->effects; 3779 3780 ret = nvme_mpath_alloc_disk(ctrl, head); 3781 if (ret) 3782 goto out_cleanup_srcu; 3783 3784 list_add_tail(&head->entry, &ctrl->subsys->nsheads); 3785 3786 kref_get(&ctrl->subsys->ref); 3787 3788 return head; 3789 out_cleanup_srcu: 3790 cleanup_srcu_struct(&head->srcu); 3791 out_ida_remove: 3792 ida_simple_remove(&ctrl->subsys->ns_ida, head->instance); 3793 out_free_head: 3794 kfree(head); 3795 out: 3796 if (ret > 0) 3797 ret = blk_status_to_errno(nvme_error_status(ret)); 3798 return ERR_PTR(ret); 3799 } 3800 3801 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, 3802 struct nvme_ns_ids *ids, bool is_shared) 3803 { 3804 struct nvme_ctrl *ctrl = ns->ctrl; 3805 struct nvme_ns_head *head = NULL; 3806 int ret = 0; 3807 3808 mutex_lock(&ctrl->subsys->lock); 3809 head = nvme_find_ns_head(ctrl->subsys, nsid); 3810 if (!head) { 3811 head = nvme_alloc_ns_head(ctrl, nsid, ids); 3812 if (IS_ERR(head)) { 3813 ret = PTR_ERR(head); 3814 goto out_unlock; 3815 } 3816 head->shared = is_shared; 3817 } else { 3818 ret = -EINVAL; 3819 if (!is_shared || !head->shared) { 3820 dev_err(ctrl->device, 3821 "Duplicate unshared namespace %d\n", nsid); 3822 goto out_put_ns_head; 3823 } 3824 if (!nvme_ns_ids_equal(&head->ids, ids)) { 3825 dev_err(ctrl->device, 3826 "IDs don't match for shared namespace %d\n", 3827 nsid); 3828 goto out_put_ns_head; 3829 } 3830 } 3831 3832 list_add_tail_rcu(&ns->siblings, &head->list); 3833 ns->head = head; 3834 mutex_unlock(&ctrl->subsys->lock); 3835 return 0; 3836 3837 out_put_ns_head: 3838 nvme_put_ns_head(head); 3839 out_unlock: 3840 mutex_unlock(&ctrl->subsys->lock); 3841 return ret; 3842 } 3843 3844 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b) 3845 { 3846 struct nvme_ns *nsa = container_of(a, struct nvme_ns, list); 3847 struct nvme_ns *nsb = container_of(b, struct nvme_ns, list); 3848 3849 return nsa->head->ns_id - nsb->head->ns_id; 3850 } 3851 3852 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid) 3853 { 3854 struct nvme_ns *ns, *ret = NULL; 3855 3856 down_read(&ctrl->namespaces_rwsem); 3857 list_for_each_entry(ns, &ctrl->namespaces, list) { 3858 if (ns->head->ns_id == nsid) { 3859 if (!kref_get_unless_zero(&ns->kref)) 3860 continue; 3861 ret = ns; 3862 break; 3863 } 3864 if (ns->head->ns_id > nsid) 3865 break; 3866 } 3867 up_read(&ctrl->namespaces_rwsem); 3868 return ret; 3869 } 3870 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU); 3871 3872 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid, 3873 struct nvme_ns_ids *ids) 3874 { 3875 struct nvme_ns *ns; 3876 struct gendisk *disk; 3877 struct nvme_id_ns *id; 3878 char disk_name[DISK_NAME_LEN]; 3879 int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT; 3880 3881 if (nvme_identify_ns(ctrl, nsid, ids, &id)) 3882 return; 3883 3884 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node); 3885 if (!ns) 3886 goto out_free_id; 3887 3888 ns->queue = blk_mq_init_queue(ctrl->tagset); 3889 if (IS_ERR(ns->queue)) 3890 goto out_free_ns; 3891 3892 if (ctrl->opts && ctrl->opts->data_digest) 3893 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue); 3894 3895 blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue); 3896 if (ctrl->ops->flags & NVME_F_PCI_P2PDMA) 3897 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue); 3898 3899 ns->queue->queuedata = ns; 3900 ns->ctrl = ctrl; 3901 kref_init(&ns->kref); 3902 3903 if (nvme_init_ns_head(ns, nsid, ids, id->nmic & NVME_NS_NMIC_SHARED)) 3904 goto out_free_queue; 3905 nvme_set_disk_name(disk_name, ns, ctrl, &flags); 3906 3907 disk = alloc_disk_node(0, node); 3908 if (!disk) 3909 goto out_unlink_ns; 3910 3911 disk->fops = &nvme_bdev_ops; 3912 disk->private_data = ns; 3913 disk->queue = ns->queue; 3914 disk->flags = flags; 3915 memcpy(disk->disk_name, disk_name, DISK_NAME_LEN); 3916 ns->disk = disk; 3917 3918 if (nvme_update_ns_info(ns, id)) 3919 goto out_put_disk; 3920 3921 if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) { 3922 if (nvme_nvm_register(ns, disk_name, node)) { 3923 dev_warn(ctrl->device, "LightNVM init failure\n"); 3924 goto out_put_disk; 3925 } 3926 } 3927 3928 down_write(&ctrl->namespaces_rwsem); 3929 list_add_tail(&ns->list, &ctrl->namespaces); 3930 up_write(&ctrl->namespaces_rwsem); 3931 3932 nvme_get_ctrl(ctrl); 3933 3934 device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups); 3935 3936 nvme_mpath_add_disk(ns, id); 3937 nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name); 3938 kfree(id); 3939 3940 return; 3941 out_put_disk: 3942 /* prevent double queue cleanup */ 3943 ns->disk->queue = NULL; 3944 put_disk(ns->disk); 3945 out_unlink_ns: 3946 mutex_lock(&ctrl->subsys->lock); 3947 list_del_rcu(&ns->siblings); 3948 if (list_empty(&ns->head->list)) 3949 list_del_init(&ns->head->entry); 3950 mutex_unlock(&ctrl->subsys->lock); 3951 nvme_put_ns_head(ns->head); 3952 out_free_queue: 3953 blk_cleanup_queue(ns->queue); 3954 out_free_ns: 3955 kfree(ns); 3956 out_free_id: 3957 kfree(id); 3958 } 3959 3960 static void nvme_ns_remove(struct nvme_ns *ns) 3961 { 3962 if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags)) 3963 return; 3964 3965 set_capacity(ns->disk, 0); 3966 nvme_fault_inject_fini(&ns->fault_inject); 3967 3968 mutex_lock(&ns->ctrl->subsys->lock); 3969 list_del_rcu(&ns->siblings); 3970 if (list_empty(&ns->head->list)) 3971 list_del_init(&ns->head->entry); 3972 mutex_unlock(&ns->ctrl->subsys->lock); 3973 3974 synchronize_rcu(); /* guarantee not available in head->list */ 3975 nvme_mpath_clear_current_path(ns); 3976 synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */ 3977 3978 if (ns->disk->flags & GENHD_FL_UP) { 3979 del_gendisk(ns->disk); 3980 blk_cleanup_queue(ns->queue); 3981 if (blk_get_integrity(ns->disk)) 3982 blk_integrity_unregister(ns->disk); 3983 } 3984 3985 down_write(&ns->ctrl->namespaces_rwsem); 3986 list_del_init(&ns->list); 3987 up_write(&ns->ctrl->namespaces_rwsem); 3988 3989 nvme_mpath_check_last_path(ns); 3990 nvme_put_ns(ns); 3991 } 3992 3993 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid) 3994 { 3995 struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid); 3996 3997 if (ns) { 3998 nvme_ns_remove(ns); 3999 nvme_put_ns(ns); 4000 } 4001 } 4002 4003 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_ids *ids) 4004 { 4005 struct nvme_id_ns *id; 4006 int ret = -ENODEV; 4007 4008 if (test_bit(NVME_NS_DEAD, &ns->flags)) 4009 goto out; 4010 4011 ret = nvme_identify_ns(ns->ctrl, ns->head->ns_id, ids, &id); 4012 if (ret) 4013 goto out; 4014 4015 ret = -ENODEV; 4016 if (!nvme_ns_ids_equal(&ns->head->ids, ids)) { 4017 dev_err(ns->ctrl->device, 4018 "identifiers changed for nsid %d\n", ns->head->ns_id); 4019 goto out_free_id; 4020 } 4021 4022 ret = nvme_update_ns_info(ns, id); 4023 4024 out_free_id: 4025 kfree(id); 4026 out: 4027 /* 4028 * Only remove the namespace if we got a fatal error back from the 4029 * device, otherwise ignore the error and just move on. 4030 * 4031 * TODO: we should probably schedule a delayed retry here. 4032 */ 4033 if (ret && ret != -ENOMEM && !(ret > 0 && !(ret & NVME_SC_DNR))) 4034 nvme_ns_remove(ns); 4035 } 4036 4037 static void nvme_validate_or_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) 4038 { 4039 struct nvme_ns_ids ids = { }; 4040 struct nvme_ns *ns; 4041 4042 if (nvme_identify_ns_descs(ctrl, nsid, &ids)) 4043 return; 4044 4045 ns = nvme_find_get_ns(ctrl, nsid); 4046 if (ns) { 4047 nvme_validate_ns(ns, &ids); 4048 nvme_put_ns(ns); 4049 return; 4050 } 4051 4052 switch (ids.csi) { 4053 case NVME_CSI_NVM: 4054 nvme_alloc_ns(ctrl, nsid, &ids); 4055 break; 4056 case NVME_CSI_ZNS: 4057 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) { 4058 dev_warn(ctrl->device, 4059 "nsid %u not supported without CONFIG_BLK_DEV_ZONED\n", 4060 nsid); 4061 break; 4062 } 4063 nvme_alloc_ns(ctrl, nsid, &ids); 4064 break; 4065 default: 4066 dev_warn(ctrl->device, "unknown csi %u for nsid %u\n", 4067 ids.csi, nsid); 4068 break; 4069 } 4070 } 4071 4072 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, 4073 unsigned nsid) 4074 { 4075 struct nvme_ns *ns, *next; 4076 LIST_HEAD(rm_list); 4077 4078 down_write(&ctrl->namespaces_rwsem); 4079 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) { 4080 if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags)) 4081 list_move_tail(&ns->list, &rm_list); 4082 } 4083 up_write(&ctrl->namespaces_rwsem); 4084 4085 list_for_each_entry_safe(ns, next, &rm_list, list) 4086 nvme_ns_remove(ns); 4087 4088 } 4089 4090 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl) 4091 { 4092 const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32); 4093 __le32 *ns_list; 4094 u32 prev = 0; 4095 int ret = 0, i; 4096 4097 if (nvme_ctrl_limited_cns(ctrl)) 4098 return -EOPNOTSUPP; 4099 4100 ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL); 4101 if (!ns_list) 4102 return -ENOMEM; 4103 4104 for (;;) { 4105 struct nvme_command cmd = { 4106 .identify.opcode = nvme_admin_identify, 4107 .identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST, 4108 .identify.nsid = cpu_to_le32(prev), 4109 }; 4110 4111 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list, 4112 NVME_IDENTIFY_DATA_SIZE); 4113 if (ret) { 4114 dev_warn(ctrl->device, 4115 "Identify NS List failed (status=0x%x)\n", ret); 4116 goto free; 4117 } 4118 4119 for (i = 0; i < nr_entries; i++) { 4120 u32 nsid = le32_to_cpu(ns_list[i]); 4121 4122 if (!nsid) /* end of the list? */ 4123 goto out; 4124 nvme_validate_or_alloc_ns(ctrl, nsid); 4125 while (++prev < nsid) 4126 nvme_ns_remove_by_nsid(ctrl, prev); 4127 } 4128 } 4129 out: 4130 nvme_remove_invalid_namespaces(ctrl, prev); 4131 free: 4132 kfree(ns_list); 4133 return ret; 4134 } 4135 4136 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl) 4137 { 4138 struct nvme_id_ctrl *id; 4139 u32 nn, i; 4140 4141 if (nvme_identify_ctrl(ctrl, &id)) 4142 return; 4143 nn = le32_to_cpu(id->nn); 4144 kfree(id); 4145 4146 for (i = 1; i <= nn; i++) 4147 nvme_validate_or_alloc_ns(ctrl, i); 4148 4149 nvme_remove_invalid_namespaces(ctrl, nn); 4150 } 4151 4152 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl) 4153 { 4154 size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32); 4155 __le32 *log; 4156 int error; 4157 4158 log = kzalloc(log_size, GFP_KERNEL); 4159 if (!log) 4160 return; 4161 4162 /* 4163 * We need to read the log to clear the AEN, but we don't want to rely 4164 * on it for the changed namespace information as userspace could have 4165 * raced with us in reading the log page, which could cause us to miss 4166 * updates. 4167 */ 4168 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0, 4169 NVME_CSI_NVM, log, log_size, 0); 4170 if (error) 4171 dev_warn(ctrl->device, 4172 "reading changed ns log failed: %d\n", error); 4173 4174 kfree(log); 4175 } 4176 4177 static void nvme_scan_work(struct work_struct *work) 4178 { 4179 struct nvme_ctrl *ctrl = 4180 container_of(work, struct nvme_ctrl, scan_work); 4181 4182 /* No tagset on a live ctrl means IO queues could not created */ 4183 if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset) 4184 return; 4185 4186 if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) { 4187 dev_info(ctrl->device, "rescanning namespaces.\n"); 4188 nvme_clear_changed_ns_log(ctrl); 4189 } 4190 4191 mutex_lock(&ctrl->scan_lock); 4192 if (nvme_scan_ns_list(ctrl) != 0) 4193 nvme_scan_ns_sequential(ctrl); 4194 mutex_unlock(&ctrl->scan_lock); 4195 4196 down_write(&ctrl->namespaces_rwsem); 4197 list_sort(NULL, &ctrl->namespaces, ns_cmp); 4198 up_write(&ctrl->namespaces_rwsem); 4199 } 4200 4201 /* 4202 * This function iterates the namespace list unlocked to allow recovery from 4203 * controller failure. It is up to the caller to ensure the namespace list is 4204 * not modified by scan work while this function is executing. 4205 */ 4206 void nvme_remove_namespaces(struct nvme_ctrl *ctrl) 4207 { 4208 struct nvme_ns *ns, *next; 4209 LIST_HEAD(ns_list); 4210 4211 /* 4212 * make sure to requeue I/O to all namespaces as these 4213 * might result from the scan itself and must complete 4214 * for the scan_work to make progress 4215 */ 4216 nvme_mpath_clear_ctrl_paths(ctrl); 4217 4218 /* prevent racing with ns scanning */ 4219 flush_work(&ctrl->scan_work); 4220 4221 /* 4222 * The dead states indicates the controller was not gracefully 4223 * disconnected. In that case, we won't be able to flush any data while 4224 * removing the namespaces' disks; fail all the queues now to avoid 4225 * potentially having to clean up the failed sync later. 4226 */ 4227 if (ctrl->state == NVME_CTRL_DEAD) 4228 nvme_kill_queues(ctrl); 4229 4230 /* this is a no-op when called from the controller reset handler */ 4231 nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO); 4232 4233 down_write(&ctrl->namespaces_rwsem); 4234 list_splice_init(&ctrl->namespaces, &ns_list); 4235 up_write(&ctrl->namespaces_rwsem); 4236 4237 list_for_each_entry_safe(ns, next, &ns_list, list) 4238 nvme_ns_remove(ns); 4239 } 4240 EXPORT_SYMBOL_GPL(nvme_remove_namespaces); 4241 4242 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env) 4243 { 4244 struct nvme_ctrl *ctrl = 4245 container_of(dev, struct nvme_ctrl, ctrl_device); 4246 struct nvmf_ctrl_options *opts = ctrl->opts; 4247 int ret; 4248 4249 ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name); 4250 if (ret) 4251 return ret; 4252 4253 if (opts) { 4254 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr); 4255 if (ret) 4256 return ret; 4257 4258 ret = add_uevent_var(env, "NVME_TRSVCID=%s", 4259 opts->trsvcid ?: "none"); 4260 if (ret) 4261 return ret; 4262 4263 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s", 4264 opts->host_traddr ?: "none"); 4265 } 4266 return ret; 4267 } 4268 4269 static void nvme_aen_uevent(struct nvme_ctrl *ctrl) 4270 { 4271 char *envp[2] = { NULL, NULL }; 4272 u32 aen_result = ctrl->aen_result; 4273 4274 ctrl->aen_result = 0; 4275 if (!aen_result) 4276 return; 4277 4278 envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result); 4279 if (!envp[0]) 4280 return; 4281 kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp); 4282 kfree(envp[0]); 4283 } 4284 4285 static void nvme_async_event_work(struct work_struct *work) 4286 { 4287 struct nvme_ctrl *ctrl = 4288 container_of(work, struct nvme_ctrl, async_event_work); 4289 4290 nvme_aen_uevent(ctrl); 4291 ctrl->ops->submit_async_event(ctrl); 4292 } 4293 4294 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl) 4295 { 4296 4297 u32 csts; 4298 4299 if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) 4300 return false; 4301 4302 if (csts == ~0) 4303 return false; 4304 4305 return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP)); 4306 } 4307 4308 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl) 4309 { 4310 struct nvme_fw_slot_info_log *log; 4311 4312 log = kmalloc(sizeof(*log), GFP_KERNEL); 4313 if (!log) 4314 return; 4315 4316 if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM, 4317 log, sizeof(*log), 0)) 4318 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n"); 4319 kfree(log); 4320 } 4321 4322 static void nvme_fw_act_work(struct work_struct *work) 4323 { 4324 struct nvme_ctrl *ctrl = container_of(work, 4325 struct nvme_ctrl, fw_act_work); 4326 unsigned long fw_act_timeout; 4327 4328 if (ctrl->mtfa) 4329 fw_act_timeout = jiffies + 4330 msecs_to_jiffies(ctrl->mtfa * 100); 4331 else 4332 fw_act_timeout = jiffies + 4333 msecs_to_jiffies(admin_timeout * 1000); 4334 4335 nvme_stop_queues(ctrl); 4336 while (nvme_ctrl_pp_status(ctrl)) { 4337 if (time_after(jiffies, fw_act_timeout)) { 4338 dev_warn(ctrl->device, 4339 "Fw activation timeout, reset controller\n"); 4340 nvme_try_sched_reset(ctrl); 4341 return; 4342 } 4343 msleep(100); 4344 } 4345 4346 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) 4347 return; 4348 4349 nvme_start_queues(ctrl); 4350 /* read FW slot information to clear the AER */ 4351 nvme_get_fw_slot_info(ctrl); 4352 } 4353 4354 static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result) 4355 { 4356 u32 aer_notice_type = (result & 0xff00) >> 8; 4357 4358 trace_nvme_async_event(ctrl, aer_notice_type); 4359 4360 switch (aer_notice_type) { 4361 case NVME_AER_NOTICE_NS_CHANGED: 4362 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events); 4363 nvme_queue_scan(ctrl); 4364 break; 4365 case NVME_AER_NOTICE_FW_ACT_STARTING: 4366 /* 4367 * We are (ab)using the RESETTING state to prevent subsequent 4368 * recovery actions from interfering with the controller's 4369 * firmware activation. 4370 */ 4371 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 4372 queue_work(nvme_wq, &ctrl->fw_act_work); 4373 break; 4374 #ifdef CONFIG_NVME_MULTIPATH 4375 case NVME_AER_NOTICE_ANA: 4376 if (!ctrl->ana_log_buf) 4377 break; 4378 queue_work(nvme_wq, &ctrl->ana_work); 4379 break; 4380 #endif 4381 case NVME_AER_NOTICE_DISC_CHANGED: 4382 ctrl->aen_result = result; 4383 break; 4384 default: 4385 dev_warn(ctrl->device, "async event result %08x\n", result); 4386 } 4387 } 4388 4389 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status, 4390 volatile union nvme_result *res) 4391 { 4392 u32 result = le32_to_cpu(res->u32); 4393 u32 aer_type = result & 0x07; 4394 4395 if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS) 4396 return; 4397 4398 switch (aer_type) { 4399 case NVME_AER_NOTICE: 4400 nvme_handle_aen_notice(ctrl, result); 4401 break; 4402 case NVME_AER_ERROR: 4403 case NVME_AER_SMART: 4404 case NVME_AER_CSS: 4405 case NVME_AER_VS: 4406 trace_nvme_async_event(ctrl, aer_type); 4407 ctrl->aen_result = result; 4408 break; 4409 default: 4410 break; 4411 } 4412 queue_work(nvme_wq, &ctrl->async_event_work); 4413 } 4414 EXPORT_SYMBOL_GPL(nvme_complete_async_event); 4415 4416 void nvme_stop_ctrl(struct nvme_ctrl *ctrl) 4417 { 4418 nvme_mpath_stop(ctrl); 4419 nvme_stop_keep_alive(ctrl); 4420 nvme_stop_failfast_work(ctrl); 4421 flush_work(&ctrl->async_event_work); 4422 cancel_work_sync(&ctrl->fw_act_work); 4423 } 4424 EXPORT_SYMBOL_GPL(nvme_stop_ctrl); 4425 4426 void nvme_start_ctrl(struct nvme_ctrl *ctrl) 4427 { 4428 nvme_start_keep_alive(ctrl); 4429 4430 nvme_enable_aen(ctrl); 4431 4432 if (ctrl->queue_count > 1) { 4433 nvme_queue_scan(ctrl); 4434 nvme_start_queues(ctrl); 4435 } 4436 } 4437 EXPORT_SYMBOL_GPL(nvme_start_ctrl); 4438 4439 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl) 4440 { 4441 nvme_fault_inject_fini(&ctrl->fault_inject); 4442 dev_pm_qos_hide_latency_tolerance(ctrl->device); 4443 cdev_device_del(&ctrl->cdev, ctrl->device); 4444 nvme_put_ctrl(ctrl); 4445 } 4446 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl); 4447 4448 static void nvme_free_cels(struct nvme_ctrl *ctrl) 4449 { 4450 struct nvme_effects_log *cel; 4451 unsigned long i; 4452 4453 xa_for_each (&ctrl->cels, i, cel) { 4454 xa_erase(&ctrl->cels, i); 4455 kfree(cel); 4456 } 4457 4458 xa_destroy(&ctrl->cels); 4459 } 4460 4461 static void nvme_free_ctrl(struct device *dev) 4462 { 4463 struct nvme_ctrl *ctrl = 4464 container_of(dev, struct nvme_ctrl, ctrl_device); 4465 struct nvme_subsystem *subsys = ctrl->subsys; 4466 4467 if (!subsys || ctrl->instance != subsys->instance) 4468 ida_simple_remove(&nvme_instance_ida, ctrl->instance); 4469 4470 nvme_free_cels(ctrl); 4471 nvme_mpath_uninit(ctrl); 4472 __free_page(ctrl->discard_page); 4473 4474 if (subsys) { 4475 mutex_lock(&nvme_subsystems_lock); 4476 list_del(&ctrl->subsys_entry); 4477 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device)); 4478 mutex_unlock(&nvme_subsystems_lock); 4479 } 4480 4481 ctrl->ops->free_ctrl(ctrl); 4482 4483 if (subsys) 4484 nvme_put_subsystem(subsys); 4485 } 4486 4487 /* 4488 * Initialize a NVMe controller structures. This needs to be called during 4489 * earliest initialization so that we have the initialized structured around 4490 * during probing. 4491 */ 4492 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, 4493 const struct nvme_ctrl_ops *ops, unsigned long quirks) 4494 { 4495 int ret; 4496 4497 ctrl->state = NVME_CTRL_NEW; 4498 clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags); 4499 spin_lock_init(&ctrl->lock); 4500 mutex_init(&ctrl->scan_lock); 4501 INIT_LIST_HEAD(&ctrl->namespaces); 4502 xa_init(&ctrl->cels); 4503 init_rwsem(&ctrl->namespaces_rwsem); 4504 ctrl->dev = dev; 4505 ctrl->ops = ops; 4506 ctrl->quirks = quirks; 4507 ctrl->numa_node = NUMA_NO_NODE; 4508 INIT_WORK(&ctrl->scan_work, nvme_scan_work); 4509 INIT_WORK(&ctrl->async_event_work, nvme_async_event_work); 4510 INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work); 4511 INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work); 4512 init_waitqueue_head(&ctrl->state_wq); 4513 4514 INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work); 4515 INIT_DELAYED_WORK(&ctrl->failfast_work, nvme_failfast_work); 4516 memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd)); 4517 ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive; 4518 4519 BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) > 4520 PAGE_SIZE); 4521 ctrl->discard_page = alloc_page(GFP_KERNEL); 4522 if (!ctrl->discard_page) { 4523 ret = -ENOMEM; 4524 goto out; 4525 } 4526 4527 ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL); 4528 if (ret < 0) 4529 goto out; 4530 ctrl->instance = ret; 4531 4532 device_initialize(&ctrl->ctrl_device); 4533 ctrl->device = &ctrl->ctrl_device; 4534 ctrl->device->devt = MKDEV(MAJOR(nvme_ctrl_base_chr_devt), 4535 ctrl->instance); 4536 ctrl->device->class = nvme_class; 4537 ctrl->device->parent = ctrl->dev; 4538 ctrl->device->groups = nvme_dev_attr_groups; 4539 ctrl->device->release = nvme_free_ctrl; 4540 dev_set_drvdata(ctrl->device, ctrl); 4541 ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance); 4542 if (ret) 4543 goto out_release_instance; 4544 4545 nvme_get_ctrl(ctrl); 4546 cdev_init(&ctrl->cdev, &nvme_dev_fops); 4547 ctrl->cdev.owner = ops->module; 4548 ret = cdev_device_add(&ctrl->cdev, ctrl->device); 4549 if (ret) 4550 goto out_free_name; 4551 4552 /* 4553 * Initialize latency tolerance controls. The sysfs files won't 4554 * be visible to userspace unless the device actually supports APST. 4555 */ 4556 ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance; 4557 dev_pm_qos_update_user_latency_tolerance(ctrl->device, 4558 min(default_ps_max_latency_us, (unsigned long)S32_MAX)); 4559 4560 nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device)); 4561 4562 return 0; 4563 out_free_name: 4564 nvme_put_ctrl(ctrl); 4565 kfree_const(ctrl->device->kobj.name); 4566 out_release_instance: 4567 ida_simple_remove(&nvme_instance_ida, ctrl->instance); 4568 out: 4569 if (ctrl->discard_page) 4570 __free_page(ctrl->discard_page); 4571 return ret; 4572 } 4573 EXPORT_SYMBOL_GPL(nvme_init_ctrl); 4574 4575 /** 4576 * nvme_kill_queues(): Ends all namespace queues 4577 * @ctrl: the dead controller that needs to end 4578 * 4579 * Call this function when the driver determines it is unable to get the 4580 * controller in a state capable of servicing IO. 4581 */ 4582 void nvme_kill_queues(struct nvme_ctrl *ctrl) 4583 { 4584 struct nvme_ns *ns; 4585 4586 down_read(&ctrl->namespaces_rwsem); 4587 4588 /* Forcibly unquiesce queues to avoid blocking dispatch */ 4589 if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q)) 4590 blk_mq_unquiesce_queue(ctrl->admin_q); 4591 4592 list_for_each_entry(ns, &ctrl->namespaces, list) 4593 nvme_set_queue_dying(ns); 4594 4595 up_read(&ctrl->namespaces_rwsem); 4596 } 4597 EXPORT_SYMBOL_GPL(nvme_kill_queues); 4598 4599 void nvme_unfreeze(struct nvme_ctrl *ctrl) 4600 { 4601 struct nvme_ns *ns; 4602 4603 down_read(&ctrl->namespaces_rwsem); 4604 list_for_each_entry(ns, &ctrl->namespaces, list) 4605 blk_mq_unfreeze_queue(ns->queue); 4606 up_read(&ctrl->namespaces_rwsem); 4607 } 4608 EXPORT_SYMBOL_GPL(nvme_unfreeze); 4609 4610 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout) 4611 { 4612 struct nvme_ns *ns; 4613 4614 down_read(&ctrl->namespaces_rwsem); 4615 list_for_each_entry(ns, &ctrl->namespaces, list) { 4616 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout); 4617 if (timeout <= 0) 4618 break; 4619 } 4620 up_read(&ctrl->namespaces_rwsem); 4621 return timeout; 4622 } 4623 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout); 4624 4625 void nvme_wait_freeze(struct nvme_ctrl *ctrl) 4626 { 4627 struct nvme_ns *ns; 4628 4629 down_read(&ctrl->namespaces_rwsem); 4630 list_for_each_entry(ns, &ctrl->namespaces, list) 4631 blk_mq_freeze_queue_wait(ns->queue); 4632 up_read(&ctrl->namespaces_rwsem); 4633 } 4634 EXPORT_SYMBOL_GPL(nvme_wait_freeze); 4635 4636 void nvme_start_freeze(struct nvme_ctrl *ctrl) 4637 { 4638 struct nvme_ns *ns; 4639 4640 down_read(&ctrl->namespaces_rwsem); 4641 list_for_each_entry(ns, &ctrl->namespaces, list) 4642 blk_freeze_queue_start(ns->queue); 4643 up_read(&ctrl->namespaces_rwsem); 4644 } 4645 EXPORT_SYMBOL_GPL(nvme_start_freeze); 4646 4647 void nvme_stop_queues(struct nvme_ctrl *ctrl) 4648 { 4649 struct nvme_ns *ns; 4650 4651 down_read(&ctrl->namespaces_rwsem); 4652 list_for_each_entry(ns, &ctrl->namespaces, list) 4653 blk_mq_quiesce_queue(ns->queue); 4654 up_read(&ctrl->namespaces_rwsem); 4655 } 4656 EXPORT_SYMBOL_GPL(nvme_stop_queues); 4657 4658 void nvme_start_queues(struct nvme_ctrl *ctrl) 4659 { 4660 struct nvme_ns *ns; 4661 4662 down_read(&ctrl->namespaces_rwsem); 4663 list_for_each_entry(ns, &ctrl->namespaces, list) 4664 blk_mq_unquiesce_queue(ns->queue); 4665 up_read(&ctrl->namespaces_rwsem); 4666 } 4667 EXPORT_SYMBOL_GPL(nvme_start_queues); 4668 4669 void nvme_sync_io_queues(struct nvme_ctrl *ctrl) 4670 { 4671 struct nvme_ns *ns; 4672 4673 down_read(&ctrl->namespaces_rwsem); 4674 list_for_each_entry(ns, &ctrl->namespaces, list) 4675 blk_sync_queue(ns->queue); 4676 up_read(&ctrl->namespaces_rwsem); 4677 } 4678 EXPORT_SYMBOL_GPL(nvme_sync_io_queues); 4679 4680 void nvme_sync_queues(struct nvme_ctrl *ctrl) 4681 { 4682 nvme_sync_io_queues(ctrl); 4683 if (ctrl->admin_q) 4684 blk_sync_queue(ctrl->admin_q); 4685 } 4686 EXPORT_SYMBOL_GPL(nvme_sync_queues); 4687 4688 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file) 4689 { 4690 if (file->f_op != &nvme_dev_fops) 4691 return NULL; 4692 return file->private_data; 4693 } 4694 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU); 4695 4696 /* 4697 * Check we didn't inadvertently grow the command structure sizes: 4698 */ 4699 static inline void _nvme_check_size(void) 4700 { 4701 BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64); 4702 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64); 4703 BUILD_BUG_ON(sizeof(struct nvme_identify) != 64); 4704 BUILD_BUG_ON(sizeof(struct nvme_features) != 64); 4705 BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64); 4706 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64); 4707 BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64); 4708 BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64); 4709 BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64); 4710 BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64); 4711 BUILD_BUG_ON(sizeof(struct nvme_command) != 64); 4712 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE); 4713 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE); 4714 BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE); 4715 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE); 4716 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); 4717 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); 4718 BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64); 4719 BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64); 4720 } 4721 4722 4723 static int __init nvme_core_init(void) 4724 { 4725 int result = -ENOMEM; 4726 4727 _nvme_check_size(); 4728 4729 nvme_wq = alloc_workqueue("nvme-wq", 4730 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 4731 if (!nvme_wq) 4732 goto out; 4733 4734 nvme_reset_wq = alloc_workqueue("nvme-reset-wq", 4735 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 4736 if (!nvme_reset_wq) 4737 goto destroy_wq; 4738 4739 nvme_delete_wq = alloc_workqueue("nvme-delete-wq", 4740 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 4741 if (!nvme_delete_wq) 4742 goto destroy_reset_wq; 4743 4744 result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0, 4745 NVME_MINORS, "nvme"); 4746 if (result < 0) 4747 goto destroy_delete_wq; 4748 4749 nvme_class = class_create(THIS_MODULE, "nvme"); 4750 if (IS_ERR(nvme_class)) { 4751 result = PTR_ERR(nvme_class); 4752 goto unregister_chrdev; 4753 } 4754 nvme_class->dev_uevent = nvme_class_uevent; 4755 4756 nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem"); 4757 if (IS_ERR(nvme_subsys_class)) { 4758 result = PTR_ERR(nvme_subsys_class); 4759 goto destroy_class; 4760 } 4761 return 0; 4762 4763 destroy_class: 4764 class_destroy(nvme_class); 4765 unregister_chrdev: 4766 unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS); 4767 destroy_delete_wq: 4768 destroy_workqueue(nvme_delete_wq); 4769 destroy_reset_wq: 4770 destroy_workqueue(nvme_reset_wq); 4771 destroy_wq: 4772 destroy_workqueue(nvme_wq); 4773 out: 4774 return result; 4775 } 4776 4777 static void __exit nvme_core_exit(void) 4778 { 4779 class_destroy(nvme_subsys_class); 4780 class_destroy(nvme_class); 4781 unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS); 4782 destroy_workqueue(nvme_delete_wq); 4783 destroy_workqueue(nvme_reset_wq); 4784 destroy_workqueue(nvme_wq); 4785 ida_destroy(&nvme_instance_ida); 4786 } 4787 4788 MODULE_LICENSE("GPL"); 4789 MODULE_VERSION("1.0"); 4790 module_init(nvme_core_init); 4791 module_exit(nvme_core_exit); 4792