1 /* 2 * NVM Express device driver 3 * Copyright (c) 2011-2014, Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 */ 14 15 #include <linux/blkdev.h> 16 #include <linux/blk-mq.h> 17 #include <linux/delay.h> 18 #include <linux/errno.h> 19 #include <linux/hdreg.h> 20 #include <linux/kernel.h> 21 #include <linux/module.h> 22 #include <linux/list_sort.h> 23 #include <linux/slab.h> 24 #include <linux/types.h> 25 #include <linux/pr.h> 26 #include <linux/ptrace.h> 27 #include <linux/nvme_ioctl.h> 28 #include <linux/t10-pi.h> 29 #include <linux/pm_qos.h> 30 #include <asm/unaligned.h> 31 32 #define CREATE_TRACE_POINTS 33 #include "trace.h" 34 35 #include "nvme.h" 36 #include "fabrics.h" 37 38 #define NVME_MINORS (1U << MINORBITS) 39 40 unsigned int admin_timeout = 60; 41 module_param(admin_timeout, uint, 0644); 42 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands"); 43 EXPORT_SYMBOL_GPL(admin_timeout); 44 45 unsigned int nvme_io_timeout = 30; 46 module_param_named(io_timeout, nvme_io_timeout, uint, 0644); 47 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O"); 48 EXPORT_SYMBOL_GPL(nvme_io_timeout); 49 50 static unsigned char shutdown_timeout = 5; 51 module_param(shutdown_timeout, byte, 0644); 52 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown"); 53 54 static u8 nvme_max_retries = 5; 55 module_param_named(max_retries, nvme_max_retries, byte, 0644); 56 MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); 57 58 static unsigned long default_ps_max_latency_us = 100000; 59 module_param(default_ps_max_latency_us, ulong, 0644); 60 MODULE_PARM_DESC(default_ps_max_latency_us, 61 "max power saving latency for new devices; use PM QOS to change per device"); 62 63 static bool force_apst; 64 module_param(force_apst, bool, 0644); 65 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off"); 66 67 static bool streams; 68 module_param(streams, bool, 0644); 69 MODULE_PARM_DESC(streams, "turn on support for Streams write directives"); 70 71 /* 72 * nvme_wq - hosts nvme related works that are not reset or delete 73 * nvme_reset_wq - hosts nvme reset works 74 * nvme_delete_wq - hosts nvme delete works 75 * 76 * nvme_wq will host works such are scan, aen handling, fw activation, 77 * keep-alive error recovery, periodic reconnects etc. nvme_reset_wq 78 * runs reset works which also flush works hosted on nvme_wq for 79 * serialization purposes. nvme_delete_wq host controller deletion 80 * works which flush reset works for serialization. 81 */ 82 struct workqueue_struct *nvme_wq; 83 EXPORT_SYMBOL_GPL(nvme_wq); 84 85 struct workqueue_struct *nvme_reset_wq; 86 EXPORT_SYMBOL_GPL(nvme_reset_wq); 87 88 struct workqueue_struct *nvme_delete_wq; 89 EXPORT_SYMBOL_GPL(nvme_delete_wq); 90 91 static DEFINE_IDA(nvme_subsystems_ida); 92 static LIST_HEAD(nvme_subsystems); 93 static DEFINE_MUTEX(nvme_subsystems_lock); 94 95 static DEFINE_IDA(nvme_instance_ida); 96 static dev_t nvme_chr_devt; 97 static struct class *nvme_class; 98 static struct class *nvme_subsys_class; 99 100 static void nvme_ns_remove(struct nvme_ns *ns); 101 static int nvme_revalidate_disk(struct gendisk *disk); 102 103 int nvme_reset_ctrl(struct nvme_ctrl *ctrl) 104 { 105 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 106 return -EBUSY; 107 if (!queue_work(nvme_reset_wq, &ctrl->reset_work)) 108 return -EBUSY; 109 return 0; 110 } 111 EXPORT_SYMBOL_GPL(nvme_reset_ctrl); 112 113 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl) 114 { 115 int ret; 116 117 ret = nvme_reset_ctrl(ctrl); 118 if (!ret) { 119 flush_work(&ctrl->reset_work); 120 if (ctrl->state != NVME_CTRL_LIVE) 121 ret = -ENETRESET; 122 } 123 124 return ret; 125 } 126 EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync); 127 128 static void nvme_delete_ctrl_work(struct work_struct *work) 129 { 130 struct nvme_ctrl *ctrl = 131 container_of(work, struct nvme_ctrl, delete_work); 132 133 dev_info(ctrl->device, 134 "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn); 135 136 flush_work(&ctrl->reset_work); 137 nvme_stop_ctrl(ctrl); 138 nvme_remove_namespaces(ctrl); 139 ctrl->ops->delete_ctrl(ctrl); 140 nvme_uninit_ctrl(ctrl); 141 nvme_put_ctrl(ctrl); 142 } 143 144 int nvme_delete_ctrl(struct nvme_ctrl *ctrl) 145 { 146 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING)) 147 return -EBUSY; 148 if (!queue_work(nvme_delete_wq, &ctrl->delete_work)) 149 return -EBUSY; 150 return 0; 151 } 152 EXPORT_SYMBOL_GPL(nvme_delete_ctrl); 153 154 int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl) 155 { 156 int ret = 0; 157 158 /* 159 * Keep a reference until the work is flushed since ->delete_ctrl 160 * can free the controller. 161 */ 162 nvme_get_ctrl(ctrl); 163 ret = nvme_delete_ctrl(ctrl); 164 if (!ret) 165 flush_work(&ctrl->delete_work); 166 nvme_put_ctrl(ctrl); 167 return ret; 168 } 169 EXPORT_SYMBOL_GPL(nvme_delete_ctrl_sync); 170 171 static inline bool nvme_ns_has_pi(struct nvme_ns *ns) 172 { 173 return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple); 174 } 175 176 static blk_status_t nvme_error_status(struct request *req) 177 { 178 switch (nvme_req(req)->status & 0x7ff) { 179 case NVME_SC_SUCCESS: 180 return BLK_STS_OK; 181 case NVME_SC_CAP_EXCEEDED: 182 return BLK_STS_NOSPC; 183 case NVME_SC_LBA_RANGE: 184 return BLK_STS_TARGET; 185 case NVME_SC_BAD_ATTRIBUTES: 186 case NVME_SC_ONCS_NOT_SUPPORTED: 187 case NVME_SC_INVALID_OPCODE: 188 case NVME_SC_INVALID_FIELD: 189 case NVME_SC_INVALID_NS: 190 return BLK_STS_NOTSUPP; 191 case NVME_SC_WRITE_FAULT: 192 case NVME_SC_READ_ERROR: 193 case NVME_SC_UNWRITTEN_BLOCK: 194 case NVME_SC_ACCESS_DENIED: 195 case NVME_SC_READ_ONLY: 196 case NVME_SC_COMPARE_FAILED: 197 return BLK_STS_MEDIUM; 198 case NVME_SC_GUARD_CHECK: 199 case NVME_SC_APPTAG_CHECK: 200 case NVME_SC_REFTAG_CHECK: 201 case NVME_SC_INVALID_PI: 202 return BLK_STS_PROTECTION; 203 case NVME_SC_RESERVATION_CONFLICT: 204 return BLK_STS_NEXUS; 205 default: 206 return BLK_STS_IOERR; 207 } 208 } 209 210 static inline bool nvme_req_needs_retry(struct request *req) 211 { 212 if (blk_noretry_request(req)) 213 return false; 214 if (nvme_req(req)->status & NVME_SC_DNR) 215 return false; 216 if (nvme_req(req)->retries >= nvme_max_retries) 217 return false; 218 return true; 219 } 220 221 void nvme_complete_rq(struct request *req) 222 { 223 blk_status_t status = nvme_error_status(req); 224 225 trace_nvme_complete_rq(req); 226 227 if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) { 228 if (nvme_req_needs_failover(req, status)) { 229 nvme_failover_req(req); 230 return; 231 } 232 233 if (!blk_queue_dying(req->q)) { 234 nvme_req(req)->retries++; 235 blk_mq_requeue_request(req, true); 236 return; 237 } 238 } 239 blk_mq_end_request(req, status); 240 } 241 EXPORT_SYMBOL_GPL(nvme_complete_rq); 242 243 void nvme_cancel_request(struct request *req, void *data, bool reserved) 244 { 245 if (!blk_mq_request_started(req)) 246 return; 247 248 dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device, 249 "Cancelling I/O %d", req->tag); 250 251 nvme_req(req)->status = NVME_SC_ABORT_REQ; 252 blk_mq_complete_request(req); 253 254 } 255 EXPORT_SYMBOL_GPL(nvme_cancel_request); 256 257 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, 258 enum nvme_ctrl_state new_state) 259 { 260 enum nvme_ctrl_state old_state; 261 unsigned long flags; 262 bool changed = false; 263 264 spin_lock_irqsave(&ctrl->lock, flags); 265 266 old_state = ctrl->state; 267 switch (new_state) { 268 case NVME_CTRL_ADMIN_ONLY: 269 switch (old_state) { 270 case NVME_CTRL_CONNECTING: 271 changed = true; 272 /* FALLTHRU */ 273 default: 274 break; 275 } 276 break; 277 case NVME_CTRL_LIVE: 278 switch (old_state) { 279 case NVME_CTRL_NEW: 280 case NVME_CTRL_RESETTING: 281 case NVME_CTRL_CONNECTING: 282 changed = true; 283 /* FALLTHRU */ 284 default: 285 break; 286 } 287 break; 288 case NVME_CTRL_RESETTING: 289 switch (old_state) { 290 case NVME_CTRL_NEW: 291 case NVME_CTRL_LIVE: 292 case NVME_CTRL_ADMIN_ONLY: 293 changed = true; 294 /* FALLTHRU */ 295 default: 296 break; 297 } 298 break; 299 case NVME_CTRL_CONNECTING: 300 switch (old_state) { 301 case NVME_CTRL_NEW: 302 case NVME_CTRL_RESETTING: 303 changed = true; 304 /* FALLTHRU */ 305 default: 306 break; 307 } 308 break; 309 case NVME_CTRL_DELETING: 310 switch (old_state) { 311 case NVME_CTRL_LIVE: 312 case NVME_CTRL_ADMIN_ONLY: 313 case NVME_CTRL_RESETTING: 314 case NVME_CTRL_CONNECTING: 315 changed = true; 316 /* FALLTHRU */ 317 default: 318 break; 319 } 320 break; 321 case NVME_CTRL_DEAD: 322 switch (old_state) { 323 case NVME_CTRL_DELETING: 324 changed = true; 325 /* FALLTHRU */ 326 default: 327 break; 328 } 329 break; 330 default: 331 break; 332 } 333 334 if (changed) 335 ctrl->state = new_state; 336 337 spin_unlock_irqrestore(&ctrl->lock, flags); 338 if (changed && ctrl->state == NVME_CTRL_LIVE) 339 nvme_kick_requeue_lists(ctrl); 340 return changed; 341 } 342 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state); 343 344 static void nvme_free_ns_head(struct kref *ref) 345 { 346 struct nvme_ns_head *head = 347 container_of(ref, struct nvme_ns_head, ref); 348 349 nvme_mpath_remove_disk(head); 350 ida_simple_remove(&head->subsys->ns_ida, head->instance); 351 list_del_init(&head->entry); 352 cleanup_srcu_struct(&head->srcu); 353 kfree(head); 354 } 355 356 static void nvme_put_ns_head(struct nvme_ns_head *head) 357 { 358 kref_put(&head->ref, nvme_free_ns_head); 359 } 360 361 static void nvme_free_ns(struct kref *kref) 362 { 363 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref); 364 365 if (ns->ndev) 366 nvme_nvm_unregister(ns); 367 368 put_disk(ns->disk); 369 nvme_put_ns_head(ns->head); 370 nvme_put_ctrl(ns->ctrl); 371 kfree(ns); 372 } 373 374 static void nvme_put_ns(struct nvme_ns *ns) 375 { 376 kref_put(&ns->kref, nvme_free_ns); 377 } 378 379 static inline void nvme_clear_nvme_request(struct request *req) 380 { 381 if (!(req->rq_flags & RQF_DONTPREP)) { 382 nvme_req(req)->retries = 0; 383 nvme_req(req)->flags = 0; 384 req->rq_flags |= RQF_DONTPREP; 385 } 386 } 387 388 struct request *nvme_alloc_request(struct request_queue *q, 389 struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid) 390 { 391 unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN; 392 struct request *req; 393 394 if (qid == NVME_QID_ANY) { 395 req = blk_mq_alloc_request(q, op, flags); 396 } else { 397 req = blk_mq_alloc_request_hctx(q, op, flags, 398 qid ? qid - 1 : 0); 399 } 400 if (IS_ERR(req)) 401 return req; 402 403 req->cmd_flags |= REQ_FAILFAST_DRIVER; 404 nvme_clear_nvme_request(req); 405 nvme_req(req)->cmd = cmd; 406 407 return req; 408 } 409 EXPORT_SYMBOL_GPL(nvme_alloc_request); 410 411 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable) 412 { 413 struct nvme_command c; 414 415 memset(&c, 0, sizeof(c)); 416 417 c.directive.opcode = nvme_admin_directive_send; 418 c.directive.nsid = cpu_to_le32(NVME_NSID_ALL); 419 c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE; 420 c.directive.dtype = NVME_DIR_IDENTIFY; 421 c.directive.tdtype = NVME_DIR_STREAMS; 422 c.directive.endir = enable ? NVME_DIR_ENDIR : 0; 423 424 return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0); 425 } 426 427 static int nvme_disable_streams(struct nvme_ctrl *ctrl) 428 { 429 return nvme_toggle_streams(ctrl, false); 430 } 431 432 static int nvme_enable_streams(struct nvme_ctrl *ctrl) 433 { 434 return nvme_toggle_streams(ctrl, true); 435 } 436 437 static int nvme_get_stream_params(struct nvme_ctrl *ctrl, 438 struct streams_directive_params *s, u32 nsid) 439 { 440 struct nvme_command c; 441 442 memset(&c, 0, sizeof(c)); 443 memset(s, 0, sizeof(*s)); 444 445 c.directive.opcode = nvme_admin_directive_recv; 446 c.directive.nsid = cpu_to_le32(nsid); 447 c.directive.numd = cpu_to_le32((sizeof(*s) >> 2) - 1); 448 c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM; 449 c.directive.dtype = NVME_DIR_STREAMS; 450 451 return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s)); 452 } 453 454 static int nvme_configure_directives(struct nvme_ctrl *ctrl) 455 { 456 struct streams_directive_params s; 457 int ret; 458 459 if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES)) 460 return 0; 461 if (!streams) 462 return 0; 463 464 ret = nvme_enable_streams(ctrl); 465 if (ret) 466 return ret; 467 468 ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL); 469 if (ret) 470 return ret; 471 472 ctrl->nssa = le16_to_cpu(s.nssa); 473 if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) { 474 dev_info(ctrl->device, "too few streams (%u) available\n", 475 ctrl->nssa); 476 nvme_disable_streams(ctrl); 477 return 0; 478 } 479 480 ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1); 481 dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams); 482 return 0; 483 } 484 485 /* 486 * Check if 'req' has a write hint associated with it. If it does, assign 487 * a valid namespace stream to the write. 488 */ 489 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl, 490 struct request *req, u16 *control, 491 u32 *dsmgmt) 492 { 493 enum rw_hint streamid = req->write_hint; 494 495 if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE) 496 streamid = 0; 497 else { 498 streamid--; 499 if (WARN_ON_ONCE(streamid > ctrl->nr_streams)) 500 return; 501 502 *control |= NVME_RW_DTYPE_STREAMS; 503 *dsmgmt |= streamid << 16; 504 } 505 506 if (streamid < ARRAY_SIZE(req->q->write_hints)) 507 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9; 508 } 509 510 static inline void nvme_setup_flush(struct nvme_ns *ns, 511 struct nvme_command *cmnd) 512 { 513 memset(cmnd, 0, sizeof(*cmnd)); 514 cmnd->common.opcode = nvme_cmd_flush; 515 cmnd->common.nsid = cpu_to_le32(ns->head->ns_id); 516 } 517 518 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req, 519 struct nvme_command *cmnd) 520 { 521 unsigned short segments = blk_rq_nr_discard_segments(req), n = 0; 522 struct nvme_dsm_range *range; 523 struct bio *bio; 524 525 range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC); 526 if (!range) 527 return BLK_STS_RESOURCE; 528 529 __rq_for_each_bio(bio, req) { 530 u64 slba = nvme_block_nr(ns, bio->bi_iter.bi_sector); 531 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift; 532 533 if (n < segments) { 534 range[n].cattr = cpu_to_le32(0); 535 range[n].nlb = cpu_to_le32(nlb); 536 range[n].slba = cpu_to_le64(slba); 537 } 538 n++; 539 } 540 541 if (WARN_ON_ONCE(n != segments)) { 542 kfree(range); 543 return BLK_STS_IOERR; 544 } 545 546 memset(cmnd, 0, sizeof(*cmnd)); 547 cmnd->dsm.opcode = nvme_cmd_dsm; 548 cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id); 549 cmnd->dsm.nr = cpu_to_le32(segments - 1); 550 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD); 551 552 req->special_vec.bv_page = virt_to_page(range); 553 req->special_vec.bv_offset = offset_in_page(range); 554 req->special_vec.bv_len = sizeof(*range) * segments; 555 req->rq_flags |= RQF_SPECIAL_PAYLOAD; 556 557 return BLK_STS_OK; 558 } 559 560 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns, 561 struct request *req, struct nvme_command *cmnd) 562 { 563 struct nvme_ctrl *ctrl = ns->ctrl; 564 u16 control = 0; 565 u32 dsmgmt = 0; 566 567 if (req->cmd_flags & REQ_FUA) 568 control |= NVME_RW_FUA; 569 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD)) 570 control |= NVME_RW_LR; 571 572 if (req->cmd_flags & REQ_RAHEAD) 573 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; 574 575 memset(cmnd, 0, sizeof(*cmnd)); 576 cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read); 577 cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id); 578 cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req))); 579 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); 580 581 if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams) 582 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt); 583 584 if (ns->ms) { 585 /* 586 * If formated with metadata, the block layer always provides a 587 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled. Else 588 * we enable the PRACT bit for protection information or set the 589 * namespace capacity to zero to prevent any I/O. 590 */ 591 if (!blk_integrity_rq(req)) { 592 if (WARN_ON_ONCE(!nvme_ns_has_pi(ns))) 593 return BLK_STS_NOTSUPP; 594 control |= NVME_RW_PRINFO_PRACT; 595 } 596 597 switch (ns->pi_type) { 598 case NVME_NS_DPS_PI_TYPE3: 599 control |= NVME_RW_PRINFO_PRCHK_GUARD; 600 break; 601 case NVME_NS_DPS_PI_TYPE1: 602 case NVME_NS_DPS_PI_TYPE2: 603 control |= NVME_RW_PRINFO_PRCHK_GUARD | 604 NVME_RW_PRINFO_PRCHK_REF; 605 cmnd->rw.reftag = cpu_to_le32( 606 nvme_block_nr(ns, blk_rq_pos(req))); 607 break; 608 } 609 } 610 611 cmnd->rw.control = cpu_to_le16(control); 612 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); 613 return 0; 614 } 615 616 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req, 617 struct nvme_command *cmd) 618 { 619 blk_status_t ret = BLK_STS_OK; 620 621 nvme_clear_nvme_request(req); 622 623 switch (req_op(req)) { 624 case REQ_OP_DRV_IN: 625 case REQ_OP_DRV_OUT: 626 memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd)); 627 break; 628 case REQ_OP_FLUSH: 629 nvme_setup_flush(ns, cmd); 630 break; 631 case REQ_OP_WRITE_ZEROES: 632 /* currently only aliased to deallocate for a few ctrls: */ 633 case REQ_OP_DISCARD: 634 ret = nvme_setup_discard(ns, req, cmd); 635 break; 636 case REQ_OP_READ: 637 case REQ_OP_WRITE: 638 ret = nvme_setup_rw(ns, req, cmd); 639 break; 640 default: 641 WARN_ON_ONCE(1); 642 return BLK_STS_IOERR; 643 } 644 645 cmd->common.command_id = req->tag; 646 if (ns) 647 trace_nvme_setup_nvm_cmd(req->q->id, cmd); 648 else 649 trace_nvme_setup_admin_cmd(cmd); 650 return ret; 651 } 652 EXPORT_SYMBOL_GPL(nvme_setup_cmd); 653 654 /* 655 * Returns 0 on success. If the result is negative, it's a Linux error code; 656 * if the result is positive, it's an NVM Express status code 657 */ 658 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 659 union nvme_result *result, void *buffer, unsigned bufflen, 660 unsigned timeout, int qid, int at_head, 661 blk_mq_req_flags_t flags) 662 { 663 struct request *req; 664 int ret; 665 666 req = nvme_alloc_request(q, cmd, flags, qid); 667 if (IS_ERR(req)) 668 return PTR_ERR(req); 669 670 req->timeout = timeout ? timeout : ADMIN_TIMEOUT; 671 672 if (buffer && bufflen) { 673 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL); 674 if (ret) 675 goto out; 676 } 677 678 blk_execute_rq(req->q, NULL, req, at_head); 679 if (result) 680 *result = nvme_req(req)->result; 681 if (nvme_req(req)->flags & NVME_REQ_CANCELLED) 682 ret = -EINTR; 683 else 684 ret = nvme_req(req)->status; 685 out: 686 blk_mq_free_request(req); 687 return ret; 688 } 689 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd); 690 691 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, 692 void *buffer, unsigned bufflen) 693 { 694 return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0, 695 NVME_QID_ANY, 0, 0); 696 } 697 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd); 698 699 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf, 700 unsigned len, u32 seed, bool write) 701 { 702 struct bio_integrity_payload *bip; 703 int ret = -ENOMEM; 704 void *buf; 705 706 buf = kmalloc(len, GFP_KERNEL); 707 if (!buf) 708 goto out; 709 710 ret = -EFAULT; 711 if (write && copy_from_user(buf, ubuf, len)) 712 goto out_free_meta; 713 714 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1); 715 if (IS_ERR(bip)) { 716 ret = PTR_ERR(bip); 717 goto out_free_meta; 718 } 719 720 bip->bip_iter.bi_size = len; 721 bip->bip_iter.bi_sector = seed; 722 ret = bio_integrity_add_page(bio, virt_to_page(buf), len, 723 offset_in_page(buf)); 724 if (ret == len) 725 return buf; 726 ret = -ENOMEM; 727 out_free_meta: 728 kfree(buf); 729 out: 730 return ERR_PTR(ret); 731 } 732 733 static int nvme_submit_user_cmd(struct request_queue *q, 734 struct nvme_command *cmd, void __user *ubuffer, 735 unsigned bufflen, void __user *meta_buffer, unsigned meta_len, 736 u32 meta_seed, u32 *result, unsigned timeout) 737 { 738 bool write = nvme_is_write(cmd); 739 struct nvme_ns *ns = q->queuedata; 740 struct gendisk *disk = ns ? ns->disk : NULL; 741 struct request *req; 742 struct bio *bio = NULL; 743 void *meta = NULL; 744 int ret; 745 746 req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY); 747 if (IS_ERR(req)) 748 return PTR_ERR(req); 749 750 req->timeout = timeout ? timeout : ADMIN_TIMEOUT; 751 nvme_req(req)->flags |= NVME_REQ_USERCMD; 752 753 if (ubuffer && bufflen) { 754 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen, 755 GFP_KERNEL); 756 if (ret) 757 goto out; 758 bio = req->bio; 759 bio->bi_disk = disk; 760 if (disk && meta_buffer && meta_len) { 761 meta = nvme_add_user_metadata(bio, meta_buffer, meta_len, 762 meta_seed, write); 763 if (IS_ERR(meta)) { 764 ret = PTR_ERR(meta); 765 goto out_unmap; 766 } 767 } 768 } 769 770 blk_execute_rq(req->q, disk, req, 0); 771 if (nvme_req(req)->flags & NVME_REQ_CANCELLED) 772 ret = -EINTR; 773 else 774 ret = nvme_req(req)->status; 775 if (result) 776 *result = le32_to_cpu(nvme_req(req)->result.u32); 777 if (meta && !ret && !write) { 778 if (copy_to_user(meta_buffer, meta, meta_len)) 779 ret = -EFAULT; 780 } 781 kfree(meta); 782 out_unmap: 783 if (bio) 784 blk_rq_unmap_user(bio); 785 out: 786 blk_mq_free_request(req); 787 return ret; 788 } 789 790 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status) 791 { 792 struct nvme_ctrl *ctrl = rq->end_io_data; 793 794 blk_mq_free_request(rq); 795 796 if (status) { 797 dev_err(ctrl->device, 798 "failed nvme_keep_alive_end_io error=%d\n", 799 status); 800 return; 801 } 802 803 schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ); 804 } 805 806 static int nvme_keep_alive(struct nvme_ctrl *ctrl) 807 { 808 struct request *rq; 809 810 rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED, 811 NVME_QID_ANY); 812 if (IS_ERR(rq)) 813 return PTR_ERR(rq); 814 815 rq->timeout = ctrl->kato * HZ; 816 rq->end_io_data = ctrl; 817 818 blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io); 819 820 return 0; 821 } 822 823 static void nvme_keep_alive_work(struct work_struct *work) 824 { 825 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work), 826 struct nvme_ctrl, ka_work); 827 828 if (nvme_keep_alive(ctrl)) { 829 /* allocation failure, reset the controller */ 830 dev_err(ctrl->device, "keep-alive failed\n"); 831 nvme_reset_ctrl(ctrl); 832 return; 833 } 834 } 835 836 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl) 837 { 838 if (unlikely(ctrl->kato == 0)) 839 return; 840 841 INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work); 842 memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd)); 843 ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive; 844 schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ); 845 } 846 847 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl) 848 { 849 if (unlikely(ctrl->kato == 0)) 850 return; 851 852 cancel_delayed_work_sync(&ctrl->ka_work); 853 } 854 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive); 855 856 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id) 857 { 858 struct nvme_command c = { }; 859 int error; 860 861 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */ 862 c.identify.opcode = nvme_admin_identify; 863 c.identify.cns = NVME_ID_CNS_CTRL; 864 865 *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL); 866 if (!*id) 867 return -ENOMEM; 868 869 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id, 870 sizeof(struct nvme_id_ctrl)); 871 if (error) 872 kfree(*id); 873 return error; 874 } 875 876 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid, 877 struct nvme_ns_ids *ids) 878 { 879 struct nvme_command c = { }; 880 int status; 881 void *data; 882 int pos; 883 int len; 884 885 c.identify.opcode = nvme_admin_identify; 886 c.identify.nsid = cpu_to_le32(nsid); 887 c.identify.cns = NVME_ID_CNS_NS_DESC_LIST; 888 889 data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL); 890 if (!data) 891 return -ENOMEM; 892 893 status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data, 894 NVME_IDENTIFY_DATA_SIZE); 895 if (status) 896 goto free_data; 897 898 for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) { 899 struct nvme_ns_id_desc *cur = data + pos; 900 901 if (cur->nidl == 0) 902 break; 903 904 switch (cur->nidt) { 905 case NVME_NIDT_EUI64: 906 if (cur->nidl != NVME_NIDT_EUI64_LEN) { 907 dev_warn(ctrl->device, 908 "ctrl returned bogus length: %d for NVME_NIDT_EUI64\n", 909 cur->nidl); 910 goto free_data; 911 } 912 len = NVME_NIDT_EUI64_LEN; 913 memcpy(ids->eui64, data + pos + sizeof(*cur), len); 914 break; 915 case NVME_NIDT_NGUID: 916 if (cur->nidl != NVME_NIDT_NGUID_LEN) { 917 dev_warn(ctrl->device, 918 "ctrl returned bogus length: %d for NVME_NIDT_NGUID\n", 919 cur->nidl); 920 goto free_data; 921 } 922 len = NVME_NIDT_NGUID_LEN; 923 memcpy(ids->nguid, data + pos + sizeof(*cur), len); 924 break; 925 case NVME_NIDT_UUID: 926 if (cur->nidl != NVME_NIDT_UUID_LEN) { 927 dev_warn(ctrl->device, 928 "ctrl returned bogus length: %d for NVME_NIDT_UUID\n", 929 cur->nidl); 930 goto free_data; 931 } 932 len = NVME_NIDT_UUID_LEN; 933 uuid_copy(&ids->uuid, data + pos + sizeof(*cur)); 934 break; 935 default: 936 /* Skip unnkown types */ 937 len = cur->nidl; 938 break; 939 } 940 941 len += sizeof(*cur); 942 } 943 free_data: 944 kfree(data); 945 return status; 946 } 947 948 static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list) 949 { 950 struct nvme_command c = { }; 951 952 c.identify.opcode = nvme_admin_identify; 953 c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST; 954 c.identify.nsid = cpu_to_le32(nsid); 955 return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 956 NVME_IDENTIFY_DATA_SIZE); 957 } 958 959 static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl, 960 unsigned nsid) 961 { 962 struct nvme_id_ns *id; 963 struct nvme_command c = { }; 964 int error; 965 966 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */ 967 c.identify.opcode = nvme_admin_identify; 968 c.identify.nsid = cpu_to_le32(nsid); 969 c.identify.cns = NVME_ID_CNS_NS; 970 971 id = kmalloc(sizeof(*id), GFP_KERNEL); 972 if (!id) 973 return NULL; 974 975 error = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id)); 976 if (error) { 977 dev_warn(ctrl->device, "Identify namespace failed\n"); 978 kfree(id); 979 return NULL; 980 } 981 982 return id; 983 } 984 985 static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, 986 void *buffer, size_t buflen, u32 *result) 987 { 988 struct nvme_command c; 989 union nvme_result res; 990 int ret; 991 992 memset(&c, 0, sizeof(c)); 993 c.features.opcode = nvme_admin_set_features; 994 c.features.fid = cpu_to_le32(fid); 995 c.features.dword11 = cpu_to_le32(dword11); 996 997 ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res, 998 buffer, buflen, 0, NVME_QID_ANY, 0, 0); 999 if (ret >= 0 && result) 1000 *result = le32_to_cpu(res.u32); 1001 return ret; 1002 } 1003 1004 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count) 1005 { 1006 u32 q_count = (*count - 1) | ((*count - 1) << 16); 1007 u32 result; 1008 int status, nr_io_queues; 1009 1010 status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0, 1011 &result); 1012 if (status < 0) 1013 return status; 1014 1015 /* 1016 * Degraded controllers might return an error when setting the queue 1017 * count. We still want to be able to bring them online and offer 1018 * access to the admin queue, as that might be only way to fix them up. 1019 */ 1020 if (status > 0) { 1021 dev_err(ctrl->device, "Could not set queue count (%d)\n", status); 1022 *count = 0; 1023 } else { 1024 nr_io_queues = min(result & 0xffff, result >> 16) + 1; 1025 *count = min(*count, nr_io_queues); 1026 } 1027 1028 return 0; 1029 } 1030 EXPORT_SYMBOL_GPL(nvme_set_queue_count); 1031 1032 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 1033 { 1034 struct nvme_user_io io; 1035 struct nvme_command c; 1036 unsigned length, meta_len; 1037 void __user *metadata; 1038 1039 if (copy_from_user(&io, uio, sizeof(io))) 1040 return -EFAULT; 1041 if (io.flags) 1042 return -EINVAL; 1043 1044 switch (io.opcode) { 1045 case nvme_cmd_write: 1046 case nvme_cmd_read: 1047 case nvme_cmd_compare: 1048 break; 1049 default: 1050 return -EINVAL; 1051 } 1052 1053 length = (io.nblocks + 1) << ns->lba_shift; 1054 meta_len = (io.nblocks + 1) * ns->ms; 1055 metadata = (void __user *)(uintptr_t)io.metadata; 1056 1057 if (ns->ext) { 1058 length += meta_len; 1059 meta_len = 0; 1060 } else if (meta_len) { 1061 if ((io.metadata & 3) || !io.metadata) 1062 return -EINVAL; 1063 } 1064 1065 memset(&c, 0, sizeof(c)); 1066 c.rw.opcode = io.opcode; 1067 c.rw.flags = io.flags; 1068 c.rw.nsid = cpu_to_le32(ns->head->ns_id); 1069 c.rw.slba = cpu_to_le64(io.slba); 1070 c.rw.length = cpu_to_le16(io.nblocks); 1071 c.rw.control = cpu_to_le16(io.control); 1072 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 1073 c.rw.reftag = cpu_to_le32(io.reftag); 1074 c.rw.apptag = cpu_to_le16(io.apptag); 1075 c.rw.appmask = cpu_to_le16(io.appmask); 1076 1077 return nvme_submit_user_cmd(ns->queue, &c, 1078 (void __user *)(uintptr_t)io.addr, length, 1079 metadata, meta_len, io.slba, NULL, 0); 1080 } 1081 1082 static u32 nvme_known_admin_effects(u8 opcode) 1083 { 1084 switch (opcode) { 1085 case nvme_admin_format_nvm: 1086 return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC | 1087 NVME_CMD_EFFECTS_CSE_MASK; 1088 case nvme_admin_sanitize_nvm: 1089 return NVME_CMD_EFFECTS_CSE_MASK; 1090 default: 1091 break; 1092 } 1093 return 0; 1094 } 1095 1096 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 1097 u8 opcode) 1098 { 1099 u32 effects = 0; 1100 1101 if (ns) { 1102 if (ctrl->effects) 1103 effects = le32_to_cpu(ctrl->effects->iocs[opcode]); 1104 if (effects & ~NVME_CMD_EFFECTS_CSUPP) 1105 dev_warn(ctrl->device, 1106 "IO command:%02x has unhandled effects:%08x\n", 1107 opcode, effects); 1108 return 0; 1109 } 1110 1111 if (ctrl->effects) 1112 effects = le32_to_cpu(ctrl->effects->acs[opcode]); 1113 else 1114 effects = nvme_known_admin_effects(opcode); 1115 1116 /* 1117 * For simplicity, IO to all namespaces is quiesced even if the command 1118 * effects say only one namespace is affected. 1119 */ 1120 if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) { 1121 nvme_start_freeze(ctrl); 1122 nvme_wait_freeze(ctrl); 1123 } 1124 return effects; 1125 } 1126 1127 static void nvme_update_formats(struct nvme_ctrl *ctrl) 1128 { 1129 struct nvme_ns *ns, *next; 1130 LIST_HEAD(rm_list); 1131 1132 down_write(&ctrl->namespaces_rwsem); 1133 list_for_each_entry(ns, &ctrl->namespaces, list) { 1134 if (ns->disk && nvme_revalidate_disk(ns->disk)) { 1135 list_move_tail(&ns->list, &rm_list); 1136 } 1137 } 1138 up_write(&ctrl->namespaces_rwsem); 1139 1140 list_for_each_entry_safe(ns, next, &rm_list, list) 1141 nvme_ns_remove(ns); 1142 } 1143 1144 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects) 1145 { 1146 /* 1147 * Revalidate LBA changes prior to unfreezing. This is necessary to 1148 * prevent memory corruption if a logical block size was changed by 1149 * this command. 1150 */ 1151 if (effects & NVME_CMD_EFFECTS_LBCC) 1152 nvme_update_formats(ctrl); 1153 if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) 1154 nvme_unfreeze(ctrl); 1155 if (effects & NVME_CMD_EFFECTS_CCC) 1156 nvme_init_identify(ctrl); 1157 if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) 1158 nvme_queue_scan(ctrl); 1159 } 1160 1161 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 1162 struct nvme_passthru_cmd __user *ucmd) 1163 { 1164 struct nvme_passthru_cmd cmd; 1165 struct nvme_command c; 1166 unsigned timeout = 0; 1167 u32 effects; 1168 int status; 1169 1170 if (!capable(CAP_SYS_ADMIN)) 1171 return -EACCES; 1172 if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 1173 return -EFAULT; 1174 if (cmd.flags) 1175 return -EINVAL; 1176 1177 memset(&c, 0, sizeof(c)); 1178 c.common.opcode = cmd.opcode; 1179 c.common.flags = cmd.flags; 1180 c.common.nsid = cpu_to_le32(cmd.nsid); 1181 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 1182 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 1183 c.common.cdw10[0] = cpu_to_le32(cmd.cdw10); 1184 c.common.cdw10[1] = cpu_to_le32(cmd.cdw11); 1185 c.common.cdw10[2] = cpu_to_le32(cmd.cdw12); 1186 c.common.cdw10[3] = cpu_to_le32(cmd.cdw13); 1187 c.common.cdw10[4] = cpu_to_le32(cmd.cdw14); 1188 c.common.cdw10[5] = cpu_to_le32(cmd.cdw15); 1189 1190 if (cmd.timeout_ms) 1191 timeout = msecs_to_jiffies(cmd.timeout_ms); 1192 1193 effects = nvme_passthru_start(ctrl, ns, cmd.opcode); 1194 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, 1195 (void __user *)(uintptr_t)cmd.addr, cmd.data_len, 1196 (void __user *)(uintptr_t)cmd.metadata, cmd.metadata, 1197 0, &cmd.result, timeout); 1198 nvme_passthru_end(ctrl, effects); 1199 1200 if (status >= 0) { 1201 if (put_user(cmd.result, &ucmd->result)) 1202 return -EFAULT; 1203 } 1204 1205 return status; 1206 } 1207 1208 /* 1209 * Issue ioctl requests on the first available path. Note that unlike normal 1210 * block layer requests we will not retry failed request on another controller. 1211 */ 1212 static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk, 1213 struct nvme_ns_head **head, int *srcu_idx) 1214 { 1215 #ifdef CONFIG_NVME_MULTIPATH 1216 if (disk->fops == &nvme_ns_head_ops) { 1217 *head = disk->private_data; 1218 *srcu_idx = srcu_read_lock(&(*head)->srcu); 1219 return nvme_find_path(*head); 1220 } 1221 #endif 1222 *head = NULL; 1223 *srcu_idx = -1; 1224 return disk->private_data; 1225 } 1226 1227 static void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx) 1228 { 1229 if (head) 1230 srcu_read_unlock(&head->srcu, idx); 1231 } 1232 1233 static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned cmd, unsigned long arg) 1234 { 1235 switch (cmd) { 1236 case NVME_IOCTL_ID: 1237 force_successful_syscall_return(); 1238 return ns->head->ns_id; 1239 case NVME_IOCTL_ADMIN_CMD: 1240 return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg); 1241 case NVME_IOCTL_IO_CMD: 1242 return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg); 1243 case NVME_IOCTL_SUBMIT_IO: 1244 return nvme_submit_io(ns, (void __user *)arg); 1245 default: 1246 #ifdef CONFIG_NVM 1247 if (ns->ndev) 1248 return nvme_nvm_ioctl(ns, cmd, arg); 1249 #endif 1250 if (is_sed_ioctl(cmd)) 1251 return sed_ioctl(ns->ctrl->opal_dev, cmd, 1252 (void __user *) arg); 1253 return -ENOTTY; 1254 } 1255 } 1256 1257 static int nvme_ioctl(struct block_device *bdev, fmode_t mode, 1258 unsigned int cmd, unsigned long arg) 1259 { 1260 struct nvme_ns_head *head = NULL; 1261 struct nvme_ns *ns; 1262 int srcu_idx, ret; 1263 1264 ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx); 1265 if (unlikely(!ns)) 1266 ret = -EWOULDBLOCK; 1267 else 1268 ret = nvme_ns_ioctl(ns, cmd, arg); 1269 nvme_put_ns_from_disk(head, srcu_idx); 1270 return ret; 1271 } 1272 1273 static int nvme_open(struct block_device *bdev, fmode_t mode) 1274 { 1275 struct nvme_ns *ns = bdev->bd_disk->private_data; 1276 1277 #ifdef CONFIG_NVME_MULTIPATH 1278 /* should never be called due to GENHD_FL_HIDDEN */ 1279 if (WARN_ON_ONCE(ns->head->disk)) 1280 goto fail; 1281 #endif 1282 if (!kref_get_unless_zero(&ns->kref)) 1283 goto fail; 1284 if (!try_module_get(ns->ctrl->ops->module)) 1285 goto fail_put_ns; 1286 1287 return 0; 1288 1289 fail_put_ns: 1290 nvme_put_ns(ns); 1291 fail: 1292 return -ENXIO; 1293 } 1294 1295 static void nvme_release(struct gendisk *disk, fmode_t mode) 1296 { 1297 struct nvme_ns *ns = disk->private_data; 1298 1299 module_put(ns->ctrl->ops->module); 1300 nvme_put_ns(ns); 1301 } 1302 1303 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo) 1304 { 1305 /* some standard values */ 1306 geo->heads = 1 << 6; 1307 geo->sectors = 1 << 5; 1308 geo->cylinders = get_capacity(bdev->bd_disk) >> 11; 1309 return 0; 1310 } 1311 1312 #ifdef CONFIG_BLK_DEV_INTEGRITY 1313 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type) 1314 { 1315 struct blk_integrity integrity; 1316 1317 memset(&integrity, 0, sizeof(integrity)); 1318 switch (pi_type) { 1319 case NVME_NS_DPS_PI_TYPE3: 1320 integrity.profile = &t10_pi_type3_crc; 1321 integrity.tag_size = sizeof(u16) + sizeof(u32); 1322 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE; 1323 break; 1324 case NVME_NS_DPS_PI_TYPE1: 1325 case NVME_NS_DPS_PI_TYPE2: 1326 integrity.profile = &t10_pi_type1_crc; 1327 integrity.tag_size = sizeof(u16); 1328 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE; 1329 break; 1330 default: 1331 integrity.profile = NULL; 1332 break; 1333 } 1334 integrity.tuple_size = ms; 1335 blk_integrity_register(disk, &integrity); 1336 blk_queue_max_integrity_segments(disk->queue, 1); 1337 } 1338 #else 1339 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type) 1340 { 1341 } 1342 #endif /* CONFIG_BLK_DEV_INTEGRITY */ 1343 1344 static void nvme_set_chunk_size(struct nvme_ns *ns) 1345 { 1346 u32 chunk_size = (((u32)ns->noiob) << (ns->lba_shift - 9)); 1347 blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size)); 1348 } 1349 1350 static void nvme_config_discard(struct nvme_ctrl *ctrl, 1351 unsigned stream_alignment, struct request_queue *queue) 1352 { 1353 u32 size = queue_logical_block_size(queue); 1354 1355 if (stream_alignment) 1356 size *= stream_alignment; 1357 1358 BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) < 1359 NVME_DSM_MAX_RANGES); 1360 1361 queue->limits.discard_alignment = 0; 1362 queue->limits.discard_granularity = size; 1363 1364 blk_queue_max_discard_sectors(queue, UINT_MAX); 1365 blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES); 1366 blk_queue_flag_set(QUEUE_FLAG_DISCARD, queue); 1367 1368 if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES) 1369 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX); 1370 } 1371 1372 static void nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid, 1373 struct nvme_id_ns *id, struct nvme_ns_ids *ids) 1374 { 1375 memset(ids, 0, sizeof(*ids)); 1376 1377 if (ctrl->vs >= NVME_VS(1, 1, 0)) 1378 memcpy(ids->eui64, id->eui64, sizeof(id->eui64)); 1379 if (ctrl->vs >= NVME_VS(1, 2, 0)) 1380 memcpy(ids->nguid, id->nguid, sizeof(id->nguid)); 1381 if (ctrl->vs >= NVME_VS(1, 3, 0)) { 1382 /* Don't treat error as fatal we potentially 1383 * already have a NGUID or EUI-64 1384 */ 1385 if (nvme_identify_ns_descs(ctrl, nsid, ids)) 1386 dev_warn(ctrl->device, 1387 "%s: Identify Descriptors failed\n", __func__); 1388 } 1389 } 1390 1391 static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids) 1392 { 1393 return !uuid_is_null(&ids->uuid) || 1394 memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) || 1395 memchr_inv(ids->eui64, 0, sizeof(ids->eui64)); 1396 } 1397 1398 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b) 1399 { 1400 return uuid_equal(&a->uuid, &b->uuid) && 1401 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 && 1402 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0; 1403 } 1404 1405 static void nvme_update_disk_info(struct gendisk *disk, 1406 struct nvme_ns *ns, struct nvme_id_ns *id) 1407 { 1408 sector_t capacity = le64_to_cpup(&id->nsze) << (ns->lba_shift - 9); 1409 unsigned short bs = 1 << ns->lba_shift; 1410 unsigned stream_alignment = 0; 1411 1412 if (ns->ctrl->nr_streams && ns->sws && ns->sgs) 1413 stream_alignment = ns->sws * ns->sgs; 1414 1415 blk_mq_freeze_queue(disk->queue); 1416 blk_integrity_unregister(disk); 1417 1418 blk_queue_logical_block_size(disk->queue, bs); 1419 blk_queue_physical_block_size(disk->queue, bs); 1420 blk_queue_io_min(disk->queue, bs); 1421 1422 if (ns->ms && !ns->ext && 1423 (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED)) 1424 nvme_init_integrity(disk, ns->ms, ns->pi_type); 1425 if (ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk)) 1426 capacity = 0; 1427 set_capacity(disk, capacity); 1428 1429 if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM) 1430 nvme_config_discard(ns->ctrl, stream_alignment, disk->queue); 1431 blk_mq_unfreeze_queue(disk->queue); 1432 } 1433 1434 static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) 1435 { 1436 struct nvme_ns *ns = disk->private_data; 1437 1438 /* 1439 * If identify namespace failed, use default 512 byte block size so 1440 * block layer can use before failing read/write for 0 capacity. 1441 */ 1442 ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds; 1443 if (ns->lba_shift == 0) 1444 ns->lba_shift = 9; 1445 ns->noiob = le16_to_cpu(id->noiob); 1446 ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT); 1447 ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms); 1448 /* the PI implementation requires metadata equal t10 pi tuple size */ 1449 if (ns->ms == sizeof(struct t10_pi_tuple)) 1450 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK; 1451 else 1452 ns->pi_type = 0; 1453 1454 if (ns->noiob) 1455 nvme_set_chunk_size(ns); 1456 nvme_update_disk_info(disk, ns, id); 1457 if (ns->ndev) 1458 nvme_nvm_update_nvm_info(ns); 1459 #ifdef CONFIG_NVME_MULTIPATH 1460 if (ns->head->disk) 1461 nvme_update_disk_info(ns->head->disk, ns, id); 1462 #endif 1463 } 1464 1465 static int nvme_revalidate_disk(struct gendisk *disk) 1466 { 1467 struct nvme_ns *ns = disk->private_data; 1468 struct nvme_ctrl *ctrl = ns->ctrl; 1469 struct nvme_id_ns *id; 1470 struct nvme_ns_ids ids; 1471 int ret = 0; 1472 1473 if (test_bit(NVME_NS_DEAD, &ns->flags)) { 1474 set_capacity(disk, 0); 1475 return -ENODEV; 1476 } 1477 1478 id = nvme_identify_ns(ctrl, ns->head->ns_id); 1479 if (!id) 1480 return -ENODEV; 1481 1482 if (id->ncap == 0) { 1483 ret = -ENODEV; 1484 goto out; 1485 } 1486 1487 __nvme_revalidate_disk(disk, id); 1488 nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids); 1489 if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) { 1490 dev_err(ctrl->device, 1491 "identifiers changed for nsid %d\n", ns->head->ns_id); 1492 ret = -ENODEV; 1493 } 1494 1495 out: 1496 kfree(id); 1497 return ret; 1498 } 1499 1500 static char nvme_pr_type(enum pr_type type) 1501 { 1502 switch (type) { 1503 case PR_WRITE_EXCLUSIVE: 1504 return 1; 1505 case PR_EXCLUSIVE_ACCESS: 1506 return 2; 1507 case PR_WRITE_EXCLUSIVE_REG_ONLY: 1508 return 3; 1509 case PR_EXCLUSIVE_ACCESS_REG_ONLY: 1510 return 4; 1511 case PR_WRITE_EXCLUSIVE_ALL_REGS: 1512 return 5; 1513 case PR_EXCLUSIVE_ACCESS_ALL_REGS: 1514 return 6; 1515 default: 1516 return 0; 1517 } 1518 }; 1519 1520 static int nvme_pr_command(struct block_device *bdev, u32 cdw10, 1521 u64 key, u64 sa_key, u8 op) 1522 { 1523 struct nvme_ns_head *head = NULL; 1524 struct nvme_ns *ns; 1525 struct nvme_command c; 1526 int srcu_idx, ret; 1527 u8 data[16] = { 0, }; 1528 1529 ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx); 1530 if (unlikely(!ns)) 1531 return -EWOULDBLOCK; 1532 1533 put_unaligned_le64(key, &data[0]); 1534 put_unaligned_le64(sa_key, &data[8]); 1535 1536 memset(&c, 0, sizeof(c)); 1537 c.common.opcode = op; 1538 c.common.nsid = cpu_to_le32(ns->head->ns_id); 1539 c.common.cdw10[0] = cpu_to_le32(cdw10); 1540 1541 ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16); 1542 nvme_put_ns_from_disk(head, srcu_idx); 1543 return ret; 1544 } 1545 1546 static int nvme_pr_register(struct block_device *bdev, u64 old, 1547 u64 new, unsigned flags) 1548 { 1549 u32 cdw10; 1550 1551 if (flags & ~PR_FL_IGNORE_KEY) 1552 return -EOPNOTSUPP; 1553 1554 cdw10 = old ? 2 : 0; 1555 cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0; 1556 cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */ 1557 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register); 1558 } 1559 1560 static int nvme_pr_reserve(struct block_device *bdev, u64 key, 1561 enum pr_type type, unsigned flags) 1562 { 1563 u32 cdw10; 1564 1565 if (flags & ~PR_FL_IGNORE_KEY) 1566 return -EOPNOTSUPP; 1567 1568 cdw10 = nvme_pr_type(type) << 8; 1569 cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0); 1570 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire); 1571 } 1572 1573 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new, 1574 enum pr_type type, bool abort) 1575 { 1576 u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1; 1577 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire); 1578 } 1579 1580 static int nvme_pr_clear(struct block_device *bdev, u64 key) 1581 { 1582 u32 cdw10 = 1 | (key ? 1 << 3 : 0); 1583 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register); 1584 } 1585 1586 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 1587 { 1588 u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0; 1589 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release); 1590 } 1591 1592 static const struct pr_ops nvme_pr_ops = { 1593 .pr_register = nvme_pr_register, 1594 .pr_reserve = nvme_pr_reserve, 1595 .pr_release = nvme_pr_release, 1596 .pr_preempt = nvme_pr_preempt, 1597 .pr_clear = nvme_pr_clear, 1598 }; 1599 1600 #ifdef CONFIG_BLK_SED_OPAL 1601 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len, 1602 bool send) 1603 { 1604 struct nvme_ctrl *ctrl = data; 1605 struct nvme_command cmd; 1606 1607 memset(&cmd, 0, sizeof(cmd)); 1608 if (send) 1609 cmd.common.opcode = nvme_admin_security_send; 1610 else 1611 cmd.common.opcode = nvme_admin_security_recv; 1612 cmd.common.nsid = 0; 1613 cmd.common.cdw10[0] = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8); 1614 cmd.common.cdw10[1] = cpu_to_le32(len); 1615 1616 return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len, 1617 ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0); 1618 } 1619 EXPORT_SYMBOL_GPL(nvme_sec_submit); 1620 #endif /* CONFIG_BLK_SED_OPAL */ 1621 1622 static const struct block_device_operations nvme_fops = { 1623 .owner = THIS_MODULE, 1624 .ioctl = nvme_ioctl, 1625 .compat_ioctl = nvme_ioctl, 1626 .open = nvme_open, 1627 .release = nvme_release, 1628 .getgeo = nvme_getgeo, 1629 .revalidate_disk= nvme_revalidate_disk, 1630 .pr_ops = &nvme_pr_ops, 1631 }; 1632 1633 #ifdef CONFIG_NVME_MULTIPATH 1634 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode) 1635 { 1636 struct nvme_ns_head *head = bdev->bd_disk->private_data; 1637 1638 if (!kref_get_unless_zero(&head->ref)) 1639 return -ENXIO; 1640 return 0; 1641 } 1642 1643 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode) 1644 { 1645 nvme_put_ns_head(disk->private_data); 1646 } 1647 1648 const struct block_device_operations nvme_ns_head_ops = { 1649 .owner = THIS_MODULE, 1650 .open = nvme_ns_head_open, 1651 .release = nvme_ns_head_release, 1652 .ioctl = nvme_ioctl, 1653 .compat_ioctl = nvme_ioctl, 1654 .getgeo = nvme_getgeo, 1655 .pr_ops = &nvme_pr_ops, 1656 }; 1657 #endif /* CONFIG_NVME_MULTIPATH */ 1658 1659 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled) 1660 { 1661 unsigned long timeout = 1662 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; 1663 u32 csts, bit = enabled ? NVME_CSTS_RDY : 0; 1664 int ret; 1665 1666 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) { 1667 if (csts == ~0) 1668 return -ENODEV; 1669 if ((csts & NVME_CSTS_RDY) == bit) 1670 break; 1671 1672 msleep(100); 1673 if (fatal_signal_pending(current)) 1674 return -EINTR; 1675 if (time_after(jiffies, timeout)) { 1676 dev_err(ctrl->device, 1677 "Device not ready; aborting %s\n", enabled ? 1678 "initialisation" : "reset"); 1679 return -ENODEV; 1680 } 1681 } 1682 1683 return ret; 1684 } 1685 1686 /* 1687 * If the device has been passed off to us in an enabled state, just clear 1688 * the enabled bit. The spec says we should set the 'shutdown notification 1689 * bits', but doing so may cause the device to complete commands to the 1690 * admin queue ... and we don't know what memory that might be pointing at! 1691 */ 1692 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap) 1693 { 1694 int ret; 1695 1696 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK; 1697 ctrl->ctrl_config &= ~NVME_CC_ENABLE; 1698 1699 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config); 1700 if (ret) 1701 return ret; 1702 1703 if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) 1704 msleep(NVME_QUIRK_DELAY_AMOUNT); 1705 1706 return nvme_wait_ready(ctrl, cap, false); 1707 } 1708 EXPORT_SYMBOL_GPL(nvme_disable_ctrl); 1709 1710 int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap) 1711 { 1712 /* 1713 * Default to a 4K page size, with the intention to update this 1714 * path in the future to accomodate architectures with differing 1715 * kernel and IO page sizes. 1716 */ 1717 unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12; 1718 int ret; 1719 1720 if (page_shift < dev_page_min) { 1721 dev_err(ctrl->device, 1722 "Minimum device page size %u too large for host (%u)\n", 1723 1 << dev_page_min, 1 << page_shift); 1724 return -ENODEV; 1725 } 1726 1727 ctrl->page_size = 1 << page_shift; 1728 1729 ctrl->ctrl_config = NVME_CC_CSS_NVM; 1730 ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT; 1731 ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE; 1732 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; 1733 ctrl->ctrl_config |= NVME_CC_ENABLE; 1734 1735 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config); 1736 if (ret) 1737 return ret; 1738 return nvme_wait_ready(ctrl, cap, true); 1739 } 1740 EXPORT_SYMBOL_GPL(nvme_enable_ctrl); 1741 1742 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl) 1743 { 1744 unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ); 1745 u32 csts; 1746 int ret; 1747 1748 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK; 1749 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL; 1750 1751 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config); 1752 if (ret) 1753 return ret; 1754 1755 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) { 1756 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT) 1757 break; 1758 1759 msleep(100); 1760 if (fatal_signal_pending(current)) 1761 return -EINTR; 1762 if (time_after(jiffies, timeout)) { 1763 dev_err(ctrl->device, 1764 "Device shutdown incomplete; abort shutdown\n"); 1765 return -ENODEV; 1766 } 1767 } 1768 1769 return ret; 1770 } 1771 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl); 1772 1773 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl, 1774 struct request_queue *q) 1775 { 1776 bool vwc = false; 1777 1778 if (ctrl->max_hw_sectors) { 1779 u32 max_segments = 1780 (ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1; 1781 1782 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors); 1783 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX)); 1784 } 1785 if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && 1786 is_power_of_2(ctrl->max_hw_sectors)) 1787 blk_queue_chunk_sectors(q, ctrl->max_hw_sectors); 1788 blk_queue_virt_boundary(q, ctrl->page_size - 1); 1789 if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) 1790 vwc = true; 1791 blk_queue_write_cache(q, vwc, vwc); 1792 } 1793 1794 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl) 1795 { 1796 __le64 ts; 1797 int ret; 1798 1799 if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP)) 1800 return 0; 1801 1802 ts = cpu_to_le64(ktime_to_ms(ktime_get_real())); 1803 ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts), 1804 NULL); 1805 if (ret) 1806 dev_warn_once(ctrl->device, 1807 "could not set timestamp (%d)\n", ret); 1808 return ret; 1809 } 1810 1811 static int nvme_configure_apst(struct nvme_ctrl *ctrl) 1812 { 1813 /* 1814 * APST (Autonomous Power State Transition) lets us program a 1815 * table of power state transitions that the controller will 1816 * perform automatically. We configure it with a simple 1817 * heuristic: we are willing to spend at most 2% of the time 1818 * transitioning between power states. Therefore, when running 1819 * in any given state, we will enter the next lower-power 1820 * non-operational state after waiting 50 * (enlat + exlat) 1821 * microseconds, as long as that state's exit latency is under 1822 * the requested maximum latency. 1823 * 1824 * We will not autonomously enter any non-operational state for 1825 * which the total latency exceeds ps_max_latency_us. Users 1826 * can set ps_max_latency_us to zero to turn off APST. 1827 */ 1828 1829 unsigned apste; 1830 struct nvme_feat_auto_pst *table; 1831 u64 max_lat_us = 0; 1832 int max_ps = -1; 1833 int ret; 1834 1835 /* 1836 * If APST isn't supported or if we haven't been initialized yet, 1837 * then don't do anything. 1838 */ 1839 if (!ctrl->apsta) 1840 return 0; 1841 1842 if (ctrl->npss > 31) { 1843 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n"); 1844 return 0; 1845 } 1846 1847 table = kzalloc(sizeof(*table), GFP_KERNEL); 1848 if (!table) 1849 return 0; 1850 1851 if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) { 1852 /* Turn off APST. */ 1853 apste = 0; 1854 dev_dbg(ctrl->device, "APST disabled\n"); 1855 } else { 1856 __le64 target = cpu_to_le64(0); 1857 int state; 1858 1859 /* 1860 * Walk through all states from lowest- to highest-power. 1861 * According to the spec, lower-numbered states use more 1862 * power. NPSS, despite the name, is the index of the 1863 * lowest-power state, not the number of states. 1864 */ 1865 for (state = (int)ctrl->npss; state >= 0; state--) { 1866 u64 total_latency_us, exit_latency_us, transition_ms; 1867 1868 if (target) 1869 table->entries[state] = target; 1870 1871 /* 1872 * Don't allow transitions to the deepest state 1873 * if it's quirked off. 1874 */ 1875 if (state == ctrl->npss && 1876 (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) 1877 continue; 1878 1879 /* 1880 * Is this state a useful non-operational state for 1881 * higher-power states to autonomously transition to? 1882 */ 1883 if (!(ctrl->psd[state].flags & 1884 NVME_PS_FLAGS_NON_OP_STATE)) 1885 continue; 1886 1887 exit_latency_us = 1888 (u64)le32_to_cpu(ctrl->psd[state].exit_lat); 1889 if (exit_latency_us > ctrl->ps_max_latency_us) 1890 continue; 1891 1892 total_latency_us = 1893 exit_latency_us + 1894 le32_to_cpu(ctrl->psd[state].entry_lat); 1895 1896 /* 1897 * This state is good. Use it as the APST idle 1898 * target for higher power states. 1899 */ 1900 transition_ms = total_latency_us + 19; 1901 do_div(transition_ms, 20); 1902 if (transition_ms > (1 << 24) - 1) 1903 transition_ms = (1 << 24) - 1; 1904 1905 target = cpu_to_le64((state << 3) | 1906 (transition_ms << 8)); 1907 1908 if (max_ps == -1) 1909 max_ps = state; 1910 1911 if (total_latency_us > max_lat_us) 1912 max_lat_us = total_latency_us; 1913 } 1914 1915 apste = 1; 1916 1917 if (max_ps == -1) { 1918 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n"); 1919 } else { 1920 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n", 1921 max_ps, max_lat_us, (int)sizeof(*table), table); 1922 } 1923 } 1924 1925 ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste, 1926 table, sizeof(*table), NULL); 1927 if (ret) 1928 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret); 1929 1930 kfree(table); 1931 return ret; 1932 } 1933 1934 static void nvme_set_latency_tolerance(struct device *dev, s32 val) 1935 { 1936 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 1937 u64 latency; 1938 1939 switch (val) { 1940 case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT: 1941 case PM_QOS_LATENCY_ANY: 1942 latency = U64_MAX; 1943 break; 1944 1945 default: 1946 latency = val; 1947 } 1948 1949 if (ctrl->ps_max_latency_us != latency) { 1950 ctrl->ps_max_latency_us = latency; 1951 nvme_configure_apst(ctrl); 1952 } 1953 } 1954 1955 struct nvme_core_quirk_entry { 1956 /* 1957 * NVMe model and firmware strings are padded with spaces. For 1958 * simplicity, strings in the quirk table are padded with NULLs 1959 * instead. 1960 */ 1961 u16 vid; 1962 const char *mn; 1963 const char *fr; 1964 unsigned long quirks; 1965 }; 1966 1967 static const struct nvme_core_quirk_entry core_quirks[] = { 1968 { 1969 /* 1970 * This Toshiba device seems to die using any APST states. See: 1971 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11 1972 */ 1973 .vid = 0x1179, 1974 .mn = "THNSF5256GPUK TOSHIBA", 1975 .quirks = NVME_QUIRK_NO_APST, 1976 } 1977 }; 1978 1979 /* match is null-terminated but idstr is space-padded. */ 1980 static bool string_matches(const char *idstr, const char *match, size_t len) 1981 { 1982 size_t matchlen; 1983 1984 if (!match) 1985 return true; 1986 1987 matchlen = strlen(match); 1988 WARN_ON_ONCE(matchlen > len); 1989 1990 if (memcmp(idstr, match, matchlen)) 1991 return false; 1992 1993 for (; matchlen < len; matchlen++) 1994 if (idstr[matchlen] != ' ') 1995 return false; 1996 1997 return true; 1998 } 1999 2000 static bool quirk_matches(const struct nvme_id_ctrl *id, 2001 const struct nvme_core_quirk_entry *q) 2002 { 2003 return q->vid == le16_to_cpu(id->vid) && 2004 string_matches(id->mn, q->mn, sizeof(id->mn)) && 2005 string_matches(id->fr, q->fr, sizeof(id->fr)); 2006 } 2007 2008 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl, 2009 struct nvme_id_ctrl *id) 2010 { 2011 size_t nqnlen; 2012 int off; 2013 2014 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE); 2015 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) { 2016 strncpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE); 2017 return; 2018 } 2019 2020 if (ctrl->vs >= NVME_VS(1, 2, 1)) 2021 dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n"); 2022 2023 /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */ 2024 off = snprintf(subsys->subnqn, NVMF_NQN_SIZE, 2025 "nqn.2014.08.org.nvmexpress:%4x%4x", 2026 le16_to_cpu(id->vid), le16_to_cpu(id->ssvid)); 2027 memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn)); 2028 off += sizeof(id->sn); 2029 memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn)); 2030 off += sizeof(id->mn); 2031 memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off); 2032 } 2033 2034 static void __nvme_release_subsystem(struct nvme_subsystem *subsys) 2035 { 2036 ida_simple_remove(&nvme_subsystems_ida, subsys->instance); 2037 kfree(subsys); 2038 } 2039 2040 static void nvme_release_subsystem(struct device *dev) 2041 { 2042 __nvme_release_subsystem(container_of(dev, struct nvme_subsystem, dev)); 2043 } 2044 2045 static void nvme_destroy_subsystem(struct kref *ref) 2046 { 2047 struct nvme_subsystem *subsys = 2048 container_of(ref, struct nvme_subsystem, ref); 2049 2050 mutex_lock(&nvme_subsystems_lock); 2051 list_del(&subsys->entry); 2052 mutex_unlock(&nvme_subsystems_lock); 2053 2054 ida_destroy(&subsys->ns_ida); 2055 device_del(&subsys->dev); 2056 put_device(&subsys->dev); 2057 } 2058 2059 static void nvme_put_subsystem(struct nvme_subsystem *subsys) 2060 { 2061 kref_put(&subsys->ref, nvme_destroy_subsystem); 2062 } 2063 2064 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn) 2065 { 2066 struct nvme_subsystem *subsys; 2067 2068 lockdep_assert_held(&nvme_subsystems_lock); 2069 2070 list_for_each_entry(subsys, &nvme_subsystems, entry) { 2071 if (strcmp(subsys->subnqn, subsysnqn)) 2072 continue; 2073 if (!kref_get_unless_zero(&subsys->ref)) 2074 continue; 2075 return subsys; 2076 } 2077 2078 return NULL; 2079 } 2080 2081 #define SUBSYS_ATTR_RO(_name, _mode, _show) \ 2082 struct device_attribute subsys_attr_##_name = \ 2083 __ATTR(_name, _mode, _show, NULL) 2084 2085 static ssize_t nvme_subsys_show_nqn(struct device *dev, 2086 struct device_attribute *attr, 2087 char *buf) 2088 { 2089 struct nvme_subsystem *subsys = 2090 container_of(dev, struct nvme_subsystem, dev); 2091 2092 return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn); 2093 } 2094 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn); 2095 2096 #define nvme_subsys_show_str_function(field) \ 2097 static ssize_t subsys_##field##_show(struct device *dev, \ 2098 struct device_attribute *attr, char *buf) \ 2099 { \ 2100 struct nvme_subsystem *subsys = \ 2101 container_of(dev, struct nvme_subsystem, dev); \ 2102 return sprintf(buf, "%.*s\n", \ 2103 (int)sizeof(subsys->field), subsys->field); \ 2104 } \ 2105 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show); 2106 2107 nvme_subsys_show_str_function(model); 2108 nvme_subsys_show_str_function(serial); 2109 nvme_subsys_show_str_function(firmware_rev); 2110 2111 static struct attribute *nvme_subsys_attrs[] = { 2112 &subsys_attr_model.attr, 2113 &subsys_attr_serial.attr, 2114 &subsys_attr_firmware_rev.attr, 2115 &subsys_attr_subsysnqn.attr, 2116 NULL, 2117 }; 2118 2119 static struct attribute_group nvme_subsys_attrs_group = { 2120 .attrs = nvme_subsys_attrs, 2121 }; 2122 2123 static const struct attribute_group *nvme_subsys_attrs_groups[] = { 2124 &nvme_subsys_attrs_group, 2125 NULL, 2126 }; 2127 2128 static int nvme_active_ctrls(struct nvme_subsystem *subsys) 2129 { 2130 int count = 0; 2131 struct nvme_ctrl *ctrl; 2132 2133 mutex_lock(&subsys->lock); 2134 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 2135 if (ctrl->state != NVME_CTRL_DELETING && 2136 ctrl->state != NVME_CTRL_DEAD) 2137 count++; 2138 } 2139 mutex_unlock(&subsys->lock); 2140 2141 return count; 2142 } 2143 2144 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 2145 { 2146 struct nvme_subsystem *subsys, *found; 2147 int ret; 2148 2149 subsys = kzalloc(sizeof(*subsys), GFP_KERNEL); 2150 if (!subsys) 2151 return -ENOMEM; 2152 ret = ida_simple_get(&nvme_subsystems_ida, 0, 0, GFP_KERNEL); 2153 if (ret < 0) { 2154 kfree(subsys); 2155 return ret; 2156 } 2157 subsys->instance = ret; 2158 mutex_init(&subsys->lock); 2159 kref_init(&subsys->ref); 2160 INIT_LIST_HEAD(&subsys->ctrls); 2161 INIT_LIST_HEAD(&subsys->nsheads); 2162 nvme_init_subnqn(subsys, ctrl, id); 2163 memcpy(subsys->serial, id->sn, sizeof(subsys->serial)); 2164 memcpy(subsys->model, id->mn, sizeof(subsys->model)); 2165 memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev)); 2166 subsys->vendor_id = le16_to_cpu(id->vid); 2167 subsys->cmic = id->cmic; 2168 2169 subsys->dev.class = nvme_subsys_class; 2170 subsys->dev.release = nvme_release_subsystem; 2171 subsys->dev.groups = nvme_subsys_attrs_groups; 2172 dev_set_name(&subsys->dev, "nvme-subsys%d", subsys->instance); 2173 device_initialize(&subsys->dev); 2174 2175 mutex_lock(&nvme_subsystems_lock); 2176 found = __nvme_find_get_subsystem(subsys->subnqn); 2177 if (found) { 2178 /* 2179 * Verify that the subsystem actually supports multiple 2180 * controllers, else bail out. 2181 */ 2182 if (nvme_active_ctrls(found) && !(id->cmic & (1 << 1))) { 2183 dev_err(ctrl->device, 2184 "ignoring ctrl due to duplicate subnqn (%s).\n", 2185 found->subnqn); 2186 nvme_put_subsystem(found); 2187 ret = -EINVAL; 2188 goto out_unlock; 2189 } 2190 2191 __nvme_release_subsystem(subsys); 2192 subsys = found; 2193 } else { 2194 ret = device_add(&subsys->dev); 2195 if (ret) { 2196 dev_err(ctrl->device, 2197 "failed to register subsystem device.\n"); 2198 goto out_unlock; 2199 } 2200 ida_init(&subsys->ns_ida); 2201 list_add_tail(&subsys->entry, &nvme_subsystems); 2202 } 2203 2204 ctrl->subsys = subsys; 2205 mutex_unlock(&nvme_subsystems_lock); 2206 2207 if (sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj, 2208 dev_name(ctrl->device))) { 2209 dev_err(ctrl->device, 2210 "failed to create sysfs link from subsystem.\n"); 2211 /* the transport driver will eventually put the subsystem */ 2212 return -EINVAL; 2213 } 2214 2215 mutex_lock(&subsys->lock); 2216 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls); 2217 mutex_unlock(&subsys->lock); 2218 2219 return 0; 2220 2221 out_unlock: 2222 mutex_unlock(&nvme_subsystems_lock); 2223 put_device(&subsys->dev); 2224 return ret; 2225 } 2226 2227 int nvme_get_log_ext(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 2228 u8 log_page, void *log, 2229 size_t size, u64 offset) 2230 { 2231 struct nvme_command c = { }; 2232 unsigned long dwlen = size / 4 - 1; 2233 2234 c.get_log_page.opcode = nvme_admin_get_log_page; 2235 2236 if (ns) 2237 c.get_log_page.nsid = cpu_to_le32(ns->head->ns_id); 2238 else 2239 c.get_log_page.nsid = cpu_to_le32(NVME_NSID_ALL); 2240 2241 c.get_log_page.lid = log_page; 2242 c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1)); 2243 c.get_log_page.numdu = cpu_to_le16(dwlen >> 16); 2244 c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset)); 2245 c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset)); 2246 2247 return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size); 2248 } 2249 2250 static int nvme_get_log(struct nvme_ctrl *ctrl, u8 log_page, void *log, 2251 size_t size) 2252 { 2253 return nvme_get_log_ext(ctrl, NULL, log_page, log, size, 0); 2254 } 2255 2256 static int nvme_get_effects_log(struct nvme_ctrl *ctrl) 2257 { 2258 int ret; 2259 2260 if (!ctrl->effects) 2261 ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL); 2262 2263 if (!ctrl->effects) 2264 return 0; 2265 2266 ret = nvme_get_log(ctrl, NVME_LOG_CMD_EFFECTS, ctrl->effects, 2267 sizeof(*ctrl->effects)); 2268 if (ret) { 2269 kfree(ctrl->effects); 2270 ctrl->effects = NULL; 2271 } 2272 return ret; 2273 } 2274 2275 /* 2276 * Initialize the cached copies of the Identify data and various controller 2277 * register in our nvme_ctrl structure. This should be called as soon as 2278 * the admin queue is fully up and running. 2279 */ 2280 int nvme_init_identify(struct nvme_ctrl *ctrl) 2281 { 2282 struct nvme_id_ctrl *id; 2283 u64 cap; 2284 int ret, page_shift; 2285 u32 max_hw_sectors; 2286 bool prev_apst_enabled; 2287 2288 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs); 2289 if (ret) { 2290 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret); 2291 return ret; 2292 } 2293 2294 ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap); 2295 if (ret) { 2296 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret); 2297 return ret; 2298 } 2299 page_shift = NVME_CAP_MPSMIN(cap) + 12; 2300 2301 if (ctrl->vs >= NVME_VS(1, 1, 0)) 2302 ctrl->subsystem = NVME_CAP_NSSRC(cap); 2303 2304 ret = nvme_identify_ctrl(ctrl, &id); 2305 if (ret) { 2306 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret); 2307 return -EIO; 2308 } 2309 2310 if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) { 2311 ret = nvme_get_effects_log(ctrl); 2312 if (ret < 0) 2313 return ret; 2314 } 2315 2316 if (!ctrl->identified) { 2317 int i; 2318 2319 ret = nvme_init_subsystem(ctrl, id); 2320 if (ret) 2321 goto out_free; 2322 2323 /* 2324 * Check for quirks. Quirk can depend on firmware version, 2325 * so, in principle, the set of quirks present can change 2326 * across a reset. As a possible future enhancement, we 2327 * could re-scan for quirks every time we reinitialize 2328 * the device, but we'd have to make sure that the driver 2329 * behaves intelligently if the quirks change. 2330 */ 2331 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) { 2332 if (quirk_matches(id, &core_quirks[i])) 2333 ctrl->quirks |= core_quirks[i].quirks; 2334 } 2335 } 2336 2337 if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) { 2338 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n"); 2339 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS; 2340 } 2341 2342 ctrl->oacs = le16_to_cpu(id->oacs); 2343 ctrl->oncs = le16_to_cpup(&id->oncs); 2344 atomic_set(&ctrl->abort_limit, id->acl + 1); 2345 ctrl->vwc = id->vwc; 2346 ctrl->cntlid = le16_to_cpup(&id->cntlid); 2347 if (id->mdts) 2348 max_hw_sectors = 1 << (id->mdts + page_shift - 9); 2349 else 2350 max_hw_sectors = UINT_MAX; 2351 ctrl->max_hw_sectors = 2352 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors); 2353 2354 nvme_set_queue_limits(ctrl, ctrl->admin_q); 2355 ctrl->sgls = le32_to_cpu(id->sgls); 2356 ctrl->kas = le16_to_cpu(id->kas); 2357 2358 if (id->rtd3e) { 2359 /* us -> s */ 2360 u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000; 2361 2362 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time, 2363 shutdown_timeout, 60); 2364 2365 if (ctrl->shutdown_timeout != shutdown_timeout) 2366 dev_info(ctrl->device, 2367 "Shutdown timeout set to %u seconds\n", 2368 ctrl->shutdown_timeout); 2369 } else 2370 ctrl->shutdown_timeout = shutdown_timeout; 2371 2372 ctrl->npss = id->npss; 2373 ctrl->apsta = id->apsta; 2374 prev_apst_enabled = ctrl->apst_enabled; 2375 if (ctrl->quirks & NVME_QUIRK_NO_APST) { 2376 if (force_apst && id->apsta) { 2377 dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n"); 2378 ctrl->apst_enabled = true; 2379 } else { 2380 ctrl->apst_enabled = false; 2381 } 2382 } else { 2383 ctrl->apst_enabled = id->apsta; 2384 } 2385 memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd)); 2386 2387 if (ctrl->ops->flags & NVME_F_FABRICS) { 2388 ctrl->icdoff = le16_to_cpu(id->icdoff); 2389 ctrl->ioccsz = le32_to_cpu(id->ioccsz); 2390 ctrl->iorcsz = le32_to_cpu(id->iorcsz); 2391 ctrl->maxcmd = le16_to_cpu(id->maxcmd); 2392 2393 /* 2394 * In fabrics we need to verify the cntlid matches the 2395 * admin connect 2396 */ 2397 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) { 2398 ret = -EINVAL; 2399 goto out_free; 2400 } 2401 2402 if (!ctrl->opts->discovery_nqn && !ctrl->kas) { 2403 dev_err(ctrl->device, 2404 "keep-alive support is mandatory for fabrics\n"); 2405 ret = -EINVAL; 2406 goto out_free; 2407 } 2408 } else { 2409 ctrl->cntlid = le16_to_cpu(id->cntlid); 2410 ctrl->hmpre = le32_to_cpu(id->hmpre); 2411 ctrl->hmmin = le32_to_cpu(id->hmmin); 2412 ctrl->hmminds = le32_to_cpu(id->hmminds); 2413 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd); 2414 } 2415 2416 kfree(id); 2417 2418 if (ctrl->apst_enabled && !prev_apst_enabled) 2419 dev_pm_qos_expose_latency_tolerance(ctrl->device); 2420 else if (!ctrl->apst_enabled && prev_apst_enabled) 2421 dev_pm_qos_hide_latency_tolerance(ctrl->device); 2422 2423 ret = nvme_configure_apst(ctrl); 2424 if (ret < 0) 2425 return ret; 2426 2427 ret = nvme_configure_timestamp(ctrl); 2428 if (ret < 0) 2429 return ret; 2430 2431 ret = nvme_configure_directives(ctrl); 2432 if (ret < 0) 2433 return ret; 2434 2435 ctrl->identified = true; 2436 2437 return 0; 2438 2439 out_free: 2440 kfree(id); 2441 return ret; 2442 } 2443 EXPORT_SYMBOL_GPL(nvme_init_identify); 2444 2445 static int nvme_dev_open(struct inode *inode, struct file *file) 2446 { 2447 struct nvme_ctrl *ctrl = 2448 container_of(inode->i_cdev, struct nvme_ctrl, cdev); 2449 2450 switch (ctrl->state) { 2451 case NVME_CTRL_LIVE: 2452 case NVME_CTRL_ADMIN_ONLY: 2453 break; 2454 default: 2455 return -EWOULDBLOCK; 2456 } 2457 2458 file->private_data = ctrl; 2459 return 0; 2460 } 2461 2462 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp) 2463 { 2464 struct nvme_ns *ns; 2465 int ret; 2466 2467 down_read(&ctrl->namespaces_rwsem); 2468 if (list_empty(&ctrl->namespaces)) { 2469 ret = -ENOTTY; 2470 goto out_unlock; 2471 } 2472 2473 ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list); 2474 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) { 2475 dev_warn(ctrl->device, 2476 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n"); 2477 ret = -EINVAL; 2478 goto out_unlock; 2479 } 2480 2481 dev_warn(ctrl->device, 2482 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n"); 2483 kref_get(&ns->kref); 2484 up_read(&ctrl->namespaces_rwsem); 2485 2486 ret = nvme_user_cmd(ctrl, ns, argp); 2487 nvme_put_ns(ns); 2488 return ret; 2489 2490 out_unlock: 2491 up_read(&ctrl->namespaces_rwsem); 2492 return ret; 2493 } 2494 2495 static long nvme_dev_ioctl(struct file *file, unsigned int cmd, 2496 unsigned long arg) 2497 { 2498 struct nvme_ctrl *ctrl = file->private_data; 2499 void __user *argp = (void __user *)arg; 2500 2501 switch (cmd) { 2502 case NVME_IOCTL_ADMIN_CMD: 2503 return nvme_user_cmd(ctrl, NULL, argp); 2504 case NVME_IOCTL_IO_CMD: 2505 return nvme_dev_user_cmd(ctrl, argp); 2506 case NVME_IOCTL_RESET: 2507 dev_warn(ctrl->device, "resetting controller\n"); 2508 return nvme_reset_ctrl_sync(ctrl); 2509 case NVME_IOCTL_SUBSYS_RESET: 2510 return nvme_reset_subsystem(ctrl); 2511 case NVME_IOCTL_RESCAN: 2512 nvme_queue_scan(ctrl); 2513 return 0; 2514 default: 2515 return -ENOTTY; 2516 } 2517 } 2518 2519 static const struct file_operations nvme_dev_fops = { 2520 .owner = THIS_MODULE, 2521 .open = nvme_dev_open, 2522 .unlocked_ioctl = nvme_dev_ioctl, 2523 .compat_ioctl = nvme_dev_ioctl, 2524 }; 2525 2526 static ssize_t nvme_sysfs_reset(struct device *dev, 2527 struct device_attribute *attr, const char *buf, 2528 size_t count) 2529 { 2530 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 2531 int ret; 2532 2533 ret = nvme_reset_ctrl_sync(ctrl); 2534 if (ret < 0) 2535 return ret; 2536 return count; 2537 } 2538 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset); 2539 2540 static ssize_t nvme_sysfs_rescan(struct device *dev, 2541 struct device_attribute *attr, const char *buf, 2542 size_t count) 2543 { 2544 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 2545 2546 nvme_queue_scan(ctrl); 2547 return count; 2548 } 2549 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan); 2550 2551 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev) 2552 { 2553 struct gendisk *disk = dev_to_disk(dev); 2554 2555 if (disk->fops == &nvme_fops) 2556 return nvme_get_ns_from_dev(dev)->head; 2557 else 2558 return disk->private_data; 2559 } 2560 2561 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr, 2562 char *buf) 2563 { 2564 struct nvme_ns_head *head = dev_to_ns_head(dev); 2565 struct nvme_ns_ids *ids = &head->ids; 2566 struct nvme_subsystem *subsys = head->subsys; 2567 int serial_len = sizeof(subsys->serial); 2568 int model_len = sizeof(subsys->model); 2569 2570 if (!uuid_is_null(&ids->uuid)) 2571 return sprintf(buf, "uuid.%pU\n", &ids->uuid); 2572 2573 if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid))) 2574 return sprintf(buf, "eui.%16phN\n", ids->nguid); 2575 2576 if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64))) 2577 return sprintf(buf, "eui.%8phN\n", ids->eui64); 2578 2579 while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' || 2580 subsys->serial[serial_len - 1] == '\0')) 2581 serial_len--; 2582 while (model_len > 0 && (subsys->model[model_len - 1] == ' ' || 2583 subsys->model[model_len - 1] == '\0')) 2584 model_len--; 2585 2586 return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id, 2587 serial_len, subsys->serial, model_len, subsys->model, 2588 head->ns_id); 2589 } 2590 static DEVICE_ATTR_RO(wwid); 2591 2592 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr, 2593 char *buf) 2594 { 2595 return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid); 2596 } 2597 static DEVICE_ATTR_RO(nguid); 2598 2599 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, 2600 char *buf) 2601 { 2602 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids; 2603 2604 /* For backward compatibility expose the NGUID to userspace if 2605 * we have no UUID set 2606 */ 2607 if (uuid_is_null(&ids->uuid)) { 2608 printk_ratelimited(KERN_WARNING 2609 "No UUID available providing old NGUID\n"); 2610 return sprintf(buf, "%pU\n", ids->nguid); 2611 } 2612 return sprintf(buf, "%pU\n", &ids->uuid); 2613 } 2614 static DEVICE_ATTR_RO(uuid); 2615 2616 static ssize_t eui_show(struct device *dev, struct device_attribute *attr, 2617 char *buf) 2618 { 2619 return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64); 2620 } 2621 static DEVICE_ATTR_RO(eui); 2622 2623 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr, 2624 char *buf) 2625 { 2626 return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id); 2627 } 2628 static DEVICE_ATTR_RO(nsid); 2629 2630 static struct attribute *nvme_ns_id_attrs[] = { 2631 &dev_attr_wwid.attr, 2632 &dev_attr_uuid.attr, 2633 &dev_attr_nguid.attr, 2634 &dev_attr_eui.attr, 2635 &dev_attr_nsid.attr, 2636 NULL, 2637 }; 2638 2639 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj, 2640 struct attribute *a, int n) 2641 { 2642 struct device *dev = container_of(kobj, struct device, kobj); 2643 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids; 2644 2645 if (a == &dev_attr_uuid.attr) { 2646 if (uuid_is_null(&ids->uuid) && 2647 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid))) 2648 return 0; 2649 } 2650 if (a == &dev_attr_nguid.attr) { 2651 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid))) 2652 return 0; 2653 } 2654 if (a == &dev_attr_eui.attr) { 2655 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64))) 2656 return 0; 2657 } 2658 return a->mode; 2659 } 2660 2661 const struct attribute_group nvme_ns_id_attr_group = { 2662 .attrs = nvme_ns_id_attrs, 2663 .is_visible = nvme_ns_id_attrs_are_visible, 2664 }; 2665 2666 #define nvme_show_str_function(field) \ 2667 static ssize_t field##_show(struct device *dev, \ 2668 struct device_attribute *attr, char *buf) \ 2669 { \ 2670 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \ 2671 return sprintf(buf, "%.*s\n", \ 2672 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field); \ 2673 } \ 2674 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL); 2675 2676 nvme_show_str_function(model); 2677 nvme_show_str_function(serial); 2678 nvme_show_str_function(firmware_rev); 2679 2680 #define nvme_show_int_function(field) \ 2681 static ssize_t field##_show(struct device *dev, \ 2682 struct device_attribute *attr, char *buf) \ 2683 { \ 2684 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \ 2685 return sprintf(buf, "%d\n", ctrl->field); \ 2686 } \ 2687 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL); 2688 2689 nvme_show_int_function(cntlid); 2690 2691 static ssize_t nvme_sysfs_delete(struct device *dev, 2692 struct device_attribute *attr, const char *buf, 2693 size_t count) 2694 { 2695 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 2696 2697 if (device_remove_file_self(dev, attr)) 2698 nvme_delete_ctrl_sync(ctrl); 2699 return count; 2700 } 2701 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete); 2702 2703 static ssize_t nvme_sysfs_show_transport(struct device *dev, 2704 struct device_attribute *attr, 2705 char *buf) 2706 { 2707 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 2708 2709 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name); 2710 } 2711 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL); 2712 2713 static ssize_t nvme_sysfs_show_state(struct device *dev, 2714 struct device_attribute *attr, 2715 char *buf) 2716 { 2717 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 2718 static const char *const state_name[] = { 2719 [NVME_CTRL_NEW] = "new", 2720 [NVME_CTRL_LIVE] = "live", 2721 [NVME_CTRL_ADMIN_ONLY] = "only-admin", 2722 [NVME_CTRL_RESETTING] = "resetting", 2723 [NVME_CTRL_CONNECTING] = "connecting", 2724 [NVME_CTRL_DELETING] = "deleting", 2725 [NVME_CTRL_DEAD] = "dead", 2726 }; 2727 2728 if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) && 2729 state_name[ctrl->state]) 2730 return sprintf(buf, "%s\n", state_name[ctrl->state]); 2731 2732 return sprintf(buf, "unknown state\n"); 2733 } 2734 2735 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL); 2736 2737 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev, 2738 struct device_attribute *attr, 2739 char *buf) 2740 { 2741 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 2742 2743 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn); 2744 } 2745 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL); 2746 2747 static ssize_t nvme_sysfs_show_address(struct device *dev, 2748 struct device_attribute *attr, 2749 char *buf) 2750 { 2751 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 2752 2753 return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE); 2754 } 2755 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL); 2756 2757 static struct attribute *nvme_dev_attrs[] = { 2758 &dev_attr_reset_controller.attr, 2759 &dev_attr_rescan_controller.attr, 2760 &dev_attr_model.attr, 2761 &dev_attr_serial.attr, 2762 &dev_attr_firmware_rev.attr, 2763 &dev_attr_cntlid.attr, 2764 &dev_attr_delete_controller.attr, 2765 &dev_attr_transport.attr, 2766 &dev_attr_subsysnqn.attr, 2767 &dev_attr_address.attr, 2768 &dev_attr_state.attr, 2769 NULL 2770 }; 2771 2772 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj, 2773 struct attribute *a, int n) 2774 { 2775 struct device *dev = container_of(kobj, struct device, kobj); 2776 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); 2777 2778 if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl) 2779 return 0; 2780 if (a == &dev_attr_address.attr && !ctrl->ops->get_address) 2781 return 0; 2782 2783 return a->mode; 2784 } 2785 2786 static struct attribute_group nvme_dev_attrs_group = { 2787 .attrs = nvme_dev_attrs, 2788 .is_visible = nvme_dev_attrs_are_visible, 2789 }; 2790 2791 static const struct attribute_group *nvme_dev_attr_groups[] = { 2792 &nvme_dev_attrs_group, 2793 NULL, 2794 }; 2795 2796 static struct nvme_ns_head *__nvme_find_ns_head(struct nvme_subsystem *subsys, 2797 unsigned nsid) 2798 { 2799 struct nvme_ns_head *h; 2800 2801 lockdep_assert_held(&subsys->lock); 2802 2803 list_for_each_entry(h, &subsys->nsheads, entry) { 2804 if (h->ns_id == nsid && kref_get_unless_zero(&h->ref)) 2805 return h; 2806 } 2807 2808 return NULL; 2809 } 2810 2811 static int __nvme_check_ids(struct nvme_subsystem *subsys, 2812 struct nvme_ns_head *new) 2813 { 2814 struct nvme_ns_head *h; 2815 2816 lockdep_assert_held(&subsys->lock); 2817 2818 list_for_each_entry(h, &subsys->nsheads, entry) { 2819 if (nvme_ns_ids_valid(&new->ids) && 2820 !list_empty(&h->list) && 2821 nvme_ns_ids_equal(&new->ids, &h->ids)) 2822 return -EINVAL; 2823 } 2824 2825 return 0; 2826 } 2827 2828 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, 2829 unsigned nsid, struct nvme_id_ns *id) 2830 { 2831 struct nvme_ns_head *head; 2832 int ret = -ENOMEM; 2833 2834 head = kzalloc(sizeof(*head), GFP_KERNEL); 2835 if (!head) 2836 goto out; 2837 ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL); 2838 if (ret < 0) 2839 goto out_free_head; 2840 head->instance = ret; 2841 INIT_LIST_HEAD(&head->list); 2842 ret = init_srcu_struct(&head->srcu); 2843 if (ret) 2844 goto out_ida_remove; 2845 head->subsys = ctrl->subsys; 2846 head->ns_id = nsid; 2847 kref_init(&head->ref); 2848 2849 nvme_report_ns_ids(ctrl, nsid, id, &head->ids); 2850 2851 ret = __nvme_check_ids(ctrl->subsys, head); 2852 if (ret) { 2853 dev_err(ctrl->device, 2854 "duplicate IDs for nsid %d\n", nsid); 2855 goto out_cleanup_srcu; 2856 } 2857 2858 ret = nvme_mpath_alloc_disk(ctrl, head); 2859 if (ret) 2860 goto out_cleanup_srcu; 2861 2862 list_add_tail(&head->entry, &ctrl->subsys->nsheads); 2863 return head; 2864 out_cleanup_srcu: 2865 cleanup_srcu_struct(&head->srcu); 2866 out_ida_remove: 2867 ida_simple_remove(&ctrl->subsys->ns_ida, head->instance); 2868 out_free_head: 2869 kfree(head); 2870 out: 2871 return ERR_PTR(ret); 2872 } 2873 2874 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, 2875 struct nvme_id_ns *id) 2876 { 2877 struct nvme_ctrl *ctrl = ns->ctrl; 2878 bool is_shared = id->nmic & (1 << 0); 2879 struct nvme_ns_head *head = NULL; 2880 int ret = 0; 2881 2882 mutex_lock(&ctrl->subsys->lock); 2883 if (is_shared) 2884 head = __nvme_find_ns_head(ctrl->subsys, nsid); 2885 if (!head) { 2886 head = nvme_alloc_ns_head(ctrl, nsid, id); 2887 if (IS_ERR(head)) { 2888 ret = PTR_ERR(head); 2889 goto out_unlock; 2890 } 2891 } else { 2892 struct nvme_ns_ids ids; 2893 2894 nvme_report_ns_ids(ctrl, nsid, id, &ids); 2895 if (!nvme_ns_ids_equal(&head->ids, &ids)) { 2896 dev_err(ctrl->device, 2897 "IDs don't match for shared namespace %d\n", 2898 nsid); 2899 ret = -EINVAL; 2900 goto out_unlock; 2901 } 2902 } 2903 2904 list_add_tail(&ns->siblings, &head->list); 2905 ns->head = head; 2906 2907 out_unlock: 2908 mutex_unlock(&ctrl->subsys->lock); 2909 return ret; 2910 } 2911 2912 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b) 2913 { 2914 struct nvme_ns *nsa = container_of(a, struct nvme_ns, list); 2915 struct nvme_ns *nsb = container_of(b, struct nvme_ns, list); 2916 2917 return nsa->head->ns_id - nsb->head->ns_id; 2918 } 2919 2920 static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid) 2921 { 2922 struct nvme_ns *ns, *ret = NULL; 2923 2924 down_read(&ctrl->namespaces_rwsem); 2925 list_for_each_entry(ns, &ctrl->namespaces, list) { 2926 if (ns->head->ns_id == nsid) { 2927 if (!kref_get_unless_zero(&ns->kref)) 2928 continue; 2929 ret = ns; 2930 break; 2931 } 2932 if (ns->head->ns_id > nsid) 2933 break; 2934 } 2935 up_read(&ctrl->namespaces_rwsem); 2936 return ret; 2937 } 2938 2939 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns) 2940 { 2941 struct streams_directive_params s; 2942 int ret; 2943 2944 if (!ctrl->nr_streams) 2945 return 0; 2946 2947 ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id); 2948 if (ret) 2949 return ret; 2950 2951 ns->sws = le32_to_cpu(s.sws); 2952 ns->sgs = le16_to_cpu(s.sgs); 2953 2954 if (ns->sws) { 2955 unsigned int bs = 1 << ns->lba_shift; 2956 2957 blk_queue_io_min(ns->queue, bs * ns->sws); 2958 if (ns->sgs) 2959 blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs); 2960 } 2961 2962 return 0; 2963 } 2964 2965 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) 2966 { 2967 struct nvme_ns *ns; 2968 struct gendisk *disk; 2969 struct nvme_id_ns *id; 2970 char disk_name[DISK_NAME_LEN]; 2971 int node = dev_to_node(ctrl->dev), flags = GENHD_FL_EXT_DEVT; 2972 2973 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node); 2974 if (!ns) 2975 return; 2976 2977 ns->queue = blk_mq_init_queue(ctrl->tagset); 2978 if (IS_ERR(ns->queue)) 2979 goto out_free_ns; 2980 blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue); 2981 ns->queue->queuedata = ns; 2982 ns->ctrl = ctrl; 2983 2984 kref_init(&ns->kref); 2985 ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */ 2986 2987 blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); 2988 nvme_set_queue_limits(ctrl, ns->queue); 2989 2990 id = nvme_identify_ns(ctrl, nsid); 2991 if (!id) 2992 goto out_free_queue; 2993 2994 if (id->ncap == 0) 2995 goto out_free_id; 2996 2997 if (nvme_init_ns_head(ns, nsid, id)) 2998 goto out_free_id; 2999 nvme_setup_streams_ns(ctrl, ns); 3000 3001 #ifdef CONFIG_NVME_MULTIPATH 3002 /* 3003 * If multipathing is enabled we need to always use the subsystem 3004 * instance number for numbering our devices to avoid conflicts 3005 * between subsystems that have multiple controllers and thus use 3006 * the multipath-aware subsystem node and those that have a single 3007 * controller and use the controller node directly. 3008 */ 3009 if (ns->head->disk) { 3010 sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance, 3011 ctrl->cntlid, ns->head->instance); 3012 flags = GENHD_FL_HIDDEN; 3013 } else { 3014 sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance, 3015 ns->head->instance); 3016 } 3017 #else 3018 /* 3019 * But without the multipath code enabled, multiple controller per 3020 * subsystems are visible as devices and thus we cannot use the 3021 * subsystem instance. 3022 */ 3023 sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance); 3024 #endif 3025 3026 if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) { 3027 if (nvme_nvm_register(ns, disk_name, node)) { 3028 dev_warn(ctrl->device, "LightNVM init failure\n"); 3029 goto out_unlink_ns; 3030 } 3031 } 3032 3033 disk = alloc_disk_node(0, node); 3034 if (!disk) 3035 goto out_unlink_ns; 3036 3037 disk->fops = &nvme_fops; 3038 disk->private_data = ns; 3039 disk->queue = ns->queue; 3040 disk->flags = flags; 3041 memcpy(disk->disk_name, disk_name, DISK_NAME_LEN); 3042 ns->disk = disk; 3043 3044 __nvme_revalidate_disk(disk, id); 3045 3046 down_write(&ctrl->namespaces_rwsem); 3047 list_add_tail(&ns->list, &ctrl->namespaces); 3048 up_write(&ctrl->namespaces_rwsem); 3049 3050 nvme_get_ctrl(ctrl); 3051 3052 kfree(id); 3053 3054 device_add_disk(ctrl->device, ns->disk); 3055 if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj, 3056 &nvme_ns_id_attr_group)) 3057 pr_warn("%s: failed to create sysfs group for identification\n", 3058 ns->disk->disk_name); 3059 if (ns->ndev && nvme_nvm_register_sysfs(ns)) 3060 pr_warn("%s: failed to register lightnvm sysfs group for identification\n", 3061 ns->disk->disk_name); 3062 3063 nvme_mpath_add_disk(ns->head); 3064 nvme_fault_inject_init(ns); 3065 return; 3066 out_unlink_ns: 3067 mutex_lock(&ctrl->subsys->lock); 3068 list_del_rcu(&ns->siblings); 3069 mutex_unlock(&ctrl->subsys->lock); 3070 out_free_id: 3071 kfree(id); 3072 out_free_queue: 3073 blk_cleanup_queue(ns->queue); 3074 out_free_ns: 3075 kfree(ns); 3076 } 3077 3078 static void nvme_ns_remove(struct nvme_ns *ns) 3079 { 3080 if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags)) 3081 return; 3082 3083 nvme_fault_inject_fini(ns); 3084 if (ns->disk && ns->disk->flags & GENHD_FL_UP) { 3085 sysfs_remove_group(&disk_to_dev(ns->disk)->kobj, 3086 &nvme_ns_id_attr_group); 3087 if (ns->ndev) 3088 nvme_nvm_unregister_sysfs(ns); 3089 del_gendisk(ns->disk); 3090 blk_cleanup_queue(ns->queue); 3091 if (blk_get_integrity(ns->disk)) 3092 blk_integrity_unregister(ns->disk); 3093 } 3094 3095 mutex_lock(&ns->ctrl->subsys->lock); 3096 nvme_mpath_clear_current_path(ns); 3097 list_del_rcu(&ns->siblings); 3098 mutex_unlock(&ns->ctrl->subsys->lock); 3099 3100 down_write(&ns->ctrl->namespaces_rwsem); 3101 list_del_init(&ns->list); 3102 up_write(&ns->ctrl->namespaces_rwsem); 3103 3104 synchronize_srcu(&ns->head->srcu); 3105 nvme_mpath_check_last_path(ns); 3106 nvme_put_ns(ns); 3107 } 3108 3109 static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid) 3110 { 3111 struct nvme_ns *ns; 3112 3113 ns = nvme_find_get_ns(ctrl, nsid); 3114 if (ns) { 3115 if (ns->disk && revalidate_disk(ns->disk)) 3116 nvme_ns_remove(ns); 3117 nvme_put_ns(ns); 3118 } else 3119 nvme_alloc_ns(ctrl, nsid); 3120 } 3121 3122 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, 3123 unsigned nsid) 3124 { 3125 struct nvme_ns *ns, *next; 3126 LIST_HEAD(rm_list); 3127 3128 down_write(&ctrl->namespaces_rwsem); 3129 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) { 3130 if (ns->head->ns_id > nsid) 3131 list_move_tail(&ns->list, &rm_list); 3132 } 3133 up_write(&ctrl->namespaces_rwsem); 3134 3135 list_for_each_entry_safe(ns, next, &rm_list, list) 3136 nvme_ns_remove(ns); 3137 3138 } 3139 3140 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn) 3141 { 3142 struct nvme_ns *ns; 3143 __le32 *ns_list; 3144 unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024); 3145 int ret = 0; 3146 3147 ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL); 3148 if (!ns_list) 3149 return -ENOMEM; 3150 3151 for (i = 0; i < num_lists; i++) { 3152 ret = nvme_identify_ns_list(ctrl, prev, ns_list); 3153 if (ret) 3154 goto free; 3155 3156 for (j = 0; j < min(nn, 1024U); j++) { 3157 nsid = le32_to_cpu(ns_list[j]); 3158 if (!nsid) 3159 goto out; 3160 3161 nvme_validate_ns(ctrl, nsid); 3162 3163 while (++prev < nsid) { 3164 ns = nvme_find_get_ns(ctrl, prev); 3165 if (ns) { 3166 nvme_ns_remove(ns); 3167 nvme_put_ns(ns); 3168 } 3169 } 3170 } 3171 nn -= j; 3172 } 3173 out: 3174 nvme_remove_invalid_namespaces(ctrl, prev); 3175 free: 3176 kfree(ns_list); 3177 return ret; 3178 } 3179 3180 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn) 3181 { 3182 unsigned i; 3183 3184 for (i = 1; i <= nn; i++) 3185 nvme_validate_ns(ctrl, i); 3186 3187 nvme_remove_invalid_namespaces(ctrl, nn); 3188 } 3189 3190 static void nvme_scan_work(struct work_struct *work) 3191 { 3192 struct nvme_ctrl *ctrl = 3193 container_of(work, struct nvme_ctrl, scan_work); 3194 struct nvme_id_ctrl *id; 3195 unsigned nn; 3196 3197 if (ctrl->state != NVME_CTRL_LIVE) 3198 return; 3199 3200 WARN_ON_ONCE(!ctrl->tagset); 3201 3202 if (nvme_identify_ctrl(ctrl, &id)) 3203 return; 3204 3205 nn = le32_to_cpu(id->nn); 3206 if (ctrl->vs >= NVME_VS(1, 1, 0) && 3207 !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) { 3208 if (!nvme_scan_ns_list(ctrl, nn)) 3209 goto done; 3210 } 3211 nvme_scan_ns_sequential(ctrl, nn); 3212 done: 3213 down_write(&ctrl->namespaces_rwsem); 3214 list_sort(NULL, &ctrl->namespaces, ns_cmp); 3215 up_write(&ctrl->namespaces_rwsem); 3216 kfree(id); 3217 } 3218 3219 void nvme_queue_scan(struct nvme_ctrl *ctrl) 3220 { 3221 /* 3222 * Only new queue scan work when admin and IO queues are both alive 3223 */ 3224 if (ctrl->state == NVME_CTRL_LIVE) 3225 queue_work(nvme_wq, &ctrl->scan_work); 3226 } 3227 EXPORT_SYMBOL_GPL(nvme_queue_scan); 3228 3229 /* 3230 * This function iterates the namespace list unlocked to allow recovery from 3231 * controller failure. It is up to the caller to ensure the namespace list is 3232 * not modified by scan work while this function is executing. 3233 */ 3234 void nvme_remove_namespaces(struct nvme_ctrl *ctrl) 3235 { 3236 struct nvme_ns *ns, *next; 3237 LIST_HEAD(ns_list); 3238 3239 /* 3240 * The dead states indicates the controller was not gracefully 3241 * disconnected. In that case, we won't be able to flush any data while 3242 * removing the namespaces' disks; fail all the queues now to avoid 3243 * potentially having to clean up the failed sync later. 3244 */ 3245 if (ctrl->state == NVME_CTRL_DEAD) 3246 nvme_kill_queues(ctrl); 3247 3248 down_write(&ctrl->namespaces_rwsem); 3249 list_splice_init(&ctrl->namespaces, &ns_list); 3250 up_write(&ctrl->namespaces_rwsem); 3251 3252 list_for_each_entry_safe(ns, next, &ns_list, list) 3253 nvme_ns_remove(ns); 3254 } 3255 EXPORT_SYMBOL_GPL(nvme_remove_namespaces); 3256 3257 static void nvme_aen_uevent(struct nvme_ctrl *ctrl) 3258 { 3259 char *envp[2] = { NULL, NULL }; 3260 u32 aen_result = ctrl->aen_result; 3261 3262 ctrl->aen_result = 0; 3263 if (!aen_result) 3264 return; 3265 3266 envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result); 3267 if (!envp[0]) 3268 return; 3269 kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp); 3270 kfree(envp[0]); 3271 } 3272 3273 static void nvme_async_event_work(struct work_struct *work) 3274 { 3275 struct nvme_ctrl *ctrl = 3276 container_of(work, struct nvme_ctrl, async_event_work); 3277 3278 nvme_aen_uevent(ctrl); 3279 ctrl->ops->submit_async_event(ctrl); 3280 } 3281 3282 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl) 3283 { 3284 3285 u32 csts; 3286 3287 if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) 3288 return false; 3289 3290 if (csts == ~0) 3291 return false; 3292 3293 return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP)); 3294 } 3295 3296 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl) 3297 { 3298 struct nvme_fw_slot_info_log *log; 3299 3300 log = kmalloc(sizeof(*log), GFP_KERNEL); 3301 if (!log) 3302 return; 3303 3304 if (nvme_get_log(ctrl, NVME_LOG_FW_SLOT, log, sizeof(*log))) 3305 dev_warn(ctrl->device, 3306 "Get FW SLOT INFO log error\n"); 3307 kfree(log); 3308 } 3309 3310 static void nvme_fw_act_work(struct work_struct *work) 3311 { 3312 struct nvme_ctrl *ctrl = container_of(work, 3313 struct nvme_ctrl, fw_act_work); 3314 unsigned long fw_act_timeout; 3315 3316 if (ctrl->mtfa) 3317 fw_act_timeout = jiffies + 3318 msecs_to_jiffies(ctrl->mtfa * 100); 3319 else 3320 fw_act_timeout = jiffies + 3321 msecs_to_jiffies(admin_timeout * 1000); 3322 3323 nvme_stop_queues(ctrl); 3324 while (nvme_ctrl_pp_status(ctrl)) { 3325 if (time_after(jiffies, fw_act_timeout)) { 3326 dev_warn(ctrl->device, 3327 "Fw activation timeout, reset controller\n"); 3328 nvme_reset_ctrl(ctrl); 3329 break; 3330 } 3331 msleep(100); 3332 } 3333 3334 if (ctrl->state != NVME_CTRL_LIVE) 3335 return; 3336 3337 nvme_start_queues(ctrl); 3338 /* read FW slot information to clear the AER */ 3339 nvme_get_fw_slot_info(ctrl); 3340 } 3341 3342 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status, 3343 union nvme_result *res) 3344 { 3345 u32 result = le32_to_cpu(res->u32); 3346 3347 if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS) 3348 return; 3349 3350 switch (result & 0x7) { 3351 case NVME_AER_ERROR: 3352 case NVME_AER_SMART: 3353 case NVME_AER_CSS: 3354 case NVME_AER_VS: 3355 ctrl->aen_result = result; 3356 break; 3357 default: 3358 break; 3359 } 3360 3361 switch (result & 0xff07) { 3362 case NVME_AER_NOTICE_NS_CHANGED: 3363 dev_info(ctrl->device, "rescanning\n"); 3364 nvme_queue_scan(ctrl); 3365 break; 3366 case NVME_AER_NOTICE_FW_ACT_STARTING: 3367 queue_work(nvme_wq, &ctrl->fw_act_work); 3368 break; 3369 default: 3370 dev_warn(ctrl->device, "async event result %08x\n", result); 3371 } 3372 queue_work(nvme_wq, &ctrl->async_event_work); 3373 } 3374 EXPORT_SYMBOL_GPL(nvme_complete_async_event); 3375 3376 void nvme_stop_ctrl(struct nvme_ctrl *ctrl) 3377 { 3378 nvme_stop_keep_alive(ctrl); 3379 flush_work(&ctrl->async_event_work); 3380 flush_work(&ctrl->scan_work); 3381 cancel_work_sync(&ctrl->fw_act_work); 3382 if (ctrl->ops->stop_ctrl) 3383 ctrl->ops->stop_ctrl(ctrl); 3384 } 3385 EXPORT_SYMBOL_GPL(nvme_stop_ctrl); 3386 3387 void nvme_start_ctrl(struct nvme_ctrl *ctrl) 3388 { 3389 if (ctrl->kato) 3390 nvme_start_keep_alive(ctrl); 3391 3392 if (ctrl->queue_count > 1) { 3393 nvme_queue_scan(ctrl); 3394 queue_work(nvme_wq, &ctrl->async_event_work); 3395 nvme_start_queues(ctrl); 3396 } 3397 } 3398 EXPORT_SYMBOL_GPL(nvme_start_ctrl); 3399 3400 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl) 3401 { 3402 cdev_device_del(&ctrl->cdev, ctrl->device); 3403 } 3404 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl); 3405 3406 static void nvme_free_ctrl(struct device *dev) 3407 { 3408 struct nvme_ctrl *ctrl = 3409 container_of(dev, struct nvme_ctrl, ctrl_device); 3410 struct nvme_subsystem *subsys = ctrl->subsys; 3411 3412 ida_simple_remove(&nvme_instance_ida, ctrl->instance); 3413 kfree(ctrl->effects); 3414 3415 if (subsys) { 3416 mutex_lock(&subsys->lock); 3417 list_del(&ctrl->subsys_entry); 3418 mutex_unlock(&subsys->lock); 3419 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device)); 3420 } 3421 3422 ctrl->ops->free_ctrl(ctrl); 3423 3424 if (subsys) 3425 nvme_put_subsystem(subsys); 3426 } 3427 3428 /* 3429 * Initialize a NVMe controller structures. This needs to be called during 3430 * earliest initialization so that we have the initialized structured around 3431 * during probing. 3432 */ 3433 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, 3434 const struct nvme_ctrl_ops *ops, unsigned long quirks) 3435 { 3436 int ret; 3437 3438 ctrl->state = NVME_CTRL_NEW; 3439 spin_lock_init(&ctrl->lock); 3440 INIT_LIST_HEAD(&ctrl->namespaces); 3441 init_rwsem(&ctrl->namespaces_rwsem); 3442 ctrl->dev = dev; 3443 ctrl->ops = ops; 3444 ctrl->quirks = quirks; 3445 INIT_WORK(&ctrl->scan_work, nvme_scan_work); 3446 INIT_WORK(&ctrl->async_event_work, nvme_async_event_work); 3447 INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work); 3448 INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work); 3449 3450 ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL); 3451 if (ret < 0) 3452 goto out; 3453 ctrl->instance = ret; 3454 3455 device_initialize(&ctrl->ctrl_device); 3456 ctrl->device = &ctrl->ctrl_device; 3457 ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance); 3458 ctrl->device->class = nvme_class; 3459 ctrl->device->parent = ctrl->dev; 3460 ctrl->device->groups = nvme_dev_attr_groups; 3461 ctrl->device->release = nvme_free_ctrl; 3462 dev_set_drvdata(ctrl->device, ctrl); 3463 ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance); 3464 if (ret) 3465 goto out_release_instance; 3466 3467 cdev_init(&ctrl->cdev, &nvme_dev_fops); 3468 ctrl->cdev.owner = ops->module; 3469 ret = cdev_device_add(&ctrl->cdev, ctrl->device); 3470 if (ret) 3471 goto out_free_name; 3472 3473 /* 3474 * Initialize latency tolerance controls. The sysfs files won't 3475 * be visible to userspace unless the device actually supports APST. 3476 */ 3477 ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance; 3478 dev_pm_qos_update_user_latency_tolerance(ctrl->device, 3479 min(default_ps_max_latency_us, (unsigned long)S32_MAX)); 3480 3481 return 0; 3482 out_free_name: 3483 kfree_const(dev->kobj.name); 3484 out_release_instance: 3485 ida_simple_remove(&nvme_instance_ida, ctrl->instance); 3486 out: 3487 return ret; 3488 } 3489 EXPORT_SYMBOL_GPL(nvme_init_ctrl); 3490 3491 /** 3492 * nvme_kill_queues(): Ends all namespace queues 3493 * @ctrl: the dead controller that needs to end 3494 * 3495 * Call this function when the driver determines it is unable to get the 3496 * controller in a state capable of servicing IO. 3497 */ 3498 void nvme_kill_queues(struct nvme_ctrl *ctrl) 3499 { 3500 struct nvme_ns *ns; 3501 3502 down_read(&ctrl->namespaces_rwsem); 3503 3504 /* Forcibly unquiesce queues to avoid blocking dispatch */ 3505 if (ctrl->admin_q) 3506 blk_mq_unquiesce_queue(ctrl->admin_q); 3507 3508 list_for_each_entry(ns, &ctrl->namespaces, list) { 3509 /* 3510 * Revalidating a dead namespace sets capacity to 0. This will 3511 * end buffered writers dirtying pages that can't be synced. 3512 */ 3513 if (!ns->disk || test_and_set_bit(NVME_NS_DEAD, &ns->flags)) 3514 continue; 3515 revalidate_disk(ns->disk); 3516 blk_set_queue_dying(ns->queue); 3517 3518 /* Forcibly unquiesce queues to avoid blocking dispatch */ 3519 blk_mq_unquiesce_queue(ns->queue); 3520 } 3521 up_read(&ctrl->namespaces_rwsem); 3522 } 3523 EXPORT_SYMBOL_GPL(nvme_kill_queues); 3524 3525 void nvme_unfreeze(struct nvme_ctrl *ctrl) 3526 { 3527 struct nvme_ns *ns; 3528 3529 down_read(&ctrl->namespaces_rwsem); 3530 list_for_each_entry(ns, &ctrl->namespaces, list) 3531 blk_mq_unfreeze_queue(ns->queue); 3532 up_read(&ctrl->namespaces_rwsem); 3533 } 3534 EXPORT_SYMBOL_GPL(nvme_unfreeze); 3535 3536 void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout) 3537 { 3538 struct nvme_ns *ns; 3539 3540 down_read(&ctrl->namespaces_rwsem); 3541 list_for_each_entry(ns, &ctrl->namespaces, list) { 3542 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout); 3543 if (timeout <= 0) 3544 break; 3545 } 3546 up_read(&ctrl->namespaces_rwsem); 3547 } 3548 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout); 3549 3550 void nvme_wait_freeze(struct nvme_ctrl *ctrl) 3551 { 3552 struct nvme_ns *ns; 3553 3554 down_read(&ctrl->namespaces_rwsem); 3555 list_for_each_entry(ns, &ctrl->namespaces, list) 3556 blk_mq_freeze_queue_wait(ns->queue); 3557 up_read(&ctrl->namespaces_rwsem); 3558 } 3559 EXPORT_SYMBOL_GPL(nvme_wait_freeze); 3560 3561 void nvme_start_freeze(struct nvme_ctrl *ctrl) 3562 { 3563 struct nvme_ns *ns; 3564 3565 down_read(&ctrl->namespaces_rwsem); 3566 list_for_each_entry(ns, &ctrl->namespaces, list) 3567 blk_freeze_queue_start(ns->queue); 3568 up_read(&ctrl->namespaces_rwsem); 3569 } 3570 EXPORT_SYMBOL_GPL(nvme_start_freeze); 3571 3572 void nvme_stop_queues(struct nvme_ctrl *ctrl) 3573 { 3574 struct nvme_ns *ns; 3575 3576 down_read(&ctrl->namespaces_rwsem); 3577 list_for_each_entry(ns, &ctrl->namespaces, list) 3578 blk_mq_quiesce_queue(ns->queue); 3579 up_read(&ctrl->namespaces_rwsem); 3580 } 3581 EXPORT_SYMBOL_GPL(nvme_stop_queues); 3582 3583 void nvme_start_queues(struct nvme_ctrl *ctrl) 3584 { 3585 struct nvme_ns *ns; 3586 3587 down_read(&ctrl->namespaces_rwsem); 3588 list_for_each_entry(ns, &ctrl->namespaces, list) 3589 blk_mq_unquiesce_queue(ns->queue); 3590 up_read(&ctrl->namespaces_rwsem); 3591 } 3592 EXPORT_SYMBOL_GPL(nvme_start_queues); 3593 3594 int nvme_reinit_tagset(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set) 3595 { 3596 if (!ctrl->ops->reinit_request) 3597 return 0; 3598 3599 return blk_mq_tagset_iter(set, set->driver_data, 3600 ctrl->ops->reinit_request); 3601 } 3602 EXPORT_SYMBOL_GPL(nvme_reinit_tagset); 3603 3604 int __init nvme_core_init(void) 3605 { 3606 int result = -ENOMEM; 3607 3608 nvme_wq = alloc_workqueue("nvme-wq", 3609 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 3610 if (!nvme_wq) 3611 goto out; 3612 3613 nvme_reset_wq = alloc_workqueue("nvme-reset-wq", 3614 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 3615 if (!nvme_reset_wq) 3616 goto destroy_wq; 3617 3618 nvme_delete_wq = alloc_workqueue("nvme-delete-wq", 3619 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 3620 if (!nvme_delete_wq) 3621 goto destroy_reset_wq; 3622 3623 result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme"); 3624 if (result < 0) 3625 goto destroy_delete_wq; 3626 3627 nvme_class = class_create(THIS_MODULE, "nvme"); 3628 if (IS_ERR(nvme_class)) { 3629 result = PTR_ERR(nvme_class); 3630 goto unregister_chrdev; 3631 } 3632 3633 nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem"); 3634 if (IS_ERR(nvme_subsys_class)) { 3635 result = PTR_ERR(nvme_subsys_class); 3636 goto destroy_class; 3637 } 3638 return 0; 3639 3640 destroy_class: 3641 class_destroy(nvme_class); 3642 unregister_chrdev: 3643 unregister_chrdev_region(nvme_chr_devt, NVME_MINORS); 3644 destroy_delete_wq: 3645 destroy_workqueue(nvme_delete_wq); 3646 destroy_reset_wq: 3647 destroy_workqueue(nvme_reset_wq); 3648 destroy_wq: 3649 destroy_workqueue(nvme_wq); 3650 out: 3651 return result; 3652 } 3653 3654 void nvme_core_exit(void) 3655 { 3656 ida_destroy(&nvme_subsystems_ida); 3657 class_destroy(nvme_subsys_class); 3658 class_destroy(nvme_class); 3659 unregister_chrdev_region(nvme_chr_devt, NVME_MINORS); 3660 destroy_workqueue(nvme_delete_wq); 3661 destroy_workqueue(nvme_reset_wq); 3662 destroy_workqueue(nvme_wq); 3663 } 3664 3665 MODULE_LICENSE("GPL"); 3666 MODULE_VERSION("1.0"); 3667 module_init(nvme_core_init); 3668 module_exit(nvme_core_exit); 3669