1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2017-2018 Christoph Hellwig. 4 */ 5 6 #include <linux/backing-dev.h> 7 #include <linux/moduleparam.h> 8 #include <linux/vmalloc.h> 9 #include <trace/events/block.h> 10 #include "nvme.h" 11 12 bool multipath = true; 13 module_param(multipath, bool, 0444); 14 MODULE_PARM_DESC(multipath, 15 "turn on native support for multiple controllers per subsystem"); 16 17 static const char *nvme_iopolicy_names[] = { 18 [NVME_IOPOLICY_NUMA] = "numa", 19 [NVME_IOPOLICY_RR] = "round-robin", 20 [NVME_IOPOLICY_QD] = "queue-depth", 21 }; 22 23 static int iopolicy = NVME_IOPOLICY_NUMA; 24 25 static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp) 26 { 27 if (!val) 28 return -EINVAL; 29 if (!strncmp(val, "numa", 4)) 30 iopolicy = NVME_IOPOLICY_NUMA; 31 else if (!strncmp(val, "round-robin", 11)) 32 iopolicy = NVME_IOPOLICY_RR; 33 else if (!strncmp(val, "queue-depth", 11)) 34 iopolicy = NVME_IOPOLICY_QD; 35 else 36 return -EINVAL; 37 38 return 0; 39 } 40 41 static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp) 42 { 43 return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]); 44 } 45 46 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy, 47 &iopolicy, 0644); 48 MODULE_PARM_DESC(iopolicy, 49 "Default multipath I/O policy; 'numa' (default), 'round-robin' or 'queue-depth'"); 50 51 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys) 52 { 53 subsys->iopolicy = iopolicy; 54 } 55 56 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) 57 { 58 struct nvme_ns_head *h; 59 60 lockdep_assert_held(&subsys->lock); 61 list_for_each_entry(h, &subsys->nsheads, entry) 62 if (h->disk) 63 blk_mq_unfreeze_queue(h->disk->queue); 64 } 65 66 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) 67 { 68 struct nvme_ns_head *h; 69 70 lockdep_assert_held(&subsys->lock); 71 list_for_each_entry(h, &subsys->nsheads, entry) 72 if (h->disk) 73 blk_mq_freeze_queue_wait(h->disk->queue); 74 } 75 76 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) 77 { 78 struct nvme_ns_head *h; 79 80 lockdep_assert_held(&subsys->lock); 81 list_for_each_entry(h, &subsys->nsheads, entry) 82 if (h->disk) 83 blk_freeze_queue_start(h->disk->queue); 84 } 85 86 void nvme_failover_req(struct request *req) 87 { 88 struct nvme_ns *ns = req->q->queuedata; 89 u16 status = nvme_req(req)->status & 0x7ff; 90 unsigned long flags; 91 struct bio *bio; 92 93 nvme_mpath_clear_current_path(ns); 94 95 /* 96 * If we got back an ANA error, we know the controller is alive but not 97 * ready to serve this namespace. Kick of a re-read of the ANA 98 * information page, and just try any other available path for now. 99 */ 100 if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) { 101 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 102 queue_work(nvme_wq, &ns->ctrl->ana_work); 103 } 104 105 spin_lock_irqsave(&ns->head->requeue_lock, flags); 106 for (bio = req->bio; bio; bio = bio->bi_next) { 107 bio_set_dev(bio, ns->head->disk->part0); 108 if (bio->bi_opf & REQ_POLLED) { 109 bio->bi_opf &= ~REQ_POLLED; 110 bio->bi_cookie = BLK_QC_T_NONE; 111 } 112 /* 113 * The alternate request queue that we may end up submitting 114 * the bio to may be frozen temporarily, in this case REQ_NOWAIT 115 * will fail the I/O immediately with EAGAIN to the issuer. 116 * We are not in the issuer context which cannot block. Clear 117 * the flag to avoid spurious EAGAIN I/O failures. 118 */ 119 bio->bi_opf &= ~REQ_NOWAIT; 120 } 121 blk_steal_bios(&ns->head->requeue_list, req); 122 spin_unlock_irqrestore(&ns->head->requeue_lock, flags); 123 124 nvme_req(req)->status = 0; 125 nvme_end_req(req); 126 kblockd_schedule_work(&ns->head->requeue_work); 127 } 128 129 void nvme_mpath_start_request(struct request *rq) 130 { 131 struct nvme_ns *ns = rq->q->queuedata; 132 struct gendisk *disk = ns->head->disk; 133 134 if (READ_ONCE(ns->head->subsys->iopolicy) == NVME_IOPOLICY_QD) { 135 atomic_inc(&ns->ctrl->nr_active); 136 nvme_req(rq)->flags |= NVME_MPATH_CNT_ACTIVE; 137 } 138 139 if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq)) 140 return; 141 142 nvme_req(rq)->flags |= NVME_MPATH_IO_STATS; 143 nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq), 144 jiffies); 145 } 146 EXPORT_SYMBOL_GPL(nvme_mpath_start_request); 147 148 void nvme_mpath_end_request(struct request *rq) 149 { 150 struct nvme_ns *ns = rq->q->queuedata; 151 152 if (nvme_req(rq)->flags & NVME_MPATH_CNT_ACTIVE) 153 atomic_dec_if_positive(&ns->ctrl->nr_active); 154 155 if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS)) 156 return; 157 bdev_end_io_acct(ns->head->disk->part0, req_op(rq), 158 blk_rq_bytes(rq) >> SECTOR_SHIFT, 159 nvme_req(rq)->start_time); 160 } 161 162 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) 163 { 164 struct nvme_ns *ns; 165 int srcu_idx; 166 167 srcu_idx = srcu_read_lock(&ctrl->srcu); 168 list_for_each_entry_srcu(ns, &ctrl->namespaces, list, 169 srcu_read_lock_held(&ctrl->srcu)) { 170 if (!ns->head->disk) 171 continue; 172 kblockd_schedule_work(&ns->head->requeue_work); 173 if (ctrl->state == NVME_CTRL_LIVE) 174 disk_uevent(ns->head->disk, KOBJ_CHANGE); 175 } 176 srcu_read_unlock(&ctrl->srcu, srcu_idx); 177 } 178 179 static const char *nvme_ana_state_names[] = { 180 [0] = "invalid state", 181 [NVME_ANA_OPTIMIZED] = "optimized", 182 [NVME_ANA_NONOPTIMIZED] = "non-optimized", 183 [NVME_ANA_INACCESSIBLE] = "inaccessible", 184 [NVME_ANA_PERSISTENT_LOSS] = "persistent-loss", 185 [NVME_ANA_CHANGE] = "change", 186 }; 187 188 bool nvme_mpath_clear_current_path(struct nvme_ns *ns) 189 { 190 struct nvme_ns_head *head = ns->head; 191 bool changed = false; 192 int node; 193 194 if (!head) 195 goto out; 196 197 for_each_node(node) { 198 if (ns == rcu_access_pointer(head->current_path[node])) { 199 rcu_assign_pointer(head->current_path[node], NULL); 200 changed = true; 201 } 202 } 203 out: 204 return changed; 205 } 206 207 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) 208 { 209 struct nvme_ns *ns; 210 int srcu_idx; 211 212 srcu_idx = srcu_read_lock(&ctrl->srcu); 213 list_for_each_entry_srcu(ns, &ctrl->namespaces, list, 214 srcu_read_lock_held(&ctrl->srcu)) { 215 nvme_mpath_clear_current_path(ns); 216 kblockd_schedule_work(&ns->head->requeue_work); 217 } 218 srcu_read_unlock(&ctrl->srcu, srcu_idx); 219 } 220 221 void nvme_mpath_revalidate_paths(struct nvme_ns *ns) 222 { 223 struct nvme_ns_head *head = ns->head; 224 sector_t capacity = get_capacity(head->disk); 225 int node; 226 int srcu_idx; 227 228 srcu_idx = srcu_read_lock(&head->srcu); 229 list_for_each_entry_srcu(ns, &head->list, siblings, 230 srcu_read_lock_held(&head->srcu)) { 231 if (capacity != get_capacity(ns->disk)) 232 clear_bit(NVME_NS_READY, &ns->flags); 233 } 234 srcu_read_unlock(&head->srcu, srcu_idx); 235 236 for_each_node(node) 237 rcu_assign_pointer(head->current_path[node], NULL); 238 kblockd_schedule_work(&head->requeue_work); 239 } 240 241 static bool nvme_path_is_disabled(struct nvme_ns *ns) 242 { 243 /* 244 * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should 245 * still be able to complete assuming that the controller is connected. 246 * Otherwise it will fail immediately and return to the requeue list. 247 */ 248 if (ns->ctrl->state != NVME_CTRL_LIVE && 249 ns->ctrl->state != NVME_CTRL_DELETING) 250 return true; 251 if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) || 252 !test_bit(NVME_NS_READY, &ns->flags)) 253 return true; 254 return false; 255 } 256 257 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) 258 { 259 int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; 260 struct nvme_ns *found = NULL, *fallback = NULL, *ns; 261 262 list_for_each_entry_srcu(ns, &head->list, siblings, 263 srcu_read_lock_held(&head->srcu)) { 264 if (nvme_path_is_disabled(ns)) 265 continue; 266 267 if (ns->ctrl->numa_node != NUMA_NO_NODE && 268 READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA) 269 distance = node_distance(node, ns->ctrl->numa_node); 270 else 271 distance = LOCAL_DISTANCE; 272 273 switch (ns->ana_state) { 274 case NVME_ANA_OPTIMIZED: 275 if (distance < found_distance) { 276 found_distance = distance; 277 found = ns; 278 } 279 break; 280 case NVME_ANA_NONOPTIMIZED: 281 if (distance < fallback_distance) { 282 fallback_distance = distance; 283 fallback = ns; 284 } 285 break; 286 default: 287 break; 288 } 289 } 290 291 if (!found) 292 found = fallback; 293 if (found) 294 rcu_assign_pointer(head->current_path[node], found); 295 return found; 296 } 297 298 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, 299 struct nvme_ns *ns) 300 { 301 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, 302 siblings); 303 if (ns) 304 return ns; 305 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings); 306 } 307 308 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head) 309 { 310 struct nvme_ns *ns, *found = NULL; 311 int node = numa_node_id(); 312 struct nvme_ns *old = srcu_dereference(head->current_path[node], 313 &head->srcu); 314 315 if (unlikely(!old)) 316 return __nvme_find_path(head, node); 317 318 if (list_is_singular(&head->list)) { 319 if (nvme_path_is_disabled(old)) 320 return NULL; 321 return old; 322 } 323 324 for (ns = nvme_next_ns(head, old); 325 ns && ns != old; 326 ns = nvme_next_ns(head, ns)) { 327 if (nvme_path_is_disabled(ns)) 328 continue; 329 330 if (ns->ana_state == NVME_ANA_OPTIMIZED) { 331 found = ns; 332 goto out; 333 } 334 if (ns->ana_state == NVME_ANA_NONOPTIMIZED) 335 found = ns; 336 } 337 338 /* 339 * The loop above skips the current path for round-robin semantics. 340 * Fall back to the current path if either: 341 * - no other optimized path found and current is optimized, 342 * - no other usable path found and current is usable. 343 */ 344 if (!nvme_path_is_disabled(old) && 345 (old->ana_state == NVME_ANA_OPTIMIZED || 346 (!found && old->ana_state == NVME_ANA_NONOPTIMIZED))) 347 return old; 348 349 if (!found) 350 return NULL; 351 out: 352 rcu_assign_pointer(head->current_path[node], found); 353 return found; 354 } 355 356 static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head) 357 { 358 struct nvme_ns *best_opt = NULL, *best_nonopt = NULL, *ns; 359 unsigned int min_depth_opt = UINT_MAX, min_depth_nonopt = UINT_MAX; 360 unsigned int depth; 361 362 list_for_each_entry_srcu(ns, &head->list, siblings, 363 srcu_read_lock_held(&head->srcu)) { 364 if (nvme_path_is_disabled(ns)) 365 continue; 366 367 depth = atomic_read(&ns->ctrl->nr_active); 368 369 switch (ns->ana_state) { 370 case NVME_ANA_OPTIMIZED: 371 if (depth < min_depth_opt) { 372 min_depth_opt = depth; 373 best_opt = ns; 374 } 375 break; 376 case NVME_ANA_NONOPTIMIZED: 377 if (depth < min_depth_nonopt) { 378 min_depth_nonopt = depth; 379 best_nonopt = ns; 380 } 381 break; 382 default: 383 break; 384 } 385 386 if (min_depth_opt == 0) 387 return best_opt; 388 } 389 390 return best_opt ? best_opt : best_nonopt; 391 } 392 393 static inline bool nvme_path_is_optimized(struct nvme_ns *ns) 394 { 395 return ns->ctrl->state == NVME_CTRL_LIVE && 396 ns->ana_state == NVME_ANA_OPTIMIZED; 397 } 398 399 static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head) 400 { 401 int node = numa_node_id(); 402 struct nvme_ns *ns; 403 404 ns = srcu_dereference(head->current_path[node], &head->srcu); 405 if (unlikely(!ns)) 406 return __nvme_find_path(head, node); 407 if (unlikely(!nvme_path_is_optimized(ns))) 408 return __nvme_find_path(head, node); 409 return ns; 410 } 411 412 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) 413 { 414 switch (READ_ONCE(head->subsys->iopolicy)) { 415 case NVME_IOPOLICY_QD: 416 return nvme_queue_depth_path(head); 417 case NVME_IOPOLICY_RR: 418 return nvme_round_robin_path(head); 419 default: 420 return nvme_numa_path(head); 421 } 422 } 423 424 static bool nvme_available_path(struct nvme_ns_head *head) 425 { 426 struct nvme_ns *ns; 427 428 if (!test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) 429 return NULL; 430 431 list_for_each_entry_srcu(ns, &head->list, siblings, 432 srcu_read_lock_held(&head->srcu)) { 433 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags)) 434 continue; 435 switch (ns->ctrl->state) { 436 case NVME_CTRL_LIVE: 437 case NVME_CTRL_RESETTING: 438 case NVME_CTRL_CONNECTING: 439 /* fallthru */ 440 return true; 441 default: 442 break; 443 } 444 } 445 return false; 446 } 447 448 static void nvme_ns_head_submit_bio(struct bio *bio) 449 { 450 struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; 451 struct device *dev = disk_to_dev(head->disk); 452 struct nvme_ns *ns; 453 int srcu_idx; 454 455 /* 456 * The namespace might be going away and the bio might be moved to a 457 * different queue via blk_steal_bios(), so we need to use the bio_split 458 * pool from the original queue to allocate the bvecs from. 459 */ 460 bio = bio_split_to_limits(bio); 461 if (!bio) 462 return; 463 464 srcu_idx = srcu_read_lock(&head->srcu); 465 ns = nvme_find_path(head); 466 if (likely(ns)) { 467 bio_set_dev(bio, ns->disk->part0); 468 bio->bi_opf |= REQ_NVME_MPATH; 469 trace_block_bio_remap(bio, disk_devt(ns->head->disk), 470 bio->bi_iter.bi_sector); 471 submit_bio_noacct(bio); 472 } else if (nvme_available_path(head)) { 473 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); 474 475 spin_lock_irq(&head->requeue_lock); 476 bio_list_add(&head->requeue_list, bio); 477 spin_unlock_irq(&head->requeue_lock); 478 } else { 479 dev_warn_ratelimited(dev, "no available path - failing I/O\n"); 480 481 bio_io_error(bio); 482 } 483 484 srcu_read_unlock(&head->srcu, srcu_idx); 485 } 486 487 static int nvme_ns_head_open(struct gendisk *disk, blk_mode_t mode) 488 { 489 if (!nvme_tryget_ns_head(disk->private_data)) 490 return -ENXIO; 491 return 0; 492 } 493 494 static void nvme_ns_head_release(struct gendisk *disk) 495 { 496 nvme_put_ns_head(disk->private_data); 497 } 498 499 #ifdef CONFIG_BLK_DEV_ZONED 500 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector, 501 unsigned int nr_zones, report_zones_cb cb, void *data) 502 { 503 struct nvme_ns_head *head = disk->private_data; 504 struct nvme_ns *ns; 505 int srcu_idx, ret = -EWOULDBLOCK; 506 507 srcu_idx = srcu_read_lock(&head->srcu); 508 ns = nvme_find_path(head); 509 if (ns) 510 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data); 511 srcu_read_unlock(&head->srcu, srcu_idx); 512 return ret; 513 } 514 #else 515 #define nvme_ns_head_report_zones NULL 516 #endif /* CONFIG_BLK_DEV_ZONED */ 517 518 const struct block_device_operations nvme_ns_head_ops = { 519 .owner = THIS_MODULE, 520 .submit_bio = nvme_ns_head_submit_bio, 521 .open = nvme_ns_head_open, 522 .release = nvme_ns_head_release, 523 .ioctl = nvme_ns_head_ioctl, 524 .compat_ioctl = blkdev_compat_ptr_ioctl, 525 .getgeo = nvme_getgeo, 526 .report_zones = nvme_ns_head_report_zones, 527 .pr_ops = &nvme_pr_ops, 528 }; 529 530 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev) 531 { 532 return container_of(cdev, struct nvme_ns_head, cdev); 533 } 534 535 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file) 536 { 537 if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev))) 538 return -ENXIO; 539 return 0; 540 } 541 542 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file) 543 { 544 nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev)); 545 return 0; 546 } 547 548 static const struct file_operations nvme_ns_head_chr_fops = { 549 .owner = THIS_MODULE, 550 .open = nvme_ns_head_chr_open, 551 .release = nvme_ns_head_chr_release, 552 .unlocked_ioctl = nvme_ns_head_chr_ioctl, 553 .compat_ioctl = compat_ptr_ioctl, 554 .uring_cmd = nvme_ns_head_chr_uring_cmd, 555 .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll, 556 }; 557 558 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) 559 { 560 int ret; 561 562 head->cdev_device.parent = &head->subsys->dev; 563 ret = dev_set_name(&head->cdev_device, "ng%dn%d", 564 head->subsys->instance, head->instance); 565 if (ret) 566 return ret; 567 ret = nvme_cdev_add(&head->cdev, &head->cdev_device, 568 &nvme_ns_head_chr_fops, THIS_MODULE); 569 return ret; 570 } 571 572 static void nvme_partition_scan_work(struct work_struct *work) 573 { 574 struct nvme_ns_head *head = 575 container_of(work, struct nvme_ns_head, partition_scan_work); 576 577 if (WARN_ON_ONCE(!test_and_clear_bit(GD_SUPPRESS_PART_SCAN, 578 &head->disk->state))) 579 return; 580 581 mutex_lock(&head->disk->open_mutex); 582 bdev_disk_changed(head->disk, false); 583 mutex_unlock(&head->disk->open_mutex); 584 } 585 586 static void nvme_requeue_work(struct work_struct *work) 587 { 588 struct nvme_ns_head *head = 589 container_of(work, struct nvme_ns_head, requeue_work); 590 struct bio *bio, *next; 591 592 spin_lock_irq(&head->requeue_lock); 593 next = bio_list_get(&head->requeue_list); 594 spin_unlock_irq(&head->requeue_lock); 595 596 while ((bio = next) != NULL) { 597 next = bio->bi_next; 598 bio->bi_next = NULL; 599 600 submit_bio_noacct(bio); 601 } 602 } 603 604 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) 605 { 606 bool vwc = false; 607 608 mutex_init(&head->lock); 609 bio_list_init(&head->requeue_list); 610 spin_lock_init(&head->requeue_lock); 611 INIT_WORK(&head->requeue_work, nvme_requeue_work); 612 INIT_WORK(&head->partition_scan_work, nvme_partition_scan_work); 613 614 /* 615 * Add a multipath node if the subsystems supports multiple controllers. 616 * We also do this for private namespaces as the namespace sharing flag 617 * could change after a rescan. 618 */ 619 if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || 620 !nvme_is_unique_nsid(ctrl, head) || !multipath) 621 return 0; 622 623 head->disk = blk_alloc_disk(ctrl->numa_node); 624 if (!head->disk) 625 return -ENOMEM; 626 head->disk->fops = &nvme_ns_head_ops; 627 head->disk->private_data = head; 628 629 /* 630 * We need to suppress the partition scan from occuring within the 631 * controller's scan_work context. If a path error occurs here, the IO 632 * will wait until a path becomes available or all paths are torn down, 633 * but that action also occurs within scan_work, so it would deadlock. 634 * Defer the partion scan to a different context that does not block 635 * scan_work. 636 */ 637 set_bit(GD_SUPPRESS_PART_SCAN, &head->disk->state); 638 sprintf(head->disk->disk_name, "nvme%dn%d", 639 ctrl->subsys->instance, head->instance); 640 641 blk_queue_flag_set(QUEUE_FLAG_NONROT, head->disk->queue); 642 blk_queue_flag_set(QUEUE_FLAG_NOWAIT, head->disk->queue); 643 blk_queue_flag_set(QUEUE_FLAG_IO_STAT, head->disk->queue); 644 /* 645 * This assumes all controllers that refer to a namespace either 646 * support poll queues or not. That is not a strict guarantee, 647 * but if the assumption is wrong the effect is only suboptimal 648 * performance but not correctness problem. 649 */ 650 if (ctrl->tagset->nr_maps > HCTX_TYPE_POLL && 651 ctrl->tagset->map[HCTX_TYPE_POLL].nr_queues) 652 blk_queue_flag_set(QUEUE_FLAG_POLL, head->disk->queue); 653 654 /* set to a default value of 512 until the disk is validated */ 655 blk_queue_logical_block_size(head->disk->queue, 512); 656 blk_set_stacking_limits(&head->disk->queue->limits); 657 blk_queue_dma_alignment(head->disk->queue, 3); 658 659 /* we need to propagate up the VMC settings */ 660 if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) 661 vwc = true; 662 blk_queue_write_cache(head->disk->queue, vwc, vwc); 663 return 0; 664 } 665 666 static void nvme_mpath_set_live(struct nvme_ns *ns) 667 { 668 struct nvme_ns_head *head = ns->head; 669 int rc; 670 671 if (!head->disk) 672 return; 673 674 /* 675 * test_and_set_bit() is used because it is protecting against two nvme 676 * paths simultaneously calling device_add_disk() on the same namespace 677 * head. 678 */ 679 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 680 rc = device_add_disk(&head->subsys->dev, head->disk, 681 nvme_ns_id_attr_groups); 682 if (rc) { 683 clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags); 684 return; 685 } 686 nvme_add_ns_head_cdev(head); 687 kblockd_schedule_work(&head->partition_scan_work); 688 } 689 690 mutex_lock(&head->lock); 691 if (nvme_path_is_optimized(ns)) { 692 int node, srcu_idx; 693 694 srcu_idx = srcu_read_lock(&head->srcu); 695 for_each_online_node(node) 696 __nvme_find_path(head, node); 697 srcu_read_unlock(&head->srcu, srcu_idx); 698 } 699 mutex_unlock(&head->lock); 700 701 synchronize_srcu(&head->srcu); 702 kblockd_schedule_work(&head->requeue_work); 703 } 704 705 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, 706 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, 707 void *)) 708 { 709 void *base = ctrl->ana_log_buf; 710 size_t offset = sizeof(struct nvme_ana_rsp_hdr); 711 int error, i; 712 713 lockdep_assert_held(&ctrl->ana_lock); 714 715 for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) { 716 struct nvme_ana_group_desc *desc = base + offset; 717 u32 nr_nsids; 718 size_t nsid_buf_size; 719 720 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc))) 721 return -EINVAL; 722 723 nr_nsids = le32_to_cpu(desc->nnsids); 724 nsid_buf_size = flex_array_size(desc, nsids, nr_nsids); 725 726 if (WARN_ON_ONCE(desc->grpid == 0)) 727 return -EINVAL; 728 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax)) 729 return -EINVAL; 730 if (WARN_ON_ONCE(desc->state == 0)) 731 return -EINVAL; 732 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE)) 733 return -EINVAL; 734 735 offset += sizeof(*desc); 736 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size)) 737 return -EINVAL; 738 739 error = cb(ctrl, desc, data); 740 if (error) 741 return error; 742 743 offset += nsid_buf_size; 744 } 745 746 return 0; 747 } 748 749 static inline bool nvme_state_is_live(enum nvme_ana_state state) 750 { 751 return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED; 752 } 753 754 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, 755 struct nvme_ns *ns) 756 { 757 ns->ana_grpid = le32_to_cpu(desc->grpid); 758 ns->ana_state = desc->state; 759 clear_bit(NVME_NS_ANA_PENDING, &ns->flags); 760 /* 761 * nvme_mpath_set_live() will trigger I/O to the multipath path device 762 * and in turn to this path device. However we cannot accept this I/O 763 * if the controller is not live. This may deadlock if called from 764 * nvme_mpath_init_identify() and the ctrl will never complete 765 * initialization, preventing I/O from completing. For this case we 766 * will reprocess the ANA log page in nvme_mpath_update() once the 767 * controller is ready. 768 */ 769 if (nvme_state_is_live(ns->ana_state) && 770 ns->ctrl->state == NVME_CTRL_LIVE) 771 nvme_mpath_set_live(ns); 772 } 773 774 static int nvme_update_ana_state(struct nvme_ctrl *ctrl, 775 struct nvme_ana_group_desc *desc, void *data) 776 { 777 u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0; 778 unsigned *nr_change_groups = data; 779 struct nvme_ns *ns; 780 int srcu_idx; 781 782 dev_dbg(ctrl->device, "ANA group %d: %s.\n", 783 le32_to_cpu(desc->grpid), 784 nvme_ana_state_names[desc->state]); 785 786 if (desc->state == NVME_ANA_CHANGE) 787 (*nr_change_groups)++; 788 789 if (!nr_nsids) 790 return 0; 791 792 srcu_idx = srcu_read_lock(&ctrl->srcu); 793 list_for_each_entry_srcu(ns, &ctrl->namespaces, list, 794 srcu_read_lock_held(&ctrl->srcu)) { 795 unsigned nsid; 796 again: 797 nsid = le32_to_cpu(desc->nsids[n]); 798 if (ns->head->ns_id < nsid) 799 continue; 800 if (ns->head->ns_id == nsid) 801 nvme_update_ns_ana_state(desc, ns); 802 if (++n == nr_nsids) 803 break; 804 if (ns->head->ns_id > nsid) 805 goto again; 806 } 807 srcu_read_unlock(&ctrl->srcu, srcu_idx); 808 return 0; 809 } 810 811 static int nvme_read_ana_log(struct nvme_ctrl *ctrl) 812 { 813 u32 nr_change_groups = 0; 814 int error; 815 816 mutex_lock(&ctrl->ana_lock); 817 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM, 818 ctrl->ana_log_buf, ctrl->ana_log_size, 0); 819 if (error) { 820 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); 821 goto out_unlock; 822 } 823 824 error = nvme_parse_ana_log(ctrl, &nr_change_groups, 825 nvme_update_ana_state); 826 if (error) 827 goto out_unlock; 828 829 /* 830 * In theory we should have an ANATT timer per group as they might enter 831 * the change state at different times. But that is a lot of overhead 832 * just to protect against a target that keeps entering new changes 833 * states while never finishing previous ones. But we'll still 834 * eventually time out once all groups are in change state, so this 835 * isn't a big deal. 836 * 837 * We also double the ANATT value to provide some slack for transports 838 * or AEN processing overhead. 839 */ 840 if (nr_change_groups) 841 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies); 842 else 843 del_timer_sync(&ctrl->anatt_timer); 844 out_unlock: 845 mutex_unlock(&ctrl->ana_lock); 846 return error; 847 } 848 849 static void nvme_ana_work(struct work_struct *work) 850 { 851 struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work); 852 853 if (ctrl->state != NVME_CTRL_LIVE) 854 return; 855 856 nvme_read_ana_log(ctrl); 857 } 858 859 void nvme_mpath_update(struct nvme_ctrl *ctrl) 860 { 861 u32 nr_change_groups = 0; 862 863 if (!ctrl->ana_log_buf) 864 return; 865 866 mutex_lock(&ctrl->ana_lock); 867 nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state); 868 mutex_unlock(&ctrl->ana_lock); 869 } 870 871 static void nvme_anatt_timeout(struct timer_list *t) 872 { 873 struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer); 874 875 dev_info(ctrl->device, "ANATT timeout, resetting controller.\n"); 876 nvme_reset_ctrl(ctrl); 877 } 878 879 void nvme_mpath_stop(struct nvme_ctrl *ctrl) 880 { 881 if (!nvme_ctrl_use_ana(ctrl)) 882 return; 883 del_timer_sync(&ctrl->anatt_timer); 884 cancel_work_sync(&ctrl->ana_work); 885 } 886 887 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store) \ 888 struct device_attribute subsys_attr_##_name = \ 889 __ATTR(_name, _mode, _show, _store) 890 891 static ssize_t nvme_subsys_iopolicy_show(struct device *dev, 892 struct device_attribute *attr, char *buf) 893 { 894 struct nvme_subsystem *subsys = 895 container_of(dev, struct nvme_subsystem, dev); 896 897 return sysfs_emit(buf, "%s\n", 898 nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); 899 } 900 901 static void nvme_subsys_iopolicy_update(struct nvme_subsystem *subsys, 902 int iopolicy) 903 { 904 struct nvme_ctrl *ctrl; 905 int old_iopolicy = READ_ONCE(subsys->iopolicy); 906 907 if (old_iopolicy == iopolicy) 908 return; 909 910 WRITE_ONCE(subsys->iopolicy, iopolicy); 911 912 /* iopolicy changes clear the mpath by design */ 913 mutex_lock(&nvme_subsystems_lock); 914 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) 915 nvme_mpath_clear_ctrl_paths(ctrl); 916 mutex_unlock(&nvme_subsystems_lock); 917 918 pr_notice("subsysnqn %s iopolicy changed from %s to %s\n", 919 subsys->subnqn, 920 nvme_iopolicy_names[old_iopolicy], 921 nvme_iopolicy_names[iopolicy]); 922 } 923 924 static ssize_t nvme_subsys_iopolicy_store(struct device *dev, 925 struct device_attribute *attr, const char *buf, size_t count) 926 { 927 struct nvme_subsystem *subsys = 928 container_of(dev, struct nvme_subsystem, dev); 929 int i; 930 931 for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) { 932 if (sysfs_streq(buf, nvme_iopolicy_names[i])) { 933 nvme_subsys_iopolicy_update(subsys, i); 934 return count; 935 } 936 } 937 938 return -EINVAL; 939 } 940 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR, 941 nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store); 942 943 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr, 944 char *buf) 945 { 946 return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid); 947 } 948 DEVICE_ATTR_RO(ana_grpid); 949 950 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr, 951 char *buf) 952 { 953 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); 954 955 return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]); 956 } 957 DEVICE_ATTR_RO(ana_state); 958 959 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl, 960 struct nvme_ana_group_desc *desc, void *data) 961 { 962 struct nvme_ana_group_desc *dst = data; 963 964 if (desc->grpid != dst->grpid) 965 return 0; 966 967 *dst = *desc; 968 return -ENXIO; /* just break out of the loop */ 969 } 970 971 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid) 972 { 973 if (nvme_ctrl_use_ana(ns->ctrl)) { 974 struct nvme_ana_group_desc desc = { 975 .grpid = anagrpid, 976 .state = 0, 977 }; 978 979 mutex_lock(&ns->ctrl->ana_lock); 980 ns->ana_grpid = le32_to_cpu(anagrpid); 981 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc); 982 mutex_unlock(&ns->ctrl->ana_lock); 983 if (desc.state) { 984 /* found the group desc: update */ 985 nvme_update_ns_ana_state(&desc, ns); 986 } else { 987 /* group desc not found: trigger a re-read */ 988 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 989 queue_work(nvme_wq, &ns->ctrl->ana_work); 990 } 991 } else { 992 ns->ana_state = NVME_ANA_OPTIMIZED; 993 nvme_mpath_set_live(ns); 994 } 995 996 if (blk_queue_stable_writes(ns->queue) && ns->head->disk) 997 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, 998 ns->head->disk->queue); 999 #ifdef CONFIG_BLK_DEV_ZONED 1000 if (blk_queue_is_zoned(ns->queue) && ns->head->disk) 1001 ns->head->disk->nr_zones = ns->disk->nr_zones; 1002 #endif 1003 } 1004 1005 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head) 1006 { 1007 if (!head->disk) 1008 return; 1009 if (test_and_clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 1010 nvme_cdev_del(&head->cdev, &head->cdev_device); 1011 /* 1012 * requeue I/O after NVME_NSHEAD_DISK_LIVE has been cleared 1013 * to allow multipath to fail all I/O. 1014 */ 1015 synchronize_srcu(&head->srcu); 1016 kblockd_schedule_work(&head->requeue_work); 1017 del_gendisk(head->disk); 1018 } 1019 /* 1020 * requeue I/O after NVME_NSHEAD_DISK_LIVE has been cleared 1021 * to allow multipath to fail all I/O. 1022 */ 1023 synchronize_srcu(&head->srcu); 1024 kblockd_schedule_work(&head->requeue_work); 1025 } 1026 1027 void nvme_mpath_remove_disk(struct nvme_ns_head *head) 1028 { 1029 if (!head->disk) 1030 return; 1031 /* make sure all pending bios are cleaned up */ 1032 kblockd_schedule_work(&head->requeue_work); 1033 flush_work(&head->requeue_work); 1034 flush_work(&head->partition_scan_work); 1035 put_disk(head->disk); 1036 } 1037 1038 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) 1039 { 1040 mutex_init(&ctrl->ana_lock); 1041 timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0); 1042 INIT_WORK(&ctrl->ana_work, nvme_ana_work); 1043 } 1044 1045 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 1046 { 1047 size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT; 1048 size_t ana_log_size; 1049 int error = 0; 1050 1051 /* check if multipath is enabled and we have the capability */ 1052 if (!multipath || !ctrl->subsys || 1053 !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)) 1054 return 0; 1055 1056 /* initialize this in the identify path to cover controller resets */ 1057 atomic_set(&ctrl->nr_active, 0); 1058 1059 if (!ctrl->max_namespaces || 1060 ctrl->max_namespaces > le32_to_cpu(id->nn)) { 1061 dev_err(ctrl->device, 1062 "Invalid MNAN value %u\n", ctrl->max_namespaces); 1063 return -EINVAL; 1064 } 1065 1066 ctrl->anacap = id->anacap; 1067 ctrl->anatt = id->anatt; 1068 ctrl->nanagrpid = le32_to_cpu(id->nanagrpid); 1069 ctrl->anagrpmax = le32_to_cpu(id->anagrpmax); 1070 1071 ana_log_size = sizeof(struct nvme_ana_rsp_hdr) + 1072 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) + 1073 ctrl->max_namespaces * sizeof(__le32); 1074 if (ana_log_size > max_transfer_size) { 1075 dev_err(ctrl->device, 1076 "ANA log page size (%zd) larger than MDTS (%zd).\n", 1077 ana_log_size, max_transfer_size); 1078 dev_err(ctrl->device, "disabling ANA support.\n"); 1079 goto out_uninit; 1080 } 1081 if (ana_log_size > ctrl->ana_log_size) { 1082 nvme_mpath_stop(ctrl); 1083 nvme_mpath_uninit(ctrl); 1084 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL); 1085 if (!ctrl->ana_log_buf) 1086 return -ENOMEM; 1087 } 1088 ctrl->ana_log_size = ana_log_size; 1089 error = nvme_read_ana_log(ctrl); 1090 if (error) 1091 goto out_uninit; 1092 return 0; 1093 1094 out_uninit: 1095 nvme_mpath_uninit(ctrl); 1096 return error; 1097 } 1098 1099 void nvme_mpath_uninit(struct nvme_ctrl *ctrl) 1100 { 1101 kvfree(ctrl->ana_log_buf); 1102 ctrl->ana_log_buf = NULL; 1103 ctrl->ana_log_size = 0; 1104 } 1105