1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2017-2018 Christoph Hellwig. 4 */ 5 6 #include <linux/backing-dev.h> 7 #include <linux/moduleparam.h> 8 #include <linux/vmalloc.h> 9 #include <trace/events/block.h> 10 #include "nvme.h" 11 12 bool multipath = true; 13 module_param(multipath, bool, 0444); 14 MODULE_PARM_DESC(multipath, 15 "turn on native support for multiple controllers per subsystem"); 16 17 static const char *nvme_iopolicy_names[] = { 18 [NVME_IOPOLICY_NUMA] = "numa", 19 [NVME_IOPOLICY_RR] = "round-robin", 20 }; 21 22 static int iopolicy = NVME_IOPOLICY_NUMA; 23 24 static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp) 25 { 26 if (!val) 27 return -EINVAL; 28 if (!strncmp(val, "numa", 4)) 29 iopolicy = NVME_IOPOLICY_NUMA; 30 else if (!strncmp(val, "round-robin", 11)) 31 iopolicy = NVME_IOPOLICY_RR; 32 else 33 return -EINVAL; 34 35 return 0; 36 } 37 38 static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp) 39 { 40 return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]); 41 } 42 43 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy, 44 &iopolicy, 0644); 45 MODULE_PARM_DESC(iopolicy, 46 "Default multipath I/O policy; 'numa' (default) or 'round-robin'"); 47 48 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys) 49 { 50 subsys->iopolicy = iopolicy; 51 } 52 53 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) 54 { 55 struct nvme_ns_head *h; 56 57 lockdep_assert_held(&subsys->lock); 58 list_for_each_entry(h, &subsys->nsheads, entry) 59 if (h->disk) 60 blk_mq_unfreeze_queue(h->disk->queue); 61 } 62 63 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) 64 { 65 struct nvme_ns_head *h; 66 67 lockdep_assert_held(&subsys->lock); 68 list_for_each_entry(h, &subsys->nsheads, entry) 69 if (h->disk) 70 blk_mq_freeze_queue_wait(h->disk->queue); 71 } 72 73 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) 74 { 75 struct nvme_ns_head *h; 76 77 lockdep_assert_held(&subsys->lock); 78 list_for_each_entry(h, &subsys->nsheads, entry) 79 if (h->disk) 80 blk_freeze_queue_start(h->disk->queue); 81 } 82 83 void nvme_failover_req(struct request *req) 84 { 85 struct nvme_ns *ns = req->q->queuedata; 86 u16 status = nvme_req(req)->status & 0x7ff; 87 unsigned long flags; 88 struct bio *bio; 89 90 nvme_mpath_clear_current_path(ns); 91 92 /* 93 * If we got back an ANA error, we know the controller is alive but not 94 * ready to serve this namespace. Kick of a re-read of the ANA 95 * information page, and just try any other available path for now. 96 */ 97 if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) { 98 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 99 queue_work(nvme_wq, &ns->ctrl->ana_work); 100 } 101 102 spin_lock_irqsave(&ns->head->requeue_lock, flags); 103 for (bio = req->bio; bio; bio = bio->bi_next) { 104 bio_set_dev(bio, ns->head->disk->part0); 105 if (bio->bi_opf & REQ_POLLED) { 106 bio->bi_opf &= ~REQ_POLLED; 107 bio->bi_cookie = BLK_QC_T_NONE; 108 } 109 /* 110 * The alternate request queue that we may end up submitting 111 * the bio to may be frozen temporarily, in this case REQ_NOWAIT 112 * will fail the I/O immediately with EAGAIN to the issuer. 113 * We are not in the issuer context which cannot block. Clear 114 * the flag to avoid spurious EAGAIN I/O failures. 115 */ 116 bio->bi_opf &= ~REQ_NOWAIT; 117 } 118 blk_steal_bios(&ns->head->requeue_list, req); 119 spin_unlock_irqrestore(&ns->head->requeue_lock, flags); 120 121 blk_mq_end_request(req, 0); 122 kblockd_schedule_work(&ns->head->requeue_work); 123 } 124 125 void nvme_mpath_start_request(struct request *rq) 126 { 127 struct nvme_ns *ns = rq->q->queuedata; 128 struct gendisk *disk = ns->head->disk; 129 130 if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq)) 131 return; 132 133 nvme_req(rq)->flags |= NVME_MPATH_IO_STATS; 134 nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq), 135 jiffies); 136 } 137 EXPORT_SYMBOL_GPL(nvme_mpath_start_request); 138 139 void nvme_mpath_end_request(struct request *rq) 140 { 141 struct nvme_ns *ns = rq->q->queuedata; 142 143 if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS)) 144 return; 145 bdev_end_io_acct(ns->head->disk->part0, req_op(rq), 146 blk_rq_bytes(rq) >> SECTOR_SHIFT, 147 nvme_req(rq)->start_time); 148 } 149 150 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) 151 { 152 struct nvme_ns *ns; 153 154 down_read(&ctrl->namespaces_rwsem); 155 list_for_each_entry(ns, &ctrl->namespaces, list) { 156 if (!ns->head->disk) 157 continue; 158 kblockd_schedule_work(&ns->head->requeue_work); 159 if (ctrl->state == NVME_CTRL_LIVE) 160 disk_uevent(ns->head->disk, KOBJ_CHANGE); 161 } 162 up_read(&ctrl->namespaces_rwsem); 163 } 164 165 static const char *nvme_ana_state_names[] = { 166 [0] = "invalid state", 167 [NVME_ANA_OPTIMIZED] = "optimized", 168 [NVME_ANA_NONOPTIMIZED] = "non-optimized", 169 [NVME_ANA_INACCESSIBLE] = "inaccessible", 170 [NVME_ANA_PERSISTENT_LOSS] = "persistent-loss", 171 [NVME_ANA_CHANGE] = "change", 172 }; 173 174 bool nvme_mpath_clear_current_path(struct nvme_ns *ns) 175 { 176 struct nvme_ns_head *head = ns->head; 177 bool changed = false; 178 int node; 179 180 if (!head) 181 goto out; 182 183 for_each_node(node) { 184 if (ns == rcu_access_pointer(head->current_path[node])) { 185 rcu_assign_pointer(head->current_path[node], NULL); 186 changed = true; 187 } 188 } 189 out: 190 return changed; 191 } 192 193 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) 194 { 195 struct nvme_ns *ns; 196 197 down_read(&ctrl->namespaces_rwsem); 198 list_for_each_entry(ns, &ctrl->namespaces, list) { 199 nvme_mpath_clear_current_path(ns); 200 kblockd_schedule_work(&ns->head->requeue_work); 201 } 202 up_read(&ctrl->namespaces_rwsem); 203 } 204 205 void nvme_mpath_revalidate_paths(struct nvme_ns *ns) 206 { 207 struct nvme_ns_head *head = ns->head; 208 sector_t capacity = get_capacity(head->disk); 209 int node; 210 int srcu_idx; 211 212 srcu_idx = srcu_read_lock(&head->srcu); 213 list_for_each_entry_rcu(ns, &head->list, siblings) { 214 if (capacity != get_capacity(ns->disk)) 215 clear_bit(NVME_NS_READY, &ns->flags); 216 } 217 srcu_read_unlock(&head->srcu, srcu_idx); 218 219 for_each_node(node) 220 rcu_assign_pointer(head->current_path[node], NULL); 221 kblockd_schedule_work(&head->requeue_work); 222 } 223 224 static bool nvme_path_is_disabled(struct nvme_ns *ns) 225 { 226 /* 227 * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should 228 * still be able to complete assuming that the controller is connected. 229 * Otherwise it will fail immediately and return to the requeue list. 230 */ 231 if (ns->ctrl->state != NVME_CTRL_LIVE && 232 ns->ctrl->state != NVME_CTRL_DELETING) 233 return true; 234 if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) || 235 !test_bit(NVME_NS_READY, &ns->flags)) 236 return true; 237 return false; 238 } 239 240 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) 241 { 242 int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; 243 struct nvme_ns *found = NULL, *fallback = NULL, *ns; 244 245 list_for_each_entry_rcu(ns, &head->list, siblings) { 246 if (nvme_path_is_disabled(ns)) 247 continue; 248 249 if (ns->ctrl->numa_node != NUMA_NO_NODE && 250 READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA) 251 distance = node_distance(node, ns->ctrl->numa_node); 252 else 253 distance = LOCAL_DISTANCE; 254 255 switch (ns->ana_state) { 256 case NVME_ANA_OPTIMIZED: 257 if (distance < found_distance) { 258 found_distance = distance; 259 found = ns; 260 } 261 break; 262 case NVME_ANA_NONOPTIMIZED: 263 if (distance < fallback_distance) { 264 fallback_distance = distance; 265 fallback = ns; 266 } 267 break; 268 default: 269 break; 270 } 271 } 272 273 if (!found) 274 found = fallback; 275 if (found) 276 rcu_assign_pointer(head->current_path[node], found); 277 return found; 278 } 279 280 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, 281 struct nvme_ns *ns) 282 { 283 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, 284 siblings); 285 if (ns) 286 return ns; 287 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings); 288 } 289 290 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, 291 int node, struct nvme_ns *old) 292 { 293 struct nvme_ns *ns, *found = NULL; 294 295 if (list_is_singular(&head->list)) { 296 if (nvme_path_is_disabled(old)) 297 return NULL; 298 return old; 299 } 300 301 for (ns = nvme_next_ns(head, old); 302 ns && ns != old; 303 ns = nvme_next_ns(head, ns)) { 304 if (nvme_path_is_disabled(ns)) 305 continue; 306 307 if (ns->ana_state == NVME_ANA_OPTIMIZED) { 308 found = ns; 309 goto out; 310 } 311 if (ns->ana_state == NVME_ANA_NONOPTIMIZED) 312 found = ns; 313 } 314 315 /* 316 * The loop above skips the current path for round-robin semantics. 317 * Fall back to the current path if either: 318 * - no other optimized path found and current is optimized, 319 * - no other usable path found and current is usable. 320 */ 321 if (!nvme_path_is_disabled(old) && 322 (old->ana_state == NVME_ANA_OPTIMIZED || 323 (!found && old->ana_state == NVME_ANA_NONOPTIMIZED))) 324 return old; 325 326 if (!found) 327 return NULL; 328 out: 329 rcu_assign_pointer(head->current_path[node], found); 330 return found; 331 } 332 333 static inline bool nvme_path_is_optimized(struct nvme_ns *ns) 334 { 335 return ns->ctrl->state == NVME_CTRL_LIVE && 336 ns->ana_state == NVME_ANA_OPTIMIZED; 337 } 338 339 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) 340 { 341 int node = numa_node_id(); 342 struct nvme_ns *ns; 343 344 ns = srcu_dereference(head->current_path[node], &head->srcu); 345 if (unlikely(!ns)) 346 return __nvme_find_path(head, node); 347 348 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR) 349 return nvme_round_robin_path(head, node, ns); 350 if (unlikely(!nvme_path_is_optimized(ns))) 351 return __nvme_find_path(head, node); 352 return ns; 353 } 354 355 static bool nvme_available_path(struct nvme_ns_head *head) 356 { 357 struct nvme_ns *ns; 358 359 list_for_each_entry_rcu(ns, &head->list, siblings) { 360 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags)) 361 continue; 362 switch (ns->ctrl->state) { 363 case NVME_CTRL_LIVE: 364 case NVME_CTRL_RESETTING: 365 case NVME_CTRL_CONNECTING: 366 /* fallthru */ 367 return true; 368 default: 369 break; 370 } 371 } 372 return false; 373 } 374 375 static void nvme_ns_head_submit_bio(struct bio *bio) 376 { 377 struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; 378 struct device *dev = disk_to_dev(head->disk); 379 struct nvme_ns *ns; 380 int srcu_idx; 381 382 /* 383 * The namespace might be going away and the bio might be moved to a 384 * different queue via blk_steal_bios(), so we need to use the bio_split 385 * pool from the original queue to allocate the bvecs from. 386 */ 387 bio = bio_split_to_limits(bio); 388 if (!bio) 389 return; 390 391 srcu_idx = srcu_read_lock(&head->srcu); 392 ns = nvme_find_path(head); 393 if (likely(ns)) { 394 bio_set_dev(bio, ns->disk->part0); 395 bio->bi_opf |= REQ_NVME_MPATH; 396 trace_block_bio_remap(bio, disk_devt(ns->head->disk), 397 bio->bi_iter.bi_sector); 398 submit_bio_noacct(bio); 399 } else if (nvme_available_path(head)) { 400 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); 401 402 spin_lock_irq(&head->requeue_lock); 403 bio_list_add(&head->requeue_list, bio); 404 spin_unlock_irq(&head->requeue_lock); 405 } else { 406 dev_warn_ratelimited(dev, "no available path - failing I/O\n"); 407 408 bio_io_error(bio); 409 } 410 411 srcu_read_unlock(&head->srcu, srcu_idx); 412 } 413 414 static int nvme_ns_head_open(struct gendisk *disk, blk_mode_t mode) 415 { 416 if (!nvme_tryget_ns_head(disk->private_data)) 417 return -ENXIO; 418 return 0; 419 } 420 421 static void nvme_ns_head_release(struct gendisk *disk) 422 { 423 nvme_put_ns_head(disk->private_data); 424 } 425 426 #ifdef CONFIG_BLK_DEV_ZONED 427 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector, 428 unsigned int nr_zones, report_zones_cb cb, void *data) 429 { 430 struct nvme_ns_head *head = disk->private_data; 431 struct nvme_ns *ns; 432 int srcu_idx, ret = -EWOULDBLOCK; 433 434 srcu_idx = srcu_read_lock(&head->srcu); 435 ns = nvme_find_path(head); 436 if (ns) 437 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data); 438 srcu_read_unlock(&head->srcu, srcu_idx); 439 return ret; 440 } 441 #else 442 #define nvme_ns_head_report_zones NULL 443 #endif /* CONFIG_BLK_DEV_ZONED */ 444 445 const struct block_device_operations nvme_ns_head_ops = { 446 .owner = THIS_MODULE, 447 .submit_bio = nvme_ns_head_submit_bio, 448 .open = nvme_ns_head_open, 449 .release = nvme_ns_head_release, 450 .ioctl = nvme_ns_head_ioctl, 451 .compat_ioctl = blkdev_compat_ptr_ioctl, 452 .getgeo = nvme_getgeo, 453 .report_zones = nvme_ns_head_report_zones, 454 .pr_ops = &nvme_pr_ops, 455 }; 456 457 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev) 458 { 459 return container_of(cdev, struct nvme_ns_head, cdev); 460 } 461 462 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file) 463 { 464 if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev))) 465 return -ENXIO; 466 return 0; 467 } 468 469 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file) 470 { 471 nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev)); 472 return 0; 473 } 474 475 static const struct file_operations nvme_ns_head_chr_fops = { 476 .owner = THIS_MODULE, 477 .open = nvme_ns_head_chr_open, 478 .release = nvme_ns_head_chr_release, 479 .unlocked_ioctl = nvme_ns_head_chr_ioctl, 480 .compat_ioctl = compat_ptr_ioctl, 481 .uring_cmd = nvme_ns_head_chr_uring_cmd, 482 .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll, 483 }; 484 485 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) 486 { 487 int ret; 488 489 head->cdev_device.parent = &head->subsys->dev; 490 ret = dev_set_name(&head->cdev_device, "ng%dn%d", 491 head->subsys->instance, head->instance); 492 if (ret) 493 return ret; 494 ret = nvme_cdev_add(&head->cdev, &head->cdev_device, 495 &nvme_ns_head_chr_fops, THIS_MODULE); 496 return ret; 497 } 498 499 static void nvme_requeue_work(struct work_struct *work) 500 { 501 struct nvme_ns_head *head = 502 container_of(work, struct nvme_ns_head, requeue_work); 503 struct bio *bio, *next; 504 505 spin_lock_irq(&head->requeue_lock); 506 next = bio_list_get(&head->requeue_list); 507 spin_unlock_irq(&head->requeue_lock); 508 509 while ((bio = next) != NULL) { 510 next = bio->bi_next; 511 bio->bi_next = NULL; 512 513 submit_bio_noacct(bio); 514 } 515 } 516 517 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) 518 { 519 bool vwc = false; 520 521 mutex_init(&head->lock); 522 bio_list_init(&head->requeue_list); 523 spin_lock_init(&head->requeue_lock); 524 INIT_WORK(&head->requeue_work, nvme_requeue_work); 525 526 /* 527 * Add a multipath node if the subsystems supports multiple controllers. 528 * We also do this for private namespaces as the namespace sharing flag 529 * could change after a rescan. 530 */ 531 if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || 532 !nvme_is_unique_nsid(ctrl, head) || !multipath) 533 return 0; 534 535 head->disk = blk_alloc_disk(ctrl->numa_node); 536 if (!head->disk) 537 return -ENOMEM; 538 head->disk->fops = &nvme_ns_head_ops; 539 head->disk->private_data = head; 540 sprintf(head->disk->disk_name, "nvme%dn%d", 541 ctrl->subsys->instance, head->instance); 542 543 blk_queue_flag_set(QUEUE_FLAG_NONROT, head->disk->queue); 544 blk_queue_flag_set(QUEUE_FLAG_NOWAIT, head->disk->queue); 545 blk_queue_flag_set(QUEUE_FLAG_IO_STAT, head->disk->queue); 546 /* 547 * This assumes all controllers that refer to a namespace either 548 * support poll queues or not. That is not a strict guarantee, 549 * but if the assumption is wrong the effect is only suboptimal 550 * performance but not correctness problem. 551 */ 552 if (ctrl->tagset->nr_maps > HCTX_TYPE_POLL && 553 ctrl->tagset->map[HCTX_TYPE_POLL].nr_queues) 554 blk_queue_flag_set(QUEUE_FLAG_POLL, head->disk->queue); 555 556 /* set to a default value of 512 until the disk is validated */ 557 blk_queue_logical_block_size(head->disk->queue, 512); 558 blk_set_stacking_limits(&head->disk->queue->limits); 559 blk_queue_dma_alignment(head->disk->queue, 3); 560 561 /* we need to propagate up the VMC settings */ 562 if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) 563 vwc = true; 564 blk_queue_write_cache(head->disk->queue, vwc, vwc); 565 return 0; 566 } 567 568 static void nvme_mpath_set_live(struct nvme_ns *ns) 569 { 570 struct nvme_ns_head *head = ns->head; 571 int rc; 572 573 if (!head->disk) 574 return; 575 576 /* 577 * test_and_set_bit() is used because it is protecting against two nvme 578 * paths simultaneously calling device_add_disk() on the same namespace 579 * head. 580 */ 581 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 582 rc = device_add_disk(&head->subsys->dev, head->disk, 583 nvme_ns_id_attr_groups); 584 if (rc) { 585 clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags); 586 return; 587 } 588 nvme_add_ns_head_cdev(head); 589 } 590 591 mutex_lock(&head->lock); 592 if (nvme_path_is_optimized(ns)) { 593 int node, srcu_idx; 594 595 srcu_idx = srcu_read_lock(&head->srcu); 596 for_each_node(node) 597 __nvme_find_path(head, node); 598 srcu_read_unlock(&head->srcu, srcu_idx); 599 } 600 mutex_unlock(&head->lock); 601 602 synchronize_srcu(&head->srcu); 603 kblockd_schedule_work(&head->requeue_work); 604 } 605 606 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, 607 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, 608 void *)) 609 { 610 void *base = ctrl->ana_log_buf; 611 size_t offset = sizeof(struct nvme_ana_rsp_hdr); 612 int error, i; 613 614 lockdep_assert_held(&ctrl->ana_lock); 615 616 for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) { 617 struct nvme_ana_group_desc *desc = base + offset; 618 u32 nr_nsids; 619 size_t nsid_buf_size; 620 621 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc))) 622 return -EINVAL; 623 624 nr_nsids = le32_to_cpu(desc->nnsids); 625 nsid_buf_size = flex_array_size(desc, nsids, nr_nsids); 626 627 if (WARN_ON_ONCE(desc->grpid == 0)) 628 return -EINVAL; 629 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax)) 630 return -EINVAL; 631 if (WARN_ON_ONCE(desc->state == 0)) 632 return -EINVAL; 633 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE)) 634 return -EINVAL; 635 636 offset += sizeof(*desc); 637 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size)) 638 return -EINVAL; 639 640 error = cb(ctrl, desc, data); 641 if (error) 642 return error; 643 644 offset += nsid_buf_size; 645 } 646 647 return 0; 648 } 649 650 static inline bool nvme_state_is_live(enum nvme_ana_state state) 651 { 652 return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED; 653 } 654 655 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, 656 struct nvme_ns *ns) 657 { 658 ns->ana_grpid = le32_to_cpu(desc->grpid); 659 ns->ana_state = desc->state; 660 clear_bit(NVME_NS_ANA_PENDING, &ns->flags); 661 /* 662 * nvme_mpath_set_live() will trigger I/O to the multipath path device 663 * and in turn to this path device. However we cannot accept this I/O 664 * if the controller is not live. This may deadlock if called from 665 * nvme_mpath_init_identify() and the ctrl will never complete 666 * initialization, preventing I/O from completing. For this case we 667 * will reprocess the ANA log page in nvme_mpath_update() once the 668 * controller is ready. 669 */ 670 if (nvme_state_is_live(ns->ana_state) && 671 ns->ctrl->state == NVME_CTRL_LIVE) 672 nvme_mpath_set_live(ns); 673 } 674 675 static int nvme_update_ana_state(struct nvme_ctrl *ctrl, 676 struct nvme_ana_group_desc *desc, void *data) 677 { 678 u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0; 679 unsigned *nr_change_groups = data; 680 struct nvme_ns *ns; 681 682 dev_dbg(ctrl->device, "ANA group %d: %s.\n", 683 le32_to_cpu(desc->grpid), 684 nvme_ana_state_names[desc->state]); 685 686 if (desc->state == NVME_ANA_CHANGE) 687 (*nr_change_groups)++; 688 689 if (!nr_nsids) 690 return 0; 691 692 down_read(&ctrl->namespaces_rwsem); 693 list_for_each_entry(ns, &ctrl->namespaces, list) { 694 unsigned nsid; 695 again: 696 nsid = le32_to_cpu(desc->nsids[n]); 697 if (ns->head->ns_id < nsid) 698 continue; 699 if (ns->head->ns_id == nsid) 700 nvme_update_ns_ana_state(desc, ns); 701 if (++n == nr_nsids) 702 break; 703 if (ns->head->ns_id > nsid) 704 goto again; 705 } 706 up_read(&ctrl->namespaces_rwsem); 707 return 0; 708 } 709 710 static int nvme_read_ana_log(struct nvme_ctrl *ctrl) 711 { 712 u32 nr_change_groups = 0; 713 int error; 714 715 mutex_lock(&ctrl->ana_lock); 716 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM, 717 ctrl->ana_log_buf, ctrl->ana_log_size, 0); 718 if (error) { 719 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); 720 goto out_unlock; 721 } 722 723 error = nvme_parse_ana_log(ctrl, &nr_change_groups, 724 nvme_update_ana_state); 725 if (error) 726 goto out_unlock; 727 728 /* 729 * In theory we should have an ANATT timer per group as they might enter 730 * the change state at different times. But that is a lot of overhead 731 * just to protect against a target that keeps entering new changes 732 * states while never finishing previous ones. But we'll still 733 * eventually time out once all groups are in change state, so this 734 * isn't a big deal. 735 * 736 * We also double the ANATT value to provide some slack for transports 737 * or AEN processing overhead. 738 */ 739 if (nr_change_groups) 740 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies); 741 else 742 del_timer_sync(&ctrl->anatt_timer); 743 out_unlock: 744 mutex_unlock(&ctrl->ana_lock); 745 return error; 746 } 747 748 static void nvme_ana_work(struct work_struct *work) 749 { 750 struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work); 751 752 if (ctrl->state != NVME_CTRL_LIVE) 753 return; 754 755 nvme_read_ana_log(ctrl); 756 } 757 758 void nvme_mpath_update(struct nvme_ctrl *ctrl) 759 { 760 u32 nr_change_groups = 0; 761 762 if (!ctrl->ana_log_buf) 763 return; 764 765 mutex_lock(&ctrl->ana_lock); 766 nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state); 767 mutex_unlock(&ctrl->ana_lock); 768 } 769 770 static void nvme_anatt_timeout(struct timer_list *t) 771 { 772 struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer); 773 774 dev_info(ctrl->device, "ANATT timeout, resetting controller.\n"); 775 nvme_reset_ctrl(ctrl); 776 } 777 778 void nvme_mpath_stop(struct nvme_ctrl *ctrl) 779 { 780 if (!nvme_ctrl_use_ana(ctrl)) 781 return; 782 del_timer_sync(&ctrl->anatt_timer); 783 cancel_work_sync(&ctrl->ana_work); 784 } 785 786 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store) \ 787 struct device_attribute subsys_attr_##_name = \ 788 __ATTR(_name, _mode, _show, _store) 789 790 static ssize_t nvme_subsys_iopolicy_show(struct device *dev, 791 struct device_attribute *attr, char *buf) 792 { 793 struct nvme_subsystem *subsys = 794 container_of(dev, struct nvme_subsystem, dev); 795 796 return sysfs_emit(buf, "%s\n", 797 nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); 798 } 799 800 static ssize_t nvme_subsys_iopolicy_store(struct device *dev, 801 struct device_attribute *attr, const char *buf, size_t count) 802 { 803 struct nvme_subsystem *subsys = 804 container_of(dev, struct nvme_subsystem, dev); 805 int i; 806 807 for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) { 808 if (sysfs_streq(buf, nvme_iopolicy_names[i])) { 809 WRITE_ONCE(subsys->iopolicy, i); 810 return count; 811 } 812 } 813 814 return -EINVAL; 815 } 816 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR, 817 nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store); 818 819 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr, 820 char *buf) 821 { 822 return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid); 823 } 824 DEVICE_ATTR_RO(ana_grpid); 825 826 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr, 827 char *buf) 828 { 829 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); 830 831 return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]); 832 } 833 DEVICE_ATTR_RO(ana_state); 834 835 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl, 836 struct nvme_ana_group_desc *desc, void *data) 837 { 838 struct nvme_ana_group_desc *dst = data; 839 840 if (desc->grpid != dst->grpid) 841 return 0; 842 843 *dst = *desc; 844 return -ENXIO; /* just break out of the loop */ 845 } 846 847 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid) 848 { 849 if (nvme_ctrl_use_ana(ns->ctrl)) { 850 struct nvme_ana_group_desc desc = { 851 .grpid = anagrpid, 852 .state = 0, 853 }; 854 855 mutex_lock(&ns->ctrl->ana_lock); 856 ns->ana_grpid = le32_to_cpu(anagrpid); 857 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc); 858 mutex_unlock(&ns->ctrl->ana_lock); 859 if (desc.state) { 860 /* found the group desc: update */ 861 nvme_update_ns_ana_state(&desc, ns); 862 } else { 863 /* group desc not found: trigger a re-read */ 864 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 865 queue_work(nvme_wq, &ns->ctrl->ana_work); 866 } 867 } else { 868 ns->ana_state = NVME_ANA_OPTIMIZED; 869 nvme_mpath_set_live(ns); 870 } 871 872 if (blk_queue_stable_writes(ns->queue) && ns->head->disk) 873 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, 874 ns->head->disk->queue); 875 #ifdef CONFIG_BLK_DEV_ZONED 876 if (blk_queue_is_zoned(ns->queue) && ns->head->disk) 877 ns->head->disk->nr_zones = ns->disk->nr_zones; 878 #endif 879 } 880 881 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head) 882 { 883 if (!head->disk) 884 return; 885 kblockd_schedule_work(&head->requeue_work); 886 if (test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 887 nvme_cdev_del(&head->cdev, &head->cdev_device); 888 del_gendisk(head->disk); 889 } 890 } 891 892 void nvme_mpath_remove_disk(struct nvme_ns_head *head) 893 { 894 if (!head->disk) 895 return; 896 /* make sure all pending bios are cleaned up */ 897 kblockd_schedule_work(&head->requeue_work); 898 flush_work(&head->requeue_work); 899 put_disk(head->disk); 900 } 901 902 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) 903 { 904 mutex_init(&ctrl->ana_lock); 905 timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0); 906 INIT_WORK(&ctrl->ana_work, nvme_ana_work); 907 } 908 909 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 910 { 911 size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT; 912 size_t ana_log_size; 913 int error = 0; 914 915 /* check if multipath is enabled and we have the capability */ 916 if (!multipath || !ctrl->subsys || 917 !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)) 918 return 0; 919 920 if (!ctrl->max_namespaces || 921 ctrl->max_namespaces > le32_to_cpu(id->nn)) { 922 dev_err(ctrl->device, 923 "Invalid MNAN value %u\n", ctrl->max_namespaces); 924 return -EINVAL; 925 } 926 927 ctrl->anacap = id->anacap; 928 ctrl->anatt = id->anatt; 929 ctrl->nanagrpid = le32_to_cpu(id->nanagrpid); 930 ctrl->anagrpmax = le32_to_cpu(id->anagrpmax); 931 932 ana_log_size = sizeof(struct nvme_ana_rsp_hdr) + 933 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) + 934 ctrl->max_namespaces * sizeof(__le32); 935 if (ana_log_size > max_transfer_size) { 936 dev_err(ctrl->device, 937 "ANA log page size (%zd) larger than MDTS (%zd).\n", 938 ana_log_size, max_transfer_size); 939 dev_err(ctrl->device, "disabling ANA support.\n"); 940 goto out_uninit; 941 } 942 if (ana_log_size > ctrl->ana_log_size) { 943 nvme_mpath_stop(ctrl); 944 nvme_mpath_uninit(ctrl); 945 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL); 946 if (!ctrl->ana_log_buf) 947 return -ENOMEM; 948 } 949 ctrl->ana_log_size = ana_log_size; 950 error = nvme_read_ana_log(ctrl); 951 if (error) 952 goto out_uninit; 953 return 0; 954 955 out_uninit: 956 nvme_mpath_uninit(ctrl); 957 return error; 958 } 959 960 void nvme_mpath_uninit(struct nvme_ctrl *ctrl) 961 { 962 kvfree(ctrl->ana_log_buf); 963 ctrl->ana_log_buf = NULL; 964 ctrl->ana_log_size = 0; 965 } 966