1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2017-2018 Christoph Hellwig. 4 */ 5 6 #include <linux/backing-dev.h> 7 #include <linux/moduleparam.h> 8 #include <linux/vmalloc.h> 9 #include <trace/events/block.h> 10 #include "nvme.h" 11 12 bool multipath = true; 13 module_param(multipath, bool, 0444); 14 MODULE_PARM_DESC(multipath, 15 "turn on native support for multiple controllers per subsystem"); 16 17 static const char *nvme_iopolicy_names[] = { 18 [NVME_IOPOLICY_NUMA] = "numa", 19 [NVME_IOPOLICY_RR] = "round-robin", 20 }; 21 22 static int iopolicy = NVME_IOPOLICY_NUMA; 23 24 static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp) 25 { 26 if (!val) 27 return -EINVAL; 28 if (!strncmp(val, "numa", 4)) 29 iopolicy = NVME_IOPOLICY_NUMA; 30 else if (!strncmp(val, "round-robin", 11)) 31 iopolicy = NVME_IOPOLICY_RR; 32 else 33 return -EINVAL; 34 35 return 0; 36 } 37 38 static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp) 39 { 40 return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]); 41 } 42 43 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy, 44 &iopolicy, 0644); 45 MODULE_PARM_DESC(iopolicy, 46 "Default multipath I/O policy; 'numa' (default) or 'round-robin'"); 47 48 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys) 49 { 50 subsys->iopolicy = iopolicy; 51 } 52 53 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) 54 { 55 struct nvme_ns_head *h; 56 57 lockdep_assert_held(&subsys->lock); 58 list_for_each_entry(h, &subsys->nsheads, entry) 59 if (h->disk) 60 blk_mq_unfreeze_queue(h->disk->queue); 61 } 62 63 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) 64 { 65 struct nvme_ns_head *h; 66 67 lockdep_assert_held(&subsys->lock); 68 list_for_each_entry(h, &subsys->nsheads, entry) 69 if (h->disk) 70 blk_mq_freeze_queue_wait(h->disk->queue); 71 } 72 73 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) 74 { 75 struct nvme_ns_head *h; 76 77 lockdep_assert_held(&subsys->lock); 78 list_for_each_entry(h, &subsys->nsheads, entry) 79 if (h->disk) 80 blk_freeze_queue_start(h->disk->queue); 81 } 82 83 void nvme_failover_req(struct request *req) 84 { 85 struct nvme_ns *ns = req->q->queuedata; 86 u16 status = nvme_req(req)->status & 0x7ff; 87 unsigned long flags; 88 struct bio *bio; 89 90 nvme_mpath_clear_current_path(ns); 91 92 /* 93 * If we got back an ANA error, we know the controller is alive but not 94 * ready to serve this namespace. Kick of a re-read of the ANA 95 * information page, and just try any other available path for now. 96 */ 97 if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) { 98 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 99 queue_work(nvme_wq, &ns->ctrl->ana_work); 100 } 101 102 spin_lock_irqsave(&ns->head->requeue_lock, flags); 103 for (bio = req->bio; bio; bio = bio->bi_next) { 104 bio_set_dev(bio, ns->head->disk->part0); 105 if (bio->bi_opf & REQ_POLLED) { 106 bio->bi_opf &= ~REQ_POLLED; 107 bio->bi_cookie = BLK_QC_T_NONE; 108 } 109 } 110 blk_steal_bios(&ns->head->requeue_list, req); 111 spin_unlock_irqrestore(&ns->head->requeue_lock, flags); 112 113 blk_mq_end_request(req, 0); 114 kblockd_schedule_work(&ns->head->requeue_work); 115 } 116 117 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) 118 { 119 struct nvme_ns *ns; 120 121 down_read(&ctrl->namespaces_rwsem); 122 list_for_each_entry(ns, &ctrl->namespaces, list) { 123 if (!ns->head->disk) 124 continue; 125 kblockd_schedule_work(&ns->head->requeue_work); 126 if (ctrl->state == NVME_CTRL_LIVE) 127 disk_uevent(ns->head->disk, KOBJ_CHANGE); 128 } 129 up_read(&ctrl->namespaces_rwsem); 130 } 131 132 static const char *nvme_ana_state_names[] = { 133 [0] = "invalid state", 134 [NVME_ANA_OPTIMIZED] = "optimized", 135 [NVME_ANA_NONOPTIMIZED] = "non-optimized", 136 [NVME_ANA_INACCESSIBLE] = "inaccessible", 137 [NVME_ANA_PERSISTENT_LOSS] = "persistent-loss", 138 [NVME_ANA_CHANGE] = "change", 139 }; 140 141 bool nvme_mpath_clear_current_path(struct nvme_ns *ns) 142 { 143 struct nvme_ns_head *head = ns->head; 144 bool changed = false; 145 int node; 146 147 if (!head) 148 goto out; 149 150 for_each_node(node) { 151 if (ns == rcu_access_pointer(head->current_path[node])) { 152 rcu_assign_pointer(head->current_path[node], NULL); 153 changed = true; 154 } 155 } 156 out: 157 return changed; 158 } 159 160 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) 161 { 162 struct nvme_ns *ns; 163 164 down_read(&ctrl->namespaces_rwsem); 165 list_for_each_entry(ns, &ctrl->namespaces, list) { 166 nvme_mpath_clear_current_path(ns); 167 kblockd_schedule_work(&ns->head->requeue_work); 168 } 169 up_read(&ctrl->namespaces_rwsem); 170 } 171 172 void nvme_mpath_revalidate_paths(struct nvme_ns *ns) 173 { 174 struct nvme_ns_head *head = ns->head; 175 sector_t capacity = get_capacity(head->disk); 176 int node; 177 178 list_for_each_entry_rcu(ns, &head->list, siblings) { 179 if (capacity != get_capacity(ns->disk)) 180 clear_bit(NVME_NS_READY, &ns->flags); 181 } 182 183 for_each_node(node) 184 rcu_assign_pointer(head->current_path[node], NULL); 185 kblockd_schedule_work(&head->requeue_work); 186 } 187 188 static bool nvme_path_is_disabled(struct nvme_ns *ns) 189 { 190 /* 191 * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should 192 * still be able to complete assuming that the controller is connected. 193 * Otherwise it will fail immediately and return to the requeue list. 194 */ 195 if (ns->ctrl->state != NVME_CTRL_LIVE && 196 ns->ctrl->state != NVME_CTRL_DELETING) 197 return true; 198 if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) || 199 !test_bit(NVME_NS_READY, &ns->flags)) 200 return true; 201 return false; 202 } 203 204 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) 205 { 206 int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; 207 struct nvme_ns *found = NULL, *fallback = NULL, *ns; 208 209 list_for_each_entry_rcu(ns, &head->list, siblings) { 210 if (nvme_path_is_disabled(ns)) 211 continue; 212 213 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA) 214 distance = node_distance(node, ns->ctrl->numa_node); 215 else 216 distance = LOCAL_DISTANCE; 217 218 switch (ns->ana_state) { 219 case NVME_ANA_OPTIMIZED: 220 if (distance < found_distance) { 221 found_distance = distance; 222 found = ns; 223 } 224 break; 225 case NVME_ANA_NONOPTIMIZED: 226 if (distance < fallback_distance) { 227 fallback_distance = distance; 228 fallback = ns; 229 } 230 break; 231 default: 232 break; 233 } 234 } 235 236 if (!found) 237 found = fallback; 238 if (found) 239 rcu_assign_pointer(head->current_path[node], found); 240 return found; 241 } 242 243 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, 244 struct nvme_ns *ns) 245 { 246 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, 247 siblings); 248 if (ns) 249 return ns; 250 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings); 251 } 252 253 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, 254 int node, struct nvme_ns *old) 255 { 256 struct nvme_ns *ns, *found = NULL; 257 258 if (list_is_singular(&head->list)) { 259 if (nvme_path_is_disabled(old)) 260 return NULL; 261 return old; 262 } 263 264 for (ns = nvme_next_ns(head, old); 265 ns && ns != old; 266 ns = nvme_next_ns(head, ns)) { 267 if (nvme_path_is_disabled(ns)) 268 continue; 269 270 if (ns->ana_state == NVME_ANA_OPTIMIZED) { 271 found = ns; 272 goto out; 273 } 274 if (ns->ana_state == NVME_ANA_NONOPTIMIZED) 275 found = ns; 276 } 277 278 /* 279 * The loop above skips the current path for round-robin semantics. 280 * Fall back to the current path if either: 281 * - no other optimized path found and current is optimized, 282 * - no other usable path found and current is usable. 283 */ 284 if (!nvme_path_is_disabled(old) && 285 (old->ana_state == NVME_ANA_OPTIMIZED || 286 (!found && old->ana_state == NVME_ANA_NONOPTIMIZED))) 287 return old; 288 289 if (!found) 290 return NULL; 291 out: 292 rcu_assign_pointer(head->current_path[node], found); 293 return found; 294 } 295 296 static inline bool nvme_path_is_optimized(struct nvme_ns *ns) 297 { 298 return ns->ctrl->state == NVME_CTRL_LIVE && 299 ns->ana_state == NVME_ANA_OPTIMIZED; 300 } 301 302 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) 303 { 304 int node = numa_node_id(); 305 struct nvme_ns *ns; 306 307 ns = srcu_dereference(head->current_path[node], &head->srcu); 308 if (unlikely(!ns)) 309 return __nvme_find_path(head, node); 310 311 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR) 312 return nvme_round_robin_path(head, node, ns); 313 if (unlikely(!nvme_path_is_optimized(ns))) 314 return __nvme_find_path(head, node); 315 return ns; 316 } 317 318 static bool nvme_available_path(struct nvme_ns_head *head) 319 { 320 struct nvme_ns *ns; 321 322 list_for_each_entry_rcu(ns, &head->list, siblings) { 323 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags)) 324 continue; 325 switch (ns->ctrl->state) { 326 case NVME_CTRL_LIVE: 327 case NVME_CTRL_RESETTING: 328 case NVME_CTRL_CONNECTING: 329 /* fallthru */ 330 return true; 331 default: 332 break; 333 } 334 } 335 return false; 336 } 337 338 static void nvme_ns_head_submit_bio(struct bio *bio) 339 { 340 struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; 341 struct device *dev = disk_to_dev(head->disk); 342 struct nvme_ns *ns; 343 int srcu_idx; 344 345 /* 346 * The namespace might be going away and the bio might be moved to a 347 * different queue via blk_steal_bios(), so we need to use the bio_split 348 * pool from the original queue to allocate the bvecs from. 349 */ 350 bio = bio_split_to_limits(bio); 351 352 srcu_idx = srcu_read_lock(&head->srcu); 353 ns = nvme_find_path(head); 354 if (likely(ns)) { 355 bio_set_dev(bio, ns->disk->part0); 356 bio->bi_opf |= REQ_NVME_MPATH; 357 trace_block_bio_remap(bio, disk_devt(ns->head->disk), 358 bio->bi_iter.bi_sector); 359 submit_bio_noacct(bio); 360 } else if (nvme_available_path(head)) { 361 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); 362 363 spin_lock_irq(&head->requeue_lock); 364 bio_list_add(&head->requeue_list, bio); 365 spin_unlock_irq(&head->requeue_lock); 366 } else { 367 dev_warn_ratelimited(dev, "no available path - failing I/O\n"); 368 369 bio_io_error(bio); 370 } 371 372 srcu_read_unlock(&head->srcu, srcu_idx); 373 } 374 375 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode) 376 { 377 if (!nvme_tryget_ns_head(bdev->bd_disk->private_data)) 378 return -ENXIO; 379 return 0; 380 } 381 382 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode) 383 { 384 nvme_put_ns_head(disk->private_data); 385 } 386 387 #ifdef CONFIG_BLK_DEV_ZONED 388 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector, 389 unsigned int nr_zones, report_zones_cb cb, void *data) 390 { 391 struct nvme_ns_head *head = disk->private_data; 392 struct nvme_ns *ns; 393 int srcu_idx, ret = -EWOULDBLOCK; 394 395 srcu_idx = srcu_read_lock(&head->srcu); 396 ns = nvme_find_path(head); 397 if (ns) 398 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data); 399 srcu_read_unlock(&head->srcu, srcu_idx); 400 return ret; 401 } 402 #else 403 #define nvme_ns_head_report_zones NULL 404 #endif /* CONFIG_BLK_DEV_ZONED */ 405 406 const struct block_device_operations nvme_ns_head_ops = { 407 .owner = THIS_MODULE, 408 .submit_bio = nvme_ns_head_submit_bio, 409 .open = nvme_ns_head_open, 410 .release = nvme_ns_head_release, 411 .ioctl = nvme_ns_head_ioctl, 412 .compat_ioctl = blkdev_compat_ptr_ioctl, 413 .getgeo = nvme_getgeo, 414 .report_zones = nvme_ns_head_report_zones, 415 .pr_ops = &nvme_pr_ops, 416 }; 417 418 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev) 419 { 420 return container_of(cdev, struct nvme_ns_head, cdev); 421 } 422 423 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file) 424 { 425 if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev))) 426 return -ENXIO; 427 return 0; 428 } 429 430 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file) 431 { 432 nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev)); 433 return 0; 434 } 435 436 static const struct file_operations nvme_ns_head_chr_fops = { 437 .owner = THIS_MODULE, 438 .open = nvme_ns_head_chr_open, 439 .release = nvme_ns_head_chr_release, 440 .unlocked_ioctl = nvme_ns_head_chr_ioctl, 441 .compat_ioctl = compat_ptr_ioctl, 442 .uring_cmd = nvme_ns_head_chr_uring_cmd, 443 .uring_cmd_iopoll = nvme_ns_head_chr_uring_cmd_iopoll, 444 }; 445 446 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) 447 { 448 int ret; 449 450 head->cdev_device.parent = &head->subsys->dev; 451 ret = dev_set_name(&head->cdev_device, "ng%dn%d", 452 head->subsys->instance, head->instance); 453 if (ret) 454 return ret; 455 ret = nvme_cdev_add(&head->cdev, &head->cdev_device, 456 &nvme_ns_head_chr_fops, THIS_MODULE); 457 return ret; 458 } 459 460 static void nvme_requeue_work(struct work_struct *work) 461 { 462 struct nvme_ns_head *head = 463 container_of(work, struct nvme_ns_head, requeue_work); 464 struct bio *bio, *next; 465 466 spin_lock_irq(&head->requeue_lock); 467 next = bio_list_get(&head->requeue_list); 468 spin_unlock_irq(&head->requeue_lock); 469 470 while ((bio = next) != NULL) { 471 next = bio->bi_next; 472 bio->bi_next = NULL; 473 474 submit_bio_noacct(bio); 475 } 476 } 477 478 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) 479 { 480 bool vwc = false; 481 482 mutex_init(&head->lock); 483 bio_list_init(&head->requeue_list); 484 spin_lock_init(&head->requeue_lock); 485 INIT_WORK(&head->requeue_work, nvme_requeue_work); 486 487 /* 488 * Add a multipath node if the subsystems supports multiple controllers. 489 * We also do this for private namespaces as the namespace sharing flag 490 * could change after a rescan. 491 */ 492 if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || 493 !nvme_is_unique_nsid(ctrl, head) || !multipath) 494 return 0; 495 496 head->disk = blk_alloc_disk(ctrl->numa_node); 497 if (!head->disk) 498 return -ENOMEM; 499 head->disk->fops = &nvme_ns_head_ops; 500 head->disk->private_data = head; 501 sprintf(head->disk->disk_name, "nvme%dn%d", 502 ctrl->subsys->instance, head->instance); 503 504 blk_queue_flag_set(QUEUE_FLAG_NONROT, head->disk->queue); 505 blk_queue_flag_set(QUEUE_FLAG_NOWAIT, head->disk->queue); 506 /* 507 * This assumes all controllers that refer to a namespace either 508 * support poll queues or not. That is not a strict guarantee, 509 * but if the assumption is wrong the effect is only suboptimal 510 * performance but not correctness problem. 511 */ 512 if (ctrl->tagset->nr_maps > HCTX_TYPE_POLL && 513 ctrl->tagset->map[HCTX_TYPE_POLL].nr_queues) 514 blk_queue_flag_set(QUEUE_FLAG_POLL, head->disk->queue); 515 516 /* set to a default value of 512 until the disk is validated */ 517 blk_queue_logical_block_size(head->disk->queue, 512); 518 blk_set_stacking_limits(&head->disk->queue->limits); 519 520 /* we need to propagate up the VMC settings */ 521 if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) 522 vwc = true; 523 blk_queue_write_cache(head->disk->queue, vwc, vwc); 524 return 0; 525 } 526 527 static void nvme_mpath_set_live(struct nvme_ns *ns) 528 { 529 struct nvme_ns_head *head = ns->head; 530 int rc; 531 532 if (!head->disk) 533 return; 534 535 /* 536 * test_and_set_bit() is used because it is protecting against two nvme 537 * paths simultaneously calling device_add_disk() on the same namespace 538 * head. 539 */ 540 if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 541 rc = device_add_disk(&head->subsys->dev, head->disk, 542 nvme_ns_id_attr_groups); 543 if (rc) { 544 clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags); 545 return; 546 } 547 nvme_add_ns_head_cdev(head); 548 } 549 550 mutex_lock(&head->lock); 551 if (nvme_path_is_optimized(ns)) { 552 int node, srcu_idx; 553 554 srcu_idx = srcu_read_lock(&head->srcu); 555 for_each_node(node) 556 __nvme_find_path(head, node); 557 srcu_read_unlock(&head->srcu, srcu_idx); 558 } 559 mutex_unlock(&head->lock); 560 561 synchronize_srcu(&head->srcu); 562 kblockd_schedule_work(&head->requeue_work); 563 } 564 565 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, 566 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, 567 void *)) 568 { 569 void *base = ctrl->ana_log_buf; 570 size_t offset = sizeof(struct nvme_ana_rsp_hdr); 571 int error, i; 572 573 lockdep_assert_held(&ctrl->ana_lock); 574 575 for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) { 576 struct nvme_ana_group_desc *desc = base + offset; 577 u32 nr_nsids; 578 size_t nsid_buf_size; 579 580 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc))) 581 return -EINVAL; 582 583 nr_nsids = le32_to_cpu(desc->nnsids); 584 nsid_buf_size = flex_array_size(desc, nsids, nr_nsids); 585 586 if (WARN_ON_ONCE(desc->grpid == 0)) 587 return -EINVAL; 588 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax)) 589 return -EINVAL; 590 if (WARN_ON_ONCE(desc->state == 0)) 591 return -EINVAL; 592 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE)) 593 return -EINVAL; 594 595 offset += sizeof(*desc); 596 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size)) 597 return -EINVAL; 598 599 error = cb(ctrl, desc, data); 600 if (error) 601 return error; 602 603 offset += nsid_buf_size; 604 } 605 606 return 0; 607 } 608 609 static inline bool nvme_state_is_live(enum nvme_ana_state state) 610 { 611 return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED; 612 } 613 614 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, 615 struct nvme_ns *ns) 616 { 617 ns->ana_grpid = le32_to_cpu(desc->grpid); 618 ns->ana_state = desc->state; 619 clear_bit(NVME_NS_ANA_PENDING, &ns->flags); 620 /* 621 * nvme_mpath_set_live() will trigger I/O to the multipath path device 622 * and in turn to this path device. However we cannot accept this I/O 623 * if the controller is not live. This may deadlock if called from 624 * nvme_mpath_init_identify() and the ctrl will never complete 625 * initialization, preventing I/O from completing. For this case we 626 * will reprocess the ANA log page in nvme_mpath_update() once the 627 * controller is ready. 628 */ 629 if (nvme_state_is_live(ns->ana_state) && 630 ns->ctrl->state == NVME_CTRL_LIVE) 631 nvme_mpath_set_live(ns); 632 } 633 634 static int nvme_update_ana_state(struct nvme_ctrl *ctrl, 635 struct nvme_ana_group_desc *desc, void *data) 636 { 637 u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0; 638 unsigned *nr_change_groups = data; 639 struct nvme_ns *ns; 640 641 dev_dbg(ctrl->device, "ANA group %d: %s.\n", 642 le32_to_cpu(desc->grpid), 643 nvme_ana_state_names[desc->state]); 644 645 if (desc->state == NVME_ANA_CHANGE) 646 (*nr_change_groups)++; 647 648 if (!nr_nsids) 649 return 0; 650 651 down_read(&ctrl->namespaces_rwsem); 652 list_for_each_entry(ns, &ctrl->namespaces, list) { 653 unsigned nsid; 654 again: 655 nsid = le32_to_cpu(desc->nsids[n]); 656 if (ns->head->ns_id < nsid) 657 continue; 658 if (ns->head->ns_id == nsid) 659 nvme_update_ns_ana_state(desc, ns); 660 if (++n == nr_nsids) 661 break; 662 if (ns->head->ns_id > nsid) 663 goto again; 664 } 665 up_read(&ctrl->namespaces_rwsem); 666 return 0; 667 } 668 669 static int nvme_read_ana_log(struct nvme_ctrl *ctrl) 670 { 671 u32 nr_change_groups = 0; 672 int error; 673 674 mutex_lock(&ctrl->ana_lock); 675 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM, 676 ctrl->ana_log_buf, ctrl->ana_log_size, 0); 677 if (error) { 678 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); 679 goto out_unlock; 680 } 681 682 error = nvme_parse_ana_log(ctrl, &nr_change_groups, 683 nvme_update_ana_state); 684 if (error) 685 goto out_unlock; 686 687 /* 688 * In theory we should have an ANATT timer per group as they might enter 689 * the change state at different times. But that is a lot of overhead 690 * just to protect against a target that keeps entering new changes 691 * states while never finishing previous ones. But we'll still 692 * eventually time out once all groups are in change state, so this 693 * isn't a big deal. 694 * 695 * We also double the ANATT value to provide some slack for transports 696 * or AEN processing overhead. 697 */ 698 if (nr_change_groups) 699 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies); 700 else 701 del_timer_sync(&ctrl->anatt_timer); 702 out_unlock: 703 mutex_unlock(&ctrl->ana_lock); 704 return error; 705 } 706 707 static void nvme_ana_work(struct work_struct *work) 708 { 709 struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work); 710 711 if (ctrl->state != NVME_CTRL_LIVE) 712 return; 713 714 nvme_read_ana_log(ctrl); 715 } 716 717 void nvme_mpath_update(struct nvme_ctrl *ctrl) 718 { 719 u32 nr_change_groups = 0; 720 721 if (!ctrl->ana_log_buf) 722 return; 723 724 mutex_lock(&ctrl->ana_lock); 725 nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state); 726 mutex_unlock(&ctrl->ana_lock); 727 } 728 729 static void nvme_anatt_timeout(struct timer_list *t) 730 { 731 struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer); 732 733 dev_info(ctrl->device, "ANATT timeout, resetting controller.\n"); 734 nvme_reset_ctrl(ctrl); 735 } 736 737 void nvme_mpath_stop(struct nvme_ctrl *ctrl) 738 { 739 if (!nvme_ctrl_use_ana(ctrl)) 740 return; 741 del_timer_sync(&ctrl->anatt_timer); 742 cancel_work_sync(&ctrl->ana_work); 743 } 744 745 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store) \ 746 struct device_attribute subsys_attr_##_name = \ 747 __ATTR(_name, _mode, _show, _store) 748 749 static ssize_t nvme_subsys_iopolicy_show(struct device *dev, 750 struct device_attribute *attr, char *buf) 751 { 752 struct nvme_subsystem *subsys = 753 container_of(dev, struct nvme_subsystem, dev); 754 755 return sysfs_emit(buf, "%s\n", 756 nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); 757 } 758 759 static ssize_t nvme_subsys_iopolicy_store(struct device *dev, 760 struct device_attribute *attr, const char *buf, size_t count) 761 { 762 struct nvme_subsystem *subsys = 763 container_of(dev, struct nvme_subsystem, dev); 764 int i; 765 766 for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) { 767 if (sysfs_streq(buf, nvme_iopolicy_names[i])) { 768 WRITE_ONCE(subsys->iopolicy, i); 769 return count; 770 } 771 } 772 773 return -EINVAL; 774 } 775 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR, 776 nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store); 777 778 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr, 779 char *buf) 780 { 781 return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid); 782 } 783 DEVICE_ATTR_RO(ana_grpid); 784 785 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr, 786 char *buf) 787 { 788 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); 789 790 return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]); 791 } 792 DEVICE_ATTR_RO(ana_state); 793 794 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl, 795 struct nvme_ana_group_desc *desc, void *data) 796 { 797 struct nvme_ana_group_desc *dst = data; 798 799 if (desc->grpid != dst->grpid) 800 return 0; 801 802 *dst = *desc; 803 return -ENXIO; /* just break out of the loop */ 804 } 805 806 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid) 807 { 808 if (nvme_ctrl_use_ana(ns->ctrl)) { 809 struct nvme_ana_group_desc desc = { 810 .grpid = anagrpid, 811 .state = 0, 812 }; 813 814 mutex_lock(&ns->ctrl->ana_lock); 815 ns->ana_grpid = le32_to_cpu(anagrpid); 816 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc); 817 mutex_unlock(&ns->ctrl->ana_lock); 818 if (desc.state) { 819 /* found the group desc: update */ 820 nvme_update_ns_ana_state(&desc, ns); 821 } else { 822 /* group desc not found: trigger a re-read */ 823 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 824 queue_work(nvme_wq, &ns->ctrl->ana_work); 825 } 826 } else { 827 ns->ana_state = NVME_ANA_OPTIMIZED; 828 nvme_mpath_set_live(ns); 829 } 830 831 if (blk_queue_stable_writes(ns->queue) && ns->head->disk) 832 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, 833 ns->head->disk->queue); 834 #ifdef CONFIG_BLK_DEV_ZONED 835 if (blk_queue_is_zoned(ns->queue) && ns->head->disk) 836 ns->head->disk->nr_zones = ns->disk->nr_zones; 837 #endif 838 } 839 840 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head) 841 { 842 if (!head->disk) 843 return; 844 kblockd_schedule_work(&head->requeue_work); 845 if (test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { 846 nvme_cdev_del(&head->cdev, &head->cdev_device); 847 del_gendisk(head->disk); 848 } 849 } 850 851 void nvme_mpath_remove_disk(struct nvme_ns_head *head) 852 { 853 if (!head->disk) 854 return; 855 blk_mark_disk_dead(head->disk); 856 /* make sure all pending bios are cleaned up */ 857 kblockd_schedule_work(&head->requeue_work); 858 flush_work(&head->requeue_work); 859 put_disk(head->disk); 860 } 861 862 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) 863 { 864 mutex_init(&ctrl->ana_lock); 865 timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0); 866 INIT_WORK(&ctrl->ana_work, nvme_ana_work); 867 } 868 869 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 870 { 871 size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT; 872 size_t ana_log_size; 873 int error = 0; 874 875 /* check if multipath is enabled and we have the capability */ 876 if (!multipath || !ctrl->subsys || 877 !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)) 878 return 0; 879 880 if (!ctrl->max_namespaces || 881 ctrl->max_namespaces > le32_to_cpu(id->nn)) { 882 dev_err(ctrl->device, 883 "Invalid MNAN value %u\n", ctrl->max_namespaces); 884 return -EINVAL; 885 } 886 887 ctrl->anacap = id->anacap; 888 ctrl->anatt = id->anatt; 889 ctrl->nanagrpid = le32_to_cpu(id->nanagrpid); 890 ctrl->anagrpmax = le32_to_cpu(id->anagrpmax); 891 892 ana_log_size = sizeof(struct nvme_ana_rsp_hdr) + 893 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) + 894 ctrl->max_namespaces * sizeof(__le32); 895 if (ana_log_size > max_transfer_size) { 896 dev_err(ctrl->device, 897 "ANA log page size (%zd) larger than MDTS (%zd).\n", 898 ana_log_size, max_transfer_size); 899 dev_err(ctrl->device, "disabling ANA support.\n"); 900 goto out_uninit; 901 } 902 if (ana_log_size > ctrl->ana_log_size) { 903 nvme_mpath_stop(ctrl); 904 nvme_mpath_uninit(ctrl); 905 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL); 906 if (!ctrl->ana_log_buf) 907 return -ENOMEM; 908 } 909 ctrl->ana_log_size = ana_log_size; 910 error = nvme_read_ana_log(ctrl); 911 if (error) 912 goto out_uninit; 913 return 0; 914 915 out_uninit: 916 nvme_mpath_uninit(ctrl); 917 return error; 918 } 919 920 void nvme_mpath_uninit(struct nvme_ctrl *ctrl) 921 { 922 kvfree(ctrl->ana_log_buf); 923 ctrl->ana_log_buf = NULL; 924 ctrl->ana_log_size = 0; 925 } 926