1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2017-2018 Christoph Hellwig. 4 */ 5 6 #include <linux/moduleparam.h> 7 #include <trace/events/block.h> 8 #include "nvme.h" 9 10 static bool multipath = true; 11 module_param(multipath, bool, 0444); 12 MODULE_PARM_DESC(multipath, 13 "turn on native support for multiple controllers per subsystem"); 14 15 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) 16 { 17 struct nvme_ns_head *h; 18 19 lockdep_assert_held(&subsys->lock); 20 list_for_each_entry(h, &subsys->nsheads, entry) 21 if (h->disk) 22 blk_mq_unfreeze_queue(h->disk->queue); 23 } 24 25 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) 26 { 27 struct nvme_ns_head *h; 28 29 lockdep_assert_held(&subsys->lock); 30 list_for_each_entry(h, &subsys->nsheads, entry) 31 if (h->disk) 32 blk_mq_freeze_queue_wait(h->disk->queue); 33 } 34 35 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) 36 { 37 struct nvme_ns_head *h; 38 39 lockdep_assert_held(&subsys->lock); 40 list_for_each_entry(h, &subsys->nsheads, entry) 41 if (h->disk) 42 blk_freeze_queue_start(h->disk->queue); 43 } 44 45 /* 46 * If multipathing is enabled we need to always use the subsystem instance 47 * number for numbering our devices to avoid conflicts between subsystems that 48 * have multiple controllers and thus use the multipath-aware subsystem node 49 * and those that have a single controller and use the controller node 50 * directly. 51 */ 52 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns, 53 struct nvme_ctrl *ctrl, int *flags) 54 { 55 if (!multipath) { 56 sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance); 57 } else if (ns->head->disk) { 58 sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance, 59 ctrl->instance, ns->head->instance); 60 *flags = GENHD_FL_HIDDEN; 61 } else { 62 sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance, 63 ns->head->instance); 64 } 65 } 66 67 bool nvme_failover_req(struct request *req) 68 { 69 struct nvme_ns *ns = req->q->queuedata; 70 u16 status = nvme_req(req)->status; 71 unsigned long flags; 72 73 switch (status & 0x7ff) { 74 case NVME_SC_ANA_TRANSITION: 75 case NVME_SC_ANA_INACCESSIBLE: 76 case NVME_SC_ANA_PERSISTENT_LOSS: 77 /* 78 * If we got back an ANA error we know the controller is alive, 79 * but not ready to serve this namespaces. The spec suggests 80 * we should update our general state here, but due to the fact 81 * that the admin and I/O queues are not serialized that is 82 * fundamentally racy. So instead just clear the current path, 83 * mark the the path as pending and kick of a re-read of the ANA 84 * log page ASAP. 85 */ 86 nvme_mpath_clear_current_path(ns); 87 if (ns->ctrl->ana_log_buf) { 88 set_bit(NVME_NS_ANA_PENDING, &ns->flags); 89 queue_work(nvme_wq, &ns->ctrl->ana_work); 90 } 91 break; 92 case NVME_SC_HOST_PATH_ERROR: 93 case NVME_SC_HOST_ABORTED_CMD: 94 /* 95 * Temporary transport disruption in talking to the controller. 96 * Try to send on a new path. 97 */ 98 nvme_mpath_clear_current_path(ns); 99 break; 100 default: 101 /* This was a non-ANA error so follow the normal error path. */ 102 return false; 103 } 104 105 spin_lock_irqsave(&ns->head->requeue_lock, flags); 106 blk_steal_bios(&ns->head->requeue_list, req); 107 spin_unlock_irqrestore(&ns->head->requeue_lock, flags); 108 blk_mq_end_request(req, 0); 109 110 kblockd_schedule_work(&ns->head->requeue_work); 111 return true; 112 } 113 114 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) 115 { 116 struct nvme_ns *ns; 117 118 down_read(&ctrl->namespaces_rwsem); 119 list_for_each_entry(ns, &ctrl->namespaces, list) { 120 if (ns->head->disk) 121 kblockd_schedule_work(&ns->head->requeue_work); 122 } 123 up_read(&ctrl->namespaces_rwsem); 124 } 125 126 static const char *nvme_ana_state_names[] = { 127 [0] = "invalid state", 128 [NVME_ANA_OPTIMIZED] = "optimized", 129 [NVME_ANA_NONOPTIMIZED] = "non-optimized", 130 [NVME_ANA_INACCESSIBLE] = "inaccessible", 131 [NVME_ANA_PERSISTENT_LOSS] = "persistent-loss", 132 [NVME_ANA_CHANGE] = "change", 133 }; 134 135 bool nvme_mpath_clear_current_path(struct nvme_ns *ns) 136 { 137 struct nvme_ns_head *head = ns->head; 138 bool changed = false; 139 int node; 140 141 if (!head) 142 goto out; 143 144 for_each_node(node) { 145 if (ns == rcu_access_pointer(head->current_path[node])) { 146 rcu_assign_pointer(head->current_path[node], NULL); 147 changed = true; 148 } 149 } 150 out: 151 return changed; 152 } 153 154 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) 155 { 156 struct nvme_ns *ns; 157 158 mutex_lock(&ctrl->scan_lock); 159 down_read(&ctrl->namespaces_rwsem); 160 list_for_each_entry(ns, &ctrl->namespaces, list) 161 if (nvme_mpath_clear_current_path(ns)) 162 kblockd_schedule_work(&ns->head->requeue_work); 163 up_read(&ctrl->namespaces_rwsem); 164 mutex_unlock(&ctrl->scan_lock); 165 } 166 167 static bool nvme_path_is_disabled(struct nvme_ns *ns) 168 { 169 return ns->ctrl->state != NVME_CTRL_LIVE || 170 test_bit(NVME_NS_ANA_PENDING, &ns->flags) || 171 test_bit(NVME_NS_REMOVING, &ns->flags); 172 } 173 174 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) 175 { 176 int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; 177 struct nvme_ns *found = NULL, *fallback = NULL, *ns; 178 179 list_for_each_entry_rcu(ns, &head->list, siblings) { 180 if (nvme_path_is_disabled(ns)) 181 continue; 182 183 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA) 184 distance = node_distance(node, ns->ctrl->numa_node); 185 else 186 distance = LOCAL_DISTANCE; 187 188 switch (ns->ana_state) { 189 case NVME_ANA_OPTIMIZED: 190 if (distance < found_distance) { 191 found_distance = distance; 192 found = ns; 193 } 194 break; 195 case NVME_ANA_NONOPTIMIZED: 196 if (distance < fallback_distance) { 197 fallback_distance = distance; 198 fallback = ns; 199 } 200 break; 201 default: 202 break; 203 } 204 } 205 206 if (!found) 207 found = fallback; 208 if (found) 209 rcu_assign_pointer(head->current_path[node], found); 210 return found; 211 } 212 213 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, 214 struct nvme_ns *ns) 215 { 216 ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, 217 siblings); 218 if (ns) 219 return ns; 220 return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings); 221 } 222 223 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head, 224 int node, struct nvme_ns *old) 225 { 226 struct nvme_ns *ns, *found, *fallback = NULL; 227 228 if (list_is_singular(&head->list)) { 229 if (nvme_path_is_disabled(old)) 230 return NULL; 231 return old; 232 } 233 234 for (ns = nvme_next_ns(head, old); 235 ns != old; 236 ns = nvme_next_ns(head, ns)) { 237 if (nvme_path_is_disabled(ns)) 238 continue; 239 240 if (ns->ana_state == NVME_ANA_OPTIMIZED) { 241 found = ns; 242 goto out; 243 } 244 if (ns->ana_state == NVME_ANA_NONOPTIMIZED) 245 fallback = ns; 246 } 247 248 if (!fallback) 249 return NULL; 250 found = fallback; 251 out: 252 rcu_assign_pointer(head->current_path[node], found); 253 return found; 254 } 255 256 static inline bool nvme_path_is_optimized(struct nvme_ns *ns) 257 { 258 return ns->ctrl->state == NVME_CTRL_LIVE && 259 ns->ana_state == NVME_ANA_OPTIMIZED; 260 } 261 262 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) 263 { 264 int node = numa_node_id(); 265 struct nvme_ns *ns; 266 267 ns = srcu_dereference(head->current_path[node], &head->srcu); 268 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR && ns) 269 ns = nvme_round_robin_path(head, node, ns); 270 if (unlikely(!ns || !nvme_path_is_optimized(ns))) 271 ns = __nvme_find_path(head, node); 272 return ns; 273 } 274 275 static bool nvme_available_path(struct nvme_ns_head *head) 276 { 277 struct nvme_ns *ns; 278 279 list_for_each_entry_rcu(ns, &head->list, siblings) { 280 switch (ns->ctrl->state) { 281 case NVME_CTRL_LIVE: 282 case NVME_CTRL_RESETTING: 283 case NVME_CTRL_CONNECTING: 284 /* fallthru */ 285 return true; 286 default: 287 break; 288 } 289 } 290 return false; 291 } 292 293 static blk_qc_t nvme_ns_head_make_request(struct request_queue *q, 294 struct bio *bio) 295 { 296 struct nvme_ns_head *head = q->queuedata; 297 struct device *dev = disk_to_dev(head->disk); 298 struct nvme_ns *ns; 299 blk_qc_t ret = BLK_QC_T_NONE; 300 int srcu_idx; 301 302 /* 303 * The namespace might be going away and the bio might 304 * be moved to a different queue via blk_steal_bios(), 305 * so we need to use the bio_split pool from the original 306 * queue to allocate the bvecs from. 307 */ 308 blk_queue_split(q, &bio); 309 310 srcu_idx = srcu_read_lock(&head->srcu); 311 ns = nvme_find_path(head); 312 if (likely(ns)) { 313 bio->bi_disk = ns->disk; 314 bio->bi_opf |= REQ_NVME_MPATH; 315 trace_block_bio_remap(bio->bi_disk->queue, bio, 316 disk_devt(ns->head->disk), 317 bio->bi_iter.bi_sector); 318 ret = direct_make_request(bio); 319 } else if (nvme_available_path(head)) { 320 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); 321 322 spin_lock_irq(&head->requeue_lock); 323 bio_list_add(&head->requeue_list, bio); 324 spin_unlock_irq(&head->requeue_lock); 325 } else { 326 dev_warn_ratelimited(dev, "no available path - failing I/O\n"); 327 328 bio->bi_status = BLK_STS_IOERR; 329 bio_endio(bio); 330 } 331 332 srcu_read_unlock(&head->srcu, srcu_idx); 333 return ret; 334 } 335 336 static void nvme_requeue_work(struct work_struct *work) 337 { 338 struct nvme_ns_head *head = 339 container_of(work, struct nvme_ns_head, requeue_work); 340 struct bio *bio, *next; 341 342 spin_lock_irq(&head->requeue_lock); 343 next = bio_list_get(&head->requeue_list); 344 spin_unlock_irq(&head->requeue_lock); 345 346 while ((bio = next) != NULL) { 347 next = bio->bi_next; 348 bio->bi_next = NULL; 349 350 /* 351 * Reset disk to the mpath node and resubmit to select a new 352 * path. 353 */ 354 bio->bi_disk = head->disk; 355 generic_make_request(bio); 356 } 357 } 358 359 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) 360 { 361 struct request_queue *q; 362 bool vwc = false; 363 364 mutex_init(&head->lock); 365 bio_list_init(&head->requeue_list); 366 spin_lock_init(&head->requeue_lock); 367 INIT_WORK(&head->requeue_work, nvme_requeue_work); 368 369 /* 370 * Add a multipath node if the subsystems supports multiple controllers. 371 * We also do this for private namespaces as the namespace sharing data could 372 * change after a rescan. 373 */ 374 if (!(ctrl->subsys->cmic & (1 << 1)) || !multipath) 375 return 0; 376 377 q = blk_alloc_queue(nvme_ns_head_make_request, ctrl->numa_node); 378 if (!q) 379 goto out; 380 q->queuedata = head; 381 blk_queue_flag_set(QUEUE_FLAG_NONROT, q); 382 /* set to a default value for 512 until disk is validated */ 383 blk_queue_logical_block_size(q, 512); 384 blk_set_stacking_limits(&q->limits); 385 386 /* we need to propagate up the VMC settings */ 387 if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) 388 vwc = true; 389 blk_queue_write_cache(q, vwc, vwc); 390 391 head->disk = alloc_disk(0); 392 if (!head->disk) 393 goto out_cleanup_queue; 394 head->disk->fops = &nvme_ns_head_ops; 395 head->disk->private_data = head; 396 head->disk->queue = q; 397 head->disk->flags = GENHD_FL_EXT_DEVT; 398 sprintf(head->disk->disk_name, "nvme%dn%d", 399 ctrl->subsys->instance, head->instance); 400 return 0; 401 402 out_cleanup_queue: 403 blk_cleanup_queue(q); 404 out: 405 return -ENOMEM; 406 } 407 408 static void nvme_mpath_set_live(struct nvme_ns *ns) 409 { 410 struct nvme_ns_head *head = ns->head; 411 412 lockdep_assert_held(&ns->head->lock); 413 414 if (!head->disk) 415 return; 416 417 if (!(head->disk->flags & GENHD_FL_UP)) 418 device_add_disk(&head->subsys->dev, head->disk, 419 nvme_ns_id_attr_groups); 420 421 if (nvme_path_is_optimized(ns)) { 422 int node, srcu_idx; 423 424 srcu_idx = srcu_read_lock(&head->srcu); 425 for_each_node(node) 426 __nvme_find_path(head, node); 427 srcu_read_unlock(&head->srcu, srcu_idx); 428 } 429 430 synchronize_srcu(&ns->head->srcu); 431 kblockd_schedule_work(&ns->head->requeue_work); 432 } 433 434 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, 435 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, 436 void *)) 437 { 438 void *base = ctrl->ana_log_buf; 439 size_t offset = sizeof(struct nvme_ana_rsp_hdr); 440 int error, i; 441 442 lockdep_assert_held(&ctrl->ana_lock); 443 444 for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) { 445 struct nvme_ana_group_desc *desc = base + offset; 446 u32 nr_nsids; 447 size_t nsid_buf_size; 448 449 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc))) 450 return -EINVAL; 451 452 nr_nsids = le32_to_cpu(desc->nnsids); 453 nsid_buf_size = nr_nsids * sizeof(__le32); 454 455 if (WARN_ON_ONCE(desc->grpid == 0)) 456 return -EINVAL; 457 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax)) 458 return -EINVAL; 459 if (WARN_ON_ONCE(desc->state == 0)) 460 return -EINVAL; 461 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE)) 462 return -EINVAL; 463 464 offset += sizeof(*desc); 465 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size)) 466 return -EINVAL; 467 468 error = cb(ctrl, desc, data); 469 if (error) 470 return error; 471 472 offset += nsid_buf_size; 473 } 474 475 return 0; 476 } 477 478 static inline bool nvme_state_is_live(enum nvme_ana_state state) 479 { 480 return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED; 481 } 482 483 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc, 484 struct nvme_ns *ns) 485 { 486 mutex_lock(&ns->head->lock); 487 ns->ana_grpid = le32_to_cpu(desc->grpid); 488 ns->ana_state = desc->state; 489 clear_bit(NVME_NS_ANA_PENDING, &ns->flags); 490 491 if (nvme_state_is_live(ns->ana_state)) 492 nvme_mpath_set_live(ns); 493 mutex_unlock(&ns->head->lock); 494 } 495 496 static int nvme_update_ana_state(struct nvme_ctrl *ctrl, 497 struct nvme_ana_group_desc *desc, void *data) 498 { 499 u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0; 500 unsigned *nr_change_groups = data; 501 struct nvme_ns *ns; 502 503 dev_dbg(ctrl->device, "ANA group %d: %s.\n", 504 le32_to_cpu(desc->grpid), 505 nvme_ana_state_names[desc->state]); 506 507 if (desc->state == NVME_ANA_CHANGE) 508 (*nr_change_groups)++; 509 510 if (!nr_nsids) 511 return 0; 512 513 down_read(&ctrl->namespaces_rwsem); 514 list_for_each_entry(ns, &ctrl->namespaces, list) { 515 unsigned nsid = le32_to_cpu(desc->nsids[n]); 516 517 if (ns->head->ns_id < nsid) 518 continue; 519 if (ns->head->ns_id == nsid) 520 nvme_update_ns_ana_state(desc, ns); 521 if (++n == nr_nsids) 522 break; 523 } 524 up_read(&ctrl->namespaces_rwsem); 525 return 0; 526 } 527 528 static int nvme_read_ana_log(struct nvme_ctrl *ctrl) 529 { 530 u32 nr_change_groups = 0; 531 int error; 532 533 mutex_lock(&ctrl->ana_lock); 534 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, 535 ctrl->ana_log_buf, ctrl->ana_log_size, 0); 536 if (error) { 537 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); 538 goto out_unlock; 539 } 540 541 error = nvme_parse_ana_log(ctrl, &nr_change_groups, 542 nvme_update_ana_state); 543 if (error) 544 goto out_unlock; 545 546 /* 547 * In theory we should have an ANATT timer per group as they might enter 548 * the change state at different times. But that is a lot of overhead 549 * just to protect against a target that keeps entering new changes 550 * states while never finishing previous ones. But we'll still 551 * eventually time out once all groups are in change state, so this 552 * isn't a big deal. 553 * 554 * We also double the ANATT value to provide some slack for transports 555 * or AEN processing overhead. 556 */ 557 if (nr_change_groups) 558 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies); 559 else 560 del_timer_sync(&ctrl->anatt_timer); 561 out_unlock: 562 mutex_unlock(&ctrl->ana_lock); 563 return error; 564 } 565 566 static void nvme_ana_work(struct work_struct *work) 567 { 568 struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work); 569 570 nvme_read_ana_log(ctrl); 571 } 572 573 static void nvme_anatt_timeout(struct timer_list *t) 574 { 575 struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer); 576 577 dev_info(ctrl->device, "ANATT timeout, resetting controller.\n"); 578 nvme_reset_ctrl(ctrl); 579 } 580 581 void nvme_mpath_stop(struct nvme_ctrl *ctrl) 582 { 583 if (!nvme_ctrl_use_ana(ctrl)) 584 return; 585 del_timer_sync(&ctrl->anatt_timer); 586 cancel_work_sync(&ctrl->ana_work); 587 } 588 589 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store) \ 590 struct device_attribute subsys_attr_##_name = \ 591 __ATTR(_name, _mode, _show, _store) 592 593 static const char *nvme_iopolicy_names[] = { 594 [NVME_IOPOLICY_NUMA] = "numa", 595 [NVME_IOPOLICY_RR] = "round-robin", 596 }; 597 598 static ssize_t nvme_subsys_iopolicy_show(struct device *dev, 599 struct device_attribute *attr, char *buf) 600 { 601 struct nvme_subsystem *subsys = 602 container_of(dev, struct nvme_subsystem, dev); 603 604 return sprintf(buf, "%s\n", 605 nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]); 606 } 607 608 static ssize_t nvme_subsys_iopolicy_store(struct device *dev, 609 struct device_attribute *attr, const char *buf, size_t count) 610 { 611 struct nvme_subsystem *subsys = 612 container_of(dev, struct nvme_subsystem, dev); 613 int i; 614 615 for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) { 616 if (sysfs_streq(buf, nvme_iopolicy_names[i])) { 617 WRITE_ONCE(subsys->iopolicy, i); 618 return count; 619 } 620 } 621 622 return -EINVAL; 623 } 624 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR, 625 nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store); 626 627 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr, 628 char *buf) 629 { 630 return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid); 631 } 632 DEVICE_ATTR_RO(ana_grpid); 633 634 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr, 635 char *buf) 636 { 637 struct nvme_ns *ns = nvme_get_ns_from_dev(dev); 638 639 return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]); 640 } 641 DEVICE_ATTR_RO(ana_state); 642 643 static int nvme_set_ns_ana_state(struct nvme_ctrl *ctrl, 644 struct nvme_ana_group_desc *desc, void *data) 645 { 646 struct nvme_ns *ns = data; 647 648 if (ns->ana_grpid == le32_to_cpu(desc->grpid)) { 649 nvme_update_ns_ana_state(desc, ns); 650 return -ENXIO; /* just break out of the loop */ 651 } 652 653 return 0; 654 } 655 656 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) 657 { 658 if (nvme_ctrl_use_ana(ns->ctrl)) { 659 mutex_lock(&ns->ctrl->ana_lock); 660 ns->ana_grpid = le32_to_cpu(id->anagrpid); 661 nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state); 662 mutex_unlock(&ns->ctrl->ana_lock); 663 } else { 664 mutex_lock(&ns->head->lock); 665 ns->ana_state = NVME_ANA_OPTIMIZED; 666 nvme_mpath_set_live(ns); 667 mutex_unlock(&ns->head->lock); 668 } 669 } 670 671 void nvme_mpath_remove_disk(struct nvme_ns_head *head) 672 { 673 if (!head->disk) 674 return; 675 if (head->disk->flags & GENHD_FL_UP) 676 del_gendisk(head->disk); 677 blk_set_queue_dying(head->disk->queue); 678 /* make sure all pending bios are cleaned up */ 679 kblockd_schedule_work(&head->requeue_work); 680 flush_work(&head->requeue_work); 681 blk_cleanup_queue(head->disk->queue); 682 put_disk(head->disk); 683 } 684 685 int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) 686 { 687 int error; 688 689 /* check if multipath is enabled and we have the capability */ 690 if (!multipath || !ctrl->subsys || !(ctrl->subsys->cmic & (1 << 3))) 691 return 0; 692 693 ctrl->anacap = id->anacap; 694 ctrl->anatt = id->anatt; 695 ctrl->nanagrpid = le32_to_cpu(id->nanagrpid); 696 ctrl->anagrpmax = le32_to_cpu(id->anagrpmax); 697 698 mutex_init(&ctrl->ana_lock); 699 timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0); 700 ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) + 701 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc); 702 ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32); 703 704 if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) { 705 dev_err(ctrl->device, 706 "ANA log page size (%zd) larger than MDTS (%d).\n", 707 ctrl->ana_log_size, 708 ctrl->max_hw_sectors << SECTOR_SHIFT); 709 dev_err(ctrl->device, "disabling ANA support.\n"); 710 return 0; 711 } 712 713 INIT_WORK(&ctrl->ana_work, nvme_ana_work); 714 kfree(ctrl->ana_log_buf); 715 ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL); 716 if (!ctrl->ana_log_buf) { 717 error = -ENOMEM; 718 goto out; 719 } 720 721 error = nvme_read_ana_log(ctrl); 722 if (error) 723 goto out_free_ana_log_buf; 724 return 0; 725 out_free_ana_log_buf: 726 kfree(ctrl->ana_log_buf); 727 ctrl->ana_log_buf = NULL; 728 out: 729 return error; 730 } 731 732 void nvme_mpath_uninit(struct nvme_ctrl *ctrl) 733 { 734 kfree(ctrl->ana_log_buf); 735 ctrl->ana_log_buf = NULL; 736 } 737 738