1 /* 2 * Copyright (C) 2001 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include "dm.h" 9 10 #include <linux/module.h> 11 #include <linux/vmalloc.h> 12 #include <linux/blkdev.h> 13 #include <linux/namei.h> 14 #include <linux/ctype.h> 15 #include <linux/string.h> 16 #include <linux/slab.h> 17 #include <linux/interrupt.h> 18 #include <linux/mutex.h> 19 #include <linux/delay.h> 20 #include <linux/atomic.h> 21 22 #define DM_MSG_PREFIX "table" 23 24 #define MAX_DEPTH 16 25 #define NODE_SIZE L1_CACHE_BYTES 26 #define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t)) 27 #define CHILDREN_PER_NODE (KEYS_PER_NODE + 1) 28 29 /* 30 * The table has always exactly one reference from either mapped_device->map 31 * or hash_cell->new_map. This reference is not counted in table->holders. 32 * A pair of dm_create_table/dm_destroy_table functions is used for table 33 * creation/destruction. 34 * 35 * Temporary references from the other code increase table->holders. A pair 36 * of dm_table_get/dm_table_put functions is used to manipulate it. 37 * 38 * When the table is about to be destroyed, we wait for table->holders to 39 * drop to zero. 40 */ 41 42 struct dm_table { 43 struct mapped_device *md; 44 atomic_t holders; 45 unsigned type; 46 47 /* btree table */ 48 unsigned int depth; 49 unsigned int counts[MAX_DEPTH]; /* in nodes */ 50 sector_t *index[MAX_DEPTH]; 51 52 unsigned int num_targets; 53 unsigned int num_allocated; 54 sector_t *highs; 55 struct dm_target *targets; 56 57 unsigned integrity_supported:1; 58 59 /* 60 * Indicates the rw permissions for the new logical 61 * device. This should be a combination of FMODE_READ 62 * and FMODE_WRITE. 63 */ 64 fmode_t mode; 65 66 /* a list of devices used by this table */ 67 struct list_head devices; 68 69 /* events get handed up using this callback */ 70 void (*event_fn)(void *); 71 void *event_context; 72 73 struct dm_md_mempools *mempools; 74 75 struct list_head target_callbacks; 76 }; 77 78 /* 79 * Similar to ceiling(log_size(n)) 80 */ 81 static unsigned int int_log(unsigned int n, unsigned int base) 82 { 83 int result = 0; 84 85 while (n > 1) { 86 n = dm_div_up(n, base); 87 result++; 88 } 89 90 return result; 91 } 92 93 /* 94 * Calculate the index of the child node of the n'th node k'th key. 95 */ 96 static inline unsigned int get_child(unsigned int n, unsigned int k) 97 { 98 return (n * CHILDREN_PER_NODE) + k; 99 } 100 101 /* 102 * Return the n'th node of level l from table t. 103 */ 104 static inline sector_t *get_node(struct dm_table *t, 105 unsigned int l, unsigned int n) 106 { 107 return t->index[l] + (n * KEYS_PER_NODE); 108 } 109 110 /* 111 * Return the highest key that you could lookup from the n'th 112 * node on level l of the btree. 113 */ 114 static sector_t high(struct dm_table *t, unsigned int l, unsigned int n) 115 { 116 for (; l < t->depth - 1; l++) 117 n = get_child(n, CHILDREN_PER_NODE - 1); 118 119 if (n >= t->counts[l]) 120 return (sector_t) - 1; 121 122 return get_node(t, l, n)[KEYS_PER_NODE - 1]; 123 } 124 125 /* 126 * Fills in a level of the btree based on the highs of the level 127 * below it. 128 */ 129 static int setup_btree_index(unsigned int l, struct dm_table *t) 130 { 131 unsigned int n, k; 132 sector_t *node; 133 134 for (n = 0U; n < t->counts[l]; n++) { 135 node = get_node(t, l, n); 136 137 for (k = 0U; k < KEYS_PER_NODE; k++) 138 node[k] = high(t, l + 1, get_child(n, k)); 139 } 140 141 return 0; 142 } 143 144 void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size) 145 { 146 unsigned long size; 147 void *addr; 148 149 /* 150 * Check that we're not going to overflow. 151 */ 152 if (nmemb > (ULONG_MAX / elem_size)) 153 return NULL; 154 155 size = nmemb * elem_size; 156 addr = vzalloc(size); 157 158 return addr; 159 } 160 EXPORT_SYMBOL(dm_vcalloc); 161 162 /* 163 * highs, and targets are managed as dynamic arrays during a 164 * table load. 165 */ 166 static int alloc_targets(struct dm_table *t, unsigned int num) 167 { 168 sector_t *n_highs; 169 struct dm_target *n_targets; 170 int n = t->num_targets; 171 172 /* 173 * Allocate both the target array and offset array at once. 174 * Append an empty entry to catch sectors beyond the end of 175 * the device. 176 */ 177 n_highs = (sector_t *) dm_vcalloc(num + 1, sizeof(struct dm_target) + 178 sizeof(sector_t)); 179 if (!n_highs) 180 return -ENOMEM; 181 182 n_targets = (struct dm_target *) (n_highs + num); 183 184 if (n) { 185 memcpy(n_highs, t->highs, sizeof(*n_highs) * n); 186 memcpy(n_targets, t->targets, sizeof(*n_targets) * n); 187 } 188 189 memset(n_highs + n, -1, sizeof(*n_highs) * (num - n)); 190 vfree(t->highs); 191 192 t->num_allocated = num; 193 t->highs = n_highs; 194 t->targets = n_targets; 195 196 return 0; 197 } 198 199 int dm_table_create(struct dm_table **result, fmode_t mode, 200 unsigned num_targets, struct mapped_device *md) 201 { 202 struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL); 203 204 if (!t) 205 return -ENOMEM; 206 207 INIT_LIST_HEAD(&t->devices); 208 INIT_LIST_HEAD(&t->target_callbacks); 209 atomic_set(&t->holders, 0); 210 211 if (!num_targets) 212 num_targets = KEYS_PER_NODE; 213 214 num_targets = dm_round_up(num_targets, KEYS_PER_NODE); 215 216 if (alloc_targets(t, num_targets)) { 217 kfree(t); 218 t = NULL; 219 return -ENOMEM; 220 } 221 222 t->mode = mode; 223 t->md = md; 224 *result = t; 225 return 0; 226 } 227 228 static void free_devices(struct list_head *devices) 229 { 230 struct list_head *tmp, *next; 231 232 list_for_each_safe(tmp, next, devices) { 233 struct dm_dev_internal *dd = 234 list_entry(tmp, struct dm_dev_internal, list); 235 DMWARN("dm_table_destroy: dm_put_device call missing for %s", 236 dd->dm_dev.name); 237 kfree(dd); 238 } 239 } 240 241 void dm_table_destroy(struct dm_table *t) 242 { 243 unsigned int i; 244 245 if (!t) 246 return; 247 248 while (atomic_read(&t->holders)) 249 msleep(1); 250 smp_mb(); 251 252 /* free the indexes */ 253 if (t->depth >= 2) 254 vfree(t->index[t->depth - 2]); 255 256 /* free the targets */ 257 for (i = 0; i < t->num_targets; i++) { 258 struct dm_target *tgt = t->targets + i; 259 260 if (tgt->type->dtr) 261 tgt->type->dtr(tgt); 262 263 dm_put_target_type(tgt->type); 264 } 265 266 vfree(t->highs); 267 268 /* free the device list */ 269 if (t->devices.next != &t->devices) 270 free_devices(&t->devices); 271 272 dm_free_md_mempools(t->mempools); 273 274 kfree(t); 275 } 276 277 void dm_table_get(struct dm_table *t) 278 { 279 atomic_inc(&t->holders); 280 } 281 EXPORT_SYMBOL(dm_table_get); 282 283 void dm_table_put(struct dm_table *t) 284 { 285 if (!t) 286 return; 287 288 smp_mb__before_atomic_dec(); 289 atomic_dec(&t->holders); 290 } 291 EXPORT_SYMBOL(dm_table_put); 292 293 /* 294 * Checks to see if we need to extend highs or targets. 295 */ 296 static inline int check_space(struct dm_table *t) 297 { 298 if (t->num_targets >= t->num_allocated) 299 return alloc_targets(t, t->num_allocated * 2); 300 301 return 0; 302 } 303 304 /* 305 * See if we've already got a device in the list. 306 */ 307 static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev) 308 { 309 struct dm_dev_internal *dd; 310 311 list_for_each_entry (dd, l, list) 312 if (dd->dm_dev.bdev->bd_dev == dev) 313 return dd; 314 315 return NULL; 316 } 317 318 /* 319 * Open a device so we can use it as a map destination. 320 */ 321 static int open_dev(struct dm_dev_internal *d, dev_t dev, 322 struct mapped_device *md) 323 { 324 static char *_claim_ptr = "I belong to device-mapper"; 325 struct block_device *bdev; 326 327 int r; 328 329 BUG_ON(d->dm_dev.bdev); 330 331 bdev = blkdev_get_by_dev(dev, d->dm_dev.mode | FMODE_EXCL, _claim_ptr); 332 if (IS_ERR(bdev)) 333 return PTR_ERR(bdev); 334 335 r = bd_link_disk_holder(bdev, dm_disk(md)); 336 if (r) { 337 blkdev_put(bdev, d->dm_dev.mode | FMODE_EXCL); 338 return r; 339 } 340 341 d->dm_dev.bdev = bdev; 342 return 0; 343 } 344 345 /* 346 * Close a device that we've been using. 347 */ 348 static void close_dev(struct dm_dev_internal *d, struct mapped_device *md) 349 { 350 if (!d->dm_dev.bdev) 351 return; 352 353 bd_unlink_disk_holder(d->dm_dev.bdev, dm_disk(md)); 354 blkdev_put(d->dm_dev.bdev, d->dm_dev.mode | FMODE_EXCL); 355 d->dm_dev.bdev = NULL; 356 } 357 358 /* 359 * If possible, this checks an area of a destination device is invalid. 360 */ 361 static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev, 362 sector_t start, sector_t len, void *data) 363 { 364 struct request_queue *q; 365 struct queue_limits *limits = data; 366 struct block_device *bdev = dev->bdev; 367 sector_t dev_size = 368 i_size_read(bdev->bd_inode) >> SECTOR_SHIFT; 369 unsigned short logical_block_size_sectors = 370 limits->logical_block_size >> SECTOR_SHIFT; 371 char b[BDEVNAME_SIZE]; 372 373 /* 374 * Some devices exist without request functions, 375 * such as loop devices not yet bound to backing files. 376 * Forbid the use of such devices. 377 */ 378 q = bdev_get_queue(bdev); 379 if (!q || !q->make_request_fn) { 380 DMWARN("%s: %s is not yet initialised: " 381 "start=%llu, len=%llu, dev_size=%llu", 382 dm_device_name(ti->table->md), bdevname(bdev, b), 383 (unsigned long long)start, 384 (unsigned long long)len, 385 (unsigned long long)dev_size); 386 return 1; 387 } 388 389 if (!dev_size) 390 return 0; 391 392 if ((start >= dev_size) || (start + len > dev_size)) { 393 DMWARN("%s: %s too small for target: " 394 "start=%llu, len=%llu, dev_size=%llu", 395 dm_device_name(ti->table->md), bdevname(bdev, b), 396 (unsigned long long)start, 397 (unsigned long long)len, 398 (unsigned long long)dev_size); 399 return 1; 400 } 401 402 if (logical_block_size_sectors <= 1) 403 return 0; 404 405 if (start & (logical_block_size_sectors - 1)) { 406 DMWARN("%s: start=%llu not aligned to h/w " 407 "logical block size %u of %s", 408 dm_device_name(ti->table->md), 409 (unsigned long long)start, 410 limits->logical_block_size, bdevname(bdev, b)); 411 return 1; 412 } 413 414 if (len & (logical_block_size_sectors - 1)) { 415 DMWARN("%s: len=%llu not aligned to h/w " 416 "logical block size %u of %s", 417 dm_device_name(ti->table->md), 418 (unsigned long long)len, 419 limits->logical_block_size, bdevname(bdev, b)); 420 return 1; 421 } 422 423 return 0; 424 } 425 426 /* 427 * This upgrades the mode on an already open dm_dev, being 428 * careful to leave things as they were if we fail to reopen the 429 * device and not to touch the existing bdev field in case 430 * it is accessed concurrently inside dm_table_any_congested(). 431 */ 432 static int upgrade_mode(struct dm_dev_internal *dd, fmode_t new_mode, 433 struct mapped_device *md) 434 { 435 int r; 436 struct dm_dev_internal dd_new, dd_old; 437 438 dd_new = dd_old = *dd; 439 440 dd_new.dm_dev.mode |= new_mode; 441 dd_new.dm_dev.bdev = NULL; 442 443 r = open_dev(&dd_new, dd->dm_dev.bdev->bd_dev, md); 444 if (r) 445 return r; 446 447 dd->dm_dev.mode |= new_mode; 448 close_dev(&dd_old, md); 449 450 return 0; 451 } 452 453 /* 454 * Add a device to the list, or just increment the usage count if 455 * it's already present. 456 */ 457 int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode, 458 struct dm_dev **result) 459 { 460 int r; 461 dev_t uninitialized_var(dev); 462 struct dm_dev_internal *dd; 463 unsigned int major, minor; 464 struct dm_table *t = ti->table; 465 466 BUG_ON(!t); 467 468 if (sscanf(path, "%u:%u", &major, &minor) == 2) { 469 /* Extract the major/minor numbers */ 470 dev = MKDEV(major, minor); 471 if (MAJOR(dev) != major || MINOR(dev) != minor) 472 return -EOVERFLOW; 473 } else { 474 /* convert the path to a device */ 475 struct block_device *bdev = lookup_bdev(path); 476 477 if (IS_ERR(bdev)) 478 return PTR_ERR(bdev); 479 dev = bdev->bd_dev; 480 bdput(bdev); 481 } 482 483 dd = find_device(&t->devices, dev); 484 if (!dd) { 485 dd = kmalloc(sizeof(*dd), GFP_KERNEL); 486 if (!dd) 487 return -ENOMEM; 488 489 dd->dm_dev.mode = mode; 490 dd->dm_dev.bdev = NULL; 491 492 if ((r = open_dev(dd, dev, t->md))) { 493 kfree(dd); 494 return r; 495 } 496 497 format_dev_t(dd->dm_dev.name, dev); 498 499 atomic_set(&dd->count, 0); 500 list_add(&dd->list, &t->devices); 501 502 } else if (dd->dm_dev.mode != (mode | dd->dm_dev.mode)) { 503 r = upgrade_mode(dd, mode, t->md); 504 if (r) 505 return r; 506 } 507 atomic_inc(&dd->count); 508 509 *result = &dd->dm_dev; 510 return 0; 511 } 512 EXPORT_SYMBOL(dm_get_device); 513 514 int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev, 515 sector_t start, sector_t len, void *data) 516 { 517 struct queue_limits *limits = data; 518 struct block_device *bdev = dev->bdev; 519 struct request_queue *q = bdev_get_queue(bdev); 520 char b[BDEVNAME_SIZE]; 521 522 if (unlikely(!q)) { 523 DMWARN("%s: Cannot set limits for nonexistent device %s", 524 dm_device_name(ti->table->md), bdevname(bdev, b)); 525 return 0; 526 } 527 528 if (bdev_stack_limits(limits, bdev, start) < 0) 529 DMWARN("%s: adding target device %s caused an alignment inconsistency: " 530 "physical_block_size=%u, logical_block_size=%u, " 531 "alignment_offset=%u, start=%llu", 532 dm_device_name(ti->table->md), bdevname(bdev, b), 533 q->limits.physical_block_size, 534 q->limits.logical_block_size, 535 q->limits.alignment_offset, 536 (unsigned long long) start << SECTOR_SHIFT); 537 538 /* 539 * Check if merge fn is supported. 540 * If not we'll force DM to use PAGE_SIZE or 541 * smaller I/O, just to be safe. 542 */ 543 if (dm_queue_merge_is_compulsory(q) && !ti->type->merge) 544 blk_limits_max_hw_sectors(limits, 545 (unsigned int) (PAGE_SIZE >> 9)); 546 return 0; 547 } 548 EXPORT_SYMBOL_GPL(dm_set_device_limits); 549 550 /* 551 * Decrement a device's use count and remove it if necessary. 552 */ 553 void dm_put_device(struct dm_target *ti, struct dm_dev *d) 554 { 555 struct dm_dev_internal *dd = container_of(d, struct dm_dev_internal, 556 dm_dev); 557 558 if (atomic_dec_and_test(&dd->count)) { 559 close_dev(dd, ti->table->md); 560 list_del(&dd->list); 561 kfree(dd); 562 } 563 } 564 EXPORT_SYMBOL(dm_put_device); 565 566 /* 567 * Checks to see if the target joins onto the end of the table. 568 */ 569 static int adjoin(struct dm_table *table, struct dm_target *ti) 570 { 571 struct dm_target *prev; 572 573 if (!table->num_targets) 574 return !ti->begin; 575 576 prev = &table->targets[table->num_targets - 1]; 577 return (ti->begin == (prev->begin + prev->len)); 578 } 579 580 /* 581 * Used to dynamically allocate the arg array. 582 */ 583 static char **realloc_argv(unsigned *array_size, char **old_argv) 584 { 585 char **argv; 586 unsigned new_size; 587 588 new_size = *array_size ? *array_size * 2 : 64; 589 argv = kmalloc(new_size * sizeof(*argv), GFP_KERNEL); 590 if (argv) { 591 memcpy(argv, old_argv, *array_size * sizeof(*argv)); 592 *array_size = new_size; 593 } 594 595 kfree(old_argv); 596 return argv; 597 } 598 599 /* 600 * Destructively splits up the argument list to pass to ctr. 601 */ 602 int dm_split_args(int *argc, char ***argvp, char *input) 603 { 604 char *start, *end = input, *out, **argv = NULL; 605 unsigned array_size = 0; 606 607 *argc = 0; 608 609 if (!input) { 610 *argvp = NULL; 611 return 0; 612 } 613 614 argv = realloc_argv(&array_size, argv); 615 if (!argv) 616 return -ENOMEM; 617 618 while (1) { 619 /* Skip whitespace */ 620 start = skip_spaces(end); 621 622 if (!*start) 623 break; /* success, we hit the end */ 624 625 /* 'out' is used to remove any back-quotes */ 626 end = out = start; 627 while (*end) { 628 /* Everything apart from '\0' can be quoted */ 629 if (*end == '\\' && *(end + 1)) { 630 *out++ = *(end + 1); 631 end += 2; 632 continue; 633 } 634 635 if (isspace(*end)) 636 break; /* end of token */ 637 638 *out++ = *end++; 639 } 640 641 /* have we already filled the array ? */ 642 if ((*argc + 1) > array_size) { 643 argv = realloc_argv(&array_size, argv); 644 if (!argv) 645 return -ENOMEM; 646 } 647 648 /* we know this is whitespace */ 649 if (*end) 650 end++; 651 652 /* terminate the string and put it in the array */ 653 *out = '\0'; 654 argv[*argc] = start; 655 (*argc)++; 656 } 657 658 *argvp = argv; 659 return 0; 660 } 661 662 /* 663 * Impose necessary and sufficient conditions on a devices's table such 664 * that any incoming bio which respects its logical_block_size can be 665 * processed successfully. If it falls across the boundary between 666 * two or more targets, the size of each piece it gets split into must 667 * be compatible with the logical_block_size of the target processing it. 668 */ 669 static int validate_hardware_logical_block_alignment(struct dm_table *table, 670 struct queue_limits *limits) 671 { 672 /* 673 * This function uses arithmetic modulo the logical_block_size 674 * (in units of 512-byte sectors). 675 */ 676 unsigned short device_logical_block_size_sects = 677 limits->logical_block_size >> SECTOR_SHIFT; 678 679 /* 680 * Offset of the start of the next table entry, mod logical_block_size. 681 */ 682 unsigned short next_target_start = 0; 683 684 /* 685 * Given an aligned bio that extends beyond the end of a 686 * target, how many sectors must the next target handle? 687 */ 688 unsigned short remaining = 0; 689 690 struct dm_target *uninitialized_var(ti); 691 struct queue_limits ti_limits; 692 unsigned i = 0; 693 694 /* 695 * Check each entry in the table in turn. 696 */ 697 while (i < dm_table_get_num_targets(table)) { 698 ti = dm_table_get_target(table, i++); 699 700 blk_set_default_limits(&ti_limits); 701 702 /* combine all target devices' limits */ 703 if (ti->type->iterate_devices) 704 ti->type->iterate_devices(ti, dm_set_device_limits, 705 &ti_limits); 706 707 /* 708 * If the remaining sectors fall entirely within this 709 * table entry are they compatible with its logical_block_size? 710 */ 711 if (remaining < ti->len && 712 remaining & ((ti_limits.logical_block_size >> 713 SECTOR_SHIFT) - 1)) 714 break; /* Error */ 715 716 next_target_start = 717 (unsigned short) ((next_target_start + ti->len) & 718 (device_logical_block_size_sects - 1)); 719 remaining = next_target_start ? 720 device_logical_block_size_sects - next_target_start : 0; 721 } 722 723 if (remaining) { 724 DMWARN("%s: table line %u (start sect %llu len %llu) " 725 "not aligned to h/w logical block size %u", 726 dm_device_name(table->md), i, 727 (unsigned long long) ti->begin, 728 (unsigned long long) ti->len, 729 limits->logical_block_size); 730 return -EINVAL; 731 } 732 733 return 0; 734 } 735 736 int dm_table_add_target(struct dm_table *t, const char *type, 737 sector_t start, sector_t len, char *params) 738 { 739 int r = -EINVAL, argc; 740 char **argv; 741 struct dm_target *tgt; 742 743 if ((r = check_space(t))) 744 return r; 745 746 tgt = t->targets + t->num_targets; 747 memset(tgt, 0, sizeof(*tgt)); 748 749 if (!len) { 750 DMERR("%s: zero-length target", dm_device_name(t->md)); 751 return -EINVAL; 752 } 753 754 tgt->type = dm_get_target_type(type); 755 if (!tgt->type) { 756 DMERR("%s: %s: unknown target type", dm_device_name(t->md), 757 type); 758 return -EINVAL; 759 } 760 761 tgt->table = t; 762 tgt->begin = start; 763 tgt->len = len; 764 tgt->error = "Unknown error"; 765 766 /* 767 * Does this target adjoin the previous one ? 768 */ 769 if (!adjoin(t, tgt)) { 770 tgt->error = "Gap in table"; 771 r = -EINVAL; 772 goto bad; 773 } 774 775 r = dm_split_args(&argc, &argv, params); 776 if (r) { 777 tgt->error = "couldn't split parameters (insufficient memory)"; 778 goto bad; 779 } 780 781 r = tgt->type->ctr(tgt, argc, argv); 782 kfree(argv); 783 if (r) 784 goto bad; 785 786 t->highs[t->num_targets++] = tgt->begin + tgt->len - 1; 787 788 if (!tgt->num_discard_requests && tgt->discards_supported) 789 DMWARN("%s: %s: ignoring discards_supported because num_discard_requests is zero.", 790 dm_device_name(t->md), type); 791 792 return 0; 793 794 bad: 795 DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error); 796 dm_put_target_type(tgt->type); 797 return r; 798 } 799 800 /* 801 * Target argument parsing helpers. 802 */ 803 static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, 804 unsigned *value, char **error, unsigned grouped) 805 { 806 const char *arg_str = dm_shift_arg(arg_set); 807 808 if (!arg_str || 809 (sscanf(arg_str, "%u", value) != 1) || 810 (*value < arg->min) || 811 (*value > arg->max) || 812 (grouped && arg_set->argc < *value)) { 813 *error = arg->error; 814 return -EINVAL; 815 } 816 817 return 0; 818 } 819 820 int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, 821 unsigned *value, char **error) 822 { 823 return validate_next_arg(arg, arg_set, value, error, 0); 824 } 825 EXPORT_SYMBOL(dm_read_arg); 826 827 int dm_read_arg_group(struct dm_arg *arg, struct dm_arg_set *arg_set, 828 unsigned *value, char **error) 829 { 830 return validate_next_arg(arg, arg_set, value, error, 1); 831 } 832 EXPORT_SYMBOL(dm_read_arg_group); 833 834 const char *dm_shift_arg(struct dm_arg_set *as) 835 { 836 char *r; 837 838 if (as->argc) { 839 as->argc--; 840 r = *as->argv; 841 as->argv++; 842 return r; 843 } 844 845 return NULL; 846 } 847 EXPORT_SYMBOL(dm_shift_arg); 848 849 void dm_consume_args(struct dm_arg_set *as, unsigned num_args) 850 { 851 BUG_ON(as->argc < num_args); 852 as->argc -= num_args; 853 as->argv += num_args; 854 } 855 EXPORT_SYMBOL(dm_consume_args); 856 857 static int dm_table_set_type(struct dm_table *t) 858 { 859 unsigned i; 860 unsigned bio_based = 0, request_based = 0; 861 struct dm_target *tgt; 862 struct dm_dev_internal *dd; 863 struct list_head *devices; 864 865 for (i = 0; i < t->num_targets; i++) { 866 tgt = t->targets + i; 867 if (dm_target_request_based(tgt)) 868 request_based = 1; 869 else 870 bio_based = 1; 871 872 if (bio_based && request_based) { 873 DMWARN("Inconsistent table: different target types" 874 " can't be mixed up"); 875 return -EINVAL; 876 } 877 } 878 879 if (bio_based) { 880 /* We must use this table as bio-based */ 881 t->type = DM_TYPE_BIO_BASED; 882 return 0; 883 } 884 885 BUG_ON(!request_based); /* No targets in this table */ 886 887 /* Non-request-stackable devices can't be used for request-based dm */ 888 devices = dm_table_get_devices(t); 889 list_for_each_entry(dd, devices, list) { 890 if (!blk_queue_stackable(bdev_get_queue(dd->dm_dev.bdev))) { 891 DMWARN("table load rejected: including" 892 " non-request-stackable devices"); 893 return -EINVAL; 894 } 895 } 896 897 /* 898 * Request-based dm supports only tables that have a single target now. 899 * To support multiple targets, request splitting support is needed, 900 * and that needs lots of changes in the block-layer. 901 * (e.g. request completion process for partial completion.) 902 */ 903 if (t->num_targets > 1) { 904 DMWARN("Request-based dm doesn't support multiple targets yet"); 905 return -EINVAL; 906 } 907 908 t->type = DM_TYPE_REQUEST_BASED; 909 910 return 0; 911 } 912 913 unsigned dm_table_get_type(struct dm_table *t) 914 { 915 return t->type; 916 } 917 918 bool dm_table_request_based(struct dm_table *t) 919 { 920 return dm_table_get_type(t) == DM_TYPE_REQUEST_BASED; 921 } 922 923 int dm_table_alloc_md_mempools(struct dm_table *t) 924 { 925 unsigned type = dm_table_get_type(t); 926 927 if (unlikely(type == DM_TYPE_NONE)) { 928 DMWARN("no table type is set, can't allocate mempools"); 929 return -EINVAL; 930 } 931 932 t->mempools = dm_alloc_md_mempools(type, t->integrity_supported); 933 if (!t->mempools) 934 return -ENOMEM; 935 936 return 0; 937 } 938 939 void dm_table_free_md_mempools(struct dm_table *t) 940 { 941 dm_free_md_mempools(t->mempools); 942 t->mempools = NULL; 943 } 944 945 struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t) 946 { 947 return t->mempools; 948 } 949 950 static int setup_indexes(struct dm_table *t) 951 { 952 int i; 953 unsigned int total = 0; 954 sector_t *indexes; 955 956 /* allocate the space for *all* the indexes */ 957 for (i = t->depth - 2; i >= 0; i--) { 958 t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE); 959 total += t->counts[i]; 960 } 961 962 indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE); 963 if (!indexes) 964 return -ENOMEM; 965 966 /* set up internal nodes, bottom-up */ 967 for (i = t->depth - 2; i >= 0; i--) { 968 t->index[i] = indexes; 969 indexes += (KEYS_PER_NODE * t->counts[i]); 970 setup_btree_index(i, t); 971 } 972 973 return 0; 974 } 975 976 /* 977 * Builds the btree to index the map. 978 */ 979 static int dm_table_build_index(struct dm_table *t) 980 { 981 int r = 0; 982 unsigned int leaf_nodes; 983 984 /* how many indexes will the btree have ? */ 985 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); 986 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); 987 988 /* leaf layer has already been set up */ 989 t->counts[t->depth - 1] = leaf_nodes; 990 t->index[t->depth - 1] = t->highs; 991 992 if (t->depth >= 2) 993 r = setup_indexes(t); 994 995 return r; 996 } 997 998 /* 999 * Get a disk whose integrity profile reflects the table's profile. 1000 * If %match_all is true, all devices' profiles must match. 1001 * If %match_all is false, all devices must at least have an 1002 * allocated integrity profile; but uninitialized is ok. 1003 * Returns NULL if integrity support was inconsistent or unavailable. 1004 */ 1005 static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t, 1006 bool match_all) 1007 { 1008 struct list_head *devices = dm_table_get_devices(t); 1009 struct dm_dev_internal *dd = NULL; 1010 struct gendisk *prev_disk = NULL, *template_disk = NULL; 1011 1012 list_for_each_entry(dd, devices, list) { 1013 template_disk = dd->dm_dev.bdev->bd_disk; 1014 if (!blk_get_integrity(template_disk)) 1015 goto no_integrity; 1016 if (!match_all && !blk_integrity_is_initialized(template_disk)) 1017 continue; /* skip uninitialized profiles */ 1018 else if (prev_disk && 1019 blk_integrity_compare(prev_disk, template_disk) < 0) 1020 goto no_integrity; 1021 prev_disk = template_disk; 1022 } 1023 1024 return template_disk; 1025 1026 no_integrity: 1027 if (prev_disk) 1028 DMWARN("%s: integrity not set: %s and %s profile mismatch", 1029 dm_device_name(t->md), 1030 prev_disk->disk_name, 1031 template_disk->disk_name); 1032 return NULL; 1033 } 1034 1035 /* 1036 * Register the mapped device for blk_integrity support if 1037 * the underlying devices have an integrity profile. But all devices 1038 * may not have matching profiles (checking all devices isn't reliable 1039 * during table load because this table may use other DM device(s) which 1040 * must be resumed before they will have an initialized integity profile). 1041 * Stacked DM devices force a 2 stage integrity profile validation: 1042 * 1 - during load, validate all initialized integrity profiles match 1043 * 2 - during resume, validate all integrity profiles match 1044 */ 1045 static int dm_table_prealloc_integrity(struct dm_table *t, struct mapped_device *md) 1046 { 1047 struct gendisk *template_disk = NULL; 1048 1049 template_disk = dm_table_get_integrity_disk(t, false); 1050 if (!template_disk) 1051 return 0; 1052 1053 if (!blk_integrity_is_initialized(dm_disk(md))) { 1054 t->integrity_supported = 1; 1055 return blk_integrity_register(dm_disk(md), NULL); 1056 } 1057 1058 /* 1059 * If DM device already has an initalized integrity 1060 * profile the new profile should not conflict. 1061 */ 1062 if (blk_integrity_is_initialized(template_disk) && 1063 blk_integrity_compare(dm_disk(md), template_disk) < 0) { 1064 DMWARN("%s: conflict with existing integrity profile: " 1065 "%s profile mismatch", 1066 dm_device_name(t->md), 1067 template_disk->disk_name); 1068 return 1; 1069 } 1070 1071 /* Preserve existing initialized integrity profile */ 1072 t->integrity_supported = 1; 1073 return 0; 1074 } 1075 1076 /* 1077 * Prepares the table for use by building the indices, 1078 * setting the type, and allocating mempools. 1079 */ 1080 int dm_table_complete(struct dm_table *t) 1081 { 1082 int r; 1083 1084 r = dm_table_set_type(t); 1085 if (r) { 1086 DMERR("unable to set table type"); 1087 return r; 1088 } 1089 1090 r = dm_table_build_index(t); 1091 if (r) { 1092 DMERR("unable to build btrees"); 1093 return r; 1094 } 1095 1096 r = dm_table_prealloc_integrity(t, t->md); 1097 if (r) { 1098 DMERR("could not register integrity profile."); 1099 return r; 1100 } 1101 1102 r = dm_table_alloc_md_mempools(t); 1103 if (r) 1104 DMERR("unable to allocate mempools"); 1105 1106 return r; 1107 } 1108 1109 static DEFINE_MUTEX(_event_lock); 1110 void dm_table_event_callback(struct dm_table *t, 1111 void (*fn)(void *), void *context) 1112 { 1113 mutex_lock(&_event_lock); 1114 t->event_fn = fn; 1115 t->event_context = context; 1116 mutex_unlock(&_event_lock); 1117 } 1118 1119 void dm_table_event(struct dm_table *t) 1120 { 1121 /* 1122 * You can no longer call dm_table_event() from interrupt 1123 * context, use a bottom half instead. 1124 */ 1125 BUG_ON(in_interrupt()); 1126 1127 mutex_lock(&_event_lock); 1128 if (t->event_fn) 1129 t->event_fn(t->event_context); 1130 mutex_unlock(&_event_lock); 1131 } 1132 EXPORT_SYMBOL(dm_table_event); 1133 1134 sector_t dm_table_get_size(struct dm_table *t) 1135 { 1136 return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0; 1137 } 1138 EXPORT_SYMBOL(dm_table_get_size); 1139 1140 struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index) 1141 { 1142 if (index >= t->num_targets) 1143 return NULL; 1144 1145 return t->targets + index; 1146 } 1147 1148 /* 1149 * Search the btree for the correct target. 1150 * 1151 * Caller should check returned pointer with dm_target_is_valid() 1152 * to trap I/O beyond end of device. 1153 */ 1154 struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector) 1155 { 1156 unsigned int l, n = 0, k = 0; 1157 sector_t *node; 1158 1159 for (l = 0; l < t->depth; l++) { 1160 n = get_child(n, k); 1161 node = get_node(t, l, n); 1162 1163 for (k = 0; k < KEYS_PER_NODE; k++) 1164 if (node[k] >= sector) 1165 break; 1166 } 1167 1168 return &t->targets[(KEYS_PER_NODE * n) + k]; 1169 } 1170 1171 /* 1172 * Establish the new table's queue_limits and validate them. 1173 */ 1174 int dm_calculate_queue_limits(struct dm_table *table, 1175 struct queue_limits *limits) 1176 { 1177 struct dm_target *uninitialized_var(ti); 1178 struct queue_limits ti_limits; 1179 unsigned i = 0; 1180 1181 blk_set_default_limits(limits); 1182 1183 while (i < dm_table_get_num_targets(table)) { 1184 blk_set_default_limits(&ti_limits); 1185 1186 ti = dm_table_get_target(table, i++); 1187 1188 if (!ti->type->iterate_devices) 1189 goto combine_limits; 1190 1191 /* 1192 * Combine queue limits of all the devices this target uses. 1193 */ 1194 ti->type->iterate_devices(ti, dm_set_device_limits, 1195 &ti_limits); 1196 1197 /* Set I/O hints portion of queue limits */ 1198 if (ti->type->io_hints) 1199 ti->type->io_hints(ti, &ti_limits); 1200 1201 /* 1202 * Check each device area is consistent with the target's 1203 * overall queue limits. 1204 */ 1205 if (ti->type->iterate_devices(ti, device_area_is_invalid, 1206 &ti_limits)) 1207 return -EINVAL; 1208 1209 combine_limits: 1210 /* 1211 * Merge this target's queue limits into the overall limits 1212 * for the table. 1213 */ 1214 if (blk_stack_limits(limits, &ti_limits, 0) < 0) 1215 DMWARN("%s: adding target device " 1216 "(start sect %llu len %llu) " 1217 "caused an alignment inconsistency", 1218 dm_device_name(table->md), 1219 (unsigned long long) ti->begin, 1220 (unsigned long long) ti->len); 1221 } 1222 1223 return validate_hardware_logical_block_alignment(table, limits); 1224 } 1225 1226 /* 1227 * Set the integrity profile for this device if all devices used have 1228 * matching profiles. We're quite deep in the resume path but still 1229 * don't know if all devices (particularly DM devices this device 1230 * may be stacked on) have matching profiles. Even if the profiles 1231 * don't match we have no way to fail (to resume) at this point. 1232 */ 1233 static void dm_table_set_integrity(struct dm_table *t) 1234 { 1235 struct gendisk *template_disk = NULL; 1236 1237 if (!blk_get_integrity(dm_disk(t->md))) 1238 return; 1239 1240 template_disk = dm_table_get_integrity_disk(t, true); 1241 if (!template_disk && 1242 blk_integrity_is_initialized(dm_disk(t->md))) { 1243 DMWARN("%s: device no longer has a valid integrity profile", 1244 dm_device_name(t->md)); 1245 return; 1246 } 1247 blk_integrity_register(dm_disk(t->md), 1248 blk_get_integrity(template_disk)); 1249 } 1250 1251 static int device_flush_capable(struct dm_target *ti, struct dm_dev *dev, 1252 sector_t start, sector_t len, void *data) 1253 { 1254 unsigned flush = (*(unsigned *)data); 1255 struct request_queue *q = bdev_get_queue(dev->bdev); 1256 1257 return q && (q->flush_flags & flush); 1258 } 1259 1260 static bool dm_table_supports_flush(struct dm_table *t, unsigned flush) 1261 { 1262 struct dm_target *ti; 1263 unsigned i = 0; 1264 1265 /* 1266 * Require at least one underlying device to support flushes. 1267 * t->devices includes internal dm devices such as mirror logs 1268 * so we need to use iterate_devices here, which targets 1269 * supporting flushes must provide. 1270 */ 1271 while (i < dm_table_get_num_targets(t)) { 1272 ti = dm_table_get_target(t, i++); 1273 1274 if (!ti->num_flush_requests) 1275 continue; 1276 1277 if (ti->type->iterate_devices && 1278 ti->type->iterate_devices(ti, device_flush_capable, &flush)) 1279 return 1; 1280 } 1281 1282 return 0; 1283 } 1284 1285 void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, 1286 struct queue_limits *limits) 1287 { 1288 unsigned flush = 0; 1289 1290 /* 1291 * Copy table's limits to the DM device's request_queue 1292 */ 1293 q->limits = *limits; 1294 1295 if (!dm_table_supports_discards(t)) 1296 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); 1297 else 1298 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 1299 1300 if (dm_table_supports_flush(t, REQ_FLUSH)) { 1301 flush |= REQ_FLUSH; 1302 if (dm_table_supports_flush(t, REQ_FUA)) 1303 flush |= REQ_FUA; 1304 } 1305 blk_queue_flush(q, flush); 1306 1307 dm_table_set_integrity(t); 1308 1309 /* 1310 * QUEUE_FLAG_STACKABLE must be set after all queue settings are 1311 * visible to other CPUs because, once the flag is set, incoming bios 1312 * are processed by request-based dm, which refers to the queue 1313 * settings. 1314 * Until the flag set, bios are passed to bio-based dm and queued to 1315 * md->deferred where queue settings are not needed yet. 1316 * Those bios are passed to request-based dm at the resume time. 1317 */ 1318 smp_mb(); 1319 if (dm_table_request_based(t)) 1320 queue_flag_set_unlocked(QUEUE_FLAG_STACKABLE, q); 1321 } 1322 1323 unsigned int dm_table_get_num_targets(struct dm_table *t) 1324 { 1325 return t->num_targets; 1326 } 1327 1328 struct list_head *dm_table_get_devices(struct dm_table *t) 1329 { 1330 return &t->devices; 1331 } 1332 1333 fmode_t dm_table_get_mode(struct dm_table *t) 1334 { 1335 return t->mode; 1336 } 1337 EXPORT_SYMBOL(dm_table_get_mode); 1338 1339 static void suspend_targets(struct dm_table *t, unsigned postsuspend) 1340 { 1341 int i = t->num_targets; 1342 struct dm_target *ti = t->targets; 1343 1344 while (i--) { 1345 if (postsuspend) { 1346 if (ti->type->postsuspend) 1347 ti->type->postsuspend(ti); 1348 } else if (ti->type->presuspend) 1349 ti->type->presuspend(ti); 1350 1351 ti++; 1352 } 1353 } 1354 1355 void dm_table_presuspend_targets(struct dm_table *t) 1356 { 1357 if (!t) 1358 return; 1359 1360 suspend_targets(t, 0); 1361 } 1362 1363 void dm_table_postsuspend_targets(struct dm_table *t) 1364 { 1365 if (!t) 1366 return; 1367 1368 suspend_targets(t, 1); 1369 } 1370 1371 int dm_table_resume_targets(struct dm_table *t) 1372 { 1373 int i, r = 0; 1374 1375 for (i = 0; i < t->num_targets; i++) { 1376 struct dm_target *ti = t->targets + i; 1377 1378 if (!ti->type->preresume) 1379 continue; 1380 1381 r = ti->type->preresume(ti); 1382 if (r) 1383 return r; 1384 } 1385 1386 for (i = 0; i < t->num_targets; i++) { 1387 struct dm_target *ti = t->targets + i; 1388 1389 if (ti->type->resume) 1390 ti->type->resume(ti); 1391 } 1392 1393 return 0; 1394 } 1395 1396 void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb) 1397 { 1398 list_add(&cb->list, &t->target_callbacks); 1399 } 1400 EXPORT_SYMBOL_GPL(dm_table_add_target_callbacks); 1401 1402 int dm_table_any_congested(struct dm_table *t, int bdi_bits) 1403 { 1404 struct dm_dev_internal *dd; 1405 struct list_head *devices = dm_table_get_devices(t); 1406 struct dm_target_callbacks *cb; 1407 int r = 0; 1408 1409 list_for_each_entry(dd, devices, list) { 1410 struct request_queue *q = bdev_get_queue(dd->dm_dev.bdev); 1411 char b[BDEVNAME_SIZE]; 1412 1413 if (likely(q)) 1414 r |= bdi_congested(&q->backing_dev_info, bdi_bits); 1415 else 1416 DMWARN_LIMIT("%s: any_congested: nonexistent device %s", 1417 dm_device_name(t->md), 1418 bdevname(dd->dm_dev.bdev, b)); 1419 } 1420 1421 list_for_each_entry(cb, &t->target_callbacks, list) 1422 if (cb->congested_fn) 1423 r |= cb->congested_fn(cb, bdi_bits); 1424 1425 return r; 1426 } 1427 1428 int dm_table_any_busy_target(struct dm_table *t) 1429 { 1430 unsigned i; 1431 struct dm_target *ti; 1432 1433 for (i = 0; i < t->num_targets; i++) { 1434 ti = t->targets + i; 1435 if (ti->type->busy && ti->type->busy(ti)) 1436 return 1; 1437 } 1438 1439 return 0; 1440 } 1441 1442 struct mapped_device *dm_table_get_md(struct dm_table *t) 1443 { 1444 return t->md; 1445 } 1446 EXPORT_SYMBOL(dm_table_get_md); 1447 1448 static int device_discard_capable(struct dm_target *ti, struct dm_dev *dev, 1449 sector_t start, sector_t len, void *data) 1450 { 1451 struct request_queue *q = bdev_get_queue(dev->bdev); 1452 1453 return q && blk_queue_discard(q); 1454 } 1455 1456 bool dm_table_supports_discards(struct dm_table *t) 1457 { 1458 struct dm_target *ti; 1459 unsigned i = 0; 1460 1461 /* 1462 * Unless any target used by the table set discards_supported, 1463 * require at least one underlying device to support discards. 1464 * t->devices includes internal dm devices such as mirror logs 1465 * so we need to use iterate_devices here, which targets 1466 * supporting discard selectively must provide. 1467 */ 1468 while (i < dm_table_get_num_targets(t)) { 1469 ti = dm_table_get_target(t, i++); 1470 1471 if (!ti->num_discard_requests) 1472 continue; 1473 1474 if (ti->discards_supported) 1475 return 1; 1476 1477 if (ti->type->iterate_devices && 1478 ti->type->iterate_devices(ti, device_discard_capable, NULL)) 1479 return 1; 1480 } 1481 1482 return 0; 1483 } 1484