1 /* 2 * Copyright (C) 2009-2011 Red Hat, Inc. 3 * 4 * Author: Mikulas Patocka <mpatocka@redhat.com> 5 * 6 * This file is released under the GPL. 7 */ 8 9 #include <linux/dm-bufio.h> 10 11 #include <linux/device-mapper.h> 12 #include <linux/dm-io.h> 13 #include <linux/slab.h> 14 #include <linux/sched/mm.h> 15 #include <linux/jiffies.h> 16 #include <linux/vmalloc.h> 17 #include <linux/shrinker.h> 18 #include <linux/module.h> 19 #include <linux/rbtree.h> 20 #include <linux/stacktrace.h> 21 22 #define DM_MSG_PREFIX "bufio" 23 24 /* 25 * Memory management policy: 26 * Limit the number of buffers to DM_BUFIO_MEMORY_PERCENT of main memory 27 * or DM_BUFIO_VMALLOC_PERCENT of vmalloc memory (whichever is lower). 28 * Always allocate at least DM_BUFIO_MIN_BUFFERS buffers. 29 * Start background writeback when there are DM_BUFIO_WRITEBACK_PERCENT 30 * dirty buffers. 31 */ 32 #define DM_BUFIO_MIN_BUFFERS 8 33 34 #define DM_BUFIO_MEMORY_PERCENT 2 35 #define DM_BUFIO_VMALLOC_PERCENT 25 36 #define DM_BUFIO_WRITEBACK_PERCENT 75 37 38 /* 39 * Check buffer ages in this interval (seconds) 40 */ 41 #define DM_BUFIO_WORK_TIMER_SECS 30 42 43 /* 44 * Free buffers when they are older than this (seconds) 45 */ 46 #define DM_BUFIO_DEFAULT_AGE_SECS 300 47 48 /* 49 * The nr of bytes of cached data to keep around. 50 */ 51 #define DM_BUFIO_DEFAULT_RETAIN_BYTES (256 * 1024) 52 53 /* 54 * Align buffer writes to this boundary. 55 * Tests show that SSDs have the highest IOPS when using 4k writes. 56 */ 57 #define DM_BUFIO_WRITE_ALIGN 4096 58 59 /* 60 * dm_buffer->list_mode 61 */ 62 #define LIST_CLEAN 0 63 #define LIST_DIRTY 1 64 #define LIST_SIZE 2 65 66 /* 67 * Linking of buffers: 68 * All buffers are linked to buffer_tree with their node field. 69 * 70 * Clean buffers that are not being written (B_WRITING not set) 71 * are linked to lru[LIST_CLEAN] with their lru_list field. 72 * 73 * Dirty and clean buffers that are being written are linked to 74 * lru[LIST_DIRTY] with their lru_list field. When the write 75 * finishes, the buffer cannot be relinked immediately (because we 76 * are in an interrupt context and relinking requires process 77 * context), so some clean-not-writing buffers can be held on 78 * dirty_lru too. They are later added to lru in the process 79 * context. 80 */ 81 struct dm_bufio_client { 82 struct mutex lock; 83 84 struct list_head lru[LIST_SIZE]; 85 unsigned long n_buffers[LIST_SIZE]; 86 87 struct block_device *bdev; 88 unsigned block_size; 89 s8 sectors_per_block_bits; 90 void (*alloc_callback)(struct dm_buffer *); 91 void (*write_callback)(struct dm_buffer *); 92 93 struct kmem_cache *slab_buffer; 94 struct kmem_cache *slab_cache; 95 struct dm_io_client *dm_io; 96 97 struct list_head reserved_buffers; 98 unsigned need_reserved_buffers; 99 100 unsigned minimum_buffers; 101 102 struct rb_root buffer_tree; 103 wait_queue_head_t free_buffer_wait; 104 105 sector_t start; 106 107 int async_write_error; 108 109 struct list_head client_list; 110 struct shrinker shrinker; 111 }; 112 113 /* 114 * Buffer state bits. 115 */ 116 #define B_READING 0 117 #define B_WRITING 1 118 #define B_DIRTY 2 119 120 /* 121 * Describes how the block was allocated: 122 * kmem_cache_alloc(), __get_free_pages() or vmalloc(). 123 * See the comment at alloc_buffer_data. 124 */ 125 enum data_mode { 126 DATA_MODE_SLAB = 0, 127 DATA_MODE_GET_FREE_PAGES = 1, 128 DATA_MODE_VMALLOC = 2, 129 DATA_MODE_LIMIT = 3 130 }; 131 132 struct dm_buffer { 133 struct rb_node node; 134 struct list_head lru_list; 135 sector_t block; 136 void *data; 137 unsigned char data_mode; /* DATA_MODE_* */ 138 unsigned char list_mode; /* LIST_* */ 139 blk_status_t read_error; 140 blk_status_t write_error; 141 unsigned hold_count; 142 unsigned long state; 143 unsigned long last_accessed; 144 unsigned dirty_start; 145 unsigned dirty_end; 146 unsigned write_start; 147 unsigned write_end; 148 struct dm_bufio_client *c; 149 struct list_head write_list; 150 void (*end_io)(struct dm_buffer *, blk_status_t); 151 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 152 #define MAX_STACK 10 153 unsigned int stack_len; 154 unsigned long stack_entries[MAX_STACK]; 155 #endif 156 }; 157 158 /*----------------------------------------------------------------*/ 159 160 #define dm_bufio_in_request() (!!current->bio_list) 161 162 static void dm_bufio_lock(struct dm_bufio_client *c) 163 { 164 mutex_lock_nested(&c->lock, dm_bufio_in_request()); 165 } 166 167 static int dm_bufio_trylock(struct dm_bufio_client *c) 168 { 169 return mutex_trylock(&c->lock); 170 } 171 172 static void dm_bufio_unlock(struct dm_bufio_client *c) 173 { 174 mutex_unlock(&c->lock); 175 } 176 177 /*----------------------------------------------------------------*/ 178 179 /* 180 * Default cache size: available memory divided by the ratio. 181 */ 182 static unsigned long dm_bufio_default_cache_size; 183 184 /* 185 * Total cache size set by the user. 186 */ 187 static unsigned long dm_bufio_cache_size; 188 189 /* 190 * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change 191 * at any time. If it disagrees, the user has changed cache size. 192 */ 193 static unsigned long dm_bufio_cache_size_latch; 194 195 static DEFINE_SPINLOCK(param_spinlock); 196 197 /* 198 * Buffers are freed after this timeout 199 */ 200 static unsigned dm_bufio_max_age = DM_BUFIO_DEFAULT_AGE_SECS; 201 static unsigned long dm_bufio_retain_bytes = DM_BUFIO_DEFAULT_RETAIN_BYTES; 202 203 static unsigned long dm_bufio_peak_allocated; 204 static unsigned long dm_bufio_allocated_kmem_cache; 205 static unsigned long dm_bufio_allocated_get_free_pages; 206 static unsigned long dm_bufio_allocated_vmalloc; 207 static unsigned long dm_bufio_current_allocated; 208 209 /*----------------------------------------------------------------*/ 210 211 /* 212 * Per-client cache: dm_bufio_cache_size / dm_bufio_client_count 213 */ 214 static unsigned long dm_bufio_cache_size_per_client; 215 216 /* 217 * The current number of clients. 218 */ 219 static int dm_bufio_client_count; 220 221 /* 222 * The list of all clients. 223 */ 224 static LIST_HEAD(dm_bufio_all_clients); 225 226 /* 227 * This mutex protects dm_bufio_cache_size_latch, 228 * dm_bufio_cache_size_per_client and dm_bufio_client_count 229 */ 230 static DEFINE_MUTEX(dm_bufio_clients_lock); 231 232 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 233 static void buffer_record_stack(struct dm_buffer *b) 234 { 235 b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2); 236 } 237 #endif 238 239 /*---------------------------------------------------------------- 240 * A red/black tree acts as an index for all the buffers. 241 *--------------------------------------------------------------*/ 242 static struct dm_buffer *__find(struct dm_bufio_client *c, sector_t block) 243 { 244 struct rb_node *n = c->buffer_tree.rb_node; 245 struct dm_buffer *b; 246 247 while (n) { 248 b = container_of(n, struct dm_buffer, node); 249 250 if (b->block == block) 251 return b; 252 253 n = (b->block < block) ? n->rb_left : n->rb_right; 254 } 255 256 return NULL; 257 } 258 259 static void __insert(struct dm_bufio_client *c, struct dm_buffer *b) 260 { 261 struct rb_node **new = &c->buffer_tree.rb_node, *parent = NULL; 262 struct dm_buffer *found; 263 264 while (*new) { 265 found = container_of(*new, struct dm_buffer, node); 266 267 if (found->block == b->block) { 268 BUG_ON(found != b); 269 return; 270 } 271 272 parent = *new; 273 new = (found->block < b->block) ? 274 &((*new)->rb_left) : &((*new)->rb_right); 275 } 276 277 rb_link_node(&b->node, parent, new); 278 rb_insert_color(&b->node, &c->buffer_tree); 279 } 280 281 static void __remove(struct dm_bufio_client *c, struct dm_buffer *b) 282 { 283 rb_erase(&b->node, &c->buffer_tree); 284 } 285 286 /*----------------------------------------------------------------*/ 287 288 static void adjust_total_allocated(unsigned char data_mode, long diff) 289 { 290 static unsigned long * const class_ptr[DATA_MODE_LIMIT] = { 291 &dm_bufio_allocated_kmem_cache, 292 &dm_bufio_allocated_get_free_pages, 293 &dm_bufio_allocated_vmalloc, 294 }; 295 296 spin_lock(¶m_spinlock); 297 298 *class_ptr[data_mode] += diff; 299 300 dm_bufio_current_allocated += diff; 301 302 if (dm_bufio_current_allocated > dm_bufio_peak_allocated) 303 dm_bufio_peak_allocated = dm_bufio_current_allocated; 304 305 spin_unlock(¶m_spinlock); 306 } 307 308 /* 309 * Change the number of clients and recalculate per-client limit. 310 */ 311 static void __cache_size_refresh(void) 312 { 313 BUG_ON(!mutex_is_locked(&dm_bufio_clients_lock)); 314 BUG_ON(dm_bufio_client_count < 0); 315 316 dm_bufio_cache_size_latch = READ_ONCE(dm_bufio_cache_size); 317 318 /* 319 * Use default if set to 0 and report the actual cache size used. 320 */ 321 if (!dm_bufio_cache_size_latch) { 322 (void)cmpxchg(&dm_bufio_cache_size, 0, 323 dm_bufio_default_cache_size); 324 dm_bufio_cache_size_latch = dm_bufio_default_cache_size; 325 } 326 327 dm_bufio_cache_size_per_client = dm_bufio_cache_size_latch / 328 (dm_bufio_client_count ? : 1); 329 } 330 331 /* 332 * Allocating buffer data. 333 * 334 * Small buffers are allocated with kmem_cache, to use space optimally. 335 * 336 * For large buffers, we choose between get_free_pages and vmalloc. 337 * Each has advantages and disadvantages. 338 * 339 * __get_free_pages can randomly fail if the memory is fragmented. 340 * __vmalloc won't randomly fail, but vmalloc space is limited (it may be 341 * as low as 128M) so using it for caching is not appropriate. 342 * 343 * If the allocation may fail we use __get_free_pages. Memory fragmentation 344 * won't have a fatal effect here, but it just causes flushes of some other 345 * buffers and more I/O will be performed. Don't use __get_free_pages if it 346 * always fails (i.e. order >= MAX_ORDER). 347 * 348 * If the allocation shouldn't fail we use __vmalloc. This is only for the 349 * initial reserve allocation, so there's no risk of wasting all vmalloc 350 * space. 351 */ 352 static void *alloc_buffer_data(struct dm_bufio_client *c, gfp_t gfp_mask, 353 unsigned char *data_mode) 354 { 355 if (unlikely(c->slab_cache != NULL)) { 356 *data_mode = DATA_MODE_SLAB; 357 return kmem_cache_alloc(c->slab_cache, gfp_mask); 358 } 359 360 if (c->block_size <= KMALLOC_MAX_SIZE && 361 gfp_mask & __GFP_NORETRY) { 362 *data_mode = DATA_MODE_GET_FREE_PAGES; 363 return (void *)__get_free_pages(gfp_mask, 364 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); 365 } 366 367 *data_mode = DATA_MODE_VMALLOC; 368 369 /* 370 * __vmalloc allocates the data pages and auxiliary structures with 371 * gfp_flags that were specified, but pagetables are always allocated 372 * with GFP_KERNEL, no matter what was specified as gfp_mask. 373 * 374 * Consequently, we must set per-process flag PF_MEMALLOC_NOIO so that 375 * all allocations done by this process (including pagetables) are done 376 * as if GFP_NOIO was specified. 377 */ 378 if (gfp_mask & __GFP_NORETRY) { 379 unsigned noio_flag = memalloc_noio_save(); 380 void *ptr = __vmalloc(c->block_size, gfp_mask, PAGE_KERNEL); 381 382 memalloc_noio_restore(noio_flag); 383 return ptr; 384 } 385 386 return __vmalloc(c->block_size, gfp_mask, PAGE_KERNEL); 387 } 388 389 /* 390 * Free buffer's data. 391 */ 392 static void free_buffer_data(struct dm_bufio_client *c, 393 void *data, unsigned char data_mode) 394 { 395 switch (data_mode) { 396 case DATA_MODE_SLAB: 397 kmem_cache_free(c->slab_cache, data); 398 break; 399 400 case DATA_MODE_GET_FREE_PAGES: 401 free_pages((unsigned long)data, 402 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); 403 break; 404 405 case DATA_MODE_VMALLOC: 406 vfree(data); 407 break; 408 409 default: 410 DMCRIT("dm_bufio_free_buffer_data: bad data mode: %d", 411 data_mode); 412 BUG(); 413 } 414 } 415 416 /* 417 * Allocate buffer and its data. 418 */ 419 static struct dm_buffer *alloc_buffer(struct dm_bufio_client *c, gfp_t gfp_mask) 420 { 421 struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask); 422 423 if (!b) 424 return NULL; 425 426 b->c = c; 427 428 b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode); 429 if (!b->data) { 430 kmem_cache_free(c->slab_buffer, b); 431 return NULL; 432 } 433 434 adjust_total_allocated(b->data_mode, (long)c->block_size); 435 436 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 437 b->stack_len = 0; 438 #endif 439 return b; 440 } 441 442 /* 443 * Free buffer and its data. 444 */ 445 static void free_buffer(struct dm_buffer *b) 446 { 447 struct dm_bufio_client *c = b->c; 448 449 adjust_total_allocated(b->data_mode, -(long)c->block_size); 450 451 free_buffer_data(c, b->data, b->data_mode); 452 kmem_cache_free(c->slab_buffer, b); 453 } 454 455 /* 456 * Link buffer to the buffer tree and clean or dirty queue. 457 */ 458 static void __link_buffer(struct dm_buffer *b, sector_t block, int dirty) 459 { 460 struct dm_bufio_client *c = b->c; 461 462 c->n_buffers[dirty]++; 463 b->block = block; 464 b->list_mode = dirty; 465 list_add(&b->lru_list, &c->lru[dirty]); 466 __insert(b->c, b); 467 b->last_accessed = jiffies; 468 } 469 470 /* 471 * Unlink buffer from the buffer tree and dirty or clean queue. 472 */ 473 static void __unlink_buffer(struct dm_buffer *b) 474 { 475 struct dm_bufio_client *c = b->c; 476 477 BUG_ON(!c->n_buffers[b->list_mode]); 478 479 c->n_buffers[b->list_mode]--; 480 __remove(b->c, b); 481 list_del(&b->lru_list); 482 } 483 484 /* 485 * Place the buffer to the head of dirty or clean LRU queue. 486 */ 487 static void __relink_lru(struct dm_buffer *b, int dirty) 488 { 489 struct dm_bufio_client *c = b->c; 490 491 BUG_ON(!c->n_buffers[b->list_mode]); 492 493 c->n_buffers[b->list_mode]--; 494 c->n_buffers[dirty]++; 495 b->list_mode = dirty; 496 list_move(&b->lru_list, &c->lru[dirty]); 497 b->last_accessed = jiffies; 498 } 499 500 /*---------------------------------------------------------------- 501 * Submit I/O on the buffer. 502 * 503 * Bio interface is faster but it has some problems: 504 * the vector list is limited (increasing this limit increases 505 * memory-consumption per buffer, so it is not viable); 506 * 507 * the memory must be direct-mapped, not vmalloced; 508 * 509 * If the buffer is small enough (up to DM_BUFIO_INLINE_VECS pages) and 510 * it is not vmalloced, try using the bio interface. 511 * 512 * If the buffer is big, if it is vmalloced or if the underlying device 513 * rejects the bio because it is too large, use dm-io layer to do the I/O. 514 * The dm-io layer splits the I/O into multiple requests, avoiding the above 515 * shortcomings. 516 *--------------------------------------------------------------*/ 517 518 /* 519 * dm-io completion routine. It just calls b->bio.bi_end_io, pretending 520 * that the request was handled directly with bio interface. 521 */ 522 static void dmio_complete(unsigned long error, void *context) 523 { 524 struct dm_buffer *b = context; 525 526 b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0); 527 } 528 529 static void use_dmio(struct dm_buffer *b, int rw, sector_t sector, 530 unsigned n_sectors, unsigned offset) 531 { 532 int r; 533 struct dm_io_request io_req = { 534 .bi_op = rw, 535 .bi_op_flags = 0, 536 .notify.fn = dmio_complete, 537 .notify.context = b, 538 .client = b->c->dm_io, 539 }; 540 struct dm_io_region region = { 541 .bdev = b->c->bdev, 542 .sector = sector, 543 .count = n_sectors, 544 }; 545 546 if (b->data_mode != DATA_MODE_VMALLOC) { 547 io_req.mem.type = DM_IO_KMEM; 548 io_req.mem.ptr.addr = (char *)b->data + offset; 549 } else { 550 io_req.mem.type = DM_IO_VMA; 551 io_req.mem.ptr.vma = (char *)b->data + offset; 552 } 553 554 r = dm_io(&io_req, 1, ®ion, NULL); 555 if (unlikely(r)) 556 b->end_io(b, errno_to_blk_status(r)); 557 } 558 559 static void bio_complete(struct bio *bio) 560 { 561 struct dm_buffer *b = bio->bi_private; 562 blk_status_t status = bio->bi_status; 563 bio_put(bio); 564 b->end_io(b, status); 565 } 566 567 static void use_bio(struct dm_buffer *b, int rw, sector_t sector, 568 unsigned n_sectors, unsigned offset) 569 { 570 struct bio *bio; 571 char *ptr; 572 unsigned vec_size, len; 573 574 vec_size = b->c->block_size >> PAGE_SHIFT; 575 if (unlikely(b->c->sectors_per_block_bits < PAGE_SHIFT - SECTOR_SHIFT)) 576 vec_size += 2; 577 578 bio = bio_kmalloc(GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN, vec_size); 579 if (!bio) { 580 dmio: 581 use_dmio(b, rw, sector, n_sectors, offset); 582 return; 583 } 584 585 bio->bi_iter.bi_sector = sector; 586 bio_set_dev(bio, b->c->bdev); 587 bio_set_op_attrs(bio, rw, 0); 588 bio->bi_end_io = bio_complete; 589 bio->bi_private = b; 590 591 ptr = (char *)b->data + offset; 592 len = n_sectors << SECTOR_SHIFT; 593 594 do { 595 unsigned this_step = min((unsigned)(PAGE_SIZE - offset_in_page(ptr)), len); 596 if (!bio_add_page(bio, virt_to_page(ptr), this_step, 597 offset_in_page(ptr))) { 598 bio_put(bio); 599 goto dmio; 600 } 601 602 len -= this_step; 603 ptr += this_step; 604 } while (len > 0); 605 606 submit_bio(bio); 607 } 608 609 static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buffer *, blk_status_t)) 610 { 611 unsigned n_sectors; 612 sector_t sector; 613 unsigned offset, end; 614 615 b->end_io = end_io; 616 617 if (likely(b->c->sectors_per_block_bits >= 0)) 618 sector = b->block << b->c->sectors_per_block_bits; 619 else 620 sector = b->block * (b->c->block_size >> SECTOR_SHIFT); 621 sector += b->c->start; 622 623 if (rw != REQ_OP_WRITE) { 624 n_sectors = b->c->block_size >> SECTOR_SHIFT; 625 offset = 0; 626 } else { 627 if (b->c->write_callback) 628 b->c->write_callback(b); 629 offset = b->write_start; 630 end = b->write_end; 631 offset &= -DM_BUFIO_WRITE_ALIGN; 632 end += DM_BUFIO_WRITE_ALIGN - 1; 633 end &= -DM_BUFIO_WRITE_ALIGN; 634 if (unlikely(end > b->c->block_size)) 635 end = b->c->block_size; 636 637 sector += offset >> SECTOR_SHIFT; 638 n_sectors = (end - offset) >> SECTOR_SHIFT; 639 } 640 641 if (b->data_mode != DATA_MODE_VMALLOC) 642 use_bio(b, rw, sector, n_sectors, offset); 643 else 644 use_dmio(b, rw, sector, n_sectors, offset); 645 } 646 647 /*---------------------------------------------------------------- 648 * Writing dirty buffers 649 *--------------------------------------------------------------*/ 650 651 /* 652 * The endio routine for write. 653 * 654 * Set the error, clear B_WRITING bit and wake anyone who was waiting on 655 * it. 656 */ 657 static void write_endio(struct dm_buffer *b, blk_status_t status) 658 { 659 b->write_error = status; 660 if (unlikely(status)) { 661 struct dm_bufio_client *c = b->c; 662 663 (void)cmpxchg(&c->async_write_error, 0, 664 blk_status_to_errno(status)); 665 } 666 667 BUG_ON(!test_bit(B_WRITING, &b->state)); 668 669 smp_mb__before_atomic(); 670 clear_bit(B_WRITING, &b->state); 671 smp_mb__after_atomic(); 672 673 wake_up_bit(&b->state, B_WRITING); 674 } 675 676 /* 677 * Initiate a write on a dirty buffer, but don't wait for it. 678 * 679 * - If the buffer is not dirty, exit. 680 * - If there some previous write going on, wait for it to finish (we can't 681 * have two writes on the same buffer simultaneously). 682 * - Submit our write and don't wait on it. We set B_WRITING indicating 683 * that there is a write in progress. 684 */ 685 static void __write_dirty_buffer(struct dm_buffer *b, 686 struct list_head *write_list) 687 { 688 if (!test_bit(B_DIRTY, &b->state)) 689 return; 690 691 clear_bit(B_DIRTY, &b->state); 692 wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); 693 694 b->write_start = b->dirty_start; 695 b->write_end = b->dirty_end; 696 697 if (!write_list) 698 submit_io(b, REQ_OP_WRITE, write_endio); 699 else 700 list_add_tail(&b->write_list, write_list); 701 } 702 703 static void __flush_write_list(struct list_head *write_list) 704 { 705 struct blk_plug plug; 706 blk_start_plug(&plug); 707 while (!list_empty(write_list)) { 708 struct dm_buffer *b = 709 list_entry(write_list->next, struct dm_buffer, write_list); 710 list_del(&b->write_list); 711 submit_io(b, REQ_OP_WRITE, write_endio); 712 cond_resched(); 713 } 714 blk_finish_plug(&plug); 715 } 716 717 /* 718 * Wait until any activity on the buffer finishes. Possibly write the 719 * buffer if it is dirty. When this function finishes, there is no I/O 720 * running on the buffer and the buffer is not dirty. 721 */ 722 static void __make_buffer_clean(struct dm_buffer *b) 723 { 724 BUG_ON(b->hold_count); 725 726 if (!b->state) /* fast case */ 727 return; 728 729 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); 730 __write_dirty_buffer(b, NULL); 731 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); 732 } 733 734 /* 735 * Find some buffer that is not held by anybody, clean it, unlink it and 736 * return it. 737 */ 738 static struct dm_buffer *__get_unclaimed_buffer(struct dm_bufio_client *c) 739 { 740 struct dm_buffer *b; 741 742 list_for_each_entry_reverse(b, &c->lru[LIST_CLEAN], lru_list) { 743 BUG_ON(test_bit(B_WRITING, &b->state)); 744 BUG_ON(test_bit(B_DIRTY, &b->state)); 745 746 if (!b->hold_count) { 747 __make_buffer_clean(b); 748 __unlink_buffer(b); 749 return b; 750 } 751 cond_resched(); 752 } 753 754 list_for_each_entry_reverse(b, &c->lru[LIST_DIRTY], lru_list) { 755 BUG_ON(test_bit(B_READING, &b->state)); 756 757 if (!b->hold_count) { 758 __make_buffer_clean(b); 759 __unlink_buffer(b); 760 return b; 761 } 762 cond_resched(); 763 } 764 765 return NULL; 766 } 767 768 /* 769 * Wait until some other threads free some buffer or release hold count on 770 * some buffer. 771 * 772 * This function is entered with c->lock held, drops it and regains it 773 * before exiting. 774 */ 775 static void __wait_for_free_buffer(struct dm_bufio_client *c) 776 { 777 DECLARE_WAITQUEUE(wait, current); 778 779 add_wait_queue(&c->free_buffer_wait, &wait); 780 set_current_state(TASK_UNINTERRUPTIBLE); 781 dm_bufio_unlock(c); 782 783 io_schedule(); 784 785 remove_wait_queue(&c->free_buffer_wait, &wait); 786 787 dm_bufio_lock(c); 788 } 789 790 enum new_flag { 791 NF_FRESH = 0, 792 NF_READ = 1, 793 NF_GET = 2, 794 NF_PREFETCH = 3 795 }; 796 797 /* 798 * Allocate a new buffer. If the allocation is not possible, wait until 799 * some other thread frees a buffer. 800 * 801 * May drop the lock and regain it. 802 */ 803 static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf) 804 { 805 struct dm_buffer *b; 806 bool tried_noio_alloc = false; 807 808 /* 809 * dm-bufio is resistant to allocation failures (it just keeps 810 * one buffer reserved in cases all the allocations fail). 811 * So set flags to not try too hard: 812 * GFP_NOWAIT: don't wait; if we need to sleep we'll release our 813 * mutex and wait ourselves. 814 * __GFP_NORETRY: don't retry and rather return failure 815 * __GFP_NOMEMALLOC: don't use emergency reserves 816 * __GFP_NOWARN: don't print a warning in case of failure 817 * 818 * For debugging, if we set the cache size to 1, no new buffers will 819 * be allocated. 820 */ 821 while (1) { 822 if (dm_bufio_cache_size_latch != 1) { 823 b = alloc_buffer(c, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 824 if (b) 825 return b; 826 } 827 828 if (nf == NF_PREFETCH) 829 return NULL; 830 831 if (dm_bufio_cache_size_latch != 1 && !tried_noio_alloc) { 832 dm_bufio_unlock(c); 833 b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); 834 dm_bufio_lock(c); 835 if (b) 836 return b; 837 tried_noio_alloc = true; 838 } 839 840 if (!list_empty(&c->reserved_buffers)) { 841 b = list_entry(c->reserved_buffers.next, 842 struct dm_buffer, lru_list); 843 list_del(&b->lru_list); 844 c->need_reserved_buffers++; 845 846 return b; 847 } 848 849 b = __get_unclaimed_buffer(c); 850 if (b) 851 return b; 852 853 __wait_for_free_buffer(c); 854 } 855 } 856 857 static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf) 858 { 859 struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf); 860 861 if (!b) 862 return NULL; 863 864 if (c->alloc_callback) 865 c->alloc_callback(b); 866 867 return b; 868 } 869 870 /* 871 * Free a buffer and wake other threads waiting for free buffers. 872 */ 873 static void __free_buffer_wake(struct dm_buffer *b) 874 { 875 struct dm_bufio_client *c = b->c; 876 877 if (!c->need_reserved_buffers) 878 free_buffer(b); 879 else { 880 list_add(&b->lru_list, &c->reserved_buffers); 881 c->need_reserved_buffers--; 882 } 883 884 wake_up(&c->free_buffer_wait); 885 } 886 887 static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait, 888 struct list_head *write_list) 889 { 890 struct dm_buffer *b, *tmp; 891 892 list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) { 893 BUG_ON(test_bit(B_READING, &b->state)); 894 895 if (!test_bit(B_DIRTY, &b->state) && 896 !test_bit(B_WRITING, &b->state)) { 897 __relink_lru(b, LIST_CLEAN); 898 continue; 899 } 900 901 if (no_wait && test_bit(B_WRITING, &b->state)) 902 return; 903 904 __write_dirty_buffer(b, write_list); 905 cond_resched(); 906 } 907 } 908 909 /* 910 * Get writeback threshold and buffer limit for a given client. 911 */ 912 static void __get_memory_limit(struct dm_bufio_client *c, 913 unsigned long *threshold_buffers, 914 unsigned long *limit_buffers) 915 { 916 unsigned long buffers; 917 918 if (unlikely(READ_ONCE(dm_bufio_cache_size) != dm_bufio_cache_size_latch)) { 919 if (mutex_trylock(&dm_bufio_clients_lock)) { 920 __cache_size_refresh(); 921 mutex_unlock(&dm_bufio_clients_lock); 922 } 923 } 924 925 buffers = dm_bufio_cache_size_per_client; 926 if (likely(c->sectors_per_block_bits >= 0)) 927 buffers >>= c->sectors_per_block_bits + SECTOR_SHIFT; 928 else 929 buffers /= c->block_size; 930 931 if (buffers < c->minimum_buffers) 932 buffers = c->minimum_buffers; 933 934 *limit_buffers = buffers; 935 *threshold_buffers = mult_frac(buffers, 936 DM_BUFIO_WRITEBACK_PERCENT, 100); 937 } 938 939 /* 940 * Check if we're over watermark. 941 * If we are over threshold_buffers, start freeing buffers. 942 * If we're over "limit_buffers", block until we get under the limit. 943 */ 944 static void __check_watermark(struct dm_bufio_client *c, 945 struct list_head *write_list) 946 { 947 unsigned long threshold_buffers, limit_buffers; 948 949 __get_memory_limit(c, &threshold_buffers, &limit_buffers); 950 951 while (c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY] > 952 limit_buffers) { 953 954 struct dm_buffer *b = __get_unclaimed_buffer(c); 955 956 if (!b) 957 return; 958 959 __free_buffer_wake(b); 960 cond_resched(); 961 } 962 963 if (c->n_buffers[LIST_DIRTY] > threshold_buffers) 964 __write_dirty_buffers_async(c, 1, write_list); 965 } 966 967 /*---------------------------------------------------------------- 968 * Getting a buffer 969 *--------------------------------------------------------------*/ 970 971 static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block, 972 enum new_flag nf, int *need_submit, 973 struct list_head *write_list) 974 { 975 struct dm_buffer *b, *new_b = NULL; 976 977 *need_submit = 0; 978 979 b = __find(c, block); 980 if (b) 981 goto found_buffer; 982 983 if (nf == NF_GET) 984 return NULL; 985 986 new_b = __alloc_buffer_wait(c, nf); 987 if (!new_b) 988 return NULL; 989 990 /* 991 * We've had a period where the mutex was unlocked, so need to 992 * recheck the buffer tree. 993 */ 994 b = __find(c, block); 995 if (b) { 996 __free_buffer_wake(new_b); 997 goto found_buffer; 998 } 999 1000 __check_watermark(c, write_list); 1001 1002 b = new_b; 1003 b->hold_count = 1; 1004 b->read_error = 0; 1005 b->write_error = 0; 1006 __link_buffer(b, block, LIST_CLEAN); 1007 1008 if (nf == NF_FRESH) { 1009 b->state = 0; 1010 return b; 1011 } 1012 1013 b->state = 1 << B_READING; 1014 *need_submit = 1; 1015 1016 return b; 1017 1018 found_buffer: 1019 if (nf == NF_PREFETCH) 1020 return NULL; 1021 /* 1022 * Note: it is essential that we don't wait for the buffer to be 1023 * read if dm_bufio_get function is used. Both dm_bufio_get and 1024 * dm_bufio_prefetch can be used in the driver request routine. 1025 * If the user called both dm_bufio_prefetch and dm_bufio_get on 1026 * the same buffer, it would deadlock if we waited. 1027 */ 1028 if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state))) 1029 return NULL; 1030 1031 b->hold_count++; 1032 __relink_lru(b, test_bit(B_DIRTY, &b->state) || 1033 test_bit(B_WRITING, &b->state)); 1034 return b; 1035 } 1036 1037 /* 1038 * The endio routine for reading: set the error, clear the bit and wake up 1039 * anyone waiting on the buffer. 1040 */ 1041 static void read_endio(struct dm_buffer *b, blk_status_t status) 1042 { 1043 b->read_error = status; 1044 1045 BUG_ON(!test_bit(B_READING, &b->state)); 1046 1047 smp_mb__before_atomic(); 1048 clear_bit(B_READING, &b->state); 1049 smp_mb__after_atomic(); 1050 1051 wake_up_bit(&b->state, B_READING); 1052 } 1053 1054 /* 1055 * A common routine for dm_bufio_new and dm_bufio_read. Operation of these 1056 * functions is similar except that dm_bufio_new doesn't read the 1057 * buffer from the disk (assuming that the caller overwrites all the data 1058 * and uses dm_bufio_mark_buffer_dirty to write new data back). 1059 */ 1060 static void *new_read(struct dm_bufio_client *c, sector_t block, 1061 enum new_flag nf, struct dm_buffer **bp) 1062 { 1063 int need_submit; 1064 struct dm_buffer *b; 1065 1066 LIST_HEAD(write_list); 1067 1068 dm_bufio_lock(c); 1069 b = __bufio_new(c, block, nf, &need_submit, &write_list); 1070 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 1071 if (b && b->hold_count == 1) 1072 buffer_record_stack(b); 1073 #endif 1074 dm_bufio_unlock(c); 1075 1076 __flush_write_list(&write_list); 1077 1078 if (!b) 1079 return NULL; 1080 1081 if (need_submit) 1082 submit_io(b, REQ_OP_READ, read_endio); 1083 1084 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); 1085 1086 if (b->read_error) { 1087 int error = blk_status_to_errno(b->read_error); 1088 1089 dm_bufio_release(b); 1090 1091 return ERR_PTR(error); 1092 } 1093 1094 *bp = b; 1095 1096 return b->data; 1097 } 1098 1099 void *dm_bufio_get(struct dm_bufio_client *c, sector_t block, 1100 struct dm_buffer **bp) 1101 { 1102 return new_read(c, block, NF_GET, bp); 1103 } 1104 EXPORT_SYMBOL_GPL(dm_bufio_get); 1105 1106 void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, 1107 struct dm_buffer **bp) 1108 { 1109 BUG_ON(dm_bufio_in_request()); 1110 1111 return new_read(c, block, NF_READ, bp); 1112 } 1113 EXPORT_SYMBOL_GPL(dm_bufio_read); 1114 1115 void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, 1116 struct dm_buffer **bp) 1117 { 1118 BUG_ON(dm_bufio_in_request()); 1119 1120 return new_read(c, block, NF_FRESH, bp); 1121 } 1122 EXPORT_SYMBOL_GPL(dm_bufio_new); 1123 1124 void dm_bufio_prefetch(struct dm_bufio_client *c, 1125 sector_t block, unsigned n_blocks) 1126 { 1127 struct blk_plug plug; 1128 1129 LIST_HEAD(write_list); 1130 1131 BUG_ON(dm_bufio_in_request()); 1132 1133 blk_start_plug(&plug); 1134 dm_bufio_lock(c); 1135 1136 for (; n_blocks--; block++) { 1137 int need_submit; 1138 struct dm_buffer *b; 1139 b = __bufio_new(c, block, NF_PREFETCH, &need_submit, 1140 &write_list); 1141 if (unlikely(!list_empty(&write_list))) { 1142 dm_bufio_unlock(c); 1143 blk_finish_plug(&plug); 1144 __flush_write_list(&write_list); 1145 blk_start_plug(&plug); 1146 dm_bufio_lock(c); 1147 } 1148 if (unlikely(b != NULL)) { 1149 dm_bufio_unlock(c); 1150 1151 if (need_submit) 1152 submit_io(b, REQ_OP_READ, read_endio); 1153 dm_bufio_release(b); 1154 1155 cond_resched(); 1156 1157 if (!n_blocks) 1158 goto flush_plug; 1159 dm_bufio_lock(c); 1160 } 1161 } 1162 1163 dm_bufio_unlock(c); 1164 1165 flush_plug: 1166 blk_finish_plug(&plug); 1167 } 1168 EXPORT_SYMBOL_GPL(dm_bufio_prefetch); 1169 1170 void dm_bufio_release(struct dm_buffer *b) 1171 { 1172 struct dm_bufio_client *c = b->c; 1173 1174 dm_bufio_lock(c); 1175 1176 BUG_ON(!b->hold_count); 1177 1178 b->hold_count--; 1179 if (!b->hold_count) { 1180 wake_up(&c->free_buffer_wait); 1181 1182 /* 1183 * If there were errors on the buffer, and the buffer is not 1184 * to be written, free the buffer. There is no point in caching 1185 * invalid buffer. 1186 */ 1187 if ((b->read_error || b->write_error) && 1188 !test_bit(B_READING, &b->state) && 1189 !test_bit(B_WRITING, &b->state) && 1190 !test_bit(B_DIRTY, &b->state)) { 1191 __unlink_buffer(b); 1192 __free_buffer_wake(b); 1193 } 1194 } 1195 1196 dm_bufio_unlock(c); 1197 } 1198 EXPORT_SYMBOL_GPL(dm_bufio_release); 1199 1200 void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b, 1201 unsigned start, unsigned end) 1202 { 1203 struct dm_bufio_client *c = b->c; 1204 1205 BUG_ON(start >= end); 1206 BUG_ON(end > b->c->block_size); 1207 1208 dm_bufio_lock(c); 1209 1210 BUG_ON(test_bit(B_READING, &b->state)); 1211 1212 if (!test_and_set_bit(B_DIRTY, &b->state)) { 1213 b->dirty_start = start; 1214 b->dirty_end = end; 1215 __relink_lru(b, LIST_DIRTY); 1216 } else { 1217 if (start < b->dirty_start) 1218 b->dirty_start = start; 1219 if (end > b->dirty_end) 1220 b->dirty_end = end; 1221 } 1222 1223 dm_bufio_unlock(c); 1224 } 1225 EXPORT_SYMBOL_GPL(dm_bufio_mark_partial_buffer_dirty); 1226 1227 void dm_bufio_mark_buffer_dirty(struct dm_buffer *b) 1228 { 1229 dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size); 1230 } 1231 EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty); 1232 1233 void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c) 1234 { 1235 LIST_HEAD(write_list); 1236 1237 BUG_ON(dm_bufio_in_request()); 1238 1239 dm_bufio_lock(c); 1240 __write_dirty_buffers_async(c, 0, &write_list); 1241 dm_bufio_unlock(c); 1242 __flush_write_list(&write_list); 1243 } 1244 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers_async); 1245 1246 /* 1247 * For performance, it is essential that the buffers are written asynchronously 1248 * and simultaneously (so that the block layer can merge the writes) and then 1249 * waited upon. 1250 * 1251 * Finally, we flush hardware disk cache. 1252 */ 1253 int dm_bufio_write_dirty_buffers(struct dm_bufio_client *c) 1254 { 1255 int a, f; 1256 unsigned long buffers_processed = 0; 1257 struct dm_buffer *b, *tmp; 1258 1259 LIST_HEAD(write_list); 1260 1261 dm_bufio_lock(c); 1262 __write_dirty_buffers_async(c, 0, &write_list); 1263 dm_bufio_unlock(c); 1264 __flush_write_list(&write_list); 1265 dm_bufio_lock(c); 1266 1267 again: 1268 list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) { 1269 int dropped_lock = 0; 1270 1271 if (buffers_processed < c->n_buffers[LIST_DIRTY]) 1272 buffers_processed++; 1273 1274 BUG_ON(test_bit(B_READING, &b->state)); 1275 1276 if (test_bit(B_WRITING, &b->state)) { 1277 if (buffers_processed < c->n_buffers[LIST_DIRTY]) { 1278 dropped_lock = 1; 1279 b->hold_count++; 1280 dm_bufio_unlock(c); 1281 wait_on_bit_io(&b->state, B_WRITING, 1282 TASK_UNINTERRUPTIBLE); 1283 dm_bufio_lock(c); 1284 b->hold_count--; 1285 } else 1286 wait_on_bit_io(&b->state, B_WRITING, 1287 TASK_UNINTERRUPTIBLE); 1288 } 1289 1290 if (!test_bit(B_DIRTY, &b->state) && 1291 !test_bit(B_WRITING, &b->state)) 1292 __relink_lru(b, LIST_CLEAN); 1293 1294 cond_resched(); 1295 1296 /* 1297 * If we dropped the lock, the list is no longer consistent, 1298 * so we must restart the search. 1299 * 1300 * In the most common case, the buffer just processed is 1301 * relinked to the clean list, so we won't loop scanning the 1302 * same buffer again and again. 1303 * 1304 * This may livelock if there is another thread simultaneously 1305 * dirtying buffers, so we count the number of buffers walked 1306 * and if it exceeds the total number of buffers, it means that 1307 * someone is doing some writes simultaneously with us. In 1308 * this case, stop, dropping the lock. 1309 */ 1310 if (dropped_lock) 1311 goto again; 1312 } 1313 wake_up(&c->free_buffer_wait); 1314 dm_bufio_unlock(c); 1315 1316 a = xchg(&c->async_write_error, 0); 1317 f = dm_bufio_issue_flush(c); 1318 if (a) 1319 return a; 1320 1321 return f; 1322 } 1323 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers); 1324 1325 /* 1326 * Use dm-io to send an empty barrier to flush the device. 1327 */ 1328 int dm_bufio_issue_flush(struct dm_bufio_client *c) 1329 { 1330 struct dm_io_request io_req = { 1331 .bi_op = REQ_OP_WRITE, 1332 .bi_op_flags = REQ_PREFLUSH | REQ_SYNC, 1333 .mem.type = DM_IO_KMEM, 1334 .mem.ptr.addr = NULL, 1335 .client = c->dm_io, 1336 }; 1337 struct dm_io_region io_reg = { 1338 .bdev = c->bdev, 1339 .sector = 0, 1340 .count = 0, 1341 }; 1342 1343 BUG_ON(dm_bufio_in_request()); 1344 1345 return dm_io(&io_req, 1, &io_reg, NULL); 1346 } 1347 EXPORT_SYMBOL_GPL(dm_bufio_issue_flush); 1348 1349 /* 1350 * We first delete any other buffer that may be at that new location. 1351 * 1352 * Then, we write the buffer to the original location if it was dirty. 1353 * 1354 * Then, if we are the only one who is holding the buffer, relink the buffer 1355 * in the buffer tree for the new location. 1356 * 1357 * If there was someone else holding the buffer, we write it to the new 1358 * location but not relink it, because that other user needs to have the buffer 1359 * at the same place. 1360 */ 1361 void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block) 1362 { 1363 struct dm_bufio_client *c = b->c; 1364 struct dm_buffer *new; 1365 1366 BUG_ON(dm_bufio_in_request()); 1367 1368 dm_bufio_lock(c); 1369 1370 retry: 1371 new = __find(c, new_block); 1372 if (new) { 1373 if (new->hold_count) { 1374 __wait_for_free_buffer(c); 1375 goto retry; 1376 } 1377 1378 /* 1379 * FIXME: Is there any point waiting for a write that's going 1380 * to be overwritten in a bit? 1381 */ 1382 __make_buffer_clean(new); 1383 __unlink_buffer(new); 1384 __free_buffer_wake(new); 1385 } 1386 1387 BUG_ON(!b->hold_count); 1388 BUG_ON(test_bit(B_READING, &b->state)); 1389 1390 __write_dirty_buffer(b, NULL); 1391 if (b->hold_count == 1) { 1392 wait_on_bit_io(&b->state, B_WRITING, 1393 TASK_UNINTERRUPTIBLE); 1394 set_bit(B_DIRTY, &b->state); 1395 b->dirty_start = 0; 1396 b->dirty_end = c->block_size; 1397 __unlink_buffer(b); 1398 __link_buffer(b, new_block, LIST_DIRTY); 1399 } else { 1400 sector_t old_block; 1401 wait_on_bit_lock_io(&b->state, B_WRITING, 1402 TASK_UNINTERRUPTIBLE); 1403 /* 1404 * Relink buffer to "new_block" so that write_callback 1405 * sees "new_block" as a block number. 1406 * After the write, link the buffer back to old_block. 1407 * All this must be done in bufio lock, so that block number 1408 * change isn't visible to other threads. 1409 */ 1410 old_block = b->block; 1411 __unlink_buffer(b); 1412 __link_buffer(b, new_block, b->list_mode); 1413 submit_io(b, REQ_OP_WRITE, write_endio); 1414 wait_on_bit_io(&b->state, B_WRITING, 1415 TASK_UNINTERRUPTIBLE); 1416 __unlink_buffer(b); 1417 __link_buffer(b, old_block, b->list_mode); 1418 } 1419 1420 dm_bufio_unlock(c); 1421 dm_bufio_release(b); 1422 } 1423 EXPORT_SYMBOL_GPL(dm_bufio_release_move); 1424 1425 /* 1426 * Free the given buffer. 1427 * 1428 * This is just a hint, if the buffer is in use or dirty, this function 1429 * does nothing. 1430 */ 1431 void dm_bufio_forget(struct dm_bufio_client *c, sector_t block) 1432 { 1433 struct dm_buffer *b; 1434 1435 dm_bufio_lock(c); 1436 1437 b = __find(c, block); 1438 if (b && likely(!b->hold_count) && likely(!b->state)) { 1439 __unlink_buffer(b); 1440 __free_buffer_wake(b); 1441 } 1442 1443 dm_bufio_unlock(c); 1444 } 1445 EXPORT_SYMBOL_GPL(dm_bufio_forget); 1446 1447 void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned n) 1448 { 1449 c->minimum_buffers = n; 1450 } 1451 EXPORT_SYMBOL_GPL(dm_bufio_set_minimum_buffers); 1452 1453 unsigned dm_bufio_get_block_size(struct dm_bufio_client *c) 1454 { 1455 return c->block_size; 1456 } 1457 EXPORT_SYMBOL_GPL(dm_bufio_get_block_size); 1458 1459 sector_t dm_bufio_get_device_size(struct dm_bufio_client *c) 1460 { 1461 sector_t s = i_size_read(c->bdev->bd_inode) >> SECTOR_SHIFT; 1462 if (likely(c->sectors_per_block_bits >= 0)) 1463 s >>= c->sectors_per_block_bits; 1464 else 1465 sector_div(s, c->block_size >> SECTOR_SHIFT); 1466 return s; 1467 } 1468 EXPORT_SYMBOL_GPL(dm_bufio_get_device_size); 1469 1470 sector_t dm_bufio_get_block_number(struct dm_buffer *b) 1471 { 1472 return b->block; 1473 } 1474 EXPORT_SYMBOL_GPL(dm_bufio_get_block_number); 1475 1476 void *dm_bufio_get_block_data(struct dm_buffer *b) 1477 { 1478 return b->data; 1479 } 1480 EXPORT_SYMBOL_GPL(dm_bufio_get_block_data); 1481 1482 void *dm_bufio_get_aux_data(struct dm_buffer *b) 1483 { 1484 return b + 1; 1485 } 1486 EXPORT_SYMBOL_GPL(dm_bufio_get_aux_data); 1487 1488 struct dm_bufio_client *dm_bufio_get_client(struct dm_buffer *b) 1489 { 1490 return b->c; 1491 } 1492 EXPORT_SYMBOL_GPL(dm_bufio_get_client); 1493 1494 static void drop_buffers(struct dm_bufio_client *c) 1495 { 1496 struct dm_buffer *b; 1497 int i; 1498 bool warned = false; 1499 1500 BUG_ON(dm_bufio_in_request()); 1501 1502 /* 1503 * An optimization so that the buffers are not written one-by-one. 1504 */ 1505 dm_bufio_write_dirty_buffers_async(c); 1506 1507 dm_bufio_lock(c); 1508 1509 while ((b = __get_unclaimed_buffer(c))) 1510 __free_buffer_wake(b); 1511 1512 for (i = 0; i < LIST_SIZE; i++) 1513 list_for_each_entry(b, &c->lru[i], lru_list) { 1514 WARN_ON(!warned); 1515 warned = true; 1516 DMERR("leaked buffer %llx, hold count %u, list %d", 1517 (unsigned long long)b->block, b->hold_count, i); 1518 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 1519 stack_trace_print(b->stack_entries, b->stack_len, 1); 1520 /* mark unclaimed to avoid BUG_ON below */ 1521 b->hold_count = 0; 1522 #endif 1523 } 1524 1525 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING 1526 while ((b = __get_unclaimed_buffer(c))) 1527 __free_buffer_wake(b); 1528 #endif 1529 1530 for (i = 0; i < LIST_SIZE; i++) 1531 BUG_ON(!list_empty(&c->lru[i])); 1532 1533 dm_bufio_unlock(c); 1534 } 1535 1536 /* 1537 * We may not be able to evict this buffer if IO pending or the client 1538 * is still using it. Caller is expected to know buffer is too old. 1539 * 1540 * And if GFP_NOFS is used, we must not do any I/O because we hold 1541 * dm_bufio_clients_lock and we would risk deadlock if the I/O gets 1542 * rerouted to different bufio client. 1543 */ 1544 static bool __try_evict_buffer(struct dm_buffer *b, gfp_t gfp) 1545 { 1546 if (!(gfp & __GFP_FS)) { 1547 if (test_bit(B_READING, &b->state) || 1548 test_bit(B_WRITING, &b->state) || 1549 test_bit(B_DIRTY, &b->state)) 1550 return false; 1551 } 1552 1553 if (b->hold_count) 1554 return false; 1555 1556 __make_buffer_clean(b); 1557 __unlink_buffer(b); 1558 __free_buffer_wake(b); 1559 1560 return true; 1561 } 1562 1563 static unsigned long get_retain_buffers(struct dm_bufio_client *c) 1564 { 1565 unsigned long retain_bytes = READ_ONCE(dm_bufio_retain_bytes); 1566 if (likely(c->sectors_per_block_bits >= 0)) 1567 retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT; 1568 else 1569 retain_bytes /= c->block_size; 1570 return retain_bytes; 1571 } 1572 1573 static unsigned long __scan(struct dm_bufio_client *c, unsigned long nr_to_scan, 1574 gfp_t gfp_mask) 1575 { 1576 int l; 1577 struct dm_buffer *b, *tmp; 1578 unsigned long freed = 0; 1579 unsigned long count = c->n_buffers[LIST_CLEAN] + 1580 c->n_buffers[LIST_DIRTY]; 1581 unsigned long retain_target = get_retain_buffers(c); 1582 1583 for (l = 0; l < LIST_SIZE; l++) { 1584 list_for_each_entry_safe_reverse(b, tmp, &c->lru[l], lru_list) { 1585 if (__try_evict_buffer(b, gfp_mask)) 1586 freed++; 1587 if (!--nr_to_scan || ((count - freed) <= retain_target)) 1588 return freed; 1589 cond_resched(); 1590 } 1591 } 1592 return freed; 1593 } 1594 1595 static unsigned long 1596 dm_bufio_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 1597 { 1598 struct dm_bufio_client *c; 1599 unsigned long freed; 1600 1601 c = container_of(shrink, struct dm_bufio_client, shrinker); 1602 if (!dm_bufio_trylock(c)) 1603 return SHRINK_STOP; 1604 1605 freed = __scan(c, sc->nr_to_scan, sc->gfp_mask); 1606 dm_bufio_unlock(c); 1607 return freed; 1608 } 1609 1610 static unsigned long 1611 dm_bufio_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 1612 { 1613 struct dm_bufio_client *c = container_of(shrink, struct dm_bufio_client, shrinker); 1614 unsigned long count = READ_ONCE(c->n_buffers[LIST_CLEAN]) + 1615 READ_ONCE(c->n_buffers[LIST_DIRTY]); 1616 unsigned long retain_target = get_retain_buffers(c); 1617 1618 return (count < retain_target) ? 0 : (count - retain_target); 1619 } 1620 1621 /* 1622 * Create the buffering interface 1623 */ 1624 struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsigned block_size, 1625 unsigned reserved_buffers, unsigned aux_size, 1626 void (*alloc_callback)(struct dm_buffer *), 1627 void (*write_callback)(struct dm_buffer *)) 1628 { 1629 int r; 1630 struct dm_bufio_client *c; 1631 unsigned i; 1632 char slab_name[27]; 1633 1634 if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) { 1635 DMERR("%s: block size not specified or is not multiple of 512b", __func__); 1636 r = -EINVAL; 1637 goto bad_client; 1638 } 1639 1640 c = kzalloc(sizeof(*c), GFP_KERNEL); 1641 if (!c) { 1642 r = -ENOMEM; 1643 goto bad_client; 1644 } 1645 c->buffer_tree = RB_ROOT; 1646 1647 c->bdev = bdev; 1648 c->block_size = block_size; 1649 if (is_power_of_2(block_size)) 1650 c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT; 1651 else 1652 c->sectors_per_block_bits = -1; 1653 1654 c->alloc_callback = alloc_callback; 1655 c->write_callback = write_callback; 1656 1657 for (i = 0; i < LIST_SIZE; i++) { 1658 INIT_LIST_HEAD(&c->lru[i]); 1659 c->n_buffers[i] = 0; 1660 } 1661 1662 mutex_init(&c->lock); 1663 INIT_LIST_HEAD(&c->reserved_buffers); 1664 c->need_reserved_buffers = reserved_buffers; 1665 1666 dm_bufio_set_minimum_buffers(c, DM_BUFIO_MIN_BUFFERS); 1667 1668 init_waitqueue_head(&c->free_buffer_wait); 1669 c->async_write_error = 0; 1670 1671 c->dm_io = dm_io_client_create(); 1672 if (IS_ERR(c->dm_io)) { 1673 r = PTR_ERR(c->dm_io); 1674 goto bad_dm_io; 1675 } 1676 1677 if (block_size <= KMALLOC_MAX_SIZE && 1678 (block_size < PAGE_SIZE || !is_power_of_2(block_size))) { 1679 unsigned align = min(1U << __ffs(block_size), (unsigned)PAGE_SIZE); 1680 snprintf(slab_name, sizeof slab_name, "dm_bufio_cache-%u", block_size); 1681 c->slab_cache = kmem_cache_create(slab_name, block_size, align, 1682 SLAB_RECLAIM_ACCOUNT, NULL); 1683 if (!c->slab_cache) { 1684 r = -ENOMEM; 1685 goto bad; 1686 } 1687 } 1688 if (aux_size) 1689 snprintf(slab_name, sizeof slab_name, "dm_bufio_buffer-%u", aux_size); 1690 else 1691 snprintf(slab_name, sizeof slab_name, "dm_bufio_buffer"); 1692 c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size, 1693 0, SLAB_RECLAIM_ACCOUNT, NULL); 1694 if (!c->slab_buffer) { 1695 r = -ENOMEM; 1696 goto bad; 1697 } 1698 1699 while (c->need_reserved_buffers) { 1700 struct dm_buffer *b = alloc_buffer(c, GFP_KERNEL); 1701 1702 if (!b) { 1703 r = -ENOMEM; 1704 goto bad; 1705 } 1706 __free_buffer_wake(b); 1707 } 1708 1709 c->shrinker.count_objects = dm_bufio_shrink_count; 1710 c->shrinker.scan_objects = dm_bufio_shrink_scan; 1711 c->shrinker.seeks = 1; 1712 c->shrinker.batch = 0; 1713 r = register_shrinker(&c->shrinker); 1714 if (r) 1715 goto bad; 1716 1717 mutex_lock(&dm_bufio_clients_lock); 1718 dm_bufio_client_count++; 1719 list_add(&c->client_list, &dm_bufio_all_clients); 1720 __cache_size_refresh(); 1721 mutex_unlock(&dm_bufio_clients_lock); 1722 1723 return c; 1724 1725 bad: 1726 while (!list_empty(&c->reserved_buffers)) { 1727 struct dm_buffer *b = list_entry(c->reserved_buffers.next, 1728 struct dm_buffer, lru_list); 1729 list_del(&b->lru_list); 1730 free_buffer(b); 1731 } 1732 kmem_cache_destroy(c->slab_cache); 1733 kmem_cache_destroy(c->slab_buffer); 1734 dm_io_client_destroy(c->dm_io); 1735 bad_dm_io: 1736 mutex_destroy(&c->lock); 1737 kfree(c); 1738 bad_client: 1739 return ERR_PTR(r); 1740 } 1741 EXPORT_SYMBOL_GPL(dm_bufio_client_create); 1742 1743 /* 1744 * Free the buffering interface. 1745 * It is required that there are no references on any buffers. 1746 */ 1747 void dm_bufio_client_destroy(struct dm_bufio_client *c) 1748 { 1749 unsigned i; 1750 1751 drop_buffers(c); 1752 1753 unregister_shrinker(&c->shrinker); 1754 1755 mutex_lock(&dm_bufio_clients_lock); 1756 1757 list_del(&c->client_list); 1758 dm_bufio_client_count--; 1759 __cache_size_refresh(); 1760 1761 mutex_unlock(&dm_bufio_clients_lock); 1762 1763 BUG_ON(!RB_EMPTY_ROOT(&c->buffer_tree)); 1764 BUG_ON(c->need_reserved_buffers); 1765 1766 while (!list_empty(&c->reserved_buffers)) { 1767 struct dm_buffer *b = list_entry(c->reserved_buffers.next, 1768 struct dm_buffer, lru_list); 1769 list_del(&b->lru_list); 1770 free_buffer(b); 1771 } 1772 1773 for (i = 0; i < LIST_SIZE; i++) 1774 if (c->n_buffers[i]) 1775 DMERR("leaked buffer count %d: %ld", i, c->n_buffers[i]); 1776 1777 for (i = 0; i < LIST_SIZE; i++) 1778 BUG_ON(c->n_buffers[i]); 1779 1780 kmem_cache_destroy(c->slab_cache); 1781 kmem_cache_destroy(c->slab_buffer); 1782 dm_io_client_destroy(c->dm_io); 1783 mutex_destroy(&c->lock); 1784 kfree(c); 1785 } 1786 EXPORT_SYMBOL_GPL(dm_bufio_client_destroy); 1787 1788 void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start) 1789 { 1790 c->start = start; 1791 } 1792 EXPORT_SYMBOL_GPL(dm_bufio_set_sector_offset); 1793 1794 static unsigned get_max_age_hz(void) 1795 { 1796 unsigned max_age = READ_ONCE(dm_bufio_max_age); 1797 1798 if (max_age > UINT_MAX / HZ) 1799 max_age = UINT_MAX / HZ; 1800 1801 return max_age * HZ; 1802 } 1803 1804 static bool older_than(struct dm_buffer *b, unsigned long age_hz) 1805 { 1806 return time_after_eq(jiffies, b->last_accessed + age_hz); 1807 } 1808 1809 static void __evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz) 1810 { 1811 struct dm_buffer *b, *tmp; 1812 unsigned long retain_target = get_retain_buffers(c); 1813 unsigned long count; 1814 LIST_HEAD(write_list); 1815 1816 dm_bufio_lock(c); 1817 1818 __check_watermark(c, &write_list); 1819 if (unlikely(!list_empty(&write_list))) { 1820 dm_bufio_unlock(c); 1821 __flush_write_list(&write_list); 1822 dm_bufio_lock(c); 1823 } 1824 1825 count = c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY]; 1826 list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_CLEAN], lru_list) { 1827 if (count <= retain_target) 1828 break; 1829 1830 if (!older_than(b, age_hz)) 1831 break; 1832 1833 if (__try_evict_buffer(b, 0)) 1834 count--; 1835 1836 cond_resched(); 1837 } 1838 1839 dm_bufio_unlock(c); 1840 } 1841 1842 static void cleanup_old_buffers(void) 1843 { 1844 unsigned long max_age_hz = get_max_age_hz(); 1845 struct dm_bufio_client *c; 1846 1847 mutex_lock(&dm_bufio_clients_lock); 1848 1849 __cache_size_refresh(); 1850 1851 list_for_each_entry(c, &dm_bufio_all_clients, client_list) 1852 __evict_old_buffers(c, max_age_hz); 1853 1854 mutex_unlock(&dm_bufio_clients_lock); 1855 } 1856 1857 static struct workqueue_struct *dm_bufio_wq; 1858 static struct delayed_work dm_bufio_work; 1859 1860 static void work_fn(struct work_struct *w) 1861 { 1862 cleanup_old_buffers(); 1863 1864 queue_delayed_work(dm_bufio_wq, &dm_bufio_work, 1865 DM_BUFIO_WORK_TIMER_SECS * HZ); 1866 } 1867 1868 /*---------------------------------------------------------------- 1869 * Module setup 1870 *--------------------------------------------------------------*/ 1871 1872 /* 1873 * This is called only once for the whole dm_bufio module. 1874 * It initializes memory limit. 1875 */ 1876 static int __init dm_bufio_init(void) 1877 { 1878 __u64 mem; 1879 1880 dm_bufio_allocated_kmem_cache = 0; 1881 dm_bufio_allocated_get_free_pages = 0; 1882 dm_bufio_allocated_vmalloc = 0; 1883 dm_bufio_current_allocated = 0; 1884 1885 mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(), 1886 DM_BUFIO_MEMORY_PERCENT, 100) << PAGE_SHIFT; 1887 1888 if (mem > ULONG_MAX) 1889 mem = ULONG_MAX; 1890 1891 #ifdef CONFIG_MMU 1892 if (mem > mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100)) 1893 mem = mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100); 1894 #endif 1895 1896 dm_bufio_default_cache_size = mem; 1897 1898 mutex_lock(&dm_bufio_clients_lock); 1899 __cache_size_refresh(); 1900 mutex_unlock(&dm_bufio_clients_lock); 1901 1902 dm_bufio_wq = alloc_workqueue("dm_bufio_cache", WQ_MEM_RECLAIM, 0); 1903 if (!dm_bufio_wq) 1904 return -ENOMEM; 1905 1906 INIT_DELAYED_WORK(&dm_bufio_work, work_fn); 1907 queue_delayed_work(dm_bufio_wq, &dm_bufio_work, 1908 DM_BUFIO_WORK_TIMER_SECS * HZ); 1909 1910 return 0; 1911 } 1912 1913 /* 1914 * This is called once when unloading the dm_bufio module. 1915 */ 1916 static void __exit dm_bufio_exit(void) 1917 { 1918 int bug = 0; 1919 1920 cancel_delayed_work_sync(&dm_bufio_work); 1921 destroy_workqueue(dm_bufio_wq); 1922 1923 if (dm_bufio_client_count) { 1924 DMCRIT("%s: dm_bufio_client_count leaked: %d", 1925 __func__, dm_bufio_client_count); 1926 bug = 1; 1927 } 1928 1929 if (dm_bufio_current_allocated) { 1930 DMCRIT("%s: dm_bufio_current_allocated leaked: %lu", 1931 __func__, dm_bufio_current_allocated); 1932 bug = 1; 1933 } 1934 1935 if (dm_bufio_allocated_get_free_pages) { 1936 DMCRIT("%s: dm_bufio_allocated_get_free_pages leaked: %lu", 1937 __func__, dm_bufio_allocated_get_free_pages); 1938 bug = 1; 1939 } 1940 1941 if (dm_bufio_allocated_vmalloc) { 1942 DMCRIT("%s: dm_bufio_vmalloc leaked: %lu", 1943 __func__, dm_bufio_allocated_vmalloc); 1944 bug = 1; 1945 } 1946 1947 BUG_ON(bug); 1948 } 1949 1950 module_init(dm_bufio_init) 1951 module_exit(dm_bufio_exit) 1952 1953 module_param_named(max_cache_size_bytes, dm_bufio_cache_size, ulong, S_IRUGO | S_IWUSR); 1954 MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache"); 1955 1956 module_param_named(max_age_seconds, dm_bufio_max_age, uint, S_IRUGO | S_IWUSR); 1957 MODULE_PARM_DESC(max_age_seconds, "Max age of a buffer in seconds"); 1958 1959 module_param_named(retain_bytes, dm_bufio_retain_bytes, ulong, S_IRUGO | S_IWUSR); 1960 MODULE_PARM_DESC(retain_bytes, "Try to keep at least this many bytes cached in memory"); 1961 1962 module_param_named(peak_allocated_bytes, dm_bufio_peak_allocated, ulong, S_IRUGO | S_IWUSR); 1963 MODULE_PARM_DESC(peak_allocated_bytes, "Tracks the maximum allocated memory"); 1964 1965 module_param_named(allocated_kmem_cache_bytes, dm_bufio_allocated_kmem_cache, ulong, S_IRUGO); 1966 MODULE_PARM_DESC(allocated_kmem_cache_bytes, "Memory allocated with kmem_cache_alloc"); 1967 1968 module_param_named(allocated_get_free_pages_bytes, dm_bufio_allocated_get_free_pages, ulong, S_IRUGO); 1969 MODULE_PARM_DESC(allocated_get_free_pages_bytes, "Memory allocated with get_free_pages"); 1970 1971 module_param_named(allocated_vmalloc_bytes, dm_bufio_allocated_vmalloc, ulong, S_IRUGO); 1972 MODULE_PARM_DESC(allocated_vmalloc_bytes, "Memory allocated with vmalloc"); 1973 1974 module_param_named(current_allocated_bytes, dm_bufio_current_allocated, ulong, S_IRUGO); 1975 MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache"); 1976 1977 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>"); 1978 MODULE_DESCRIPTION(DM_NAME " buffered I/O library"); 1979 MODULE_LICENSE("GPL"); 1980