Lines Matching +full:cache +full:- +full:op +full:- +full:block +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2009-2011 Red Hat, Inc.
10 #include <linux/dm-bufio.h>
12 #include <linux/device-mapper.h>
13 #include <linux/dm-io.h>
65 * dm_buffer->list_mode
71 /*--------------------------------------------------------------*/
99 /*--------------*/
103 lru->cursor = NULL; in lru_init()
104 lru->count = 0; in lru_init()
105 INIT_LIST_HEAD(&lru->iterators); in lru_init()
110 WARN_ON_ONCE(lru->cursor); in lru_destroy()
111 WARN_ON_ONCE(!list_empty(&lru->iterators)); in lru_destroy()
123 atomic_set(&le->referenced, 0); in lru_insert()
125 if (lru->cursor) { in lru_insert()
126 list_add_tail(&le->list, lru->cursor); in lru_insert()
128 INIT_LIST_HEAD(&le->list); in lru_insert()
129 lru->cursor = &le->list; in lru_insert()
131 lru->count++; in lru_insert()
134 /*--------------*/
149 it->lru = lru; in lru_iter_begin()
150 it->stop = lru->cursor ? to_le(lru->cursor->prev) : NULL; in lru_iter_begin()
151 it->e = lru->cursor ? to_le(lru->cursor) : NULL; in lru_iter_begin()
152 list_add(&it->list, &lru->iterators); in lru_iter_begin()
160 list_del(&it->list); in lru_iter_end()
176 while (it->e) { in lru_iter_next()
177 e = it->e; in lru_iter_next()
180 if (it->e == it->stop) in lru_iter_next()
181 it->e = NULL; in lru_iter_next()
183 it->e = to_le(it->e->list.next); in lru_iter_next()
200 list_for_each_entry(it, &lru->iterators, list) { in lru_iter_invalidate()
201 /* Move c->e forwards if necc. */ in lru_iter_invalidate()
202 if (it->e == e) { in lru_iter_invalidate()
203 it->e = to_le(it->e->list.next); in lru_iter_invalidate()
204 if (it->e == e) in lru_iter_invalidate()
205 it->e = NULL; in lru_iter_invalidate()
208 /* Move it->stop backwards if necc. */ in lru_iter_invalidate()
209 if (it->stop == e) { in lru_iter_invalidate()
210 it->stop = to_le(it->stop->list.prev); in lru_iter_invalidate()
211 if (it->stop == e) in lru_iter_invalidate()
212 it->stop = NULL; in lru_iter_invalidate()
217 /*--------------*/
225 if (lru->count == 1) { in lru_remove()
226 lru->cursor = NULL; in lru_remove()
228 if (lru->cursor == &le->list) in lru_remove()
229 lru->cursor = lru->cursor->next; in lru_remove()
230 list_del(&le->list); in lru_remove()
232 lru->count--; in lru_remove()
240 atomic_set(&le->referenced, 1); in lru_reference()
243 /*--------------*/
260 struct list_head *h = lru->cursor;
270 while (tested < lru->count) {
273 if (atomic_read(&le->referenced)) {
274 atomic_set(&le->referenced, 0);
283 lru->cursor = le->list.next;
291 lru->cursor = le->list.next;
296 h = h->next;
305 /*--------------------------------------------------------------*/
315 * Describes how the block was allocated:
331 sector_t block; member
365 /*--------------------------------------------------------------*/
368 * The buffer cache manages buffers, particularly:
369 * - inc/dec of holder count
370 * - setting the last_accessed field
371 * - maintains clean/dirty state along with lru
372 * - selecting buffers that match predicates
375 * - allocation/freeing of buffers.
376 * - IO
377 * - Eviction or cache sizing.
406 static inline unsigned int cache_index(sector_t block, unsigned int num_locks) in cache_index() argument
408 return dm_hash_locks_index(block, num_locks); in cache_index()
411 static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block) in cache_read_lock() argument
413 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_read_lock()
414 read_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_read_lock()
416 down_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_read_lock()
419 static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block) in cache_read_unlock() argument
421 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_read_unlock()
422 read_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_read_unlock()
424 up_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_read_unlock()
427 static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block) in cache_write_lock() argument
429 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_write_lock()
430 write_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_write_lock()
432 down_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_write_lock()
435 static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block) in cache_write_unlock() argument
437 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_write_unlock()
438 write_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_write_unlock()
440 up_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_write_unlock()
448 struct dm_buffer_cache *cache; member
454 static void lh_init(struct lock_history *lh, struct dm_buffer_cache *cache, bool write) in lh_init() argument
456 lh->cache = cache; in lh_init()
457 lh->write = write; in lh_init()
458 lh->no_previous = cache->num_locks; in lh_init()
459 lh->previous = lh->no_previous; in lh_init()
464 if (lh->write) { in __lh_lock()
465 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_lock()
466 write_lock_bh(&lh->cache->trees[index].u.spinlock); in __lh_lock()
468 down_write(&lh->cache->trees[index].u.lock); in __lh_lock()
470 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_lock()
471 read_lock_bh(&lh->cache->trees[index].u.spinlock); in __lh_lock()
473 down_read(&lh->cache->trees[index].u.lock); in __lh_lock()
479 if (lh->write) { in __lh_unlock()
480 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_unlock()
481 write_unlock_bh(&lh->cache->trees[index].u.spinlock); in __lh_unlock()
483 up_write(&lh->cache->trees[index].u.lock); in __lh_unlock()
485 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_unlock()
486 read_unlock_bh(&lh->cache->trees[index].u.spinlock); in __lh_unlock()
488 up_read(&lh->cache->trees[index].u.lock); in __lh_unlock()
497 if (lh->previous != lh->no_previous) { in lh_exit()
498 __lh_unlock(lh, lh->previous); in lh_exit()
499 lh->previous = lh->no_previous; in lh_exit()
509 unsigned int index = cache_index(b, lh->no_previous); /* no_previous is num_locks */ in lh_next()
511 if (lh->previous != lh->no_previous) { in lh_next()
512 if (lh->previous != index) { in lh_next()
513 __lh_unlock(lh, lh->previous); in lh_next()
515 lh->previous = index; in lh_next()
519 lh->previous = index; in lh_next()
542 bc->num_locks = num_locks; in cache_init()
543 bc->no_sleep = no_sleep; in cache_init()
545 for (i = 0; i < bc->num_locks; i++) { in cache_init()
547 rwlock_init(&bc->trees[i].u.spinlock); in cache_init()
549 init_rwsem(&bc->trees[i].u.lock); in cache_init()
550 bc->trees[i].root = RB_ROOT; in cache_init()
553 lru_init(&bc->lru[LIST_CLEAN]); in cache_init()
554 lru_init(&bc->lru[LIST_DIRTY]); in cache_init()
561 for (i = 0; i < bc->num_locks; i++) in cache_destroy()
562 WARN_ON_ONCE(!RB_EMPTY_ROOT(&bc->trees[i].root)); in cache_destroy()
564 lru_destroy(&bc->lru[LIST_CLEAN]); in cache_destroy()
565 lru_destroy(&bc->lru[LIST_DIRTY]); in cache_destroy()
568 /*--------------*/
575 return bc->lru[list_mode].count; in cache_count()
583 /*--------------*/
586 * Gets a specific buffer, indexed by block.
592 static struct dm_buffer *__cache_get(const struct rb_root *root, sector_t block) in __cache_get() argument
594 struct rb_node *n = root->rb_node; in __cache_get()
600 if (b->block == block) in __cache_get()
603 n = block < b->block ? n->rb_left : n->rb_right; in __cache_get()
611 atomic_inc(&b->hold_count); in __cache_inc_buffer()
612 WRITE_ONCE(b->last_accessed, jiffies); in __cache_inc_buffer()
615 static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t block) in cache_get() argument
619 cache_read_lock(bc, block); in cache_get()
620 b = __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, block); in cache_get()
622 lru_reference(&b->lru); in cache_get()
625 cache_read_unlock(bc, block); in cache_get()
630 /*--------------*/
640 cache_read_lock(bc, b->block); in cache_put()
641 BUG_ON(!atomic_read(&b->hold_count)); in cache_put()
642 r = atomic_dec_and_test(&b->hold_count); in cache_put()
643 cache_read_unlock(bc, b->block); in cache_put()
648 /*--------------*/
672 lh_next(w->lh, b->block); in __evict_pred()
674 if (atomic_read(&b->hold_count)) in __evict_pred()
677 return w->pred(b, w->context); in __evict_pred()
688 le = lru_evict(&bc->lru[list_mode], __evict_pred, &w, bc->no_sleep); in __cache_evict()
694 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); in __cache_evict()
712 /*--------------*/
719 cache_write_lock(bc, b->block); in cache_mark()
720 if (list_mode != b->list_mode) { in cache_mark()
721 lru_remove(&bc->lru[b->list_mode], &b->lru); in cache_mark()
722 b->list_mode = list_mode; in cache_mark()
723 lru_insert(&bc->lru[b->list_mode], &b->lru); in cache_mark()
725 cache_write_unlock(bc, b->block); in cache_mark()
728 /*--------------*/
742 le = lru_evict(&bc->lru[old_mode], __evict_pred, &w, bc->no_sleep); in __cache_mark_many()
747 b->list_mode = new_mode; in __cache_mark_many()
748 lru_insert(&bc->lru[b->list_mode], &b->lru); in __cache_mark_many()
762 /*--------------*/
783 struct lru *lru = &bc->lru[list_mode];
786 if (!lru->cursor)
789 first = le = to_le(lru->cursor);
793 lh_next(lh, b->block);
804 le = to_le(le->list.next);
818 /*--------------*/
821 * Passes ownership of the buffer to the cache. Returns false if the
831 struct rb_node **new = &root->rb_node, *parent = NULL; in __cache_insert()
837 if (found->block == b->block) in __cache_insert()
841 new = b->block < found->block ? in __cache_insert()
842 &found->node.rb_left : &found->node.rb_right; in __cache_insert()
845 rb_link_node(&b->node, parent, new); in __cache_insert()
846 rb_insert_color(&b->node, root); in __cache_insert()
855 if (WARN_ON_ONCE(b->list_mode >= LIST_SIZE)) in cache_insert()
858 cache_write_lock(bc, b->block); in cache_insert()
859 BUG_ON(atomic_read(&b->hold_count) != 1); in cache_insert()
860 r = __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].root, b); in cache_insert()
862 lru_insert(&bc->lru[b->list_mode], &b->lru); in cache_insert()
863 cache_write_unlock(bc, b->block); in cache_insert()
868 /*--------------*/
871 * Removes buffer from cache, ownership of the buffer passes back to the caller.
880 cache_write_lock(bc, b->block); in cache_remove()
882 if (atomic_read(&b->hold_count) != 1) { in cache_remove()
886 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); in cache_remove()
887 lru_remove(&bc->lru[b->list_mode], &b->lru); in cache_remove()
890 cache_write_unlock(bc, b->block); in cache_remove()
895 /*--------------*/
899 static struct dm_buffer *__find_next(struct rb_root *root, sector_t block) in __find_next() argument
901 struct rb_node *n = root->rb_node; in __find_next()
908 if (b->block == block) in __find_next()
911 if (block <= b->block) { in __find_next()
912 n = n->rb_left; in __find_next()
915 n = n->rb_right; in __find_next()
933 if (!b || (b->block >= end)) in __remove_range()
936 begin = b->block + 1; in __remove_range()
938 if (atomic_read(&b->hold_count)) in __remove_range()
942 rb_erase(&b->node, root); in __remove_range()
943 lru_remove(&bc->lru[b->list_mode], &b->lru); in __remove_range()
955 BUG_ON(bc->no_sleep); in cache_remove_range()
956 for (i = 0; i < bc->num_locks; i++) { in cache_remove_range()
957 down_write(&bc->trees[i].u.lock); in cache_remove_range()
958 __remove_range(bc, &bc->trees[i].root, begin, end, pred, release); in cache_remove_range()
959 up_write(&bc->trees[i].u.lock); in cache_remove_range()
963 /*----------------------------------------------------------------*/
976 * context), so some clean-not-writing buffers can be held on
1017 struct dm_buffer_cache cache; /* must be last member */ member
1020 /*----------------------------------------------------------------*/
1022 #define dm_bufio_in_request() (!!current->bio_list)
1026 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) in dm_bufio_lock()
1027 spin_lock_bh(&c->spinlock); in dm_bufio_lock()
1029 mutex_lock_nested(&c->lock, dm_bufio_in_request()); in dm_bufio_lock()
1034 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) in dm_bufio_unlock()
1035 spin_unlock_bh(&c->spinlock); in dm_bufio_unlock()
1037 mutex_unlock(&c->lock); in dm_bufio_unlock()
1040 /*----------------------------------------------------------------*/
1043 * Default cache size: available memory divided by the ratio.
1048 * Total cache size set by the user.
1054 * at any time. If it disagrees, the user has changed cache size.
1072 /*----------------------------------------------------------------*/
1097 b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2); in buffer_record_stack()
1101 /*----------------------------------------------------------------*/
1114 data_mode = b->data_mode; in adjust_total_allocated()
1115 diff = (long)b->c->block_size; in adjust_total_allocated()
1117 diff = -diff; in adjust_total_allocated()
1137 * Change the number of clients and recalculate per-client limit.
1149 * Use default if set to 0 and report the actual cache size used. in __cache_size_refresh()
1182 if (unlikely(c->slab_cache != NULL)) { in alloc_buffer_data()
1184 return kmem_cache_alloc(c->slab_cache, gfp_mask); in alloc_buffer_data()
1187 if (c->block_size <= KMALLOC_MAX_SIZE && in alloc_buffer_data()
1191 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); in alloc_buffer_data()
1196 return __vmalloc(c->block_size, gfp_mask); in alloc_buffer_data()
1207 kmem_cache_free(c->slab_cache, data); in free_buffer_data()
1212 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); in free_buffer_data()
1231 struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask); in alloc_buffer()
1236 b->c = c; in alloc_buffer()
1238 b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode); in alloc_buffer()
1239 if (!b->data) { in alloc_buffer()
1240 kmem_cache_free(c->slab_buffer, b); in alloc_buffer()
1246 b->stack_len = 0; in alloc_buffer()
1256 struct dm_bufio_client *c = b->c; in free_buffer()
1259 free_buffer_data(c, b->data, b->data_mode); in free_buffer()
1260 kmem_cache_free(c->slab_buffer, b); in free_buffer()
1264 *--------------------------------------------------------------------------
1269 * memory-consumption per buffer, so it is not viable);
1271 * the memory must be direct-mapped, not vmalloced;
1277 * rejects the bio because it is too large, use dm-io layer to do the I/O.
1278 * The dm-io layer splits the I/O into multiple requests, avoiding the above
1280 *--------------------------------------------------------------------------
1284 * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
1291 b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0); in dmio_complete()
1294 static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, in use_dmio() argument
1299 .bi_opf = op, in use_dmio()
1302 .client = b->c->dm_io, in use_dmio()
1305 .bdev = b->c->bdev, in use_dmio()
1310 if (b->data_mode != DATA_MODE_VMALLOC) { in use_dmio()
1312 io_req.mem.ptr.addr = (char *)b->data + offset; in use_dmio()
1315 io_req.mem.ptr.vma = (char *)b->data + offset; in use_dmio()
1320 b->end_io(b, errno_to_blk_status(r)); in use_dmio()
1325 struct dm_buffer *b = bio->bi_private; in bio_complete()
1326 blk_status_t status = bio->bi_status; in bio_complete()
1330 b->end_io(b, status); in bio_complete()
1333 static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, in use_bio() argument
1342 use_dmio(b, op, sector, n_sectors, offset); in use_bio()
1345 bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op); in use_bio()
1346 bio->bi_iter.bi_sector = sector; in use_bio()
1347 bio->bi_end_io = bio_complete; in use_bio()
1348 bio->bi_private = b; in use_bio()
1350 ptr = (char *)b->data + offset; in use_bio()
1358 static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block) in block_to_sector() argument
1362 if (likely(c->sectors_per_block_bits >= 0)) in block_to_sector()
1363 sector = block << c->sectors_per_block_bits; in block_to_sector()
1365 sector = block * (c->block_size >> SECTOR_SHIFT); in block_to_sector()
1366 sector += c->start; in block_to_sector()
1371 static void submit_io(struct dm_buffer *b, enum req_op op, in submit_io() argument
1378 b->end_io = end_io; in submit_io()
1380 sector = block_to_sector(b->c, b->block); in submit_io()
1382 if (op != REQ_OP_WRITE) { in submit_io()
1383 n_sectors = b->c->block_size >> SECTOR_SHIFT; in submit_io()
1386 if (b->c->write_callback) in submit_io()
1387 b->c->write_callback(b); in submit_io()
1388 offset = b->write_start; in submit_io()
1389 end = b->write_end; in submit_io()
1390 offset &= -DM_BUFIO_WRITE_ALIGN; in submit_io()
1391 end += DM_BUFIO_WRITE_ALIGN - 1; in submit_io()
1392 end &= -DM_BUFIO_WRITE_ALIGN; in submit_io()
1393 if (unlikely(end > b->c->block_size)) in submit_io()
1394 end = b->c->block_size; in submit_io()
1397 n_sectors = (end - offset) >> SECTOR_SHIFT; in submit_io()
1400 if (b->data_mode != DATA_MODE_VMALLOC) in submit_io()
1401 use_bio(b, op, sector, n_sectors, offset); in submit_io()
1403 use_dmio(b, op, sector, n_sectors, offset); in submit_io()
1407 *--------------------------------------------------------------
1409 *--------------------------------------------------------------
1420 b->write_error = status; in write_endio()
1422 struct dm_bufio_client *c = b->c; in write_endio()
1424 (void)cmpxchg(&c->async_write_error, 0, in write_endio()
1428 BUG_ON(!test_bit(B_WRITING, &b->state)); in write_endio()
1431 clear_bit(B_WRITING, &b->state); in write_endio()
1434 wake_up_bit(&b->state, B_WRITING); in write_endio()
1440 * - If the buffer is not dirty, exit.
1441 * - If there some previous write going on, wait for it to finish (we can't
1443 * - Submit our write and don't wait on it. We set B_WRITING indicating
1449 if (!test_bit(B_DIRTY, &b->state)) in __write_dirty_buffer()
1452 clear_bit(B_DIRTY, &b->state); in __write_dirty_buffer()
1453 wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __write_dirty_buffer()
1455 b->write_start = b->dirty_start; in __write_dirty_buffer()
1456 b->write_end = b->dirty_end; in __write_dirty_buffer()
1461 list_add_tail(&b->write_list, write_list); in __write_dirty_buffer()
1471 list_entry(write_list->next, struct dm_buffer, write_list); in __flush_write_list()
1472 list_del(&b->write_list); in __flush_write_list()
1486 BUG_ON(atomic_read(&b->hold_count)); in __make_buffer_clean()
1489 if (!smp_load_acquire(&b->state)) /* fast case */ in __make_buffer_clean()
1492 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
1494 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
1502 if (WARN_ON_ONCE(test_bit(B_WRITING, &b->state))) in is_clean()
1504 if (WARN_ON_ONCE(test_bit(B_DIRTY, &b->state))) in is_clean()
1506 if (WARN_ON_ONCE(b->list_mode != LIST_CLEAN)) in is_clean()
1509 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep && in is_clean()
1510 unlikely(test_bit(B_READING, &b->state))) in is_clean()
1519 if (WARN_ON_ONCE(test_bit(B_READING, &b->state))) in is_dirty()
1521 if (WARN_ON_ONCE(b->list_mode != LIST_DIRTY)) in is_dirty()
1535 b = cache_evict(&c->cache, LIST_CLEAN, is_clean, c); in __get_unclaimed_buffer()
1542 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) in __get_unclaimed_buffer()
1545 b = cache_evict(&c->cache, LIST_DIRTY, is_dirty, NULL); in __get_unclaimed_buffer()
1558 * This function is entered with c->lock held, drops it and regains it
1565 add_wait_queue(&c->free_buffer_wait, &wait); in __wait_for_free_buffer()
1571 * hold c->lock when wake_up is called. So we have a timeout here, in __wait_for_free_buffer()
1576 remove_wait_queue(&c->free_buffer_wait, &wait); in __wait_for_free_buffer()
1600 * dm-bufio is resistant to allocation failures (it just keeps in __alloc_buffer_wait_no_callback()
1609 * For debugging, if we set the cache size to 1, no new buffers will in __alloc_buffer_wait_no_callback()
1631 if (!list_empty(&c->reserved_buffers)) { in __alloc_buffer_wait_no_callback()
1632 b = list_to_buffer(c->reserved_buffers.next); in __alloc_buffer_wait_no_callback()
1633 list_del(&b->lru.list); in __alloc_buffer_wait_no_callback()
1634 c->need_reserved_buffers++; in __alloc_buffer_wait_no_callback()
1654 if (c->alloc_callback) in __alloc_buffer_wait()
1655 c->alloc_callback(b); in __alloc_buffer_wait()
1665 struct dm_bufio_client *c = b->c; in __free_buffer_wake()
1667 b->block = -1; in __free_buffer_wake()
1668 if (!c->need_reserved_buffers) in __free_buffer_wake()
1671 list_add(&b->lru.list, &c->reserved_buffers); in __free_buffer_wake()
1672 c->need_reserved_buffers--; in __free_buffer_wake()
1679 if (unlikely(waitqueue_active(&c->free_buffer_wait))) in __free_buffer_wake()
1680 wake_up(&c->free_buffer_wait); in __free_buffer_wake()
1685 if (WARN_ON_ONCE(test_bit(B_READING, &b->state))) in cleaned()
1688 if (test_bit(B_DIRTY, &b->state) || test_bit(B_WRITING, &b->state)) in cleaned()
1696 cache_mark_many(&c->cache, LIST_DIRTY, LIST_CLEAN, cleaned, NULL); in __move_clean_buffers()
1708 if (wc->no_wait && test_bit(B_WRITING, &b->state)) in write_one()
1711 __write_dirty_buffer(b, wc->write_list); in write_one()
1721 cache_iterate(&c->cache, LIST_DIRTY, write_one, &wc); in __write_dirty_buffers_async()
1727 * If we're over "limit_buffers", block until we get under the limit.
1732 if (cache_count(&c->cache, LIST_DIRTY) > in __check_watermark()
1733 cache_count(&c->cache, LIST_CLEAN) * DM_BUFIO_WRITEBACK_RATIO) in __check_watermark()
1738 *--------------------------------------------------------------
1740 *--------------------------------------------------------------
1749 if (cache_put(&c->cache, b) && in cache_put_and_wake()
1750 unlikely(waitqueue_active(&c->free_buffer_wait))) in cache_put_and_wake()
1751 wake_up(&c->free_buffer_wait); in cache_put_and_wake()
1755 * This assumes you have already checked the cache to see if the buffer
1758 static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block, in __bufio_new() argument
1778 b = cache_get(&c->cache, block); in __bufio_new()
1787 atomic_set(&b->hold_count, 1); in __bufio_new()
1788 WRITE_ONCE(b->last_accessed, jiffies); in __bufio_new()
1789 b->block = block; in __bufio_new()
1790 b->read_error = 0; in __bufio_new()
1791 b->write_error = 0; in __bufio_new()
1792 b->list_mode = LIST_CLEAN; in __bufio_new()
1795 b->state = 0; in __bufio_new()
1797 b->state = 1 << B_READING; in __bufio_new()
1802 * We mustn't insert into the cache until the B_READING state in __bufio_new()
1806 cache_insert(&c->cache, b); in __bufio_new()
1823 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { in __bufio_new()
1837 b->read_error = status; in read_endio()
1839 BUG_ON(!test_bit(B_READING, &b->state)); in read_endio()
1842 clear_bit(B_READING, &b->state); in read_endio()
1845 wake_up_bit(&b->state, B_READING); in read_endio()
1854 static void *new_read(struct dm_bufio_client *c, sector_t block, in new_read() argument
1865 * Fast path, hopefully the block is already in the cache. No need in new_read()
1868 b = cache_get(&c->cache, block); in new_read()
1882 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { in new_read()
1893 b = __bufio_new(c, block, nf, &need_submit, &write_list); in new_read()
1898 if (b && (atomic_read(&b->hold_count) == 1)) in new_read()
1911 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in new_read()
1913 if (b->read_error) { in new_read()
1914 int error = blk_status_to_errno(b->read_error); in new_read()
1923 return b->data; in new_read()
1926 void *dm_bufio_get(struct dm_bufio_client *c, sector_t block, in dm_bufio_get() argument
1929 return new_read(c, block, NF_GET, bp); in dm_bufio_get()
1933 void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, in dm_bufio_read() argument
1937 return ERR_PTR(-EINVAL); in dm_bufio_read()
1939 return new_read(c, block, NF_READ, bp); in dm_bufio_read()
1943 void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, in dm_bufio_new() argument
1947 return ERR_PTR(-EINVAL); in dm_bufio_new()
1949 return new_read(c, block, NF_FRESH, bp); in dm_bufio_new()
1954 sector_t block, unsigned int n_blocks) in dm_bufio_prefetch() argument
1965 for (; n_blocks--; block++) { in dm_bufio_prefetch()
1969 b = cache_get(&c->cache, block); in dm_bufio_prefetch()
1971 /* already in cache */ in dm_bufio_prefetch()
1977 b = __bufio_new(c, block, NF_PREFETCH, &need_submit, in dm_bufio_prefetch()
2009 struct dm_bufio_client *c = b->c; in dm_bufio_release()
2016 if ((b->read_error || b->write_error) && in dm_bufio_release()
2017 !test_bit_acquire(B_READING, &b->state) && in dm_bufio_release()
2018 !test_bit(B_WRITING, &b->state) && in dm_bufio_release()
2019 !test_bit(B_DIRTY, &b->state)) { in dm_bufio_release()
2022 /* cache remove can fail if there are other holders */ in dm_bufio_release()
2023 if (cache_remove(&c->cache, b)) { in dm_bufio_release()
2039 struct dm_bufio_client *c = b->c; in dm_bufio_mark_partial_buffer_dirty()
2042 BUG_ON(end > b->c->block_size); in dm_bufio_mark_partial_buffer_dirty()
2046 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_mark_partial_buffer_dirty()
2048 if (!test_and_set_bit(B_DIRTY, &b->state)) { in dm_bufio_mark_partial_buffer_dirty()
2049 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
2050 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
2051 cache_mark(&c->cache, b, LIST_DIRTY); in dm_bufio_mark_partial_buffer_dirty()
2053 if (start < b->dirty_start) in dm_bufio_mark_partial_buffer_dirty()
2054 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
2055 if (end > b->dirty_end) in dm_bufio_mark_partial_buffer_dirty()
2056 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
2065 dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size); in dm_bufio_mark_buffer_dirty()
2085 * and simultaneously (so that the block layer can merge the writes) and then
2088 * Finally, we flush hardware disk cache.
2094 return test_bit(B_WRITING, &b->state); in is_writing()
2112 nr_buffers = cache_count(&c->cache, LIST_DIRTY); in dm_bufio_write_dirty_buffers()
2113 lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it); in dm_bufio_write_dirty_buffers()
2118 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_write_dirty_buffers()
2121 nr_buffers--; in dm_bufio_write_dirty_buffers()
2123 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in dm_bufio_write_dirty_buffers()
2126 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in dm_bufio_write_dirty_buffers()
2129 if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state)) in dm_bufio_write_dirty_buffers()
2130 cache_mark(&c->cache, b, LIST_CLEAN); in dm_bufio_write_dirty_buffers()
2138 wake_up(&c->free_buffer_wait); in dm_bufio_write_dirty_buffers()
2141 a = xchg(&c->async_write_error, 0); in dm_bufio_write_dirty_buffers()
2151 * Use dm-io to send an empty barrier to flush the device.
2159 .client = c->dm_io, in dm_bufio_issue_flush()
2162 .bdev = c->bdev, in dm_bufio_issue_flush()
2168 return -EINVAL; in dm_bufio_issue_flush()
2175 * Use dm-io to send a discard request to flush the device.
2177 int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count) in dm_bufio_issue_discard() argument
2183 .client = c->dm_io, in dm_bufio_issue_discard()
2186 .bdev = c->bdev, in dm_bufio_issue_discard()
2187 .sector = block_to_sector(c, block), in dm_bufio_issue_discard()
2192 return -EINVAL; /* discards are optional */ in dm_bufio_issue_discard()
2198 static bool forget_buffer(struct dm_bufio_client *c, sector_t block) in forget_buffer() argument
2202 b = cache_get(&c->cache, block); in forget_buffer()
2204 if (likely(!smp_load_acquire(&b->state))) { in forget_buffer()
2205 if (cache_remove(&c->cache, b)) in forget_buffer()
2223 void dm_bufio_forget(struct dm_bufio_client *c, sector_t block) in dm_bufio_forget() argument
2226 forget_buffer(c, block); in dm_bufio_forget()
2233 return b->state ? ER_DONT_EVICT : ER_EVICT; in idle()
2236 void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t n_blocks) in dm_bufio_forget_buffers() argument
2239 cache_remove_range(&c->cache, block, block + n_blocks, idle, __free_buffer_wake); in dm_bufio_forget_buffers()
2246 c->minimum_buffers = n; in dm_bufio_set_minimum_buffers()
2252 return c->block_size; in dm_bufio_get_block_size()
2258 sector_t s = bdev_nr_sectors(c->bdev); in dm_bufio_get_device_size()
2260 if (s >= c->start) in dm_bufio_get_device_size()
2261 s -= c->start; in dm_bufio_get_device_size()
2264 if (likely(c->sectors_per_block_bits >= 0)) in dm_bufio_get_device_size()
2265 s >>= c->sectors_per_block_bits; in dm_bufio_get_device_size()
2267 sector_div(s, c->block_size >> SECTOR_SHIFT); in dm_bufio_get_device_size()
2274 return c->dm_io; in dm_bufio_get_dm_io_client()
2280 return b->block; in dm_bufio_get_block_number()
2286 return b->data; in dm_bufio_get_block_data()
2298 return b->c; in dm_bufio_get_client()
2309 (unsigned long long)b->block, atomic_read(&b->hold_count), b->list_mode); in warn_leak()
2311 stack_trace_print(b->stack_entries, b->stack_len, 1); in warn_leak()
2313 atomic_set(&b->hold_count, 0); in warn_leak()
2327 * An optimization so that the buffers are not written one-by-one. in drop_buffers()
2339 cache_iterate(&c->cache, i, warn_leak, &warned); in drop_buffers()
2348 WARN_ON(cache_count(&c->cache, i)); in drop_buffers()
2357 if (likely(c->sectors_per_block_bits >= 0)) in get_retain_buffers()
2358 retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT; in get_retain_buffers()
2360 retain_bytes /= c->block_size; in get_retain_buffers()
2371 unsigned long count = cache_total(&c->cache); in __scan()
2375 if (count - freed <= retain_target) in __scan()
2376 atomic_long_set(&c->need_shrink, 0); in __scan()
2377 if (!atomic_long_read(&c->need_shrink)) in __scan()
2380 b = cache_evict(&c->cache, l, in __scan()
2388 atomic_long_dec(&c->need_shrink); in __scan()
2409 atomic_long_add(sc->nr_to_scan, &c->need_shrink); in dm_bufio_shrink_scan()
2410 queue_work(dm_bufio_wq, &c->shrink_work); in dm_bufio_shrink_scan()
2412 return sc->nr_to_scan; in dm_bufio_shrink_scan()
2418 unsigned long count = cache_total(&c->cache); in dm_bufio_shrink_count()
2420 unsigned long queued_for_cleanup = atomic_long_read(&c->need_shrink); in dm_bufio_shrink_count()
2425 count -= retain_target; in dm_bufio_shrink_count()
2430 count -= queued_for_cleanup; in dm_bufio_shrink_count()
2450 if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) { in dm_bufio_client_create()
2451 DMERR("%s: block size not specified or is not multiple of 512b", __func__); in dm_bufio_client_create()
2452 r = -EINVAL; in dm_bufio_client_create()
2459 r = -ENOMEM; in dm_bufio_client_create()
2462 cache_init(&c->cache, num_locks, (flags & DM_BUFIO_CLIENT_NO_SLEEP) != 0); in dm_bufio_client_create()
2464 c->bdev = bdev; in dm_bufio_client_create()
2465 c->block_size = block_size; in dm_bufio_client_create()
2467 c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT; in dm_bufio_client_create()
2469 c->sectors_per_block_bits = -1; in dm_bufio_client_create()
2471 c->alloc_callback = alloc_callback; in dm_bufio_client_create()
2472 c->write_callback = write_callback; in dm_bufio_client_create()
2475 c->no_sleep = true; in dm_bufio_client_create()
2479 mutex_init(&c->lock); in dm_bufio_client_create()
2480 spin_lock_init(&c->spinlock); in dm_bufio_client_create()
2481 INIT_LIST_HEAD(&c->reserved_buffers); in dm_bufio_client_create()
2482 c->need_reserved_buffers = reserved_buffers; in dm_bufio_client_create()
2486 init_waitqueue_head(&c->free_buffer_wait); in dm_bufio_client_create()
2487 c->async_write_error = 0; in dm_bufio_client_create()
2489 c->dm_io = dm_io_client_create(); in dm_bufio_client_create()
2490 if (IS_ERR(c->dm_io)) { in dm_bufio_client_create()
2491 r = PTR_ERR(c->dm_io); in dm_bufio_client_create()
2499 snprintf(slab_name, sizeof(slab_name), "dm_bufio_cache-%u-%u", in dm_bufio_client_create()
2501 c->slab_cache = kmem_cache_create(slab_name, block_size, align, in dm_bufio_client_create()
2503 if (!c->slab_cache) { in dm_bufio_client_create()
2504 r = -ENOMEM; in dm_bufio_client_create()
2509 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer-%u-%u", in dm_bufio_client_create()
2512 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer-%u", in dm_bufio_client_create()
2514 c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size, in dm_bufio_client_create()
2516 if (!c->slab_buffer) { in dm_bufio_client_create()
2517 r = -ENOMEM; in dm_bufio_client_create()
2521 while (c->need_reserved_buffers) { in dm_bufio_client_create()
2525 r = -ENOMEM; in dm_bufio_client_create()
2531 INIT_WORK(&c->shrink_work, shrink_work); in dm_bufio_client_create()
2532 atomic_long_set(&c->need_shrink, 0); in dm_bufio_client_create()
2534 c->shrinker.count_objects = dm_bufio_shrink_count; in dm_bufio_client_create()
2535 c->shrinker.scan_objects = dm_bufio_shrink_scan; in dm_bufio_client_create()
2536 c->shrinker.seeks = 1; in dm_bufio_client_create()
2537 c->shrinker.batch = 0; in dm_bufio_client_create()
2538 r = register_shrinker(&c->shrinker, "dm-bufio:(%u:%u)", in dm_bufio_client_create()
2539 MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev)); in dm_bufio_client_create()
2545 list_add(&c->client_list, &dm_bufio_all_clients); in dm_bufio_client_create()
2552 while (!list_empty(&c->reserved_buffers)) { in dm_bufio_client_create()
2553 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next); in dm_bufio_client_create()
2555 list_del(&b->lru.list); in dm_bufio_client_create()
2558 kmem_cache_destroy(c->slab_cache); in dm_bufio_client_create()
2559 kmem_cache_destroy(c->slab_buffer); in dm_bufio_client_create()
2560 dm_io_client_destroy(c->dm_io); in dm_bufio_client_create()
2562 mutex_destroy(&c->lock); in dm_bufio_client_create()
2563 if (c->no_sleep) in dm_bufio_client_create()
2581 unregister_shrinker(&c->shrinker); in dm_bufio_client_destroy()
2582 flush_work(&c->shrink_work); in dm_bufio_client_destroy()
2586 list_del(&c->client_list); in dm_bufio_client_destroy()
2587 dm_bufio_client_count--; in dm_bufio_client_destroy()
2592 WARN_ON(c->need_reserved_buffers); in dm_bufio_client_destroy()
2594 while (!list_empty(&c->reserved_buffers)) { in dm_bufio_client_destroy()
2595 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next); in dm_bufio_client_destroy()
2597 list_del(&b->lru.list); in dm_bufio_client_destroy()
2602 if (cache_count(&c->cache, i)) in dm_bufio_client_destroy()
2603 DMERR("leaked buffer count %d: %lu", i, cache_count(&c->cache, i)); in dm_bufio_client_destroy()
2606 WARN_ON(cache_count(&c->cache, i)); in dm_bufio_client_destroy()
2608 cache_destroy(&c->cache); in dm_bufio_client_destroy()
2609 kmem_cache_destroy(c->slab_cache); in dm_bufio_client_destroy()
2610 kmem_cache_destroy(c->slab_buffer); in dm_bufio_client_destroy()
2611 dm_io_client_destroy(c->dm_io); in dm_bufio_client_destroy()
2612 mutex_destroy(&c->lock); in dm_bufio_client_destroy()
2613 if (c->no_sleep) in dm_bufio_client_destroy()
2622 flush_work(&c->shrink_work); in dm_bufio_client_reset()
2628 c->start = start; in dm_bufio_set_sector_offset()
2632 /*--------------------------------------------------------------*/
2646 return time_after_eq(jiffies, READ_ONCE(b->last_accessed) + age_hz); in older_than()
2673 if (!(params->gfp & __GFP_FS) || in select_for_evict()
2674 (static_branch_unlikely(&no_sleep_enabled) && b->c->no_sleep)) { in select_for_evict()
2675 if (test_bit_acquire(B_READING, &b->state) || in select_for_evict()
2676 test_bit(B_WRITING, &b->state) || in select_for_evict()
2677 test_bit(B_DIRTY, &b->state)) in select_for_evict()
2681 return older_than(b, params->age_hz) ? ER_EVICT : ER_STOP; in select_for_evict()
2693 b = cache_evict(&c->cache, list_mode, select_for_evict, params); in __evict_many()
2697 last_accessed = READ_ONCE(b->last_accessed); in __evict_many()
2698 if (time_after_eq(params->last_accessed, last_accessed)) in __evict_many()
2699 params->last_accessed = last_accessed; in __evict_many()
2726 count = cache_total(&c->cache); in evict_old_buffers()
2728 __evict_many(c, ¶ms, LIST_CLEAN, count - retain); in evict_old_buffers()
2756 /*--------------------------------------------------------------*/
2787 if (time_after_eq(c->oldest_buffer, new_client->oldest_buffer)) in __insert_client()
2789 h = h->next; in __insert_client()
2792 list_add_tail(&new_client->client_list, h); in __insert_client()
2815 c->oldest_buffer = params.last_accessed; in __evict_a_few()
2839 unsigned long threshold = dm_bufio_cache_size - in evict_old()
2858 *--------------------------------------------------------------
2860 *--------------------------------------------------------------
2876 mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(), in dm_bufio_init()
2895 return -ENOMEM; in dm_bufio_init()
2946 MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache");
2967 MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache");
2969 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");