1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef BTRFS_EXTENT_IO_H 4 #define BTRFS_EXTENT_IO_H 5 6 #include <linux/rbtree.h> 7 #include <linux/refcount.h> 8 #include "ulist.h" 9 10 /* 11 * flags for bio submission. The high bits indicate the compression 12 * type for this bio 13 */ 14 #define EXTENT_BIO_COMPRESSED 1 15 #define EXTENT_BIO_FLAG_SHIFT 16 16 17 enum { 18 EXTENT_BUFFER_UPTODATE, 19 EXTENT_BUFFER_DIRTY, 20 EXTENT_BUFFER_CORRUPT, 21 /* this got triggered by readahead */ 22 EXTENT_BUFFER_READAHEAD, 23 EXTENT_BUFFER_TREE_REF, 24 EXTENT_BUFFER_STALE, 25 EXTENT_BUFFER_WRITEBACK, 26 /* read IO error */ 27 EXTENT_BUFFER_READ_ERR, 28 EXTENT_BUFFER_UNMAPPED, 29 EXTENT_BUFFER_IN_TREE, 30 /* write IO error */ 31 EXTENT_BUFFER_WRITE_ERR, 32 }; 33 34 /* these are flags for __process_pages_contig */ 35 #define PAGE_UNLOCK (1 << 0) 36 #define PAGE_CLEAR_DIRTY (1 << 1) 37 #define PAGE_SET_WRITEBACK (1 << 2) 38 #define PAGE_END_WRITEBACK (1 << 3) 39 #define PAGE_SET_PRIVATE2 (1 << 4) 40 #define PAGE_SET_ERROR (1 << 5) 41 #define PAGE_LOCK (1 << 6) 42 43 /* 44 * page->private values. Every page that is controlled by the extent 45 * map has page->private set to one. 46 */ 47 #define EXTENT_PAGE_PRIVATE 1 48 49 /* 50 * The extent buffer bitmap operations are done with byte granularity instead of 51 * word granularity for two reasons: 52 * 1. The bitmaps must be little-endian on disk. 53 * 2. Bitmap items are not guaranteed to be aligned to a word and therefore a 54 * single word in a bitmap may straddle two pages in the extent buffer. 55 */ 56 #define BIT_BYTE(nr) ((nr) / BITS_PER_BYTE) 57 #define BYTE_MASK ((1 << BITS_PER_BYTE) - 1) 58 #define BITMAP_FIRST_BYTE_MASK(start) \ 59 ((BYTE_MASK << ((start) & (BITS_PER_BYTE - 1))) & BYTE_MASK) 60 #define BITMAP_LAST_BYTE_MASK(nbits) \ 61 (BYTE_MASK >> (-(nbits) & (BITS_PER_BYTE - 1))) 62 63 struct btrfs_root; 64 struct btrfs_inode; 65 struct btrfs_io_bio; 66 struct io_failure_record; 67 struct extent_io_tree; 68 69 typedef blk_status_t (extent_submit_bio_start_t)(void *private_data, 70 struct bio *bio, u64 bio_offset); 71 72 struct extent_io_ops { 73 /* 74 * The following callbacks must be always defined, the function 75 * pointer will be called unconditionally. 76 */ 77 blk_status_t (*submit_bio_hook)(struct inode *inode, struct bio *bio, 78 int mirror_num, unsigned long bio_flags); 79 int (*readpage_end_io_hook)(struct btrfs_io_bio *io_bio, u64 phy_offset, 80 struct page *page, u64 start, u64 end, 81 int mirror); 82 }; 83 84 85 #define INLINE_EXTENT_BUFFER_PAGES 16 86 #define MAX_INLINE_EXTENT_BUFFER_SIZE (INLINE_EXTENT_BUFFER_PAGES * PAGE_SIZE) 87 struct extent_buffer { 88 u64 start; 89 unsigned long len; 90 unsigned long bflags; 91 struct btrfs_fs_info *fs_info; 92 spinlock_t refs_lock; 93 atomic_t refs; 94 atomic_t io_pages; 95 int read_mirror; 96 struct rcu_head rcu_head; 97 pid_t lock_owner; 98 99 int blocking_writers; 100 atomic_t blocking_readers; 101 bool lock_nested; 102 /* >= 0 if eb belongs to a log tree, -1 otherwise */ 103 short log_index; 104 105 /* protects write locks */ 106 rwlock_t lock; 107 108 /* readers use lock_wq while they wait for the write 109 * lock holders to unlock 110 */ 111 wait_queue_head_t write_lock_wq; 112 113 /* writers use read_lock_wq while they wait for readers 114 * to unlock 115 */ 116 wait_queue_head_t read_lock_wq; 117 struct page *pages[INLINE_EXTENT_BUFFER_PAGES]; 118 #ifdef CONFIG_BTRFS_DEBUG 119 int spinning_writers; 120 atomic_t spinning_readers; 121 atomic_t read_locks; 122 int write_locks; 123 struct list_head leak_list; 124 #endif 125 }; 126 127 /* 128 * Structure to record how many bytes and which ranges are set/cleared 129 */ 130 struct extent_changeset { 131 /* How many bytes are set/cleared in this operation */ 132 unsigned int bytes_changed; 133 134 /* Changed ranges */ 135 struct ulist range_changed; 136 }; 137 138 static inline void extent_changeset_init(struct extent_changeset *changeset) 139 { 140 changeset->bytes_changed = 0; 141 ulist_init(&changeset->range_changed); 142 } 143 144 static inline struct extent_changeset *extent_changeset_alloc(void) 145 { 146 struct extent_changeset *ret; 147 148 ret = kmalloc(sizeof(*ret), GFP_KERNEL); 149 if (!ret) 150 return NULL; 151 152 extent_changeset_init(ret); 153 return ret; 154 } 155 156 static inline void extent_changeset_release(struct extent_changeset *changeset) 157 { 158 if (!changeset) 159 return; 160 changeset->bytes_changed = 0; 161 ulist_release(&changeset->range_changed); 162 } 163 164 static inline void extent_changeset_free(struct extent_changeset *changeset) 165 { 166 if (!changeset) 167 return; 168 extent_changeset_release(changeset); 169 kfree(changeset); 170 } 171 172 static inline void extent_set_compress_type(unsigned long *bio_flags, 173 int compress_type) 174 { 175 *bio_flags |= compress_type << EXTENT_BIO_FLAG_SHIFT; 176 } 177 178 static inline int extent_compress_type(unsigned long bio_flags) 179 { 180 return bio_flags >> EXTENT_BIO_FLAG_SHIFT; 181 } 182 183 struct extent_map_tree; 184 185 typedef struct extent_map *(get_extent_t)(struct btrfs_inode *inode, 186 struct page *page, size_t pg_offset, 187 u64 start, u64 len); 188 189 int try_release_extent_mapping(struct page *page, gfp_t mask); 190 int try_release_extent_buffer(struct page *page); 191 192 int extent_read_full_page(struct page *page, get_extent_t *get_extent, 193 int mirror_num); 194 int extent_write_full_page(struct page *page, struct writeback_control *wbc); 195 int extent_write_locked_range(struct inode *inode, u64 start, u64 end, 196 int mode); 197 int extent_writepages(struct address_space *mapping, 198 struct writeback_control *wbc); 199 int btree_write_cache_pages(struct address_space *mapping, 200 struct writeback_control *wbc); 201 int extent_readpages(struct address_space *mapping, struct list_head *pages, 202 unsigned nr_pages); 203 int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 204 __u64 start, __u64 len); 205 void set_page_extent_mapped(struct page *page); 206 207 struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, 208 u64 start); 209 struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, 210 u64 start, unsigned long len); 211 struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, 212 u64 start); 213 struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src); 214 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info, 215 u64 start); 216 void free_extent_buffer(struct extent_buffer *eb); 217 void free_extent_buffer_stale(struct extent_buffer *eb); 218 #define WAIT_NONE 0 219 #define WAIT_COMPLETE 1 220 #define WAIT_PAGE_LOCK 2 221 int read_extent_buffer_pages(struct extent_buffer *eb, int wait, 222 int mirror_num); 223 void wait_on_extent_buffer_writeback(struct extent_buffer *eb); 224 225 static inline int num_extent_pages(const struct extent_buffer *eb) 226 { 227 return (round_up(eb->start + eb->len, PAGE_SIZE) >> PAGE_SHIFT) - 228 (eb->start >> PAGE_SHIFT); 229 } 230 231 static inline int extent_buffer_uptodate(struct extent_buffer *eb) 232 { 233 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 234 } 235 236 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, 237 unsigned long start, unsigned long len); 238 void read_extent_buffer(const struct extent_buffer *eb, void *dst, 239 unsigned long start, 240 unsigned long len); 241 int read_extent_buffer_to_user(const struct extent_buffer *eb, 242 void __user *dst, unsigned long start, 243 unsigned long len); 244 void write_extent_buffer_fsid(struct extent_buffer *eb, const void *src); 245 void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb, 246 const void *src); 247 void write_extent_buffer(struct extent_buffer *eb, const void *src, 248 unsigned long start, unsigned long len); 249 void copy_extent_buffer_full(struct extent_buffer *dst, 250 struct extent_buffer *src); 251 void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src, 252 unsigned long dst_offset, unsigned long src_offset, 253 unsigned long len); 254 void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, 255 unsigned long src_offset, unsigned long len); 256 void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, 257 unsigned long src_offset, unsigned long len); 258 void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start, 259 unsigned long len); 260 int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start, 261 unsigned long pos); 262 void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start, 263 unsigned long pos, unsigned long len); 264 void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start, 265 unsigned long pos, unsigned long len); 266 void clear_extent_buffer_dirty(struct extent_buffer *eb); 267 bool set_extent_buffer_dirty(struct extent_buffer *eb); 268 void set_extent_buffer_uptodate(struct extent_buffer *eb); 269 void clear_extent_buffer_uptodate(struct extent_buffer *eb); 270 int extent_buffer_under_io(struct extent_buffer *eb); 271 int map_private_extent_buffer(const struct extent_buffer *eb, 272 unsigned long offset, unsigned long min_len, 273 char **map, unsigned long *map_start, 274 unsigned long *map_len); 275 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end); 276 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end); 277 void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end, 278 struct page *locked_page, 279 unsigned bits_to_clear, 280 unsigned long page_ops); 281 struct bio *btrfs_bio_alloc(u64 first_byte); 282 struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs); 283 struct bio *btrfs_bio_clone(struct bio *bio); 284 struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size); 285 286 struct btrfs_fs_info; 287 struct btrfs_inode; 288 289 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, 290 u64 length, u64 logical, struct page *page, 291 unsigned int pg_offset, int mirror_num); 292 void end_extent_writepage(struct page *page, int err, u64 start, u64 end); 293 int btrfs_repair_eb_io_failure(struct extent_buffer *eb, int mirror_num); 294 295 /* 296 * When IO fails, either with EIO or csum verification fails, we 297 * try other mirrors that might have a good copy of the data. This 298 * io_failure_record is used to record state as we go through all the 299 * mirrors. If another mirror has good data, the page is set up to date 300 * and things continue. If a good mirror can't be found, the original 301 * bio end_io callback is called to indicate things have failed. 302 */ 303 struct io_failure_record { 304 struct page *page; 305 u64 start; 306 u64 len; 307 u64 logical; 308 unsigned long bio_flags; 309 int this_mirror; 310 int failed_mirror; 311 int in_validation; 312 }; 313 314 315 bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages, 316 struct io_failure_record *failrec, int fail_mirror); 317 struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio, 318 struct io_failure_record *failrec, 319 struct page *page, int pg_offset, int icsum, 320 bio_end_io_t *endio_func, void *data); 321 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 322 bool find_lock_delalloc_range(struct inode *inode, 323 struct page *locked_page, u64 *start, 324 u64 *end); 325 #endif 326 struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info, 327 u64 start); 328 329 #ifdef CONFIG_BTRFS_DEBUG 330 void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info); 331 #else 332 #define btrfs_extent_buffer_leak_debug_check(fs_info) do {} while (0) 333 #endif 334 335 #endif 336